diff --git a/docs/apps-script/Config.gs b/docs/apps-script/Config.gs new file mode 100644 index 000000000..b83bc8da9 --- /dev/null +++ b/docs/apps-script/Config.gs @@ -0,0 +1,11 @@ +// Config.gs +// +// The Gemini key is NOT in this file. It lives in Script Properties under +// "GEMINI_API_KEY" (Project Settings -> Script Properties), the same place +// SERVICE_ACCOUNT_KEY is kept. A literal here is readable by anyone with edit access to +// the project, and it also rides along into every clasp clone and Drive mirror of it. +function geminiApiKey_() { + const key = PropertiesService.getScriptProperties().getProperty("GEMINI_API_KEY"); + if (!key) throw new Error("GEMINI_API_KEY not set in Script Properties."); + return key; +} diff --git a/docs/apps-script/appsscript.json b/docs/apps-script/appsscript.json new file mode 100644 index 000000000..cc74eb2c7 --- /dev/null +++ b/docs/apps-script/appsscript.json @@ -0,0 +1,33 @@ +{ + "timeZone": "America/Los_Angeles", + "dependencies": { + "libraries": [ + { + "userSymbol": "OAuth2", + "version": "43", + "libraryId": "1B7FSrk5Zi6L1rSxxTDgDEUsPzlukDsi4KGuTMorsTQHhGBzBkMun4iDF" + } + ], + "enabledAdvancedServices": [ + { + "userSymbol": "Gmail", + "version": "v1", + "serviceId": "gmail" + } + ] + }, + "exceptionLogging": "STACKDRIVER", + "runtimeVersion": "V8", + "oauthScopes": [ + "https://www.googleapis.com/auth/script.external_request", + "https://www.googleapis.com/auth/drive", + "https://www.googleapis.com/auth/gmail.send", + "https://www.googleapis.com/auth/gmail.modify", + "https://www.googleapis.com/auth/script.scriptapp", + "https://www.googleapis.com/auth/script.send_mail", + "https://www.googleapis.com/auth/spreadsheets", + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/chat.spaces.readonly", + "https://www.googleapis.com/auth/chat.messages" + ] +} \ No newline at end of file diff --git a/docs/apps-script/auditNeedsReview.gs b/docs/apps-script/auditNeedsReview.gs new file mode 100644 index 000000000..3abdc090b --- /dev/null +++ b/docs/apps-script/auditNeedsReview.gs @@ -0,0 +1,211 @@ +/** + * GOLDEN TOUCH — "_Needs Review" BACKLOG AUDIT [read-only, one-off] + * ================================================================== + * Answers one question for every file sitting in "_Needs Review": + * + * Did this receipt ever reach QuickBooks, and was anyone ever told about it? + * + * Why this is needed: "_Needs Review" starts with "_" and is SKIPPED by the folder scan + * in runReceiptAutomation(). Before v3.5, several paths moved a file there BEFORE sending + * their alert email. If that email failed (a blown daily MailApp quota is the realistic + * cause), the receipt ended up in a folder the trigger never revisits, with nobody told — + * neither booked nor surfaced. v3.5 closed the hole going forward. This script finds the + * ones it already swallowed. + * + * READ-ONLY BY CONSTRUCTION. It never moves a file, never edits a description, and never + * emails QuickBooks. The only write it can perform is one summary email to you, and only + * if you call auditNeedsReviewAndEmail() instead of auditNeedsReview(). + * + * HOW TO RUN + * 1. Paste this as a NEW file in the same Apps Script project (it reuses that project's + * NEW_RECEIPTS_FOLDER_ID / NEEDS_REVIEW_NAME constants, so it must live alongside + * runReceiptAutomation.gs). + * 2. Select auditNeedsReview in the function dropdown and press Run. + * 3. Read the execution log. Or run auditNeedsReviewAndEmail() to also get it by email. + * + * HOW TO READ THE VERDICTS (v2, 2026-08-14 — "emailed" no longer counts as "booked") + * CONFIRMED the API path created a QBO Purchase and recorded its id — it IS in + * the books. Do not re-enter it. See REASON for why it is parked. + * QUEUE ONLY the email fallback delivered the document to QBO's receipts INBOX. + * That is NOT a booked transaction — it books only when a human + * reviews it in the QBO Receipts queue. Check the queue, not the GL. + * AMBIGUOUS a send was attempted but never confirmed (emailing set, emailed + * not). It may or may not have arrived. Look it up before entering. + * PARKED never sent, parked for an actionable reason (gave up, multi-doc, + * unreadable total, bad format, possible duplicate). Needs a human. + * NON-PURCH deliberately not a purchase for this pipeline: payroll/non-receipt, + * or an Amazon document owned by the Amazon Business QBO app. + * UNKNOWN no usable state on the file (dropped in by hand, or pre-dates state + * tracking). Judge it from the document. + * Bank-feed match correctness (matched-to-the-right-line) is NOT this script's job — + * that lives in the ProBuild bank ledger / wrong-match detector. + * + * And the column that matters most for the bug being audited: + * ALERTED yes — a human was told why this file is here. + * NO — nobody was ever told. These are the silent losses. If such a file is + * also NOT BOOKED, that expense is missing from QuickBooks and no email + * about it was ever sent. Those are listed again under ACTION REQUIRED. + */ + +/** Run this one. Logs the report. */ +function auditNeedsReview() { + const report = buildNeedsReviewAudit_(); + Logger.log(report); + return report; +} + +/** Same audit, but also emails the report to ALERT_EMAIL. */ +function auditNeedsReviewAndEmail() { + const report = buildNeedsReviewAudit_(); + Logger.log(report); + if (MailApp.getRemainingDailyQuota() <= 0) { + Logger.log("\n[NOT EMAILED] Daily mail quota is exhausted — the report above is complete; re-run tomorrow to receive it by email."); + return report; + } + MailApp.sendEmail(ALERT_EMAIL, "Receipt bot: _Needs Review backlog audit", report); + return report; +} + +function buildNeedsReviewAudit_() { + const root = DriveApp.getFolderById(NEW_RECEIPTS_FOLDER_ID); + const folders = root.getFoldersByName(NEEDS_REVIEW_NAME); + if (!folders.hasNext()) return 'No "' + NEEDS_REVIEW_NAME + '" folder exists yet — nothing to audit.'; + + const rows = []; + const files = folders.next().getFiles(); + while (files.hasNext()) rows.push(auditOneFile_(files.next())); + + if (!rows.length) return '"' + NEEDS_REVIEW_NAME + '" is empty — nothing to audit.'; + + // Oldest first: the silent losses are the old ones, and they are what you want to see. + rows.sort(function (a, b) { return a.created < b.created ? -1 : (a.created > b.created ? 1 : 0); }); + + const counts = { CONFIRMED: 0, "QUEUE ONLY": 0, AMBIGUOUS: 0, PARKED: 0, "NON-PURCH": 0, UNKNOWN: 0 }; + const silent = []; + let money = 0; + + const lines = rows.map(function (r) { + counts[r.verdict] = (counts[r.verdict] || 0) + 1; + if (!r.alerted && r.verdict !== "CONFIRMED" && r.verdict !== "NON-PURCH") { + silent.push(r); + const n = parseFloat(r.amount); + if (!isNaN(n)) money += n; + } + return " " + pad_(r.created, 10) + " " + pad_(r.verdict, 11) + " " + + pad_(r.alerted ? "alerted" : "NO ALERT", 9) + " " + + pad_("$" + r.amount, 11) + " " + pad_(r.reason, 26) + " " + r.name; + }); + + let out = ""; + out += "_NEEDS REVIEW BACKLOG AUDIT — " + todayStr() + "\n"; + out += "=".repeat(82) + "\n"; + out += rows.length + " file(s) in \"" + NEEDS_REVIEW_NAME + "\".\n\n"; + out += " " + pad_("UPLOADED", 10) + " " + pad_("IN QBO?", 11) + " " + pad_("TOLD?", 9) + + " " + pad_("AMOUNT", 11) + " " + pad_("REASON", 26) + " FILE\n"; + out += " " + "-".repeat(80) + "\n"; + out += lines.join("\n") + "\n\n"; + + out += "SUMMARY\n"; + out += " API-confirmed QBO Purchase (leave alone): " + (counts["CONFIRMED"] || 0) + "\n"; + out += " Emailed to QBO inbox, NOT booked (queue): " + (counts["QUEUE ONLY"] || 0) + "\n"; + out += " Send attempted, unconfirmed (check QBO): " + (counts["AMBIGUOUS"] || 0) + "\n"; + out += " Parked, actionable (needs a human): " + (counts["PARKED"] || 0) + "\n"; + out += " Deliberately non-purchase (payroll/Amazon): " + (counts["NON-PURCH"] || 0) + "\n"; + out += " No state on file (judge by eye): " + (counts["UNKNOWN"] || 0) + "\n\n"; + + if (silent.length) { + out += "ACTION REQUIRED — " + silent.length + " file(s) reached this folder with NOBODY TOLD.\n"; + out += "These are the silent losses. Each is missing from QuickBooks (or unconfirmed) AND\n"; + out += "never generated an alert email, so nothing in your inbox points at them.\n"; + if (money > 0) out += "Approximate total at stake: $" + money.toFixed(2) + "\n"; + out += "\n"; + silent.forEach(function (r, i) { + out += " " + (i + 1) + ". " + r.name + "\n"; + out += " uploaded " + r.created + " | " + r.verdict + " | vendor " + r.vendor + + " | date " + r.docDate + " | $" + r.amount + "\n"; + }); + out += "\n"; + } else { + out += "No silent losses found — every unbooked file here generated an alert.\n\n"; + } + + out += "Verdicts: CONFIRMED = API-created QBO Purchase (id on file), do not re-enter.\n"; + out += "QUEUE ONLY = delivered to QBO's receipts inbox but NOT booked — check the queue.\n"; + out += "AMBIGUOUS = send attempted, unconfirmed — look it up first. PARKED = never sent,\n"; + out += "needs a human. NON-PURCH = payroll/non-receipt or Amazon-app-owned. UNKNOWN = no state.\n"; + out += 'A "*" on the reason means it was inferred from the saved reading rather than\n'; + out += "recorded — those files were parked before v3.5 started writing down the reason.\n"; + return out; +} + +function auditOneFile_(file) { + const state = getState(file); // read-only helper from runReceiptAutomation.gs + const d = state.data || {}; + const hasState = Object.keys(state).length > 0; + + // "emailed" alone proves DELIVERY, not BOOKING. Only an API-recorded purchase id + // proves a GL transaction exists; the email fallback lands in QBO's review inbox. + const apiConfirmed = !!(state.emailed && state.qboApi && + !/^email-fallback:/.test(String(state.qboApi))); + let verdict; + if (apiConfirmed) verdict = "CONFIRMED"; + else if (state.emailed) verdict = "QUEUE ONLY"; + else if (state.emailing) verdict = "AMBIGUOUS"; + else if (state.amazonAppOwned || state.nonReceipt || + String((state.data || {}).doc_type || "").toLowerCase() === "non_receipt") verdict = "NON-PURCH"; + else if (hasState) verdict = "PARKED"; + else verdict = "UNKNOWN"; + + // Was a human ever told? Any one of the alert flags counts. A file with no state at all + // is reported as un-alerted, which is the safe reading — we cannot prove anyone was told. + const alerted = !!(state.parkAlerted || state.nonReceiptAlerted || state.badFormatAlerted || + state.weakDuplicateAlerted || state.refundAlerted); + + return { + name: file.getName(), + created: Utilities.formatDate(file.getDateCreated(), Session.getScriptTimeZone(), "yyyy-MM-dd"), + verdict: verdict, + alerted: alerted, + reason: auditReason_(state), + vendor: d.vendor || "?", + docDate: normalizeDateStr(d.date) || d.date || "?", + amount: d.total_amount ? cleanMoney(d.total_amount) : "?" + }; +} + +// Why is this file parked? Most specific reason first. +// +// The inference block matters more than the explicit flags: a file parked BEFORE v3.5 has +// no parkReason at all (the old code moved it without recording why), and those pre-v3.5 +// files are precisely the silent losses this audit exists to find. Reporting them as +// "unclear" would be the least useful possible answer, so where the flag is missing the +// reason is reconstructed from the extraction the AI already saved on the file. +function auditReason_(state) { + // Explicit, recorded by v3.5+. + if (state.parkReason === "zeroTotal") return "unreadable $0.00 total"; + if (state.parkReason === "multiDoc") return "multiple receipts in one"; + if (state.parkReason === "gaveUp") return "gave up (retry limit)"; + if (state.parkReason) return String(state.parkReason); + if (state.nonReceipt) return "not a receipt (payroll?)"; + if (state.badFormat) return "format QBO can't read"; + if (state.dedupWeak || state.duplicateOf) return "possible duplicate"; + if (state.refund) return "refund / credit"; + + // Inferred, for files parked before v3.5 recorded a reason. + const d = state.data || {}; + const docType = String(d.doc_type || "").toLowerCase(); + if (docType === "multi") return "multiple receipts in one*"; + if (docType === "non_receipt") return "not a receipt (payroll?)*"; + if (d.total_amount !== undefined && d.total_amount !== null && + cleanMoney(d.total_amount) === "0.00") return "unreadable $0.00 total*"; + + if ((state.attempts || 0) >= 3 || (state.runs || 0) >= 6) return "gave up (retry limit)"; + if (Object.keys(state).length === 0) return "no state recorded"; + return "unclear — read the file"; +} + +function pad_(s, n) { + s = String(s === undefined || s === null ? "" : s); + if (s.length >= n) return s.slice(0, n); + return s + " ".repeat(n - s.length); +} diff --git a/docs/apps-script/googleChat.gs b/docs/apps-script/googleChat.gs new file mode 100644 index 000000000..7cbcf9fe5 --- /dev/null +++ b/docs/apps-script/googleChat.gs @@ -0,0 +1,281 @@ +/** + * googleChat.gs — native Google Chat from Apps Script. + * + * WHY THIS FILE EXISTS + * Justin: "my AI always gets confused about how to connect with Google's chat." + * That confusion is real and it has a specific cause, recorded here so nobody + * (human or AI) re-derives it: + * + * Composio LISTS google_chat as a toolkit and even exposes 8 GOOGLE_CHAT_* + * tools, but the toolkit has composio_managed_auth_schemes: []. There is NO + * OAuth flow behind it. Those tools return "No connected account found" + * forever, and no amount of clicking in the dashboard fixes it. Verified + * 2026-08-20 against the live Composio API. + * + * The working path is the one below: Apps Script already runs AS the user, so + * adding the chat.spaces scope to appsscript.json is the entire "connection". + * No service account, no domain-wide delegation, no third-party hub. + * + * TWO WAYS TO POST, and they are NOT interchangeable: + * + * 1. WEBHOOK (postToChatWebhook_) + * A per-space URL you create in the space's own menu. No OAuth at all. + * Post-only: cannot read, cannot list spaces, cannot DM. + * Best for "the pipeline shouts into a room." + * + * 2. CHAT API as the user (chatListSpaces / chatPostMessage) + * Real API, needs the chat.spaces + chat.messages scopes in the manifest. + * Can list spaces and read membership. Posting to a space still requires + * that the app/user be a member of it. + * + * Webhook is what actually gets used day to day. The API functions are here + * because "list the spaces" is how you find out where to put the webhook. + */ + +const CHAT_WEBHOOK_PROP = "GOOGLE_CHAT_WEBHOOK_URL"; + +/** + * Post to a Google Chat space via an incoming webhook. + * Returns true on success. Never throws — alerting must not break a pipeline. + */ +function postToChatWebhook_(text, opts) { + opts = opts || {}; + const props = PropertiesService.getScriptProperties(); + const url = opts.url || props.getProperty(CHAT_WEBHOOK_PROP); + if (!url) { + Logger.log("[CHAT] no webhook configured (" + CHAT_WEBHOOK_PROP + " unset)"); + return false; + } + try { + const payload = { text: String(text).slice(0, 4000) }; + // threadKey groups related messages into one thread instead of spamming + // the space with disconnected posts. + const fullUrl = opts.threadKey + ? url + "&threadKey=" + encodeURIComponent(opts.threadKey) + + "&messageReplyOption=REPLY_MESSAGE_FALLBACK_TO_NEW_THREAD" + : url; + const res = UrlFetchApp.fetch(fullUrl, { + method: "post", + contentType: "application/json; charset=UTF-8", + payload: JSON.stringify(payload), + muteHttpExceptions: true + }); + const code = res.getResponseCode(); + if (code !== 200) { + Logger.log("[CHAT] webhook HTTP " + code + ": " + res.getContentText().slice(0, 200)); + return false; + } + return true; + } catch (e) { + Logger.log("[CHAT] webhook failed: " + e); + return false; + } +} + +/** + * List the Chat spaces this user can see. Requires the chat.spaces scope in + * appsscript.json — if it is missing this throws a clear authorization error + * rather than returning nothing. + * + * Use this to FIND the space you want, then create a webhook inside it. + */ +function chatListSpaces() { + const token = ScriptApp.getOAuthToken(); + const res = UrlFetchApp.fetch( + "https://chat.googleapis.com/v1/spaces?pageSize=100", { + method: "get", + headers: { Authorization: "Bearer " + token }, + muteHttpExceptions: true + }); + const code = res.getResponseCode(); + const body = res.getContentText(); + if (code !== 200) { + Logger.log("[CHAT] list spaces HTTP " + code); + Logger.log(body.slice(0, 400)); + if (code === 403) { + Logger.log(""); + Logger.log("403 usually means the chat.spaces scope is missing from"); + Logger.log("appsscript.json, OR the Chat API is not enabled on the"); + Logger.log("Cloud project behind this script."); + } + return null; + } + const spaces = (JSON.parse(body).spaces) || []; + Logger.log("--- GOOGLE CHAT SPACES (" + spaces.length + ") ---"); + spaces.forEach(function (s) { + Logger.log(" " + (s.displayName || "(direct message)") + + " name=" + s.name + " type=" + (s.spaceType || s.type)); + }); + Logger.log(""); + Logger.log("To wire a webhook: open the space in Chat -> space name ->"); + Logger.log("Apps & integrations -> Webhooks -> Add webhook -> copy the URL,"); + Logger.log("then run setChatWebhook() with it."); + return spaces; +} + +/** + * Post as the USER via the Chat API (not a webhook). Requires chat.messages + * scope AND that this account is a member of the space. + */ +function chatPostMessage(spaceName, text) { + if (!spaceName) { Logger.log("spaceName required, e.g. spaces/AAAA1234"); return false; } + const token = ScriptApp.getOAuthToken(); + const res = UrlFetchApp.fetch( + "https://chat.googleapis.com/v1/" + spaceName + "/messages", { + method: "post", + contentType: "application/json; charset=UTF-8", + headers: { Authorization: "Bearer " + token }, + payload: JSON.stringify({ text: String(text).slice(0, 4000) }), + muteHttpExceptions: true + }); + const code = res.getResponseCode(); + if (code !== 200) { + Logger.log("[CHAT] post HTTP " + code + ": " + res.getContentText().slice(0, 300)); + return false; + } + Logger.log("Posted to " + spaceName); + return true; +} + +/** Store the webhook URL. Run once, then blank the argument out of history. */ +function setChatWebhook(url) { + if (!url) { + Logger.log("Pass the webhook URL: setChatWebhook('https://chat.googleapis.com/v1/spaces/.../messages?key=...&token=...')"); + return; + } + PropertiesService.getScriptProperties().setProperty(CHAT_WEBHOOK_PROP, url); + Logger.log("Stored " + CHAT_WEBHOOK_PROP + "."); + const ok = postToChatWebhook_( + "Receipt bot is connected to this space.\n\n" + + "You'll see a message here when receipts need a human decision, or when " + + "something breaks. Silence means it's running clean."); + Logger.log(ok ? "Test message sent — check the space." + : "Test FAILED. Re-copy the webhook URL (it must include both key= and token=)."); +} + +/** Read-only: is Chat wired up? Never prints the full URL. */ +function checkChatSetup() { + const url = PropertiesService.getScriptProperties().getProperty(CHAT_WEBHOOK_PROP); + if (!url) { + Logger.log(CHAT_WEBHOOK_PROP + ": MISSING — run chatListSpaces() to find your space,"); + Logger.log("then create a webhook in it and call setChatWebhook(url)."); + return false; + } + const m = url.match(/spaces\/([^\/]+)/); + Logger.log(CHAT_WEBHOOK_PROP + ": set (space " + (m ? m[1] : "?") + ")"); + return true; +} + +/* ───────────────────────────────────────────────────────────────────────── + * DIRECT MESSAGES — a different mechanism from webhooks. Read this. + * + * You CANNOT webhook a DM. An incoming webhook is registered inside a + * specific space through that space's own UI, so there is no way to create + * one for a 1:1 conversation with another person. Every "just make a webhook + * to DM someone" attempt dead-ends here. + * + * The real path is the Chat API with USER auth: + * 1. spaces.findDirectMessage?name=users/ -> the DM space id + * 2. spaces.messages.create on that space + * + * Both run as the signed-in Apps Script user, so the message genuinely comes + * from that person's account. Requires chat.spaces.readonly + chat.messages + * in appsscript.json, and the Chat API enabled on the script's Cloud project. + * ───────────────────────────────────────────────────────────────────────── */ + +/** + * Find the 1:1 DM space between the running user and someone else. + * Returns "spaces/XXXX" or null. Never throws. + */ +function chatFindDm(email) { + if (!email) { Logger.log("chatFindDm needs an email"); return null; } + try { + const res = UrlFetchApp.fetch( + "https://chat.googleapis.com/v1/spaces:findDirectMessage?name=" + + encodeURIComponent("users/" + email), { + method: "get", + headers: { Authorization: "Bearer " + ScriptApp.getOAuthToken() }, + muteHttpExceptions: true + }); + const code = res.getResponseCode(); + const body = res.getContentText(); + if (code !== 200) { + Logger.log("[CHAT] findDirectMessage HTTP " + code); + Logger.log(body.slice(0, 400)); + if (code === 403) { + Logger.log(""); + Logger.log("403 means one of two things, and the error does not say which:"); + Logger.log(" a) the chat.spaces.readonly scope is missing -> re-authorize"); + Logger.log(" b) the Chat API is not enabled on this script's Cloud project"); + Logger.log(" -> Project Settings -> Google Cloud Platform project, then"); + Logger.log(" enable 'Google Chat API' in that project."); + } + if (code === 404) { + Logger.log("No DM exists yet. Open Chat and send this person one message"); + Logger.log("by hand, then this will find it."); + } + return null; + } + const space = JSON.parse(body); + Logger.log("DM space with " + email + ": " + space.name); + return space.name; + } catch (e) { + Logger.log("[CHAT] findDirectMessage failed: " + e); + return null; + } +} + +/** + * Send a direct message to one person, as the running user. + * Returns true on success. + */ +function chatSendDm(email, text) { + const space = chatFindDm(email); + if (!space) return false; + return chatPostMessage(space, text); +} + +/** + * THE ONE TO RUN: DM Marge the receipt-process update. + * + * Everything is baked in so this is a single click. The message is written to + * be openly AI-attributed — Justin asked for that explicitly ("let her know + * it's AI that's saying it"), and it matters: a process change that lands + * without a named author reads as a decree from nowhere. + */ +function dmMargeReceiptUpdate() { + const MARGE = "gtrsupport@goldentouchremodeling.com"; + + const msg = +"*Hey Marge — this is Justin's AI, writing on his behalf.* He asked me to let you know what changed with receipts.\n" + +"\n" + +"You know how you've been hunting receipts one bank line at a time? Your Aug 14 note said you got through 08/04 and were stuck on four of them — checked Drive, checked Lowe's.com, checked email. That part is done now. The system does it.\n" + +"\n" + +"*What runs on its own*\n" + +"A receipt shows up — email, photo, or dropped in a job folder. It gets read, named Job_Date_Vendor_Invoice_$Amount, checked against everything already filed, then sent to QuickBooks and archived. *14 receipts filed in the last two days with nobody touching them.*\n" + +"\n" + +"*What it now refuses to do*\n" + +"• Book a Cash App payment or payroll advance as an expense — those get held and flagged for Gusto\n" + +"• Book an Amazon order twice — Intuit's own app owns those now\n" + +"• Book the same charge twice\n" + +"\n" + +"*What broke before and doesn't now*\n" + +"The reader died for nine days in August. Receipts quietly gave up, and the only sign was email piling up. It now retries on its own and sends a text if it can't. Ten minutes is the longest a failure can go unnoticed.\n" + +"\n" + +"*What still needs you*\n" + +"• The items sitting in _Needs Review — that's real judgment work\n" + +"• Missing receipt memos — only a person can write and sign those\n" + +"• Assigning jobs inside QuickBooks — still manual\n" + +"\n" + +"*What you can stop doing*\n" + +"Working the bank line by line looking for receipts. What's left in _Needs Review is the actual work now.\n" + +"\n" + +"One honest note: this is intake through QuickBooks, not the whole receipt problem. The duplicate groups you found in your audit still need reconciling, and job assignment in QBO is still by hand.\n" + +"\n" + +"— Justin's AI"; + + const ok = chatSendDm(MARGE, msg); + Logger.log(ok ? "Sent to " + MARGE + " — check Chat." + : "FAILED. Read the log above; it names the cause."); + return ok; +} diff --git a/docs/apps-script/requeueParkedReceipts.gs b/docs/apps-script/requeueParkedReceipts.gs index 474314457..e72489f19 100644 --- a/docs/apps-script/requeueParkedReceipts.gs +++ b/docs/apps-script/requeueParkedReceipts.gs @@ -42,6 +42,12 @@ const REQUEUE_CLEARED_KEYS = [ "lastError", "lastErrorAt", ]; +// Files parked as non-receipts or duplicates are NOT outage victims — the bot +// judged them correctly. Requeueing those would re-alert and re-duplicate, which +// is the noise we just fixed. Only clear files whose park was caused by the API +// being dead. +const REQUEUE_SKIP_IF = ["nonReceipt", "duplicateOf", "amazonAppOwned", "emailed"]; + function previewParkedReceipts() { requeueParkedReceipts_(true); } @@ -84,6 +90,17 @@ function requeueParkedReceipts_(dryRun) { continue; } + // Never requeue a file the bot judged correctly. + var judged = null; + for (var si = 0; si < REQUEUE_SKIP_IF.length; si++) { + if (state[REQUEUE_SKIP_IF[si]]) { judged = REQUEUE_SKIP_IF[si]; break; } + } + if (judged) { + skipped++; + Logger.log(" skip (" + judged + ", not an outage victim): " + name); + continue; + } + Logger.log(" " + (dryRun ? "would clear" : "CLEARED") + ": " + name + " [attempts=" + attempts + " runs=" + runs + " busy=" + busy + " park=" + (park || "-") + "]"); diff --git a/docs/apps-script/runReceiptAutomation.gs b/docs/apps-script/runReceiptAutomation.gs index a43a34e40..8e7dc9e58 100644 --- a/docs/apps-script/runReceiptAutomation.gs +++ b/docs/apps-script/runReceiptAutomation.gs @@ -29,7 +29,7 @@ * { runs, attempts, data, dedupOwned, dedupPk, dedupWeakOwned, dedupWeakPk, dedupWeak, * dedupWeakReason, emailing, emailed, refund, refundAlerted, nonReceipt, * nonReceiptAlerted, badFormat, badFormatAlerted, duplicateOf, weakDuplicateAlerted, - * parkReason, parkAlerted }. + * parkReason, parkAlerted, amazonAppOwned }. * * PARKED (not wired into this flow): * - WA "tax paid at source" recovery math -> taxPaidAtSource.parked.gs @@ -104,26 +104,14 @@ const QBO_OK_MIMES = ["application/pdf", "image/jpeg", "image/png"]; // The Gemini key comes from geminiApiKey_() in Config.gs, which reads Script Properties. // Models are tried IN ORDER — if one is overloaded (HTTP 503) or rate-limited (429), // the read falls through to the next, so one busy model never sinks the run. -// -// Model fallback chain, verified live 2026-08-19 against a real receipt. -// -// OUTAGE (2026-08-10 → 08-19): the previous chain was -// ["gemini-2.5-pro", "gemini-2.5-flash"]. BOTH died at once — 2.5-pro was -// retired ("no longer available to new users") while the whole project was -// separately blocked with 403 PERMISSION_DENIED for unlinked billing. Every -// receipt failed 3/3 for nine days and the only signal was mail piling up -// in Justin's inbox. Billing is now on Paid Tier and the 403s are gone, but -// gemini-2.5-pro is STILL 404 — a retired model never comes back. -// -// The chain therefore leads with a "-latest" alias, which Google repoints -// as models retire, so a single retirement can no longer take the pipeline -// down. Ordered fastest-verified first: gemini-flash-latest read this -// receipt in 2.0s, 2.5-flash in 2.6s, pro-latest in 4.5s. -// -// WARNING: the /models LISTING endpoint lies. It happily returned -// gemini-2.5-pro and gemini-2.5-flash all through the outage while -// generateContent 404'd and 403'd them. Any health check MUST make a real -// generateContent call — see checkVisionModels_() and the nightly watchdog. +// Chain verified live 2026-08-19 against a real receipt. The previous chain +// ["gemini-2.5-pro","gemini-2.5-flash"] died ENTIRELY on 08-10: 2.5-pro was +// RETIRED (404) while the project was simultaneously blocked (403), so nine +// days of receipts failed with no survivor. Leading with a "-latest" alias +// means Google repoints it as models retire, so one retirement can no longer +// take the pipeline down. WARNING: the /models LISTING endpoint LIES - it +// returned both dead models throughout the outage while generateContent 404d +// and 403d them. Health checks must call generateContent for real. const GEMINI_MODELS = ["gemini-flash-latest", "gemini-2.5-flash", "gemini-pro-latest"]; const MAX_AI_ATTEMPTS = 3; // failed AI reads per file, across runs const MAX_TOTAL_RUNS = 6; // total processing passes per file, across runs @@ -461,6 +449,54 @@ function processSingleFile(file, ctx, archive, needsReview) { if (docType === "non_receipt") { state.nonReceipt = true; setState(file, state); + + // DEDUP BEFORE PARKING (fixed 2026-08-19). This branch used to return here + // WITHOUT ever claiming a dedup key - the claim lives ~60 lines further down, + // past this early return. So a payroll screenshot was treated as brand new on + // every 10-minute pass: the same $973.25 CJ Havens PDF alerted at 2:59pm and + // again at 3:10pm and landed in _Needs Review twice, byte-identical. + // Left alone it emails forever and breeds copies. + // + // A non-receipt has no invoice number, so the strong key is unavailable; the + // weak key (vendor + date + amount) is the right identity. Claiming it makes + // the SECOND copy recognise the first as owner and stay silent. + var nrVendor = sanitize(aiData && aiData.vendor) || "Unknown"; + var nrDateRaw = normalizeDateStr(aiData && aiData.date); + var nrDate = isValidDate(nrDateRaw) ? nrDateRaw : driveDateStr(file); + var nrAmount = cleanMoney(aiData && aiData.total_amount); + // Only claim when the identity is REAL. A null/garbage vendor or a 0.00 + // amount collapses every unreadable non-receipt onto ONE key, so the second + // payroll screenshot of the day gets silently swallowed as a duplicate of an + // unrelated one. Better to alert twice than to lose a document. (Kimi.) + var nrIdentityUsable = isValidDate(nrDate) && + nrVendor !== "Unknown" && nrVendor.length > 2 && + nrAmount && nrAmount !== "0.00"; + if (!state.dedupWeakOwned && nrIdentityUsable) { + // NAMESPACED so a non-receipt can never block a real purchase. + // (Kimi BLOCKER: the first version claimed the ordinary weak key + // vendor|date|amount. A payroll advance to Charles Havens for $973.25 + // on 08-19 would then permanently quarantine a genuine receipt sharing + // those three values - and because this branch never calls + // releaseDedupClaims_, the block is FOREVER. A silently unbooked expense + // is far worse than the duplicate email this fix set out to stop.) + var nrKey = dedupPropKey("nonreceipt|" + makeWeakDedupKey(nrVendor, nrDate, nrAmount)); + state.dedupWeakPk = nrKey; // persist BEFORE claiming so a crash cannot orphan it + setState(file, state); + var nrOwner = claimDedupKey(nrKey, file.getId(), nrAmount); + if (nrOwner) { + // An earlier copy already owns this identity. Park quietly - alerting + // again is the exact noise this fix exists to stop. + state.duplicateOf = nrOwner.fileId; + state.nonReceiptAlerted = true; + setState(file, state); + file.moveTo(needsReview); + Logger.log(" > [NON-RECEIPT DUPLICATE] already owned by " + nrOwner.fileId + " - parked silently, no second alert."); + return; + } + state.dedupWeakOwned = true; + setState(file, state); + } + sendNonReceiptAlertIfNeeded(file, state, ctx, originalName); file.moveTo(needsReview); Logger.log(" > [NON-RECEIPT] parked in " + NEEDS_REVIEW_NAME + "; route to payroll (Gusto)."); @@ -599,7 +635,21 @@ function processSingleFile(file, ctx, archive, needsReview) { // it and skips — two runs can't both email the same document. "emailing" is marked // BEFORE the send; a crash mid-send leaves emailing=true/emailed=false so the next // pass re-sends WITH a duplicate warning. - if (!state.emailed || (state.refund && !state.refundAlerted)) { + // AMAZON: the native Intuit "Amazon Business Purchases" app (connected 2026-08-14) + // is the single writer for Amazon in QuickBooks. The bot must NOT also book these — + // two writers on one vendor is exactly how duplicates happen. The file still + // archives to the Drive receipt archive below (source of truth for + // expense-by-project) and keeps its dedup claim so stray copies cannot book either. + // Flip to false to hand booking back to the bot. + const AMAZON_APP_OWNS_BOOKING = true; + if (AMAZON_APP_OWNS_BOOKING && !state.emailed && + /amazon|amzn/i.test(String((aiData && aiData.vendor) || ""))) { + if (!state.amazonAppOwned) { + state.amazonAppOwned = true; + setState(file, state); + } + Logger.log(" > [AMAZON] Booking owned by the Amazon Business QBO app — archiving only, no QBO send."); + } else if (!state.emailed || (state.refund && !state.refundAlerted)) { const sendLock = RECEIPT_RUN_LOCK_HELD_ ? null : LockService.getScriptLock(); if (sendLock) sendLock.waitLock(30000); try { @@ -1110,15 +1160,9 @@ function analyzeDriveFileWithGemini(file, ctx) { const part = cand && cand.content && cand.content.parts && cand.content.parts[0]; const text = part && part.text; // The model answered; it just could not turn THIS document into usable data. - // An empty/blocked response means the model produced nothing. That is a - // MODEL failure, not proof this document is unreadable — so it falls - // through to the next model without charging the file. Only if EVERY - // model comes back empty does the busy ceiling eventually park it. - // (Kimi review #6: a garbage 200 is model-side, not document-side.) - if (!text) { Logger.log(" > [SERVICE] " + model + " returned no text (empty/safety-blocked). Trying next model."); break; } + if (!text) { Logger.log(" > [SERVICE] (" + model + ") returned no text (empty/safety-blocked). Trying next model."); break; } try { return JSON.parse(text); } - // Invalid JSON is the same class: the model failed to follow the contract. - catch (parseErr) { Logger.log(" > [SERVICE] " + model + " returned invalid JSON: " + parseErr + ". Trying next model."); break; } + catch (parseErr) { Logger.log(" > [SERVICE] (" + model + ") returned invalid JSON: " + parseErr + ". Trying next model."); break; } } if (code === 429 || code === 503) { // overloaded / rate-limited -> back off, then fall to next model @@ -1130,48 +1174,30 @@ function analyzeDriveFileWithGemini(file, ctx) { } if (code === 404) { // model id not available for this key -> try the next one - // SERVICE failure, never the document's fault. A 404 means the model was - // RETIRED — the document was never even read. - // - // This line used to set sawDecisiveFailure = true, and that is precisely - // the bug that parked five perfectly legible receipts on 2026-08-10..19: - // gemini-2.5-pro was retired (404) while the project was simultaneously - // blocked (403), so every file burned all three attempts against an API - // that never looked at it. The Fred Meyer receipt it "gave up" on reads - // in 1.4 seconds. - // - // The rule (Kimi review 2026-08-19): a per-model SERVICE failure must not - // charge document-side attempts. Falling through to the next model is the - // whole point of a chain — a 404 on ONE model while another works is - // harmless. If EVERY model fails this way, no attempt is charged either; - // the busy ceiling catches it and parks with aiUnavailable, which is a - // "park and escalate" state rather than "park and forget". - Logger.log(" > [SERVICE] " + model + " returned 404 (model retired/unavailable). Trying next model."); + // Decisive, NOT "busy": if every model 404s the project is misconfigured, and + // quietly retrying that for hours per file would delay the one alert that tells + // someone to fix it. Still not the document's fault — the alert says so. + // SERVICE failure - the document was never read, so it must not cost + // this file an attempt. This line used to set sawDecisiveFailure = true, + // and that is exactly what parked five legible receipts during the + // 2026-08-10..19 outage. A 404 on ONE model while another works is + // harmless - that is what a chain is for. + Logger.log(" > [SKIP] " + model + " not available (HTTP 404). Trying next model."); break; } - // 401/403 (revoked key, blocked project, not authorised) is a SERVICE failure: - // the document was never read, so it must not cost this file an attempt. - // - // The old code returned null here — a decisive "this file failed" verdict — - // with a comment arguing that retrying would "hide the outage". That reasoning - // was wrong and it cost nine days: on 2026-08-10 the project was blocked with - // 403 PERMISSION_DENIED, and because 403 was fatal, every receipt burned its - // three attempts and parked permanently. The outage was hidden anyway, because - // the alert was an email to an inbox nobody watches. - // - // Surfacing an outage is the WATCHDOG's job (an hourly real generateContent - // canary that pages Telegram), not the retry counter's. The counter's only job - // is to distinguish "this document is unreadable" from "the service is down". - // Breaking here falls through to the next model; if all models fail this way - // the pass is charged as a busy pass, not an attempt. + // 401/403 (revoked key, blocked project) is a SERVICE failure: the + // document was never read, so it must not cost this file an attempt. + // The old code returned null here, arguing a retry would "hide the + // outage". Wrong twice: the outage was hidden anyway (the alert was + // email to an unwatched inbox), and surfacing an outage is the + // WATCHDOG's job, not the retry counter's. if (code === 401 || code === 403) { - Logger.log(" > [SERVICE] " + model + " HTTP " + code + " (auth/project blocked). Not the document's fault. Trying next model."); + Logger.log(" > [SERVICE] " + model + " HTTP " + code + " (auth/project blocked). Trying next model."); break; } - // 400 = oversized or undecodable payload. THAT is about this document, so it - // stays decisive. + // 400 = oversized/undecodable payload. THAT is about this document. sawDecisiveFailure = true; Logger.log("API Error (Fatal, " + model + "): " + response.getContentText()); return null; @@ -1373,7 +1399,15 @@ function analyzeMultiPageMapWithGemini(file) { } if (code === 429 || code === 503) { Utilities.sleep(Math.pow(2, a + 1) * 1000); continue; } if (code === 404) break; // model id not on this key -> next model - return null; // 400/403/... -> fatal + // 401/403 = auth/project blocked. SERVICE failure, not this document's + // fault - fall through to the next model, exactly like the main + // classifier. (Kimi SHOULD-FIX: this path still treated 403 as fatal, so + // during the billing outage a multi-page PDF would have been parked as + // unsplittable while single-page reads were correctly retried. Two + // classifiers disagreeing about one HTTP code is how the original bug + // survived review.) + if (code === 401 || code === 403) break; + return null; // 400 (oversized/undecodable payload) -> fatal } catch (e) { Utilities.sleep(2000); } } Logger.log(" > [SPLIT-MAP] " + model + " unavailable; trying next model."); diff --git a/docs/apps-script/selfHeal.gs b/docs/apps-script/selfHeal.gs new file mode 100644 index 000000000..77674039f --- /dev/null +++ b/docs/apps-script/selfHeal.gs @@ -0,0 +1,346 @@ +/** + * selfHeal.gs — the receipt pipeline watches and repairs itself. + * + * Justin, 2026-08-19: "chron job watching it every 10 minutes, fixing it as + * needed, improving it." + * + * WHAT THIS EXISTS TO PREVENT + * From 08-10 to 08-19 the bot read nothing. gemini-2.5-pro was retired (404) + * while the Cloud project was blocked for billing (403). Both entries in the + * model chain died at once, every receipt burned its three attempts against a + * dead API, and eleven good documents parked permanently. The only signal was + * mail piling up in an inbox nobody watches. Nine days. + * + * Three functions, each safe to run on a trigger: + * + * pipelineSelfHeal() every 10 min, right after runReceiptAutomation + * pipelineHealthCheck() hourly — the canary + * pipelineDailyReport() once a day — the human summary + * + * DESIGN RULES (from the Kimi review of this incident) + * - Auto-requeue ONLY files parked because the AI was unavailable. A file the + * bot judged (non-receipt, duplicate, already emailed) is never touched. + * - Requeue only after PROOF the API works — a real generateContent call, not + * the /models listing endpoint, which lied throughout the outage. + * - Bound everything. Max 2 auto-requeues per file, max 10 files per pass. + * A wrong classifier must not create an infinite loop. + * - Escalate rather than retry silently. Repeated failure pages a human. + */ + +const SELFHEAL_MAX_REQUEUES_PER_FILE = 2; +const SELFHEAL_MAX_FILES_PER_PASS = 10; +const SELFHEAL_BACKLOG_ALERT = 20; +const SELFHEAL_PROP_LAST_HEALTHY = "selfheal_last_healthy_iso"; +const SELFHEAL_PROP_LAST_ALERT = "selfheal_last_alert_iso"; +const SELFHEAL_ALERT_COOLDOWN_MIN = 120; // don't nag more than every 2h + +/** A 1x1 white JPEG — enough to prove the endpoint accepts an image request. */ +const SELFHEAL_TINY_JPEG = + "/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0a" + + "HBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAABAAEBAREA/8QAFAABAAAAAAAA" + + "AAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8AKp//2Q=="; + +/** + * Can each model in the chain actually read an image RIGHT NOW? + * + * Deliberately calls generateContent. The /models listing endpoint returned + * gemini-2.5-pro and gemini-2.5-flash happily all through the outage while + * every real call 404'd and 403'd. Listing a model is not being able to use it. + */ +function checkVisionModels_() { + const key = geminiApiKey_(); + const results = []; + for (let i = 0; i < GEMINI_MODELS.length; i++) { + const model = GEMINI_MODELS[i]; + const url = "https://generativelanguage.googleapis.com/v1beta/models/" + + model + ":generateContent?key=" + key; + const payload = { + contents: [{ + parts: [ + { text: "Reply with the single word: OK" }, + { inline_data: { mime_type: "image/jpeg", data: SELFHEAL_TINY_JPEG } } + ] + }] + }; + let code = 0, detail = ""; + try { + const res = UrlFetchApp.fetch(url, { + method: "post", + contentType: "application/json", + payload: JSON.stringify(payload), + muteHttpExceptions: true + }); + code = res.getResponseCode(); + if (code !== 200) { + try { + const err = JSON.parse(res.getContentText()).error || {}; + detail = (err.status || "") + " " + String(err.message || "").slice(0, 110); + } catch (e) { detail = res.getContentText().slice(0, 110); } + } + } catch (netErr) { + detail = "network: " + String(netErr).slice(0, 100); + } + // 429 is quota — transient and self-clearing, not an outage. + const ok = (code === 200 || code === 429); + results.push({ model: model, ok: ok, code: code, detail: detail }); + } + return results; +} + +/** Files parked because the AI was UNAVAILABLE — the only auto-requeue class. */ +function outageVictims_(folder) { + const out = []; + const files = folder.getFiles(); + while (files.hasNext()) { + const f = files.next(); + if (f.getName() === "desktop.ini") continue; + let st = {}; + try { st = JSON.parse(f.getDescription() || "{}") || {}; } catch (e) { continue; } + + // Never touch a file the bot judged correctly. + if (st.nonReceipt || st.duplicateOf || st.amazonAppOwned || st.emailed) continue; + // ONLY outage victims. A file parked for zeroTotal or multiDoc was judged + // on its CONTENT - requeueing replays the same verdict and burns the bounded + // retry budget for nothing. (Kimi SHOULD-FIX: the first version requeued + // every parked file, so a $0.00 receipt ate both auto-requeues re-deciding + // what was already decided.) + if (st.parkReason !== PARK_AI_UNAVAILABLE && st.parkReason !== PARK_GAVE_UP) continue; + + // A gaveUp park is only an outage victim if NOTHING decisive was ever learned. + // If the AI truly read it and failed, a healthy API changes nothing - a human + // must look. + if (st.parkReason === PARK_GAVE_UP && Number(st.busyPasses || 0) === 0) continue; + // Bound the loop. + if (Number(st.autoRequeues || 0) >= SELFHEAL_MAX_REQUEUES_PER_FILE) continue; + + out.push({ file: f, state: st }); + } + return out; +} + +/** + * Runs every 10 minutes after the main scan. Silent when healthy. + */ +function pipelineSelfHeal() { + const props = PropertiesService.getScriptProperties(); + const health = checkVisionModels_(); + const working = health.filter(function (h) { return h.ok; }); + + if (!working.length) { + // Nothing can be read. Do NOT requeue into a dead API — that would burn + // the bounded requeue budget for no reason. Alert and wait. + const lines = health.map(function (h) { + return " " + h.model + ": HTTP " + h.code + " " + h.detail; + }).join("\n"); + selfHealAlert_( + "Receipt bot: AI IS DOWN — nothing can be read", + "Every vision model failed a real read test.\n\n" + lines + "\n\n" + + "Receipts will pile up in _Needs Review until this is fixed.\n" + + "403 = the Google Cloud project is blocked (check billing at\n" + + " https://aistudio.google.com/apikey)\n" + + "404 = the model was retired; update GEMINI_MODELS.\n\n" + + "Nothing was requeued — that would waste the retry budget on a dead API."); + return; + } + + props.setProperty(SELFHEAL_PROP_LAST_HEALTHY, new Date().toISOString()); + + // Some models dead but not all: worth knowing, not worth stopping for. + const dead = health.filter(function (h) { return !h.ok; }); + if (dead.length) { + Logger.log("[SELFHEAL] degraded chain: " + + dead.map(function (d) { return d.model + " (" + d.code + ")"; }).join(", ")); + } + + // The API works — so anything parked for unavailability deserves another go. + const folder = getOrCreateFolder( + DriveApp.getFolderById(NEW_RECEIPTS_FOLDER_ID), NEEDS_REVIEW_NAME); + const victims = outageVictims_(folder).slice(0, SELFHEAL_MAX_FILES_PER_PASS); + if (!victims.length) return; // healthy and nothing stuck: stay silent + + const inbox = DriveApp.getFolderById(NEW_RECEIPTS_FOLDER_ID); + const freed = []; + for (let i = 0; i < victims.length; i++) { + const v = victims[i]; + const next = {}; + for (const k in v.state) { + if (["attempts", "runs", "busyPasses", "parkReason", "parkAlerted", + "lastError", "lastErrorAt"].indexOf(k) === -1) next[k] = v.state[k]; + } + next.autoRequeues = Number(v.state.autoRequeues || 0) + 1; + next.requeuedAt = new Date().toISOString(); + try { + v.file.setDescription(JSON.stringify(next)); + v.file.moveTo(inbox); // back into the scan path + freed.push(v.file.getName()); + } catch (e) { + Logger.log("[SELFHEAL] could not requeue " + v.file.getName() + ": " + e); + } + } + + if (freed.length) { + Logger.log("[SELFHEAL] requeued " + freed.length + " file(s): " + freed.join(", ")); + selfHealAlert_( + "Receipt bot: recovered " + freed.length + " receipt(s) automatically", + "The AI is readable again, so receipts parked during the outage were put\n" + + "back in the queue. They will process on the next run.\n\n " + + freed.join("\n ") + "\n\n" + + "No action needed — this message exists so the recovery is visible."); + } +} + +/** Hourly canary. Alerts the moment ANY model stops working. */ +function pipelineHealthCheck() { + const health = checkVisionModels_(); + const dead = health.filter(function (h) { return !h.ok; }); + if (!dead.length) return; // silence = healthy + + const allDead = dead.length === health.length; + const lines = health.map(function (h) { + return " " + h.model + ": " + (h.ok ? "OK" : "HTTP " + h.code + " " + h.detail); + }).join("\n"); + + selfHealAlert_( + allDead ? "Receipt bot: ALL vision models are down" + : "Receipt bot: " + dead.length + " model(s) degraded", + (allDead + ? "Nothing can be read. Receipts are piling up.\n\n" + : "The chain still works, but fix this before the rest go.\n\n") + + lines + "\n\nChecked with a real image read, not the models list."); +} + +/** Once a day: backlog and staleness. The lagging indicator, kept honest. */ +function pipelineDailyReport() { + const props = PropertiesService.getScriptProperties(); + const folder = getOrCreateFolder( + DriveApp.getFolderById(NEW_RECEIPTS_FOLDER_ID), NEEDS_REVIEW_NAME); + + let total = 0, stuck = 0; + const files = folder.getFiles(); + while (files.hasNext()) { + const f = files.next(); + if (f.getName() === "desktop.ini") continue; + total++; + let st = {}; + try { st = JSON.parse(f.getDescription() || "{}") || {}; } catch (e) {} + if (st.parkReason && !st.nonReceipt && !st.duplicateOf) stuck++; + } + + const lastHealthy = props.getProperty(SELFHEAL_PROP_LAST_HEALTHY); + const hoursSince = lastHealthy + ? Math.round((Date.now() - new Date(lastHealthy).getTime()) / 3600000) + : null; + + const problems = []; + if (total >= SELFHEAL_BACKLOG_ALERT) { + problems.push(total + " files in _Needs Review (" + stuck + " actionable)."); + } + if (hoursSince !== null && hoursSince > 6) { + problems.push("The AI has not passed a health check in " + hoursSince + " hours."); + } + if (!problems.length) return; // silence = healthy + + selfHealAlert_("Receipt bot: daily check", problems.join("\n") + + "\n\nRun auditNeedsReview() for a file-by-file verdict."); +} + +/** + * Send to Telegram as well as email. + * + * Kimi's sharpest finding: the whole 9-day outage happened because the ONLY + * signal was mail to ALERT_EMAIL, an inbox nobody watches. Alerting to that + * same inbox fixes nothing. Telegram is what Justin actually reads. + * Credentials live in Script Properties (TELEGRAM_BOT_TOKEN / + * TELEGRAM_CHAT_ID) - never in code. Skipped silently if unset, and a + * Telegram failure never blocks the email. + */ +function telegramAlert_(text) { + try { + const props = PropertiesService.getScriptProperties(); + const token = props.getProperty("TELEGRAM_BOT_TOKEN"); + const chat = props.getProperty("TELEGRAM_CHAT_ID"); + if (!token || !chat) return false; + const res = UrlFetchApp.fetch( + "https://api.telegram.org/bot" + token + "/sendMessage", { + method: "post", + contentType: "application/json", + payload: JSON.stringify({ chat_id: chat, text: text.slice(0, 3900) }), + muteHttpExceptions: true + }); + return res.getResponseCode() === 200; + } catch (e) { + Logger.log("[SELFHEAL] telegram failed: " + e); + return false; + } +} + +/** One alert channel, rate-limited so a broken pipeline cannot spam. */ +function selfHealAlert_(subject, body) { + const props = PropertiesService.getScriptProperties(); + const last = props.getProperty(SELFHEAL_PROP_LAST_ALERT); + if (last) { + const mins = (Date.now() - new Date(last).getTime()) / 60000; + if (mins < SELFHEAL_ALERT_COOLDOWN_MIN) { + Logger.log("[SELFHEAL] alert suppressed (cooldown): " + subject); + return; + } + } + props.setProperty(SELFHEAL_PROP_LAST_ALERT, new Date().toISOString()); + // Fan out to every channel that is actually watched. Each is + // independent: one failing must never suppress the others, and none + // of them may throw - a broken alert path cannot break the pipeline. + // Google Chat is where the OFFICE sees it (Marge); Telegram is where + // Justin sees it; email is the archive of record. + try { postToChatWebhook_(subject + "\n\n" + body, { threadKey: "receipt-bot" }); } + catch (e) { Logger.log("[SELFHEAL] chat alert failed: " + e); } + // Telegram FIRST - it is the channel that gets read. + telegramAlert_(subject + "\n\n" + body); + try { MailApp.sendEmail(ALERT_EMAIL, subject, body); } + catch (e) { Logger.log("[SELFHEAL] alert email failed: " + e); } +} + +/** + * Install the triggers. Run ONCE by hand; it clears its own duplicates first. + */ +function installSelfHealTriggers() { + const wanted = { + pipelineSelfHeal: "every10", + pipelineHealthCheck: "hourly", + pipelineDailyReport: "daily" + }; + const existing = ScriptApp.getProjectTriggers(); + for (let i = 0; i < existing.length; i++) { + if (wanted[existing[i].getHandlerFunction()]) { + ScriptApp.deleteTrigger(existing[i]); + } + } + ScriptApp.newTrigger("pipelineSelfHeal").timeBased().everyMinutes(10).create(); + ScriptApp.newTrigger("pipelineHealthCheck").timeBased().everyHours(1).create(); + ScriptApp.newTrigger("pipelineDailyReport").timeBased().everyDays(1).atHour(7).create(); + Logger.log("Installed: pipelineSelfHeal (10 min), pipelineHealthCheck (hourly), " + + "pipelineDailyReport (daily 7am)."); +} + +/** Read-only: what would self-heal do right now? Changes nothing. */ +function previewSelfHeal() { + const health = checkVisionModels_(); + Logger.log("--- MODEL HEALTH (real image read) ---"); + for (let i = 0; i < health.length; i++) { + const h = health[i]; + Logger.log(" " + (h.ok ? "OK " : "DEAD ") + h.model + + " HTTP " + h.code + (h.detail ? " " + h.detail : "")); + } + const folder = getOrCreateFolder( + DriveApp.getFolderById(NEW_RECEIPTS_FOLDER_ID), NEEDS_REVIEW_NAME); + const victims = outageVictims_(folder); + Logger.log(""); + Logger.log("--- WOULD REQUEUE (" + victims.length + ") ---"); + for (let i = 0; i < victims.length; i++) { + Logger.log(" " + victims[i].file.getName() + + " [park=" + victims[i].state.parkReason + + " attempts=" + (victims[i].state.attempts || 0) + + " autoRequeues=" + (victims[i].state.autoRequeues || 0) + "]"); + } + Logger.log(""); + Logger.log("Nothing was changed. Run pipelineSelfHeal() to apply."); +} diff --git a/docs/apps-script/setupTelegramAlerts.gs b/docs/apps-script/setupTelegramAlerts.gs new file mode 100644 index 000000000..108ddd375 --- /dev/null +++ b/docs/apps-script/setupTelegramAlerts.gs @@ -0,0 +1,59 @@ +/** + * setupTelegramAlerts.gs — one-time credential install + trigger setup. + * + * WHY THIS FILE EXISTS + * The 9-day receipt outage (2026-08-10..19) was invisible because the ONLY + * alert channel was email to ALERT_EMAIL, an inbox nobody watches. Fixing + * the models without fixing the channel would have set up the next silent + * outage. Telegram is the channel Justin actually reads. + * + * Credentials go in Script Properties, never in source — this file is in a + * git repo. Paste the token below, run once, then CLEAR IT AGAIN before + * committing (or just never commit the filled-in version). + * + * HOW TO RUN + * 1. Fill in TG_TOKEN below. + * 2. Select setupTelegramAlerts -> Run. It sends a test message. + * 3. Blank TG_TOKEN out again. + * 4. Select installSelfHealTriggers -> Run. (Registers the 10-min + * self-heal, hourly canary, and daily report. Apps Script does not + * allow creating triggers remotely, so this step is manual, once.) + */ + +// Paste, run, then blank. Never commit a real value here. +const TG_TOKEN = ""; +const TG_CHAT = "8681967411"; // Justin's Telegram chat id + +function setupTelegramAlerts() { + if (!TG_TOKEN) { + Logger.log("TG_TOKEN is empty — paste the bot token at the top of this file first."); + return; + } + const props = PropertiesService.getScriptProperties(); + props.setProperty("TELEGRAM_BOT_TOKEN", TG_TOKEN); + props.setProperty("TELEGRAM_CHAT_ID", TG_CHAT); + Logger.log("Stored TELEGRAM_BOT_TOKEN and TELEGRAM_CHAT_ID in Script Properties."); + + const ok = telegramAlert_( + "Receipt bot: alerts are wired to Telegram.\n\n" + + "This is the channel that gets read. From now on a dead vision API, a " + + "growing backlog, or an auto-recovery shows up here instead of dying " + + "quietly in an inbox.\n\n" + + "Silence means healthy."); + Logger.log(ok ? "Test message sent — check Telegram." + : "Test message FAILED. Check the token and chat id."); +} + +/** Read-only: are the credentials present? Never prints the token. */ +function checkTelegramSetup() { + const props = PropertiesService.getScriptProperties(); + const tok = props.getProperty("TELEGRAM_BOT_TOKEN"); + const chat = props.getProperty("TELEGRAM_CHAT_ID"); + Logger.log("TELEGRAM_BOT_TOKEN: " + (tok ? "set (" + tok.length + " chars)" : "MISSING")); + Logger.log("TELEGRAM_CHAT_ID: " + (chat || "MISSING")); + if (tok && chat) { + Logger.log("Alerts will reach Telegram."); + } else { + Logger.log("Alerts fall back to email only — run setupTelegramAlerts()."); + } +} diff --git a/package.json b/package.json index 9a0e29808..0bc269229 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "lint": "eslint", "test:qbo-expense-sync": "tsx --test tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts", "test:qbo-receipt-push": "tsx --test tests/qbo-receipt-push.test.ts", - "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/deposit-attribution.test.ts", + "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", "test:ai-review-reasonableness": "tsx --test tests/ai-review-reasonableness.test.ts", @@ -20,7 +20,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", + "test:unit": "tsx --test tests/pdf-watermark.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 7c639df61..34dc3aa72 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -2865,6 +2865,17 @@ model BankImage { /// ledger movement. amountCents Int? + /// Check-payer extraction (scripts/extract-check-payers.mjs, applied via + /// scripts/apply-check-payer-extraction.mjs). payerName = who wrote the + /// check (top-left block); memoText = the memo/"for" line. The MICR line / + /// routing / account numbers are NEVER extracted or stored — banned in the + /// prompt and scrubbed in code. extractedAt+extractionModel are stamped + /// together (CHECK BankImage_extraction_pair) and make replays a no-op. + payerName String? + memoText String? + extractedAt DateTime? @db.Timestamptz(6) + extractionModel String? + createdAt DateTime @default(now()) @db.Timestamptz(6) updatedAt DateTime @default(now()) @updatedAt @db.Timestamptz(6) diff --git a/scripts/apply-check-payer-extraction.mjs b/scripts/apply-check-payer-extraction.mjs new file mode 100644 index 000000000..c323ad076 --- /dev/null +++ b/scripts/apply-check-payer-extraction.mjs @@ -0,0 +1,151 @@ +// Additive migration: check-payer extraction columns on BankImage. +// +// WHY (2026-08-22): the ledger's inbound lines say only "DEPOSIT - DDA/MMKT". +// The image names WHO PAID US — extracting the payer name and memo line lets +// Beverly (bookkeeping) attribute deposits to clients and jobs. Four nullable +// columns, nothing else: +// +// payerName TEXT — name block top-left of the check +// memoText TEXT — the "memo"/"for" line +// extractedAt TIMESTAMPTZ — idempotency marker for the extractor +// extractionModel TEXT — which model produced the values +// +// DELIBERATELY ABSENT: routing number, account number, MICR line. Those are +// NEVER extracted and NEVER stored — see scripts/extract-check-payers.mjs, +// which bans them in the prompt AND drops them in code. Do not add columns +// for them. +// +// Additive and idempotent: ADD COLUMN IF NOT EXISTS only. No existing column +// or row is touched. Safe to re-run. +// +// node scripts/apply-check-payer-extraction.mjs --dry-run +// node scripts/apply-check-payer-extraction.mjs --yes --expect-db --expect-host +// +// --expect-db and --expect-host are BOTH required alongside --yes, matching +// scripts/apply-bank-image.mjs: "--yes" alone only proves you meant to run +// something, and a database NAME alone doesn't prove which SERVER it's on. +// +// Apply BEFORE deploying any build whose Prisma client selects these columns +// (P2022), and BEFORE running extract-check-payers.mjs with --commit. +import { PrismaClient } from "@prisma/client"; +import fs from "node:fs"; +import { fileURLToPath } from "node:url"; + +export function resolveDatabaseUrl() { + if (process.env.DATABASE_URL) return { url: process.env.DATABASE_URL, from: "process.env.DATABASE_URL" }; + for (const file of [".env.local", ".env"]) { + if (!fs.existsSync(file)) continue; + const match = fs.readFileSync(file, "utf8").match(/^DATABASE_URL\s*=\s*"?([^"\n]+)"?/m); + if (match) return { url: match[1], from: file }; + } + throw new Error("DATABASE_URL not found in process.env, .env.local, or .env"); +} + +export function maskUrl(url) { + return url.replace(/:[^:@]*@/, ":****@"); +} + +function readFlagValue(flag) { + const idx = process.argv.indexOf(flag); + return idx >= 0 ? process.argv[idx + 1] : undefined; +} + +/** Pure comparison, exported for unit testing (mirrors apply-bank-image.mjs). */ +export function targetMatches(actual, expectDb, expectHost) { + if (!actual || typeof actual !== "object") return false; + if (String(actual.db ?? "") !== String(expectDb ?? "")) return false; + const host = String(actual.host ?? ""); + const wanted = String(expectHost ?? ""); + if (host === wanted) return true; + return host !== "" && wanted !== "" && (host.includes(wanted) || wanted.includes(host)); +} + +export const NEW_COLUMNS = ["payerName", "memoText", "extractedAt", "extractionModel"]; + +export const statements = [ + `ALTER TABLE "BankImage" ADD COLUMN IF NOT EXISTS "payerName" TEXT`, + `ALTER TABLE "BankImage" ADD COLUMN IF NOT EXISTS "memoText" TEXT`, + `ALTER TABLE "BankImage" ADD COLUMN IF NOT EXISTS "extractedAt" TIMESTAMPTZ(6)`, + `ALTER TABLE "BankImage" ADD COLUMN IF NOT EXISTS "extractionModel" TEXT`, + + // The extractor stamps extractedAt and extractionModel together — a row + // claiming extraction without saying what did it (or vice versa) is a bug. + `DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'BankImage_extraction_pair') THEN + ALTER TABLE "BankImage" ADD CONSTRAINT "BankImage_extraction_pair" + CHECK (("extractedAt" IS NULL) = ("extractionModel" IS NULL)); + END IF; + END $$`, +]; + +async function main() { + if (process.argv.includes("--dry-run")) { + console.log("DRY RUN — the following SQL would be applied (nothing executed):\n"); + for (const sql of statements) console.log(sql.replace(/\n\s+/g, "\n ") + ";\n"); + console.log("Re-run with --yes --expect-db --expect-host to apply."); + return; + } + + if (!process.argv.includes("--yes")) { + console.error("Refusing to run without --yes (and --expect-db / --expect-host). Use --dry-run to preview."); + process.exit(1); + } + const expectDb = readFlagValue("--expect-db") ?? process.env.BANK_LEDGER_EXPECT_DB; + const expectHost = readFlagValue("--expect-host") ?? process.env.BANK_LEDGER_EXPECT_HOST; + if (!expectDb || !expectHost) { + console.error("Both --expect-db and --expect-host are required (or BANK_LEDGER_EXPECT_DB / BANK_LEDGER_EXPECT_HOST)."); + process.exit(1); + } + + const { url, from } = resolveDatabaseUrl(); + console.log(`DATABASE_URL from ${from}: ${maskUrl(url)}`); + const prisma = new PrismaClient({ datasources: { db: { url } } }); + + try { + const [actual] = await prisma.$queryRawUnsafe( + `SELECT current_database() AS db, COALESCE(host(inet_server_addr()), '') AS host`, + ); + console.log(`connected to db="${actual.db}" host="${actual.host}"`); + if (!targetMatches(actual, expectDb, expectHost)) { + console.error(`REFUSING: expected db="${expectDb}" host="${expectHost}" but connected to db="${actual.db}" host="${actual.host}".`); + process.exit(1); + } + + for (const sql of statements) { + const label = sql.replace(/\s+/g, " ").slice(0, 84); + process.stdout.write(` ${label} ... `); + await prisma.$executeRawUnsafe(sql); + console.log("ok"); + } + + // Verify shape rather than trusting the run. + const rows = await prisma.$queryRawUnsafe( + `SELECT column_name FROM information_schema.columns WHERE table_schema='public' AND table_name='BankImage'`, + ); + const found = new Set(rows.map(r => r.column_name)); + const missing = NEW_COLUMNS.filter(c => !found.has(c)); + if (missing.length) { + console.error(`VERIFY FAILED: BankImage missing columns: ${missing.join(", ")}`); + process.exit(1); + } + const [pair] = await prisma.$queryRawUnsafe( + `SELECT 1 AS ok FROM pg_constraint WHERE conname = 'BankImage_extraction_pair'`, + ); + if (!pair) { + console.error("VERIFY FAILED: constraint BankImage_extraction_pair missing"); + process.exit(1); + } + console.log(`verified ${NEW_COLUMNS.length} columns + 1 constraint`); + console.log("\nCheck-payer extraction migration applied and verified."); + } finally { + await prisma.$disconnect(); + } +} + +const isMainModule = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]; +if (isMainModule) { + main().catch(error => { + console.error(error); + process.exitCode = 1; + }); +} diff --git a/scripts/extract-check-payers.mjs b/scripts/extract-check-payers.mjs new file mode 100644 index 000000000..cbb01eab7 --- /dev/null +++ b/scripts/extract-check-payers.mjs @@ -0,0 +1,516 @@ +// Extract WHO PAID US from check/deposit images via Gemini Vision. +// +// A WTB inbound line says only "OTHER DEPOSITS DEPOSIT - DDA/MMKT". The image +// is the only evidence naming the payer, and the memo line often names the +// job. This reads BankImage rows (kind CHECK_FRONT / DEPOSIT_PHOTO by +// default) that have not been extracted yet, sends each image to Gemini +// Vision, and stores ONLY payerName + memoText (+ extractedAt + +// extractionModel) via scripts/apply-check-payer-extraction.mjs's columns. +// +// ── HARD PRIVACY RULE ──────────────────────────────────────────────────── +// The MICR line / routing number / account number are NEVER extracted, +// NEVER stored, NEVER printed. This is enforced twice: +// 1. The prompt forbids reading the bottom MICR strip at all. +// 2. scrubExtraction() drops ANY field whose digit content — after +// collapsing EVERY common separator (space dash dot slash comma parens) +// AND interleaved letters — totals 8 or more digits and is not the known +// check number or amount, and logs a warning. Recognizable calendar +// dates (ISO 2026-08-13 / US 8/13/2026) are exempted before counting. +// A field dropped by layer 2 also flips needsReview: --commit then stores +// NULL for payerName AND memoText on that row (extractedAt/extractionModel +// still set so replay skips it) and logs the row for human review. +// Do not weaken either layer. Do not add columns for these values. +// ───────────────────────────────────────────────────────────────────────── +// +// IDEMPOTENT: rows with extractedAt NOT NULL are skipped. Re-running is a +// no-op for already-extracted images. +// +// node scripts/extract-check-payers.mjs --dry-run # default; nothing written +// node scripts/extract-check-payers.mjs --dry-run --from-manifest # pre-DDL test straight from the Drive manifest +// node scripts/extract-check-payers.mjs --commit --limit 10 # write results (requires the DDL applied) +// node scripts/extract-check-payers.mjs --report # REVIEW REPORT only: suggest payer→Client / memo→Project matches +// +// The REVIEW REPORT is print-only. It NEVER writes BankImageMatch — that +// table means a HUMAN said yes (see prisma/schema.prisma). Beverly/Justin +// confirm matches; this script only suggests. +// +// DATABASE_URL / GEMINI_API_KEY come from the environment or .env.local, +// never argv. +import { PrismaClient } from "@prisma/client"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +export const IMAGES_DIR = + "I:/My Drive/2025 Reconciliation/Washington Trust Bank/Check Images"; +export const MANIFEST_PATH = path.join(IMAGES_DIR, "_manifest.json"); + +export const GEMINI_MODEL = "gemini-3-flash-preview"; +export const DEFAULT_KINDS = ["CHECK_FRONT", "DEPOSIT_PHOTO"]; +export const ALL_KINDS = ["CHECK_FRONT", "CHECK_BACK", "DEPOSIT_SLIP", "DEPOSIT_PHOTO"]; +export const DEFAULT_LIMIT = 25; + +// ── env ────────────────────────────────────────────────────────────────── +export function resolveEnv(name) { + if (process.env[name]) return process.env[name]; + for (const file of [".env.local", ".env"]) { + if (!fs.existsSync(file)) continue; + const match = fs.readFileSync(file, "utf8").match(new RegExp(`^${name}\\s*=\\s*"?([^"\\r\\n]+)"?`, "m")); + if (match) return match[1]; + } + return null; +} + +// ── MICR / routing / account guard ─────────────────────────────────────── +// Routing numbers are 9 digits; account numbers 8-12+. A check number is +// short (typically 3-5 digits) and the amount's digit string is 3-7. So any +// candidate run whose TOTAL digit count reaches 8 — after stripping every +// common separator (space, dash, dot, slash, comma, parens) AND interleaved +// letters — is treated as banked-number leakage unless the digit string is +// explicitly allowed, and the WHOLE field is dropped. Consecutive-run-only +// checks are NOT enough: "1234 5678", "123.456.789", "123/456/789" and +// "A1B2C3D4E5F6G7H8" all hide an account/routing number without ever showing +// 8 consecutive digits. +export const BANNED_DIGIT_RUN = /\d{8,}/g; // kept for reference/compat; superseded by the total-count rule below +export const BANNED_TOTAL_DIGITS = 8; +const MICR_SYMBOLS = /[\u2446\u2447\u2448\u2449]/; // ⑆⑇⑈⑉ MICR transit/on-us glyphs + +// Plausible calendar dates are exempt from digit counting (an ISO date is 8 +// digits and must survive). Month/day ranges are validated so an account +// disguised as "12/34/5678" does NOT qualify. +const ISO_DATE = /\b(?:19|20)\d{2}-(?:0?[1-9]|1[0-2])-(?:0?[1-9]|[12]\d|3[01])\b/g; +const US_DATE = /\b(?:0?[1-9]|1[0-2])[/-](?:0?[1-9]|[12]\d|3[01])[/-](?:19|20)?\d{2}\b/g; + +// A candidate token run: maximal chunk of letters/digits joined by the +// common separators. Letters are included so "A1B2C3D4E5F6G7H8" is one run. +const TOKEN_RUN = /[0-9A-Za-z][0-9A-Za-z\s\-./,()]*/g; + +/** + * Pure. Returns { value, dropped, truncated } — dropped is a reason string + * when the field was discarded; truncated is true when maxLen was applied. + * `allow` lists digit strings that are legitimately long (never expected in + * practice, but the check number is allowed on principle: it is already + * stored openly on the row). The allow check runs on the SEPARATOR-STRIPPED + * digit string, so an allowed "12345678" also covers "1234 5678". + * + * @param {unknown} value + * @param {string[]} [allow] + * @param {number | null} [maxLen] + */ +export function scrubField(value, allow = [], maxLen = null) { + if (value === null || value === undefined) return { value: null, dropped: null, truncated: false }; + const text = String(value).trim(); + if (!text) return { value: null, dropped: null, truncated: false }; + if (MICR_SYMBOLS.test(text)) { + return { value: null, dropped: "contains MICR symbols", truncated: false }; + } + // Blank out plausible calendar dates, then count TOTAL digits per token + // run after stripping ALL non-digits (separators and letters alike). + const dateless = text.replace(ISO_DATE, " ").replace(US_DATE, " "); + const banned = (dateless.match(TOKEN_RUN) ?? []) + .map(run => run.replace(/\D/g, "")) + .filter(digits => digits.length >= BANNED_TOTAL_DIGITS && !allow.includes(digits)); + if (banned.length) { + return { + value: null, + dropped: `contains ${banned.length} banned digit run(s) (routing/account pattern)`, + truncated: false, + }; + } + if (maxLen !== null && text.length > maxLen) { + return { value: text.slice(0, maxLen).trimEnd(), dropped: null, truncated: true }; + } + return { value: text, dropped: null, truncated: false }; +} + +export const PAYER_NAME_MAX = 120; +export const MEMO_TEXT_MAX = 200; + +/** + * Pure. Scrubs a raw Gemini response into the ONLY values we keep: + * payerName and memoText. Everything else (date, amount, check number) is + * used for cross-check logging only and is never stored by this script. + * + * Length caps: payerName 120 chars, memoText 200 chars — anything longer is + * truncated and flagged (a real name/memo line never approaches these; an + * overrun means the model transcribed something it shouldn't have). + * + * needsReview is true when ANY field was dropped or truncated: the caller + * must then store NULL for payerName/memoText and log for human review + * instead of trusting the surviving values. + * + * @param {Record | null | undefined} raw + * @param {{ checkNumber?: string | null, amountCents?: number | null }} [opts] + */ +export function scrubExtraction(raw, { checkNumber = null, amountCents = null } = {}) { + const allow = []; + if (checkNumber) allow.push(String(checkNumber).replace(/\D/g, "")); + if (amountCents !== null && amountCents !== undefined) allow.push(String(amountCents)); + + const warnings = []; + let needsReview = false; + const take = (field, maxLen = null) => { + const { value, dropped, truncated } = scrubField(raw?.[field], allow, maxLen); + if (dropped) { + warnings.push(`${field} DROPPED: ${dropped}`); + needsReview = true; + } + if (truncated) { + warnings.push(`${field} TRUNCATED to ${maxLen} chars`); + needsReview = true; + } + return value; + }; + + const payerName = take("payerName", PAYER_NAME_MAX); + const memoText = take("memoText", MEMO_TEXT_MAX); + // Cross-check-only fields go through the same guard so a leaked account + // number can never even reach a console.log. + const documentDate = take("documentDate"); + const amount = take("amount"); + const checkNo = take("checkNumber"); + + return { payerName, memoText, documentDate, amount, checkNumber: checkNo, warnings, needsReview }; +} + +// ── Gemini Vision (repo's REST pattern, src/lib/actions.ts aiGeneratePunchlist) ── +const EXTRACTION_PROMPT = `You are reading ONE side of a bank document (a check front or a deposit photo) for a construction company's bookkeeping. + +Extract ONLY these fields: +- payerName: the person or company name printed in the top-left name/address block (who wrote the check). Name only — no street address. +- documentDate: the written or printed date, as YYYY-MM-DD. +- amount: the dollar amount from the courtesy box, e.g. "6037.15". +- memoText: the handwriting or print on the "memo" / "for" line, or null if blank. +- checkNumber: the check number from the TOP RIGHT corner ONLY. + +ABSOLUTE PROHIBITION — read carefully: +Do NOT read, extract, transcribe, or output the MICR line: the row of numbers printed along the BOTTOM edge of the check in magnetic ink. That includes the routing number, the bank account number, and any number printed between ⑆ ⑈ ⑉ symbols. Never output any number with 8 or more consecutive digits from the bottom strip of the document, in any field. If you cannot fill a field without using the bottom strip, output null for that field. This is a privacy requirement and overrides completeness. + +Return ONLY a JSON object, nothing else: +{"payerName": string|null, "documentDate": string|null, "amount": string|null, "memoText": string|null, "checkNumber": string|null}`; + +/** + * Pure. Parse a model response as JSON, tolerating a ```json ... ``` (or + * bare ```) fence wrapper — models sometimes emit fences even with + * responseMimeType: application/json. + */ +export function parseModelJson(rawText) { + let text = String(rawText ?? "").trim(); + const fenced = text.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?\s*```$/i); + if (fenced) text = fenced[1].trim(); + return JSON.parse(text); +} + +export async function extractViaGemini(apiKey, imageBytes, mime) { + const res = await fetch( + `https://generativelanguage.googleapis.com/v1beta/models/${GEMINI_MODEL}:generateContent?key=${apiKey}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + contents: [{ + parts: [ + { text: EXTRACTION_PROMPT }, + { inline_data: { mime_type: mime || "image/jpeg", data: imageBytes.toString("base64") } }, + ], + }], + generationConfig: { temperature: 0, responseMimeType: "application/json" }, + }), + }, + ); + if (!res.ok) throw new Error(`Gemini HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`); + const data = await res.json(); + const rawText = data.candidates?.[0]?.content?.parts?.[0]?.text; + if (!rawText) throw new Error("No AI response"); + return parseModelJson(rawText); +} + +// ── fuzzy matching for the REVIEW REPORT ───────────────────────────────── +const NAME_NOISE = new Set(["llc", "inc", "co", "corp", "ltd", "the", "and", "&", "of", "mr", "mrs", "ms", "dr", "or"]); + +export function nameTokens(s) { + return String(s ?? "") + .toLowerCase() + .replace(/[^a-z0-9\s]/g, " ") + .split(/\s+/) + .filter(t => t && !NAME_NOISE.has(t)); +} + +/** Pure. 0..1 similarity: token-set Jaccard plus a containment bonus. */ +export function nameSimilarity(a, b) { + const ta = nameTokens(a), tb = nameTokens(b); + if (!ta.length || !tb.length) return 0; + const sa = new Set(ta), sb = new Set(tb); + const inter = [...sa].filter(t => sb.has(t)).length; + const union = new Set([...sa, ...sb]).size; + const jaccard = inter / union; + const containment = inter / Math.min(sa.size, sb.size); + return Math.max(jaccard, containment * 0.85); +} + +/** + * Pure. Suggest candidate matches for one extraction. Returns + * { payerMatches: [{id,name,score}], memoMatches: [...] } sorted by score, + * top 3 each, threshold 0.4. Suggestion only — the caller must NEVER write + * BankImageMatch from this. + */ +export function suggestMatches({ payerName, memoText }, clients, projects) { + const rank = (text, rows) => rows + .map(r => ({ id: r.id, name: r.name, score: nameSimilarity(text, r.name) })) + .filter(m => m.score >= 0.4) + .sort((x, y) => y.score - x.score) + .slice(0, 3); + return { + payerMatches: payerName ? rank(payerName, clients) : [], + memoMatches: memoText ? rank(memoText, projects) : [], + }; +} + +// ── candidate loading ──────────────────────────────────────────────────── +function parseKinds() { + const idx = process.argv.indexOf("--kinds"); + if (idx < 0) return DEFAULT_KINDS; + const kinds = String(process.argv[idx + 1] ?? "").split(",").map(s => s.trim()).filter(Boolean); + const bad = kinds.filter(k => !ALL_KINDS.includes(k)); + if (bad.length) throw new Error(`Unknown kind(s): ${bad.join(", ")}. Allowed: ${ALL_KINDS.join(", ")}`); + return kinds; +} + +function parseLimit() { + const idx = process.argv.indexOf("--limit"); + if (idx < 0) return DEFAULT_LIMIT; + const n = Number(process.argv[idx + 1]); + if (!Number.isInteger(n) || n < 1) throw new Error(`--limit must be a positive integer, got "${process.argv[idx + 1]}"`); + return n; +} + +async function hasExtractionColumns(prisma) { + const rows = await prisma.$queryRawUnsafe( + `SELECT column_name FROM information_schema.columns + WHERE table_schema='public' AND table_name='BankImage' + AND column_name IN ('payerName','memoText','extractedAt','extractionModel')`, + ); + return rows.length === 4; +} + +async function loadCandidatesFromDb(prisma, kinds, limit, columnsReady) { + // kinds is validated against ALL_KINDS above, limit is a checked integer — + // safe to inline. extractedAt only exists after the DDL is applied; before + // that, dry-run falls back to "everything is unextracted". + const kindList = kinds.map(k => `'${k}'`).join(","); + const where = columnsReady ? `AND "extractedAt" IS NULL` : ""; + return prisma.$queryRawUnsafe( + `SELECT "id", "kind", "sourceExternalId", "fileName", "mime", + "normalizedCheckNumber", "amountCents", "documentDate" + FROM "BankImage" + WHERE "kind" IN (${kindList}) ${where} + ORDER BY "capturedAt" ASC + LIMIT ${limit}`, + ); +} + +/** Pre-DDL escape hatch: derive candidates straight from the Drive manifest. */ +export function loadCandidatesFromManifest(manifest, kinds, limit) { + const rows = []; + for (const entry of Object.values(manifest.images ?? {})) { + const files = Array.isArray(entry.files) ? entry.files : []; + const isCheck = !!String(entry.checkNumber ?? "").replace(/\D/g, "").replace(/^0+/, ""); + files.forEach((f, i) => { + const kind = isCheck + ? (f.side === "front" || i === 0 ? "CHECK_FRONT" : "CHECK_BACK") + : (i === 0 ? "DEPOSIT_SLIP" : "DEPOSIT_PHOTO"); + if (!kinds.includes(kind)) return; + rows.push({ + id: `manifest:${entry.bankReference}:${f.side ?? `img${i + 1}`}`, + kind, + sourceExternalId: `${entry.bankReference}:${f.side ?? `img${i + 1}`}`, + fileName: f.fileName, + mime: "image/jpeg", + normalizedCheckNumber: isCheck ? String(entry.checkNumber).replace(/\D/g, "").replace(/^0+/, "") : null, + amountCents: entry.amountCents ?? null, + documentDate: null, + }); + }); + } + return rows.slice(0, limit); +} + +// ── review report ──────────────────────────────────────────────────────── +function printReviewReport(results, clients, projects) { + console.log("\n════════ REVIEW REPORT — suggestions only, NOTHING written ════════"); + console.log("BankImageMatch is human-confirmation-only; confirm in the app, not here.\n"); + let any = false; + for (const r of results) { + if (!r.payerName && !r.memoText) continue; + const { payerMatches, memoMatches } = suggestMatches(r, clients, projects); + any = true; + console.log(` ${r.sourceExternalId} (${r.kind}${r.normalizedCheckNumber ? `, chk#${r.normalizedCheckNumber}` : ""})`); + console.log(` payer: ${r.payerName ?? "(none)"} | memo: ${r.memoText ?? "(none)"}`); + if (payerMatches.length) { + for (const m of payerMatches) console.log(` payer → Client "${m.name}" score ${m.score.toFixed(2)} [${m.id}]`); + } else console.log(" payer → no Client match ≥ 0.40"); + if (r.memoText) { + if (memoMatches.length) { + for (const m of memoMatches) console.log(` memo → Project "${m.name}" score ${m.score.toFixed(2)} [${m.id}]`); + } else console.log(" memo → no Project match ≥ 0.40"); + } + console.log(""); + } + if (!any) console.log(" (no extracted payer/memo values to match)\n"); +} + +// ── main ───────────────────────────────────────────────────────────────── +async function main() { + const commit = process.argv.includes("--commit"); + const reportOnly = process.argv.includes("--report") && !commit; + const dryRun = !commit; + const fromManifest = process.argv.includes("--from-manifest"); + const kinds = parseKinds(); + const limit = parseLimit(); + + if (fromManifest && commit) { + console.error("--from-manifest is a pre-DDL dry-run aid; it cannot be combined with --commit."); + process.exit(1); + } + + const dbUrl = resolveEnv("DATABASE_URL"); + if (!dbUrl && !fromManifest) { + console.error("DATABASE_URL not found (env, .env.local, .env)."); + process.exit(1); + } + const prisma = dbUrl ? new PrismaClient({ datasources: { db: { url: dbUrl } } }) : null; + + try { + // ── report-only mode reads already-extracted rows and exits ── + if (reportOnly && !fromManifest) { + if (!prisma) throw new Error("Report mode needs a database connection."); + const columnsReady = await hasExtractionColumns(prisma); + if (!columnsReady) { + console.error("Extraction columns not applied yet — run scripts/apply-check-payer-extraction.mjs first."); + process.exit(1); + } + const rows = await prisma.$queryRawUnsafe( + `SELECT "id", "kind", "sourceExternalId", "normalizedCheckNumber", "payerName", "memoText" + FROM "BankImage" WHERE "extractedAt" IS NOT NULL ORDER BY "capturedAt" ASC LIMIT ${limit}`, + ); + const clients = await prisma.$queryRawUnsafe(`SELECT "id", "name" FROM "Client"`); + const projects = await prisma.$queryRawUnsafe(`SELECT "id", "name" FROM "Project"`); + console.log(`report: ${rows.length} extracted image(s), ${clients.length} client(s), ${projects.length} project(s)`); + printReviewReport(rows, clients, projects); + return; + } + + // ── candidate selection ── + let candidates; + let columnsReady = false; + if (fromManifest) { + if (!fs.existsSync(MANIFEST_PATH)) { + console.error(`No manifest at ${MANIFEST_PATH}`); + process.exit(1); + } + const manifest = JSON.parse(fs.readFileSync(MANIFEST_PATH, "utf8")); + candidates = loadCandidatesFromManifest(manifest, kinds, limit); + console.log(`manifest: ${candidates.length} candidate image(s) [kinds: ${kinds.join(", ")}]`); + } else { + columnsReady = await hasExtractionColumns(prisma); + if (!columnsReady) { + if (commit) { + console.error("Extraction columns missing — apply scripts/apply-check-payer-extraction.mjs before --commit."); + process.exit(1); + } + console.log("NOTE: extraction columns not applied yet; dry-run treats every row as unextracted."); + } + candidates = await loadCandidatesFromDb(prisma, kinds, limit, columnsReady); + console.log(`db: ${candidates.length} unextracted image(s) [kinds: ${kinds.join(", ")}, limit ${limit}]`); + } + + if (!candidates.length) { + console.log("Nothing to extract."); + return; + } + + const apiKey = resolveEnv("GEMINI_API_KEY"); + if (!apiKey) { + console.error("GEMINI_API_KEY not found (env, .env.local, .env)."); + process.exit(1); + } + + // ── extraction loop ── + const results = []; + for (const row of candidates) { + const imagePath = path.join(IMAGES_DIR, row.fileName); + if (!fs.existsSync(imagePath)) { + console.warn(` SKIP ${row.sourceExternalId}: image file not found at ${imagePath}`); + continue; + } + process.stdout.write(` ${row.sourceExternalId} (${row.kind}) ... `); + let raw; + try { + raw = await extractViaGemini(apiKey, fs.readFileSync(imagePath), row.mime); + } catch (err) { + console.log(`FAILED: ${err.message}`); + continue; + } + const scrubbed = scrubExtraction(raw, { + checkNumber: row.normalizedCheckNumber, + amountCents: row.amountCents, + }); + for (const w of scrubbed.warnings) console.warn(`\n WARNING ${row.sourceExternalId}: ${w}`); + console.log(`payer="${scrubbed.payerName ?? ""}" memo="${scrubbed.memoText ?? ""}"` + + (scrubbed.documentDate ? ` date=${scrubbed.documentDate}` : "") + + (scrubbed.amount ? ` amt=${scrubbed.amount}` : "") + + (scrubbed.checkNumber ? ` chk#${scrubbed.checkNumber}` : "")); + + // Cross-checks are advisory: log disagreement, store nothing extra. + if (row.normalizedCheckNumber && scrubbed.checkNumber && + scrubbed.checkNumber.replace(/\D/g, "").replace(/^0+/, "") !== row.normalizedCheckNumber) { + console.warn(` NOTE: image check# ${scrubbed.checkNumber} != row check# ${row.normalizedCheckNumber}`); + } + + results.push({ ...row, payerName: scrubbed.payerName, memoText: scrubbed.memoText, needsReview: scrubbed.needsReview }); + + if (commit) { + // needsReview rows store NULL for BOTH kept fields: a scrub + // warning means the extraction is suspect (dropped digit run, + // truncated overrun), so nothing from it is trusted. The row + // is still stamped extractedAt/extractionModel so replay + // skips it; a human re-reads the image instead. + const storePayer = scrubbed.needsReview ? null : scrubbed.payerName; + const storeMemo = scrubbed.needsReview ? null : scrubbed.memoText; + if (scrubbed.needsReview) { + console.warn(` NEEDS REVIEW ${row.sourceExternalId}: stored NULL payer/memo — a human must read this image (file: ${row.fileName})`); + } + await prisma.$executeRaw` + UPDATE "BankImage" + SET "payerName" = ${storePayer}, + "memoText" = ${storeMemo}, + "extractedAt" = now(), + "extractionModel" = ${GEMINI_MODEL}, + "updatedAt" = now() + WHERE "id" = ${row.id} AND "extractedAt" IS NULL`; + } + } + + if (dryRun) console.log(`\nDRY RUN — nothing written. Re-run with --commit${columnsReady ? "" : " after applying the DDL"}.`); + else console.log(`\nwrote ${results.length} extraction(s) [model ${GEMINI_MODEL}] (replay skips them)`); + + // ── review report ── + if (prisma) { + const clients = await prisma.$queryRawUnsafe(`SELECT "id", "name" FROM "Client"`); + const projects = await prisma.$queryRawUnsafe(`SELECT "id", "name" FROM "Project"`); + printReviewReport(results, clients, projects); + } else { + console.log("\n(no DATABASE_URL — skipping the review report's Client/Project matching)"); + } + } finally { + if (prisma) await prisma.$disconnect(); + } +} + +const isMainModule = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]; +if (isMainModule) { + main().catch(error => { + console.error(error); + process.exitCode = 1; + }); +} diff --git a/scripts/pull-qbo-bank-register.mjs b/scripts/pull-qbo-bank-register.mjs new file mode 100644 index 000000000..c522a0469 --- /dev/null +++ b/scripts/pull-qbo-bank-register.mjs @@ -0,0 +1,394 @@ +// QBO bank REGISTER REPORT for the WTB checking account (…0723). +// +// WHY THIS EXISTS (2026-08-21): Justin rotated the Washington Trust password, +// so the browser-based daily CSV export (skill wtb-daily-bank-export) has no +// valid credentials and is dead as the primary bank-register source. The +// QuickBooks bank feed carries the same transactions over OAuth — no login +// page, no 2FA, no session to lose. This script is the human-readable +// per-day register view of that feed. +// +// WHAT IT SHOWS: per calendar day — transaction count, money in, money out, +// net, and (when QBO's running-balance column reconciles exactly) the derived +// per-day opening/closing balance. Every figure is integer cents from QBO's +// decimal strings; no float math on the way in. +// +// HONESTY CONTRACT (same as src/lib/qbo-bank-register.ts): this is the BOOKS +// view — what QuickBooks has POSTED to the account. It cannot see WTB +// transactions that are pending, excluded from the feed, or absent from +// QuickBooks, and it does not prove bank clearance. The derived balances are +// QBO's book balances, NOT the bank's statement OPENING/CLOSING LEDGER. +// +// WHY --post IS BLOCKED (deliberate, not unfinished): +// 1. The ingest route's STATEMENT source is reserved for the bank's own +// statement — true north. QBO book balances routinely differ from the +// bank's ledger balances (feed lag ~1 day, uncleared checks, excluded +// feed rows), so a QBO-derived "statement" would assert control totals +// the bank never published. That is exactly the faked-balance failure +// docs/BANK-REGISTER-PLAN.md forbids. +// 2. STATEMENT days for WTB-0723 are already minted by +// scripts/parse-wtb-daily-csv.mjs under the route's uniqueness key +// (account, periodStart, periodEnd). QBO-derived one-day statements for +// the same account would either 409 against every existing day (books +// vs bank content differs) or, on uncovered days, mint canonical +// BankLines from non-bank evidence — the cross-source double-minting +// the Codex B1 review exists to prevent. +// 3. The sanctioned QBO→ledger path already exists and is live: +// scripts/post-qbo-register.mjs posts source=QBO_REGISTER observation +// rows (idempotent by qbTxnId, 409 on content change), and the +// reconcile route links them to canonical statement lines. +// If the bank-statement balance source is ever restored (CSV re-enabled, +// or an OFX pull lands), STATEMENT posting belongs there — not here. +// +// Usage (tsx required — the imported src/lib modules are TypeScript with +// "@/*" path aliases that bare node cannot resolve): +// npx tsx scripts/pull-qbo-bank-register.mjs # last 14 days +// npx tsx scripts/pull-qbo-bank-register.mjs --days 30 +// npx tsx scripts/pull-qbo-bank-register.mjs --start 2026-08-01 --end 2026-08-20 +// npx tsx scripts/pull-qbo-bank-register.mjs --txns # per-transaction detail +// +// Env: loads .env.production.local, then .env.local, then .env itself +// (first file that defines a key wins; a value already in process.env always +// wins). .env.production.local is preferred because the QBO OAuth client +// creds (QB_CLIENT_ID/QB_CLIENT_SECRET) and the NEXTAUTH_SECRET that +// decrypts the stored token row exist only in the prod env pull. + +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const REPO_ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), ".."); +const DEFAULT_DAYS = 14; +// fetchBankRegister enforces 92; mirror it so the error is ours, not a stack trace. +const MAX_RANGE_DAYS = 92; +// The account this report is for. Refuse to render anything else so a future +// second bank account can't silently masquerade as WTB checking. +const EXPECTED_LAST4 = "0723"; + +// ── env ───────────────────────────────────────────────────────────────────── + +/** + * Minimal .env parser (KEY=value, optional double quotes, # comments). + * Precedence: existing process.env > .env.production.local > .env.local > .env + * — prod values first because this script only makes sense against prod QBO, + * and the prod NEXTAUTH_SECRET is the one that decrypts the Integration row. + */ +function loadEnvFiles() { + const loaded = []; + for (const file of [".env.production.local", ".env.local", ".env"]) { + const full = path.join(REPO_ROOT, file); + if (!fs.existsSync(full)) continue; + const text = fs.readFileSync(full, "utf8").replace(/^\uFEFF/, ""); + for (const rawLine of text.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line || line.startsWith("#")) continue; + const m = /^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line); + if (!m) continue; + let [, key, value] = m; + if (value.startsWith('"') && value.endsWith('"') && value.length >= 2) { + value = value.slice(1, -1); + } + if (!(key in process.env)) process.env[key] = value; + } + loaded.push(file); + } + return loaded; +} + +// ── args ──────────────────────────────────────────────────────────────────── + +function parseArgs(argv) { + const args = { days: DEFAULT_DAYS, start: null, end: null, txns: false, post: null }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--txns") args.txns = true; + else if (arg === "--days") { + const v = Number(argv[++i]); + if (!Number.isInteger(v) || v < 1 || v > MAX_RANGE_DAYS) throw new Error(`--days must be 1..${MAX_RANGE_DAYS}`); + args.days = v; + } else if (arg === "--start") { + const v = argv[++i]; + if (!isYmd(v)) throw new Error("--start must be YYYY-MM-DD"); + args.start = v; + } else if (arg === "--end") { + const v = argv[++i]; + if (!isYmd(v)) throw new Error("--end must be YYYY-MM-DD"); + args.end = v; + } else if (arg === "--post") { + // Consume the value if present so the refusal message is accurate, + // but never act on it — see the header comment. + const v = argv[i + 1]; + if (v !== undefined && !v.startsWith("--")) i++; + args.post = v ?? "(no url)"; + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + if ((args.start === null) !== (args.end === null)) throw new Error("--start and --end must be given together"); + return args; +} + +/** Strict YYYY-MM-DD with calendar round-trip ("2026-02-30" must fail). */ +function isYmd(s) { + if (typeof s !== "string" || !/^\d{4}-\d{2}-\d{2}$/.test(s)) return false; + const t = Date.parse(`${s}T00:00:00Z`); + return Number.isFinite(t) && new Date(t).toISOString().slice(0, 10) === s; +} + +/** UTC-only date math — local timezone must never shift a posting date. */ +function ymdDaysAgo(days) { + return new Date(Date.now() - days * 86_400_000).toISOString().slice(0, 10); +} + +// ── money ─────────────────────────────────────────────────────────────────── + +/** + * Exact decimal-string → integer cents. Refuses anything that isn't a plain + * signed decimal with ≤2 fraction digits (financial-data-pipelines rule 1). + * QBO report cells arrive as decimal strings; never parseFloat them. + */ +export function toCents(raw) { + const s = String(raw ?? "").trim(); + const m = /^(-?)(\d+)(?:\.(\d{1,2}))?$/.exec(s); + if (!m) return null; + const [, sign, whole, frac = ""] = m; + const cents = Number(whole) * 100 + Number(frac.padEnd(2, "0")); + if (!Number.isSafeInteger(cents)) return null; + return sign === "-" ? -cents : cents; +} + +const money = c => (c / 100).toLocaleString("en-US", { minimumFractionDigits: 2, maximumFractionDigits: 2 }); + +// ── balance derivation (GL report with running-balance column) ────────────── + +/** + * Fetch the GL report AGAIN, this time asking for the running-balance column + * (rbal_nat_amount) so per-day book balances can be DERIVED rather than + * guessed. Returns { openingCents, rows: [{date, amountCents, runbalCents}] } + * or null when the report doesn't carry a usable balance chain. + * + * The derivation is only trusted when it proves itself: every row must + * satisfy previous_balance + amount == running_balance in exact integer + * cents, chaining from the report's Beginning Balance row. One break → the + * whole chain is discarded and the report prints "n/a" for balances. + * (financial-data-pipelines rule 2: control totals refuse to ship.) + */ +async function fetchBalanceChain(qbFetch, tokens, accountId, startDate, endDate) { + const params = new URLSearchParams({ + start_date: startDate, + end_date: endDate, + account: accountId, + columns: "tx_date,txn_type,subt_nat_amount,rbal_nat_amount", + }); + const res = await qbFetch(`/reports/GeneralLedger?${params}`, tokens); + if (!res.ok) return { ok: false, reason: `GL balance report HTTP ${res.status}` }; + const report = await res.json(); + + // Column order comes from the report's own Columns block, never assumed. + const idx = new Map(); + (report.Columns?.Column ?? []).forEach((col, i) => { + for (const meta of col.MetaData ?? []) { + if (meta.Name === "ColKey" && meta.Value) idx.set(meta.Value, i); + } + }); + if (!idx.has("tx_date") || !idx.has("subt_nat_amount") || !idx.has("rbal_nat_amount")) { + return { ok: false, reason: "GL report did not return the running-balance column" }; + } + + const flat = []; + (function walk(rows) { + for (const row of rows ?? []) { + if (row.ColData) flat.push(row.ColData); + if (row.Rows?.Row) walk(row.Rows.Row); + } + })(report.Rows?.Row ?? []); + + let openingCents = null; + const rows = []; + for (const cols of flat) { + const cell = key => cols[idx.get(key)]?.value; + const label = String(cell("tx_date") ?? "").trim(); + const txnType = String(cols[idx.get("txn_type")]?.value ?? "").trim(); + const runbal = toCents(cell("rbal_nat_amount")); + // The section's Beginning Balance line: no txn type, carries the + // starting running balance. QBO renders its label in the first column. + if (!txnType && /beginning balance/i.test(label) && runbal !== null) { + if (openingCents !== null && openingCents !== runbal) { + return { ok: false, reason: "conflicting Beginning Balance rows" }; + } + openingCents = runbal; + continue; + } + if (!txnType) continue; // other summary/total lines + const date = isYmd(label) ? label : null; + const amount = toCents(cell("subt_nat_amount")); + if (!date || amount === null || runbal === null) { + return { ok: false, reason: `unparseable GL balance row (date "${label}")` }; + } + rows.push({ date, amountCents: amount, runbalCents: runbal }); + } + if (openingCents === null) return { ok: false, reason: "no Beginning Balance row in GL report" }; + + // The chain must PROVE itself: opening + each amount → each running balance. + let bal = openingCents; + for (const row of rows) { + bal += row.amountCents; + if (bal !== row.runbalCents) { + return { ok: false, reason: `balance chain breaks at ${row.date}: expected ${bal} got ${row.runbalCents}` }; + } + } + return { ok: true, openingCents, rows }; +} + +// ── report ────────────────────────────────────────────────────────────────── + +function buildDaySummaries(rows) { + // rows arrive newest-first from fetchBankRegister; group by date ascending. + const byDay = new Map(); + for (const row of rows) { + if (!byDay.has(row.date)) byDay.set(row.date, { count: 0, inCents: 0, outCents: 0 }); + const d = byDay.get(row.date); + d.count++; + if (row.amountCents >= 0) d.inCents += row.amountCents; + else d.outCents += row.amountCents; + } + return [...byDay.entries()].sort(([a], [b]) => a.localeCompare(b)) + .map(([date, d]) => ({ date, ...d, netCents: d.inCents + d.outCents })); +} + +function fail(msg) { + console.error(`GATE FAILED: ${msg}`); + process.exitCode = 1; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + if (args.post !== null) { + console.error("--post is BLOCKED for this script, deliberately:"); + console.error(" • QBO is the BOOKS view. Its balances are not the bank's OPENING/CLOSING"); + console.error(" LEDGER, so a QBO-derived STATEMENT would assert control totals the bank"); + console.error(" never published (feed lag, uncleared checks, excluded rows)."); + console.error(" • STATEMENT days for WTB-0723 are owned by scripts/parse-wtb-daily-csv.mjs"); + console.error(" under the (account, periodStart, periodEnd) uniqueness key — QBO-derived"); + console.error(" days would 409 against them or double-mint canonical BankLines."); + console.error(" • The sanctioned QBO→ledger path already exists:"); + console.error(" npx tsx scripts/post-qbo-register.mjs --days 30 --post "); + console.error(" (source=QBO_REGISTER observations, idempotent by qbTxnId)."); + console.error(" Restore a real bank-statement source (CSV/OFX) for STATEMENT posting."); + process.exit(1); + } + + const loaded = loadEnvFiles(); + if (loaded.length === 0) fail("no .env file found next to the repo root"); + for (const key of ["DATABASE_URL", "NEXTAUTH_SECRET", "QB_CLIENT_ID", "QB_CLIENT_SECRET"]) { + if (!process.env[key]) return fail(`${key} missing — run 'vercel env pull .env.production.local' first (loaded: ${loaded.join(", ")})`); + } + if (!/pgbouncer=true/.test(process.env.DATABASE_URL)) { + console.error("WARNING: DATABASE_URL lacks ?pgbouncer=true — Supabase pooler + Prisma needs it."); + } + + const endDate = args.end ?? new Date().toISOString().slice(0, 10); + const startDate = args.start ?? ymdDaysAgo(args.days); + if (startDate > endDate) return fail("start date is after end date"); + + // Imported AFTER env is loaded — src/lib/prisma reads DATABASE_URL at + // import time, and integration-store decrypts with NEXTAUTH_SECRET. + // These are TypeScript with "@/*" aliases: run this script under tsx. + const { fetchBankRegister, bankAccountId } = await import("../src/lib/qbo-bank-register.ts"); + const { getFreshQBTokens } = await import("../src/lib/quickbooks-payments.ts"); + const { qbFetch, qbQuery } = await import("../src/lib/quickbooks.ts"); + + const tokens = await getFreshQBTokens(); + + // 1) Account verification: the register must provably be the WTB …0723 + // checking account, not whatever account id happens to be configured. + const accountId = bankAccountId(); + const bankAccounts = await qbQuery(tokens, "SELECT * FROM Account WHERE AccountType = 'Bank' MAXRESULTS 100"); + const target = bankAccounts.find(a => String(a.Id) === String(accountId)); + if (!target) return fail(`configured bank account id ${accountId} not found among ${bankAccounts.length} QBO bank account(s)`); + const acctNum = String(target.AcctNum ?? ""); + const acctLabel = `${target.Name}${acctNum ? ` (#…${acctNum.slice(-4)})` : ""}`; + const looks0723 = acctNum.endsWith(EXPECTED_LAST4) || new RegExp(EXPECTED_LAST4).test(target.Name ?? ""); + if (!looks0723) { + const other = bankAccounts.find(a => String(a.AcctNum ?? "").endsWith(EXPECTED_LAST4) || new RegExp(EXPECTED_LAST4).test(a.Name ?? "")); + if (other) return fail(`configured account ${accountId} (${acctLabel}) is NOT the …${EXPECTED_LAST4} account — QBO account ${other.Id} (${other.Name}) is. Set QBO_RECEIPT_BANK_ACCOUNT_ID.`); + console.error(`WARNING: cannot confirm account ${accountId} (${acctLabel}) is …${EXPECTED_LAST4} — no QBO bank account carries that number. Proceeding; verify in QBO.`); + } + + // 2) The register rows (same proven path the /automation/bank page uses). + const result = await fetchBankRegister(() => Promise.resolve(tokens), startDate, endDate); + const rows = result.rows ?? []; + + // 3) Balance chain (books view) — display-only, and only if it proves itself. + let chain = { ok: false, reason: "not attempted" }; + try { + chain = await fetchBalanceChain(qbFetch, tokens, accountId, startDate, endDate); + } catch (error) { + chain = { ok: false, reason: error instanceof Error ? error.message : String(error) }; + } + + // 4) Print. + const days = buildDaySummaries(rows); + console.log(`REGISTER REPORT — QBO books view of ${acctLabel}`); + console.log(` realm ${tokens.realmId}, QBO account id ${accountId}`); + console.log(` window ${startDate} → ${endDate}${result.stale ? " [STALE CACHE — QBO errored, showing last good fetch]" : ""}`); + console.log(` ${rows.length} posted row(s) across ${days.length} day(s). Books balance chain: ${chain.ok ? "VERIFIED (exact cents)" : `n/a — ${chain.reason}`}`); + if (typeof target.CurrentBalance !== "undefined") { + console.log(` QBO CurrentBalance (books, as of now): ${money(toCents(String(target.CurrentBalance)) ?? NaN)}`); + } + console.log(" NOTE: books view only — pending/unfed bank activity is invisible; balances are NOT the bank's ledger."); + console.log(""); + console.log(" date txns money in money out net" + (chain.ok ? " open(books) close(books)" : "")); + console.log(" ---------- ----- ------------ ------------ ------------" + (chain.ok ? " -------------- --------------" : "")); + + // Per-day close from the verified chain: opening + cumulative net of every + // chain row dated ≤ that day (chain rows are the same GL rows, so the two + // fetches agree; if they ever disagree the sums below expose it). + let closeByDay = new Map(); + if (chain.ok) { + let bal = chain.openingCents; + const sortedChain = [...chain.rows].sort((a, b) => a.date.localeCompare(b.date)); + for (const row of sortedChain) { + bal += row.amountCents; + closeByDay.set(row.date, bal); + } + } + let prevClose = chain.ok ? chain.openingCents : null; + for (const d of days) { + let balCols = ""; + if (chain.ok) { + const close = closeByDay.get(d.date); + if (close !== undefined && prevClose !== null && prevClose + d.netCents === close) { + balCols = ` ${money(prevClose).padStart(14)} ${money(close).padStart(14)}`; + prevClose = close; + } else { + // The two GL fetches disagreed for this day — say so, never guess. + balCols = " " + "chain mismatch".padStart(30); + prevClose = close ?? prevClose; + } + } + console.log(` ${d.date} ${String(d.count).padStart(5)} ${money(d.inCents).padStart(12)} ${money(d.outCents).padStart(12)} ${money(d.netCents).padStart(12)}${balCols}`); + } + const totalIn = days.reduce((a, d) => a + d.inCents, 0); + const totalOut = days.reduce((a, d) => a + d.outCents, 0); + console.log(" ---------- ----- ------------ ------------ ------------"); + console.log(` TOTAL ${String(rows.length).padStart(5)} ${money(totalIn).padStart(12)} ${money(totalOut).padStart(12)} ${money(totalIn + totalOut).padStart(12)}`); + + if (args.txns) { + console.log(""); + console.log(" TRANSACTIONS (newest first)"); + for (const row of rows) { + console.log(` ${row.date} ${money(row.amountCents).padStart(12)} ${(row.qbType ?? "").padEnd(18)} ${(row.name ?? "").slice(0, 40)}${row.qbTxnId ? ` [${row.qbTxnId}]` : ""}`); + } + } + + console.log(""); + console.log(" Ledger posting: use scripts/post-qbo-register.mjs (source=QBO_REGISTER observations)."); + console.log(" STATEMENT posting stays with the bank's own statement source (see header)."); +} + +// Entry check must survive being imported by tests: process.argv[1] can be +// undefined under `node -e`, and pathToFileURL(undefined) throws. +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch(error => { console.error(error); process.exit(1); }); +} diff --git a/src/app/automation/check-images-data.ts b/src/app/automation/check-images-data.ts new file mode 100644 index 000000000..b8feefa20 --- /dev/null +++ b/src/app/automation/check-images-data.ts @@ -0,0 +1,186 @@ +import { prisma } from "@/lib/prisma"; +import { + proposeImageMatches, + type BankImageCandidate, + type BankImageKind, + type BankImageLine, +} from "@/lib/bank-image"; +import { suggestMatches, type MatchSuggestion } from "@/lib/check-payer-match"; + +/** + * Data for the Automation page's "Check images" panel — the human worklist + * for the check-payer pipeline (scripts/extract-check-payers.mjs writes the + * extractions; this reads them). + * + * READ-ONLY. The only writer of BankImageMatch is the explicit + * confirmBankImageMatch server action (src/lib/actions.ts) — a row in that + * table means a HUMAN said yes (see prisma/schema.prisma). + * + * Caller (src/app/automation/page.tsx) is gated on the `financialReports` + * permission before this ever runs — internal roles only, never the client + * or sub portals. + */ + +const IMAGE_KINDS = new Set(["CHECK_FRONT", "CHECK_BACK", "DEPOSIT_SLIP", "DEPOSIT_PHOTO"]); +/** Display cap — this is a review worklist, not an archive browser. */ +export const CHECK_IMAGE_DISPLAY_LIMIT = 50; +/** Candidate ledger lines considered for proposals (most recent first). */ +const LINE_CANDIDATE_LIMIT = 1000; + +export interface CheckImageConfirmedMatch { + bankLineId: string | null; + lineDescriptor: string | null; + linePostedDate: string | null; + lineAmountCents: number | null; + confirmedBy: string; + /** ISO timestamp */ + confirmedAt: string; + note: string | null; +} + +export interface CheckImageProposedLine { + bankLineId: string; + confidence: string; + reason: string; + lineDescriptor: string; + linePostedDate: string | null; + lineAmountCents: number; +} + +export interface CheckImagePanelRow { + id: string; + kind: string; + sourceExternalId: string; + fileName: string; + driveFileId: string | null; + /** ISO timestamp */ + capturedAt: string; + /** YYYY-MM-DD */ + documentDate: string | null; + amountCents: number | null; + normalizedCheckNumber: string | null; + /** Null both when extraction hasn't run (extractedAt null) AND when it + * ran but found nothing / was scrubbed — `extracted` disambiguates. */ + payerName: string | null; + memoText: string | null; + extracted: boolean; + extractionModel: string | null; + payerMatches: MatchSuggestion[]; + memoMatches: MatchSuggestion[]; + proposal: CheckImageProposedLine | null; + /** Why no line could be proposed, when there is no proposal and no confirmation. */ + unmatchedDetail: string | null; + confirmed: CheckImageConfirmedMatch | null; +} + +function toDateOnly(value: Date | null | undefined): string | null { + return value ? value.toISOString().slice(0, 10) : null; +} + +export async function fetchCheckImagePanelData(): Promise<{ rows: CheckImagePanelRow[]; totalImages: number }> { + const [images, totalImages, lines, clients, projects] = await Promise.all([ + prisma.bankImage.findMany({ + orderBy: { capturedAt: "desc" }, + take: CHECK_IMAGE_DISPLAY_LIMIT, + include: { + matches: { + include: { + bankLine: { + select: { id: true, rawDescriptor: true, postedDate: true, amountCents: true }, + }, + }, + }, + }, + }), + prisma.bankImage.count(), + prisma.bankLine.findMany({ + orderBy: { postedDate: "desc" }, + take: LINE_CANDIDATE_LIMIT, + select: { id: true, postedDate: true, amountCents: true, rawDescriptor: true, checkNumber: true }, + }), + prisma.client.findMany({ select: { id: true, name: true } }), + prisma.project.findMany({ select: { id: true, name: true } }), + ]); + + const candidateLines: BankImageLine[] = lines.map((line) => ({ + id: line.id, + postedDate: toDateOnly(line.postedDate) ?? "", + amountCents: line.amountCents, + rawDescriptor: line.rawDescriptor, + checkNumber: line.checkNumber, + })).filter((line) => line.postedDate !== ""); + + const matchableImages: BankImageCandidate[] = images + .filter((img) => IMAGE_KINDS.has(img.kind)) + .map((img) => ({ + id: img.id, + kind: img.kind as BankImageKind, + documentDate: toDateOnly(img.documentDate), + amountCents: img.amountCents, + normalizedCheckNumber: img.normalizedCheckNumber, + })); + + const alreadyMatchedImageIds = images + .filter((img) => img.matches.length > 0) + .map((img) => img.id); + + const { proposals, unmatched } = proposeImageMatches(matchableImages, candidateLines, { + alreadyMatchedImageIds, + }); + const proposalByImage = new Map(proposals.map((p) => [p.bankImageId, p])); + const unmatchedByImage = new Map(unmatched.map((u) => [u.bankImageId, u])); + const lineById = new Map(candidateLines.map((line) => [line.id, line])); + + const rows: CheckImagePanelRow[] = images.map((img) => { + // bankImageId is @unique on BankImageMatch, so 0 or 1 rows. + const match = img.matches[0] ?? null; + const extracted = img.extractedAt !== null; + const suggestions = extracted + ? suggestMatches({ payerName: img.payerName, memoText: img.memoText }, clients, projects) + : { payerMatches: [], memoMatches: [] }; + const proposal = proposalByImage.get(img.id) ?? null; + const proposalLine = proposal ? lineById.get(proposal.bankLineId) ?? null : null; + + return { + id: img.id, + kind: img.kind, + sourceExternalId: img.sourceExternalId, + fileName: img.fileName, + driveFileId: img.driveFileId, + capturedAt: img.capturedAt.toISOString(), + documentDate: toDateOnly(img.documentDate), + amountCents: img.amountCents, + normalizedCheckNumber: img.normalizedCheckNumber, + payerName: img.payerName, + memoText: img.memoText, + extracted, + extractionModel: img.extractionModel, + payerMatches: suggestions.payerMatches, + memoMatches: suggestions.memoMatches, + proposal: proposal + ? { + bankLineId: proposal.bankLineId, + confidence: proposal.confidence, + reason: proposal.reason, + lineDescriptor: proposal.lineDescriptor, + linePostedDate: proposalLine?.postedDate ?? null, + lineAmountCents: proposal.lineAmountCents, + } + : null, + unmatchedDetail: match ? null : unmatchedByImage.get(img.id)?.detail ?? null, + confirmed: match + ? { + bankLineId: match.bankLineId, + lineDescriptor: match.bankLine?.rawDescriptor ?? null, + linePostedDate: toDateOnly(match.bankLine?.postedDate ?? null), + lineAmountCents: match.bankLine?.amountCents ?? null, + confirmedBy: match.confirmedBy, + confirmedAt: match.confirmedAt.toISOString(), + note: match.note, + } + : null, + }; + }); + + return { rows, totalImages }; +} diff --git a/src/app/automation/components/check-image-confirm-button.tsx b/src/app/automation/components/check-image-confirm-button.tsx new file mode 100644 index 000000000..b9b57a21b --- /dev/null +++ b/src/app/automation/components/check-image-confirm-button.tsx @@ -0,0 +1,58 @@ +"use client"; + +import { useState } from "react"; +import { useRouter } from "next/navigation"; +import { toast } from "sonner"; +import { confirmBankImageMatch } from "@/lib/actions"; + +/** + * The explicit human "yes" for a check-image → bank-line pairing. Clicking + * this is the ONLY path that writes BankImageMatch (via the + * confirmBankImageMatch server action) — everything else on the panel is a + * suggestion. Same "just the interactive bit" split as + * mark-reviewed-button.tsx. + */ +export default function ConfirmMatchButton({ + bankImageId, + bankLineId, + note, +}: { + bankImageId: string; + bankLineId: string; + /** The proposal's plain-language reason — stored on the match as the audit trail. */ + note: string | null; +}) { + const router = useRouter(); + const [pending, setPending] = useState(false); + const [done, setDone] = useState(false); + + async function confirm() { + setPending(true); + try { + const res = await confirmBankImageMatch({ bankImageId, bankLineId, note }); + if (res.success) { + setDone(true); + toast.success("Match confirmed"); + router.refresh(); + } else { + toast.error(res.error || "Couldn't confirm — try again"); + } + } catch { + toast.error("Couldn't confirm — try again"); + } finally { + setPending(false); + } + } + + return ( + + ); +} diff --git a/src/app/automation/components/check-images-panel.tsx b/src/app/automation/components/check-images-panel.tsx new file mode 100644 index 000000000..d231f3be2 --- /dev/null +++ b/src/app/automation/components/check-images-panel.tsx @@ -0,0 +1,185 @@ +import { formatCurrency } from "@/lib/utils"; +import type { CheckImagePanelRow } from "../check-images-data"; +import ConfirmMatchButton from "./check-image-confirm-button"; + +/** + * "Check images" panel (Automation page): the human worklist for the + * check-payer pipeline. Each card shows one BankImage with its extracted + * payer/memo evidence, the fuzzy Client/Project suggestions, and — when the + * matcher proposes exactly one ledger line — an explicit CONFIRM action. + * + * Suggestions are suggestions. Only the Confirm button (its server action) + * writes BankImageMatch, and only per image, by a signed-in internal user. + * + * Server component — the only interactive bit is ConfirmMatchButton. + */ + +const KIND_LABELS: Record = { + CHECK_FRONT: "Check front", + CHECK_BACK: "Check back", + DEPOSIT_SLIP: "Deposit slip", + DEPOSIT_PHOTO: "Deposit photo", +}; + +function fmtDate(iso: string | null): string { + if (!iso) return "—"; + const parsed = new Date(`${iso.slice(0, 10)}T00:00:00Z`); + if (Number.isNaN(parsed.getTime())) return "—"; + return parsed.toLocaleDateString("en-US", { year: "numeric", month: "short", day: "numeric", timeZone: "UTC" }); +} + +function Label({ children }: { children: React.ReactNode }) { + return

{children}

; +} + +function SuggestionList({ title, matches, hrefBase }: { + title: string; + matches: { id: string; name: string; score: number }[]; + hrefBase: string | null; +}) { + if (matches.length === 0) return null; + return ( +
+ +
    + {matches.map((m) => ( +
  • + {hrefBase ? ( + + {m.name} + + ) : ( + {m.name} + )} + {Math.round(m.score * 100)}% similar +
  • + ))} +
+
+ ); +} + +function ImageCard({ row }: { row: CheckImagePanelRow }) { + const kindLabel = KIND_LABELS[row.kind] ?? row.kind; + const hasSuggestions = row.payerMatches.length > 0 || row.memoMatches.length > 0; + + return ( +
+ {/* Identity line */} +
+
+

+ {kindLabel} + {row.normalizedCheckNumber && chk#{row.normalizedCheckNumber}} +

+

+ {row.fileName} · captured {fmtDate(row.capturedAt)} + {row.documentDate && ` · dated ${fmtDate(row.documentDate)}`} +

+
+
+ {row.amountCents !== null && ( + + {formatCurrency(row.amountCents / 100)} + + )} + {row.driveFileId && ( + + Image ↗ + + )} +
+
+ + {/* Extraction evidence */} + {!row.extracted ? ( +

Not yet extracted — run the check-payer extraction to read this image.

+ ) : ( +
+
+ +

{row.payerName ?? not readable — needs a human}

+
+
+ +

{row.memoText ?? (blank)}

+
+
+ )} + + {/* Fuzzy suggestions */} + {row.extracted && hasSuggestions && ( +
+ + +
+ )} + {row.extracted && !hasSuggestions && (row.payerName || row.memoText) && ( +

No client or project is a close enough name match to suggest.

+ )} + + {/* Confirmed / proposed / unmatched — exactly one of the three */} + {row.confirmed ? ( +
+

+ Confirmed → {row.confirmed.lineDescriptor ?? "bank line"} + {row.confirmed.lineAmountCents !== null && ` · ${formatCurrency(Math.abs(row.confirmed.lineAmountCents) / 100)}`} + {row.confirmed.linePostedDate && ` · posted ${fmtDate(row.confirmed.linePostedDate)}`} +

+

+ by {row.confirmed.confirmedBy} on {fmtDate(row.confirmed.confirmedAt)} +

+
+ ) : row.proposal ? ( +
+
+

+ Suggested bank line: {row.proposal.lineDescriptor} + {` · ${formatCurrency(Math.abs(row.proposal.lineAmountCents) / 100)}`} + {row.proposal.linePostedDate && ` · posted ${fmtDate(row.proposal.linePostedDate)}`} +

+

{row.proposal.reason}

+
+ +
+ ) : ( +

+ {row.unmatchedDetail ?? "No bank line to suggest for this image."} +

+ )} +
+ ); +} + +export function CheckImagesPanel({ rows, totalImages }: { rows: CheckImagePanelRow[]; totalImages: number }) { + return ( +
+
+

Check images

+

+ Who actually paid us, straight from the check — the bank line alone doesn't say. Extracted payer + and memo are evidence; the matches below are suggestions until a human confirms them. + {totalImages > rows.length && ` Showing the ${rows.length} most recent of ${totalImages} images.`} +

+
+ {rows.length === 0 ? ( +

+ No check or deposit images pulled from the bank yet. +

+ ) : ( +
+ {rows.map((row) => )} +
+ )} +
+ ); +} diff --git a/src/app/automation/page.tsx b/src/app/automation/page.tsx index a7e9737e7..80c16568e 100644 --- a/src/app/automation/page.tsx +++ b/src/app/automation/page.tsx @@ -48,6 +48,8 @@ import { LinksCell } from "./components/register/links-cell"; import { RowDrilldown } from "./components/register/row-drilldown"; import { matchReceiptJourney, type ReceiptJourneyMatch, type ReceiptJourneyIndex } from "./components/register/match-receipt-journey"; import { toSerializedJourney } from "./components/register/serialize-journey"; +import { fetchCheckImagePanelData, type CheckImagePanelRow } from "./check-images-data"; +import { CheckImagesPanel } from "./components/check-images-panel"; export const dynamic = "force-dynamic"; @@ -443,6 +445,28 @@ export default async function AutomationPage(props: { console.error("pipeline health inputs failed", error instanceof Error ? error.message : "UnknownError"); pipelineHealthUnavailable = true; } + + // Check images panel (check-payer pipeline worklist) — independent of the + // register/merge/pipeline-health fetches above, so it fails alone: an + // error here degrades to an honest "unavailable" card, never the page. + let checkImagesUnavailable = false; + let checkImageRows: CheckImagePanelRow[] = []; + let checkImageTotal = 0; + try { + const checkImages = await withTimeout(fetchCheckImagePanelData(), 15_000); + checkImageRows = checkImages.rows; + checkImageTotal = checkImages.totalImages; + } catch (error) { + console.error("check image panel fetch failed", error instanceof Error ? error.message : "UnknownError"); + checkImagesUnavailable = true; + } + const checkImagesSection: ReactNode = checkImagesUnavailable ? ( +
+ Check images unavailable right now — the register above is still current. +
+ ) : ( + + ); const minutesSaved = summary.pushedThisMonth * 4; const hoursSavedRaw = Math.round((minutesSaved / 60) * 2) / 2; const hoursSavedLabel = Number.isInteger(hoursSavedRaw) ? String(hoursSavedRaw) : hoursSavedRaw.toFixed(1); @@ -698,6 +722,9 @@ export default async function AutomationPage(props: { {/* Orphan receipts */} {orphanSection} + {/* Check images — check-payer pipeline worklist (human confirm) */} + {checkImagesSection} + {/* Receipt pipeline — journey list, Verify in QuickBooks + AI review (plan §3) */} {journeySection} diff --git a/src/app/projects/[id]/invoices/[invoiceId]/InvoiceEditor.tsx b/src/app/projects/[id]/invoices/[invoiceId]/InvoiceEditor.tsx index c318aff66..90cfc2298 100644 --- a/src/app/projects/[id]/invoices/[invoiceId]/InvoiceEditor.tsx +++ b/src/app/projects/[id]/invoices/[invoiceId]/InvoiceEditor.tsx @@ -13,6 +13,7 @@ import DocumentComments from "@/components/DocumentComments"; import { toast } from "sonner"; import { formatCurrency } from "@/lib/utils"; import { formatMoneyDate } from "@/lib/payment-date"; +import type { CheckEvidence } from "@/lib/check-evidence"; const METHOD_LABELS: Record = { card: "Card", @@ -30,7 +31,7 @@ function formatPaymentMethod(method: string | null | undefined, ref: string | nu return label; } -export default function InvoiceEditor({ project, initialInvoice }: { project: any, initialInvoice: any }) { +export default function InvoiceEditor({ project, initialInvoice, checkEvidence = {} }: { project: any, initialInvoice: any, checkEvidence?: Record }) { const router = useRouter(); const [isIssuing, setIsIssuing] = useState(false); const [isDeleting, setIsDeleting] = useState(false); @@ -789,6 +790,7 @@ export default function InvoiceEditor({ project, initialInvoice }: { project: an {initialInvoice.payments?.map((payment: any) => { const isPastDue = payment.dueDate && new Date(payment.dueDate) < new Date() && payment.status !== "Paid"; const methodLabel = formatPaymentMethod(payment.paymentMethod, payment.referenceNumber); + const evidence = payment.status === "Paid" ? checkEvidence[payment.id] : undefined; const receiptSentLabel = payment.receiptSentAt ? `Last sent ${new Date(payment.receiptSentAt).toLocaleString(undefined, { year: 'numeric', month: 'short', day: 'numeric', hour: 'numeric', minute: '2-digit' })}` : undefined; @@ -837,6 +839,24 @@ export default function InvoiceEditor({ project, initialInvoice }: { project: an {payment.status === 'Paid' && methodLabel && (
{methodLabel}
)} + {evidence && ( +
+ Paid by {evidence.payerName ?? "(payer not readable on image)"}, chk#{evidence.checkNumber} + {evidence.driveFileId && ( + <> + {" · "} + + check image ↗ + + + )} +
+ )} {payment.status !== 'Paid' && sentLabel && (
diff --git a/src/app/projects/[id]/invoices/[invoiceId]/page.tsx b/src/app/projects/[id]/invoices/[invoiceId]/page.tsx index 2f6cc6bf1..5c19a1aed 100644 --- a/src/app/projects/[id]/invoices/[invoiceId]/page.tsx +++ b/src/app/projects/[id]/invoices/[invoiceId]/page.tsx @@ -1,6 +1,7 @@ import { getProject } from "@/lib/actions"; import { getInvoice } from "@/lib/actions"; import { notFound } from "next/navigation"; +import { fetchCheckEvidenceForPayments, type CheckEvidence } from "@/lib/check-evidence"; import InvoiceEditor from "./InvoiceEditor"; export default async function InvoicePage({ params }: { params: Promise<{ id: string, invoiceId: string }> }) { @@ -22,9 +23,30 @@ export default async function InvoicePage({ params }: { params: Promise<{ id: st notFound(); } + // Check evidence for paid-by-check milestones: a human-confirmed + // BankImageMatch whose check number AND amount agree lets the editor show + // "Paid by , chk#" from the physical instrument. Display-only + // and best-effort — a failure here must never take down the invoice. + let checkEvidence: Record = {}; + try { + checkEvidence = await fetchCheckEvidenceForPayments( + (initialInvoice.payments ?? []) + .filter((p) => p.status === "Paid" && p.referenceNumber) + .map((p) => ({ + id: p.id, + referenceNumber: p.referenceNumber, + // Decimal dollars → integer cents; a NaN becomes null and + // the matcher skips it rather than matching loosely. + amountCents: Number.isFinite(Number(p.amount)) ? Math.round(Number(p.amount) * 100) : null, + })), + ); + } catch (error) { + console.error("check evidence fetch failed", error instanceof Error ? error.message : "UnknownError"); + } + return (
- +
); } diff --git a/src/lib/actions.ts b/src/lib/actions.ts index ec2125f8d..ba7b55c5c 100644 --- a/src/lib/actions.ts +++ b/src/lib/actions.ts @@ -2536,14 +2536,21 @@ export async function approveEstimate(estimateId: string, signatureName: string, let pdfBuffer: Buffer | null = null; let attachments: any = undefined; try { + const { applyGoldenTouchWatermarkToPdfBytes, generateEstimatePdf } = await import("./pdf"); if (capturedPdfUrl) { pdfBuffer = await downloadDocBytes(capturedPdfUrl); if (!pdfBuffer) { console.warn("[approveEstimate] Failed to read capturedPdfUrl:", capturedPdfUrl); + } else { + try { + pdfBuffer = await applyGoldenTouchWatermarkToPdfBytes(pdfBuffer); + } catch (e) { + console.warn("[approveEstimate] Captured PDF watermark failed; rebuilding from the server estimate:", e); + pdfBuffer = null; + } } } if (!pdfBuffer) { - const { generateEstimatePdf } = await import("./pdf"); pdfBuffer = await generateEstimatePdf(estimateId); } if (pdfBuffer) { @@ -5671,17 +5678,25 @@ export async function sendEstimateToClient( let pdfAttached = false; try { let pdfBuffer: Buffer | undefined; + const { applyGoldenTouchWatermarkToPdfBytes, generateEstimatePdf } = await import("./pdf"); if (capturedPdfUrl && (isSecureRef(capturedPdfUrl) || isAllowedCapturedPdfUrl(capturedPdfUrl))) { // Use the pre-captured portal PDF (high-quality, matches what client sees). // Read via the service key so this still works now that the capture lands in the // private bucket; the allowlist still gates legacy http(s) values. - pdfBuffer = (await downloadDocBytes(capturedPdfUrl)) ?? undefined; + const capturedPdf = await downloadDocBytes(capturedPdfUrl); + if (capturedPdf) { + try { + pdfBuffer = await applyGoldenTouchWatermarkToPdfBytes(capturedPdf); + } catch (e) { + console.warn("[sendEstimateToClient] Captured PDF watermark failed; rebuilding from the server estimate:", e); + pdfBuffer = undefined; + } + } } else if (capturedPdfUrl) { console.warn("[sendEstimateToClient] Rejected capturedPdfUrl (failed allowlist):", capturedPdfUrl); } if (!pdfBuffer) { // Fall back to server-side PDF generation - const { generateEstimatePdf } = await import("./pdf"); pdfBuffer = await generateEstimatePdf(estimateId); } if (pdfBuffer) { @@ -14739,3 +14754,55 @@ export async function deletePermit(permitId: string) { revalidatePath(`/projects/${target.projectId}/permits`); return { success: true }; } + +// ── Bank image match confirmation (Automation "Check images" panel) ───────── + +/** + * The ONE writer of BankImageMatch. A row in that table means a HUMAN said + * "this image explains this bank line" (prisma/schema.prisma) — the matchers + * (lib/bank-image.ts, lib/check-payer-match.ts, the extract script's + * --report) only ever suggest. Gated on financialReports like the Automation + * page itself: internal roles only, never a portal role (portal sessions + * carry no staff user, so assertActiveStaff already rejects them). + * + * bankImageId is @unique on BankImageMatch, so a second confirm for the same + * image loses the P2002 race honestly instead of double-writing. + */ +export async function confirmBankImageMatch(input: { + bankImageId: string; + bankLineId: string; + note?: string | null; +}): Promise<{ success: true } | { success: false; error: string }> { + const user = await assertFinancialPermission(); + + const bankImageId = (input.bankImageId ?? "").trim(); + const bankLineId = (input.bankLineId ?? "").trim(); + if (!bankImageId || !bankLineId) { + return { success: false, error: "Missing image or bank line id" }; + } + // The note is display/audit text from our own proposal reason — cap it so + // a mangled payload can't stuff arbitrary blobs into the audit trail. + const note = (input.note ?? "").trim().slice(0, 500) || null; + + const [image, line] = await Promise.all([ + prisma.bankImage.findUnique({ where: { id: bankImageId }, select: { id: true, sourceExternalId: true } }), + prisma.bankLine.findUnique({ where: { id: bankLineId }, select: { id: true } }), + ]); + if (!image) return { success: false, error: "That image no longer exists" }; + if (!line) return { success: false, error: "That bank line no longer exists" }; + + const confirmedBy = user.email ?? user.name ?? "staff"; + try { + await prisma.bankImageMatch.create({ + data: { bankImageId, bankLineId, confirmedBy, note }, + }); + } catch (e: any) { + if (e?.code === "P2002") { + return { success: false, error: "This image is already confirmed — refresh the page" }; + } + throw e; + } + + revalidatePath("/automation"); + return { success: true }; +} diff --git a/src/lib/check-evidence.ts b/src/lib/check-evidence.ts new file mode 100644 index 000000000..4e9a9015d --- /dev/null +++ b/src/lib/check-evidence.ts @@ -0,0 +1,146 @@ +/** + * Check evidence for invoice payments — "Paid by X, chk#N", backed by the + * physical check image a human confirmed against the bank ledger. + * + * A paid milestone recorded as method "check" carries a referenceNumber (the + * check #). A confirmed BankImageMatch means a human said "this image + * explains this bank line". This module ties the two together so the invoice + * detail can show the payer printed on the actual instrument. + * + * CHECK NUMBERS ARE NOT UNIQUE ACROSS PAYERS — every checkbook has a #1027. + * So a number match alone is never enough: the amount must corroborate + * (image amount when readable, otherwise the confirmed bank line's amount). + * No corroborating amount ⇒ no evidence shown. Honest silence beats a + * plausible wrong name on a money page. + * + * READ-ONLY and display-only: nothing here writes, settles, or notifies. + * The pure matcher is separated from the Prisma fetch for unit testing + * (tests/check-evidence.test.ts). + */ + +import { prisma } from "@/lib/prisma"; +import { normalizeCheckRef } from "@/lib/check-payer-match"; + +export interface CheckEvidence { + /** Payer printed on the check, when the extraction could read it. */ + payerName: string | null; + /** Digits-only check number (the normalized identity). */ + checkNumber: string; + /** Google Drive file id for the image, when the pull recorded one. */ + driveFileId: string | null; + fileName: string | null; + /** Who confirmed the image ↔ bank line match, and when (ISO). */ + confirmedBy: string; + confirmedAt: string; +} + +/** Confirmed image row shape the pure matcher consumes. */ +export interface ConfirmedCheckImage { + normalizedCheckNumber: string | null; + /** Positive cents printed on the document, when readable. */ + amountCents: number | null; + /** Signed cents of the confirmed bank line, when the match carries one. */ + lineAmountCents: number | null; + payerName: string | null; + driveFileId: string | null; + fileName: string | null; + confirmedBy: string; + confirmedAt: string; +} + +export interface PaymentForEvidence { + id: string; + referenceNumber: string | null; + /** Milestone amount in cents. */ + amountCents: number | null; +} + +/** + * Pure. For each payment, find the confirmed check image whose number AND + * amount both agree. Ambiguity (two confirmed images with the same number + * and amount) yields nothing — never a guess. Returns paymentId → evidence. + */ +export function matchCheckEvidence( + payments: PaymentForEvidence[], + confirmedImages: ConfirmedCheckImage[], +): Map { + const out = new Map(); + + for (const payment of payments) { + const ref = normalizeCheckRef(payment.referenceNumber); + if (!ref) continue; + const amountCents = payment.amountCents; + if (amountCents === null || !Number.isSafeInteger(amountCents) || amountCents <= 0) continue; + + const hits = confirmedImages.filter((img) => { + if (img.normalizedCheckNumber !== ref) return false; + // Amount corroboration: the document amount when readable, + // otherwise the confirmed bank line's magnitude. + if (img.amountCents !== null) return img.amountCents === amountCents; + if (img.lineAmountCents !== null) return Math.abs(img.lineAmountCents) === amountCents; + return false; + }); + + if (hits.length !== 1) continue; // 0 = no evidence; 2+ = ambiguous, never guess + const img = hits[0]; + out.set(payment.id, { + payerName: img.payerName, + checkNumber: ref, + driveFileId: img.driveFileId, + fileName: img.fileName, + confirmedBy: img.confirmedBy, + confirmedAt: img.confirmedAt, + }); + } + + return out; +} + +/** + * Server fetch for the invoice detail page. Loads only the confirmed + * check-front images whose numbers appear on the given payments, then runs + * the pure matcher. Failures are the caller's to degrade on (the invoice + * page renders fine with no evidence). + */ +export async function fetchCheckEvidenceForPayments( + payments: PaymentForEvidence[], +): Promise> { + const refs = [...new Set( + payments.map((p) => normalizeCheckRef(p.referenceNumber)).filter((r): r is string => r !== null), + )]; + if (refs.length === 0) return {}; + + const matches = await prisma.bankImageMatch.findMany({ + where: { + bankImage: { + kind: "CHECK_FRONT", + normalizedCheckNumber: { in: refs }, + }, + }, + include: { + bankImage: { + select: { + normalizedCheckNumber: true, + amountCents: true, + payerName: true, + driveFileId: true, + fileName: true, + }, + }, + bankLine: { select: { amountCents: true } }, + }, + }); + + const confirmed: ConfirmedCheckImage[] = matches.map((m) => ({ + normalizedCheckNumber: m.bankImage.normalizedCheckNumber, + amountCents: m.bankImage.amountCents, + lineAmountCents: m.bankLine?.amountCents ?? null, + payerName: m.bankImage.payerName, + driveFileId: m.bankImage.driveFileId, + fileName: m.bankImage.fileName, + confirmedBy: m.confirmedBy, + confirmedAt: m.confirmedAt.toISOString(), + })); + + return Object.fromEntries(matchCheckEvidence(payments, confirmed)); +} diff --git a/src/lib/check-payer-match.ts b/src/lib/check-payer-match.ts new file mode 100644 index 000000000..27ec30170 --- /dev/null +++ b/src/lib/check-payer-match.ts @@ -0,0 +1,93 @@ +/** + * Check-payer fuzzy matching — payerName → Client, memoText → Project. + * + * TS port of the suggestion logic in scripts/extract-check-payers.mjs + * (nameTokens / nameSimilarity / suggestMatches). The script stays the CLI + * entry point; this module is what the Automation "Check images" panel uses + * server-side. tests/check-payer-match.test.ts asserts the two stay in + * agreement, so neither can drift silently. + * + * SUGGESTION ONLY — same house rule as bank-image.ts and vendor-alias: + * nothing here may ever write BankImageMatch. That table means a HUMAN said + * yes; the confirm server action is the only writer. + * + * PURE: no Prisma, no I/O. + */ + +const NAME_NOISE = new Set([ + "llc", "inc", "co", "corp", "ltd", "the", "and", "&", "of", + "mr", "mrs", "ms", "dr", "or", +]); + +export interface NamedRow { + id: string; + name: string; +} + +export interface MatchSuggestion { + id: string; + name: string; + /** 0..1 — token-set Jaccard with a containment bonus. */ + score: number; +} + +export interface CheckMatchSuggestions { + payerMatches: MatchSuggestion[]; + memoMatches: MatchSuggestion[]; +} + +/** Minimum similarity for a suggestion to surface at all. */ +export const SUGGESTION_THRESHOLD = 0.4; +/** Suggestions shown per field. */ +export const SUGGESTION_LIMIT = 3; + +export function nameTokens(s: string | null | undefined): string[] { + return String(s ?? "") + .toLowerCase() + .replace(/[^a-z0-9\s]/g, " ") + .split(/\s+/) + .filter(t => t && !NAME_NOISE.has(t)); +} + +/** Pure. 0..1 similarity: token-set Jaccard plus a containment bonus. */ +export function nameSimilarity(a: string | null | undefined, b: string | null | undefined): number { + const ta = nameTokens(a), tb = nameTokens(b); + if (!ta.length || !tb.length) return 0; + const sa = new Set(ta), sb = new Set(tb); + const inter = [...sa].filter(t => sb.has(t)).length; + const union = new Set([...sa, ...sb]).size; + const jaccard = inter / union; + const containment = inter / Math.min(sa.size, sb.size); + return Math.max(jaccard, containment * 0.85); +} + +/** + * Pure. Suggest candidate matches for one extraction. Returns + * { payerMatches, memoMatches } sorted by score, top 3 each, threshold 0.4. + * A null/blank payer or memo yields an empty list — never a guess. + */ +export function suggestMatches( + extraction: { payerName: string | null; memoText: string | null }, + clients: NamedRow[], + projects: NamedRow[], +): CheckMatchSuggestions { + const rank = (text: string, rows: NamedRow[]): MatchSuggestion[] => rows + .map(r => ({ id: r.id, name: r.name, score: nameSimilarity(text, r.name) })) + .filter(m => m.score >= SUGGESTION_THRESHOLD) + .sort((x, y) => y.score - x.score || x.id.localeCompare(y.id)) + .slice(0, SUGGESTION_LIMIT); + return { + payerMatches: extraction.payerName ? rank(extraction.payerName, clients) : [], + memoMatches: extraction.memoText ? rank(extraction.memoText, projects) : [], + }; +} + +/** + * Normalize a check-number-ish reference to the identity every parser in + * this repo produces: digits only, leading zeros stripped. Returns null for + * anything that leaves no digits — "" must never match "" on a join. + */ +export function normalizeCheckRef(value: string | null | undefined): string | null { + const digits = String(value ?? "").replace(/\D/g, "").replace(/^0+/, ""); + return digits.length ? digits : null; +} diff --git a/src/lib/pdf.ts b/src/lib/pdf.ts index cb31d76f9..a9d9ca1a6 100644 --- a/src/lib/pdf.ts +++ b/src/lib/pdf.ts @@ -1,4 +1,4 @@ -import { PDFDocument, PDFPage, PDFFont, PDFImage, rgb, StandardFonts } from 'pdf-lib'; +import { PDFDocument, PDFPage, PDFFont, PDFImage, degrees, rgb, StandardFonts } from 'pdf-lib'; import { prisma } from './prisma'; import { toNum } from './prisma-helpers'; import { buildLetterheadConfig, type LetterheadConfig } from './letterhead'; @@ -136,6 +136,48 @@ const colors = { white: rgb(1, 1, 1), }; +const GOLDEN_TOUCH_WATERMARK = 'GOLDEN TOUCH REMODELING'; + +/** + * Stamps every finished page immediately before the PDF is saved. Keeping this + * at the document boundary means the preview/download and email-attachment + * paths use the same branded file, including documents that gained extra pages. + */ +export async function applyGoldenTouchWatermark(doc: PDFDocument): Promise { + const font = await doc.embedFont(StandardFonts.HelveticaBold); + const pages = doc.getPages(); + + for (const page of pages) { + const { width, height } = page.getSize(); + const unitTextWidth = font.widthOfTextAtSize(GOLDEN_TOUCH_WATERMARK, 1); + const fontSize = Math.min(48, (width * 0.78) / unitTextWidth); + const textWidth = font.widthOfTextAtSize(GOLDEN_TOUCH_WATERMARK, fontSize); + const angle = 35 * Math.PI / 180; + const watermarkWidth = Math.cos(angle) * textWidth + Math.sin(angle) * fontSize; + const watermarkHeight = Math.sin(angle) * textWidth + Math.cos(angle) * fontSize; + + page.drawText(GOLDEN_TOUCH_WATERMARK, { + // Center the rotated bounding box, not just the unrotated baseline. + x: (width - watermarkWidth) / 2 + Math.sin(angle) * fontSize, + y: (height - watermarkHeight) / 2, + size: fontSize, + font, + color: rgb(0.85, 0.66, 0.08), + opacity: 0.10, + rotate: degrees(35), + }); + } + + return pages.length; +} + +/** Adds the required watermark to a static portal-captured estimate PDF before it is sent or filed. */ +export async function applyGoldenTouchWatermarkToPdfBytes(pdfBytes: Buffer): Promise { + const document = await PDFDocument.load(pdfBytes, { ignoreEncryption: true }); + await applyGoldenTouchWatermark(document); + return Buffer.from(await document.save()); +} + function hexToRgb(hex: string) { const h = hex.replace('#', ''); const r = parseInt(h.substring(0, 2), 16) / 255; @@ -743,6 +785,7 @@ export async function generateEstimatePdf(estimateId: string): Promise { x: pageWidth - margin - pageLabelWidth, y: footerY, size: 7, font: helvetica, color: colors.textMuted, }); + await applyGoldenTouchWatermark(doc); const pdfBytes = await doc.save(); return Buffer.from(pdfBytes); } @@ -1184,6 +1227,7 @@ export async function generateInvoicePdf( const footerText = `Generated ${new Date().toLocaleDateString()} • ${company?.companyName || 'ProBuild'}`; page.drawText(footerText, { x: margin, y: 30, size: 7, font: helvetica, color: colors.textMuted }); + await applyGoldenTouchWatermark(doc); const pdfBytes = await doc.save(); return Buffer.from(pdfBytes); } diff --git a/tests/check-evidence.test.ts b/tests/check-evidence.test.ts new file mode 100644 index 000000000..2499643db --- /dev/null +++ b/tests/check-evidence.test.ts @@ -0,0 +1,109 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + matchCheckEvidence, + type ConfirmedCheckImage, + type PaymentForEvidence, +} from "@/lib/check-evidence"; + +// The real prod row this feature was built for: check #1027, +// "GOLDEN TOUCH RMEODELING LLC", HOPPE VANITY memo, $6,037.15. +const hoppeImage = (over: Partial = {}): ConfirmedCheckImage => ({ + normalizedCheckNumber: "1027", + amountCents: 603715, + lineAmountCents: -603715, + payerName: "GOLDEN TOUCH RMEODELING LLC", + driveFileId: "drive-abc", + fileName: "26225018006376-front.jpg", + confirmedBy: "justin@goldentouchremodeling.com", + confirmedAt: "2026-08-22T05:00:00.000Z", + ...over, +}); + +const paidByCheck = (over: Partial = {}): PaymentForEvidence => ({ + id: "pay-1", + referenceNumber: "1027", + amountCents: 603715, + ...over, +}); + +test("check number + amount both agreeing yields evidence", () => { + const map = matchCheckEvidence([paidByCheck()], [hoppeImage()]); + const ev = map.get("pay-1"); + assert.ok(ev); + assert.equal(ev.payerName, "GOLDEN TOUCH RMEODELING LLC"); + assert.equal(ev.checkNumber, "1027"); + assert.equal(ev.driveFileId, "drive-abc"); +}); + +test("referenceNumber is normalized before matching (leading zeros, decorations)", () => { + const map = matchCheckEvidence( + [paidByCheck({ referenceNumber: "chk #01027" })], + [hoppeImage()], + ); + assert.ok(map.get("pay-1")); +}); + +test("same check number but a DIFFERENT amount is NOT evidence — check numbers collide across payers", () => { + const map = matchCheckEvidence( + [paidByCheck({ amountCents: 500000 })], + [hoppeImage()], + ); + assert.equal(map.get("pay-1"), undefined); +}); + +test("image amount unreadable: the confirmed bank line's magnitude corroborates instead", () => { + const map = matchCheckEvidence( + [paidByCheck()], + [hoppeImage({ amountCents: null, lineAmountCents: -603715 })], + ); + assert.ok(map.get("pay-1")); +}); + +test("no readable image amount AND no line amount: no evidence, never a guess", () => { + const map = matchCheckEvidence( + [paidByCheck()], + [hoppeImage({ amountCents: null, lineAmountCents: null })], + ); + assert.equal(map.get("pay-1"), undefined); +}); + +test("two confirmed images with the same number and amount is ambiguous — nothing shown", () => { + const map = matchCheckEvidence( + [paidByCheck()], + [hoppeImage(), hoppeImage({ payerName: "Someone Else", driveFileId: "drive-xyz" })], + ); + assert.equal(map.get("pay-1"), undefined); +}); + +test("null referenceNumber / zero-digit reference / non-check payments are skipped", () => { + const map = matchCheckEvidence( + [ + paidByCheck({ id: "p-null", referenceNumber: null }), + paidByCheck({ id: "p-empty", referenceNumber: "0000" }), + paidByCheck({ id: "p-alpha", referenceNumber: "no digits" }), + ], + [hoppeImage()], + ); + assert.equal(map.size, 0); +}); + +test("null / non-integer / non-positive payment amounts are skipped, not matched loosely", () => { + const map = matchCheckEvidence( + [ + paidByCheck({ id: "p-null-amt", amountCents: null }), + paidByCheck({ id: "p-float", amountCents: 6037.15 as unknown as number }), + paidByCheck({ id: "p-neg", amountCents: -603715 }), + ], + [hoppeImage()], + ); + assert.equal(map.size, 0); +}); + +test("null payerName still surfaces evidence (chk# + confirmation without a name)", () => { + const map = matchCheckEvidence([paidByCheck()], [hoppeImage({ payerName: null })]); + const ev = map.get("pay-1"); + assert.ok(ev); + assert.equal(ev.payerName, null); + assert.equal(ev.checkNumber, "1027"); +}); diff --git a/tests/check-payer-match.test.ts b/tests/check-payer-match.test.ts new file mode 100644 index 000000000..5165afefd --- /dev/null +++ b/tests/check-payer-match.test.ts @@ -0,0 +1,110 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + nameTokens, + nameSimilarity, + suggestMatches, + normalizeCheckRef, + SUGGESTION_THRESHOLD, +} from "@/lib/check-payer-match"; +// The CLI script is the original implementation; the lib is its TS port for +// the Automation panel. This import keeps the two honest against each other. +import { + nameTokens as mjsNameTokens, + nameSimilarity as mjsNameSimilarity, + suggestMatches as mjsSuggestMatches, +} from "../scripts/extract-check-payers.mjs"; + +// The REAL extracted row that motivated this feature (prod, 2026-08-21): +// payer "GOLDEN TOUCH RMEODELING LLC" (typo and all), memo +// "HOPPE VANITY CONTRACT 4152", check #1027. +const REAL_PAYER = "GOLDEN TOUCH RMEODELING LLC"; +const REAL_MEMO = "HOPPE VANITY CONTRACT 4152"; + +const clients = [ + { id: "c1", name: "Golden Touch Remodeling" }, + { id: "c2", name: "Sandi Christensen" }, + { id: "c3", name: "Mesplay" }, +]; + +const projects = [ + { id: "p1", name: "Hoppe Vanity" }, + { id: "p2", name: "Christensen Kitchen" }, + { id: "p3", name: "Shop" }, +]; + +test("nameTokens drops entity noise and punctuation", () => { + assert.deepEqual(nameTokens("GOLDEN TOUCH RMEODELING LLC"), ["golden", "touch", "rmeodeling"]); + assert.deepEqual(nameTokens("The Smith & Jones Co."), ["smith", "jones"]); + assert.deepEqual(nameTokens(null), []); + assert.deepEqual(nameTokens(""), []); +}); + +test("nameSimilarity: identical token sets score 1, disjoint score 0", () => { + assert.equal(nameSimilarity("Hoppe Vanity", "Vanity Hoppe"), 1); + assert.equal(nameSimilarity("Mesplay", "Christensen"), 0); + assert.equal(nameSimilarity(null, "anything"), 0); + assert.equal(nameSimilarity("anything", ""), 0); +}); + +test("memo naming the job outranks unrelated projects", () => { + const { memoMatches } = suggestMatches({ payerName: null, memoText: REAL_MEMO }, clients, projects); + assert.ok(memoMatches.length >= 1); + assert.equal(memoMatches[0].id, "p1"); + assert.ok(memoMatches[0].score >= SUGGESTION_THRESHOLD); +}); + +test("payer with a typo still finds the client via shared tokens", () => { + const { payerMatches } = suggestMatches({ payerName: REAL_PAYER, memoText: null }, clients, projects); + assert.ok(payerMatches.length >= 1); + assert.equal(payerMatches[0].id, "c1"); +}); + +test("null payer AND memo yield no suggestions — never a guess", () => { + const { payerMatches, memoMatches } = suggestMatches({ payerName: null, memoText: null }, clients, projects); + assert.deepEqual(payerMatches, []); + assert.deepEqual(memoMatches, []); +}); + +test("scores below the threshold are dropped entirely", () => { + const { payerMatches } = suggestMatches( + { payerName: "Totally Unrelated Person", memoText: null }, + clients, + projects, + ); + assert.deepEqual(payerMatches, []); +}); + +test("normalizeCheckRef: digits only, leading zeros stripped, empty is null", () => { + assert.equal(normalizeCheckRef("chk #01027"), "1027"); + assert.equal(normalizeCheckRef("1027"), "1027"); + assert.equal(normalizeCheckRef("0000"), null); + assert.equal(normalizeCheckRef(""), null); + assert.equal(normalizeCheckRef(null), null); + assert.equal(normalizeCheckRef("no digits"), null); +}); + +// ── parity with the CLI script — the port must not drift ───────────────── +test("lib agrees with scripts/extract-check-payers.mjs on tokens and similarity", () => { + const samples: Array<[string, string]> = [ + [REAL_PAYER, "Golden Touch Remodeling"], + [REAL_MEMO, "Hoppe Vanity"], + ["Sandi Christensen", "Christensen Kitchen"], + ["The Smith & Jones Co.", "Smith Residence"], + ["", "anything"], + ]; + for (const [a, b] of samples) { + assert.deepEqual(nameTokens(a), mjsNameTokens(a), `tokens drifted for "${a}"`); + assert.equal(nameSimilarity(a, b), mjsNameSimilarity(a, b), `similarity drifted for "${a}" vs "${b}"`); + } + const libResult = suggestMatches({ payerName: REAL_PAYER, memoText: REAL_MEMO }, clients, projects); + const mjsResult = mjsSuggestMatches({ payerName: REAL_PAYER, memoText: REAL_MEMO }, clients, projects); + assert.deepEqual( + libResult.payerMatches.map(m => [m.id, m.score]), + mjsResult.payerMatches.map((m: { id: string; score: number }) => [m.id, m.score]), + ); + assert.deepEqual( + libResult.memoMatches.map(m => [m.id, m.score]), + mjsResult.memoMatches.map((m: { id: string; score: number }) => [m.id, m.score]), + ); +}); diff --git a/tests/extract-check-payers.test.ts b/tests/extract-check-payers.test.ts new file mode 100644 index 000000000..0621681d0 --- /dev/null +++ b/tests/extract-check-payers.test.ts @@ -0,0 +1,280 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + scrubField, + scrubExtraction, + parseModelJson, + nameSimilarity, + suggestMatches, + loadCandidatesFromManifest, + PAYER_NAME_MAX, + MEMO_TEXT_MAX, +} from "../scripts/extract-check-payers.mjs"; + +// ── the MICR / routing / account ban ──────────────────────────────────── +// This is the guard the whole pipeline leans on: NOTHING with a long digit +// run may survive scrubbing, no matter what the model returned. + +test("scrubField passes ordinary payer names through", () => { + assert.deepEqual(scrubField("Smith Family Trust"), { value: "Smith Family Trust", dropped: null, truncated: false }); +}); + +test("scrubField drops a 9-digit routing number", () => { + const { value, dropped } = scrubField("routing 125108272"); + assert.equal(value, null); + assert.match(dropped ?? "", /banned digit run/); +}); + +test("scrubField drops account numbers split by spaces or dashes", () => { + assert.equal(scrubField("1234 5678 9012").value, null); + assert.equal(scrubField("1234-5678-90").value, null); +}); + +test("scrubField drops MICR transit symbols outright", () => { + const { value, dropped } = scrubField("\u2446125108272\u2446"); + assert.equal(value, null); + assert.match(dropped ?? "", /MICR/); +}); + +test("scrubField allows an explicitly allowed long digit string", () => { + assert.equal(scrubField("ref 123456789", ["123456789"]).value, "ref 123456789"); +}); + +// ── Kimi review gap 1: 8-digit accounts split by separators ───────────── + +test("scrubField drops an 8-digit account split by a space", () => { + const { value, dropped } = scrubField("acct 1234 5678"); + assert.equal(value, null); + assert.match(dropped ?? "", /banned digit run/); +}); + +test("scrubField drops an 8-digit account split by dashes", () => { + assert.equal(scrubField("12-34-56-78").value, null); +}); + +// ── Kimi review gap 2: dot / slash separators ─────────────────────────── + +test("scrubField drops a dot-separated routing number", () => { + const { value, dropped } = scrubField("123.456.789"); + assert.equal(value, null); + assert.match(dropped ?? "", /banned digit run/); +}); + +test("scrubField drops a slash-separated account number", () => { + assert.equal(scrubField("123/456/789").value, null); + assert.equal(scrubField("1234/5678").value, null); +}); + +test("scrubField drops comma- and paren-separated digit groups", () => { + assert.equal(scrubField("1234,5678").value, null); + assert.equal(scrubField("(1234) 5678-90").value, null); +}); + +// ── Kimi review gap 3: letter-mixed digit runs ────────────────────────── + +test("scrubField drops letter-mixed runs hiding 8+ digits", () => { + const { value, dropped } = scrubField("A1B2C3D4E5F6G7H8"); + assert.equal(value, null); + assert.match(dropped ?? "", /banned digit run/); +}); + +test("scrubField drops an account with a letter prefix", () => { + assert.equal(scrubField("acct no. 12345678").value, null); +}); + +// ── dates must survive the total-digit rule ───────────────────────────── + +test("scrubField passes ISO and US calendar dates (8 digits, but a date)", () => { + assert.equal(scrubField("2026-08-13").value, "2026-08-13"); + assert.equal(scrubField("8/13/2026").value, "8/13/2026"); + assert.equal(scrubField("12/31/2026").value, "12/31/2026"); +}); + +test("scrubField does NOT exempt an account disguised as an invalid date", () => { + assert.equal(scrubField("12/34/5678").value, null); +}); + +test("scrubField allow-list matches the separator-stripped digit string", () => { + // Allowed "123456789" also covers its spaced form. + assert.equal(scrubField("ref 1234 56789", ["123456789"]).value, "ref 1234 56789"); +}); + +test("scrubField keeps short digit content (check numbers, amounts, addresses)", () => { + assert.equal(scrubField("chk 1027").value, "chk 1027"); + assert.equal(scrubField("6037.15").value, "6037.15"); + assert.equal(scrubField("1234 W 5th Ave").value, "1234 W 5th Ave"); +}); + +// ── length caps ───────────────────────────────────────────────────────── + +test("scrubField truncates past maxLen and flags it", () => { + const long = "X".repeat(150); + const { value, dropped, truncated } = scrubField(long, [], 120); + assert.equal(value, "X".repeat(120)); + assert.equal(dropped, null); + assert.equal(truncated, true); +}); + +test("scrubField leaves values within maxLen untouched", () => { + const { value, truncated } = scrubField("Smith Family Trust", [], 120); + assert.equal(value, "Smith Family Trust"); + assert.equal(truncated, false); +}); + +test("scrubExtraction caps payerName at 120 and memoText at 200, sets needsReview", () => { + const out = scrubExtraction({ + payerName: "P".repeat(PAYER_NAME_MAX + 40), + memoText: "m".repeat(MEMO_TEXT_MAX + 1), + documentDate: null, + amount: null, + checkNumber: null, + }); + assert.equal(out.payerName, "P".repeat(120)); + assert.equal(out.memoText, "m".repeat(200)); + assert.equal(out.needsReview, true); + assert.equal(out.warnings.length, 2); + assert.match(out.warnings[0], /payerName TRUNCATED to 120/); + assert.match(out.warnings[1], /memoText TRUNCATED to 200/); +}); + +// ── needsReview signal ────────────────────────────────────────────────── + +test("scrubExtraction reports needsReview=false on a clean extraction", () => { + const out = scrubExtraction( + { + payerName: "Henderson Kitchen LLC", + memoText: "master bath", + documentDate: "2026-08-13", + amount: "6037.15", + checkNumber: "1027", + }, + { checkNumber: "1027", amountCents: 603715 }, + ); + assert.equal(out.needsReview, false); + assert.equal(out.warnings.length, 0); +}); + +test("scrubExtraction flags needsReview when any field is dropped", () => { + const out = scrubExtraction({ + payerName: "Fine Name", + memoText: "acct 1234 5678", // spaced 8-digit leak + documentDate: null, + amount: null, + checkNumber: null, + }); + assert.equal(out.memoText, null); + assert.equal(out.needsReview, true); +}); + +// ── JSON fence tolerance ──────────────────────────────────────────────── + +test("parseModelJson parses bare JSON", () => { + assert.deepEqual(parseModelJson('{"payerName": "Smith"}'), { payerName: "Smith" }); +}); + +test("parseModelJson tolerates ```json fences", () => { + const fenced = '```json\n{"payerName": "Smith", "memoText": null}\n```'; + assert.deepEqual(parseModelJson(fenced), { payerName: "Smith", memoText: null }); +}); + +test("parseModelJson tolerates bare ``` fences and surrounding whitespace", () => { + const fenced = ' ```\n{"amount": "6037.15"}\n``` '; + assert.deepEqual(parseModelJson(fenced), { amount: "6037.15" }); +}); + +test("parseModelJson still throws on genuinely invalid JSON", () => { + assert.throws(() => parseModelJson("```json\nnot json at all\n```")); +}); + +test("scrubExtraction keeps payer/memo, drops fields carrying account-like runs", () => { + const out = scrubExtraction( + { + payerName: "Henderson Kitchen LLC", + memoText: "acct 26225018006376", // model leaked something — must die + documentDate: "2026-08-13", + amount: "6037.15", + checkNumber: "1027", + }, + { checkNumber: "1027", amountCents: 603715 }, + ); + assert.equal(out.payerName, "Henderson Kitchen LLC"); + assert.equal(out.memoText, null); + assert.equal(out.warnings.length, 1); + assert.match(out.warnings[0], /memoText DROPPED/); + assert.equal(out.documentDate, "2026-08-13"); + assert.equal(out.checkNumber, "1027"); +}); + +test("scrubExtraction never lets a routing number through any field", () => { + const out = scrubExtraction({ + payerName: "125108272", // routing number where a name should be + memoText: null, + documentDate: null, + amount: null, + checkNumber: null, + }); + assert.equal(out.payerName, null); + assert.equal(out.warnings.length, 1); +}); + +// ── fuzzy matching (review report is suggestion-only) ─────────────────── + +test("nameSimilarity: exact and noise-word-insensitive matches score high", () => { + assert.equal(nameSimilarity("Henderson", "Henderson"), 1); + assert.ok(nameSimilarity("Henderson Kitchen LLC", "The Henderson Kitchen Co") > 0.6); +}); + +test("nameSimilarity: unrelated names score low", () => { + assert.ok(nameSimilarity("Smith Family Trust", "Jones Roofing") < 0.4); +}); + +test("suggestMatches ranks clients by payer and projects by memo, threshold 0.4", () => { + const clients = [ + { id: "c1", name: "Sarah Henderson" }, + { id: "c2", name: "Bob Jones" }, + ]; + const projects = [ + { id: "p1", name: "Henderson Master Bath" }, + { id: "p2", name: "Shop" }, + ]; + const { payerMatches, memoMatches } = suggestMatches( + { payerName: "Henderson, Sarah", memoText: "master bath" }, + clients, + projects, + ); + assert.equal(payerMatches[0]?.id, "c1"); + assert.equal(memoMatches[0]?.id, "p1"); + assert.ok(!payerMatches.some((m: { id: string }) => m.id === "c2")); +}); + +// ── manifest candidate derivation (pre-DDL dry-run path) ──────────────── + +test("loadCandidatesFromManifest derives kinds like post-bank-images and respects the kind filter", () => { + const manifest = { + images: { + "26225018006376": { + bankReference: "26225018006376", + checkNumber: "1027", + files: [ + { fileName: "front.jpg", side: "front" }, + { fileName: "back.jpg", side: "back" }, + ], + }, + }, + }; + const rows = loadCandidatesFromManifest(manifest, ["CHECK_FRONT"], 10); + assert.equal(rows.length, 1); + assert.equal(rows[0].kind, "CHECK_FRONT"); + assert.equal(rows[0].normalizedCheckNumber, "1027"); + assert.equal(rows[0].sourceExternalId, "26225018006376:front"); +}); + +test("loadCandidatesFromManifest honors the limit", () => { + const manifest = { + images: { + a: { bankReference: "a", checkNumber: "1", files: [{ fileName: "1.jpg", side: "front" }] }, + b: { bankReference: "b", checkNumber: "2", files: [{ fileName: "2.jpg", side: "front" }] }, + }, + }; + assert.equal(loadCandidatesFromManifest(manifest, ["CHECK_FRONT"], 1).length, 1); +}); diff --git a/tests/pdf-watermark.test.ts b/tests/pdf-watermark.test.ts new file mode 100644 index 000000000..6adf69fce --- /dev/null +++ b/tests/pdf-watermark.test.ts @@ -0,0 +1,44 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { PDFDocument, StandardFonts } from "pdf-lib"; +import { PDFParse } from "pdf-parse"; + +import { + applyGoldenTouchWatermark, + applyGoldenTouchWatermarkToPdfBytes, +} from "../src/lib/pdf"; + +async function extractText(pdfBytes: Buffer): Promise { + const parser = new PDFParse({ data: pdfBytes }); + try { + return (await parser.getText()).text; + } finally { + await parser.destroy(); + } +} + +test("applies the Golden Touch Remodeling watermark to every PDF page", async () => { + const document = await PDFDocument.create(); + document.addPage([612, 792]); + document.addPage([612, 792]); + + const stampedPages = await applyGoldenTouchWatermark(document); + const extractedText = await extractText(Buffer.from(await document.save())); + const watermarkCount = extractedText.split("GOLDEN TOUCH REMODELING").length - 1; + + assert.equal(stampedPages, 2); + assert.equal(watermarkCount, 2, "each page must contain the Golden Touch Remodeling watermark"); +}); + +test("stamps a pre-captured PDF without dropping its existing contents", async () => { + const original = await PDFDocument.create(); + const page = original.addPage([612, 792]); + const font = await original.embedFont(StandardFonts.Helvetica); + page.drawText("SIGNED ESTIMATE #1001", { x: 72, y: 700, size: 18, font }); + + const stampedPdf = await applyGoldenTouchWatermarkToPdfBytes(Buffer.from(await original.save())); + const extractedText = await extractText(stampedPdf); + + assert.match(extractedText, /SIGNED ESTIMATE #1001/); + assert.match(extractedText, /GOLDEN TOUCH REMODELING/); +});