reconciliation-mismatch alerts

TODO.md

@@ -1,15 +1,19 @@
-# TODO
-
-## Invoice state machine exhaustive transition matrix tests
-- [ ] Read current invoice.state.test.js and understand existing coverage gaps.
-- [ ] Implement a data-driven Cartesian matrix test for every (fromState, targetState) pair across INVOICE_STATES.
-  - [ ] Assert isTransitionAllowed outcome matches VALID_TRANSITIONS, including silent-jump prevention (pending → linked_escrow) and self transitions.
-  - [ ] Assert validateTransition returns exact error codes: INVALID_TRANSITION, TERMINAL_STATE, ALREADY_IN_TARGET_STATE.
-  - [ ] Ensure terminal states reject all outgoing transitions.
-- [ ] Add executeTransition assertions:
-  - [ ] For each allowed transition, ensure it writes an audit log entry (action=STATE_TRANSITION) with correct before/after states and metadata reason/transitionType.
-  - [ ] For each disallowed transition, ensure executeTransition throws an error with correct error.code.
-- [ ] Add executeTransition/canLinkToEscrow coverage for business rule: must be approved to link.
-- [x] Run test suite and confirm invoiceStateMachine.js hits 100% branch coverage (cannot be executed here due to missing `npm` in terminal).
+# TODO - enhancement/reconciliation-14-mismatch-alerts
 
+- [ ] Inspect current Sentry wiring and matcher for captureException
+- [x] Implement structured mismatch alert helper in `src/jobs/reconcileEscrow.js`:
+  - [x] Keep existing warn log and `escrowReconciliationMismatches.inc()` untouched
+  - [x] Add Sentry capture (minimal, scrubbed fields only) behind Sentry enabled check
+  - [x] Add env configurable mismatch alert threshold + channel
+
+- [ ] Update `src/services/health.js` (`checkReconciliationHealth`) so `/ready` degrades only when `summary.mismatches >= RECONCILIATION_DRIFT_THRESHOLD`
+
+- [ ] Update tests in `tests/reconcileEscrow.test.js`:
+  - [ ] Cover mismatch alerts (log + Sentry capture)
+  - [ ] Cover Sentry disabled
+  - [ ] Cover threshold breach health degradation
+  - [ ] Cover read failure health path
+- [ ] Update documentation `docs/ops-reconcile.md` with env vars + behavior
+- [ ] Run `npm test` and `npm run lint`
+- [ ] Ensure ≥95% test coverage for impacted modules
 

src/jobs/reconcileEscrow.js

@@ -32,6 +32,8 @@
 } = require('../metrics');
 const JobQueue = require('../workers/jobQueue');
 const BackgroundWorker = require('../workers/worker');
+const sentry = require('../observability/sentry');
+
 
 /**
  * Reconciliation result status.
@@ -74,6 +76,84 @@
  */
 const DRIFT_THRESHOLD = getDriftThreshold();
 
+
+/**
+ * Reads the mismatch alert threshold for per-invoice mismatch alerts.
+
+ * Defaults to 1 so any mismatch triggers an alert.
+ *
+ * @returns {number}
+ */
+function getMismatchAlertThreshold() {
+  return Math.max(
+    1,
+    parseInt(process.env.RECONCILIATION_MISMATCH_ALERT_THRESHOLD, 10) || 1,
+  );
+}
+
+/**
+ * Returns the configured alert channel.
+ *
+ * @returns {'log'|'sentry'|'sentry+log'}
+ */
+function getMismatchAlertChannel() {
+  const raw = process.env.RECONCILIATION_MISMATCH_ALERT_CHANNEL || 'sentry+log';
+  if (raw === 'log') return 'log';
+  if (raw === 'sentry') return 'sentry';
+  if (raw === 'sentry+log') return 'sentry+log';
+  return 'sentry+log';
+}
+
+/**
+ * Raises an alert when an escrow reconciliation mismatch is detected.
+ *
+ * Security: only non-sensitive, minimal fields are emitted.
+ *
+ * @param {object} params
+ * @param {string} params.invoiceId
+ * @param {number} params.expected - Expected funded amount (DB).
+ * @param {number} params.actual - Actual funded amount (on-chain).
+ * @param {number} params.driftMagnitude
+ * @returns {void}
+ */
+function raiseReconciliationMismatchAlert({
+  invoiceId,
+  expected,
+  actual,
+  driftMagnitude,
+}) {
+  const channel = getMismatchAlertChannel();
+  const threshold = getMismatchAlertThreshold();
+
+  // Threshold determines whether escalation work is performed.
+  // Per-invoice helper treats each mismatch as count=1.
+  if (threshold > 1) return;
+
+  if (channel === 'log' || channel === 'sentry+log') {
+    logger.warn(
+      { invoiceId, expected, actual, driftMagnitude },
+      'Escrow mismatch alert escalation',
+    );
+  }
+
+  if (channel === 'sentry' || channel === 'sentry+log') {
+    if (sentry && typeof sentry.isEnabled === 'function' && sentry.isEnabled()) {
+      const error = new Error('Escrow reconciliation mismatch');
+      sentry.captureException(error, {
+        invoiceId,
+        expected,
+        actual,
+        driftMagnitude,
+      });
+    }
+  }
+}
+
+
+
+
+
+
 /**
  * Coerces a DB-sourced funded total into a finite number.
  *
@@ -101,6 +181,7 @@
  * @yields {{ id: string, fundedTotal: number }} One invoice per iteration.
  */
 async function* iterateInvoicesFromDb(options = {}) {
+
   const dbClient = options.dbClient || db;
   const rawSize = Number(options.pageSize) || DEFAULT_PAGE_SIZE;
   const pageSize = Math.min(Math.max(1, Math.trunc(rawSize)), 1000);
@@ -149,7 +230,9 @@
  * @returns {Promise<Object>} Reconciliation result (status MATCH | MISMATCH | ERROR).
  */
 async function reconcileInvoice(invoiceId, dbFundedTotal, options = {}) {
+
   try {
+
     const onChainAmount = await readFundedAmount(invoiceId, {
       escrowAdapter: options.escrowAdapter,
     });
@@ -163,6 +246,16 @@
         { invoiceId, dbFundedTotal, onChainAmount },
         `Escrow mismatch for invoice ${invoiceId}: DB=${dbFundedTotal}, OnChain=${onChainAmount}`,
       );
+
+      // Raise structured mismatch alerts once thresholds/channels allow.
+      // Existing counter increments remain unchanged.
+      raiseReconciliationMismatchAlert({
+        invoiceId,
+        expected: dbFundedTotal,
+        actual: onChainAmount,
+        driftMagnitude: Math.abs(dbFundedTotal - onChainAmount),
+      });
+
       escrowReconciliationMismatches.inc();
     }
 
@@ -175,6 +268,7 @@
       reconciledAt: new Date().toISOString(),
     };
   } catch (error) {
+
     logger.error(
       { invoiceId, dbFundedTotal, err: error?.message },
       `Error reconciling invoice ${invoiceId}: ${error.message}`,