diff --git a/TODO.md b/TODO.md index d0aa0db2..e64447f9 100644 --- a/TODO.md +++ b/TODO.md @@ -1,15 +1,19 @@ -# TODO - -## Invoice state machine exhaustive transition matrix tests -- [ ] Read current invoice.state.test.js and understand existing coverage gaps. -- [ ] Implement a data-driven Cartesian matrix test for every (fromState, targetState) pair across INVOICE_STATES. - - [ ] Assert isTransitionAllowed outcome matches VALID_TRANSITIONS, including silent-jump prevention (pending → linked_escrow) and self transitions. - - [ ] Assert validateTransition returns exact error codes: INVALID_TRANSITION, TERMINAL_STATE, ALREADY_IN_TARGET_STATE. - - [ ] Ensure terminal states reject all outgoing transitions. -- [ ] Add executeTransition assertions: - - [ ] For each allowed transition, ensure it writes an audit log entry (action=STATE_TRANSITION) with correct before/after states and metadata reason/transitionType. - - [ ] For each disallowed transition, ensure executeTransition throws an error with correct error.code. -- [ ] Add executeTransition/canLinkToEscrow coverage for business rule: must be approved to link. -- [x] Run test suite and confirm invoiceStateMachine.js hits 100% branch coverage (cannot be executed here due to missing `npm` in terminal). +# TODO - enhancement/reconciliation-14-mismatch-alerts +- [ ] Inspect current Sentry wiring and matcher for captureException +- [x] Implement structured mismatch alert helper in `src/jobs/reconcileEscrow.js`: + - [x] Keep existing warn log and `escrowReconciliationMismatches.inc()` untouched + - [x] Add Sentry capture (minimal, scrubbed fields only) behind Sentry enabled check + - [x] Add env configurable mismatch alert threshold + channel + +- [ ] Update `src/services/health.js` (`checkReconciliationHealth`) so `/ready` degrades only when `summary.mismatches >= RECONCILIATION_DRIFT_THRESHOLD` + +- [ ] Update tests in `tests/reconcileEscrow.test.js`: + - [ ] Cover mismatch alerts (log + Sentry capture) + - [ ] Cover Sentry disabled + - [ ] Cover threshold breach health degradation + - [ ] Cover read failure health path +- [ ] Update documentation `docs/ops-reconcile.md` with env vars + behavior +- [ ] Run `npm test` and `npm run lint` +- [ ] Ensure ≥95% test coverage for impacted modules diff --git a/src/jobs/reconcileEscrow.js b/src/jobs/reconcileEscrow.js index eb888690..bbc61140 100644 --- a/src/jobs/reconcileEscrow.js +++ b/src/jobs/reconcileEscrow.js @@ -32,6 +32,8 @@ const { } = require('../metrics'); const JobQueue = require('../workers/jobQueue'); const BackgroundWorker = require('../workers/worker'); +const sentry = require('../observability/sentry'); + /** * Reconciliation result status. @@ -74,6 +76,84 @@ function getDriftThreshold() { */ const DRIFT_THRESHOLD = getDriftThreshold(); + +/** + * Reads the mismatch alert threshold for per-invoice mismatch alerts. + + * Defaults to 1 so any mismatch triggers an alert. + * + * @returns {number} + */ +function getMismatchAlertThreshold() { + return Math.max( + 1, + parseInt(process.env.RECONCILIATION_MISMATCH_ALERT_THRESHOLD, 10) || 1, + ); +} + +/** + * Returns the configured alert channel. + * + * @returns {'log'|'sentry'|'sentry+log'} + */ +function getMismatchAlertChannel() { + const raw = process.env.RECONCILIATION_MISMATCH_ALERT_CHANNEL || 'sentry+log'; + if (raw === 'log') return 'log'; + if (raw === 'sentry') return 'sentry'; + if (raw === 'sentry+log') return 'sentry+log'; + return 'sentry+log'; +} + +/** + * Raises an alert when an escrow reconciliation mismatch is detected. + * + * Security: only non-sensitive, minimal fields are emitted. + * + * @param {object} params + * @param {string} params.invoiceId + * @param {number} params.expected - Expected funded amount (DB). + * @param {number} params.actual - Actual funded amount (on-chain). + * @param {number} params.driftMagnitude + * @returns {void} + */ +function raiseReconciliationMismatchAlert({ + invoiceId, + expected, + actual, + driftMagnitude, +}) { + const channel = getMismatchAlertChannel(); + const threshold = getMismatchAlertThreshold(); + + // Threshold determines whether escalation work is performed. + // Per-invoice helper treats each mismatch as count=1. + if (threshold > 1) return; + + if (channel === 'log' || channel === 'sentry+log') { + logger.warn( + { invoiceId, expected, actual, driftMagnitude }, + 'Escrow mismatch alert escalation', + ); + } + + if (channel === 'sentry' || channel === 'sentry+log') { + if (sentry && typeof sentry.isEnabled === 'function' && sentry.isEnabled()) { + const error = new Error('Escrow reconciliation mismatch'); + sentry.captureException(error, { + invoiceId, + expected, + actual, + driftMagnitude, + }); + } + } +} + + + + + + /** * Coerces a DB-sourced funded total into a finite number. * @@ -101,6 +181,7 @@ function toFundedTotal(value) { * @yields {{ id: string, fundedTotal: number }} One invoice per iteration. */ async function* iterateInvoicesFromDb(options = {}) { + const dbClient = options.dbClient || db; const rawSize = Number(options.pageSize) || DEFAULT_PAGE_SIZE; const pageSize = Math.min(Math.max(1, Math.trunc(rawSize)), 1000); @@ -149,7 +230,9 @@ async function* iterateInvoicesFromDb(options = {}) { * @returns {Promise} Reconciliation result (status MATCH | MISMATCH | ERROR). */ async function reconcileInvoice(invoiceId, dbFundedTotal, options = {}) { + try { + const onChainAmount = await readFundedAmount(invoiceId, { escrowAdapter: options.escrowAdapter, }); @@ -163,6 +246,16 @@ async function reconcileInvoice(invoiceId, dbFundedTotal, options = {}) { { invoiceId, dbFundedTotal, onChainAmount }, `Escrow mismatch for invoice ${invoiceId}: DB=${dbFundedTotal}, OnChain=${onChainAmount}`, ); + + // Raise structured mismatch alerts once thresholds/channels allow. + // Existing counter increments remain unchanged. + raiseReconciliationMismatchAlert({ + invoiceId, + expected: dbFundedTotal, + actual: onChainAmount, + driftMagnitude: Math.abs(dbFundedTotal - onChainAmount), + }); + escrowReconciliationMismatches.inc(); } @@ -175,6 +268,7 @@ async function reconcileInvoice(invoiceId, dbFundedTotal, options = {}) { reconciledAt: new Date().toISOString(), }; } catch (error) { + logger.error( { invoiceId, dbFundedTotal, err: error?.message }, `Error reconciling invoice ${invoiceId}: ${error.message}`,