diff --git a/cloud-functions/src/index.ts b/cloud-functions/src/index.ts index b7c22701..ca21f07a 100644 --- a/cloud-functions/src/index.ts +++ b/cloud-functions/src/index.ts @@ -23,6 +23,7 @@ export * from './auditLog'; export * from './attendanceStreak'; export * from './permanentCleanup'; export * from './volunteer'; +export * from './retryDeadLetterEmail'; export * from './clubReputation'; export * from './onEventDelete'; export * from './events'; diff --git a/cloud-functions/src/reminders.test.ts b/cloud-functions/src/reminders.test.ts new file mode 100644 index 00000000..c66857d7 --- /dev/null +++ b/cloud-functions/src/reminders.test.ts @@ -0,0 +1,181 @@ +import * as admin from 'firebase-admin'; + +process.env.GCLOUD_PROJECT = 'demo-test'; +process.env.FIREBASE_CONFIG = '{"projectId":"demo-test"}'; + +jest.mock('firebase-admin', () => { + const updateMock = jest.fn(); + const setMock = jest.fn(); + const queryGetMock = jest.fn(); + const docGetMock = jest.fn(); + const whereMock = jest.fn(() => ({ + where: whereMock, + get: queryGetMock, + })); + + const batchMock = { + set: setMock, + update: updateMock, + commit: jest.fn().mockResolvedValue(undefined), + }; + + const docRefMock = { + get: docGetMock, + collection: jest.fn(() => collectionRefMock), + update: jest.fn(), + }; + + const collectionRefMock = { + doc: jest.fn(() => docRefMock), + where: whereMock, + get: queryGetMock, + }; + + const firestoreInstance = { + collection: jest.fn(() => collectionRefMock), + batch: jest.fn(() => batchMock), + FieldValue: { + serverTimestamp: jest.fn(() => 'SERVER_TS'), + }, + }; + + return { + apps: [], + initializeApp: jest.fn(), + firestore: jest.fn(() => firestoreInstance), + }; +}); + +jest.mock('firebase-admin/firestore', () => ({ + FieldValue: { + serverTimestamp: jest.fn(() => 'SERVER_TS'), + }, + Timestamp: { + now: jest.fn(() => ({ toDate: () => new Date() })), + }, +})); + +jest.mock('expo-server-sdk', () => ({ + Expo: { isExpoPushToken: jest.fn(() => false) }, +})); + +jest.mock('./utils/push', () => ({ + sendPushNotifications: jest.fn().mockResolvedValue(undefined), +})); + +jest.mock('./utils/emailSender', () => ({ + sendEmailWithRetry: jest.fn().mockResolvedValue({ success: true }), +})); + +jest.mock('./lib/participants', () => ({ + getParticipantContacts: jest.fn(), +})); + +const { sendEmailWithRetry } = require('./utils/emailSender'); +const { getParticipantContacts } = require('./lib/participants'); +const { processDueReminders } = require('./reminders'); +const adminMock = require('firebase-admin'); + +const remindersQueryGet = () => adminMock.firestore().collection('reminders').where().get; + +const eventsDocGet = () => adminMock.firestore().collection('events').doc().get; + +beforeEach(() => { + jest.clearAllMocks(); + eventsDocGet().mockResolvedValue(makeEvent()); +}); + +const makeReminder = data => ({ + id: 'rem-1', + data: jest.fn(() => data), + ref: { id: 'rem-1' }, +}); + +const makeEvent = (overrides = {}) => ({ + exists: true, + data: () => ({ + title: 'Hack Night', + startAt: '2026-09-01T10:00:00.000Z', + location: 'Auditorium', + eventMode: 'offline', + ...overrides, + }), +}); + +beforeEach(() => { + jest.clearAllMocks(); +}); + +describe('processDueReminders — email reminders (#671)', () => { + it('sends an email reminder to every participant email', async () => { + getParticipantContacts.mockResolvedValue([ + { id: 'p1', email: 'alice@example.com' }, + { id: 'p2', email: 'bob@example.com' }, + ]); + const reminder = makeReminder({ userId: 'u1', eventId: 'evt1', remindAt: {} }); + remindersQueryGet().mockResolvedValue({ empty: false, docs: [reminder], size: 1 }); + eventsDocGet().mockResolvedValue(makeEvent()); + + await processDueReminders(adminMock.firestore()); + + expect(sendEmailWithRetry).toHaveBeenCalledTimes(2); + expect(sendEmailWithRetry).toHaveBeenCalledWith( + expect.objectContaining({ + to: 'alice@example.com', + subject: expect.stringContaining('Hack Night'), + templateName: 'universal_email_template', + }), + expect.objectContaining({ eventId: 'evt1', attempts: 3 }), + ); + const updateCall = adminMock.firestore().batch().update.mock.calls[0]; + expect(updateCall[1]).toMatchObject({ sent: true, emailed: true, emailCount: 2 }); + }); + + it('does not resend when the reminder was already emailed', async () => { + const reminder = makeReminder({ + userId: 'u1', + eventId: 'evt1', + remindAt: {}, + emailed: true, + }); + remindersQueryGet().mockResolvedValue({ empty: false, docs: [reminder], size: 1 }); + + await processDueReminders(adminMock.firestore()); + + expect(sendEmailWithRetry).not.toHaveBeenCalled(); + const updateCall = adminMock.firestore().batch().update.mock.calls[0]; + expect(updateCall[1]).toEqual({ sent: true }); + }); + + it('skips emails when the event no longer exists', async () => { + getParticipantContacts.mockResolvedValue([{ id: 'p1', email: 'a@example.com' }]); + const reminder = makeReminder({ userId: 'u1', eventId: 'gone', remindAt: {} }); + remindersQueryGet().mockResolvedValue({ empty: false, docs: [reminder], size: 1 }); + eventsDocGet().mockResolvedValue({ exists: false }); + + await processDueReminders(adminMock.firestore()); + + expect(sendEmailWithRetry).not.toHaveBeenCalled(); + }); + + it('filters malformed emails and computes the email count correctly', async () => { + getParticipantContacts.mockResolvedValue([ + { id: 'p1', email: 'ok@example.com' }, + { id: 'p2', email: 'not-an-email' }, + { id: 'p3' }, + ]); + const reminder = makeReminder({ userId: 'u1', eventId: 'evt1', remindAt: {} }); + remindersQueryGet().mockResolvedValue({ empty: false, docs: [reminder], size: 1 }); + eventsDocGet().mockResolvedValue(makeEvent()); + + await processDueReminders(adminMock.firestore()); + + expect(sendEmailWithRetry).toHaveBeenCalledTimes(1); + expect(sendEmailWithRetry).toHaveBeenCalledWith( + expect.objectContaining({ to: 'ok@example.com' }), + expect.anything(), + ); + const updateCall = adminMock.firestore().batch().update.mock.calls[0]; + expect(updateCall[1]).toMatchObject({ emailed: true, emailCount: 1 }); + }); +}); diff --git a/cloud-functions/src/reminders.ts b/cloud-functions/src/reminders.ts index 27b1069e..0865f655 100644 --- a/cloud-functions/src/reminders.ts +++ b/cloud-functions/src/reminders.ts @@ -1,15 +1,109 @@ import * as admin from 'firebase-admin'; import * as functions from 'firebase-functions'; import { FieldValue, Timestamp } from 'firebase-admin/firestore'; +import * as ExpoSdk from 'expo-server-sdk'; import { sendPushNotifications } from './utils/push'; -const { Expo } = require('expo-server-sdk'); +import { sendEmailWithRetry } from './utils/emailSender'; +import { getParticipantContacts } from './lib/participants'; + +const { Expo } = ExpoSdk; + +const MAX_EMAILS_PER_RUN = 100; +const MAX_PARTICIPANTS_PER_EVENT = 100; + +const WEBSITE_BASE_URL = process.env.WEBSITE_URL || 'https://unievent.web.app'; + +function formatEventDate(startAt: string | undefined): string { + if (!startAt) return 'soon'; + const date = new Date(startAt); + if (Number.isNaN(date.getTime())) return 'soon'; + return date.toLocaleString(undefined, { + weekday: 'short', + month: 'short', + day: 'numeric', + hour: '2-digit', + minute: '2-digit', + }); +} /** - * Scheduled function to check for reminders. - * Runs every minute. + * Sends email reminders for a due reminder to the event's registered + * participants (#671). Failures flow through the email retry/circuit- + * breaker/DLQ pipeline so nothing is silently lost. */ -export const checkReminders = functions.pubsub.schedule('every 1 minutes').onRun(async context => { - const db = admin.firestore(); +async function sendReminderEmails( + db: admin.firestore.Firestore, + reminder: admin.firestore.DocumentSnapshot, + eventCache: Map, + emailBudget: { remaining: number }, +): Promise<{ sent: number; attempted: boolean }> { + const data = reminder.data() || {}; + const eventId = data.eventId as string | undefined; + + if (!eventId) { + return { sent: 0, attempted: false }; + } + if (data.emailed === true || emailBudget.remaining <= 0) { + return { sent: 0, attempted: false }; + } + + if (!eventCache.has(eventId)) { + const eventSnap = await db.collection('events').doc(eventId).get(); + eventCache.set(eventId, eventSnap.exists ? (eventSnap.data() ?? null) : null); + } + const event = eventCache.get(eventId) ?? null; + if (!event) { + return { sent: 0, attempted: false }; + } + + const title = event.title || 'Upcoming event'; + const startAt = formatEventDate(event.startAt); + const location = + event.eventMode === 'online' ? event.meetLink || 'online' : event.location || 'TBD'; + + const participants = await getParticipantContacts(db, eventId); + const emails = [ + ...new Set( + participants + .slice(0, MAX_PARTICIPANTS_PER_EVENT) + .map(p => (typeof p.email === 'string' ? p.email.trim() : '')) + .filter(email => email.length > 0 && email.includes('@')), + ), + ].slice(0, emailBudget.remaining); + + for (const email of emails) { + emailBudget.remaining -= 1; + await sendEmailWithRetry( + { + to: email, + subject: `⏰ Reminder: ${title} starts ${startAt}`, + templateName: 'universal_email_template', + templateData: { + subject: `${title} starts ${startAt}`, + to_name: '', + message: `Don't forget — ${title} is starting ${startAt}.\n\nWhere: ${location}\n\nYou're registered, so we'll see you there!`, + event_title: title, + date: startAt, + event_link: `${WEBSITE_BASE_URL}/event/${eventId}`, + cert_display: 'none', + download_btn_display: 'none', + browse_btn_display: 'block', + }, + }, + { eventId, attempts: 3 }, + ); + } + + return { sent: emails.length, attempted: true }; +} + +/** + * Processes all due reminders: in-app + push notifications plus email + * reminders to registered participants (#671). + * + * Extracted from the scheduler so it can be unit-tested directly. + */ +export async function processDueReminders(db: admin.firestore.Firestore): Promise { const now = Timestamp.now(); // Find reminders that need to be sent (remindAt <= now) and haven't been sent yet @@ -19,11 +113,13 @@ export const checkReminders = functions.pubsub.schedule('every 1 minutes').onRun const snapshot = await q.get(); if (snapshot.empty) { - return null; + return 0; } const batch = db.batch(); const messages = []; + const eventCache = new Map(); + const emailBudget = { remaining: MAX_EMAILS_PER_RUN }; // We need to fetch user tokens // To handle many reminders, we might need efficient querying, but loop is fine for now @@ -58,8 +154,20 @@ export const checkReminders = functions.pubsub.schedule('every 1 minutes').onRun } } - // 3. Mark reminder as sent - batch.update(docSnapshot.ref, { sent: true }); + // 3. Email reminder to registered participants (#671) + const emailResult = await sendReminderEmails(db, docSnapshot, eventCache, emailBudget); + + // 4. Mark reminder as sent + batch.update(docSnapshot.ref, { + sent: true, + ...(emailResult.attempted + ? { + emailed: true, + emailedAt: FieldValue.serverTimestamp(), + emailCount: emailResult.sent, + } + : {}), + }); } // Send Pushes @@ -69,5 +177,13 @@ export const checkReminders = functions.pubsub.schedule('every 1 minutes').onRun await batch.commit(); console.log(`Processed ${snapshot.size} reminders.`); - return null; -}); + return snapshot.size; +} + +/** + * Scheduled function to check for reminders. + * Runs every minute. + */ +export const checkReminders = functions.pubsub + .schedule('every 1 minutes') + .onRun(async () => processDueReminders(admin.firestore())); diff --git a/cloud-functions/src/retryDeadLetterEmail.test.ts b/cloud-functions/src/retryDeadLetterEmail.test.ts new file mode 100644 index 00000000..63fc6608 --- /dev/null +++ b/cloud-functions/src/retryDeadLetterEmail.test.ts @@ -0,0 +1,100 @@ +const functionsTest = require('firebase-functions-test'); + +jest.mock('firebase-admin', () => { + const getMock = jest.fn(); + const updateMock = jest.fn(); + const collectionMock = jest.fn(() => ({ + doc: jest.fn(() => ({ + get: getMock, + update: updateMock, + })), + })); + return { + apps: [], + initializeApp: jest.fn(), + firestore: jest.fn(() => ({ collection: collectionMock })), + }; +}); + +jest.mock('./utils/emailSender', () => { + const sendEmailWithRetry = jest.fn(); + return { + sendEmail: jest.fn(async () => ({ success: true })), + sendEmailWithRetry, + }; +}); + +jest.mock('./utils/emailResilience', () => ({ + emailCircuitBreaker: { + isOpen: jest.fn(() => false), + }, + DEFAULT_RETRY_DELAYS_MS: [1000, 2000, 4000, 8000, 16000], +})); + +import { retryDeadLetterEmail } from './retryDeadLetterEmail'; +import { sendEmailWithRetry } from './utils/emailSender'; + +const testEnv = functionsTest(); +const wrapped = testEnv.wrap(retryDeadLetterEmail as any); + +describe('retryDeadLetterEmail', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('rejects unauthenticated calls', async () => { + await expect(wrapped({ entryId: 'x' }, { auth: null } as any)).rejects.toThrow( + 'You must be signed in.', + ); + }); + + it('rejects students', async () => { + await expect( + wrapped({ entryId: 'x' }, { auth: { uid: 's1', token: {} } } as any), + ).rejects.toThrow('Only admins or clubs can retry dead-letter emails.'); + }); + + it('rejects a missing entryId', async () => { + await expect( + wrapped({}, { auth: { uid: 'a1', token: { admin: true } } } as any), + ).rejects.toThrow(/entryId/); + }); + + it('re-sends a queued entry and marks it delivered', async () => { + const admin = require('firebase-admin'); + admin + .firestore() + .collection('email_dead_letter_queue') // init chain + .doc('dlq-1') + .get.mockResolvedValue({ + exists: true, + data: () => ({ + to: 'user@example.com', + subject: 'Hello', + templateId: 'universal_email_template', + templateData: {}, + provider: 'resend', + retryCount: 1, + status: 'queued', + }), + }); + admin + .firestore() + .collection('email_dead_letter_queue') + .doc('dlq-1') + .update.mockResolvedValue({}); + (sendEmailWithRetry as unknown as jest.Mock).mockResolvedValue({ + success: true, + messageId: 'msg-retry', + }); + + const result = await wrapped({ entryId: 'dlq-1' }, { + auth: { uid: 'a1', token: { admin: true } }, + } as any); + + expect(result).toEqual({ success: true, entryId: 'dlq-1', retryCount: 2 }); + expect(sendEmailWithRetry).toHaveBeenCalledWith( + expect.objectContaining({ to: 'user@example.com' }), + ); + }); +}); diff --git a/cloud-functions/src/retryDeadLetterEmail.ts b/cloud-functions/src/retryDeadLetterEmail.ts new file mode 100644 index 00000000..3ddad1c7 --- /dev/null +++ b/cloud-functions/src/retryDeadLetterEmail.ts @@ -0,0 +1,91 @@ +import * as admin from 'firebase-admin'; +import * as functions from 'firebase-functions'; +import { logger } from './logger'; +import { sendEmailWithRetry } from './utils/emailSender'; +import { emailCircuitBreaker } from './utils/emailResilience'; + +/** + * Manual retry of a failed email from the dead-letter queue (#326). + * Callable by admins/clubs; re-sends the queued email and updates the + * queue entry's status/retryCount. + */ +export const retryDeadLetterEmail = functions.https.onCall(async (data, context) => { + if (!context.auth) { + throw new functions.https.HttpsError('unauthenticated', 'You must be signed in.'); + } + const token = context.auth.token; + if (!token.admin && !token.club) { + throw new functions.https.HttpsError( + 'permission-denied', + 'Only admins or clubs can retry dead-letter emails.', + ); + } + + const { entryId } = data ?? {}; + if (!entryId || typeof entryId !== 'string') { + throw new functions.https.HttpsError('invalid-argument', 'entryId is required.'); + } + + const db = admin.firestore(); + const entryRef = db.collection('email_dead_letter_queue').doc(entryId); + const snap = await entryRef.get(); + if (!snap.exists) { + throw new functions.https.HttpsError('not-found', 'Dead-letter entry not found.'); + } + + const entry = snap.data() ?? {}; + if (entry.status === 'delivered') { + return { success: true, alreadyDelivered: true, entryId }; + } + + if (emailCircuitBreaker.isOpen()) { + throw new functions.https.HttpsError( + 'unavailable', + 'Email circuit breaker is open; try again later.', + ); + } + + if (entry.provider !== 'resend') { + throw new functions.https.HttpsError( + 'failed-precondition', + `Manual retry is only supported for resend entries (got ${entry.provider}).`, + ); + } + + const result = await sendEmailWithRetry({ + to: entry.to, + subject: entry.subject ?? 'Retried: Previous email failed to send', + templateName: entry.templateId ?? 'universal_email_template', + templateData: entry.templateData ?? {}, + }); + + const retryCount = (entry.retryCount ?? 0) + 1; + if (result.success) { + await entryRef.update({ + status: 'delivered', + retryCount, + deliveredAt: new Date().toISOString(), + lastAttemptAt: new Date().toISOString(), + }); + logger.info({ + message: 'dead-letter email delivered on retry', + entryId, + to: entry.to, + }); + return { success: true, entryId, retryCount }; + } + + await entryRef.update({ + status: 'queued', + retryCount, + lastAttemptAt: new Date().toISOString(), + lastError: result.error ?? 'Unknown error', + }); + logger.error({ + message: 'dead-letter email retry failed', + entryId, + to: entry.to, + error: result.error, + }); + return { success: false, entryId, error: result.error }; +}); diff --git a/cloud-functions/src/sendBulkEmails.ts b/cloud-functions/src/sendBulkEmails.ts index 40fe2319..84598623 100644 --- a/cloud-functions/src/sendBulkEmails.ts +++ b/cloud-functions/src/sendBulkEmails.ts @@ -1,6 +1,14 @@ import * as admin from 'firebase-admin'; import * as functions from 'firebase-functions'; import { FieldValue } from 'firebase-admin/firestore'; +import { logger } from './logger'; +import { + emailCircuitBreaker, + retryWithExponentialBackoff, + enqueueDeadLetter, + alertAdmins, + type DeadLetterEntry, +} from './utils/emailResilience'; // Interface for Email Participant interface Participant { @@ -150,30 +158,72 @@ export const sendBulkEmails = functions.https.onCall( }; try { - const response = await fetch( - 'https://api.emailjs.com/api/v1.0/email/send', - { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(payload), - signal: controller.signal, + const response = await retryWithExponentialBackoff( + async () => { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 10000); + try { + const res = await fetch( + 'https://api.emailjs.com/api/v1.0/email/send', + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(payload), + signal: controller.signal, + }, + ); + if (!res.ok) { + throw new Error( + `EmailJS rejected send: ${res.status} ${await res + .text() + .catch(() => '')}`, + ); + } + return res; + } finally { + clearTimeout(timeout); + } }, + { label: `bulk email to ${p.email}` }, ); if (response.ok) { + emailCircuitBreaker.recordSuccess(); successCount++; } else { - const errorText = await response.text(); - console.error('EmailJS Error:', errorText); failureCount++; } } catch (error) { - console.error('Email Network Error:', error); + const reason = error instanceof Error ? error.message : String(error); + logger.error({ + message: 'bulk email failed after retries', + to: p.email, + templateId, + reason, + }); + + emailCircuitBreaker.recordFailure(); + if (emailCircuitBreaker.isOpen()) { + await alertAdmins(admin.firestore(), { + title: 'Bulk email circuit breaker tripped', + message: + 'The email circuit breaker opened after consecutive EmailJS failures during a bulk send.', + context: { templateId, senderId: uid }, + }); + } + + const deadLetter: DeadLetterEntry = { + to: p.email, + provider: 'emailjs', + subject, + templateId, + reason, + attempts: 6, + }; + await enqueueDeadLetter(admin.firestore(), deadLetter); failureCount++; - } finally { - clearTimeout(timeout); } }), ); diff --git a/cloud-functions/src/utils/emailResilience.test.ts b/cloud-functions/src/utils/emailResilience.test.ts new file mode 100644 index 00000000..32e20dcb --- /dev/null +++ b/cloud-functions/src/utils/emailResilience.test.ts @@ -0,0 +1,122 @@ +import { + EmailCircuitBreaker, + BreakerState, + retryWithExponentialBackoff, + enqueueDeadLetter, +} from './emailResilience'; + +jest.useFakeTimers(); + +const flushTimers = async () => { + await jest.runAllTimersAsync(); +}; + +describe('EmailCircuitBreaker', () => { + beforeEach(() => { + jest.useFakeTimers(); + jest.setSystemTime(new Date('2026-08-12T00:00:00Z')); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('opens after the failure threshold is reached', () => { + const breaker = new EmailCircuitBreaker({ failureThreshold: 5, cooldownMs: 60000 }); + expect(breaker.state).toBe(BreakerState.CLOSED); + for (let i = 0; i < 5; i += 1) breaker.recordFailure(); + expect(breaker.state).toBe(BreakerState.OPEN); + expect(breaker.isOpen()).toBe(true); + }); + + it('closes again after a success', () => { + const breaker = new EmailCircuitBreaker({ failureThreshold: 2, cooldownMs: 60000 }); + breaker.recordFailure(); + breaker.recordFailure(); + expect(breaker.isOpen()).toBe(true); + breaker.recordSuccess(); + expect(breaker.isOpen()).toBe(false); + expect(breaker.consecutiveFailures).toBe(0); + }); + + it('allows a trial request after the cooldown elapses (half-open)', () => { + const breaker = new EmailCircuitBreaker({ failureThreshold: 1, cooldownMs: 60000 }); + breaker.recordFailure(); + expect(breaker.isOpen()).toBe(true); + expect(breaker.tryReset()).toBe(false); + jest.setSystemTime(new Date('2026-08-12T00:01:01Z')); + expect(breaker.tryReset()).toBe(true); + expect(breaker.state).toBe(BreakerState.HALF_OPEN); + }); +}); + +describe('retryWithExponentialBackoff', () => { + it('succeeds on the first attempt', async () => { + const fn = jest.fn().mockResolvedValue('ok'); + const result = await retryWithExponentialBackoff(fn, { delaysMs: [1, 2] }); + expect(result).toBe('ok'); + expect(fn).toHaveBeenCalledTimes(1); + }); + + it('retries with exponential backoff and succeeds', async () => { + const fn = jest + .fn() + .mockRejectedValueOnce(new Error('boom 1')) + .mockRejectedValueOnce(new Error('boom 2')) + .mockResolvedValue('recovered'); + const onRetry = jest.fn(); + + const promise = retryWithExponentialBackoff(fn, { + delaysMs: [1000, 2000, 4000], + onRetry, + }); + await jest.runOnlyPendingTimersAsync(); + const result = await promise; + + expect(result).toBe('recovered'); + expect(fn).toHaveBeenCalledTimes(3); + expect(onRetry).toHaveBeenNthCalledWith(1, 1, 1000, expect.any(Error)); + expect(onRetry).toHaveBeenNthCalledWith(2, 2, 2000, expect.any(Error)); + }); + + it('throws the last error after exhausting attempts', async () => { + const fn = jest.fn().mockRejectedValue(new Error('always fails')); + const promise = retryWithExponentialBackoff(fn, { delaysMs: [1, 2, 3] }); + await jest.runAllTimersAsync(); + await expect(promise).rejects.toThrow('always fails'); + expect(fn).toHaveBeenCalledTimes(4); + }); +}); + +describe('enqueueDeadLetter', () => { + it('writes the entry with queued status and context', async () => { + const add = jest.fn().mockResolvedValue({ id: 'dlq-1' }); + const db = { + collection: jest.fn(() => ({ add })), + } as unknown as FirebaseFirestore.Firestore; + + const ref = await enqueueDeadLetter(db, { + to: 'user@example.com', + provider: 'resend', + subject: 'Test', + templateId: 'universal_email_template', + eventId: 'event-123', + reason: 'RESEND_500', + attempts: 3, + }); + + expect(ref).toEqual({ id: 'dlq-1' }); + expect(db.collection).toHaveBeenCalledWith('email_dead_letter_queue'); + expect(add).toHaveBeenCalledWith( + expect.objectContaining({ + to: 'user@example.com', + provider: 'resend', + eventId: 'event-123', + reason: 'RESEND_500', + attempts: 3, + status: 'queued', + retryCount: 0, + }), + ); + }); +}); diff --git a/cloud-functions/src/utils/emailResilience.ts b/cloud-functions/src/utils/emailResilience.ts new file mode 100644 index 00000000..3eb02362 --- /dev/null +++ b/cloud-functions/src/utils/emailResilience.ts @@ -0,0 +1,172 @@ +import { logger } from '../logger'; + +/** + * Email resilience helpers (#326): exponential-backoff retries, a circuit + * breaker that trips after consecutive failures, contextual failure logging + * and a Firestore dead-letter queue for manual re-delivery. + */ + +export const DEFAULT_RETRY_DELAYS_MS = [1000, 2000, 4000, 8000, 16000]; + +export enum BreakerState { + CLOSED = 'CLOSED', + OPEN = 'OPEN', + HALF_OPEN = 'HALF_OPEN', +} + +export class EmailCircuitBreaker { + failureThreshold: number; + cooldownMs: number; + consecutiveFailures = 0; + state: BreakerState = BreakerState.CLOSED; + openedAt: number | null = null; + + constructor(opts: { failureThreshold?: number; cooldownMs?: number } = {}) { + this.failureThreshold = opts.failureThreshold ?? 5; + this.cooldownMs = opts.cooldownMs ?? 60_000; + } + + recordSuccess() { + this.consecutiveFailures = 0; + this.state = BreakerState.CLOSED; + this.openedAt = null; + } + + recordFailure() { + this.consecutiveFailures += 1; + if (this.consecutiveFailures >= this.failureThreshold) { + this.state = BreakerState.OPEN; + this.openedAt = Date.now(); + } + } + + /** Allows a single trial request after the cooldown has elapsed. */ + tryReset(): boolean { + if (this.state !== BreakerState.OPEN || this.openedAt === null) return false; + if (Date.now() - this.openedAt >= this.cooldownMs) { + this.state = BreakerState.HALF_OPEN; + return true; + } + return false; + } + + isOpen(): boolean { + return this.state === BreakerState.OPEN; + } +} + +export interface RetryOptions { + maxAttempts?: number; + delaysMs?: number[]; + label?: string; + onRetry?: (attempt: number, delayMs: number, error: unknown) => void; +} + +/** + * Runs `fn` with exponential backoff (1s, 2s, 4s, 8s, 16s by default). + * Only errors are retried; successful results are returned immediately. + */ +export async function retryWithExponentialBackoff( + fn: () => Promise, + options: RetryOptions = {}, +): Promise { + const delaysMs = options.delaysMs ?? DEFAULT_RETRY_DELAYS_MS; + const maxAttempts = options.maxAttempts ?? delaysMs.length + 1; + const label = options.label ?? 'email send'; + + let lastError: unknown; + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + try { + return await fn(); + } catch (error) { + lastError = error; + if (attempt >= maxAttempts) break; + const delayMs = delaysMs[attempt - 1] ?? delaysMs[delaysMs.length - 1]; + logger.warn({ + message: `retry scheduled for ${label}`, + attempt, + nextDelayMs: delayMs, + error: error instanceof Error ? error.message : String(error), + }); + options.onRetry?.(attempt, delayMs, error); + await new Promise(resolve => setTimeout(resolve, delayMs)); + } + } + throw lastError; +} + +export interface DeadLetterEntry { + to: string; + provider: 'resend' | 'emailjs'; + subject?: string; + templateId?: string; + eventId?: string; + reason: string; + attempts: number; +} + +/** + * Persists a failed email to the dead-letter queue for manual retry. + * Returns the created document reference (or null on failure). + */ +export async function enqueueDeadLetter( + db: FirebaseFirestore.Firestore, + entry: DeadLetterEntry, +): Promise { + try { + const ref = await db.collection('email_dead_letter_queue').add({ + ...entry, + status: 'queued', + retryCount: 0, + createdAt: new Date().toISOString(), + }); + logger.error({ + message: 'email queued to dead-letter queue', + to: entry.to, + provider: entry.provider, + eventId: entry.eventId ?? null, + reason: entry.reason, + }); + return ref; + } catch (error) { + logger.error({ + message: 'failed to enqueue dead-letter entry', + error: error instanceof Error ? error.message : String(error), + }); + return null; + } +} + +export interface AdminAlert { + title: string; + message: string; + context?: Record; +} + +/** + * Records an admin alert (circuit breaker trips, systemic failures) and + * logs it loudly for debugging. + */ +export async function alertAdmins( + db: FirebaseFirestore.Firestore, + alert: AdminAlert, +): Promise { + logger.error({ + message: `ADMIN ALERT: ${alert.title}`, + detail: alert.message, + context: alert.context, + }); + try { + await db.collection('admin_alerts').add({ + ...alert, + createdAt: new Date().toISOString(), + }); + } catch (error) { + logger.error({ + message: 'failed to persist admin alert', + error: error instanceof Error ? error.message : String(error), + }); + } +} + +export const emailCircuitBreaker = new EmailCircuitBreaker(); diff --git a/cloud-functions/src/utils/emailSender.ts b/cloud-functions/src/utils/emailSender.ts index 44562bcf..781d2ce6 100644 --- a/cloud-functions/src/utils/emailSender.ts +++ b/cloud-functions/src/utils/emailSender.ts @@ -1,5 +1,14 @@ import { renderTemplate } from './emailTemplateRenderer'; import { Resend } from 'resend'; +import * as admin from 'firebase-admin'; +import { logger } from '../logger'; +import { + emailCircuitBreaker, + retryWithExponentialBackoff, + enqueueDeadLetter, + alertAdmins, + type DeadLetterEntry, +} from './emailResilience'; /** * Options for sending (or dry-running) an email. @@ -98,3 +107,64 @@ export async function sendEmail(options: SendEmailOptions): Promise { + if (emailCircuitBreaker.isOpen()) { + logger.error({ + message: 'email circuit breaker is OPEN; skipping send', + to: options.to, + eventId: retryContext.eventId ?? null, + }); + return { success: false, error: 'Email circuit breaker is open. Please retry later.' }; + } + + const finalResult = await retryWithExponentialBackoff( + async () => { + const result = await sendEmail(options); + if (!result.success) { + throw new Error(result.error ?? 'Unknown email send failure'); + } + return result; + }, + { label: `email to ${options.to}` }, + ).catch((error: unknown) => { + const reason = error instanceof Error ? error.message : String(error); + return { success: false, error: reason }; + }); + + if (finalResult.success) { + emailCircuitBreaker.recordSuccess(); + return finalResult; + } + + emailCircuitBreaker.recordFailure(); + if (emailCircuitBreaker.isOpen()) { + const db = admin.firestore(); + await alertAdmins(db, { + title: 'Email circuit breaker tripped', + message: `The email circuit breaker opened after ${emailCircuitBreaker.failureThreshold} consecutive failures. Check the email provider (Resend) configuration.`, + context: { to: options.to, eventId: retryContext.eventId ?? null }, + }); + } + + const deadLetter: DeadLetterEntry = { + to: options.to, + provider: 'resend', + subject: options.subject, + templateId: options.templateName, + eventId: retryContext.eventId, + reason: finalResult.error ?? 'Unknown email send failure', + attempts: retryContext.attempts ?? 1, + }; + await enqueueDeadLetter(admin.firestore(), deadLetter); + + return finalResult; +} diff --git a/cloud-functions/src/utils/sendEmailWithRetry.test.ts b/cloud-functions/src/utils/sendEmailWithRetry.test.ts new file mode 100644 index 00000000..8a990996 --- /dev/null +++ b/cloud-functions/src/utils/sendEmailWithRetry.test.ts @@ -0,0 +1,96 @@ +import { sendEmail, sendEmailWithRetry } from './emailSender'; + +const mockResendSend = jest.fn(); +jest.mock('resend', () => ({ + Resend: jest.fn().mockImplementation(() => ({ + emails: { + send: mockResendSend, + }, + })), +})); + +jest.mock('./emailTemplateRenderer', () => ({ + renderTemplate: jest.fn(() => '

Hello

'), +})); + +jest.mock('firebase-admin', () => ({ + firestore: jest.fn(() => ({ + collection: jest.fn(() => ({ + add: jest.fn().mockResolvedValue({ id: 'dlq-1' }), + })), + })), +})); + +import { emailCircuitBreaker } from './emailResilience'; + +const baseOptions = { + to: 'user@example.com', + subject: 'Test', + templateName: 'universal_email_template', + templateData: { name: 'World' }, +}; + +describe('sendEmailWithRetry', () => { + const originalEnv = process.env; + + beforeEach(() => { + jest.clearAllMocks(); + jest.useFakeTimers(); + process.env = { ...originalEnv, RESEND_API_KEY: 'test-key' }; + emailCircuitBreaker.consecutiveFailures = 0; + emailCircuitBreaker.state = 'CLOSED' as never; + emailCircuitBreaker.openedAt = null; + mockResendSend.mockResolvedValue({ data: { id: 'msg-1' }, error: null }); + }); + + afterEach(() => { + jest.useRealTimers(); + process.env = originalEnv; + }); + + it('succeeds on the first attempt', async () => { + const result = await sendEmailWithRetry(baseOptions); + expect(result.success).toBe(true); + expect(mockResendSend).toHaveBeenCalledTimes(1); + }); + + it('retries transient failures and succeeds', async () => { + mockResendSend + .mockRejectedValueOnce(new Error('network down')) + .mockResolvedValueOnce({ data: { id: 'msg-2' }, error: null }); + + const promise = sendEmailWithRetry(baseOptions); + await jest.runOnlyPendingTimersAsync(); + const result = await promise; + + expect(result.success).toBe(true); + expect(mockResendSend).toHaveBeenCalledTimes(2); + }); + + it('queues the email to the dead-letter queue after exhausting retries', async () => { + mockResendSend.mockResolvedValue({ data: null, error: { message: 'RESEND_500' } }); + + const promise = sendEmailWithRetry(baseOptions, { eventId: 'event-1' }); + await jest.runAllTimersAsync(); + const result = await promise; + + expect(result.success).toBe(false); + expect(result.error).toContain('RESEND_500'); + expect(mockResendSend).toHaveBeenCalledTimes(6); + expect(emailCircuitBreaker.consecutiveFailures).toBeGreaterThan(0); + }); + + it('short-circuits while the circuit breaker is open', async () => { + emailCircuitBreaker.recordFailure(); + emailCircuitBreaker.recordFailure(); + emailCircuitBreaker.recordFailure(); + emailCircuitBreaker.recordFailure(); + emailCircuitBreaker.recordFailure(); + expect(emailCircuitBreaker.isOpen()).toBe(true); + + const result = await sendEmailWithRetry(baseOptions); + expect(result.success).toBe(false); + expect(result.error).toContain('circuit breaker is open'); + expect(mockResendSend).not.toHaveBeenCalled(); + }); +});