From 4f30f393f9a573b14763b079d4c87ddb943367ab Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Tue, 13 Oct 2026 08:48:23 +0200 Subject: [PATCH 01/23] fix: Run the Erasure Cascade in Admin User Deletion --- api/server/routes/admin/users.js | 48 ++++++++++++- packages/api/src/admin/users.spec.ts | 103 ++++++++++++++++++++++++--- packages/api/src/admin/users.ts | 79 ++++++++++++++++---- packages/api/src/user/generations.ts | 53 ++++++++++++++ packages/api/src/user/index.ts | 1 + 5 files changed, 259 insertions(+), 25 deletions(-) create mode 100644 packages/api/src/user/generations.ts diff --git a/api/server/routes/admin/users.js b/api/server/routes/admin/users.js index 2823abd9f5a..6c980783106 100644 --- a/api/server/routes/admin/users.js +++ b/api/server/routes/admin/users.js @@ -1,6 +1,12 @@ const express = require('express'); const mongoose = require('mongoose'); -const { createAdminUsersHandlers, revokeUserCodeEnvironmentWorkers } = require('@librechat/api'); +const { + GenerationJobManager, + openCheckpointDeletion, + deleteAllSharedLinksWithCleanup, + createAdminUsersHandlers, + revokeUserCodeEnvironmentWorkers, +} = require('@librechat/api'); const { SystemCapabilities } = require('@librechat/data-schemas'); const { requireCapability } = require('~/server/middleware/roles/capabilities'); const { requireJwtAuth } = require('~/server/middleware'); @@ -10,11 +16,44 @@ const { cancelAgentTriggerUserPurge, purgeAgentTriggerDeliveriesForUser, } = require('~/server/services/Agents/triggers'); +const { processDeleteRequest } = require('~/server/services/Files/process'); +const { invalidateCachedTools } = require('~/server/services/Config/getCachedTools'); +const { getMCPManager } = require('~/config'); +const subagentThreadTaskStore = require('~/server/services/Endpoints/agents/subagentThreadStore'); +const { + quiesceUserSchedules, + restoreUserSchedulesFromDeletion, +} = require('~/server/services/Schedules'); const db = require('~/models'); const { getAppConfig, invalidateCodeEnvironmentConfigCache } = require('~/server/services/Config'); const router = express.Router(); +/** Same dependencies the self-service controller passes, with storage cleanup run as the target. */ +const resolveErasure = async ({ id, role, tenantId }) => { + const appConfig = await getAppConfig({ role, userId: id, tenantId }); + const fileRequest = { + user: { id, _id: id, role, tenantId }, + baseUrl: '', + query: {}, + body: {}, + config: appConfig, + }; + return { + checkpointer: appConfig?.endpoints?.agents?.checkpointer, + deps: { + ...db, + openCheckpointDeletion, + deleteSharedLinks: deleteAllSharedLinksWithCleanup, + deleteStoredFiles: (files) => processDeleteRequest({ req: fileRequest, files }), + invalidateMCPServerTools: (serverName) => invalidateCachedTools({ userId: id, serverName }), + disconnectMCPServer: async (serverName) => { + await getMCPManager()?.disconnectUserConnection(id, serverName); + }, + }, + }; +}; + const requireAdminAccess = requireCapability(SystemCapabilities.ACCESS_ADMIN); const requireReadUsers = requireCapability(SystemCapabilities.READ_USERS); // const requireManageUsers = requireCapability(SystemCapabilities.MANAGE_USERS); @@ -40,7 +79,12 @@ const handlers = createAdminUsersHandlers({ deleteConfig: db.deleteConfig, deleteAclEntries: db.deleteAclEntries, deletePasskeysByUser: db.deletePasskeysByUser, - deleteTokens: db.deleteTokens, + resolveErasure, + cancelSubagentTasks: (userId, tenantId) => + subagentThreadTaskStore.cancelAndDrainForOwner(userId, tenantId), + quiesceUserSchedules, + restoreUserSchedulesFromDeletion, + generationJobs: GenerationJobManager, }); router.use(requireJwtAuth, requireAdminAccess); diff --git a/packages/api/src/admin/users.spec.ts b/packages/api/src/admin/users.spec.ts index 1b969cfef66..765a07a96b8 100644 --- a/packages/api/src/admin/users.spec.ts +++ b/packages/api/src/admin/users.spec.ts @@ -5,6 +5,11 @@ import type { Response } from 'express'; import type { ServerRequest } from '~/types/http'; import type { AdminUsersDeps } from './users'; import { createAdminUsersHandlers } from './users'; +import { eraseUserData } from '~/user/erasure'; + +jest.mock('~/user/erasure', () => ({ + eraseUserData: jest.fn().mockResolvedValue(undefined), +})); jest.mock('@librechat/data-schemas', () => ({ ...jest.requireActual('@librechat/data-schemas'), @@ -68,12 +73,24 @@ function createDeps(overrides: Partial = {}): AdminUsersDeps { deleteConfig: jest.fn().mockResolvedValue(null), deleteAclEntries: jest.fn().mockResolvedValue(undefined), deletePasskeysByUser: jest.fn().mockResolvedValue(0), - deleteTokens: jest.fn().mockResolvedValue({ deletedCount: 0 }), + resolveErasure: jest.fn().mockResolvedValue({ deps: {} }), + cancelSubagentTasks: jest.fn().mockResolvedValue(undefined), + quiesceUserSchedules: jest.fn().mockResolvedValue(true), + restoreUserSchedulesFromDeletion: jest.fn().mockResolvedValue(undefined), + generationJobs: { + getAccountCleanupJobIdsForUser: jest.fn().mockResolvedValue([]), + getCleanupJob: jest.fn().mockResolvedValue(undefined), + abortJob: jest.fn().mockResolvedValue(null), + }, ...overrides, }; } describe('createAdminUsersHandlers', () => { + beforeEach(() => { + jest.mocked(eraseUserData).mockClear(); + }); + describe('listUsers', () => { it('returns paginated users with total count', async () => { const users = [ @@ -462,32 +479,100 @@ describe('createAdminUsersHandlers', () => { expect(deps.deletePasskeysByUser).toHaveBeenCalledWith(validUserId); }); - it("removes the user's stored credentials before the user", async () => { - const deps = createDeps(); + it('runs the shared erasure cascade for the target before removing the user', async () => { + const checkpointer = { ttl: 60 }; + const erasureDeps = { marker: true }; + const deps = createDeps({ + findUsers: jest.fn().mockResolvedValue([mockUser({ role: 'USER', tenantId: 'tenant-1' })]), + resolveErasure: jest.fn().mockResolvedValue({ deps: erasureDeps, checkpointer }), + }); const handlers = createAdminUsersHandlers(deps); const { req, res, status } = createReqRes({ params: { id: validUserId } }); await handlers.deleteUser(req, res); expect(status).toHaveBeenCalledWith(200); - expect(deps.deleteTokens).toHaveBeenCalledWith({ userId: validUserId }); - expect(jest.mocked(deps.deleteTokens).mock.invocationCallOrder[0]).toBeLessThan( + expect(deps.resolveErasure).toHaveBeenCalledWith({ + id: validUserId, + role: 'USER', + tenantId: 'tenant-1', + }); + expect(eraseUserData).toHaveBeenCalledWith( + { userId: validUserId, tenantId: 'tenant-1', checkpointer }, + erasureDeps, + ); + expect(jest.mocked(eraseUserData).mock.invocationCallOrder[0]).toBeLessThan( jest.mocked(deps.deleteUserById).mock.invocationCallOrder[0], ); }); - it('keeps the user, and releases the fence, when its credentials cannot be removed', async () => { + it('keeps the user, and releases the fence, when the erasure cascade fails', async () => { + jest.mocked(eraseUserData).mockRejectedValueOnce(new Error('storage down')); + const deps = createDeps(); + const handlers = createAdminUsersHandlers(deps); + const { req, res, status, json } = createReqRes({ params: { id: validUserId } }); + + await handlers.deleteUser(req, res); + + expect(status).toHaveBeenCalledWith(500); + expect(json).toHaveBeenCalledWith({ error: 'Failed to delete user' }); + expect(deps.deleteUserById).not.toHaveBeenCalled(); + expect(deps.cancelAgentTriggerUserDeletion).toHaveBeenCalled(); + expect(deps.cancelAgentTriggerUserPurge).toHaveBeenCalled(); + }); + + it('suspends schedules before erasing and restores them when the deletion fails', async () => { + jest.mocked(eraseUserData).mockRejectedValueOnce(new Error('storage down')); + const deps = createDeps(); + const handlers = createAdminUsersHandlers(deps); + const { req, res, status } = createReqRes({ params: { id: validUserId } }); + + await handlers.deleteUser(req, res); + + const token = jest.mocked(deps.quiesceUserSchedules).mock.calls[0][1]; + expect(status).toHaveBeenCalledWith(500); + expect(deps.restoreUserSchedulesFromDeletion).toHaveBeenCalledWith(validUserId, token); + expect( + jest.mocked(deps.restoreUserSchedulesFromDeletion).mock.invocationCallOrder[0], + ).toBeLessThan(jest.mocked(deps.cancelAgentTriggerUserDeletion).mock.invocationCallOrder[0]); + }); + + it('does not erase when schedules cannot be confirmed stopped', async () => { + const deps = createDeps({ quiesceUserSchedules: jest.fn().mockResolvedValue(false) }); + const handlers = createAdminUsersHandlers(deps); + const { req, res, status } = createReqRes({ params: { id: validUserId } }); + + await handlers.deleteUser(req, res); + + expect(status).toHaveBeenCalledWith(500); + expect(eraseUserData).not.toHaveBeenCalled(); + expect(deps.deleteUserById).not.toHaveBeenCalled(); + }); + + it('aborts the cleanup jobs of the target and refuses to erase when a stop is unconfirmed', async () => { + const abortJob = jest.fn().mockResolvedValue(null); const deps = createDeps({ - deleteTokens: jest.fn().mockRejectedValue(new Error('mongo down')), + generationJobs: { + getAccountCleanupJobIdsForUser: jest.fn().mockResolvedValue(['s1', 's2']), + getCleanupJob: jest.fn(async (id: string) => ({ + createdAt: 5, + metadata: { userId: id === 's1' ? validUserId : 'someone-else' }, + })), + abortJob, + } as unknown as AdminUsersDeps['generationJobs'], }); const handlers = createAdminUsersHandlers(deps); const { req, res, status } = createReqRes({ params: { id: validUserId } }); await handlers.deleteUser(req, res); + expect(abortJob).toHaveBeenCalledTimes(1); + expect(abortJob).toHaveBeenCalledWith('s1', { + expectedCreatedAt: 5, + awaitProviderDrain: true, + }); expect(status).toHaveBeenCalledWith(500); - expect(deps.deleteUserById).not.toHaveBeenCalled(); - expect(deps.cancelAgentTriggerUserDeletion).toHaveBeenCalled(); + expect(eraseUserData).not.toHaveBeenCalled(); }); it('returns success even when cascade cleanup partially fails', async () => { diff --git a/packages/api/src/admin/users.ts b/packages/api/src/admin/users.ts index e6a1e5f431c..9b555cab689 100644 --- a/packages/api/src/admin/users.ts +++ b/packages/api/src/admin/users.ts @@ -1,4 +1,5 @@ import { Types } from 'mongoose'; +import { randomUUID } from 'node:crypto'; import { PrincipalType, SystemRoles } from 'librechat-data-provider'; import { logger, isValidObjectIdString } from '@librechat/data-schemas'; import type { @@ -8,15 +9,32 @@ import type { AdminUserSearchResult, UserDeleteResult, } from '@librechat/data-schemas'; +import type { TCheckpointerConfig } from 'librechat-data-provider'; import type { FilterQuery } from 'mongoose'; import type { Response } from 'express'; +import type { UserGenerationControl } from '~/user/generations'; +import type { UserErasureDeps } from '~/user/erasure'; import type { ServerRequest } from '~/types/http'; +import { stopUserGenerations } from '~/user/generations'; import { parsePagination } from './pagination'; +import { eraseUserData } from '~/user/erasure'; const MAX_SEARCH_LENGTH = 200; const USER_LIST_FIELDS = '_id name username email avatar role provider createdAt updatedAt'; +/** The target account's identity, from which the caller builds storage and MCP cleanup. */ +export interface AdminErasureTarget { + id: string; + role?: string; + tenantId?: string; +} + +export interface AdminErasureContext { + deps: UserErasureDeps; + checkpointer?: TCheckpointerConfig; +} + export interface AdminUsersDeps { findUsers: ( searchCriteria: FilterQuery, @@ -38,14 +56,22 @@ export interface AdminUsersDeps { cancelAgentTriggerUserPurge: (userId: string, fenceStartedAt: Date) => Promise; purgeAgentTriggerDeliveriesForUser: (userId: string) => Promise; revokeUserCodeEnvironmentWorkers?: (userId: string) => Promise; + /** Cancels and drains the account's subagent thread tasks. */ + cancelSubagentTasks?: (userId: string, tenantId?: string) => Promise; /** - * Thin data-layer delete — removes the User document only. - * Full cascade of user-owned resources (conversations, messages, files, tokens, etc.) - * is handled by `UserController.deleteUserController` in the self-delete flow. - * This admin endpoint fences durable triggers around the user commit and currently - * cascades Config and AclEntries. - * A future iteration should consolidate the full cascade into a shared service function. + * Reversibly suspends the account's schedules under a per-attempt token; false means the + * executions could not be confirmed stopped. Restored if the deletion fails. */ + quiesceUserSchedules: (userId: string, token: string) => Promise; + restoreUserSchedulesFromDeletion: (userId: string, token: string) => Promise; + generationJobs: UserGenerationControl; + /** + * Resolves the dependencies of the shared `eraseUserData` cascade for the account being + * deleted, the same set the self-service controller and the delete-user CLI pass. Storage + * cleanup must run as the target user, not the calling admin. + */ + resolveErasure: (target: AdminErasureTarget) => Promise; + /** Thin data-layer delete: removes the User document after the erasure cascade has run. */ deleteUserById: (userId: string) => Promise; deleteUserCodeEnvironments: (userId: string | Types.ObjectId) => Promise; invalidateCodeEnvironmentConfigCache: (tenantId?: string) => Promise; @@ -62,12 +88,6 @@ export interface AdminUsersDeps { * keep authentication material and can collide with a later registration. */ deletePasskeysByUser: (userId: string) => Promise; - /** - * Stored credentials the user holds, such as a connected account's OAuth grant. Removed before - * the user is, as the self-delete flow does, so a failure leaves the account to retry rather - * than its credentials behind. - */ - deleteTokens: (query: { userId: string }) => Promise; } export function createAdminUsersHandlers(deps: AdminUsersDeps): { @@ -91,7 +111,11 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): { deleteConfig, deleteAclEntries, deletePasskeysByUser, - deleteTokens, + resolveErasure, + cancelSubagentTasks, + quiesceUserSchedules, + restoreUserSchedulesFromDeletion, + generationJobs, } = deps; async function listUsersHandler(req: ServerRequest, res: Response) { @@ -173,6 +197,7 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): { async function deleteUserHandler(req: ServerRequest, res: Response) { let targetUserId: string | undefined; let triggerDeletionFence: Date | undefined; + let scheduleSuspensionToken: string | undefined; let userDeleted = false; try { @@ -209,7 +234,22 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): { await prepareAgentTriggerUserPurge(id, triggerDeletionFence, targetUser?.tenantId); await drainAgentTriggerDeliveriesForUser(id); - await deleteTokens({ userId: id }); + await cancelSubagentTasks?.(id, targetUser?.tenantId); + scheduleSuspensionToken = randomUUID(); + if (!(await quiesceUserSchedules(id, scheduleSuspensionToken))) { + throw new Error('Scheduled executions could not be confirmed stopped'); + } + await stopUserGenerations(id, targetUser?.tenantId, generationJobs); + + const { deps: erasureDeps, checkpointer } = await resolveErasure({ + id, + role: targetUser?.role, + tenantId: targetUser?.tenantId, + }); + await eraseUserData( + { userId: id, tenantId: targetUser?.tenantId, checkpointer }, + erasureDeps, + ); const result = await deleteUserById(id); if (result.deletedCount === 0) { @@ -256,6 +296,17 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): { return res.status(200).json({ message: result.message || 'User deleted successfully' }); } catch (error) { + /** Restore before releasing the fence, as the self-service flow does. */ + if (targetUserId != null && scheduleSuspensionToken != null && !userDeleted) { + try { + await restoreUserSchedulesFromDeletion(targetUserId, scheduleSuspensionToken); + } catch (restoreError) { + logger.error( + `[adminUsers] failed to restore suspended schedules for user ${targetUserId} under token ${scheduleSuspensionToken}:`, + restoreError, + ); + } + } if (targetUserId != null && triggerDeletionFence != null && !userDeleted) { try { await cancelAgentTriggerUserPurge(targetUserId, triggerDeletionFence); diff --git a/packages/api/src/user/generations.ts b/packages/api/src/user/generations.ts new file mode 100644 index 00000000000..9e6e658278e --- /dev/null +++ b/packages/api/src/user/generations.ts @@ -0,0 +1,53 @@ +import type { GenerationJob } from '~/types/stream'; +import { waitForGenerationPersistence } from '~/stream/persistence'; +import { isStopConfirmed } from '~/stream/interfaces/IJobStore'; + +type AbortOutcome = Parameters[0]; +type CleanupJob = Pick; + +/** The slice of the generation job manager needed to stop an account's runs. */ +export interface UserGenerationControl { + getAccountCleanupJobIdsForUser: (userId: string, tenantId?: string) => Promise; + getCleanupJob: (streamId: string) => Promise; + abortJob: ( + streamId: string, + options: { expectedCreatedAt: number; awaitProviderDrain: boolean }, + ) => Promise; +} + +/** + * Stops the user's in-flight and retained generations and waits for their final persistence, + * so nothing is written for the account after its data is erased. Throws if a stop cannot be + * confirmed, leaving the caller to keep the account and retry. + */ +export async function stopUserGenerations( + userId: string, + tenantId: string | undefined, + control: UserGenerationControl, +): Promise { + const streamIds = await control.getAccountCleanupJobIdsForUser(userId, tenantId); + const jobs = await Promise.all( + streamIds.map(async (streamId) => ({ + streamId, + job: await control.getCleanupJob(streamId), + })), + ); + const owned = jobs.filter( + ({ job }) => + job?.metadata?.userId === userId && + (!job.metadata.tenantId || job.metadata.tenantId === tenantId), + ); + const results = await Promise.all( + owned.map(({ streamId, job }) => + control.abortJob(streamId, { expectedCreatedAt: job!.createdAt, awaitProviderDrain: true }), + ), + ); + if (results.some((result) => !isStopConfirmed(result))) { + throw new Error('Agent generations could not be confirmed stopped'); + } + await Promise.all( + owned.map(({ streamId, job }) => + waitForGenerationPersistence(streamId, job!.createdAt, control.getCleanupJob), + ), + ); +} diff --git a/packages/api/src/user/index.ts b/packages/api/src/user/index.ts index 1945a8f3161..e68c9505095 100644 --- a/packages/api/src/user/index.ts +++ b/packages/api/src/user/index.ts @@ -1,2 +1,3 @@ export * from './preferences'; export * from './erasure'; +export * from './generations'; From bccc2e4fc4950bae643a41dad57cbe49e6447341 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Tue, 13 Oct 2026 08:51:06 +0200 Subject: [PATCH 02/23] refactor: Share Generation Stop Logic Between Self-Service and Admin Deletion --- api/server/controllers/UserController.js | 37 +------------------ .../controllers/__tests__/deleteUser.spec.js | 8 +--- packages/api/src/user/generations.ts | 6 +-- 3 files changed, 7 insertions(+), 44 deletions(-) diff --git a/api/server/controllers/UserController.js b/api/server/controllers/UserController.js index 3f4bdb66b67..70aa6c2db4e 100644 --- a/api/server/controllers/UserController.js +++ b/api/server/controllers/UserController.js @@ -15,8 +15,7 @@ const { getWebSearchInstallEntries, getWebSearchUninstallFields, openCheckpointDeletion, - isStopConfirmed, - waitForGenerationPersistence, + stopUserGenerations, deleteAllSharedLinksWithCleanup, eraseUserData, deleteUserMCPServers, @@ -410,39 +409,7 @@ const deleteUserController = async (req, res) => { if (!(await quiesceUserSchedules(user.id, scheduleSuspensionToken))) { throw new Error('Scheduled executions could not be confirmed stopped'); } - const activeAgentRuns = await GenerationJobManager.getAccountCleanupJobIdsForUser( - user.id, - tenantId, - ); - const activeAgentJobs = await Promise.all( - activeAgentRuns.map(async (streamId) => ({ - streamId, - job: await GenerationJobManager.getCleanupJob(streamId), - })), - ); - const ownedAgentJobs = activeAgentJobs.filter( - ({ job }) => - job?.metadata?.userId === user.id && - (!job.metadata.tenantId || job.metadata.tenantId === tenantId), - ); - const stopResults = await Promise.all( - ownedAgentJobs.map(({ streamId, job }) => - GenerationJobManager.abortJob(streamId, { - expectedCreatedAt: job.createdAt, - awaitProviderDrain: true, - }), - ), - ); - if (stopResults.some((result) => !isStopConfirmed(result))) { - throw new Error('Agent generations could not be confirmed stopped'); - } - await Promise.all( - ownedAgentJobs.map(({ streamId, job }) => - waitForGenerationPersistence(streamId, job.createdAt, (id) => - GenerationJobManager.getCleanupJob(id), - ), - ), - ); + await stopUserGenerations(user.id, tenantId, GenerationJobManager); const appConfig = req.config ?? diff --git a/api/server/controllers/__tests__/deleteUser.spec.js b/api/server/controllers/__tests__/deleteUser.spec.js index cbe5dd58ae4..13dafb794b2 100644 --- a/api/server/controllers/__tests__/deleteUser.spec.js +++ b/api/server/controllers/__tests__/deleteUser.spec.js @@ -77,13 +77,9 @@ jest.mock('@librechat/api', () => ({ abortJob: (...args) => mockAbortJob(...args), }, getOwnedAgentCheckpointScope: jest.fn(() => undefined), - isStopConfirmed: jest.fn( - (result) => result?.success === true || result?.failureReason === 'already_settled', - ), deleteOwnedAgentCheckpoints: jest.fn(), - waitForGenerationPersistence: jest.requireActual( - '../../../../packages/api/src/stream/persistence.ts', - ).waitForGenerationPersistence, + stopUserGenerations: jest.requireActual('../../../../packages/api/src/user/generations.ts') + .stopUserGenerations, openCheckpointDeletion: jest.fn(async (userId, tenantId, _root, cfg) => ({ remember: jest.fn(), cleanup: () => diff --git a/packages/api/src/user/generations.ts b/packages/api/src/user/generations.ts index 9e6e658278e..b0aeb375302 100644 --- a/packages/api/src/user/generations.ts +++ b/packages/api/src/user/generations.ts @@ -1,6 +1,6 @@ -import type { GenerationJob } from '~/types/stream'; -import { waitForGenerationPersistence } from '~/stream/persistence'; -import { isStopConfirmed } from '~/stream/interfaces/IJobStore'; +import type { GenerationJob } from '../types/stream'; +import { waitForGenerationPersistence } from '../stream/persistence'; +import { isStopConfirmed } from '../stream/interfaces/IJobStore'; type AbortOutcome = Parameters[0]; type CleanupJob = Pick; From a68a7ceb29b0270d83864669f6f9e16382cdc60b Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Tue, 13 Oct 2026 08:52:03 +0200 Subject: [PATCH 03/23] refactor: Move Admin Erasure Dependency Assembly Into packages/api --- api/server/routes/admin/users.js | 34 ++++-------- packages/api/src/user/index.ts | 1 + packages/api/src/user/resolver.spec.ts | 77 ++++++++++++++++++++++++++ packages/api/src/user/resolver.ts | 71 ++++++++++++++++++++++++ 4 files changed, 159 insertions(+), 24 deletions(-) create mode 100644 packages/api/src/user/resolver.spec.ts create mode 100644 packages/api/src/user/resolver.ts diff --git a/api/server/routes/admin/users.js b/api/server/routes/admin/users.js index 6c980783106..223a7ba8836 100644 --- a/api/server/routes/admin/users.js +++ b/api/server/routes/admin/users.js @@ -4,6 +4,7 @@ const { GenerationJobManager, openCheckpointDeletion, deleteAllSharedLinksWithCleanup, + createErasureResolver, createAdminUsersHandlers, revokeUserCodeEnvironmentWorkers, } = require('@librechat/api'); @@ -29,30 +30,15 @@ const { getAppConfig, invalidateCodeEnvironmentConfigCache } = require('~/server const router = express.Router(); -/** Same dependencies the self-service controller passes, with storage cleanup run as the target. */ -const resolveErasure = async ({ id, role, tenantId }) => { - const appConfig = await getAppConfig({ role, userId: id, tenantId }); - const fileRequest = { - user: { id, _id: id, role, tenantId }, - baseUrl: '', - query: {}, - body: {}, - config: appConfig, - }; - return { - checkpointer: appConfig?.endpoints?.agents?.checkpointer, - deps: { - ...db, - openCheckpointDeletion, - deleteSharedLinks: deleteAllSharedLinksWithCleanup, - deleteStoredFiles: (files) => processDeleteRequest({ req: fileRequest, files }), - invalidateMCPServerTools: (serverName) => invalidateCachedTools({ userId: id, serverName }), - disconnectMCPServer: async (serverName) => { - await getMCPManager()?.disconnectUserConnection(id, serverName); - }, - }, - }; -}; +const resolveErasure = createErasureResolver({ + db, + getAppConfig, + getMCPManager, + processDeleteRequest, + invalidateCachedTools, + openCheckpointDeletion, + deleteSharedLinks: deleteAllSharedLinksWithCleanup, +}); const requireAdminAccess = requireCapability(SystemCapabilities.ACCESS_ADMIN); const requireReadUsers = requireCapability(SystemCapabilities.READ_USERS); diff --git a/packages/api/src/user/index.ts b/packages/api/src/user/index.ts index e68c9505095..34607738adf 100644 --- a/packages/api/src/user/index.ts +++ b/packages/api/src/user/index.ts @@ -1,3 +1,4 @@ export * from './preferences'; export * from './erasure'; export * from './generations'; +export * from './resolver'; diff --git a/packages/api/src/user/resolver.spec.ts b/packages/api/src/user/resolver.spec.ts new file mode 100644 index 00000000000..ea70c7fe8cf --- /dev/null +++ b/packages/api/src/user/resolver.spec.ts @@ -0,0 +1,77 @@ +import type { IMongoFile } from '@librechat/data-schemas'; +import type { ErasureResolverDeps } from './resolver'; +import { createErasureResolver } from './resolver'; + +function createDeps(overrides: Partial = {}) { + const disconnectUserConnection = jest.fn().mockResolvedValue(undefined); + const deps = { + db: { deleteConvos: jest.fn() }, + getAppConfig: jest + .fn() + .mockResolvedValue({ endpoints: { agents: { checkpointer: { ttl: 5 } } } }), + processDeleteRequest: jest.fn().mockResolvedValue(undefined), + invalidateCachedTools: jest.fn().mockResolvedValue(undefined), + getMCPManager: jest.fn(() => ({ disconnectUserConnection })), + openCheckpointDeletion: jest.fn(), + deleteSharedLinks: jest.fn(), + ...overrides, + } as unknown as ErasureResolverDeps; + return { deps, disconnectUserConnection }; +} + +const target = { id: 'user-1', role: 'USER', tenantId: 'tenant-1' }; + +describe('createErasureResolver', () => { + it('resolves the target config and its checkpointer', async () => { + const { deps } = createDeps(); + const result = await createErasureResolver(deps)(target); + + expect(deps.getAppConfig).toHaveBeenCalledWith({ + role: 'USER', + userId: 'user-1', + tenantId: 'tenant-1', + }); + expect(result.checkpointer).toEqual({ ttl: 5 }); + expect(result.deps.openCheckpointDeletion).toBe(deps.openCheckpointDeletion); + expect(result.deps.deleteSharedLinks).toBe(deps.deleteSharedLinks); + }); + + it('deletes stored files with a request that carries the target user', async () => { + const { deps } = createDeps(); + const { deps: erasure } = await createErasureResolver(deps)(target); + const files = [{ file_id: 'f1' }] as IMongoFile[]; + + await erasure.deleteStoredFiles(files); + + expect(deps.processDeleteRequest).toHaveBeenCalledWith({ + req: expect.objectContaining({ + user: { id: 'user-1', _id: 'user-1', role: 'USER', tenantId: 'tenant-1' }, + body: {}, + query: {}, + baseUrl: '', + }), + files, + }); + }); + + it('scopes MCP runtime cleanup to the target user', async () => { + const { deps, disconnectUserConnection } = createDeps(); + const { deps: erasure } = await createErasureResolver(deps)(target); + + await erasure.invalidateMCPServerTools?.('srv'); + await erasure.disconnectMCPServer?.('srv'); + + expect(deps.invalidateCachedTools).toHaveBeenCalledWith({ + userId: 'user-1', + serverName: 'srv', + }); + expect(disconnectUserConnection).toHaveBeenCalledWith('user-1', 'srv'); + }); + + it('tolerates a missing MCP manager', async () => { + const { deps } = createDeps({ getMCPManager: jest.fn(() => undefined) }); + const { deps: erasure } = await createErasureResolver(deps)(target); + + await expect(erasure.disconnectMCPServer?.('srv')).resolves.toBeUndefined(); + }); +}); diff --git a/packages/api/src/user/resolver.ts b/packages/api/src/user/resolver.ts new file mode 100644 index 00000000000..44b5eba0983 --- /dev/null +++ b/packages/api/src/user/resolver.ts @@ -0,0 +1,71 @@ +import type { TCheckpointerConfig } from 'librechat-data-provider'; +import type { IMongoFile } from '@librechat/data-schemas'; +import type { AdminErasureContext, AdminErasureTarget } from '~/admin/users'; +import type { UserErasureDeps } from './erasure'; + +type ErasureBaseDeps = Omit< + UserErasureDeps, + | 'openCheckpointDeletion' + | 'deleteSharedLinks' + | 'deleteStoredFiles' + | 'invalidateMCPServerTools' + | 'disconnectMCPServer' +>; + +export interface ErasureResolverDeps { + /** Data-layer methods used by the cascade. */ + db: ErasureBaseDeps; + getAppConfig: (options: { + role?: string; + userId: string; + tenantId?: string; + }) => Promise<{ endpoints?: { agents?: { checkpointer?: TCheckpointerConfig } } } | undefined>; + processDeleteRequest: (params: { + req: Record; + files: IMongoFile[]; + }) => Promise; + invalidateCachedTools: (params: { userId: string; serverName: string }) => Promise; + getMCPManager: () => + | { disconnectUserConnection: (userId: string, serverName: string) => Promise } + | null + | undefined; + openCheckpointDeletion: UserErasureDeps['openCheckpointDeletion']; + deleteSharedLinks: UserErasureDeps['deleteSharedLinks']; +} + +/** + * Builds the `eraseUserData` dependencies for an account deleted by someone else. Stored files + * are removed with a request that carries the target user, since provider clients and + * credentials are resolved from the request user, not the calling admin. + */ +export function createErasureResolver(deps: ErasureResolverDeps) { + return async function resolveErasure({ + id, + role, + tenantId, + }: AdminErasureTarget): Promise { + const appConfig = await deps.getAppConfig({ role, userId: id, tenantId }); + const fileRequest = { + user: { id, _id: id, role, tenantId }, + /** Assistants client setup reads these Express request fields. */ + baseUrl: '', + query: {}, + body: {}, + config: appConfig, + }; + return { + checkpointer: appConfig?.endpoints?.agents?.checkpointer, + deps: { + ...deps.db, + openCheckpointDeletion: deps.openCheckpointDeletion, + deleteSharedLinks: deps.deleteSharedLinks, + deleteStoredFiles: (files) => deps.processDeleteRequest({ req: fileRequest, files }), + invalidateMCPServerTools: (serverName) => + deps.invalidateCachedTools({ userId: id, serverName }), + disconnectMCPServer: async (serverName) => { + await deps.getMCPManager()?.disconnectUserConnection(id, serverName); + }, + }, + }; + }; +} From feb0fa2fc637585d78ea342f8bd2947fdf21b79e Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Tue, 13 Oct 2026 08:52:57 +0200 Subject: [PATCH 04/23] fix: Bind getCleanupJob When Waiting for Generation Persistence --- packages/api/src/user/generations.spec.ts | 49 +++++++++++++++++++++++ packages/api/src/user/generations.ts | 2 +- 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 packages/api/src/user/generations.spec.ts diff --git a/packages/api/src/user/generations.spec.ts b/packages/api/src/user/generations.spec.ts new file mode 100644 index 00000000000..da2681a3fee --- /dev/null +++ b/packages/api/src/user/generations.spec.ts @@ -0,0 +1,49 @@ +import type { UserGenerationControl } from './generations'; +import { stopUserGenerations } from './generations'; + +/** Methods read `this`, as the real GenerationJobManager does, so an unbound call throws. */ +class FakeJobManager { + private jobs = new Map([ + ['s1', { createdAt: 7, metadata: { userId: 'u1', terminalPersistencePending: false } }], + ['s2', { createdAt: 8, metadata: { userId: 'other' } }], + ]); + + readonly aborted: string[] = []; + + async getAccountCleanupJobIdsForUser() { + return [...this.jobs.keys()]; + } + + async getCleanupJob(streamId: string) { + return this.jobs.get(streamId); + } + + async abortJob(streamId: string) { + this.aborted.push(streamId); + return { success: true }; + } +} + +describe('stopUserGenerations', () => { + it('stops only the user jobs using a manager whose methods depend on this', async () => { + const manager = new FakeJobManager(); + + await expect( + stopUserGenerations('u1', undefined, manager as unknown as UserGenerationControl), + ).resolves.toBeUndefined(); + + expect(manager.aborted).toEqual(['s1']); + }); + + it('throws when a stop is not confirmed', async () => { + const manager = new FakeJobManager(); + manager.abortJob = async function (this: FakeJobManager, streamId: string) { + this.aborted.push(streamId); + return { success: false }; + }; + + await expect( + stopUserGenerations('u1', undefined, manager as unknown as UserGenerationControl), + ).rejects.toThrow('could not be confirmed stopped'); + }); +}); diff --git a/packages/api/src/user/generations.ts b/packages/api/src/user/generations.ts index b0aeb375302..01a60af631a 100644 --- a/packages/api/src/user/generations.ts +++ b/packages/api/src/user/generations.ts @@ -47,7 +47,7 @@ export async function stopUserGenerations( } await Promise.all( owned.map(({ streamId, job }) => - waitForGenerationPersistence(streamId, job!.createdAt, control.getCleanupJob), + waitForGenerationPersistence(streamId, job!.createdAt, (id) => control.getCleanupJob(id)), ), ); } From 5388abbc7a6c620f4356a0041d9b145eafaf71f7 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Tue, 13 Oct 2026 08:58:27 +0200 Subject: [PATCH 05/23] test: Pin the Deletion Fence Before Admin Generation Stop --- packages/api/src/admin/users.spec.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/api/src/admin/users.spec.ts b/packages/api/src/admin/users.spec.ts index 765a07a96b8..78c0609561f 100644 --- a/packages/api/src/admin/users.spec.ts +++ b/packages/api/src/admin/users.spec.ts @@ -537,6 +537,20 @@ describe('createAdminUsersHandlers', () => { ).toBeLessThan(jest.mocked(deps.cancelAgentTriggerUserDeletion).mock.invocationCallOrder[0]); }); + it('takes the trigger deletion fence before listing generations to stop', async () => { + const deps = createDeps(); + const handlers = createAdminUsersHandlers(deps); + const { req, res } = createReqRes({ params: { id: validUserId } }); + + await handlers.deleteUser(req, res); + + expect( + jest.mocked(deps.beginAgentTriggerUserDeletion).mock.invocationCallOrder[0], + ).toBeLessThan( + jest.mocked(deps.generationJobs.getAccountCleanupJobIdsForUser).mock.invocationCallOrder[0], + ); + }); + it('does not erase when schedules cannot be confirmed stopped', async () => { const deps = createDeps({ quiesceUserSchedules: jest.fn().mockResolvedValue(false) }); const handlers = createAdminUsersHandlers(deps); From 429663c7d8c6e4e7a9089428fcebfc0864716ce2 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:30:37 +0200 Subject: [PATCH 06/23] fix: Resolve the Erasure Target Config Fail Closed --- packages/api/src/user/resolver.spec.ts | 8 ++++++++ packages/api/src/user/resolver.ts | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/api/src/user/resolver.spec.ts b/packages/api/src/user/resolver.spec.ts index ea70c7fe8cf..4d13aa7fd24 100644 --- a/packages/api/src/user/resolver.spec.ts +++ b/packages/api/src/user/resolver.spec.ts @@ -30,12 +30,20 @@ describe('createErasureResolver', () => { role: 'USER', userId: 'user-1', tenantId: 'tenant-1', + failClosed: true, }); expect(result.checkpointer).toEqual({ ttl: 5 }); expect(result.deps.openCheckpointDeletion).toBe(deps.openCheckpointDeletion); expect(result.deps.deleteSharedLinks).toBe(deps.deleteSharedLinks); }); + it('rejects when the target config cannot be resolved', async () => { + const { deps } = createDeps({ + getAppConfig: jest.fn().mockRejectedValue(new Error('override store down')), + }); + await expect(createErasureResolver(deps)(target)).rejects.toThrow('override store down'); + }); + it('deletes stored files with a request that carries the target user', async () => { const { deps } = createDeps(); const { deps: erasure } = await createErasureResolver(deps)(target); diff --git a/packages/api/src/user/resolver.ts b/packages/api/src/user/resolver.ts index 44b5eba0983..7f48c1983d9 100644 --- a/packages/api/src/user/resolver.ts +++ b/packages/api/src/user/resolver.ts @@ -19,6 +19,7 @@ export interface ErasureResolverDeps { role?: string; userId: string; tenantId?: string; + failClosed?: boolean; }) => Promise<{ endpoints?: { agents?: { checkpointer?: TCheckpointerConfig } } } | undefined>; processDeleteRequest: (params: { req: Record; @@ -44,7 +45,8 @@ export function createErasureResolver(deps: ErasureResolverDeps) { role, tenantId, }: AdminErasureTarget): Promise { - const appConfig = await deps.getAppConfig({ role, userId: id, tenantId }); + /** A failed override lookup must abort the deletion, not fall back to a base config that cannot reach the target's storage. */ + const appConfig = await deps.getAppConfig({ role, userId: id, tenantId, failClosed: true }); const fileRequest = { user: { id, _id: id, role, tenantId }, /** Assistants client setup reads these Express request fields. */ From 6cb7d89c2e9db357811cdfd20e8e80da9d56dbd7 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Tue, 13 Oct 2026 23:27:27 +0200 Subject: [PATCH 07/23] fix: Carry the Target Profile Into Admin Erasure Storage Cleanup --- packages/api/src/admin/users.spec.ts | 13 ++++++++----- packages/api/src/admin/users.ts | 8 +++++++- packages/api/src/user/resolver.spec.ts | 24 ++++++++++++++++++++++++ packages/api/src/user/resolver.ts | 3 ++- 4 files changed, 41 insertions(+), 7 deletions(-) diff --git a/packages/api/src/admin/users.spec.ts b/packages/api/src/admin/users.spec.ts index 78c0609561f..a96b35ff8ec 100644 --- a/packages/api/src/admin/users.spec.ts +++ b/packages/api/src/admin/users.spec.ts @@ -492,11 +492,14 @@ describe('createAdminUsersHandlers', () => { await handlers.deleteUser(req, res); expect(status).toHaveBeenCalledWith(200); - expect(deps.resolveErasure).toHaveBeenCalledWith({ - id: validUserId, - role: 'USER', - tenantId: 'tenant-1', - }); + expect(deps.resolveErasure).toHaveBeenCalledWith( + expect.objectContaining({ + id: validUserId, + role: 'USER', + tenantId: 'tenant-1', + profile: expect.objectContaining({ role: 'USER' }), + }), + ); expect(eraseUserData).toHaveBeenCalledWith( { userId: validUserId, tenantId: 'tenant-1', checkpointer }, erasureDeps, diff --git a/packages/api/src/admin/users.ts b/packages/api/src/admin/users.ts index 9b555cab689..0aab6aea101 100644 --- a/packages/api/src/admin/users.ts +++ b/packages/api/src/admin/users.ts @@ -21,6 +21,9 @@ import { eraseUserData } from '~/user/erasure'; const MAX_SEARCH_LENGTH = 200; +const ERASURE_TARGET_FIELDS = + 'role tenantId name username email provider googleId facebookId openidId samlId ldapId githubId discordId appleId emailVerified twoFactorEnabled termsAccepted termsAcceptedAt'; + const USER_LIST_FIELDS = '_id name username email avatar role provider createdAt updatedAt'; /** The target account's identity, from which the caller builds storage and MCP cleanup. */ @@ -28,6 +31,8 @@ export interface AdminErasureTarget { id: string; role?: string; tenantId?: string; + /** The target's profile fields, which storage header templates such as the user email read. */ + profile?: Partial; } export interface AdminErasureContext { @@ -213,7 +218,7 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): { return res.status(403).json({ error: 'Cannot delete your own account' }); } - const [targetUser] = await findUsers({ _id: id }, 'role tenantId', { limit: 1 }); + const [targetUser] = await findUsers({ _id: id }, ERASURE_TARGET_FIELDS, { limit: 1 }); if (targetUser?.role === SystemRoles.ADMIN) { const adminCount = await countUsers({ role: SystemRoles.ADMIN }); if (adminCount <= 1) { @@ -245,6 +250,7 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): { id, role: targetUser?.role, tenantId: targetUser?.tenantId, + profile: targetUser, }); await eraseUserData( { userId: id, tenantId: targetUser?.tenantId, checkpointer }, diff --git a/packages/api/src/user/resolver.spec.ts b/packages/api/src/user/resolver.spec.ts index 4d13aa7fd24..4b5df657afb 100644 --- a/packages/api/src/user/resolver.spec.ts +++ b/packages/api/src/user/resolver.spec.ts @@ -62,6 +62,30 @@ describe('createErasureResolver', () => { }); }); + it('carries the target profile fields for header templates', async () => { + const { deps } = createDeps(); + const { deps: erasure } = await createErasureResolver(deps)({ + ...target, + profile: { email: 'target@example.com', name: 'Target' }, + }); + + await erasure.deleteStoredFiles([]); + + expect(deps.processDeleteRequest).toHaveBeenCalledWith({ + req: expect.objectContaining({ + user: { + email: 'target@example.com', + name: 'Target', + id: 'user-1', + _id: 'user-1', + role: 'USER', + tenantId: 'tenant-1', + }, + }), + files: [], + }); + }); + it('scopes MCP runtime cleanup to the target user', async () => { const { deps, disconnectUserConnection } = createDeps(); const { deps: erasure } = await createErasureResolver(deps)(target); diff --git a/packages/api/src/user/resolver.ts b/packages/api/src/user/resolver.ts index 7f48c1983d9..ddb0faa053f 100644 --- a/packages/api/src/user/resolver.ts +++ b/packages/api/src/user/resolver.ts @@ -44,11 +44,12 @@ export function createErasureResolver(deps: ErasureResolverDeps) { id, role, tenantId, + profile, }: AdminErasureTarget): Promise { /** A failed override lookup must abort the deletion, not fall back to a base config that cannot reach the target's storage. */ const appConfig = await deps.getAppConfig({ role, userId: id, tenantId, failClosed: true }); const fileRequest = { - user: { id, _id: id, role, tenantId }, + user: { ...profile, id, _id: id, role, tenantId }, /** Assistants client setup reads these Express request fields. */ baseUrl: '', query: {}, From bf50f1ed2e33584ebf03c0cbae0262295ed81d78 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:34:04 +0200 Subject: [PATCH 08/23] fix: Keep the Erasure Target Contract Storage Neutral and Reuse the Loaded Identity --- packages/api/src/admin/users.spec.ts | 1 + packages/api/src/admin/users.ts | 33 ++++++++++++++++++++++---- packages/api/src/user/resolver.spec.ts | 9 +++++++ packages/api/src/user/resolver.ts | 10 +++++++- 4 files changed, 48 insertions(+), 5 deletions(-) diff --git a/packages/api/src/admin/users.spec.ts b/packages/api/src/admin/users.spec.ts index a96b35ff8ec..3c1aea13037 100644 --- a/packages/api/src/admin/users.spec.ts +++ b/packages/api/src/admin/users.spec.ts @@ -497,6 +497,7 @@ describe('createAdminUsersHandlers', () => { id: validUserId, role: 'USER', tenantId: 'tenant-1', + idOnTheSource: null, profile: expect.objectContaining({ role: 'USER' }), }), ); diff --git a/packages/api/src/admin/users.ts b/packages/api/src/admin/users.ts index 0aab6aea101..ae8ac19c9c4 100644 --- a/packages/api/src/admin/users.ts +++ b/packages/api/src/admin/users.ts @@ -21,8 +21,31 @@ import { eraseUserData } from '~/user/erasure'; const MAX_SEARCH_LENGTH = 200; -const ERASURE_TARGET_FIELDS = - 'role tenantId name username email provider googleId facebookId openidId samlId ldapId githubId discordId appleId emailVerified twoFactorEnabled termsAccepted termsAcceptedAt'; +const ERASURE_PROFILE_KEYS = [ + 'name', + 'username', + 'email', + 'provider', + 'googleId', + 'facebookId', + 'openidId', + 'samlId', + 'ldapId', + 'githubId', + 'discordId', + 'appleId', + 'emailVerified', + 'twoFactorEnabled', + 'termsAccepted', + 'termsAcceptedAt', +] as const; + +const ERASURE_TARGET_FIELDS = ['role', 'tenantId', 'idOnTheSource', ...ERASURE_PROFILE_KEYS].join( + ' ', +); + +/** The target's plain profile fields, which storage header templates such as the user email read. */ +export type ErasureTargetProfile = Partial>; const USER_LIST_FIELDS = '_id name username email avatar role provider createdAt updatedAt'; @@ -31,8 +54,9 @@ export interface AdminErasureTarget { id: string; role?: string; tenantId?: string; - /** The target's profile fields, which storage header templates such as the user email read. */ - profile?: Partial; + /** External identity key, so principal resolution reuses the loaded user instead of re-reading it. */ + idOnTheSource?: string | null; + profile?: ErasureTargetProfile; } export interface AdminErasureContext { @@ -250,6 +274,7 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): { id, role: targetUser?.role, tenantId: targetUser?.tenantId, + idOnTheSource: targetUser?.idOnTheSource ?? null, profile: targetUser, }); await eraseUserData( diff --git a/packages/api/src/user/resolver.spec.ts b/packages/api/src/user/resolver.spec.ts index 4b5df657afb..c061b7f0270 100644 --- a/packages/api/src/user/resolver.spec.ts +++ b/packages/api/src/user/resolver.spec.ts @@ -37,6 +37,15 @@ describe('createErasureResolver', () => { expect(result.deps.deleteSharedLinks).toBe(deps.deleteSharedLinks); }); + it('passes the loaded external identity so principal resolution does not re-read the user', async () => { + const { deps } = createDeps(); + await createErasureResolver(deps)({ ...target, idOnTheSource: 'ext-1' }); + + expect(deps.getAppConfig).toHaveBeenCalledWith( + expect.objectContaining({ userId: 'user-1', idOnTheSource: 'ext-1' }), + ); + }); + it('rejects when the target config cannot be resolved', async () => { const { deps } = createDeps({ getAppConfig: jest.fn().mockRejectedValue(new Error('override store down')), diff --git a/packages/api/src/user/resolver.ts b/packages/api/src/user/resolver.ts index ddb0faa053f..81d1d106b2f 100644 --- a/packages/api/src/user/resolver.ts +++ b/packages/api/src/user/resolver.ts @@ -19,6 +19,7 @@ export interface ErasureResolverDeps { role?: string; userId: string; tenantId?: string; + idOnTheSource?: string | null; failClosed?: boolean; }) => Promise<{ endpoints?: { agents?: { checkpointer?: TCheckpointerConfig } } } | undefined>; processDeleteRequest: (params: { @@ -44,10 +45,17 @@ export function createErasureResolver(deps: ErasureResolverDeps) { id, role, tenantId, + idOnTheSource, profile, }: AdminErasureTarget): Promise { /** A failed override lookup must abort the deletion, not fall back to a base config that cannot reach the target's storage. */ - const appConfig = await deps.getAppConfig({ role, userId: id, tenantId, failClosed: true }); + const appConfig = await deps.getAppConfig({ + role, + userId: id, + tenantId, + idOnTheSource, + failClosed: true, + }); const fileRequest = { user: { ...profile, id, _id: id, role, tenantId }, /** Assistants client setup reads these Express request fields. */ From 912506f8d338f2c643274ee2f993f9149f5acede Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:56:26 +0200 Subject: [PATCH 09/23] fix: Refuse New Generations Once Account Deletion Has Begun --- api/server/index.js | 1 + .../api/src/stream/GenerationJobManager.ts | 56 ++++++++++++---- .../stream/__tests__/admissionGuard.spec.ts | 64 +++++++++++++++++++ packages/api/src/stream/index.ts | 1 + .../api/src/stream/interfaces/IJobStore.ts | 11 ++++ 5 files changed, 120 insertions(+), 13 deletions(-) create mode 100644 packages/api/src/stream/__tests__/admissionGuard.spec.ts diff --git a/api/server/index.js b/api/server/index.js index 469f2863e62..f98c09dfd23 100644 --- a/api/server/index.js +++ b/api/server/index.js @@ -139,6 +139,7 @@ const configureGenerationStreams = () => { cleanupOnComplete: !isEnabled(process.env.STREAM_KEEP_COMPLETED_JOBS), }); GenerationJobManager.setApprovalExpiredHandler(recordExpiredScheduleApproval); + GenerationJobManager.setAdmissionGuard(agentEventMethods.isAgentTriggerPrincipalActive); GenerationJobManager.setTerminalHostActionHandler( createAgentEventTerminalHandler(agentEventMethods, { resumeDetachedAction: resumeAgentEventDetachedAction, diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts index 9f721c2a3dd..3e4ccaa331a 100644 --- a/packages/api/src/stream/GenerationJobManager.ts +++ b/packages/api/src/stream/GenerationJobManager.ts @@ -44,25 +44,26 @@ import type { RecoveredSteerPayload } from './SteerRecovery'; import type { SteerContentView } from './SteeringLifecycle'; import type { GenerationJobStore } from '~/app/metrics'; import type * as t from '~/types'; -import { - recordGenerationStreamEarlyBufferOverflow, - recordGenerationStreamResumePendingEvents, - recordGenerationStreamSubscription, - recordGenerationStreamAttachment, - recordGenerationStreamRecovery, - setGenerationJobsInFlight, - recordGenerationJob, -} from '~/app/metrics'; import { GenerationPublicationFencedError, JobCreationSupersededError, JobPredecessorMismatchError, + GenerationAdmissionDeniedError, isPendingActionStale, isPendingActionExpired, PAUSE_PERSISTENCE_TIMEOUT_ERROR, PROVIDER_DRAIN_TIMEOUT_MS, STEER_QUEUE_MAX_DEPTH, } from './interfaces/IJobStore'; +import { + recordGenerationStreamEarlyBufferOverflow, + recordGenerationStreamResumePendingEvents, + recordGenerationStreamSubscription, + recordGenerationStreamAttachment, + recordGenerationStreamRecovery, + setGenerationJobsInFlight, + recordGenerationJob, +} from '~/app/metrics'; import { isRecoveredSteerPayload, RecoveredSteerPayloadMismatchError } from './SteerRecovery'; import { projectTerminalEvent } from './terminalProjection'; import { assertJobStoreV2 } from './jobStoreCapabilities'; @@ -143,6 +144,7 @@ const IDEMPOTENCY_TTL_SECONDS = 25 * 60 * 60; const LEGACY_IDEMPOTENCY_TTL_SECONDS = 26 * 60 * 60; const OAUTH_TOOL_CALL_PREFIX = `oauth${Constants.mcp_delimiter}`; const SHUTDOWN_SUBSCRIBER_ERROR = 'Server is shutting down'; +const ACCOUNT_DELETION_JOB_ERROR = 'Generation stopped because the account is being deleted'; const SHUTDOWN_JOB_ERROR = 'Generation interrupted because its server shut down'; const SHUTTING_DOWN_ERROR = 'Generation job manager is shutting down'; /** Internal transport signal: durable terminal state exists, but its DONE @@ -886,6 +888,8 @@ type DeferredDelivery = * ``` */ class GenerationJobManagerClass { + private admissionGuard?: (userId: string) => Promise; + /** Job metadata + content state storage - swappable for Redis, etc. */ private jobStore: IJobStoreV2; /** Guarded human-review lifecycle (pause / resolve / expire) over the store. */ @@ -1142,6 +1146,15 @@ class GenerationJobManagerClass { this.approvalExpiredHandler = handler; } + /** + * Installs the account admission check. `createJob` runs it after the job is committed, so an + * account deletion either fences first and refuses this job, or starts later and finds it in its + * active-generation drain. The caller supplies the check, keeping the stream package storage-free. + */ + setAdmissionGuard(guard?: (userId: string) => Promise): void { + this.admissionGuard = guard; + } + /** Installs a durable, generation-fenced terminal lifecycle adapter. */ setTerminalHostActionHandler(handler?: TerminalHostActionHandler): void { this.terminalHostActionHandler = handler; @@ -2731,17 +2744,31 @@ class GenerationJobManagerClass { } throw new Error('Generation job was replaced during initialization'); } - if (this.shuttingDown) { + let admissionDenied = false; + let admissionFailure: unknown; + if (!this.shuttingDown && this.admissionGuard != null) { + try { + admissionDenied = !(await this.admissionGuard(userId)); + } catch (error) { + admissionFailure = error; + } + } + const admissionRefused = admissionDenied || admissionFailure != null; + if (this.shuttingDown || admissionRefused) { await this.processDurableReplacementReceipts(streamId, jobData, conversationId).catch( (handoffError) => { logger.error( - `[GenerationJobManager] Failed to hand off predecessor during shutdown ${streamId}:`, + `[GenerationJobManager] Failed to hand off predecessor during refused admission ${streamId}:`, handoffError, ); return false; }, ); - await this.completeJob(streamId, SHUTDOWN_JOB_ERROR, jobData.createdAt); + await this.completeJob( + streamId, + admissionRefused ? ACCOUNT_DELETION_JOB_ERROR : SHUTDOWN_JOB_ERROR, + jobData.createdAt, + ); if (jobData.providerExecutionId) { await this.markProviderExecutionDrained( streamId, @@ -2754,7 +2781,10 @@ class GenerationJobManagerClass { replacedLocal.abortController.abort(); this.retireStoppedPredecessorRuntime(streamId, jobData.createdAt); } - throw new Error(SHUTTING_DOWN_ERROR); + if (admissionFailure != null) { + throw admissionFailure; + } + throw admissionDenied ? new GenerationAdmissionDeniedError() : new Error(SHUTTING_DOWN_ERROR); } if (!(await this.processDurableReplacementReceipts(streamId, jobData, conversationId))) { diff --git a/packages/api/src/stream/__tests__/admissionGuard.spec.ts b/packages/api/src/stream/__tests__/admissionGuard.spec.ts new file mode 100644 index 00000000000..20f0bcd5a81 --- /dev/null +++ b/packages/api/src/stream/__tests__/admissionGuard.spec.ts @@ -0,0 +1,64 @@ +import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport'; +import { GenerationAdmissionDeniedError } from '~/stream/interfaces/IJobStore'; +import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore'; +import { GenerationJobManagerClass } from '~/stream/GenerationJobManager'; +import { stopUserGenerations } from '~/user/generations'; + +describe('GenerationJobManager admission guard', () => { + let manager: GenerationJobManagerClass; + let store: InMemoryJobStore; + + beforeEach(() => { + store = new InMemoryJobStore({ ttlAfterComplete: 60000 }); + manager = new GenerationJobManagerClass(); + manager.configure({ jobStore: store, eventTransport: new InMemoryEventTransport() }); + manager.initialize(); + }); + + afterEach(async () => { + await manager.destroy(); + }); + + it('refuses a job admitted while the account deletion has begun, leaving nothing running', async () => { + manager.setAdmissionGuard(async () => false); + + await expect(manager.createJob('denied', 'user-1', 'denied')).rejects.toBeInstanceOf( + GenerationAdmissionDeniedError, + ); + await expect(manager.createJob('denied-2', 'user-1', 'denied-2')).rejects.toMatchObject({ + code: 'ACCOUNT_DELETION_IN_PROGRESS', + }); + expect((await store.getJob('denied'))?.status).not.toBe('running'); + }); + + it('retires the committed job and propagates the failure when the guard rejects', async () => { + manager.setAdmissionGuard(async () => { + throw new Error('lookup down'); + }); + + await expect(manager.createJob('failed', 'user-1', 'failed')).rejects.toThrow('lookup down'); + expect((await store.getJob('failed'))?.status).not.toBe('running'); + }); + + it('admits jobs while the account is active', async () => { + manager.setAdmissionGuard(async () => true); + + await expect(manager.createJob('ok', 'user-1', 'ok')).resolves.toMatchObject({ + streamId: 'ok', + }); + }); + + it('stops a job admitted before the deletion fence through the drain', async () => { + let active = true; + manager.setAdmissionGuard(async () => active); + await manager.createJob('before', 'user-1', 'before'); + + active = false; + await stopUserGenerations('user-1', undefined, manager); + + expect((await store.getJob('before'))?.status).not.toBe('running'); + await expect(manager.createJob('after', 'user-1', 'after')).rejects.toBeInstanceOf( + GenerationAdmissionDeniedError, + ); + }); +}); diff --git a/packages/api/src/stream/index.ts b/packages/api/src/stream/index.ts index c26c310b7c1..eb7e4eefd45 100644 --- a/packages/api/src/stream/index.ts +++ b/packages/api/src/stream/index.ts @@ -31,6 +31,7 @@ export type { export { GenerationPublicationFencedError, JobPredecessorMismatchError, + GenerationAdmissionDeniedError, isPendingActionExpired, isPendingActionStale, } from './interfaces/IJobStore'; diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.ts index d1609332290..3b7f7e48f28 100644 --- a/packages/api/src/stream/interfaces/IJobStore.ts +++ b/packages/api/src/stream/interfaces/IJobStore.ts @@ -460,6 +460,17 @@ export class JobPredecessorMismatchError extends Error { } } +/** Thrown by `createJob` when the owner's account deletion began before the job could be admitted. */ +export class GenerationAdmissionDeniedError extends Error { + readonly code = 'ACCOUNT_DELETION_IN_PROGRESS'; + readonly status = 409; + + constructor() { + super('Account deletion is in progress'); + this.name = 'GenerationAdmissionDeniedError'; + } +} + export type JobMetadataPatch = Partial< Pick< SerializableJobData, From 8223c5bc37d6c032718a573a20c292642cd9e175 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:26:55 +0200 Subject: [PATCH 10/23] fix: Enforce Account Deletion Admission Once in createJob and Narrow the Erasure File Type --- .../__tests__/request.resumeMetadata.spec.js | 26 ++++++------------- api/server/controllers/agents/request.js | 15 ++--------- packages/api/src/user/resolver.ts | 8 +++++- 3 files changed, 17 insertions(+), 32 deletions(-) diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js index 1d0790a95aa..8b9e2196e11 100644 --- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js +++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js @@ -1100,7 +1100,12 @@ describe('ResumableAgentController resume metadata', () => { }); it('rejects any session fenced after authentication before generation execution starts', async () => { - mockIsAgentTriggerPrincipalActive.mockResolvedValue(false); + mockGenerationJobManager.createJob.mockRejectedValueOnce( + Object.assign(new Error('Account deletion is in progress'), { + code: 'ACCOUNT_DELETION_IN_PROGRESS', + status: 409, + }), + ); const req = { user: { id: 'user-123' }, body: { @@ -1118,10 +1123,6 @@ describe('ResumableAgentController resume metadata', () => { await AgentController(req, res, jest.fn(), initializeClient, null); expect(mockGenerationJobManager.createJob).toHaveBeenCalledTimes(1); - expect(mockIsAgentTriggerPrincipalActive).toHaveBeenCalledWith('user-123'); - expect(mockGenerationJobManager.createJob.mock.invocationCallOrder[0]).toBeLessThan( - mockIsAgentTriggerPrincipalActive.mock.invocationCallOrder[0], - ); expect(mockGenerationJobManager.beginProviderExecution).not.toHaveBeenCalled(); expect(res.status).toHaveBeenCalledWith(409); expect(res.json).toHaveBeenCalledWith({ @@ -1132,15 +1133,8 @@ describe('ResumableAgentController resume metadata', () => { }); expect(initializeClient).not.toHaveBeenCalled(); expect(mockAcceptAgentStartupTelemetry).not.toHaveBeenCalled(); - expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith( - 'conversation-123', - JSON.stringify({ - status: 409, - code: 'ACCOUNT_DELETION_IN_PROGRESS', - error: 'Account deletion is in progress', - }), - 1000, - ); + /** createJob retired the refused job itself, so the controller has none to complete. */ + expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled(); expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith( 'user-123', 'trigger-request', @@ -1239,10 +1233,6 @@ describe('ResumableAgentController resume metadata', () => { await AgentController(req, res, jest.fn(), initializeClient, null); - expect(mockIsAgentTriggerPrincipalActive).toHaveBeenCalledWith('user-123'); - expect(mockIsAgentTriggerPrincipalActive.mock.invocationCallOrder[0]).toBeLessThan( - mockGenerationJobManager.beginProviderExecution.mock.invocationCallOrder[0], - ); expect(initializeClient).not.toHaveBeenCalled(); expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith( 'conversation-123', diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.js index 51fa58b5333..f6cc1694ee8 100644 --- a/api/server/controllers/agents/request.js +++ b/api/server/controllers/agents/request.js @@ -100,7 +100,6 @@ const { claimAgentEventActorSuspension, settleAgentEventActorSuspension, stampConvoLastResponse, - isAgentTriggerPrincipalActive, isSubagentOwnerAdmissible, appendConvoMessageReference, } = require('~/models'); @@ -1755,18 +1754,8 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit req.turnStartedAt = jobCreatedAt; providerExecutionId = job.metadata?.providerExecutionId; - /** Authentication can precede a slow admission path. Recheck the durable - * account-deletion fence after the job is committed but before execution - * starts. This ordering closes both sides of the race for ordinary and - * trigger-scoped sessions: a fence that wins first rejects this run; a - * fence that starts after this read must observe the already-created job - * in account deletion's active-generation drain. */ - if (!(await isAgentTriggerPrincipalActive(userId))) { - throw Object.assign(new Error('Account deletion is in progress'), { - code: 'ACCOUNT_DELETION_IN_PROGRESS', - status: 409, - }); - } + /** Account-deletion admission is enforced inside `createJob`, after the job is committed, so a + * fence that wins first refuses this run and one that starts later finds the job in its drain. */ if (req._agentEventBindingParentConversationId != null) { /** The generation job is the durable marker that a deletion on another replica * can abort. Recheck only after that marker exists: either the deletion fence diff --git a/packages/api/src/user/resolver.ts b/packages/api/src/user/resolver.ts index 81d1d106b2f..b282f4c61de 100644 --- a/packages/api/src/user/resolver.ts +++ b/packages/api/src/user/resolver.ts @@ -12,6 +12,12 @@ type ErasureBaseDeps = Omit< | 'disconnectMCPServer' >; +/** The stored-file fields deletion reads, without the document members of the persisted model. */ +export type ErasureFile = Pick< + IMongoFile, + 'file_id' | 'filepath' | 'filename' | 'source' | 'type' | 'bytes' | 'embedded' | 'temp_file_id' +>; + export interface ErasureResolverDeps { /** Data-layer methods used by the cascade. */ db: ErasureBaseDeps; @@ -24,7 +30,7 @@ export interface ErasureResolverDeps { }) => Promise<{ endpoints?: { agents?: { checkpointer?: TCheckpointerConfig } } } | undefined>; processDeleteRequest: (params: { req: Record; - files: IMongoFile[]; + files: ErasureFile[]; }) => Promise; invalidateCachedTools: (params: { userId: string; serverName: string }) => Promise; getMCPManager: () => From 4600f30d2ac83c19064c79e1341ddd072c5394a6 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:44:52 +0200 Subject: [PATCH 11/23] fix: Install the Admission Guard in Every Entrypoint and Purge Refused Jobs --- api/server/experimental.js | 1 + packages/api/src/agents/remote/lifecycle.ts | 2 ++ packages/api/src/stream/GenerationJobManager.ts | 14 +++++++++++++- .../src/stream/__tests__/admissionGuard.spec.ts | 14 ++++++++++++-- 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/api/server/experimental.js b/api/server/experimental.js index c444acd04ad..1e168f404dc 100644 --- a/api/server/experimental.js +++ b/api/server/experimental.js @@ -367,6 +367,7 @@ if (cluster.isMaster) { // but an already-fired scheduled generation can still reach HITL here. Settle // its durable run when the generic approval runtime expires it. GenerationJobManager.setApprovalExpiredHandler(recordExpiredScheduleApproval); + GenerationJobManager.setAdmissionGuard(agentEventMethods.isAgentTriggerPrincipalActive); GenerationJobManager.setTerminalHostActionHandler( createAgentEventTerminalHandler(agentEventMethods, { resumeDetachedAction: resumeAgentEventDetachedAction, diff --git a/packages/api/src/agents/remote/lifecycle.ts b/packages/api/src/agents/remote/lifecycle.ts index b80aabbdb45..9457ee25348 100644 --- a/packages/api/src/agents/remote/lifecycle.ts +++ b/packages/api/src/agents/remote/lifecycle.ts @@ -203,6 +203,8 @@ export async function enrollAgentExecution( ): Promise { const { runId, userId, conversationId, agentId, protocol, isPrincipalActive } = params; const job = await deps.manager.createJob(runId, userId, conversationId, { + /** The principal check below is this path's single post-commit admission read. */ + skipAdmissionGuard: true, initialMetadata: { agent_id: agentId, endpoint: protocol, diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts index 3e4ccaa331a..c88929d12fe 100644 --- a/packages/api/src/stream/GenerationJobManager.ts +++ b/packages/api/src/stream/GenerationJobManager.ts @@ -664,6 +664,8 @@ export type TerminalHostActionHandler = ( ) => void | Promise; export interface CreateGenerationJobOptions { + /** For callers that run their own post-commit account admission check (remote agent runs). */ + skipAdmissionGuard?: boolean; startupTelemetry?: AgentStartupTelemetry; initialMetadata?: Partial; /** A terminal steer being handed off as this new normal turn. The store @@ -2746,7 +2748,7 @@ class GenerationJobManagerClass { } let admissionDenied = false; let admissionFailure: unknown; - if (!this.shuttingDown && this.admissionGuard != null) { + if (!this.shuttingDown && this.admissionGuard != null && options.skipAdmissionGuard !== true) { try { admissionDenied = !(await this.admissionGuard(userId)); } catch (error) { @@ -2781,6 +2783,16 @@ class GenerationJobManagerClass { replacedLocal.abortController.abort(); this.retireStoppedPredecessorRuntime(streamId, jobData.createdAt); } + if (admissionRefused) { + /** A refused job never ran, and its metadata holds the user's message; the erasure that + * refused it will not list it again, so purge it instead of retaining the error job. */ + await this.jobStore.deleteJob(streamId, jobData.createdAt).catch((purgeError) => { + logger.error( + `[GenerationJobManager] Failed to purge refused generation ${streamId}:`, + purgeError, + ); + }); + } if (admissionFailure != null) { throw admissionFailure; } diff --git a/packages/api/src/stream/__tests__/admissionGuard.spec.ts b/packages/api/src/stream/__tests__/admissionGuard.spec.ts index 20f0bcd5a81..b442d8a24fc 100644 --- a/packages/api/src/stream/__tests__/admissionGuard.spec.ts +++ b/packages/api/src/stream/__tests__/admissionGuard.spec.ts @@ -28,7 +28,17 @@ describe('GenerationJobManager admission guard', () => { await expect(manager.createJob('denied-2', 'user-1', 'denied-2')).rejects.toMatchObject({ code: 'ACCOUNT_DELETION_IN_PROGRESS', }); - expect((await store.getJob('denied'))?.status).not.toBe('running'); + expect(await store.getJob('denied')).toBeNull(); + }); + + it('skips the guard for callers that run their own admission check', async () => { + const guard = jest.fn(async () => false); + manager.setAdmissionGuard(guard); + + await expect( + manager.createJob('own-check', 'user-1', 'own-check', { skipAdmissionGuard: true }), + ).resolves.toMatchObject({ streamId: 'own-check' }); + expect(guard).not.toHaveBeenCalled(); }); it('retires the committed job and propagates the failure when the guard rejects', async () => { @@ -37,7 +47,7 @@ describe('GenerationJobManager admission guard', () => { }); await expect(manager.createJob('failed', 'user-1', 'failed')).rejects.toThrow('lookup down'); - expect((await store.getJob('failed'))?.status).not.toBe('running'); + expect(await store.getJob('failed')).toBeNull(); }); it('admits jobs while the account is active', async () => { From fb560c1e1ddb2129b0cf108d5180a7fff68425e0 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:12:24 +0200 Subject: [PATCH 12/23] fix: Guarantee Refused Job Purge and Sanitize Admission Failures --- .../api/src/stream/GenerationJobManager.ts | 50 ++++++++++--------- .../stream/__tests__/admissionGuard.spec.ts | 16 ++++-- packages/api/src/stream/index.ts | 1 + .../api/src/stream/interfaces/IJobStore.ts | 11 ++++ 4 files changed, 52 insertions(+), 26 deletions(-) diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts index c88929d12fe..bf921a8b5da 100644 --- a/packages/api/src/stream/GenerationJobManager.ts +++ b/packages/api/src/stream/GenerationJobManager.ts @@ -49,6 +49,7 @@ import { JobCreationSupersededError, JobPredecessorMismatchError, GenerationAdmissionDeniedError, + GenerationAdmissionUnavailableError, isPendingActionStale, isPendingActionExpired, PAUSE_PERSISTENCE_TIMEOUT_ERROR, @@ -2766,35 +2767,38 @@ class GenerationJobManagerClass { return false; }, ); - await this.completeJob( - streamId, - admissionRefused ? ACCOUNT_DELETION_JOB_ERROR : SHUTDOWN_JOB_ERROR, - jobData.createdAt, - ); - if (jobData.providerExecutionId) { - await this.markProviderExecutionDrained( + try { + await this.completeJob( streamId, + admissionRefused ? ACCOUNT_DELETION_JOB_ERROR : SHUTDOWN_JOB_ERROR, jobData.createdAt, - jobData.providerExecutionId, ); - } - const replacedLocal = this.runtimeState.get(streamId); - if (replacedLocal != null && replacedLocal.createdAt < jobData.createdAt) { - replacedLocal.abortController.abort(); - this.retireStoppedPredecessorRuntime(streamId, jobData.createdAt); - } - if (admissionRefused) { - /** A refused job never ran, and its metadata holds the user's message; the erasure that - * refused it will not list it again, so purge it instead of retaining the error job. */ - await this.jobStore.deleteJob(streamId, jobData.createdAt).catch((purgeError) => { - logger.error( - `[GenerationJobManager] Failed to purge refused generation ${streamId}:`, - purgeError, + if (jobData.providerExecutionId) { + await this.markProviderExecutionDrained( + streamId, + jobData.createdAt, + jobData.providerExecutionId, ); - }); + } + const replacedLocal = this.runtimeState.get(streamId); + if (replacedLocal != null && replacedLocal.createdAt < jobData.createdAt) { + replacedLocal.abortController.abort(); + this.retireStoppedPredecessorRuntime(streamId, jobData.createdAt); + } + } finally { + if (admissionRefused) { + /** A refused job never ran, and its metadata holds the user's message; the erasure that + * refused it will not list it again, so purge it even when retiring it failed. A purge + * failure propagates so the caller fails instead of reporting a clean refusal. */ + await this.jobStore.deleteJob(streamId, jobData.createdAt); + } } if (admissionFailure != null) { - throw admissionFailure; + logger.error( + `[GenerationJobManager] Admission check failed for ${streamId}:`, + admissionFailure, + ); + throw new GenerationAdmissionUnavailableError(); } throw admissionDenied ? new GenerationAdmissionDeniedError() : new Error(SHUTTING_DOWN_ERROR); } diff --git a/packages/api/src/stream/__tests__/admissionGuard.spec.ts b/packages/api/src/stream/__tests__/admissionGuard.spec.ts index b442d8a24fc..6261a96c97b 100644 --- a/packages/api/src/stream/__tests__/admissionGuard.spec.ts +++ b/packages/api/src/stream/__tests__/admissionGuard.spec.ts @@ -41,15 +41,25 @@ describe('GenerationJobManager admission guard', () => { expect(guard).not.toHaveBeenCalled(); }); - it('retires the committed job and propagates the failure when the guard rejects', async () => { + it('retires the committed job and refuses with a sanitized code when the guard rejects', async () => { manager.setAdmissionGuard(async () => { - throw new Error('lookup down'); + throw new Error('mongodb://internal-host:27017 lookup down'); }); - await expect(manager.createJob('failed', 'user-1', 'failed')).rejects.toThrow('lookup down'); + await expect(manager.createJob('failed', 'user-1', 'failed')).rejects.toMatchObject({ + code: 'GENERATION_ADMISSION_UNAVAILABLE', + message: 'Generation admission could not be verified', + }); expect(await store.getJob('failed')).toBeNull(); }); + it('fails the request when a refused job cannot be purged', async () => { + manager.setAdmissionGuard(async () => false); + jest.spyOn(store, 'deleteJob').mockRejectedValueOnce(new Error('store down')); + + await expect(manager.createJob('stuck', 'user-1', 'stuck')).rejects.toThrow('store down'); + }); + it('admits jobs while the account is active', async () => { manager.setAdmissionGuard(async () => true); diff --git a/packages/api/src/stream/index.ts b/packages/api/src/stream/index.ts index eb7e4eefd45..b54ffdcb0da 100644 --- a/packages/api/src/stream/index.ts +++ b/packages/api/src/stream/index.ts @@ -32,6 +32,7 @@ export { GenerationPublicationFencedError, JobPredecessorMismatchError, GenerationAdmissionDeniedError, + GenerationAdmissionUnavailableError, isPendingActionExpired, isPendingActionStale, } from './interfaces/IJobStore'; diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.ts index 3b7f7e48f28..f9e040252c0 100644 --- a/packages/api/src/stream/interfaces/IJobStore.ts +++ b/packages/api/src/stream/interfaces/IJobStore.ts @@ -471,6 +471,17 @@ export class GenerationAdmissionDeniedError extends Error { } } +/** Thrown by `createJob` when the account admission check itself failed; carries no lookup diagnostics. */ +export class GenerationAdmissionUnavailableError extends Error { + readonly code = 'GENERATION_ADMISSION_UNAVAILABLE'; + readonly status = 503; + + constructor() { + super('Generation admission could not be verified'); + this.name = 'GenerationAdmissionUnavailableError'; + } +} + export type JobMetadataPatch = Partial< Pick< SerializableJobData, From 5965bbe89f0ce0a61e6b878cb695acb651e89aef Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:45:35 +0200 Subject: [PATCH 13/23] fix: Route Remote Admission Through createJob and Keep Refusal Cleanup Sanitized --- .../api/src/agents/remote/lifecycle.spec.ts | 9 +-- packages/api/src/agents/remote/lifecycle.ts | 45 +++++++-------- .../api/src/stream/GenerationJobManager.ts | 56 +++++++++++-------- .../stream/__tests__/admissionGuard.spec.ts | 14 +++-- 4 files changed, 65 insertions(+), 59 deletions(-) diff --git a/packages/api/src/agents/remote/lifecycle.spec.ts b/packages/api/src/agents/remote/lifecycle.spec.ts index ea8400d9767..cad6969e8ed 100644 --- a/packages/api/src/agents/remote/lifecycle.spec.ts +++ b/packages/api/src/agents/remote/lifecycle.spec.ts @@ -87,13 +87,10 @@ describe('Agent execution enrollment', () => { expect(beginProviderExecution).not.toHaveBeenCalled(); await expect(manager.getCleanupBlockingJobIdsForUser('user-1')).resolves.toEqual([]); - await expect(manager.getJobStore().getJob('chatcmpl-deleting')).resolves.toMatchObject({ - status: 'error', - providerDrained: true, - }); + await expect(manager.getJobStore().getJob('chatcmpl-deleting')).resolves.toBeNull(); }); - it('preserves principal-check infrastructure failures after retiring the run', async () => { + it('reports principal-check infrastructure failures with a sanitized error after retiring the run', async () => { const infrastructureError = new Error('principal store unavailable'); await expect( @@ -104,7 +101,7 @@ describe('Agent execution enrollment', () => { }, { manager }, ), - ).rejects.toBe(infrastructureError); + ).rejects.toMatchObject({ code: 'GENERATION_ADMISSION_UNAVAILABLE' }); await expect(manager.getCleanupBlockingJobIdsForUser('user-1')).resolves.toEqual([]); }); diff --git a/packages/api/src/agents/remote/lifecycle.ts b/packages/api/src/agents/remote/lifecycle.ts index 9457ee25348..3191c753351 100644 --- a/packages/api/src/agents/remote/lifecycle.ts +++ b/packages/api/src/agents/remote/lifecycle.ts @@ -1,6 +1,6 @@ import type { GenerationJobManagerClass } from '~/stream'; import type { GenerationJob } from '~/types/stream'; -import { GenerationJobManager } from '~/stream'; +import { GenerationJobManager, GenerationAdmissionDeniedError } from '~/stream'; const ACCOUNT_DELETION_ERROR = 'Account deletion is in progress'; const REMOTE_EXECUTION_ERROR = 'Remote agent execution failed'; @@ -190,40 +190,33 @@ export class AgentExecutionEnrollment { } } -async function retireRejectedEnrollment( - manager: GenerationJobManagerClass, - job: GenerationJob, -): Promise { - await manager.completeJob(job.streamId, ACCOUNT_DELETION_ERROR, job.createdAt); -} - export async function enrollAgentExecution( params: AgentExecutionEnrollmentParams, deps: AgentExecutionEnrollmentDeps = { manager: GenerationJobManager }, ): Promise { const { runId, userId, conversationId, agentId, protocol, isPrincipalActive } = params; - const job = await deps.manager.createJob(runId, userId, conversationId, { - /** The principal check below is this path's single post-commit admission read. */ - skipAdmissionGuard: true, - initialMetadata: { - agent_id: agentId, - endpoint: protocol, - model: agentId, - responseMessageId: runId, - }, - }); - - let active = false; + let job: GenerationJob; try { - active = await isPrincipalActive(userId); + /** The principal predicate replaces the default guard, so admission is one post-commit read + * and a refused run takes the manager's retire-and-purge path. */ + job = await deps.manager.createJob(runId, userId, conversationId, { + admissionCheck: isPrincipalActive, + initialMetadata: { + agent_id: agentId, + endpoint: protocol, + model: agentId, + responseMessageId: runId, + }, + }); } catch (error) { - await retireRejectedEnrollment(deps.manager, job).catch(() => undefined); + if (error instanceof GenerationAdmissionDeniedError) { + throw new AgentExecutionAdmissionError( + ACCOUNT_DELETION_ERROR, + 'ACCOUNT_DELETION_IN_PROGRESS', + ); + } throw error; } - if (!active) { - await retireRejectedEnrollment(deps.manager, job); - throw new AgentExecutionAdmissionError(ACCOUNT_DELETION_ERROR, 'ACCOUNT_DELETION_IN_PROGRESS'); - } return new AgentExecutionEnrollment(deps.manager, job); } diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts index bf921a8b5da..a9bfc4d1cc5 100644 --- a/packages/api/src/stream/GenerationJobManager.ts +++ b/packages/api/src/stream/GenerationJobManager.ts @@ -665,8 +665,8 @@ export type TerminalHostActionHandler = ( ) => void | Promise; export interface CreateGenerationJobOptions { - /** For callers that run their own post-commit account admission check (remote agent runs). */ - skipAdmissionGuard?: boolean; + /** Replaces the installed admission guard for this job, for callers with a stricter predicate. */ + admissionCheck?: (userId: string) => Promise; startupTelemetry?: AgentStartupTelemetry; initialMetadata?: Partial; /** A terminal steer being handed off as this new normal turn. The store @@ -2749,24 +2749,27 @@ class GenerationJobManagerClass { } let admissionDenied = false; let admissionFailure: unknown; - if (!this.shuttingDown && this.admissionGuard != null && options.skipAdmissionGuard !== true) { + const admissionCheck = options.admissionCheck ?? this.admissionGuard; + if (!this.shuttingDown && admissionCheck != null) { try { - admissionDenied = !(await this.admissionGuard(userId)); + admissionDenied = !(await admissionCheck(userId)); } catch (error) { admissionFailure = error; } } const admissionRefused = admissionDenied || admissionFailure != null; if (this.shuttingDown || admissionRefused) { - await this.processDurableReplacementReceipts(streamId, jobData, conversationId).catch( - (handoffError) => { - logger.error( - `[GenerationJobManager] Failed to hand off predecessor during refused admission ${streamId}:`, - handoffError, - ); - return false; - }, - ); + const handoffConfirmed = await this.processDurableReplacementReceipts( + streamId, + jobData, + conversationId, + ).catch((handoffError) => { + logger.error( + `[GenerationJobManager] Failed to hand off predecessor during refused admission ${streamId}:`, + handoffError, + ); + return false; + }); try { await this.completeJob( streamId, @@ -2786,18 +2789,27 @@ class GenerationJobManagerClass { this.retireStoppedPredecessorRuntime(streamId, jobData.createdAt); } } finally { - if (admissionRefused) { + if (admissionRefused && handoffConfirmed) { /** A refused job never ran, and its metadata holds the user's message; the erasure that - * refused it will not list it again, so purge it even when retiring it failed. A purge - * failure propagates so the caller fails instead of reporting a clean refusal. */ - await this.jobStore.deleteJob(streamId, jobData.createdAt); + * refused it will not list it again, so purge it even when retiring it failed. It stays + * until the displaced predecessor is confirmed drained, because the replacement receipt + * is the only durable link to that provider. A purge failure fails the request. */ + await this.jobStore.deleteJob(streamId, jobData.createdAt).catch((purgeError) => { + logger.error( + `[GenerationJobManager] Failed to purge refused generation ${streamId}:`, + purgeError, + ); + throw new GenerationAdmissionUnavailableError(); + }); } } - if (admissionFailure != null) { - logger.error( - `[GenerationJobManager] Admission check failed for ${streamId}:`, - admissionFailure, - ); + if (admissionFailure != null || (admissionRefused && !handoffConfirmed)) { + if (admissionFailure != null) { + logger.error( + `[GenerationJobManager] Admission check failed for ${streamId}:`, + admissionFailure, + ); + } throw new GenerationAdmissionUnavailableError(); } throw admissionDenied ? new GenerationAdmissionDeniedError() : new Error(SHUTTING_DOWN_ERROR); diff --git a/packages/api/src/stream/__tests__/admissionGuard.spec.ts b/packages/api/src/stream/__tests__/admissionGuard.spec.ts index 6261a96c97b..d9199b9c24b 100644 --- a/packages/api/src/stream/__tests__/admissionGuard.spec.ts +++ b/packages/api/src/stream/__tests__/admissionGuard.spec.ts @@ -31,14 +31,15 @@ describe('GenerationJobManager admission guard', () => { expect(await store.getJob('denied')).toBeNull(); }); - it('skips the guard for callers that run their own admission check', async () => { - const guard = jest.fn(async () => false); + it('uses a per-job admission check in place of the installed guard', async () => { + const guard = jest.fn(async () => true); manager.setAdmissionGuard(guard); await expect( - manager.createJob('own-check', 'user-1', 'own-check', { skipAdmissionGuard: true }), - ).resolves.toMatchObject({ streamId: 'own-check' }); + manager.createJob('own-check', 'user-1', 'own-check', { admissionCheck: async () => false }), + ).rejects.toBeInstanceOf(GenerationAdmissionDeniedError); expect(guard).not.toHaveBeenCalled(); + expect(await store.getJob('own-check')).toBeNull(); }); it('retires the committed job and refuses with a sanitized code when the guard rejects', async () => { @@ -57,7 +58,10 @@ describe('GenerationJobManager admission guard', () => { manager.setAdmissionGuard(async () => false); jest.spyOn(store, 'deleteJob').mockRejectedValueOnce(new Error('store down')); - await expect(manager.createJob('stuck', 'user-1', 'stuck')).rejects.toThrow('store down'); + await expect(manager.createJob('stuck', 'user-1', 'stuck')).rejects.toMatchObject({ + code: 'GENERATION_ADMISSION_UNAVAILABLE', + message: 'Generation admission could not be verified', + }); }); it('admits jobs while the account is active', async () => { From 30de716324c7a292d717a3302259c5de14c68c76 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:23:59 +0200 Subject: [PATCH 14/23] fix: Log Admission Cleanup Failures With Safe Error Metadata --- packages/api/src/stream/GenerationJobManager.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts index a9bfc4d1cc5..2736a241234 100644 --- a/packages/api/src/stream/GenerationJobManager.ts +++ b/packages/api/src/stream/GenerationJobManager.ts @@ -68,6 +68,7 @@ import { import { isRecoveredSteerPayload, RecoveredSteerPayloadMismatchError } from './SteerRecovery'; import { projectTerminalEvent } from './terminalProjection'; import { assertJobStoreV2 } from './jobStoreCapabilities'; +import { getSafeErrorMetadata } from '~/utils/errors'; /** * Tombstone budget per generation. Twice the queue depth so a full queue's @@ -2766,7 +2767,7 @@ class GenerationJobManagerClass { ).catch((handoffError) => { logger.error( `[GenerationJobManager] Failed to hand off predecessor during refused admission ${streamId}:`, - handoffError, + getSafeErrorMetadata(handoffError), ); return false; }); @@ -2797,7 +2798,7 @@ class GenerationJobManagerClass { await this.jobStore.deleteJob(streamId, jobData.createdAt).catch((purgeError) => { logger.error( `[GenerationJobManager] Failed to purge refused generation ${streamId}:`, - purgeError, + getSafeErrorMetadata(purgeError), ); throw new GenerationAdmissionUnavailableError(); }); @@ -2807,7 +2808,7 @@ class GenerationJobManagerClass { if (admissionFailure != null) { logger.error( `[GenerationJobManager] Admission check failed for ${streamId}:`, - admissionFailure, + getSafeErrorMetadata(admissionFailure), ); } throw new GenerationAdmissionUnavailableError(); From 4cdf30cbaf7c3343f9cf716c9d340bc0337ac6e8 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:01:15 +0200 Subject: [PATCH 15/23] refactor: Check Account Deletion Before Creating a Generation and Drain Twice --- .../api/src/stream/GenerationJobManager.ts | 103 +++++++----------- .../stream/__tests__/admissionGuard.spec.ts | 47 ++++---- packages/api/src/user/generations.spec.ts | 24 +++- packages/api/src/user/generations.ts | 75 ++++++++----- 4 files changed, 130 insertions(+), 119 deletions(-) diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts index 2736a241234..c88f6de06e3 100644 --- a/packages/api/src/stream/GenerationJobManager.ts +++ b/packages/api/src/stream/GenerationJobManager.ts @@ -146,7 +146,6 @@ const IDEMPOTENCY_TTL_SECONDS = 25 * 60 * 60; const LEGACY_IDEMPOTENCY_TTL_SECONDS = 26 * 60 * 60; const OAUTH_TOOL_CALL_PREFIX = `oauth${Constants.mcp_delimiter}`; const SHUTDOWN_SUBSCRIBER_ERROR = 'Server is shutting down'; -const ACCOUNT_DELETION_JOB_ERROR = 'Generation stopped because the account is being deleted'; const SHUTDOWN_JOB_ERROR = 'Generation interrupted because its server shut down'; const SHUTTING_DOWN_ERROR = 'Generation job manager is shutting down'; /** Internal transport signal: durable terminal state exists, but its DONE @@ -1151,9 +1150,8 @@ class GenerationJobManagerClass { } /** - * Installs the account admission check. `createJob` runs it after the job is committed, so an - * account deletion either fences first and refuses this job, or starts later and finds it in its - * active-generation drain. The caller supplies the check, keeping the stream package storage-free. + * Installs the account admission check, which `createJob` runs before it creates any durable + * state. The caller supplies it, keeping the stream package storage-free. */ setAdmissionGuard(guard?: (userId: string) => Promise): void { this.admissionGuard = guard; @@ -2557,6 +2555,23 @@ class GenerationJobManagerClass { throw new Error('Invalid active generation predecessor policy'); } + const admissionCheck = options.admissionCheck ?? this.admissionGuard; + if (admissionCheck != null) { + let admitted: boolean; + try { + admitted = await admissionCheck(userId); + } catch (error) { + logger.error( + `[GenerationJobManager] Admission check failed for ${streamId}:`, + getSafeErrorMetadata(error), + ); + throw new GenerationAdmissionUnavailableError(); + } + if (!admitted) { + throw new GenerationAdmissionDeniedError(); + } + } + const tenantId = getTenantId(); const safeTenantId = tenantId && tenantId !== SYSTEM_TENANT_ID ? tenantId : undefined; const creationAttemptId = randomUUID(); @@ -2748,72 +2763,30 @@ class GenerationJobManagerClass { } throw new Error('Generation job was replaced during initialization'); } - let admissionDenied = false; - let admissionFailure: unknown; - const admissionCheck = options.admissionCheck ?? this.admissionGuard; - if (!this.shuttingDown && admissionCheck != null) { - try { - admissionDenied = !(await admissionCheck(userId)); - } catch (error) { - admissionFailure = error; - } - } - const admissionRefused = admissionDenied || admissionFailure != null; - if (this.shuttingDown || admissionRefused) { - const handoffConfirmed = await this.processDurableReplacementReceipts( - streamId, - jobData, - conversationId, - ).catch((handoffError) => { - logger.error( - `[GenerationJobManager] Failed to hand off predecessor during refused admission ${streamId}:`, - getSafeErrorMetadata(handoffError), - ); - return false; - }); - try { - await this.completeJob( + if (this.shuttingDown) { + await this.processDurableReplacementReceipts(streamId, jobData, conversationId).catch( + (handoffError) => { + logger.error( + `[GenerationJobManager] Failed to hand off predecessor during shutdown ${streamId}:`, + handoffError, + ); + return false; + }, + ); + await this.completeJob(streamId, SHUTDOWN_JOB_ERROR, jobData.createdAt); + if (jobData.providerExecutionId) { + await this.markProviderExecutionDrained( streamId, - admissionRefused ? ACCOUNT_DELETION_JOB_ERROR : SHUTDOWN_JOB_ERROR, jobData.createdAt, + jobData.providerExecutionId, ); - if (jobData.providerExecutionId) { - await this.markProviderExecutionDrained( - streamId, - jobData.createdAt, - jobData.providerExecutionId, - ); - } - const replacedLocal = this.runtimeState.get(streamId); - if (replacedLocal != null && replacedLocal.createdAt < jobData.createdAt) { - replacedLocal.abortController.abort(); - this.retireStoppedPredecessorRuntime(streamId, jobData.createdAt); - } - } finally { - if (admissionRefused && handoffConfirmed) { - /** A refused job never ran, and its metadata holds the user's message; the erasure that - * refused it will not list it again, so purge it even when retiring it failed. It stays - * until the displaced predecessor is confirmed drained, because the replacement receipt - * is the only durable link to that provider. A purge failure fails the request. */ - await this.jobStore.deleteJob(streamId, jobData.createdAt).catch((purgeError) => { - logger.error( - `[GenerationJobManager] Failed to purge refused generation ${streamId}:`, - getSafeErrorMetadata(purgeError), - ); - throw new GenerationAdmissionUnavailableError(); - }); - } } - if (admissionFailure != null || (admissionRefused && !handoffConfirmed)) { - if (admissionFailure != null) { - logger.error( - `[GenerationJobManager] Admission check failed for ${streamId}:`, - getSafeErrorMetadata(admissionFailure), - ); - } - throw new GenerationAdmissionUnavailableError(); + const replacedLocal = this.runtimeState.get(streamId); + if (replacedLocal != null && replacedLocal.createdAt < jobData.createdAt) { + replacedLocal.abortController.abort(); + this.retireStoppedPredecessorRuntime(streamId, jobData.createdAt); } - throw admissionDenied ? new GenerationAdmissionDeniedError() : new Error(SHUTTING_DOWN_ERROR); + throw new Error(SHUTTING_DOWN_ERROR); } if (!(await this.processDurableReplacementReceipts(streamId, jobData, conversationId))) { diff --git a/packages/api/src/stream/__tests__/admissionGuard.spec.ts b/packages/api/src/stream/__tests__/admissionGuard.spec.ts index d9199b9c24b..713714ea3bc 100644 --- a/packages/api/src/stream/__tests__/admissionGuard.spec.ts +++ b/packages/api/src/stream/__tests__/admissionGuard.spec.ts @@ -1,5 +1,8 @@ +import { + GenerationAdmissionDeniedError, + GenerationAdmissionUnavailableError, +} from '~/stream/interfaces/IJobStore'; import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport'; -import { GenerationAdmissionDeniedError } from '~/stream/interfaces/IJobStore'; import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore'; import { GenerationJobManagerClass } from '~/stream/GenerationJobManager'; import { stopUserGenerations } from '~/user/generations'; @@ -19,49 +22,43 @@ describe('GenerationJobManager admission guard', () => { await manager.destroy(); }); - it('refuses a job admitted while the account deletion has begun, leaving nothing running', async () => { + it('refuses before creating any job state while the account deletion has begun', async () => { manager.setAdmissionGuard(async () => false); await expect(manager.createJob('denied', 'user-1', 'denied')).rejects.toBeInstanceOf( GenerationAdmissionDeniedError, ); - await expect(manager.createJob('denied-2', 'user-1', 'denied-2')).rejects.toMatchObject({ + await expect(manager.createJob('denied', 'user-1', 'denied')).rejects.toMatchObject({ code: 'ACCOUNT_DELETION_IN_PROGRESS', + status: 409, + message: 'Account deletion is in progress', }); expect(await store.getJob('denied')).toBeNull(); }); - it('uses a per-job admission check in place of the installed guard', async () => { - const guard = jest.fn(async () => true); - manager.setAdmissionGuard(guard); - - await expect( - manager.createJob('own-check', 'user-1', 'own-check', { admissionCheck: async () => false }), - ).rejects.toBeInstanceOf(GenerationAdmissionDeniedError); - expect(guard).not.toHaveBeenCalled(); - expect(await store.getJob('own-check')).toBeNull(); - }); - - it('retires the committed job and refuses with a sanitized code when the guard rejects', async () => { + it('refuses with a sanitized code when the guard fails and creates nothing', async () => { manager.setAdmissionGuard(async () => { throw new Error('mongodb://internal-host:27017 lookup down'); }); - await expect(manager.createJob('failed', 'user-1', 'failed')).rejects.toMatchObject({ + const error = await manager.createJob('failed', 'user-1', 'failed').catch((e) => e); + + expect(error).toBeInstanceOf(GenerationAdmissionUnavailableError); + expect(error).toMatchObject({ code: 'GENERATION_ADMISSION_UNAVAILABLE', message: 'Generation admission could not be verified', }); expect(await store.getJob('failed')).toBeNull(); }); - it('fails the request when a refused job cannot be purged', async () => { - manager.setAdmissionGuard(async () => false); - jest.spyOn(store, 'deleteJob').mockRejectedValueOnce(new Error('store down')); + it('uses a per-job admission check in place of the installed guard', async () => { + const guard = jest.fn(async () => true); + manager.setAdmissionGuard(guard); - await expect(manager.createJob('stuck', 'user-1', 'stuck')).rejects.toMatchObject({ - code: 'GENERATION_ADMISSION_UNAVAILABLE', - message: 'Generation admission could not be verified', - }); + await expect( + manager.createJob('own-check', 'user-1', 'own-check', { admissionCheck: async () => false }), + ).rejects.toBeInstanceOf(GenerationAdmissionDeniedError); + expect(guard).not.toHaveBeenCalled(); }); it('admits jobs while the account is active', async () => { @@ -72,13 +69,13 @@ describe('GenerationJobManager admission guard', () => { }); }); - it('stops a job admitted before the deletion fence through the drain', async () => { + it('stops a job admitted before the fence and refuses one after it', async () => { let active = true; manager.setAdmissionGuard(async () => active); await manager.createJob('before', 'user-1', 'before'); active = false; - await stopUserGenerations('user-1', undefined, manager); + await stopUserGenerations('user-1', undefined, manager, 0); expect((await store.getJob('before'))?.status).not.toBe('running'); await expect(manager.createJob('after', 'user-1', 'after')).rejects.toBeInstanceOf( diff --git a/packages/api/src/user/generations.spec.ts b/packages/api/src/user/generations.spec.ts index da2681a3fee..fa88b0ec995 100644 --- a/packages/api/src/user/generations.spec.ts +++ b/packages/api/src/user/generations.spec.ts @@ -29,7 +29,7 @@ describe('stopUserGenerations', () => { const manager = new FakeJobManager(); await expect( - stopUserGenerations('u1', undefined, manager as unknown as UserGenerationControl), + stopUserGenerations('u1', undefined, manager as unknown as UserGenerationControl, 0), ).resolves.toBeUndefined(); expect(manager.aborted).toEqual(['s1']); @@ -43,7 +43,27 @@ describe('stopUserGenerations', () => { }; await expect( - stopUserGenerations('u1', undefined, manager as unknown as UserGenerationControl), + stopUserGenerations('u1', undefined, manager as unknown as UserGenerationControl, 0), ).rejects.toThrow('could not be confirmed stopped'); }); + + it('catches a job created right after the first drain on the second drain', async () => { + const manager = new FakeJobManager(); + const listed = manager.getAccountCleanupJobIdsForUser.bind(manager); + let calls = 0; + manager.getAccountCleanupJobIdsForUser = async () => { + calls += 1; + if (calls === 2) { + (manager as unknown as { jobs: Map }).jobs.set('late', { + createdAt: 9, + metadata: { userId: 'u1' }, + }); + } + return listed(); + }; + + await stopUserGenerations('u1', undefined, manager as unknown as UserGenerationControl, 0); + + expect(manager.aborted).toEqual(['s1', 'late']); + }); }); diff --git a/packages/api/src/user/generations.ts b/packages/api/src/user/generations.ts index 01a60af631a..7b9d7596891 100644 --- a/packages/api/src/user/generations.ts +++ b/packages/api/src/user/generations.ts @@ -15,39 +15,60 @@ export interface UserGenerationControl { ) => Promise; } +/** + * Time given to a generation that passed its admission check just before the account's deletion + * marker was set to finish creating its job. Admission and job creation are consecutive awaits in + * `createJob`, so a few seconds comfortably exceeds the window; the second drain then sees it. + */ +export const ADMISSION_GRACE_MS = 2000; + /** * Stops the user's in-flight and retained generations and waits for their final persistence, - * so nothing is written for the account after its data is erased. Throws if a stop cannot be - * confirmed, leaving the caller to keep the account and retry. + * so nothing is written for the account after its data is erased. The deletion marker must + * already be set, which makes `createJob` refuse new runs. A run admitted just before the marker + * is caught by a second drain after the grace period. Throws if a stop cannot be confirmed, + * leaving the caller to keep the account and retry. */ export async function stopUserGenerations( userId: string, tenantId: string | undefined, control: UserGenerationControl, + graceMs: number = ADMISSION_GRACE_MS, ): Promise { - const streamIds = await control.getAccountCleanupJobIdsForUser(userId, tenantId); - const jobs = await Promise.all( - streamIds.map(async (streamId) => ({ - streamId, - job: await control.getCleanupJob(streamId), - })), - ); - const owned = jobs.filter( - ({ job }) => - job?.metadata?.userId === userId && - (!job.metadata.tenantId || job.metadata.tenantId === tenantId), - ); - const results = await Promise.all( - owned.map(({ streamId, job }) => - control.abortJob(streamId, { expectedCreatedAt: job!.createdAt, awaitProviderDrain: true }), - ), - ); - if (results.some((result) => !isStopConfirmed(result))) { - throw new Error('Agent generations could not be confirmed stopped'); - } - await Promise.all( - owned.map(({ streamId, job }) => - waitForGenerationPersistence(streamId, job!.createdAt, (id) => control.getCleanupJob(id)), - ), - ); + const seen = new Set(); + const drain = async (): Promise => { + const streamIds = await control.getAccountCleanupJobIdsForUser(userId, tenantId); + const jobs = await Promise.all( + streamIds.map(async (streamId) => ({ + streamId, + job: await control.getCleanupJob(streamId), + })), + ); + const owned = jobs.filter( + ({ streamId, job }) => + job?.metadata?.userId === userId && + (!job.metadata.tenantId || job.metadata.tenantId === tenantId) && + !seen.has(`${streamId}:${job.createdAt}`), + ); + for (const { streamId, job } of owned) { + seen.add(`${streamId}:${job!.createdAt}`); + } + const results = await Promise.all( + owned.map(({ streamId, job }) => + control.abortJob(streamId, { expectedCreatedAt: job!.createdAt, awaitProviderDrain: true }), + ), + ); + if (results.some((result) => !isStopConfirmed(result))) { + throw new Error('Agent generations could not be confirmed stopped'); + } + await Promise.all( + owned.map(({ streamId, job }) => + waitForGenerationPersistence(streamId, job!.createdAt, (id) => control.getCleanupJob(id)), + ), + ); + }; + + await drain(); + await new Promise((resolve) => setTimeout(resolve, graceMs)); + await drain(); } From 3d78be3f4f77b78e74cb82fc20cb6c55b1784736 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:26:09 +0200 Subject: [PATCH 16/23] refactor: Order Admission Against the Deletion Marker Without a Grace Period --- .../api/src/stream/GenerationJobManager.ts | 67 ++++++++++++----- .../stream/__tests__/admissionGuard.spec.ts | 28 +++++-- packages/api/src/user/generations.spec.ts | 24 +----- packages/api/src/user/generations.ts | 73 +++++++------------ 4 files changed, 97 insertions(+), 95 deletions(-) diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts index c88f6de06e3..b7ff60b743d 100644 --- a/packages/api/src/stream/GenerationJobManager.ts +++ b/packages/api/src/stream/GenerationJobManager.ts @@ -146,6 +146,7 @@ const IDEMPOTENCY_TTL_SECONDS = 25 * 60 * 60; const LEGACY_IDEMPOTENCY_TTL_SECONDS = 26 * 60 * 60; const OAUTH_TOOL_CALL_PREFIX = `oauth${Constants.mcp_delimiter}`; const SHUTDOWN_SUBSCRIBER_ERROR = 'Server is shutting down'; +const ACCOUNT_DELETION_JOB_ERROR = 'Generation stopped because the account is being deleted'; const SHUTDOWN_JOB_ERROR = 'Generation interrupted because its server shut down'; const SHUTTING_DOWN_ERROR = 'Generation job manager is shutting down'; /** Internal transport signal: durable terminal state exists, but its DONE @@ -1157,6 +1158,30 @@ class GenerationJobManagerClass { this.admissionGuard = guard; } + private async assertAdmitted( + streamId: string, + userId: string, + options: CreateGenerationJobOptions, + ): Promise { + const admissionCheck = options.admissionCheck ?? this.admissionGuard; + if (admissionCheck == null) { + return; + } + let admitted: boolean; + try { + admitted = await admissionCheck(userId); + } catch (error) { + logger.error( + `[GenerationJobManager] Admission check failed for ${streamId}:`, + getSafeErrorMetadata(error), + ); + throw new GenerationAdmissionUnavailableError(); + } + if (!admitted) { + throw new GenerationAdmissionDeniedError(); + } + } + /** Installs a durable, generation-fenced terminal lifecycle adapter. */ setTerminalHostActionHandler(handler?: TerminalHostActionHandler): void { this.terminalHostActionHandler = handler; @@ -2555,22 +2580,8 @@ class GenerationJobManagerClass { throw new Error('Invalid active generation predecessor policy'); } - const admissionCheck = options.admissionCheck ?? this.admissionGuard; - if (admissionCheck != null) { - let admitted: boolean; - try { - admitted = await admissionCheck(userId); - } catch (error) { - logger.error( - `[GenerationJobManager] Admission check failed for ${streamId}:`, - getSafeErrorMetadata(error), - ); - throw new GenerationAdmissionUnavailableError(); - } - if (!admitted) { - throw new GenerationAdmissionDeniedError(); - } - } + /** Refuse before any durable state exists; the common case leaves nothing to clean up. */ + await this.assertAdmitted(streamId, userId, options); const tenantId = getTenantId(); const safeTenantId = tenantId && tenantId !== SYSTEM_TENANT_ID ? tenantId : undefined; @@ -2763,7 +2774,21 @@ class GenerationJobManagerClass { } throw new Error('Generation job was replaced during initialization'); } - if (this.shuttingDown) { + /** + * Race-free admission: account deletion sets its marker first and then lists this user's jobs. + * A job committed before that listing is stopped by the drain; a job committed after it + * checks the marker here, after the marker was set, and retires itself through the normal + * terminal path. The marker is therefore always seen by one side or the other. + */ + let postCommitRefusal: Error | undefined; + if (!this.shuttingDown) { + try { + await this.assertAdmitted(streamId, userId, options); + } catch (error) { + postCommitRefusal = error as Error; + } + } + if (this.shuttingDown || postCommitRefusal != null) { await this.processDurableReplacementReceipts(streamId, jobData, conversationId).catch( (handoffError) => { logger.error( @@ -2773,7 +2798,11 @@ class GenerationJobManagerClass { return false; }, ); - await this.completeJob(streamId, SHUTDOWN_JOB_ERROR, jobData.createdAt); + await this.completeJob( + streamId, + postCommitRefusal != null ? ACCOUNT_DELETION_JOB_ERROR : SHUTDOWN_JOB_ERROR, + jobData.createdAt, + ); if (jobData.providerExecutionId) { await this.markProviderExecutionDrained( streamId, @@ -2786,7 +2815,7 @@ class GenerationJobManagerClass { replacedLocal.abortController.abort(); this.retireStoppedPredecessorRuntime(streamId, jobData.createdAt); } - throw new Error(SHUTTING_DOWN_ERROR); + throw postCommitRefusal ?? new Error(SHUTTING_DOWN_ERROR); } if (!(await this.processDurableReplacementReceipts(streamId, jobData, conversationId))) { diff --git a/packages/api/src/stream/__tests__/admissionGuard.spec.ts b/packages/api/src/stream/__tests__/admissionGuard.spec.ts index 713714ea3bc..7b27340d632 100644 --- a/packages/api/src/stream/__tests__/admissionGuard.spec.ts +++ b/packages/api/src/stream/__tests__/admissionGuard.spec.ts @@ -69,17 +69,29 @@ describe('GenerationJobManager admission guard', () => { }); }); - it('stops a job admitted before the fence and refuses one after it', async () => { - let active = true; - manager.setAdmissionGuard(async () => active); + it('retires a job committed after the marker was set and fails with the stable code', async () => { + let marker = false; + manager.setAdmissionGuard(async () => !marker); + const created = jest.spyOn(store, 'createJob'); + created.mockImplementationOnce(async (...args) => { + marker = true; + return InMemoryJobStore.prototype.createJob.apply(store, args); + }); + + await expect(manager.createJob('late', 'user-1', 'late')).rejects.toBeInstanceOf( + GenerationAdmissionDeniedError, + ); + expect((await store.getJob('late'))?.status).toBe('error'); + }); + + it('stops a job committed before the deletion listing', async () => { + let marker = false; + manager.setAdmissionGuard(async () => !marker); await manager.createJob('before', 'user-1', 'before'); - active = false; - await stopUserGenerations('user-1', undefined, manager, 0); + marker = true; + await stopUserGenerations('user-1', undefined, manager); expect((await store.getJob('before'))?.status).not.toBe('running'); - await expect(manager.createJob('after', 'user-1', 'after')).rejects.toBeInstanceOf( - GenerationAdmissionDeniedError, - ); }); }); diff --git a/packages/api/src/user/generations.spec.ts b/packages/api/src/user/generations.spec.ts index fa88b0ec995..da2681a3fee 100644 --- a/packages/api/src/user/generations.spec.ts +++ b/packages/api/src/user/generations.spec.ts @@ -29,7 +29,7 @@ describe('stopUserGenerations', () => { const manager = new FakeJobManager(); await expect( - stopUserGenerations('u1', undefined, manager as unknown as UserGenerationControl, 0), + stopUserGenerations('u1', undefined, manager as unknown as UserGenerationControl), ).resolves.toBeUndefined(); expect(manager.aborted).toEqual(['s1']); @@ -43,27 +43,7 @@ describe('stopUserGenerations', () => { }; await expect( - stopUserGenerations('u1', undefined, manager as unknown as UserGenerationControl, 0), + stopUserGenerations('u1', undefined, manager as unknown as UserGenerationControl), ).rejects.toThrow('could not be confirmed stopped'); }); - - it('catches a job created right after the first drain on the second drain', async () => { - const manager = new FakeJobManager(); - const listed = manager.getAccountCleanupJobIdsForUser.bind(manager); - let calls = 0; - manager.getAccountCleanupJobIdsForUser = async () => { - calls += 1; - if (calls === 2) { - (manager as unknown as { jobs: Map }).jobs.set('late', { - createdAt: 9, - metadata: { userId: 'u1' }, - }); - } - return listed(); - }; - - await stopUserGenerations('u1', undefined, manager as unknown as UserGenerationControl, 0); - - expect(manager.aborted).toEqual(['s1', 'late']); - }); }); diff --git a/packages/api/src/user/generations.ts b/packages/api/src/user/generations.ts index 7b9d7596891..f57de9a79d5 100644 --- a/packages/api/src/user/generations.ts +++ b/packages/api/src/user/generations.ts @@ -15,60 +15,41 @@ export interface UserGenerationControl { ) => Promise; } -/** - * Time given to a generation that passed its admission check just before the account's deletion - * marker was set to finish creating its job. Admission and job creation are consecutive awaits in - * `createJob`, so a few seconds comfortably exceeds the window; the second drain then sees it. - */ -export const ADMISSION_GRACE_MS = 2000; - /** * Stops the user's in-flight and retained generations and waits for their final persistence, * so nothing is written for the account after its data is erased. The deletion marker must - * already be set, which makes `createJob` refuse new runs. A run admitted just before the marker - * is caught by a second drain after the grace period. Throws if a stop cannot be confirmed, + * already be set: a run committed before this listing is stopped here, and one committed after it + * checks the marker and retires itself in `createJob`. Throws if a stop cannot be confirmed, * leaving the caller to keep the account and retry. */ export async function stopUserGenerations( userId: string, tenantId: string | undefined, control: UserGenerationControl, - graceMs: number = ADMISSION_GRACE_MS, ): Promise { - const seen = new Set(); - const drain = async (): Promise => { - const streamIds = await control.getAccountCleanupJobIdsForUser(userId, tenantId); - const jobs = await Promise.all( - streamIds.map(async (streamId) => ({ - streamId, - job: await control.getCleanupJob(streamId), - })), - ); - const owned = jobs.filter( - ({ streamId, job }) => - job?.metadata?.userId === userId && - (!job.metadata.tenantId || job.metadata.tenantId === tenantId) && - !seen.has(`${streamId}:${job.createdAt}`), - ); - for (const { streamId, job } of owned) { - seen.add(`${streamId}:${job!.createdAt}`); - } - const results = await Promise.all( - owned.map(({ streamId, job }) => - control.abortJob(streamId, { expectedCreatedAt: job!.createdAt, awaitProviderDrain: true }), - ), - ); - if (results.some((result) => !isStopConfirmed(result))) { - throw new Error('Agent generations could not be confirmed stopped'); - } - await Promise.all( - owned.map(({ streamId, job }) => - waitForGenerationPersistence(streamId, job!.createdAt, (id) => control.getCleanupJob(id)), - ), - ); - }; - - await drain(); - await new Promise((resolve) => setTimeout(resolve, graceMs)); - await drain(); + const streamIds = await control.getAccountCleanupJobIdsForUser(userId, tenantId); + const jobs = await Promise.all( + streamIds.map(async (streamId) => ({ + streamId, + job: await control.getCleanupJob(streamId), + })), + ); + const owned = jobs.filter( + ({ job }) => + job?.metadata?.userId === userId && + (!job.metadata.tenantId || job.metadata.tenantId === tenantId), + ); + const results = await Promise.all( + owned.map(({ streamId, job }) => + control.abortJob(streamId, { expectedCreatedAt: job!.createdAt, awaitProviderDrain: true }), + ), + ); + if (results.some((result) => !isStopConfirmed(result))) { + throw new Error('Agent generations could not be confirmed stopped'); + } + await Promise.all( + owned.map(({ streamId, job }) => + waitForGenerationPersistence(streamId, job!.createdAt, (id) => control.getCleanupJob(id)), + ), + ); } From 076beb321158de63a3cc4b4add4370d062f77225 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Wed, 14 Oct 2026 14:47:22 +0200 Subject: [PATCH 17/23] fix: Sanitize Post-Commit Admission Retirement Failures --- .../api/src/stream/GenerationJobManager.ts | 37 ++++++++++++------- .../stream/__tests__/admissionGuard.spec.ts | 15 ++++++++ 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts index b7ff60b743d..48353dbdce0 100644 --- a/packages/api/src/stream/GenerationJobManager.ts +++ b/packages/api/src/stream/GenerationJobManager.ts @@ -2798,22 +2798,33 @@ class GenerationJobManagerClass { return false; }, ); - await this.completeJob( - streamId, - postCommitRefusal != null ? ACCOUNT_DELETION_JOB_ERROR : SHUTDOWN_JOB_ERROR, - jobData.createdAt, - ); - if (jobData.providerExecutionId) { - await this.markProviderExecutionDrained( + try { + await this.completeJob( streamId, + postCommitRefusal != null ? ACCOUNT_DELETION_JOB_ERROR : SHUTDOWN_JOB_ERROR, jobData.createdAt, - jobData.providerExecutionId, ); - } - const replacedLocal = this.runtimeState.get(streamId); - if (replacedLocal != null && replacedLocal.createdAt < jobData.createdAt) { - replacedLocal.abortController.abort(); - this.retireStoppedPredecessorRuntime(streamId, jobData.createdAt); + if (jobData.providerExecutionId) { + await this.markProviderExecutionDrained( + streamId, + jobData.createdAt, + jobData.providerExecutionId, + ); + } + const replacedLocal = this.runtimeState.get(streamId); + if (replacedLocal != null && replacedLocal.createdAt < jobData.createdAt) { + replacedLocal.abortController.abort(); + this.retireStoppedPredecessorRuntime(streamId, jobData.createdAt); + } + } catch (retireError) { + if (postCommitRefusal == null) { + throw retireError; + } + logger.error( + `[GenerationJobManager] Failed to retire refused generation ${streamId}:`, + getSafeErrorMetadata(retireError), + ); + throw new GenerationAdmissionUnavailableError(); } throw postCommitRefusal ?? new Error(SHUTTING_DOWN_ERROR); } diff --git a/packages/api/src/stream/__tests__/admissionGuard.spec.ts b/packages/api/src/stream/__tests__/admissionGuard.spec.ts index 7b27340d632..b3a78b85b64 100644 --- a/packages/api/src/stream/__tests__/admissionGuard.spec.ts +++ b/packages/api/src/stream/__tests__/admissionGuard.spec.ts @@ -84,6 +84,21 @@ describe('GenerationJobManager admission guard', () => { expect((await store.getJob('late'))?.status).toBe('error'); }); + it('maps a retirement failure after a refused commit to the sanitized error', async () => { + let marker = false; + manager.setAdmissionGuard(async () => !marker); + jest.spyOn(store, 'createJob').mockImplementationOnce(async (...args) => { + marker = true; + return InMemoryJobStore.prototype.createJob.apply(store, args); + }); + jest.spyOn(manager, 'completeJob').mockRejectedValueOnce(new Error('redis://internal:6379')); + + await expect(manager.createJob('retire-fail', 'user-1', 'retire-fail')).rejects.toMatchObject({ + code: 'GENERATION_ADMISSION_UNAVAILABLE', + message: 'Generation admission could not be verified', + }); + }); + it('stops a job committed before the deletion listing', async () => { let marker = false; manager.setAdmissionGuard(async () => !marker); From 9dc6298581513b6dff6b33253d79c8706111215d Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:21:18 +0200 Subject: [PATCH 18/23] fix: Wire Shared-Server MCP OAuth Cleanup Into Admin Erasure --- api/server/routes/admin/users.js | 8 ++++++- packages/api/src/user/resolver.spec.ts | 28 +++++++++++++++++++++++ packages/api/src/user/resolver.ts | 31 +++++++++++++++++++++++++- 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/api/server/routes/admin/users.js b/api/server/routes/admin/users.js index 223a7ba8836..528ba9b4ea8 100644 --- a/api/server/routes/admin/users.js +++ b/api/server/routes/admin/users.js @@ -19,7 +19,8 @@ const { } = require('~/server/services/Agents/triggers'); const { processDeleteRequest } = require('~/server/services/Files/process'); const { invalidateCachedTools } = require('~/server/services/Config/getCachedTools'); -const { getMCPManager } = require('~/config'); +const { getMCPManager, getMCPServersRegistry } = require('~/config'); +const { maybeUninstallOAuthMCP } = require('~/server/services/MCP/oauthCleanup'); const subagentThreadTaskStore = require('~/server/services/Endpoints/agents/subagentThreadStore'); const { quiesceUserSchedules, @@ -36,6 +37,11 @@ const resolveErasure = createErasureResolver({ getMCPManager, processDeleteRequest, invalidateCachedTools, + mcpOAuth: { + getRegistry: getMCPServersRegistry, + getUserPrincipals: db.getUserPrincipals, + uninstallOAuthMCP: maybeUninstallOAuthMCP, + }, openCheckpointDeletion, deleteSharedLinks: deleteAllSharedLinksWithCleanup, }); diff --git a/packages/api/src/user/resolver.spec.ts b/packages/api/src/user/resolver.spec.ts index c061b7f0270..1fd56af11d4 100644 --- a/packages/api/src/user/resolver.spec.ts +++ b/packages/api/src/user/resolver.spec.ts @@ -46,6 +46,34 @@ describe('createErasureResolver', () => { ); }); + it('builds the shared-server OAuth hooks for the target account', async () => { + const registry = { + getServerConfig: jest.fn().mockResolvedValue({ url: 'https://srv' }), + resolveAllowlists: jest.fn().mockResolvedValue({}), + }; + const mcpOAuth = { + getRegistry: () => registry, + getUserPrincipals: jest.fn().mockResolvedValue([]), + uninstallOAuthMCP: jest.fn(), + }; + const { deps } = createDeps({ mcpOAuth }); + const { deps: erasure } = await createErasureResolver(deps)(target); + + await erasure.mcpOAuth?.getServerConfig?.('srv'); + await erasure.mcpOAuth?.getUserPrincipals('other-user'); + await erasure.mcpOAuth?.resolveAllowlists('other-user'); + await erasure.mcpOAuth?.fenceAndDisconnectUser?.('other-user', 'srv'); + + expect(registry.getServerConfig).toHaveBeenCalledWith('srv', 'user-1'); + expect(mcpOAuth.getUserPrincipals).toHaveBeenCalledWith({ userId: 'other-user' }); + expect(registry.resolveAllowlists).toHaveBeenCalledWith({ userId: 'other-user' }); + expect(deps.invalidateCachedTools).toHaveBeenCalledWith({ + userId: 'other-user', + serverName: 'srv', + }); + expect(erasure.mcpOAuth?.uninstallOAuthMCP).toBe(mcpOAuth.uninstallOAuthMCP); + }); + it('rejects when the target config cannot be resolved', async () => { const { deps } = createDeps({ getAppConfig: jest.fn().mockRejectedValue(new Error('override store down')), diff --git a/packages/api/src/user/resolver.ts b/packages/api/src/user/resolver.ts index b282f4c61de..c2b4f280b84 100644 --- a/packages/api/src/user/resolver.ts +++ b/packages/api/src/user/resolver.ts @@ -1,7 +1,7 @@ import type { TCheckpointerConfig } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { AdminErasureContext, AdminErasureTarget } from '~/admin/users'; -import type { UserErasureDeps } from './erasure'; +import type { UserErasureDeps, MCPOAuthErasureHooks } from './erasure'; type ErasureBaseDeps = Omit< UserErasureDeps, @@ -10,6 +10,7 @@ type ErasureBaseDeps = Omit< | 'deleteStoredFiles' | 'invalidateMCPServerTools' | 'disconnectMCPServer' + | 'mcpOAuth' >; /** The stored-file fields deletion reads, without the document members of the persisted model. */ @@ -19,6 +20,22 @@ export type ErasureFile = Pick< >; export interface ErasureResolverDeps { + /** Shared-server OAuth cleanup is wired only when the host has an MCP OAuth runtime. */ + mcpOAuth?: { + getRegistry: () => { + getServerConfig: ( + serverName: string, + userId: string, + ) => ReturnType>; + resolveAllowlists: (options: { + userId: string; + }) => ReturnType; + }; + getUserPrincipals: (params: { + userId: string; + }) => ReturnType; + uninstallOAuthMCP: MCPOAuthErasureHooks['uninstallOAuthMCP']; + }; /** Data-layer methods used by the cascade. */ db: ErasureBaseDeps; getAppConfig: (options: { @@ -62,6 +79,7 @@ export function createErasureResolver(deps: ErasureResolverDeps) { idOnTheSource, failClosed: true, }); + const oauth = deps.mcpOAuth; const fileRequest = { user: { ...profile, id, _id: id, role, tenantId }, /** Assistants client setup reads these Express request fields. */ @@ -82,6 +100,17 @@ export function createErasureResolver(deps: ErasureResolverDeps) { disconnectMCPServer: async (serverName) => { await deps.getMCPManager()?.disconnectUserConnection(id, serverName); }, + mcpOAuth: oauth && { + getServerConfig: (serverName) => oauth.getRegistry().getServerConfig(serverName, id), + getUserPrincipals: (candidateId) => oauth.getUserPrincipals({ userId: candidateId }), + resolveAllowlists: (candidateId) => + oauth.getRegistry().resolveAllowlists({ userId: candidateId }), + uninstallOAuthMCP: oauth.uninstallOAuthMCP, + fenceAndDisconnectUser: async (candidateId, serverName) => { + await deps.invalidateCachedTools({ userId: candidateId, serverName }); + await deps.getMCPManager()?.disconnectUserConnection(candidateId, serverName); + }, + }, }, }; }; From 9bbd434a36757dfa765689b31d552eefa7bc262e Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:29:17 +0200 Subject: [PATCH 19/23] fix: Scope Admin Erasure Lookups to the Target Tenant and Await Shared-Link Cleanup --- api/server/routes/admin/users.js | 2 +- packages/api/src/user/resolver.spec.ts | 44 +++++++++++++++++++++++++- packages/api/src/user/resolver.ts | 27 +++++++++++++--- 3 files changed, 66 insertions(+), 7 deletions(-) diff --git a/api/server/routes/admin/users.js b/api/server/routes/admin/users.js index 528ba9b4ea8..895920ff445 100644 --- a/api/server/routes/admin/users.js +++ b/api/server/routes/admin/users.js @@ -43,7 +43,7 @@ const resolveErasure = createErasureResolver({ uninstallOAuthMCP: maybeUninstallOAuthMCP, }, openCheckpointDeletion, - deleteSharedLinks: deleteAllSharedLinksWithCleanup, + deleteSharedLinks: (userId) => deleteAllSharedLinksWithCleanup(userId, { awaitCleanup: true }), }); const requireAdminAccess = requireCapability(SystemCapabilities.ACCESS_ADMIN); diff --git a/packages/api/src/user/resolver.spec.ts b/packages/api/src/user/resolver.spec.ts index 1fd56af11d4..557515cf15b 100644 --- a/packages/api/src/user/resolver.spec.ts +++ b/packages/api/src/user/resolver.spec.ts @@ -1,3 +1,4 @@ +import { tenantStorage } from '@librechat/data-schemas'; import type { IMongoFile } from '@librechat/data-schemas'; import type { ErasureResolverDeps } from './resolver'; import { createErasureResolver } from './resolver'; @@ -71,7 +72,48 @@ describe('createErasureResolver', () => { userId: 'other-user', serverName: 'srv', }); - expect(erasure.mcpOAuth?.uninstallOAuthMCP).toBe(mcpOAuth.uninstallOAuthMCP); + await erasure.mcpOAuth?.uninstallOAuthMCP('other-user', 'mcp_srv', {}); + expect(mcpOAuth.uninstallOAuthMCP).toHaveBeenCalledWith('other-user', 'mcp_srv', {}, undefined); + }); + + it('runs id-keyed lookups in the target tenant, not the calling admin tenant', async () => { + const seen: Record = {}; + const record = (key: string) => async () => { + seen[key] = tenantStorage.getStore()?.tenantId; + return []; + }; + const registry = { + getServerConfig: jest.fn(record('config')), + resolveAllowlists: jest.fn(record('allowlists')), + }; + const { deps } = createDeps({ + db: { + deleteUserAgents: jest.fn(record('agents')), + findMCPServerOAuthTokenUserIds: jest.fn(record('tokens')), + findMCPServerViewerEntries: jest.fn(record('viewers')), + } as unknown as ErasureResolverDeps['db'], + processDeleteRequest: jest.fn(record('files')), + mcpOAuth: { + getRegistry: () => registry, + getUserPrincipals: jest.fn(record('principals')), + uninstallOAuthMCP: jest.fn(record('uninstall')), + }, + } as unknown as Partial); + const { deps: erasure } = await createErasureResolver(deps)(target); + + await tenantStorage.run({ tenantId: 'admin-tenant' }, async () => { + await erasure.deleteUserAgents('user-1'); + await erasure.findMCPServerOAuthTokenUserIds?.('srv'); + await erasure.findMCPServerViewerEntries?.('srv-id'); + await erasure.deleteStoredFiles([]); + await erasure.mcpOAuth?.getServerConfig?.('srv'); + await erasure.mcpOAuth?.getUserPrincipals('other'); + await erasure.mcpOAuth?.resolveAllowlists('other'); + await erasure.mcpOAuth?.uninstallOAuthMCP('other', 'mcp_srv', {}); + }); + + expect(Object.keys(seen)).toHaveLength(8); + expect(Object.values(seen).every((tenant) => tenant === 'tenant-1')).toBe(true); }); it('rejects when the target config cannot be resolved', async () => { diff --git a/packages/api/src/user/resolver.ts b/packages/api/src/user/resolver.ts index c2b4f280b84..12cec79cdd1 100644 --- a/packages/api/src/user/resolver.ts +++ b/packages/api/src/user/resolver.ts @@ -1,3 +1,4 @@ +import { runAsSystem, tenantStorage } from '@librechat/data-schemas'; import type { TCheckpointerConfig } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { AdminErasureContext, AdminErasureTarget } from '~/admin/users'; @@ -80,6 +81,9 @@ export function createErasureResolver(deps: ErasureResolverDeps) { failClosed: true, }); const oauth = deps.mcpOAuth; + /** The calling admin's tenant context must not decide whose rows an id-keyed lookup reads. */ + const inTargetTenant = (operation: () => Promise): Promise => + tenantId ? tenantStorage.run({ tenantId, userId: id }, operation) : runAsSystem(operation); const fileRequest = { user: { ...profile, id, _id: id, role, tenantId }, /** Assistants client setup reads these Express request fields. */ @@ -94,18 +98,31 @@ export function createErasureResolver(deps: ErasureResolverDeps) { ...deps.db, openCheckpointDeletion: deps.openCheckpointDeletion, deleteSharedLinks: deps.deleteSharedLinks, - deleteStoredFiles: (files) => deps.processDeleteRequest({ req: fileRequest, files }), + deleteUserAgents: (userId) => inTargetTenant(async () => deps.db.deleteUserAgents(userId)), + findMCPServerOAuthTokenUserIds: (serverName) => + inTargetTenant(async () => deps.db.findMCPServerOAuthTokenUserIds(serverName)), + findMCPServerViewerEntries: (serverId) => + inTargetTenant(async () => deps.db.findMCPServerViewerEntries(serverId)), + deleteStoredFiles: (files) => + inTargetTenant(async () => deps.processDeleteRequest({ req: fileRequest, files })), invalidateMCPServerTools: (serverName) => deps.invalidateCachedTools({ userId: id, serverName }), disconnectMCPServer: async (serverName) => { await deps.getMCPManager()?.disconnectUserConnection(id, serverName); }, mcpOAuth: oauth && { - getServerConfig: (serverName) => oauth.getRegistry().getServerConfig(serverName, id), - getUserPrincipals: (candidateId) => oauth.getUserPrincipals({ userId: candidateId }), + getServerConfig: (serverName) => + inTargetTenant(async () => oauth.getRegistry().getServerConfig(serverName, id)), + getUserPrincipals: (candidateId) => + inTargetTenant(async () => oauth.getUserPrincipals({ userId: candidateId })), resolveAllowlists: (candidateId) => - oauth.getRegistry().resolveAllowlists({ userId: candidateId }), - uninstallOAuthMCP: oauth.uninstallOAuthMCP, + inTargetTenant(async () => + oauth.getRegistry().resolveAllowlists({ userId: candidateId }), + ), + uninstallOAuthMCP: (candidateId, pluginKey, cleanupConfig, serverConfig) => + inTargetTenant(async () => + oauth.uninstallOAuthMCP(candidateId, pluginKey, cleanupConfig, serverConfig), + ), fenceAndDisconnectUser: async (candidateId, serverName) => { await deps.invalidateCachedTools({ userId: candidateId, serverName }); await deps.getMCPManager()?.disconnectUserConnection(candidateId, serverName); From 717e20297b9be53dd275eaa21b5471f8619f36e3 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:48:46 +0200 Subject: [PATCH 20/23] fix: Resolve the Erasure Target Config Under the Target Tenant --- packages/api/src/user/resolver.spec.ts | 16 ++++++++++++++++ packages/api/src/user/resolver.ts | 20 +++++++++++--------- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/packages/api/src/user/resolver.spec.ts b/packages/api/src/user/resolver.spec.ts index 557515cf15b..778e49c7262 100644 --- a/packages/api/src/user/resolver.spec.ts +++ b/packages/api/src/user/resolver.spec.ts @@ -116,6 +116,22 @@ describe('createErasureResolver', () => { expect(Object.values(seen).every((tenant) => tenant === 'tenant-1')).toBe(true); }); + it('resolves the target config under the target tenant, not the calling admin tenant', async () => { + let seen: string | undefined; + const { deps } = createDeps({ + getAppConfig: jest.fn(async () => { + seen = tenantStorage.getStore()?.tenantId; + return {}; + }), + }); + + await tenantStorage.run({ tenantId: 'admin-tenant' }, () => + createErasureResolver(deps)(target), + ); + + expect(seen).toBe('tenant-1'); + }); + it('rejects when the target config cannot be resolved', async () => { const { deps } = createDeps({ getAppConfig: jest.fn().mockRejectedValue(new Error('override store down')), diff --git a/packages/api/src/user/resolver.ts b/packages/api/src/user/resolver.ts index 12cec79cdd1..afdce9a141a 100644 --- a/packages/api/src/user/resolver.ts +++ b/packages/api/src/user/resolver.ts @@ -72,18 +72,20 @@ export function createErasureResolver(deps: ErasureResolverDeps) { idOnTheSource, profile, }: AdminErasureTarget): Promise { - /** A failed override lookup must abort the deletion, not fall back to a base config that cannot reach the target's storage. */ - const appConfig = await deps.getAppConfig({ - role, - userId: id, - tenantId, - idOnTheSource, - failClosed: true, - }); - const oauth = deps.mcpOAuth; /** The calling admin's tenant context must not decide whose rows an id-keyed lookup reads. */ const inTargetTenant = (operation: () => Promise): Promise => tenantId ? tenantStorage.run({ tenantId, userId: id }, operation) : runAsSystem(operation); + /** A failed override lookup must abort the deletion, not fall back to a base config that cannot reach the target's storage. */ + const appConfig = await inTargetTenant(async () => + deps.getAppConfig({ + role, + userId: id, + tenantId, + idOnTheSource, + failClosed: true, + }), + ); + const oauth = deps.mcpOAuth; const fileRequest = { user: { ...profile, id, _id: id, role, tenantId }, /** Assistants client setup reads these Express request fields. */ From 5118d041e13b931f6c911f824416a99bef276260 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Wed, 14 Oct 2026 20:08:51 +0200 Subject: [PATCH 21/23] fix: Run Admin Erasure OAuth Fencing in the Target Tenant --- packages/api/src/user/resolver.spec.ts | 4 +++- packages/api/src/user/resolver.ts | 9 +++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/api/src/user/resolver.spec.ts b/packages/api/src/user/resolver.spec.ts index 778e49c7262..f37e97e19fc 100644 --- a/packages/api/src/user/resolver.spec.ts +++ b/packages/api/src/user/resolver.spec.ts @@ -93,6 +93,7 @@ describe('createErasureResolver', () => { findMCPServerViewerEntries: jest.fn(record('viewers')), } as unknown as ErasureResolverDeps['db'], processDeleteRequest: jest.fn(record('files')), + invalidateCachedTools: jest.fn(record('fence')), mcpOAuth: { getRegistry: () => registry, getUserPrincipals: jest.fn(record('principals')), @@ -110,9 +111,10 @@ describe('createErasureResolver', () => { await erasure.mcpOAuth?.getUserPrincipals('other'); await erasure.mcpOAuth?.resolveAllowlists('other'); await erasure.mcpOAuth?.uninstallOAuthMCP('other', 'mcp_srv', {}); + await erasure.mcpOAuth?.fenceAndDisconnectUser?.('other', 'srv'); }); - expect(Object.keys(seen)).toHaveLength(8); + expect(Object.keys(seen)).toHaveLength(9); expect(Object.values(seen).every((tenant) => tenant === 'tenant-1')).toBe(true); }); diff --git a/packages/api/src/user/resolver.ts b/packages/api/src/user/resolver.ts index afdce9a141a..d6992b9600b 100644 --- a/packages/api/src/user/resolver.ts +++ b/packages/api/src/user/resolver.ts @@ -125,10 +125,11 @@ export function createErasureResolver(deps: ErasureResolverDeps) { inTargetTenant(async () => oauth.uninstallOAuthMCP(candidateId, pluginKey, cleanupConfig, serverConfig), ), - fenceAndDisconnectUser: async (candidateId, serverName) => { - await deps.invalidateCachedTools({ userId: candidateId, serverName }); - await deps.getMCPManager()?.disconnectUserConnection(candidateId, serverName); - }, + fenceAndDisconnectUser: (candidateId, serverName) => + inTargetTenant(async () => { + await deps.invalidateCachedTools({ userId: candidateId, serverName }); + await deps.getMCPManager()?.disconnectUserConnection(candidateId, serverName); + }), }, }, }; From 294e04dadc5c6d9f961ceefab7bc060b5e021a94 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Wed, 14 Oct 2026 22:17:28 +0200 Subject: [PATCH 22/23] fix: Match the Split OAuth Invalidate and Disconnect Hooks in Admin Erasure --- packages/api/src/user/resolver.spec.ts | 10 ++++++---- packages/api/src/user/resolver.ts | 13 ++++++++----- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/packages/api/src/user/resolver.spec.ts b/packages/api/src/user/resolver.spec.ts index f37e97e19fc..7277ec280dd 100644 --- a/packages/api/src/user/resolver.spec.ts +++ b/packages/api/src/user/resolver.spec.ts @@ -60,10 +60,11 @@ describe('createErasureResolver', () => { const { deps } = createDeps({ mcpOAuth }); const { deps: erasure } = await createErasureResolver(deps)(target); - await erasure.mcpOAuth?.getServerConfig?.('srv'); + await erasure.mcpOAuth?.getServerConfig?.('srv', 'srv-id'); await erasure.mcpOAuth?.getUserPrincipals('other-user'); await erasure.mcpOAuth?.resolveAllowlists('other-user'); - await erasure.mcpOAuth?.fenceAndDisconnectUser?.('other-user', 'srv'); + await erasure.mcpOAuth?.invalidateUserTools?.('other-user', 'srv'); + await erasure.mcpOAuth?.disconnectUser?.('other-user', 'srv'); expect(registry.getServerConfig).toHaveBeenCalledWith('srv', 'user-1'); expect(mcpOAuth.getUserPrincipals).toHaveBeenCalledWith({ userId: 'other-user' }); @@ -107,11 +108,12 @@ describe('createErasureResolver', () => { await erasure.findMCPServerOAuthTokenUserIds?.('srv'); await erasure.findMCPServerViewerEntries?.('srv-id'); await erasure.deleteStoredFiles([]); - await erasure.mcpOAuth?.getServerConfig?.('srv'); + await erasure.mcpOAuth?.getServerConfig?.('srv', 'srv-id'); await erasure.mcpOAuth?.getUserPrincipals('other'); await erasure.mcpOAuth?.resolveAllowlists('other'); await erasure.mcpOAuth?.uninstallOAuthMCP('other', 'mcp_srv', {}); - await erasure.mcpOAuth?.fenceAndDisconnectUser?.('other', 'srv'); + await erasure.mcpOAuth?.invalidateUserTools?.('other', 'srv'); + await erasure.mcpOAuth?.disconnectUser?.('other', 'srv'); }); expect(Object.keys(seen)).toHaveLength(9); diff --git a/packages/api/src/user/resolver.ts b/packages/api/src/user/resolver.ts index d6992b9600b..1fa9c7301ad 100644 --- a/packages/api/src/user/resolver.ts +++ b/packages/api/src/user/resolver.ts @@ -125,11 +125,14 @@ export function createErasureResolver(deps: ErasureResolverDeps) { inTargetTenant(async () => oauth.uninstallOAuthMCP(candidateId, pluginKey, cleanupConfig, serverConfig), ), - fenceAndDisconnectUser: (candidateId, serverName) => - inTargetTenant(async () => { - await deps.invalidateCachedTools({ userId: candidateId, serverName }); - await deps.getMCPManager()?.disconnectUserConnection(candidateId, serverName); - }), + invalidateUserTools: (candidateId, serverName) => + inTargetTenant(async () => + deps.invalidateCachedTools({ userId: candidateId, serverName }), + ), + disconnectUser: (candidateId, serverName) => + inTargetTenant(async () => + deps.getMCPManager()?.disconnectUserConnection(candidateId, serverName), + ), }, }, }; From f0854939801e6a378820f87d45d2dd15db13beb4 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Wed, 14 Oct 2026 22:32:24 +0200 Subject: [PATCH 23/23] refactor: Inject the Tenant Executor Into the Erasure Resolver --- api/server/routes/admin/users.js | 4 +++- packages/api/src/user/resolver.spec.ts | 4 +++- packages/api/src/user/resolver.ts | 5 +++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/api/server/routes/admin/users.js b/api/server/routes/admin/users.js index 895920ff445..efe861248ff 100644 --- a/api/server/routes/admin/users.js +++ b/api/server/routes/admin/users.js @@ -8,7 +8,7 @@ const { createAdminUsersHandlers, revokeUserCodeEnvironmentWorkers, } = require('@librechat/api'); -const { SystemCapabilities } = require('@librechat/data-schemas'); +const { SystemCapabilities, runAsSystem, tenantStorage } = require('@librechat/data-schemas'); const { requireCapability } = require('~/server/middleware/roles/capabilities'); const { requireJwtAuth } = require('~/server/middleware'); const { @@ -32,6 +32,8 @@ const { getAppConfig, invalidateCodeEnvironmentConfigCache } = require('~/server const router = express.Router(); const resolveErasure = createErasureResolver({ + withTenant: (tenantId, operation) => + tenantId ? tenantStorage.run({ tenantId }, operation) : runAsSystem(operation), db, getAppConfig, getMCPManager, diff --git a/packages/api/src/user/resolver.spec.ts b/packages/api/src/user/resolver.spec.ts index 7277ec280dd..054e7b60827 100644 --- a/packages/api/src/user/resolver.spec.ts +++ b/packages/api/src/user/resolver.spec.ts @@ -1,4 +1,4 @@ -import { tenantStorage } from '@librechat/data-schemas'; +import { tenantStorage, runAsSystem } from '@librechat/data-schemas'; import type { IMongoFile } from '@librechat/data-schemas'; import type { ErasureResolverDeps } from './resolver'; import { createErasureResolver } from './resolver'; @@ -15,6 +15,8 @@ function createDeps(overrides: Partial = {}) { getMCPManager: jest.fn(() => ({ disconnectUserConnection })), openCheckpointDeletion: jest.fn(), deleteSharedLinks: jest.fn(), + withTenant: (tenant: string | undefined, operation: () => Promise) => + tenant ? tenantStorage.run({ tenantId: tenant }, operation) : runAsSystem(operation), ...overrides, } as unknown as ErasureResolverDeps; return { deps, disconnectUserConnection }; diff --git a/packages/api/src/user/resolver.ts b/packages/api/src/user/resolver.ts index 1fa9c7301ad..ce32891d105 100644 --- a/packages/api/src/user/resolver.ts +++ b/packages/api/src/user/resolver.ts @@ -1,4 +1,3 @@ -import { runAsSystem, tenantStorage } from '@librechat/data-schemas'; import type { TCheckpointerConfig } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { AdminErasureContext, AdminErasureTarget } from '~/admin/users'; @@ -21,6 +20,8 @@ export type ErasureFile = Pick< >; export interface ErasureResolverDeps { + /** Runs an operation in the given tenant's data scope, or system scope when it has none. */ + withTenant: (tenantId: string | undefined, operation: () => Promise) => Promise; /** Shared-server OAuth cleanup is wired only when the host has an MCP OAuth runtime. */ mcpOAuth?: { getRegistry: () => { @@ -74,7 +75,7 @@ export function createErasureResolver(deps: ErasureResolverDeps) { }: AdminErasureTarget): Promise { /** The calling admin's tenant context must not decide whose rows an id-keyed lookup reads. */ const inTargetTenant = (operation: () => Promise): Promise => - tenantId ? tenantStorage.run({ tenantId, userId: id }, operation) : runAsSystem(operation); + deps.withTenant(tenantId, operation); /** A failed override lookup must abort the deletion, not fall back to a base config that cannot reach the target's storage. */ const appConfig = await inTargetTenant(async () => deps.getAppConfig({