🧹 fix: Run the Full Erasure Cascade When an Admin Deletes a User

api/server/controllers/UserController.js

@@ -15,8 +15,7 @@ const {
   getWebSearchInstallEntries,
   getWebSearchUninstallFields,
   openCheckpointDeletion,
-  isStopConfirmed,
-  waitForGenerationPersistence,
+  stopUserGenerations,
   deleteAllSharedLinksWithCleanup,
   eraseUserData,
   deleteUserMCPServers,
@@ -410,39 +409,7 @@ const deleteUserController = async (req, res) => {
     if (!(await quiesceUserSchedules(user.id, scheduleSuspensionToken))) {
       throw new Error('Scheduled executions could not be confirmed stopped');
     }
-    const activeAgentRuns = await GenerationJobManager.getAccountCleanupJobIdsForUser(
-      user.id,
-      tenantId,
-    );
-    const activeAgentJobs = await Promise.all(
-      activeAgentRuns.map(async (streamId) => ({
-        streamId,
-        job: await GenerationJobManager.getCleanupJob(streamId),
-      })),
-    );
-    const ownedAgentJobs = activeAgentJobs.filter(
-      ({ job }) =>
-        job?.metadata?.userId === user.id &&
-        (!job.metadata.tenantId || job.metadata.tenantId === tenantId),
-    );
-    const stopResults = await Promise.all(
-      ownedAgentJobs.map(({ streamId, job }) =>
-        GenerationJobManager.abortJob(streamId, {
-          expectedCreatedAt: job.createdAt,
-          awaitProviderDrain: true,
-        }),
-      ),
-    );
-    if (stopResults.some((result) => !isStopConfirmed(result))) {
-      throw new Error('Agent generations could not be confirmed stopped');
-    }
-    await Promise.all(
-      ownedAgentJobs.map(({ streamId, job }) =>
-        waitForGenerationPersistence(streamId, job.createdAt, (id) =>
-          GenerationJobManager.getCleanupJob(id),
-        ),
-      ),
-    );
+    await stopUserGenerations(user.id, tenantId, GenerationJobManager);
 
     const appConfig =
       req.config ??

api/server/controllers/__tests__/deleteUser.spec.js

@@ -77,13 +77,9 @@ jest.mock('@librechat/api', () => ({
     abortJob: (...args) => mockAbortJob(...args),
   },
   getOwnedAgentCheckpointScope: jest.fn(() => undefined),
-  isStopConfirmed: jest.fn(
-    (result) => result?.success === true || result?.failureReason === 'already_settled',
-  ),
   deleteOwnedAgentCheckpoints: jest.fn(),
-  waitForGenerationPersistence: jest.requireActual(
-    '../../../../packages/api/src/stream/persistence.ts',
-  ).waitForGenerationPersistence,
+  stopUserGenerations: jest.requireActual('../../../../packages/api/src/user/generations.ts')
+    .stopUserGenerations,
   openCheckpointDeletion: jest.fn(async (userId, tenantId, _root, cfg) => ({
     remember: jest.fn(),
     cleanup: () =>

api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js

@@ -1100,7 +1100,12 @@ describe('ResumableAgentController resume metadata', () => {
   });
 
   it('rejects any session fenced after authentication before generation execution starts', async () => {
-    mockIsAgentTriggerPrincipalActive.mockResolvedValue(false);
+    mockGenerationJobManager.createJob.mockRejectedValueOnce(
+      Object.assign(new Error('Account deletion is in progress'), {
+        code: 'ACCOUNT_DELETION_IN_PROGRESS',
+        status: 409,
+      }),
+    );
     const req = {
       user: { id: 'user-123' },
       body: {
@@ -1118,10 +1123,6 @@ describe('ResumableAgentController resume metadata', () => {
     await AgentController(req, res, jest.fn(), initializeClient, null);
 
     expect(mockGenerationJobManager.createJob).toHaveBeenCalledTimes(1);
-    expect(mockIsAgentTriggerPrincipalActive).toHaveBeenCalledWith('user-123');
-    expect(mockGenerationJobManager.createJob.mock.invocationCallOrder[0]).toBeLessThan(
-      mockIsAgentTriggerPrincipalActive.mock.invocationCallOrder[0],
-    );
     expect(mockGenerationJobManager.beginProviderExecution).not.toHaveBeenCalled();
     expect(res.status).toHaveBeenCalledWith(409);
     expect(res.json).toHaveBeenCalledWith({
@@ -1132,15 +1133,8 @@ describe('ResumableAgentController resume metadata', () => {
     });
     expect(initializeClient).not.toHaveBeenCalled();
     expect(mockAcceptAgentStartupTelemetry).not.toHaveBeenCalled();
-    expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
-      'conversation-123',
-      JSON.stringify({
-        status: 409,
-        code: 'ACCOUNT_DELETION_IN_PROGRESS',
-        error: 'Account deletion is in progress',
-      }),
-      1000,
-    );
+    /** createJob retired the refused job itself, so the controller has none to complete. */
+    expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
     expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith(
       'user-123',
       'trigger-request',
@@ -1239,10 +1233,6 @@ describe('ResumableAgentController resume metadata', () => {
 
     await AgentController(req, res, jest.fn(), initializeClient, null);
 
-    expect(mockIsAgentTriggerPrincipalActive).toHaveBeenCalledWith('user-123');
-    expect(mockIsAgentTriggerPrincipalActive.mock.invocationCallOrder[0]).toBeLessThan(
-      mockGenerationJobManager.beginProviderExecution.mock.invocationCallOrder[0],
-    );
     expect(initializeClient).not.toHaveBeenCalled();
     expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
       'conversation-123',

api/server/controllers/agents/request.js

@@ -100,7 +100,6 @@ const {
   claimAgentEventActorSuspension,
   settleAgentEventActorSuspension,
   stampConvoLastResponse,
-  isAgentTriggerPrincipalActive,
   isSubagentOwnerAdmissible,
   appendConvoMessageReference,
 } = require('~/models');
@@ -1755,18 +1754,8 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
     req.turnStartedAt = jobCreatedAt;
     providerExecutionId = job.metadata?.providerExecutionId;
 
-    /** Authentication can precede a slow admission path. Recheck the durable
-     * account-deletion fence after the job is committed but before execution
-     * starts. This ordering closes both sides of the race for ordinary and
-     * trigger-scoped sessions: a fence that wins first rejects this run; a
-     * fence that starts after this read must observe the already-created job
-     * in account deletion's active-generation drain. */
-    if (!(await isAgentTriggerPrincipalActive(userId))) {
-      throw Object.assign(new Error('Account deletion is in progress'), {
-        code: 'ACCOUNT_DELETION_IN_PROGRESS',
-        status: 409,
-      });
-    }
+    /** Account-deletion admission is enforced inside `createJob`, after the job is committed, so a
+     * fence that wins first refuses this run and one that starts later finds the job in its drain. */
     if (req._agentEventBindingParentConversationId != null) {
       /** The generation job is the durable marker that a deletion on another replica
        * can abort. Recheck only after that marker exists: either the deletion fence

api/server/experimental.js

@@ -367,6 +367,7 @@ if (cluster.isMaster) {
   // but an already-fired scheduled generation can still reach HITL here. Settle
   // its durable run when the generic approval runtime expires it.
   GenerationJobManager.setApprovalExpiredHandler(recordExpiredScheduleApproval);
+  GenerationJobManager.setAdmissionGuard(agentEventMethods.isAgentTriggerPrincipalActive);
   GenerationJobManager.setTerminalHostActionHandler(
     createAgentEventTerminalHandler(agentEventMethods, {
       resumeDetachedAction: resumeAgentEventDetachedAction,

api/server/index.js

@@ -139,6 +139,7 @@ const configureGenerationStreams = () => {
     cleanupOnComplete: !isEnabled(process.env.STREAM_KEEP_COMPLETED_JOBS),
   });
   GenerationJobManager.setApprovalExpiredHandler(recordExpiredScheduleApproval);
+  GenerationJobManager.setAdmissionGuard(agentEventMethods.isAgentTriggerPrincipalActive);
   GenerationJobManager.setTerminalHostActionHandler(
     createAgentEventTerminalHandler(agentEventMethods, {
       resumeDetachedAction: resumeAgentEventDetachedAction,

api/server/routes/admin/users.js

@@ -1,7 +1,14 @@
 const express = require('express');
 const mongoose = require('mongoose');
-const { createAdminUsersHandlers, revokeUserCodeEnvironmentWorkers } = require('@librechat/api');
-const { SystemCapabilities } = require('@librechat/data-schemas');
+const {
+  GenerationJobManager,
+  openCheckpointDeletion,
+  deleteAllSharedLinksWithCleanup,
+  createErasureResolver,
+  createAdminUsersHandlers,
+  revokeUserCodeEnvironmentWorkers,
+} = require('@librechat/api');
+const { SystemCapabilities, runAsSystem, tenantStorage } = require('@librechat/data-schemas');
 const { requireCapability } = require('~/server/middleware/roles/capabilities');
 const { requireJwtAuth } = require('~/server/middleware');
 const {
@@ -10,11 +17,37 @@ const {
   cancelAgentTriggerUserPurge,
   purgeAgentTriggerDeliveriesForUser,
 } = require('~/server/services/Agents/triggers');
+const { processDeleteRequest } = require('~/server/services/Files/process');
+const { invalidateCachedTools } = require('~/server/services/Config/getCachedTools');
+const { getMCPManager, getMCPServersRegistry } = require('~/config');
+const { maybeUninstallOAuthMCP } = require('~/server/services/MCP/oauthCleanup');
+const subagentThreadTaskStore = require('~/server/services/Endpoints/agents/subagentThreadStore');
+const {
+  quiesceUserSchedules,
+  restoreUserSchedulesFromDeletion,
+} = require('~/server/services/Schedules');
 const db = require('~/models');
 const { getAppConfig, invalidateCodeEnvironmentConfigCache } = require('~/server/services/Config');
 
 const router = express.Router();
 
+const resolveErasure = createErasureResolver({
+  withTenant: (tenantId, operation) =>
+    tenantId ? tenantStorage.run({ tenantId }, operation) : runAsSystem(operation),
+  db,
+  getAppConfig,
+  getMCPManager,
+  processDeleteRequest,
+  invalidateCachedTools,
+  mcpOAuth: {
+    getRegistry: getMCPServersRegistry,
+    getUserPrincipals: db.getUserPrincipals,
+    uninstallOAuthMCP: maybeUninstallOAuthMCP,
+  },
+  openCheckpointDeletion,
+  deleteSharedLinks: (userId) => deleteAllSharedLinksWithCleanup(userId, { awaitCleanup: true }),
+});
+
 const requireAdminAccess = requireCapability(SystemCapabilities.ACCESS_ADMIN);
 const requireReadUsers = requireCapability(SystemCapabilities.READ_USERS);
 // const requireManageUsers = requireCapability(SystemCapabilities.MANAGE_USERS);
@@ -40,7 +73,12 @@ const handlers = createAdminUsersHandlers({
   deleteConfig: db.deleteConfig,
   deleteAclEntries: db.deleteAclEntries,
   deletePasskeysByUser: db.deletePasskeysByUser,
-  deleteTokens: db.deleteTokens,
+  resolveErasure,
+  cancelSubagentTasks: (userId, tenantId) =>
+    subagentThreadTaskStore.cancelAndDrainForOwner(userId, tenantId),
+  quiesceUserSchedules,
+  restoreUserSchedulesFromDeletion,
+  generationJobs: GenerationJobManager,
 });
 
 router.use(requireJwtAuth, requireAdminAccess);

packages/api/src/admin/users.spec.ts

@@ -5,6 +5,11 @@ import type { Response } from 'express';
 import type { ServerRequest } from '~/types/http';
 import type { AdminUsersDeps } from './users';
 import { createAdminUsersHandlers } from './users';
+import { eraseUserData } from '~/user/erasure';
+
+jest.mock('~/user/erasure', () => ({
+  eraseUserData: jest.fn().mockResolvedValue(undefined),
+}));
 
 jest.mock('@librechat/data-schemas', () => ({
   ...jest.requireActual('@librechat/data-schemas'),
@@ -68,12 +73,24 @@ function createDeps(overrides: Partial<AdminUsersDeps> = {}): AdminUsersDeps {
     deleteConfig: jest.fn().mockResolvedValue(null),
     deleteAclEntries: jest.fn().mockResolvedValue(undefined),
     deletePasskeysByUser: jest.fn().mockResolvedValue(0),
-    deleteTokens: jest.fn().mockResolvedValue({ deletedCount: 0 }),
+    resolveErasure: jest.fn().mockResolvedValue({ deps: {} }),
+    cancelSubagentTasks: jest.fn().mockResolvedValue(undefined),
+    quiesceUserSchedules: jest.fn().mockResolvedValue(true),
+    restoreUserSchedulesFromDeletion: jest.fn().mockResolvedValue(undefined),
+    generationJobs: {
+      getAccountCleanupJobIdsForUser: jest.fn().mockResolvedValue([]),
+      getCleanupJob: jest.fn().mockResolvedValue(undefined),
+      abortJob: jest.fn().mockResolvedValue(null),
+    },
     ...overrides,
   };
 }
 
 describe('createAdminUsersHandlers', () => {
+  beforeEach(() => {
+    jest.mocked(eraseUserData).mockClear();
+  });
+
   describe('listUsers', () => {
     it('returns paginated users with total count', async () => {
       const users = [
@@ -462,32 +479,118 @@ describe('createAdminUsersHandlers', () => {
       expect(deps.deletePasskeysByUser).toHaveBeenCalledWith(validUserId);
     });
 
-    it("removes the user's stored credentials before the user", async () => {
-      const deps = createDeps();
+    it('runs the shared erasure cascade for the target before removing the user', async () => {
+      const checkpointer = { ttl: 60 };
+      const erasureDeps = { marker: true };
+      const deps = createDeps({
+        findUsers: jest.fn().mockResolvedValue([mockUser({ role: 'USER', tenantId: 'tenant-1' })]),
+        resolveErasure: jest.fn().mockResolvedValue({ deps: erasureDeps, checkpointer }),
+      });
       const handlers = createAdminUsersHandlers(deps);
       const { req, res, status } = createReqRes({ params: { id: validUserId } });
 
       await handlers.deleteUser(req, res);
 
       expect(status).toHaveBeenCalledWith(200);
-      expect(deps.deleteTokens).toHaveBeenCalledWith({ userId: validUserId });
-      expect(jest.mocked(deps.deleteTokens).mock.invocationCallOrder[0]).toBeLessThan(
+      expect(deps.resolveErasure).toHaveBeenCalledWith(
+        expect.objectContaining({
+          id: validUserId,
+          role: 'USER',
+          tenantId: 'tenant-1',
+          idOnTheSource: null,
+          profile: expect.objectContaining({ role: 'USER' }),
+        }),
+      );
+      expect(eraseUserData).toHaveBeenCalledWith(
+        { userId: validUserId, tenantId: 'tenant-1', checkpointer },
+        erasureDeps,
+      );
+      expect(jest.mocked(eraseUserData).mock.invocationCallOrder[0]).toBeLessThan(
         jest.mocked(deps.deleteUserById).mock.invocationCallOrder[0],
       );
     });
 
-    it('keeps the user, and releases the fence, when its credentials cannot be removed', async () => {
+    it('keeps the user, and releases the fence, when the erasure cascade fails', async () => {
+      jest.mocked(eraseUserData).mockRejectedValueOnce(new Error('storage down'));
+      const deps = createDeps();
+      const handlers = createAdminUsersHandlers(deps);
+      const { req, res, status, json } = createReqRes({ params: { id: validUserId } });
+
+      await handlers.deleteUser(req, res);
+
+      expect(status).toHaveBeenCalledWith(500);
+      expect(json).toHaveBeenCalledWith({ error: 'Failed to delete user' });
+      expect(deps.deleteUserById).not.toHaveBeenCalled();
+      expect(deps.cancelAgentTriggerUserDeletion).toHaveBeenCalled();
+      expect(deps.cancelAgentTriggerUserPurge).toHaveBeenCalled();
+    });
+
+    it('suspends schedules before erasing and restores them when the deletion fails', async () => {
+      jest.mocked(eraseUserData).mockRejectedValueOnce(new Error('storage down'));
+      const deps = createDeps();
+      const handlers = createAdminUsersHandlers(deps);
+      const { req, res, status } = createReqRes({ params: { id: validUserId } });
+
+      await handlers.deleteUser(req, res);
+
+      const token = jest.mocked(deps.quiesceUserSchedules).mock.calls[0][1];
+      expect(status).toHaveBeenCalledWith(500);
+      expect(deps.restoreUserSchedulesFromDeletion).toHaveBeenCalledWith(validUserId, token);
+      expect(
+        jest.mocked(deps.restoreUserSchedulesFromDeletion).mock.invocationCallOrder[0],
+      ).toBeLessThan(jest.mocked(deps.cancelAgentTriggerUserDeletion).mock.invocationCallOrder[0]);
+    });
+
+    it('takes the trigger deletion fence before listing generations to stop', async () => {
+      const deps = createDeps();
+      const handlers = createAdminUsersHandlers(deps);
+      const { req, res } = createReqRes({ params: { id: validUserId } });
+
+      await handlers.deleteUser(req, res);
+
+      expect(
+        jest.mocked(deps.beginAgentTriggerUserDeletion).mock.invocationCallOrder[0],
+      ).toBeLessThan(
+        jest.mocked(deps.generationJobs.getAccountCleanupJobIdsForUser).mock.invocationCallOrder[0],
+      );
+    });
+
+    it('does not erase when schedules cannot be confirmed stopped', async () => {
+      const deps = createDeps({ quiesceUserSchedules: jest.fn().mockResolvedValue(false) });
+      const handlers = createAdminUsersHandlers(deps);
+      const { req, res, status } = createReqRes({ params: { id: validUserId } });
+
+      await handlers.deleteUser(req, res);
+
+      expect(status).toHaveBeenCalledWith(500);
+      expect(eraseUserData).not.toHaveBeenCalled();
+      expect(deps.deleteUserById).not.toHaveBeenCalled();
+    });
+
+    it('aborts the cleanup jobs of the target and refuses to erase when a stop is unconfirmed', async () => {
+      const abortJob = jest.fn().mockResolvedValue(null);
       const deps = createDeps({
-        deleteTokens: jest.fn().mockRejectedValue(new Error('mongo down')),
+        generationJobs: {
+          getAccountCleanupJobIdsForUser: jest.fn().mockResolvedValue(['s1', 's2']),
+          getCleanupJob: jest.fn(async (id: string) => ({
+            createdAt: 5,
+            metadata: { userId: id === 's1' ? validUserId : 'someone-else' },
+          })),
+          abortJob,
+        } as unknown as AdminUsersDeps['generationJobs'],
       });
       const handlers = createAdminUsersHandlers(deps);
       const { req, res, status } = createReqRes({ params: { id: validUserId } });
 
       await handlers.deleteUser(req, res);
 
+      expect(abortJob).toHaveBeenCalledTimes(1);
+      expect(abortJob).toHaveBeenCalledWith('s1', {
+        expectedCreatedAt: 5,
+        awaitProviderDrain: true,
+      });
       expect(status).toHaveBeenCalledWith(500);
-      expect(deps.deleteUserById).not.toHaveBeenCalled();
-      expect(deps.cancelAgentTriggerUserDeletion).toHaveBeenCalled();
+      expect(eraseUserData).not.toHaveBeenCalled();
     });
 
     it('returns success even when cascade cleanup partially fails', async () => {