Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 83 additions & 42 deletions apps/api/src/scheduled/stuck-tasks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import {
DEFAULT_TASK_RUN_MAX_EXECUTION_MS,
DEFAULT_TASK_STUCK_DELEGATED_TIMEOUT_MS,
DEFAULT_TASK_STUCK_QUEUED_TIMEOUT_MS,
type TaskExecutionStep,
} from '@simple-agent-manager/shared';
import { drizzle } from 'drizzle-orm/d1';

Expand Down Expand Up @@ -83,6 +84,8 @@ import {
const SUPERSEDED_TERMINATION_MESSAGE =
'Superseded by a later session wake; the conversation continued in a replacement ' +
'task and has since ended.';
const TASK_RUNNER_NORMAL_HANDOFF_STEP: TaskExecutionStep = 'running';
const TASK_RUNNER_MISMATCH_RECOVERY_TYPE = 'do_task_status_mismatch';

function parseMs(value: string | undefined, fallback: number): number {
if (!value) return fallback;
Expand All @@ -109,6 +112,22 @@ function describeStep(step: string | null): string {
return STEP_DESCRIPTIONS[step] ?? step;
}

function isNormalCompletedTaskRunnerHandoff(
task: StuckTaskCandidate,
doStatus: NonNullable<TaskRunnerProbeResult['status']>
): boolean {
return task.status === 'in_progress' && doStatus.currentStep === TASK_RUNNER_NORMAL_HANDOFF_STEP;
}

function taskRunnerMismatchKind(
task: StuckTaskCandidate,
doStatus: NonNullable<TaskRunnerProbeResult['status']>
): 'completed_handoff_missing_in_d1' | 'completed_before_running_handoff' {
return doStatus.currentStep === TASK_RUNNER_NORMAL_HANDOFF_STEP && task.status !== 'in_progress'
? 'completed_handoff_missing_in_d1'
: 'completed_before_running_handoff';
}

export interface StuckTaskResult {
failedQueued: number;
failedDelegated: number;
Expand Down Expand Up @@ -1223,50 +1242,72 @@ export async function recoverStuckTasks(env: Env): Promise<StuckTaskResult> {
});
}

if (doStatus?.completed) {
// Reconcile only with conclusive dead-runtime evidence. Live or unknown
// task-scoped runtime state remains active and is logged for investigation.
// Deduplicate the persisted mismatch signal independently of reconciliation.
log.warn('stuck_task.do_completed_but_task_active', {
taskId: task.id,
taskStatus: task.status,
doCurrentStep: doStatus.currentStep,
doRetryCount: doStatus.retryCount,
});

// Deduplicate: only persist if no recent mismatch record exists for this task
const recentMismatch = await env.OBSERVABILITY_DATABASE.prepare(
`SELECT id FROM platform_errors
WHERE task_id = ? AND context LIKE ? AND timestamp > ?
LIMIT 1`
)
.bind(task.id, '%do_task_status_mismatch%', Date.now() - 30 * 60 * 1000)
.first();

if (!recentMismatch) {
await persistError(
env.OBSERVABILITY_DATABASE,
{
source: 'api',
level: 'warn',
message: `TaskRunner DO completed but task still in '${task.status}' — possible D1 update failure`,
context: {
recoveryType: 'do_task_status_mismatch',
if (doStatus?.completed && !isStuck) {
if (isNormalCompletedTaskRunnerHandoff(task, doStatus)) {
// `transitionToInProgress` deliberately stores TaskRunner
// `completed=true` at the successful handoff boundary while the D1 task
// remains active until an explicit agent/user terminal path. This is
// normal lifecycle bookkeeping, not D1 drift; production evidence on
// 2026-08-24 showed these rows were live, restorable, or live-superseded.
log.info('stuck_task.do_completed_handoff_active', {
taskId: task.id,
taskStatus: task.status,
executionStep: task.execution_step,
doCurrentStep: doStatus.currentStep,
doRetryCount: doStatus.retryCount,
livenessReason: liveness?.reason ?? null,
action: 'observed_normal_handoff',
});
} else {
const mismatchKind = taskRunnerMismatchKind(task, doStatus);
log.warn('stuck_task.do_completed_active_state_mismatch', {
taskId: task.id,
taskStatus: task.status,
executionStep: task.execution_step,
doCurrentStep: doStatus.currentStep,
doRetryCount: doStatus.retryCount,
mismatchKind,
});

// One durable diagnostic per task is enough. Repeating the same
// preserved candidate every 30 minutes caused the production noise that
// hid the real state: normal handoff and resumable/superseded sessions.
const existingMismatch = await env.OBSERVABILITY_DATABASE.prepare(
`SELECT id FROM platform_errors
WHERE task_id = ? AND context LIKE ?
LIMIT 1`
)
.bind(task.id, `%${TASK_RUNNER_MISMATCH_RECOVERY_TYPE}%`)
.first();

if (!existingMismatch) {
await persistError(
env.OBSERVABILITY_DATABASE,
{
source: 'api',
level: 'warn',
message:
`TaskRunner DO reports completed at '${doStatus.currentStep}' while ` +
`task remains '${task.status}' — active state mismatch before normal handoff convergence`,
context: {
recoveryType: TASK_RUNNER_MISMATCH_RECOVERY_TYPE,
mismatchKind,
taskId: task.id,
taskStatus: task.status,
executionStep: task.execution_step,
doCurrentStep: doStatus.currentStep,
doRetryCount: doStatus.retryCount,
timeForCheck,
taskRunnerProbeOutcome: doProbe.outcome,
livenessReason: liveness?.reason ?? null,
},
userId: task.user_id,
taskId: task.id,
taskStatus: task.status,
executionStep: task.execution_step,
doCurrentStep: doStatus.currentStep,
doRetryCount: doStatus.retryCount,
timeForCheck,
taskRunnerProbeOutcome: doProbe.outcome,
livenessReason: liveness?.reason ?? null,
sessionId: task.chat_session_id,
},
userId: task.user_id,
taskId: task.id,
sessionId: task.chat_session_id,
},
env
);
env
);
}
}
}
}
Expand Down
15 changes: 9 additions & 6 deletions apps/api/tests/unit/recovery-resilience.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -167,19 +167,22 @@ describe('stuck-tasks DO health checks (TDF-7)', () => {
expect(healthCheckSection).toContain('timeForCheck');
});

it('detects DO-completed-but-task-active mismatch', () => {
expect(stuckTasksSource).toContain('stuck_task.do_completed_but_task_active');
it('detects completed-DO active-state mismatches while recognizing normal handoff', () => {
expect(stuckTasksSource).toContain('stuck_task.do_completed_handoff_active');
expect(stuckTasksSource).toContain('stuck_task.do_completed_active_state_mismatch');
expect(stuckTasksSource).toContain('doStatus?.completed');
});

it('records DO mismatch in OBSERVABILITY_DATABASE', () => {
expect(stuckTasksSource).toContain("recoveryType: 'do_task_status_mismatch'");
expect(stuckTasksSource).toContain('TASK_RUNNER_MISMATCH_RECOVERY_TYPE');
expect(stuckTasksSource).toContain("'do_task_status_mismatch'");
});

it('deduplicates DO mismatch records (30 min window)', () => {
expect(stuckTasksSource).toContain('recentMismatch');
it('deduplicates DO mismatch records once per task', () => {
expect(stuckTasksSource).toContain('existingMismatch');
expect(stuckTasksSource).toContain('do_task_status_mismatch');
expect(stuckTasksSource).toContain('30 * 60 * 1000');
expect(stuckTasksSource).toContain('One durable diagnostic per task is enough');
expect(stuckTasksSource).not.toContain('30 * 60 * 1000');
});

it('tracks doHealthChecked count in result', () => {
Expand Down
Loading
Loading