Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 28 additions & 11 deletions apps/worker/src/run-task/__tests__/run-task.test.ts

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

30 changes: 20 additions & 10 deletions apps/worker/src/run-task/run-task.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2085,6 +2085,8 @@ export const runTask = async ({
//
// Terminal cancel must skip this handoff: publish no due sleepAt and
// finish as Canceled instead of becoming a snapshot/standby candidate.
// Failed turns still use the ordinary idle retention path so a model or
// provider error cannot discard an otherwise healthy workspace.
//
// NOTE: Snapshot creation ultimately tears down the provider runtime. Vercel
// does this as part of snapshot creation, while Modal explicitly terminates
Expand All @@ -2097,36 +2099,39 @@ export const runTask = async ({
// messages). The drain check below is kept as a fallback for edge cases where
// the snapshot fails or times out but the worker survives.
const skipSleepAfterTerminalCancel = Boolean(finalState.cancelTriggeredAt);
const skipSleepAfterTerminalFailure =
resolvedResult.status === RunStatus.Failed;
const skipSleepWithoutRetentionDeadline =
resolvedResult.status === RunStatus.Failed &&
harnessManager.getSleepAt() == null;
if (skipSleepAfterTerminalCancel) {
logger.info(
`[runTask] Skipping external sleep handoff after terminal cancel for task run ${taskRun.id}`,
);
}
if (skipSleepAfterTerminalFailure) {
if (skipSleepWithoutRetentionDeadline && !skipSleepAfterTerminalCancel) {
logger.info(
`[runTask] Skipping external sleep handoff after terminal failure for task run ${taskRun.id}`,
`[runTask] Skipping external sleep handoff without a retention deadline for task run ${taskRun.id}`,
);
}

let sleepActionTriggered = false;
let sleepActionCompleted = false;

if (
!skipExternalSleepAction &&
!skipSleepAfterTerminalCancel &&
!skipSleepAfterTerminalFailure
!skipSleepWithoutRetentionDeadline
) {
// BullMQ may claim the sleep action and snapshot the filesystem while
// the handoff helper polls below. The harness has already shut down, so
// drop on-disk credential material first; resume re-injects it from the
// dequeue response.
await scrubSandboxSecretsBeforeSnapshot(logger, { homeDir, runtimeEnv });

({ claimed: sleepActionTriggered } = await waitForExternalSleepAction({
taskRun,
logger,
}));
({ claimed: sleepActionTriggered, completed: sleepActionCompleted } =
await waitForExternalSleepAction({
taskRun,
logger,
}));
}

// Fallback: check for pending Linear messages that arrived during the snapshot
Expand Down Expand Up @@ -2209,7 +2214,12 @@ export const runTask = async ({
} else {
taskCancellation.abortController.abort();
}
return resolvedResult;
// BullMQ owns terminal state after a completed snapshot/standby handoff.
// Do not let a stale pre-handoff failure overwrite that completion if the
// provider leaves this worker alive long enough to return normally.
return sleepActionCompleted
? { status: RunStatus.Completed }
: resolvedResult;
} finally {
activeWorkerCrashContext = null;
}
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

21 changes: 10 additions & 11 deletions apps/worker/src/sandbox-server/lib/harness-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -722,12 +722,14 @@ export class HarnessManager extends EventEmitter<HarnessManagerEvents> {
return null;
}

// Failed shutdowns must reach finishRun so channel integrations can report
// the error. Snapshotting would otherwise finalize the run as completed.
// Failures without a usable runtime session still need terminal
// finalization. A provider error from an existing session is a failed turn
// instead, so it remains eligible for the ordinary idle retention path.
if (
this.phase === 'shutting_down' &&
this.state.lastErrorMessage &&
!this.state.taskFinishedAt
!this.state.taskFinishedAt &&
!this.terminalProviderErrorPending
) {
return null;
}
Expand Down Expand Up @@ -1362,16 +1364,13 @@ export class HarnessManager extends EventEmitter<HarnessManagerEvents> {
if (payload[0] === this.state.sessionId) {
this.logger.info(`[HarnessManager] Task aborted: ${payload[0]}`);

// A provider error is terminal, unlike a user-initiated abort. Preserve
// the error and shut down without setting the cancellation stamp so the
// worker resolves this run as Failed rather than Canceled.
if (this.terminalProviderErrorPending && !this.state.cancelTriggeredAt) {
this.triggerShutdown();
return;
// A terminal provider error ends the current model turn, not the live
// task session. Keep its error visible and leave the session available
// for follow-ups until the ordinary idle keepalive expires.
if (!this.terminalProviderErrorPending || this.state.cancelTriggeredAt) {
this.state.taskAbortedAt = Date.now();
}

this.state.taskAbortedAt = Date.now();

if (this.runtimeQueuedMessagesCount > 0) {
// Abort takes priority: never downgrade a deferred abort to completion.
this.deferredTurnSettlement = HarnessEvent.TaskAborted;
Expand Down
Loading