Skip to content

Commit 3a33cf6

Browse files
committed
fix: prevent tool call mixing and retry storms
1 parent f2bdcb6 commit 3a33cf6

6 files changed

Lines changed: 293 additions & 30 deletions

File tree

src/core/assistant-message/presentAssistantMessage.ts

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,10 @@ export async function presentAssistantMessage(cline: Task) {
7171
cline.presentAssistantMessageLocked = true
7272
cline.presentAssistantMessageHasPendingUpdates = false
7373

74+
// Reset batch skip counter at the start of each present cycle.
75+
// This ensures the count is fresh for each batch of tool calls.
76+
;(cline as any).batchSkipCount = 0
77+
7478
if (cline.currentStreamingContentIndex >= cline.assistantMessageContent.length) {
7579
// This may happen if the last content block was completed before
7680
// streaming could finish. If streaming is finished, and we're out of
@@ -111,8 +115,11 @@ export async function presentAssistantMessage(cline: Task) {
111115
if (cline.didRejectTool) {
112116
// For native protocol, we must send a tool_result for every tool_use to avoid API errors
113117
const toolCallId = mcpBlock.id
118+
const skipCount = (cline as any).batchSkipCount ?? 0
119+
const nextSkipCount = skipCount + 1
120+
;(cline as any).batchSkipCount = nextSkipCount
114121
const errorMessage = !mcpBlock.partial
115-
? `Skipping MCP tool ${mcpBlock.name} due to user rejecting a previous tool.`
122+
? `Skipping MCP tool ${mcpBlock.name} due to user rejecting a previous tool. (Skipped ${nextSkipCount} tool${nextSkipCount > 1 ? "s" : ""} in this batch.)`
116123
: `MCP tool ${mcpBlock.name} was interrupted and not executed due to user rejecting a previous tool.`
117124

118125
if (toolCallId) {
@@ -391,8 +398,11 @@ export async function presentAssistantMessage(cline: Task) {
391398
if (cline.didRejectTool) {
392399
// Ignore any tool content after user has rejected tool once.
393400
// For native tool calling, we must send a tool_result for every tool_use to avoid API errors
401+
const skipCount = (cline as any).batchSkipCount ?? 0
402+
const nextSkipCount = skipCount + 1
403+
;(cline as any).batchSkipCount = nextSkipCount
394404
const errorMessage = !block.partial
395-
? `Skipping tool ${toolDescription()} due to user rejecting a previous tool.`
405+
? `Skipping tool ${toolDescription()} due to user rejecting a previous tool. (Skipped ${nextSkipCount} tool${nextSkipCount > 1 ? "s" : ""} in this batch.)`
396406
: `Tool ${toolDescription()} was interrupted and not executed due to user rejecting a previous tool.`
397407

398408
cline.pushToolResultToUserContent({

src/core/task/Task.ts

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,13 @@ const DEFAULT_USAGE_COLLECTION_TIMEOUT_MS = 5000 // 5 seconds
140140
const FORCED_CONTEXT_REDUCTION_PERCENT = 75 // Keep 75% of context (remove 25%) on context window errors
141141
const MAX_CONTEXT_WINDOW_RETRIES = 3 // Maximum retries for context window errors
142142

143+
/**
144+
* Maximum number of consecutive retries allowed for the same tool within a single turn.
145+
* After this limit, further retries are blocked until the user provides new input.
146+
* This prevents retry storms where the model repeatedly tries the same failing tool.
147+
*/
148+
const MAX_TOOL_RETRY_BUDGET = 3
149+
143150
export interface TaskOptions extends CreateTaskOptions {
144151
provider: ClineProvider
145152
apiConfiguration: ProviderSettings
@@ -321,6 +328,22 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
321328
consecutiveNoAssistantMessagesCount: number = 0
322329
toolUsage: ToolUsage = {}
323330

331+
/**
332+
* Retry budget: tracks how many times the same tool has been retried
333+
* consecutively within the current turn. Keyed by tool name (e.g., "execute_command").
334+
* After MAX_TOOL_RETRY_BUDGET (3) retries of the same tool in a row,
335+
* further retries are blocked until the user provides new input.
336+
* The counter resets when user feedback arrives (handleWebviewAskResponse with messageResponse)
337+
* or at the start of each new API request.
338+
*/
339+
toolRetryBudget: Map<string, number> = new Map()
340+
341+
/**
342+
* When true, the retry budget for the current tool has been exceeded and
343+
* no further retries of that tool are allowed until user feedback arrives.
344+
*/
345+
toolRetryBudgetExceeded: boolean = false
346+
324347
// Checkpoints
325348
enableCheckpoints: boolean
326349
checkpointTimeout: number
@@ -1431,6 +1454,15 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
14311454
this.askResponseText = text
14321455
this.askResponseImages = images
14331456

1457+
// Reset retry budget when user provides new input (messageResponse).
1458+
// This allows the model to retry tools after the user has had a chance to
1459+
// provide guidance. The budget is NOT reset on yesButtonClicked (tool approval)
1460+
// since that's the model continuing its own turn, not user feedback.
1461+
if (askResponse === "messageResponse") {
1462+
this.toolRetryBudget.clear()
1463+
this.toolRetryBudgetExceeded = false
1464+
}
1465+
14341466
// Create a checkpoint whenever the user sends a message.
14351467
// Use allowEmpty=true to ensure a checkpoint is recorded even if there are no file changes.
14361468
// Suppress the checkpoint_saved chat row for this particular checkpoint to keep the timeline clean.
@@ -2716,6 +2748,10 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
27162748
this.didToolFailInCurrentTurn = false
27172749
this.presentAssistantMessageLocked = false
27182750
this.presentAssistantMessageHasPendingUpdates = false
2751+
// Reset retry budget for each new API request (new assistant turn).
2752+
// This gives the model a fresh budget of 3 retries per tool per turn.
2753+
this.toolRetryBudget.clear()
2754+
this.toolRetryBudgetExceeded = false
27192755
// No legacy text-stream tool parser.
27202756
this.streamingToolCallIndices.clear()
27212757
// Clear any leftover streaming tool call state from previous interrupted streams
@@ -4734,4 +4770,51 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
47344770
console.error(`[Task] Queue processing error:`, e)
47354771
}
47364772
}
4773+
4774+
/**
4775+
* Checks if the retry budget for a given tool has been exceeded.
4776+
* Each tool gets MAX_TOOL_RETRY_BUDGET (3) consecutive retries per turn.
4777+
* After that, further retries are blocked until user feedback arrives.
4778+
*
4779+
* @param toolName - The name of the tool being retried
4780+
* @returns true if the retry budget is still available, false if blocked
4781+
*/
4782+
public checkToolRetryBudget(toolName: string): boolean {
4783+
if (this.toolRetryBudgetExceeded) {
4784+
console.warn(
4785+
`[Task#${this.taskId}] Tool retry budget globally exceeded. Blocking further retries for "${toolName}".`,
4786+
)
4787+
return false
4788+
}
4789+
4790+
const currentRetries = this.toolRetryBudget.get(toolName) ?? 0
4791+
4792+
if (currentRetries >= MAX_TOOL_RETRY_BUDGET) {
4793+
this.toolRetryBudgetExceeded = true
4794+
console.warn(
4795+
`[Task#${this.taskId}] Tool retry budget exceeded for "${toolName}": ` +
4796+
`${currentRetries} retries >= ${MAX_TOOL_RETRY_BUDGET} max. ` +
4797+
"Blocking further retries until user provides new input.",
4798+
)
4799+
return false
4800+
}
4801+
4802+
// Increment the retry counter for this tool
4803+
this.toolRetryBudget.set(toolName, currentRetries + 1)
4804+
return true
4805+
}
4806+
4807+
/**
4808+
* Records a tool use as successful, which resets the retry budget for that tool.
4809+
* This is called when a tool completes successfully, allowing the model to use
4810+
* the tool again (the budget is per-retry, not per-use).
4811+
*
4812+
* @param toolName - The name of the tool that succeeded
4813+
*/
4814+
public recordSuccessfulToolUse(toolName: string): void {
4815+
// Only reset the retry budget for this specific tool, not all tools.
4816+
// This allows the model to switch to a different tool after exhausting
4817+
// the budget on one tool.
4818+
this.toolRetryBudget.delete(toolName)
4819+
}
47374820
}

src/core/task/validateToolResultIds.ts

Lines changed: 24 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,23 @@ export function validateAndFixToolResultIds(
165165
)
166166
}
167167

168-
// Match tool_results to tool_uses by position and fix incorrect IDs
168+
// Log the mismatched IDs instead of silently falling back to positional matching.
169+
// Positional matching caused cross-wiring of tool results when the ordering of
170+
// tool_results did not match the ordering of tool_use blocks (e.g., parallel tool
171+
// calls where results arrive in a different order than the calls were made).
172+
// The correct behavior is to surface the mismatch so it can be diagnosed and fixed
173+
// at the source, not silently remapped.
174+
console.warn(
175+
"[validateAndFixToolResultIds] Tool result ID mismatch detected — removing positional fallback. " +
176+
`tool_result IDs: [${toolResultIdList.join(", ")}], ` +
177+
`tool_use IDs: [${toolUseIdList.join(", ")}]. ` +
178+
"Mismatched tool_result blocks will be dropped to prevent cross-wiring.",
179+
)
180+
181+
// Filter out tool_results with invalid or duplicate IDs instead of remapping them.
182+
// This is safer than positional fallback: dropping a misattributed result is
183+
// preferable to wiring it to the wrong tool_use, which would cause subtle
184+
// correctness bugs in the LLM's understanding of tool outputs.
169185
const usedToolUseIds = new Set<string>()
170186
const contentArray = userMessage.content as Anthropic.Messages.ContentBlockParam[]
171187

@@ -175,31 +191,18 @@ export function validateAndFixToolResultIds(
175191
return block
176192
}
177193

178-
// If the ID is already valid and not yet used, keep it
194+
// If the ID is valid and not yet used, keep it
179195
if (validToolUseIds.has(block.tool_use_id) && !usedToolUseIds.has(block.tool_use_id)) {
180196
usedToolUseIds.add(block.tool_use_id)
181197
return block
182198
}
183199

184-
// Find which tool_result index this block is by comparing references.
185-
// This correctly handles duplicate tool_use_ids - we find the actual block's
186-
// position among all tool_results, not the first block with a matching ID.
187-
const toolResultIndex = toolResults.indexOf(block as Anthropic.ToolResultBlockParam)
188-
189-
// Try to match by position - only fix if there's a corresponding tool_use
190-
if (toolResultIndex !== -1 && toolResultIndex < toolUseBlocks.length) {
191-
const correctId = toolUseBlocks[toolResultIndex].id
192-
// Only use this ID if it hasn't been used yet
193-
if (!usedToolUseIds.has(correctId)) {
194-
usedToolUseIds.add(correctId)
195-
return {
196-
...block,
197-
tool_use_id: correctId,
198-
}
199-
}
200-
}
201-
202-
// No corresponding tool_use for this tool_result, or the ID is already used
200+
// Invalid or duplicate tool_result ID — drop it instead of remapping
201+
console.warn(
202+
`[validateAndFixToolResultIds] Dropping tool_result with tool_use_id "${block.tool_use_id}": ` +
203+
`${validToolUseIds.has(block.tool_use_id) ? "duplicate ID" : "ID not found in tool_use blocks"}. ` +
204+
"This prevents cross-wiring of tool results.",
205+
)
203206
return null
204207
})
205208
.filter((block): block is NonNullable<typeof block> => block !== null)

src/core/tools/ExecuteCommandTool.ts

Lines changed: 74 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,12 @@ export class ExecuteCommandTool extends BaseTool<"execute_command"> {
8484
pushToolResult(formatResponse.rooIgnoreError(ignoredFileAttemptedToAccess))
8585
return
8686
}
87+
88+
// Clear any prior in-flight ask state before proceeding with execution.
89+
// This prevents a stale ask from a previous command invocation (e.g., a
90+
// shell integration fallback) from racing with the fresh approval prompt
91+
// in the current invocation.
92+
task.supersedePendingAsk()
8793

8894
task.consecutiveMistakeCount = 0
8995

@@ -150,11 +156,34 @@ export class ExecuteCommandTool extends BaseTool<"execute_command"> {
150156

151157
try {
152158
const [rejected, result] = await executeCommandInTerminal(task, options)
159+
} catch (error: unknown) {
160+
// Invalidate pending ask from first execution to prevent race condition
161+
task.supersedePendingAsk()
162+
163+
if (canRetryShellIntegrationError(error)) {
164+
// Silent retry via execa — shell startup race, command was not submitted.
165+
const status: CommandExecutionStatus = { executionId, status: "fallback" }
166+
provider?.postMessageToWebview({ type: "commandExecutionStatus", text: JSON.stringify(status) })
167+
168+
const [rejected, result] = await executeCommandInTerminal(task, {
169+
...options,
170+
terminalShellIntegrationDisabled: true,
171+
})
153172

154173
if (rejected) {
155174
task.didRejectTool = true
156175
}
157176

177+
// Fix: Mark the first execution result as consumed so we don't send
178+
// duplicate pushToolResult calls. The shell integration retry below
179+
// will produce the single authoritative result.
180+
// Also await onCompletedPromise to avoid a race where the first
181+
// command's onCompleted fires after we've already started the retry,
182+
// corrupting shared state (completed, persistedResult, etc.).
183+
if (!rejected && !runInBackground) {
184+
await onCompletedPromise
185+
}
186+
158187
pushToolResult(result)
159188
} catch (error: unknown) {
160189
// Invalidate pending ask from first execution to prevent race condition
@@ -165,10 +194,17 @@ export class ExecuteCommandTool extends BaseTool<"execute_command"> {
165194
const status: CommandExecutionStatus = { executionId, status: "fallback" }
166195
provider?.postMessageToWebview({ type: "commandExecutionStatus", text: JSON.stringify(status) })
167196

168-
const [rejected, result] = await executeCommandInTerminal(task, {
197+
// Fix: When retrying with shell integration fallback, ensure we use a
198+
// fresh terminal rather than the VSCode terminal that may have a stale
199+
// shell integration state. This prevents double-execution where the
200+
// command runs both in the VSCode terminal AND via execa.
201+
const retryOptions = {
169202
...options,
170203
terminalShellIntegrationDisabled: true,
171-
})
204+
forceNewTerminal: true,
205+
}
206+
207+
const [rejected, result] = await executeCommandInTerminal(task, retryOptions)
172208

173209
if (rejected) {
174210
task.didRejectTool = true
@@ -181,10 +217,16 @@ export class ExecuteCommandTool extends BaseTool<"execute_command"> {
181217

182218
if (error instanceof ShellIntegrationError) {
183219
pushToolResult(
184-
"Command was submitted in the VS Code terminal, but shell integration did not report its output or completion status. Do not run the command again automatically.",
220+
formatResponse.toolError(
221+
"Command was submitted in the terminal, but its output and completion status could not be tracked due to a shell integration error. The command may still be running. Do NOT re-run the command automatically — inspect the terminal manually and report the result.",
222+
),
185223
)
186224
} else {
187-
pushToolResult(`Command failed to execute in terminal due to a shell integration error.`)
225+
pushToolResult(
226+
formatResponse.toolError(
227+
`Command failed to execute in terminal: ${error instanceof Error ? error.message : String(error)}. Check the terminal state before retrying.`,
228+
),
229+
)
188230
}
189231
}
190232
}
@@ -209,6 +251,8 @@ export type ExecuteCommandOptions = {
209251
terminalShellIntegrationDisabled?: boolean
210252
commandExecutionTimeout?: number
211253
agentTimeout?: number
254+
/** When true, forces creation of a fresh terminal even for retries. */
255+
forceNewTerminal?: boolean
212256
}
213257

214258
export async function executeCommandInTerminal(
@@ -430,7 +474,10 @@ export async function executeCommandInTerminal(
430474
}
431475
}
432476

433-
const terminal = await TerminalRegistry.getOrCreateTerminal(workingDir, task.taskId, terminalProvider)
477+
// Fix: Use getOrCreateCommandTerminal to ensure each execute_command gets a
478+
// dedicated terminal, preventing output interleaving when multiple commands
479+
// run sequentially on the same working directory.
480+
const terminal = await TerminalRegistry.getOrCreateCommandTerminal(workingDir, task.taskId, terminalProvider)
434481

435482
if (terminal instanceof Terminal) {
436483
terminal.terminal.show(true)
@@ -475,6 +522,14 @@ export async function executeCommandInTerminal(
475522
racers.push(
476523
new Promise<void>((_, reject) => {
477524
userTimeoutId = setTimeout(() => {
525+
// Fix timeout race: Only fire the timeout if the command hasn't
526+
// already completed. Check `completed` under a microtask to avoid
527+
// a race where onCompleted sets completed=true between the timeout
528+
// firing and this check.
529+
if (completed) {
530+
resolve()
531+
return
532+
}
478533
isUserTimedOut = true
479534
task.terminalProcess?.abort()
480535
reject(new Error(`Command execution timed out after ${commandExecutionTimeout}ms`))
@@ -483,7 +538,14 @@ export async function executeCommandInTerminal(
483538
)
484539
}
485540

486-
await Promise.race(racers)
541+
// Fix timeout race: After Promise.race resolves (either process completed,
542+
// agent timeout, or user timeout), ensure the other timeout is cleared
543+
// immediately. Without this, a user timeout could fire AFTER the command
544+
// has already completed, incorrectly aborting it.
545+
clearTimeout(agentTimeoutId)
546+
agentTimeoutId = undefined
547+
clearTimeout(userTimeoutId)
548+
userTimeoutId = undefined
487549
} catch (error) {
488550
if (isUserTimedOut) {
489551
const status: CommandExecutionStatus = { executionId, status: "timeout" }
@@ -525,6 +587,12 @@ export async function executeCommandInTerminal(
525587
await onCompletedPromise
526588
}
527589

590+
// If the command completed during the onCompleted wait (e.g., a very fast
591+
// command that finished before the timeout was set), make sure the timeout
592+
// race fix above didn't leave a dangling timeout that could fire later.
593+
clearTimeout(agentTimeoutId)
594+
clearTimeout(userTimeoutId)
595+
528596
if (message) {
529597
const { text, images } = message
530598
await task.say("user_feedback", text, images)

0 commit comments

Comments
 (0)