Add terminal test to chat scenario tests

build/azure-pipelines/linux/steps/product-build-linux-test.yml

@@ -143,6 +143,18 @@ steps:
       timeoutInMinutes: 20
       displayName: 🧪 Run smoke tests (Electron)
 
+  - ${{ if eq(parameters.VSCODE_RUN_ELECTRON_TESTS, true) }}:
+    - script: |
+        set -e
+        for i in $(seq 1 20); do
+          echo "=== Probe iteration $i/20 ==="
+          npm run smoketest-no-compile -- -g "run in terminal" --tracing --build "$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)"
+        done
+      env:
+        TMPDIR: $(Agent.TempDirectory)
+      timeoutInMinutes: 120
+      displayName: 🧪 PROBE — run "run in terminal" smoke tests 20×
+
   - ${{ if eq(parameters.VSCODE_RUN_BROWSER_TESTS, true) }}:
     - script: npm run smoketest-no-compile -- --web --tracing --headless
       env:

build/azure-pipelines/win32/steps/product-build-win32-test.yml

@@ -135,6 +135,16 @@ steps:
       displayName: 🧪 Run smoke tests (Electron)
       timeoutInMinutes: 20
 
+  - ${{ if eq(parameters.VSCODE_RUN_ELECTRON_TESTS, true) }}:
+    - powershell: |
+        for ($i = 1; $i -le 20; $i++) {
+          Write-Host "=== Probe iteration $i/20 ==="
+          npm run smoketest-no-compile -- -g "run in terminal" --tracing --build "$(agent.builddirectory)\test\VSCode-win32-$(VSCODE_ARCH)"
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+        }
+      displayName: 🧪 PROBE — run "run in terminal" smoke tests 20×
+      timeoutInMinutes: 120
+
   - ${{ if eq(parameters.VSCODE_RUN_BROWSER_TESTS, true) }}:
     - powershell: npm run smoketest-no-compile -- --web --tracing --headless
       env:

test/automation/src/agentsWindow.ts

@@ -4,6 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { Code } from './code';
+import { acceptToolConfirmationIfPresent } from './chat';
 import { QuickAccess } from './quickaccess';
 
 const AGENTS_WORKBENCH = '.agent-sessions-workbench';
@@ -428,7 +429,7 @@ export class AgentsWindow {
 	 * well after the content is on screen, so requiring `:not(.chat-response-loading)`
 	 * causes false-negative timeouts.
 	 */
-	async waitForAssistantText(predicate: RegExp | string, timeoutMs: number = 60_000): Promise<string> {
+	async waitForAssistantText(predicate: RegExp | string, timeoutMs: number = 60_000, options?: { acceptToolConfirmations?: boolean }): Promise<string> {
 		const retryCount = Math.ceil(timeoutMs / 100);
 		await this.code.waitForElement(RESPONSE, undefined, retryCount);
 
@@ -437,6 +438,12 @@ export class AgentsWindow {
 		const deadline = Date.now() + timeoutMs;
 		let lastTexts: string[] = [];
 		while (Date.now() < deadline) {
+			// When requested, accept any pending terminal tool confirmation so
+			// the agentic loop can proceed. No-op for sessions that
+			// auto-approve their shell commands.
+			if (options?.acceptToolConfirmations) {
+				await acceptToolConfirmationIfPresent(this.code);
+			}
 			// Look in BOTH the active session view and the broader workbench
 			// scope. The Agents Window can auto-swap the active slot to a
 			// fresh untitled session immediately after a follow-up commits,

test/automation/src/chat.ts

@@ -126,12 +126,12 @@ export class Chat {
 	 * the content has actually arrived (avoiding false matches on placeholder
 	 * text like "Considering" that appears before streaming begins).
 	 */
-	async waitForResponseText(predicate: string | RegExp, timeoutMs: number = 60_000): Promise<string> {
-		return await this.pollForResponseText(CHAT_RESPONSE, CHAT_RESPONSE_RENDERED, predicate, timeoutMs);
+	async waitForResponseText(predicate: string | RegExp, timeoutMs: number = 60_000, options?: { acceptToolConfirmations?: boolean }): Promise<string> {
+		return await this.pollForResponseText(CHAT_RESPONSE, CHAT_RESPONSE_RENDERED, predicate, timeoutMs, options?.acceptToolConfirmations);
 	}
 
-	async waitForEditorResponseText(predicate: string | RegExp, timeoutMs: number = 60_000): Promise<string> {
-		const matched = await this.pollForResponseText(CHAT_EDITOR_RESPONSE, CHAT_EDITOR_RESPONSE_RENDERED, predicate, timeoutMs);
+	async waitForEditorResponseText(predicate: string | RegExp, timeoutMs: number = 60_000, options?: { acceptToolConfirmations?: boolean }): Promise<string> {
+		const matched = await this.pollForResponseText(CHAT_EDITOR_RESPONSE, CHAT_EDITOR_RESPONSE_RENDERED, predicate, timeoutMs, options?.acceptToolConfirmations);
 		// After a contributed chat session (e.g. Copilot CLI, Claude) returns
 		// its first response, the workbench commits the untitled session into
 		// a real (titled) one and `replaceEditors` swaps the chat editor over.
@@ -164,10 +164,16 @@ export class Chat {
 		}
 	}
 
-	private async pollForResponseText(bubbleSelector: string, renderedSelector: string, predicate: string | RegExp, timeoutMs: number): Promise<string> {
+	private async pollForResponseText(bubbleSelector: string, renderedSelector: string, predicate: string | RegExp, timeoutMs: number, acceptToolConfirmations?: boolean): Promise<string> {
 		const deadline = Date.now() + timeoutMs;
 		const matches = (text: string) => typeof predicate === 'string' ? text.includes(predicate) : predicate.test(text);
 		while (Date.now() < deadline) {
+			// When requested, accept any pending terminal tool confirmation so
+			// the agentic loop can proceed to render the final response. This
+			// is a no-op for sessions that auto-approve their shell commands.
+			if (acceptToolConfirmations) {
+				await acceptToolConfirmationIfPresent(this.code);
+			}
 			const elements = await this.code.driver.getElements(renderedSelector, /* recursive */ true);
 			const matched = elements.map(el => el.textContent ?? '').filter(matches);
 			if (matched.length > 0) {
@@ -474,3 +480,54 @@ export class Chat {
 		}
 	}
 }
+
+/**
+ * Click the "Allow" button of a pending terminal tool confirmation if one is
+ * present. No-op when there is no confirmation (e.g. the session auto-approved
+ * its shell command). Shared by {@link Chat} and the Agents Window driver so
+ * shell-tool tests can drive the real confirmation flow without per-agent
+ * special-casing.
+ *
+ * The terminal confirmation renders as a `.chat-confirmation-widget2` whose
+ * action buttons are `.monaco-button.monaco-text-button` anchors — the primary
+ * "Allow" (rendered first), then "Skip". The dropdown chevron next to "Allow"
+ * is a `.monaco-dropdown-button.codicon` (no `.monaco-text-button`) and the
+ * carousel's "Allow All" lives on `.chat-tool-carousel-allow-all-button`, so
+ * neither is matched here. We verify the first text button actually reads
+ * "Allow" before clicking to avoid ever hitting "Skip".
+ *
+ * Uses the VS Code driver (`getElements`/`click`) rather than a raw Playwright
+ * locator: the confirmation renders outside the chat response/editor
+ * containers, and the driver reliably resolves it workbench-wide.
+ */
+export async function acceptToolConfirmationIfPresent(code: Code): Promise<void> {
+	const allowButtonSelector = '.chat-confirmation-widget2 .monaco-button.monaco-text-button';
+	try {
+		const buttons = await code.driver.getElements(allowButtonSelector, /* recursive */ true);
+		// The first text button in the widget is "Allow"; only proceed when
+		// that holds so we never accidentally hit "Skip".
+		if (!buttons || buttons.length === 0 || (buttons[0].textContent ?? '').trim() !== 'Allow') {
+			return;
+		}
+		// Filter to only visible "Allow" buttons: the DOM can contain multiple
+		// widgets (e.g. a stale one plus the currently active one, or a
+		// sandbox-prerequisite confirmation plus the terminal one), and only
+		// one is user-actionable at a time. `.first()` alone would race on
+		// element order and often pick a hidden one.
+		const allowButtons = code.driver.currentPage
+			.locator('.chat-confirmation-widget2 .monaco-button.monaco-text-button')
+			.filter({ hasText: /^Allow$/ });
+		const total = await allowButtons.count();
+		for (let i = 0; i < total; i++) {
+			const b = allowButtons.nth(i);
+			if (await b.isVisible()) {
+				await b.click({ timeout: 2_000 });
+				return;
+			}
+		}
+	} catch {
+		// Ignore: the confirmation may not be present, may not be actionable
+		// yet, or may have just been dismissed between the query and the click.
+		// The surrounding poll retries.
+	}
+}