diff --git a/build/azure-pipelines/linux/steps/product-build-linux-test.yml b/build/azure-pipelines/linux/steps/product-build-linux-test.yml index 57b712fe1c3492..6a28f5701b4d68 100644 --- a/build/azure-pipelines/linux/steps/product-build-linux-test.yml +++ b/build/azure-pipelines/linux/steps/product-build-linux-test.yml @@ -143,6 +143,18 @@ steps: timeoutInMinutes: 20 displayName: πŸ§ͺ Run smoke tests (Electron) + - ${{ if eq(parameters.VSCODE_RUN_ELECTRON_TESTS, true) }}: + - script: | + set -e + for i in $(seq 1 20); do + echo "=== Probe iteration $i/20 ===" + npm run smoketest-no-compile -- -g "run in terminal" --tracing --build "$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)" + done + env: + TMPDIR: $(Agent.TempDirectory) + timeoutInMinutes: 120 + displayName: πŸ§ͺ PROBE β€” run "run in terminal" smoke tests 20Γ— + - ${{ if eq(parameters.VSCODE_RUN_BROWSER_TESTS, true) }}: - script: npm run smoketest-no-compile -- --web --tracing --headless env: diff --git a/build/azure-pipelines/win32/steps/product-build-win32-test.yml b/build/azure-pipelines/win32/steps/product-build-win32-test.yml index 128ff569ca03ee..ca52abdb5d2447 100644 --- a/build/azure-pipelines/win32/steps/product-build-win32-test.yml +++ b/build/azure-pipelines/win32/steps/product-build-win32-test.yml @@ -135,6 +135,16 @@ steps: displayName: πŸ§ͺ Run smoke tests (Electron) timeoutInMinutes: 20 + - ${{ if eq(parameters.VSCODE_RUN_ELECTRON_TESTS, true) }}: + - powershell: | + for ($i = 1; $i -le 20; $i++) { + Write-Host "=== Probe iteration $i/20 ===" + npm run smoketest-no-compile -- -g "run in terminal" --tracing --build "$(agent.builddirectory)\test\VSCode-win32-$(VSCODE_ARCH)" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } + displayName: πŸ§ͺ PROBE β€” run "run in terminal" smoke tests 20Γ— + timeoutInMinutes: 120 + - ${{ if eq(parameters.VSCODE_RUN_BROWSER_TESTS, true) }}: - powershell: npm run smoketest-no-compile -- --web --tracing --headless env: diff --git a/test/automation/src/agentsWindow.ts b/test/automation/src/agentsWindow.ts index cad5d36b3f1f15..6d69b7b09d8a6f 100644 --- a/test/automation/src/agentsWindow.ts +++ b/test/automation/src/agentsWindow.ts @@ -4,6 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { Code } from './code'; +import { acceptToolConfirmationIfPresent } from './chat'; import { QuickAccess } from './quickaccess'; const AGENTS_WORKBENCH = '.agent-sessions-workbench'; @@ -428,7 +429,7 @@ export class AgentsWindow { * well after the content is on screen, so requiring `:not(.chat-response-loading)` * causes false-negative timeouts. */ - async waitForAssistantText(predicate: RegExp | string, timeoutMs: number = 60_000): Promise { + async waitForAssistantText(predicate: RegExp | string, timeoutMs: number = 60_000, options?: { acceptToolConfirmations?: boolean }): Promise { const retryCount = Math.ceil(timeoutMs / 100); await this.code.waitForElement(RESPONSE, undefined, retryCount); @@ -437,6 +438,12 @@ export class AgentsWindow { const deadline = Date.now() + timeoutMs; let lastTexts: string[] = []; while (Date.now() < deadline) { + // When requested, accept any pending terminal tool confirmation so + // the agentic loop can proceed. No-op for sessions that + // auto-approve their shell commands. + if (options?.acceptToolConfirmations) { + await acceptToolConfirmationIfPresent(this.code); + } // Look in BOTH the active session view and the broader workbench // scope. The Agents Window can auto-swap the active slot to a // fresh untitled session immediately after a follow-up commits, diff --git a/test/automation/src/chat.ts b/test/automation/src/chat.ts index 75a0853be5f57b..ba8b92628e478f 100644 --- a/test/automation/src/chat.ts +++ b/test/automation/src/chat.ts @@ -126,12 +126,12 @@ export class Chat { * the content has actually arrived (avoiding false matches on placeholder * text like "Considering" that appears before streaming begins). */ - async waitForResponseText(predicate: string | RegExp, timeoutMs: number = 60_000): Promise { - return await this.pollForResponseText(CHAT_RESPONSE, CHAT_RESPONSE_RENDERED, predicate, timeoutMs); + async waitForResponseText(predicate: string | RegExp, timeoutMs: number = 60_000, options?: { acceptToolConfirmations?: boolean }): Promise { + return await this.pollForResponseText(CHAT_RESPONSE, CHAT_RESPONSE_RENDERED, predicate, timeoutMs, options?.acceptToolConfirmations); } - async waitForEditorResponseText(predicate: string | RegExp, timeoutMs: number = 60_000): Promise { - const matched = await this.pollForResponseText(CHAT_EDITOR_RESPONSE, CHAT_EDITOR_RESPONSE_RENDERED, predicate, timeoutMs); + async waitForEditorResponseText(predicate: string | RegExp, timeoutMs: number = 60_000, options?: { acceptToolConfirmations?: boolean }): Promise { + const matched = await this.pollForResponseText(CHAT_EDITOR_RESPONSE, CHAT_EDITOR_RESPONSE_RENDERED, predicate, timeoutMs, options?.acceptToolConfirmations); // After a contributed chat session (e.g. Copilot CLI, Claude) returns // its first response, the workbench commits the untitled session into // a real (titled) one and `replaceEditors` swaps the chat editor over. @@ -164,10 +164,16 @@ export class Chat { } } - private async pollForResponseText(bubbleSelector: string, renderedSelector: string, predicate: string | RegExp, timeoutMs: number): Promise { + private async pollForResponseText(bubbleSelector: string, renderedSelector: string, predicate: string | RegExp, timeoutMs: number, acceptToolConfirmations?: boolean): Promise { const deadline = Date.now() + timeoutMs; const matches = (text: string) => typeof predicate === 'string' ? text.includes(predicate) : predicate.test(text); while (Date.now() < deadline) { + // When requested, accept any pending terminal tool confirmation so + // the agentic loop can proceed to render the final response. This + // is a no-op for sessions that auto-approve their shell commands. + if (acceptToolConfirmations) { + await acceptToolConfirmationIfPresent(this.code); + } const elements = await this.code.driver.getElements(renderedSelector, /* recursive */ true); const matched = elements.map(el => el.textContent ?? '').filter(matches); if (matched.length > 0) { @@ -474,3 +480,54 @@ export class Chat { } } } + +/** + * Click the "Allow" button of a pending terminal tool confirmation if one is + * present. No-op when there is no confirmation (e.g. the session auto-approved + * its shell command). Shared by {@link Chat} and the Agents Window driver so + * shell-tool tests can drive the real confirmation flow without per-agent + * special-casing. + * + * The terminal confirmation renders as a `.chat-confirmation-widget2` whose + * action buttons are `.monaco-button.monaco-text-button` anchors β€” the primary + * "Allow" (rendered first), then "Skip". The dropdown chevron next to "Allow" + * is a `.monaco-dropdown-button.codicon` (no `.monaco-text-button`) and the + * carousel's "Allow All" lives on `.chat-tool-carousel-allow-all-button`, so + * neither is matched here. We verify the first text button actually reads + * "Allow" before clicking to avoid ever hitting "Skip". + * + * Uses the VS Code driver (`getElements`/`click`) rather than a raw Playwright + * locator: the confirmation renders outside the chat response/editor + * containers, and the driver reliably resolves it workbench-wide. + */ +export async function acceptToolConfirmationIfPresent(code: Code): Promise { + const allowButtonSelector = '.chat-confirmation-widget2 .monaco-button.monaco-text-button'; + try { + const buttons = await code.driver.getElements(allowButtonSelector, /* recursive */ true); + // The first text button in the widget is "Allow"; only proceed when + // that holds so we never accidentally hit "Skip". + if (!buttons || buttons.length === 0 || (buttons[0].textContent ?? '').trim() !== 'Allow') { + return; + } + // Filter to only visible "Allow" buttons: the DOM can contain multiple + // widgets (e.g. a stale one plus the currently active one, or a + // sandbox-prerequisite confirmation plus the terminal one), and only + // one is user-actionable at a time. `.first()` alone would race on + // element order and often pick a hidden one. + const allowButtons = code.driver.currentPage + .locator('.chat-confirmation-widget2 .monaco-button.monaco-text-button') + .filter({ hasText: /^Allow$/ }); + const total = await allowButtons.count(); + for (let i = 0; i < total; i++) { + const b = allowButtons.nth(i); + if (await b.isVisible()) { + await b.click({ timeout: 2_000 }); + return; + } + } + } catch { + // Ignore: the confirmation may not be present, may not be actionable + // yet, or may have just been dismissed between the query and the click. + // The surrounding poll retries. + } +} diff --git a/test/smoke/src/areas/agentsWindow/agentsWindow.test.ts b/test/smoke/src/areas/agentsWindow/agentsWindow.test.ts index 6618ff33dd2a9c..332e7714422dd3 100644 --- a/test/smoke/src/areas/agentsWindow/agentsWindow.test.ts +++ b/test/smoke/src/areas/agentsWindow/agentsWindow.test.ts @@ -9,6 +9,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { Application, ApplicationOptions, Logger } from '../../../../automation'; import { createApp, dumpFailureDiagnostics, getCopilotSmokeTestEnv, getMockLlmServerPath, installAppAfterHandler, installDiagnosticsHandler, installAllHandlers, MockLlmServer, suiteCrashPath, suiteLogsPath } from '../../utils'; +import { shellEchoResponseMatcher, shellEchoScenario } from '../chat/shellScenarios'; // Selector for the send button in the Agents Window new-session homepage. // Kept in sync with `SEND_BUTTON_ENABLED` in `test/automation/src/agentsWindow.ts` @@ -209,6 +210,79 @@ export function setup(logger: Logger) { let mockServer: MockLlmServer; + // Shell-tool scenarios for each session type. Each entry carries + // everything the registration step and the corresponding test need β€” + // scenario id, expected echoed reply, and the mock-LLM scenario + // factory (different tool names per surface: `bash`/`pwsh`/ + // `powershell` for SDK sessions vs `run_in_terminal` for the Local + // agent), plus optional per-session hooks for cold-start warm-up and + // extra assertions. Keeping the data here avoids drift between the + // scenario registration and the test that consumes it. + interface ShellSession { + readonly name: string; + readonly sessionType: string; + readonly scenarioId: string; + readonly reply: string; + readonly scenarioFactory: (reply: string) => unknown; + /** + * Override `chat.cli.sandbox.enabled` to `'off'` for this test. + * The Agents Window suite enables the Copilot CLI sandbox at the + * suite level (for the "Test Copilot CLI session (sandbox)" + * test), but the Win32 AppContainer backend returns + * `Experimental_CreateProcessInSandbox returned E_NOTIMPL` on dev + * machines without the corresponding velocity feature flags + * (61389575, 61155944) enabled, which would fail any Copilot + * shell-tool test on Windows. Set this for non-sandbox Copilot + * tests so they exercise the plain (non-sandboxed) shell path + * everywhere β€” including Windows dev machines and CI. + */ + readonly disableCliSandbox?: boolean; + /** Optional cold-start warm-up (e.g. Claude SDK bundling). */ + readonly warmUp?: (app: Application, label: string) => Promise; + /** Optional extra assertion run after the chat reply lands. */ + readonly extraAssertion?: (app: Application) => Promise; + } + + const SHELL_SESSIONS: readonly ShellSession[] = [ + { + name: 'Copilot', + sessionType: 'Copilot', + scenarioId: 'smoke-hello-copilot-shell', + reply: 'MOCKED_COPILOT_SHELL_RESPONSE', + scenarioFactory: shellEchoScenario, + disableCliSandbox: true, + // Confirm the shell tool actually executed by checking the + // CopilotCLISession diagnostic log. We don't care whether + // the command was sandboxed for this test. + extraAssertion: async (app) => { + const chatLogPath = path.join(app.logsPath, 'window2', 'exthost', 'GitHub.copilot-chat', 'GitHub Copilot Chat.log'); + const chatLog = await fs.promises.readFile(chatLogPath, 'utf8'); + assert.match( + chatLog, + /\[CopilotCLISession\] tool\.execution_complete /, + `expected tool.execution_complete in ${chatLogPath}` + ); + }, + }, + { + name: 'Claude', + sessionType: 'Claude', + scenarioId: 'smoke-hello-claude-shell', + reply: 'MOCKED_CLAUDE_SHELL_RESPONSE', + scenarioFactory: shellEchoScenario, + // Pre-pay the Claude cold-start cost so the real assertion + // below runs against a warm pipeline (see warmUpClaudeModel). + warmUp: (app, label) => warmUpClaudeModel(app, logger, label), + }, + // Note: there is intentionally no "Local" entry. The Local agent + // in the Agents Window does not include `run_in_terminal` in its + // advertised tool set, so the model's tool call is rejected with + // "Tool run_in_terminal is currently disabled by the user". + // The Chat Sessions "Test Local session run in terminal" test + // already covers `run_in_terminal` against the regular chat panel + // where the tool is available. + ]; + // Start the mock server BEFORE installAllHandlers' `before` runs so // the mock URL is available when we configure the app's env vars via // `optionsTransform`. @@ -226,21 +300,14 @@ export function setup(logger: Logger) { registerScenario(session.scenarioId2, new ScenarioBuilder().emit(session.reply2).build()); } - registerScenario(COPILOT_SANDBOX_SCENARIO_ID, { - type: 'multi-turn', - turns: [ - { - kind: 'tool-calls', - toolCalls: [ - { - toolNamePattern: /^(bash|pwsh|powershell)$/i, - arguments: { command: `echo ${COPILOT_SANDBOX_REPLY}` }, - }, - ], - }, - { kind: 'echo-last-message' }, - ], - }); + registerScenario(COPILOT_SANDBOX_SCENARIO_ID, shellEchoScenario(COPILOT_SANDBOX_REPLY)); + + // Shell-tool scenarios for the non-sandbox shell-tool tests + // (auto-approved by the default `chat.tools.terminal.autoApprove` + // entry for `echo`). + for (const shellSession of SHELL_SESSIONS) { + registerScenario(shellSession.scenarioId, shellSession.scenarioFactory(shellSession.reply)); + } registerScenario(CLAUDE_WARMUP_SCENARIO_ID, new ScenarioBuilder().emit(CLAUDE_WARMUP_REPLY).build()); @@ -483,6 +550,61 @@ export function setup(logger: Logger) { `expected tool.execution_complete with sandboxed=true in ${chatLogPath}` ); }); + + // Shell-tool variants for each session type β€” exercise the + // model-driven shell tool (`bash` / `pwsh` / `powershell` for the SDK + // sessions) on the first prompt and verify both that the command + // actually ran (the JSON tool result contains the echoed marker) and + // that the reply rendered in the chat. These run the "non-sandbox" + // path: the shell command surfaces a terminal confirmation, which the + // wait helper accepts by clicking "Allow" (a no-op for sessions that + // auto-approve their shell commands). + for (const shellSession of SHELL_SESSIONS) { + it(`Test ${shellSession.name} session run in terminal`, async function () { + const app = this.app as Application; + const label = `Agents Window/${shellSession.name} shell`; + try { + if (shellSession.disableCliSandbox) { + // Override the suite-level `chat.cli.sandbox.enabled: 'on'` + // (set in the suite `before` for the sandbox test) so the + // SDK runs the shell tool without the Win32 AppContainer + // backend, which fails with E_NOTIMPL on dev machines and + // CI agents that lack the velocity feature flags. Write + // directly to settings.json on disk (the configuration + // service has a file watcher) rather than opening the + // settings editor β€” that would steal focus from the + // Agents Window UI under test. + await overrideUserSettingOnDisk(app, 'github.copilot.chat.cli.sandbox.enabled', 'off'); + } + await app.workbench.agentsWindow.startNewSession(); + await app.workbench.agentsWindow.waitForNewSessionView(); + if (shellSession.warmUp) { + await shellSession.warmUp(app, label); + } else { + await app.workbench.agentsWindow.selectSessionType(shellSession.sessionType); + } + + const requestsBefore = mockServer.requestCount(); + await app.workbench.agentsWindow.submitNewSessionPrompt(`hello world [scenario:${shellSession.scenarioId}]`); + + const text = await app.workbench.agentsWindow.waitForAssistantText(shellEchoResponseMatcher(shellSession.reply), 120_000, { acceptToolConfirmations: true }); + logger.log(`${label} response: ${text}`); + + assert.ok( + mockServer.requestCount() > requestsBefore, + `expected the mock LLM server to have received a new request from the ${shellSession.name} shell session` + ); + + if (shellSession.extraAssertion) { + await shellSession.extraAssertion(app); + } + } catch (error) { + logger.log(`[${label}] FAILURE: ${error instanceof Error ? error.stack ?? error.message : String(error)}`); + await dumpFailureDiagnostics(app, logger, label, { sendButtonSelector: AGENTS_SEND_BUTTON_SELECTOR }); + throw error; + } + }); + } }); describe('Agents Window (model configuration)', function () { @@ -1003,54 +1125,6 @@ export function setup(logger: Logger) { }); } -/** - * Builds a two-turn mock scenario that exercises a sandboxed shell tool: the - * model first runs `echo ` via the bash/pwsh/powershell tool, then β€” - * after the tool result round-trips β€” replays the last (tool-result) message - * back as a ```json fenced block via `echo-last-message`. - * - * The reply text therefore appears in two kinds of `.rendered-markdown` - * elements (both searched by {@link AgentsWindow.waitForAssistantText}): - * 1. the terminal tool-call's command preview β€” rendered as `echo ` - * (the bareword, no surrounding quotes), and - * 2. the final assistant response β€” the JSON dump of the tool result, an - * object whose `output` field holds the echoed `` (possibly with - * a prefix, e.g. shell-integration noise, and an `` suffix). - * To assert on the real response (2) and not the command preview (1), callers - * match with {@link shellEchoResponseMatcher} β€” see the sandbox tests. - */ -function shellEchoScenario(reply: string) { - return { - type: 'multi-turn', - turns: [ - { - kind: 'tool-calls', - toolCalls: [ - { - toolNamePattern: /^(bash|pwsh|powershell)$/i, - arguments: { command: `echo ${reply}` }, - }, - ], - }, - { kind: 'echo-last-message' }, - ], - }; -} - -/** - * Builds the {@link AgentsWindow.waitForAssistantText} matcher for a - * {@link shellEchoScenario} reply. The final response renders the tool result - * as a ```json block of the form - * `{ ..., "output": "\n" }`, so anchoring on - * `"output": ... ` matches that JSON value specifically β€” not the - * `echo ` command preview (which has no `"output"` field) β€” while still - * tolerating any prefix inside the captured output (e.g. shell-integration - * noise). `` contains no regex metacharacters. - */ -function shellEchoResponseMatcher(reply: string): RegExp { - return new RegExp(`"output":.*${reply}`); -} - /** * Primes a freshly-spawned AgentHost process's CLI model list to avoid the * cold-start "No model available" race (github/copilot-agent-runtime#9876): @@ -1286,3 +1360,34 @@ function ahpJsonlFiles(ahpLogDir: string): string[] { function readAhpFrames(ahpLogDir: string): string { return ahpJsonlFiles(ahpLogDir).map(f => fs.readFileSync(path.join(ahpLogDir, f), 'utf8')).join('\n'); } + +/** + * Override a single user-scope VS Code setting by editing + * `/User/settings.json` directly on disk. The configuration + * service watches the file and picks up the change. Preferred over + * {@link Settings.addUserSetting} when the workbench has switched to a + * secondary window (Agents Window) where opening the settings editor would + * steal focus from the UI under test. + */ +async function overrideUserSettingOnDisk(app: Application, key: string, value: unknown): Promise { + const userDataDir = app.userDataPath; + if (!userDataDir) { + throw new Error('overrideUserSettingOnDisk: app.userDataPath is unset'); + } + const settingsPath = path.join(userDataDir, 'User', 'settings.json'); + let current: Record = {}; + try { + const raw = await fs.promises.readFile(settingsPath, 'utf8'); + // Strip trailing comma the settings editor may emit and accept JSONC. + current = JSON.parse(raw.replace(/,(\s*[}\]])/g, '$1')) as Record; + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== 'ENOENT') { + throw err; + } + } + current[key] = value; + await fs.promises.writeFile(settingsPath, JSON.stringify(current, null, '\t')); + // The configuration service debounces file watcher events; give it a + // moment to pick up the change before downstream code reads the setting. + await new Promise(resolve => setTimeout(resolve, 500)); +} diff --git a/test/smoke/src/areas/chat/chatSessions.test.ts b/test/smoke/src/areas/chat/chatSessions.test.ts index e7c4a6b31d91da..db55428d21135f 100644 --- a/test/smoke/src/areas/chat/chatSessions.test.ts +++ b/test/smoke/src/areas/chat/chatSessions.test.ts @@ -6,6 +6,7 @@ import * as assert from 'assert'; import { Application, Chat, Logger } from '../../../../automation'; import { dumpFailureDiagnostics, getCopilotSmokeTestEnv, getMockLlmServerPath, installAllHandlers, MockLlmServer } from '../../utils'; +import { runInTerminalScenario, shellEchoResponseMatcher, shellEchoScenario } from './shellScenarios'; /** * Per-session scenarios. Each session uses a pair of unique scenario ids so @@ -35,7 +36,29 @@ const SESSIONS: readonly SessionConfig[] = [ { name: 'Local', command: 'workbench.action.chat.open', kind: 'view', scenarioId: 'smoke-chat-sessions-local', reply: 'MOCKED_CHAT_SESSIONS_LOCAL_RESPONSE', scenarioId2: 'smoke-chat-sessions-local-2', reply2: 'MOCKED_CHAT_SESSIONS_LOCAL_RESPONSE_2' }, ]; -async function openSession(app: Application, session: SessionConfig): Promise { +/** + * Per-session shell-tool scenarios. Each session triggers a shell tool call + * on the first prompt and verifies the echoed marker appears in the chat + * (which proves both that the command ran and that the reply was rendered). + * SDK-based sessions (Copilot CLI, Claude) advertise `bash`/`pwsh`/ + * `powershell`; the Local chat agent advertises `run_in_terminal`. + */ +interface ShellSessionConfig { + readonly name: string; + readonly command: string; + readonly kind: 'editor' | 'view'; + readonly scenarioId: string; + readonly reply: string; + readonly scenarioFactory: (reply: string) => unknown; +} + +const SHELL_SESSIONS: readonly ShellSessionConfig[] = [ + { name: 'Copilot CLI', command: 'smoketest.openCopilotCliChat', kind: 'editor', scenarioId: 'smoke-chat-sessions-copilot-cli-shell', reply: 'MOCKED_CHAT_SESSIONS_COPILOT_CLI_SHELL_RESPONSE', scenarioFactory: shellEchoScenario }, + { name: 'Claude', command: 'smoketest.openClaudeChat', kind: 'editor', scenarioId: 'smoke-chat-sessions-claude-shell', reply: 'MOCKED_CHAT_SESSIONS_CLAUDE_SHELL_RESPONSE', scenarioFactory: shellEchoScenario }, + { name: 'Local', command: 'workbench.action.chat.open', kind: 'view', scenarioId: 'smoke-chat-sessions-local-terminal', reply: 'MOCKED_CHAT_SESSIONS_LOCAL_TERMINAL_RESPONSE', scenarioFactory: runInTerminalScenario }, +]; + +async function openSession(app: Application, session: { readonly command: string; readonly kind: 'editor' | 'view' }): Promise { await app.workbench.quickaccess.runCommand(session.command); if (session.kind === 'editor') { await app.workbench.chat.waitForChatEditor(600); @@ -84,6 +107,14 @@ export function setup(logger: Logger) { registerScenario(session.scenarioId2, new ScenarioBuilder().emit(session.reply2).build()); } + // Shell-tool scenarios. `echo` is in the default + // `chat.tools.terminal.autoApprove` list, so no extra settings + // are required to auto-approve the command β€” these tests + // deliberately exercise the non-sandbox shell-tool path. + for (const shellSession of SHELL_SESSIONS) { + registerScenario(shellSession.scenarioId, shellSession.scenarioFactory(shellSession.reply)); + } + mockServer = await startServer(0, { logger: (msg: string) => logger.log(`[mock-llm] ${msg}`) }); logger.log(`[Chat Sessions] mock LLM server started at ${mockServer.url} (platform=${process.platform}, arch=${process.arch}, node=${process.version})`); logger.log(`[Chat Sessions] env: VSCODE_DEV=${process.env.VSCODE_DEV ?? ''}, VSCODE_QUALITY=${process.env.VSCODE_QUALITY ?? ''}, BUILD_SOURCEBRANCH=${process.env.BUILD_SOURCEBRANCH ?? ''}, GITHUB_RUN_ID=${process.env.GITHUB_RUN_ID ?? ''}, GITHUB_ACTIONS=${process.env.GITHUB_ACTIONS ?? ''}`); @@ -131,6 +162,13 @@ export function setup(logger: Logger) { // would route through the ms-vscode.vscode-claude-sdk extension, // which would attempt a network install during the smoke run). ['github.copilot.chat.claudeAgent.useSdkExtension', 'false'], + // Disable the LLM-generated tool risk assessment. It issues a + // separate model request whose mock reply ("OK") never resolves + // to a valid Safe/Caution/Review verdict, which would otherwise + // leave the terminal confirmation stuck in the "Assessing risk…" + // state so its "Allow" button never becomes available. The + // shell-tool tests need to click "Allow" to proceed. + ['chat.tools.riskAssessment.enabled', 'false'], ]); logger.log(`[Chat Sessions] user settings written; requestCount=${mockServer.requestCount()}`); }); @@ -187,5 +225,58 @@ export function setup(logger: Logger) { } }); } + + // Shell-tool variant per session β€” exercises the model-driven shell + // tool (`bash`/`pwsh`/`powershell` for the SDK sessions, or + // `run_in_terminal` for the Local session) on the first prompt and + // verifies both that the command actually ran (the JSON tool result + // contains the echoed marker) and that the reply was rendered in the + // chat. `echo` is in the default `chat.tools.terminal.autoApprove` + // list, so no extra auto-approve settings are required. + for (const shellSession of SHELL_SESSIONS) { + it(`Test ${shellSession.name} session run in terminal`, async function () { + const app = this.app as Application; + const requestsBefore = mockServer.requestCount(); + logger.log(`[Chat Sessions/${shellSession.name} shell] starting test; requestCount=${requestsBefore}`); + + try { + await openSession(app, shellSession); + + const prompt = `hello world [scenario:${shellSession.scenarioId}]`; + const matcher = shellEchoResponseMatcher(shellSession.reply); + let responseText: string; + if (shellSession.kind === 'editor') { + await app.workbench.chat.sendEditorMessage(prompt); + // 120s timeout β€” SDK + shell tool round-trip can be slow on cold CI. + // acceptToolConfirmations clicks "Allow" on the terminal + // confirmation so the command runs (no-op when the session + // auto-approves). + responseText = (await app.workbench.chat.waitForEditorResponseText(matcher, 120_000, { acceptToolConfirmations: true })).trim(); + } else { + await app.workbench.chat.sendMessage(prompt); + responseText = (await app.workbench.chat.waitForResponseText(matcher, 120_000, { acceptToolConfirmations: true })).trim(); + } + logger.log(`Chat Sessions (${shellSession.name} shell) response: ${responseText}`); + + assert.match( + responseText, + matcher, + `Expected ${shellSession.name} shell response to include the echoed marker "${shellSession.reply}" inside a JSON tool result string.\n\nResponse:\n${responseText}` + ); + + assert.ok( + mockServer.requestCount() > requestsBefore, + `expected the mock LLM server to have received a new request from the ${shellSession.name} shell session (before=${requestsBefore}, after=${mockServer.requestCount()})` + ); + } catch (error) { + logger.log(`[Chat Sessions/${shellSession.name} shell] FAILURE: ${error instanceof Error ? error.stack ?? error.message : String(error)}`); + logger.log(`[Chat Sessions/${shellSession.name} shell] mock server requestCount at failure: ${mockServer.requestCount()} (before=${requestsBefore})`); + await dumpFailureDiagnostics(app, logger, `Chat Sessions/${shellSession.name} shell`); + throw error; + } finally { + await app.workbench.quickaccess.runCommand('workbench.action.closeAllEditors'); + } + }); + } }); } diff --git a/test/smoke/src/areas/chat/shellScenarios.ts b/test/smoke/src/areas/chat/shellScenarios.ts new file mode 100644 index 00000000000000..6f986a4705bd6a --- /dev/null +++ b/test/smoke/src/areas/chat/shellScenarios.ts @@ -0,0 +1,97 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** + * Mock-LLM scenarios + response matchers used by smoke tests that exercise a + * model-driven shell tool invocation and verify both: + * 1. The tool actually ran (the tool result contains the echoed marker), and + * 2. The reply appears in the chat (the same marker is rendered). + * + * Two variants are provided to cover the different tool names advertised by + * each surface: + * - {@link shellEchoScenario} matches SDK-based sessions (Copilot CLI, + * Claude, AgentHost), which expose `bash` / `pwsh` / `powershell` tools. + * - {@link runInTerminalScenario} matches the VS Code built-in chat agent + * (used by the "Local" session), which exposes the `run_in_terminal` + * tool. + * + * Both scenarios use a two-turn protocol: + * - Turn 1 (`tool-calls`): the model asks the tool to execute + * `echo `. + * - Turn 2 (`echo-last-message`): the model replays the last + * (tool-result) message back as a ```json fenced block, so the reply + * appears in the assistant rendering. + * + * Importantly, `echo` is in the default `chat.tools.terminal.autoApprove` + * list, so no extra settings are required to auto-approve the command. + */ + +export function shellEchoScenario(reply: string): unknown { + return { + type: 'multi-turn', + turns: [ + { + kind: 'tool-calls', + toolCalls: [ + { + toolNamePattern: /^(bash|pwsh|powershell)$/i, + arguments: { command: `echo ${reply}` }, + }, + ], + }, + { kind: 'echo-last-message' }, + ], + }; +} + +export function runInTerminalScenario(reply: string): unknown { + return { + type: 'multi-turn', + turns: [ + { + kind: 'tool-calls', + toolCalls: [ + { + toolNamePattern: /^run_in_terminal$/, + arguments: { + command: `echo ${reply}`, + explanation: 'Smoke test echo to verify run_in_terminal', + goal: 'Echo a marker to verify terminal execution', + mode: 'sync', + }, + }, + ], + }, + { kind: 'echo-last-message' }, + ], + }; +} + +/** + * Matcher for the assistant text produced by {@link shellEchoScenario} or + * {@link runInTerminalScenario}. The final response renders the tool result + * as a ```json block; the exact format varies by surface: + * - Copilot CLI (responses-API): `{ "output": "" }` + * - Local `run_in_terminal`: `{ "output": "..." }` + * - Claude SDK Bash (messages-API): `{ "content": "" }` + * - AgentHost custom terminal tool: `{ "output": "ShellID: ...\nExitcode: 0\n" }` + * + * Anchoring on a JSON double-quote OR newline immediately preceding the reply + * matches: + * - the value side of any `"": "..."` pair (Copilot CLI, Claude, + * Local `run_in_terminal`), AND + * - the AgentHost custom-terminal format where the reply is preceded by + * `\n` inside a larger `"output"` string. + * + * This excludes the `echo ` command preview (a bareword surrounded by + * whitespace, not `"` or `\n`). `` must not contain regex + * metacharacters. + */ +export function shellEchoResponseMatcher(reply: string): RegExp { + // The rendered JSON block collapses `\n` in the string value into a real + // newline in `textContent`, but the assistant output can also contain + // literal escaped `\n` when the JSON is displayed verbatim; match either. + return new RegExp(`(?:"|\\n|\\\\n)${reply}`); +}