diff --git a/autoresearch/eval-publish.ts b/autoresearch/eval-publish.ts index ea748f51..7cb43ac5 100644 --- a/autoresearch/eval-publish.ts +++ b/autoresearch/eval-publish.ts @@ -78,7 +78,7 @@ function judge(criteria: JudgeCriteria, output: string): boolean { } function runCommand(cmd: string, timeout = 30000): string { - const localCmd = cmd.replace(/^opencli /, `node dist/main.js `); + const localCmd = cmd.replace(/^opencli /, `node dist/src/main.js `); try { return execSync(localCmd, { cwd: PROJECT_ROOT, diff --git a/autoresearch/eval-save.ts b/autoresearch/eval-save.ts index d3384079..acce88f6 100644 --- a/autoresearch/eval-save.ts +++ b/autoresearch/eval-save.ts @@ -83,7 +83,7 @@ const PROJECT_ROOT = join(__dirname, '..'); /** Run a command, using local dist/main.js instead of global opencli for consistency */ function runCommand(cmd: string, timeout = 30000): string { // Use local build so tests always run against the current source - const localCmd = cmd.replace(/^opencli /, `node dist/main.js `); + const localCmd = cmd.replace(/^opencli /, `node dist/src/main.js `); try { return execSync(localCmd, { cwd: PROJECT_ROOT, diff --git a/skills/opencli-repair/SKILL.md b/skills/opencli-repair/SKILL.md new file mode 100644 index 00000000..96df2c51 --- /dev/null +++ b/skills/opencli-repair/SKILL.md @@ -0,0 +1,203 @@ +--- +name: opencli-repair +description: Diagnose and fix broken OpenCLI adapters when websites change. Use when an opencli command fails with SELECTOR, EMPTY_RESULT, API_ERROR, or PAGE_CHANGED errors. Reads structured diagnostic output and uses browser automation to discover what changed and patch the adapter. +allowed-tools: Bash(opencli:*), Read, Edit, Write +--- + +# OpenCLI Repair — AI-Driven Adapter Self-Repair + +When an adapter breaks because a website changed its DOM, API, or auth flow, use this skill to diagnose the failure and patch the adapter. + +## Prerequisites + +```bash +opencli doctor # Verify extension + daemon connectivity +``` + +## When to Use This Skill + +Use when `opencli ` fails with errors like: +- **SELECTOR** — element not found (DOM changed) +- **EMPTY_RESULT** — no data returned (API response changed) +- **API_ERROR** / **NETWORK** — endpoint moved or broke +- **PAGE_CHANGED** — page structure no longer matches +- **COMMAND_EXEC** — runtime error in adapter logic +- **TIMEOUT** — page loads differently, adapter waits for wrong thing + +## Step 1: Collect Diagnostic Context + +Run the failing command with diagnostic mode enabled: + +```bash +OPENCLI_DIAGNOSTIC=1 opencli [args...] 2>diagnostic.json +``` + +This outputs a `RepairContext` JSON between `___OPENCLI_DIAGNOSTIC___` markers in stderr: + +```json +{ + "error": { + "code": "SELECTOR", + "message": "Could not find element: .old-selector", + "hint": "The page UI may have changed." + }, + "adapter": { + "site": "example", + "command": "example/search", + "sourcePath": "/path/to/clis/example/search.ts", + "source": "// full adapter source code" + }, + "page": { + "url": "https://example.com/search", + "snapshot": "// DOM snapshot with [N] indices", + "networkRequests": [], + "consoleErrors": [] + }, + "timestamp": "2025-01-01T00:00:00.000Z" +} +``` + +**Parse it:** +```bash +# Extract JSON between markers from stderr output +cat diagnostic.json | sed -n '/___OPENCLI_DIAGNOSTIC___/{n;p;}' +``` + +## Step 2: Analyze the Failure + +Read the diagnostic context and the adapter source. Classify the root cause: + +| Error Code | Likely Cause | Repair Strategy | +|-----------|-------------|-----------------| +| SELECTOR | DOM restructured, class/id renamed | Explore current DOM → find new selector | +| EMPTY_RESULT | API response schema changed, or data moved | Check network → find new response path | +| API_ERROR | Endpoint URL changed, new params required | Discover new API via network intercept | +| AUTH_REQUIRED | Login flow changed, cookies expired | Walk login flow with operate | +| TIMEOUT | Page loads differently, spinner/lazy-load | Add/update wait conditions | +| PAGE_CHANGED | Major redesign | May need full adapter rewrite | + +**Key questions to answer:** +1. What is the adapter trying to do? (Read the `source` field) +2. What did the page look like when it failed? (Read the `snapshot` field) +3. What network requests happened? (Read `networkRequests`) +4. What's the gap between what the adapter expects and what the page provides? + +## Step 3: Explore the Current Website + +Use `opencli operate` to inspect the live website. **Never use the broken adapter** — it will just fail again. + +### DOM changed (SELECTOR errors) + +```bash +# Open the page and inspect current DOM +opencli operate open https://example.com/target-page && opencli operate state + +# Look for elements that match the adapter's intent +# Compare the snapshot with what the adapter expects +``` + +### API changed (API_ERROR, EMPTY_RESULT) + +```bash +# Open page with network interceptor, then trigger the action manually +opencli operate open https://example.com/target-page && opencli operate state + +# Interact to trigger API calls +opencli operate click && opencli operate network + +# Inspect specific API response +opencli operate network --detail +``` + +### Auth changed (AUTH_REQUIRED) + +```bash +# Check current auth state +opencli operate open https://example.com && opencli operate state + +# If login page: inspect the login form +opencli operate state # Look for login form fields +``` + +## Step 4: Patch the Adapter + +Read the adapter source file and make targeted fixes: + +```bash +# Read the adapter +cat +``` + +### Common Fixes + +**Selector update:** +```typescript +// Before: page.evaluate('document.querySelector(".old-class")...') +// After: page.evaluate('document.querySelector(".new-class")...') +``` + +**API endpoint change:** +```typescript +// Before: const resp = await page.evaluate(`fetch('/api/v1/old-endpoint')...`) +// After: const resp = await page.evaluate(`fetch('/api/v2/new-endpoint')...`) +``` + +**Response schema change:** +```typescript +// Before: const items = data.results +// After: const items = data.data.items // API now nests under "data" +``` + +**Wait condition update:** +```typescript +// Before: await page.waitForSelector('.loading-spinner', { hidden: true }) +// After: await page.waitForSelector('[data-loaded="true"]') +``` + +### Rules for Patching + +1. **Make minimal changes** — fix only what's broken, don't refactor +2. **Keep the same output structure** — `columns` and return format must stay compatible +3. **Prefer API over DOM scraping** — if you discover a JSON API during exploration, switch to it +4. **Use `@jackwener/opencli/*` imports only** — never add third-party package imports +5. **Test after patching** — run the command again to verify + +## Step 5: Verify the Fix + +```bash +# Run the command normally (without diagnostic mode) +opencli [args...] +``` + +If it still fails, go back to Step 3 and explore further. If the website has fundamentally changed (major redesign, removed feature), report that the adapter needs a full rewrite. + +## When to Give Up + +Not all failures are repairable with a quick patch: + +- **Site requires CAPTCHA** — can't automate this +- **Feature completely removed** — the data no longer exists +- **Major redesign** — needs full adapter rewrite via `opencli-explorer` skill +- **Rate limited / IP blocked** — not an adapter issue + +In these cases, clearly communicate the situation to the user rather than making futile patches. + +## Example Repair Session + +``` +1. User runs: opencli zhihu hot + → Fails: SELECTOR "Could not find element: .HotList-item" + +2. AI runs: OPENCLI_DIAGNOSTIC=1 opencli zhihu hot 2>diag.json + → Gets RepairContext with DOM snapshot showing page loaded + +3. AI reads diagnostic: snapshot shows the page loaded but uses ".HotItem" instead of ".HotList-item" + +4. AI explores: opencli operate open https://www.zhihu.com/hot && opencli operate state + → Confirms new class name ".HotItem" with child ".HotItem-content" + +5. AI patches: Edit clis/zhihu/hot.ts — replace ".HotList-item" with ".HotItem" + +6. AI verifies: opencli zhihu hot + → Success: returns hot topics +``` diff --git a/src/diagnostic.test.ts b/src/diagnostic.test.ts new file mode 100644 index 00000000..5a7f0a70 --- /dev/null +++ b/src/diagnostic.test.ts @@ -0,0 +1,100 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { buildRepairContext, isDiagnosticEnabled, emitDiagnostic, type RepairContext } from './diagnostic.js'; +import { CliError, SelectorError, CommandExecutionError } from './errors.js'; +import type { InternalCliCommand } from './registry.js'; + +function makeCmd(overrides: Partial = {}): InternalCliCommand { + return { + site: 'test-site', + name: 'test-cmd', + description: 'test', + args: [], + ...overrides, + } as InternalCliCommand; +} + +describe('isDiagnosticEnabled', () => { + const origEnv = process.env.OPENCLI_DIAGNOSTIC; + afterEach(() => { + if (origEnv === undefined) delete process.env.OPENCLI_DIAGNOSTIC; + else process.env.OPENCLI_DIAGNOSTIC = origEnv; + }); + + it('returns false when env not set', () => { + delete process.env.OPENCLI_DIAGNOSTIC; + expect(isDiagnosticEnabled()).toBe(false); + }); + + it('returns true when env is "1"', () => { + process.env.OPENCLI_DIAGNOSTIC = '1'; + expect(isDiagnosticEnabled()).toBe(true); + }); + + it('returns false for other values', () => { + process.env.OPENCLI_DIAGNOSTIC = 'true'; + expect(isDiagnosticEnabled()).toBe(false); + }); +}); + +describe('buildRepairContext', () => { + it('captures CliError fields', () => { + const err = new SelectorError('.missing-element', 'Element removed'); + const ctx = buildRepairContext(err, makeCmd()); + + expect(ctx.error.code).toBe('SELECTOR'); + expect(ctx.error.message).toContain('.missing-element'); + expect(ctx.error.hint).toBe('Element removed'); + expect(ctx.error.stack).toBeDefined(); + expect(ctx.adapter.site).toBe('test-site'); + expect(ctx.adapter.command).toBe('test-site/test-cmd'); + expect(ctx.timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T/); + }); + + it('handles non-CliError errors', () => { + const err = new TypeError('Cannot read property "x" of undefined'); + const ctx = buildRepairContext(err, makeCmd()); + + expect(ctx.error.code).toBe('UNKNOWN'); + expect(ctx.error.message).toContain('Cannot read property'); + expect(ctx.error.hint).toBeUndefined(); + }); + + it('includes page state when provided', () => { + const pageState: RepairContext['page'] = { + url: 'https://example.com/page', + snapshot: '
...
', + networkRequests: [{ url: '/api/data', status: 200 }], + consoleErrors: ['Uncaught TypeError'], + }; + const ctx = buildRepairContext(new CommandExecutionError('boom'), makeCmd(), pageState); + + expect(ctx.page).toEqual(pageState); + }); + + it('omits page when not provided', () => { + const ctx = buildRepairContext(new Error('boom'), makeCmd()); + expect(ctx.page).toBeUndefined(); + }); +}); + +describe('emitDiagnostic', () => { + it('writes delimited JSON to stderr', () => { + const writeSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); + + const ctx = buildRepairContext(new CommandExecutionError('test error'), makeCmd()); + emitDiagnostic(ctx); + + const output = writeSpy.mock.calls.map(c => c[0]).join(''); + expect(output).toContain('___OPENCLI_DIAGNOSTIC___'); + expect(output).toContain('"code":"COMMAND_EXEC"'); + expect(output).toContain('"message":"test error"'); + + // Verify JSON is parseable between markers + const match = output.match(/___OPENCLI_DIAGNOSTIC___\n(.*)\n___OPENCLI_DIAGNOSTIC___/); + expect(match).toBeTruthy(); + const parsed = JSON.parse(match![1]); + expect(parsed.error.code).toBe('COMMAND_EXEC'); + + writeSpy.mockRestore(); + }); +}); diff --git a/src/diagnostic.ts b/src/diagnostic.ts new file mode 100644 index 00000000..131a552c --- /dev/null +++ b/src/diagnostic.ts @@ -0,0 +1,110 @@ +/** + * Structured diagnostic output for AI-driven adapter repair. + * + * When OPENCLI_DIAGNOSTIC=1, failed commands emit a JSON RepairContext to stderr + * containing the error, adapter source, and browser state (DOM snapshot, network + * requests, console errors). AI Agents consume this to diagnose and fix adapters. + */ + +import * as fs from 'node:fs'; +import type { IPage } from './types.js'; +import { CliError, getErrorMessage } from './errors.js'; +import type { InternalCliCommand } from './registry.js'; +import { fullName } from './registry.js'; + +// ── Types ──────────────────────────────────────────────────────────────────── + +export interface RepairContext { + error: { + code: string; + message: string; + hint?: string; + stack?: string; + }; + adapter: { + site: string; + command: string; + sourcePath?: string; + source?: string; + }; + page?: { + url: string; + snapshot: string; + networkRequests: unknown[]; + consoleErrors: unknown[]; + }; + timestamp: string; +} + +// ── Diagnostic collection ──────────────────────────────────────────────────── + +/** Whether diagnostic mode is enabled. */ +export function isDiagnosticEnabled(): boolean { + return process.env.OPENCLI_DIAGNOSTIC === '1'; +} + +/** Safely collect page diagnostic state. Individual failures are swallowed. */ +async function collectPageState(page: IPage): Promise { + try { + const [url, snapshot, networkRequests, consoleErrors] = await Promise.all([ + page.getCurrentUrl?.().catch(() => null) ?? Promise.resolve(null), + page.snapshot().catch(() => '(snapshot unavailable)'), + page.networkRequests().catch(() => []), + page.consoleMessages('error').catch(() => []), + ]); + return { url: url ?? 'unknown', snapshot, networkRequests, consoleErrors }; + } catch { + return undefined; + } +} + +/** Read adapter source file content. */ +function readAdapterSource(modulePath: string | undefined): string | undefined { + if (!modulePath) return undefined; + try { + return fs.readFileSync(modulePath, 'utf-8'); + } catch { + return undefined; + } +} + +/** Build a RepairContext from an error, command metadata, and optional page state. */ +export function buildRepairContext( + err: unknown, + cmd: InternalCliCommand, + pageState?: RepairContext['page'], +): RepairContext { + const isCliError = err instanceof CliError; + return { + error: { + code: isCliError ? err.code : 'UNKNOWN', + message: getErrorMessage(err), + hint: isCliError ? err.hint : undefined, + stack: err instanceof Error ? err.stack : undefined, + }, + adapter: { + site: cmd.site, + command: fullName(cmd), + sourcePath: cmd._modulePath, + source: readAdapterSource(cmd._modulePath), + }, + page: pageState, + timestamp: new Date().toISOString(), + }; +} + +/** Collect full diagnostic context including page state. */ +export async function collectDiagnostic( + err: unknown, + cmd: InternalCliCommand, + page: IPage | null, +): Promise { + const pageState = page ? await collectPageState(page) : undefined; + return buildRepairContext(err, cmd, pageState); +} + +/** Emit diagnostic JSON to stderr. */ +export function emitDiagnostic(ctx: RepairContext): void { + const marker = '___OPENCLI_DIAGNOSTIC___'; + process.stderr.write(`\n${marker}\n${JSON.stringify(ctx)}\n${marker}\n`); +} diff --git a/src/execution.ts b/src/execution.ts index efd1f1f4..fbd8226d 100644 --- a/src/execution.ts +++ b/src/execution.ts @@ -15,6 +15,7 @@ import type { IPage } from './types.js'; import { pathToFileURL } from 'node:url'; import { executePipeline } from './pipeline/index.js'; import { AdapterLoadError, ArgumentError, BrowserConnectError, CommandExecutionError, getErrorMessage } from './errors.js'; +import { isDiagnosticEnabled, collectDiagnostic, emitDiagnostic } from './diagnostic.js'; import { shouldUseBrowserSession } from './capabilityRouting.js'; import { getBrowserFactory, browserSession, runWithTimeout, DEFAULT_BROWSER_COMMAND_TIMEOUT } from './runtime.js'; import { emitHook, type HookContext } from './hooks.js'; @@ -153,6 +154,7 @@ export async function executeCommand( await emitHook('onBeforeExecute', hookCtx); let result: unknown; + let diagnosticEmitted = false; try { if (shouldUseBrowserSession(cmd)) { const electron = isElectronApp(cmd.site); @@ -204,10 +206,21 @@ export async function executeCommand( if (debug) log.debug(`[pre-nav] Failed to navigate to ${preNavUrl}: ${err instanceof Error ? err.message : err}`); } } - return runWithTimeout(runCommand(cmd, page, kwargs, debug), { - timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT, - label: fullName(cmd), - }); + try { + return await runWithTimeout(runCommand(cmd, page, kwargs, debug), { + timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT, + label: fullName(cmd), + }); + } catch (err) { + // Collect diagnostic while page is still alive (before browserSession closes it). + if (isDiagnosticEnabled()) { + const internal = cmd as InternalCliCommand; + const ctx = await collectDiagnostic(err, internal, page); + emitDiagnostic(ctx); + diagnosticEmitted = true; + } + throw err; + } }, { workspace: `site:${cmd.site}`, cdpEndpoint }); } else { // Non-browser commands: apply timeout only when explicitly configured. @@ -223,6 +236,13 @@ export async function executeCommand( } } } catch (err) { + // Emit diagnostic if not already emitted (browser session emits with page state; + // this fallback covers non-browser commands and pre-session failures like BrowserConnectError). + if (isDiagnosticEnabled() && !diagnosticEmitted) { + const internal = cmd as InternalCliCommand; + const ctx = await collectDiagnostic(err, internal, null); + emitDiagnostic(ctx); + } hookCtx.error = err; hookCtx.finishedAt = Date.now(); await emitHook('onAfterExecute', hookCtx); diff --git a/tests/e2e/helpers.ts b/tests/e2e/helpers.ts index 4fab175d..2b75b30d 100644 --- a/tests/e2e/helpers.ts +++ b/tests/e2e/helpers.ts @@ -11,7 +11,7 @@ import { fileURLToPath } from 'node:url'; const exec = promisify(execFile); const __dirname = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(__dirname, '../..'); -const MAIN = path.join(ROOT, 'dist', 'main.js'); +const MAIN = path.join(ROOT, 'dist', 'src', 'main.js'); export interface CliResult { stdout: string; diff --git a/tests/e2e/public-commands.test.ts b/tests/e2e/public-commands.test.ts index acd87a7b..6c7ce959 100644 --- a/tests/e2e/public-commands.test.ts +++ b/tests/e2e/public-commands.test.ts @@ -31,6 +31,14 @@ function isExpectedGoogleRestriction(code: number, stderr: string): boolean { return /fetch failed/.test(stderr) || /Error \[FETCH_ERROR\]: HTTP (403|429|451|503)\b/.test(stderr); } +function isExpectedBloombergRestriction(code: number, stderr: string): boolean { + if (code === 0) return false; + return /Bloomberg RSS HTTP \d+/.test(stderr) + || /Bloomberg RSS feed returned no items/.test(stderr) + || /fetch failed/.test(stderr) + || stderr.trim() === ''; +} + // Keep old name as alias for existing tests const isExpectedXiaoyuzhouRestriction = isExpectedChineseSiteRestriction; @@ -48,7 +56,11 @@ describe('public command restriction detectors', () => { describe('public commands E2E', () => { // ── bloomberg (RSS-backed, browser: false) ── it('bloomberg main returns structured headline data', async () => { - const { stdout, code } = await runCli(['bloomberg', 'main', '--limit', '1', '-f', 'json']); + const { stdout, stderr, code } = await runCli(['bloomberg', 'main', '--limit', '1', '-f', 'json']); + if (isExpectedBloombergRestriction(code, stderr)) { + console.warn(`bloomberg main skipped: ${stderr.trim()}`); + return; + } expect(code).toBe(0); const data = parseJsonOutput(stdout); expect(Array.isArray(data)).toBe(true); @@ -69,7 +81,11 @@ describe('public commands E2E', () => { 'businessweek', 'opinions', ])('bloomberg %s returns structured RSS items', async (section) => { - const { stdout, code } = await runCli(['bloomberg', section, '--limit', '1', '-f', 'json']); + const { stdout, stderr, code } = await runCli(['bloomberg', section, '--limit', '1', '-f', 'json']); + if (isExpectedBloombergRestriction(code, stderr)) { + console.warn(`bloomberg ${section} skipped: ${stderr.trim()}`); + return; + } expect(code).toBe(0); const data = parseJsonOutput(stdout); expect(Array.isArray(data)).toBe(true);