diff --git a/src/agent-command.ts b/src/agent-command.ts index 21c1a491..bd4e2873 100644 --- a/src/agent-command.ts +++ b/src/agent-command.ts @@ -92,8 +92,10 @@ export function buildRawResumeCommand( return `${AGENT_ENV} claude --resume ${sessionId}`; case "codex": return `codex resume ${sessionId}`; + // `cursor agent` exposes `--resume [chatId]`; it has no `--session` flag + // (`error: unknown option '--session'`). Verified against `cursor agent --help`. case "cursor": - return `cursor agent --session ${sessionId}`; + return `cursor agent --resume ${sessionId}`; case "gemini": return `${AGENT_ENV} gemini --resume ${sessionId}`; case "kiro": { diff --git a/src/agent-engine.ts b/src/agent-engine.ts index 707d6cb2..28bfa5a1 100644 --- a/src/agent-engine.ts +++ b/src/agent-engine.ts @@ -589,6 +589,13 @@ const DEFAULT_HALT_IDLE_WITHOUT_DONE_DWELL_MS = 15 * 60_000; const DEFAULT_HALT_WEDGED_DWELL_MS = 120_000; const DEFAULT_HALT_WEDGED_SWEEPS = 3; const MAX_AUTO_REVIVE_BACKOFF_MS = 30_000; +/** + * Harness responses that mean "I refused this resume command" -- a wrong flag, + * an unknown/expired session, or no such binary. Matched only against the screen + * tail that follows our own echoed resume command. + */ +const RESUME_REJECTION_RE = + /\b(?:unknown option|unknown argument|unknown flag|unrecognized (?:option|argument)|unexpected argument|command not found|no rollout found|failed to resume|session not found|no such session|invalid session)\b/i; const DONE_QUIESCENCE_MS = 1_500; const SESSION_ID_PATTERN = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"; @@ -3031,6 +3038,15 @@ export class AgentEngine { if (!evidence.ready && !evidence.activeCodex) { this.readyPatternMatches.delete(agent.agent_id); + // A harness that rejected the resume command is a FAILED attempt, not a + // slow boot. Record it now instead of burning the boot timeout in + // silence and then retrying the identical broken command. + const rejection = this.detectResumeRejection(agent, screen.text); + if (rejection) { + const settled = this.stateMgr.updateRecord(agent.agent_id, settlement); + this.registry.set(agent.agent_id, settled); + return this.recordAutoReviveResumeFailure(settled, rejection); + } const since = Date.parse(agent.updated_at); if ( !Number.isNaN(since) && @@ -3484,7 +3500,7 @@ export class AgentEngine { private async dispatchCliExitOutcome( agent: AgentRecord, - outcome: "revived" | "unrecoverable", + outcome: "revived" | "recovered" | "unrecoverable", ): Promise<{ record: AgentRecord; dispatched: boolean }> { if (!agent.parent_agent_id || agent.revive_notification_sent_at) { return { record: agent, dispatched: false }; @@ -3494,15 +3510,19 @@ export class AgentEngine { ? buildRawResumeCommand(agent.cli, agent.repo, agent.cli_session_id) : null; const tag = - outcome === "revived" - ? "agent_cli_exit_revived" - : "agent_cli_exit_unrecoverable"; + outcome === "unrecoverable" + ? "agent_cli_exit_unrecoverable" + : "agent_cli_exit_revived"; const task = outcome === "revived" ? `Agent ${agent.agent_id} revived automatically on attempt ${attempts} ` + `in surface ${agent.surface_id}; verified model ${agent.parsed_model ?? "unknown"}.` - : `Agent ${agent.agent_id} CLI exit is unrecoverable after ${attempts} attempts ` + - `in surface ${agent.surface_id}. Manual fallback: ${manualResumeCommand ?? "no captured session"}`; + : outcome === "recovered" + ? `Agent ${agent.agent_id} recovered in surface ${agent.surface_id} without an ` + + `engine resume after ${attempts} attempts; the pending auto-resume was ` + + `cleared before injection so nothing was typed into the live agent.` + : `Agent ${agent.agent_id} CLI exit is unrecoverable after ${attempts} attempts ` + + `in surface ${agent.surface_id}. Manual fallback: ${manualResumeCommand ?? "no captured session"}`; try { dispatchOnce( agent.parent_agent_id, @@ -3582,8 +3602,187 @@ export class AgentEngine { return completed; } + /** + * Classify what currently occupies a revive target's surface. Auto-resume may + * only type into a bare shell: between the death signal and the injection the + * pane can be revived by other means (a human running `--resume` by hand), and + * typing then lands the resume command in a working agent's composer as if it + * were a user message. Same guard class as the interactive-overlay delivery + * refusal. + */ + private async classifyReviveTarget( + agent: AgentRecord, + knownShellScreenText?: string, + ): Promise<"shell" | "live_agent" | "unverified"> { + let screenText: string; + try { + screenText = + knownShellScreenText ?? (await this.readSweepScreen(agent, {})).text; + } catch { + return "unverified"; + } + const parsed = parseScreen(screenText); + if (parsed.control_state === "shell") return "shell"; + if ( + parsed.control_state === "ready" || + parsed.control_state === "busy" || + parsed.control_state === "permission_prompt" || + parsed.control_state === "interactive_overlay" || + screenHasReadyAgentIdentity(agent.cli, screenText, parsed) + ) { + return "live_agent"; + } + return "unverified"; + } + + /** + * The pane came back without us: clear the pending resume so nothing is typed, + * and hand the record back to the ordinary boot-readiness path. + */ + private async markAutoReviveRecovered( + agent: AgentRecord, + ): Promise { + let recovered = this.stateMgr.updateRecord(agent.agent_id, { + revive_last_outcome: "revived", + revive_last_error: null, + revive_next_attempt_at: null, + revive_completed_at: new Date().toISOString(), + revive_observation_source: "screen", + revive_observed_at_ms: Date.now(), + error: null, + }); + this.registry.set(agent.agent_id, recovered); + const notification = await this.dispatchCliExitOutcome( + recovered, + "recovered", + ); + recovered = notification.record; + this.appendAutoReviveCliExitEvent( + recovered, + "revived", + notification.dispatched, + ); + try { + const creating = this.stateMgr.transition( + recovered.agent_id, + "creating", + { + error: null, + pid: null, + }, + ); + this.registry.set(creating.agent_id, creating); + const booting = this.stateMgr.transition(creating.agent_id, "booting", { + error: null, + pid: null, + }); + this.registry.set(booting.agent_id, booting); + return booting; + } catch { + return recovered; + } + } + + /** + * Hold the attempt without consuming one: we could not prove the surface is a + * bare shell, and typing on an unproven surface is the failure mode this guard + * exists to prevent. + */ + private deferAutoReviveAttempt( + agent: AgentRecord, + attempt: number, + ): AgentRecord { + try { + const deferred = this.stateMgr.updateRecord(agent.agent_id, { + revive_next_attempt_at: new Date( + Date.now() + this.autoReviveBackoffMs(attempt), + ).toISOString(), + revive_observation_source: "screen", + revive_observed_at_ms: Date.now(), + }); + this.registry.set(agent.agent_id, deferred); + return deferred; + } catch { + return agent; + } + } + + /** + * The harness rejected the resume command itself (bad flag, unknown session). + * Record the failure and back off; at the cap, escalate as unrecoverable. + */ + private async recordAutoReviveResumeFailure( + agent: AgentRecord, + reason: string, + ): Promise { + const attempts = agent.revive_attempts ?? 0; + let failed: AgentRecord; + try { + failed = + agent.state === "error" + ? agent + : this.stateMgr.transition(agent.agent_id, "error", { + error: `Auto-revive attempt ${attempts} failed: ${reason}`, + }); + failed = this.stateMgr.updateRecord(failed.agent_id, { + revive_last_outcome: "failed", + revive_last_error: reason, + revive_next_attempt_at: new Date( + Date.now() + this.autoReviveBackoffMs(Math.max(1, attempts)), + ).toISOString(), + revive_observation_source: "screen", + revive_observed_at_ms: Date.now(), + }); + this.registry.set(failed.agent_id, failed); + } catch { + return agent; + } + if (attempts >= MAX_RESPAWN_ATTEMPTS) { + return this.markAutoReviveUnrecoverable(failed, reason); + } + return failed; + } + + /** + * Detect a resume command the harness refused, by reading only the screen tail + * that followed our own echoed command. Returns the offending line, or null. + */ + private detectResumeRejection( + agent: AgentRecord, + screenText: string, + ): string | null { + if (agent.revive_last_outcome !== "pending" || !agent.cli_session_id) { + return null; + } + let resumeCommand: string; + try { + resumeCommand = buildRawResumeCommand( + agent.cli, + agent.repo, + agent.cli_session_id, + ); + } catch { + return null; + } + const echoed = screenText.lastIndexOf(resumeCommand); + if (echoed < 0) return null; + const tail = screenText.slice(echoed + resumeCommand.length); + const parsed = parseScreen(tail); + // An agent that actually came up is not a rejected resume, whatever else + // its own output happens to say. + if (screenHasReadyAgentIdentity(agent.cli, tail, parsed)) return null; + return ( + tail + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .find((line) => RESUME_REJECTION_RE.test(line)) ?? null + ); + } + private async attemptSameSurfaceAutoRevive( agent: AgentRecord, + knownShellScreenText?: string, ): Promise { const attempt = (agent.revive_attempts ?? 0) + 1; if (attempt > MAX_RESPAWN_ATTEMPTS) { @@ -3599,6 +3798,13 @@ export class AgentEngine { "captured session id is missing", ); } + const target = await this.classifyReviveTarget(agent, knownShellScreenText); + if (target === "live_agent") { + return this.markAutoReviveRecovered(agent); + } + if (target === "unverified") { + return this.deferAutoReviveAttempt(agent, attempt); + } const attemptedAt = new Date().toISOString(); let attempted = this.stateMgr.updateRecord(agent.agent_id, { revive_attempts: attempt, @@ -3795,7 +4001,9 @@ export class AgentEngine { }); this.registry.set(agent.agent_id, tracked); this.appendAutoReviveCliExitEvent(tracked, "pending", false); - return this.attemptSameSurfaceAutoRevive(tracked); + // This sweep just proved the surface is a bare shell; reuse that read + // rather than paying for (and racing on) a second one. + return this.attemptSameSurfaceAutoRevive(tracked, screenText); } let inboxDispatched = false; diff --git a/tests/agent-engine.test.ts b/tests/agent-engine.test.ts index 1e75a179..81111146 100644 --- a/tests/agent-engine.test.ts +++ b/tests/agent-engine.test.ts @@ -9365,12 +9365,17 @@ Session ID: ${sessionId}`, await engine.runSweep(); await engine.runSweep(); + // The harness refused the resume, so this is a recorded FAILURE with a + // backoff -- never a revival, and never a silent hold that burns the boot + // timeout before retrying the identical command. expect( engine.getAgentState("cmuxlayerCodex-auto-revive-failed-shell"), ).toMatchObject({ - state: "booting", + state: "error", revive_attempts: 1, - revive_last_outcome: "pending", + revive_last_outcome: "failed", + revive_last_error: expect.stringContaining("Failed to resume session"), + revive_next_attempt_at: expect.any(String), }); expect(readInbox("cmuxlayerClaude", { baseDir: TEST_DIR })).toEqual([]); }); @@ -9417,9 +9422,10 @@ Session ID: ${sessionId}`, expect( engine.getAgentState("cmuxlayerCodex-auto-revive-failed-before-shell"), ).toMatchObject({ - state: "booting", + state: "error", revive_attempts: 1, - revive_last_outcome: "pending", + revive_last_outcome: "failed", + revive_last_error: expect.stringContaining("Failed to resume session"), }); expect(readInbox("cmuxlayerClaude", { baseDir: TEST_DIR })).toEqual([]); }, @@ -9521,7 +9527,7 @@ Session ID: ${sessionId}`, cli: "cursor" as const, id: "019faccc-4848-7555-8666-777788889999", command: - "cursor agent --session 019faccc-4848-7555-8666-777788889999", + "cursor agent --resume 019faccc-4848-7555-8666-777788889999", staleReady: "Cursor Agent\ncursor>", }, { @@ -9883,6 +9889,236 @@ Session ID: ${sessionId}`, ).toEqual([]); }); + it("revives a dead cursor pane with the real --resume flag and verifies it from the post-resume screen", async () => { + const sessionId = "019faccc-6060-7555-8666-777788889999"; + stateMgr.writeState( + makeRecord({ + agent_id: "cmuxlayerCursor-auto-revive", + state: "working", + surface_id: "surface:cursor-auto-revive", + surface_provenance: "cmuxlayer_spawn", + parent_agent_id: "cmuxlayerClaude", + spawn_depth: 1, + cli: "cursor", + cli_session_id: sessionId, + auto_revive: true, + }), + ); + liveSurfaces = [makeSurface("surface:cursor-auto-revive")]; + const deadShell = "% cursor agent exited\n%"; + const readScreen = mockClient.readScreen as ReturnType; + readScreen + .mockResolvedValueOnce({ + surface: "surface:cursor-auto-revive", + text: deadShell, + lines: 80, + scrollback_used: false, + }) + .mockResolvedValueOnce({ + surface: "surface:cursor-auto-revive", + text: deadShell, + lines: 80, + scrollback_used: false, + }) + .mockResolvedValue({ + surface: "surface:cursor-auto-revive", + text: + `% cursor agent --resume ${sessionId}\n` + + "Cursor Agent\nā†’ Plan, search, build anything\nAuto\ncursor>", + lines: 80, + scrollback_used: false, + }); + await engine.getRegistry().reconstitute(); + + await engine.runSweep(); + await engine.runSweep(); + + expect(mockClient.send).toHaveBeenCalledWith( + "surface:cursor-auto-revive", + `cursor agent --resume ${sessionId}`, + expect.objectContaining({ workspace: undefined }), + ); + + await engine.runSweep(); + + expect(engine.getAgentState("cmuxlayerCursor-auto-revive")).toMatchObject( + { + state: "ready", + revive_attempts: 1, + revive_last_outcome: "revived", + revive_completed_at: expect.any(String), + }, + ); + expect(readInbox("cmuxlayerClaude", { baseDir: TEST_DIR })).toEqual([ + expect.objectContaining({ tag: "agent_cli_exit_revived" }), + ]); + }); + + it("never types a pending resume into a surface that already recovered a live agent", async () => { + const sessionId = "019faccc-6161-7555-8666-777788889999"; + engine.dispose(); + const launchCommandSender = vi.fn().mockResolvedValue(undefined); + engine = new AgentEngine(stateMgr, engine.getRegistry(), mockClient, { + spawnPreflight: async () => {}, + sessionIdentityResolver: () => null, + launchCommandSender, + autoReviveBackoffBaseMs: 0, + }); + stateMgr.writeState( + makeRecord({ + agent_id: "cmuxlayerCursor-auto-revive-recovered", + state: "error", + surface_id: "surface:cursor-auto-revive-recovered", + surface_provenance: "cmuxlayer_spawn", + parent_agent_id: "cmuxlayerClaude", + spawn_depth: 1, + cli: "cursor", + cli_session_id: sessionId, + auto_revive: true, + revive_attempts: 1, + revive_last_attempt_at: "2026-08-13T07:00:00.000Z", + revive_next_attempt_at: "2026-08-13T07:00:00.000Z", + revive_last_outcome: "failed", + revive_last_error: "readiness timed out", + revive_previous_state: "working", + error: "Auto-revive attempt 1 failed", + }), + ); + liveSurfaces = [makeSurface("surface:cursor-auto-revive-recovered")]; + // Etan resumed this pane by hand: it is no longer a bare shell. + (mockClient.readScreen as ReturnType).mockResolvedValue({ + surface: "surface:cursor-auto-revive-recovered", + text: [ + "Cursor Agent", + "ā†’ Plan, search, build anything", + "Auto", + "cursor>", + ].join("\n"), + lines: 80, + scrollback_used: false, + }); + await engine.getRegistry().reconstitute(); + + await engine.runSweep(); + await engine.runSweep(); + + expect(launchCommandSender).not.toHaveBeenCalled(); + expect(mockClient.send).not.toHaveBeenCalled(); + expect(mockClient.sendKey).not.toHaveBeenCalled(); + const recovered = engine.getAgentState( + "cmuxlayerCursor-auto-revive-recovered", + ); + expect(recovered).toMatchObject({ revive_last_outcome: "revived" }); + expect(recovered?.revive_attempts).toBe(1); + }); + + it("records a rejected cursor resume as a failure instead of waiting out the boot timeout", async () => { + const sessionId = "019faccc-6262-7555-8666-777788889999"; + engine.dispose(); + engine = new AgentEngine(stateMgr, engine.getRegistry(), mockClient, { + spawnPreflight: async () => {}, + sessionIdentityResolver: () => null, + autoReviveBackoffBaseMs: 60_000, + }); + stateMgr.writeState( + makeRecord({ + agent_id: "cmuxlayerCursor-auto-revive-rejected", + state: "booting", + surface_id: "surface:cursor-auto-revive-rejected", + surface_provenance: "cmuxlayer_spawn", + parent_agent_id: "cmuxlayerClaude", + spawn_depth: 1, + cli: "cursor", + cli_session_id: sessionId, + auto_revive: true, + revive_attempts: 1, + revive_last_outcome: "pending", + revive_previous_state: "working", + updated_at: new Date().toISOString(), + }), + ); + liveSurfaces = [makeSurface("surface:cursor-auto-revive-rejected")]; + (mockClient.readScreen as ReturnType).mockResolvedValue({ + surface: "surface:cursor-auto-revive-rejected", + text: [ + `% cursor agent --resume ${sessionId}`, + "error: unknown option '--resume'", + "(Did you mean --version?)", + "%", + ].join("\n"), + lines: 80, + scrollback_used: false, + }); + await engine.getRegistry().reconstitute(); + + await engine.runSweep(); + + expect( + engine.getAgentState("cmuxlayerCursor-auto-revive-rejected"), + ).toMatchObject({ + state: "error", + revive_attempts: 1, + revive_last_outcome: "failed", + revive_last_error: expect.stringContaining("unknown option"), + revive_next_attempt_at: expect.any(String), + }); + // Backed off, not retried in the same breath. + expect(mockClient.send).not.toHaveBeenCalled(); + expect(readInbox("cmuxlayerClaude", { baseDir: TEST_DIR })).toEqual([]); + }); + + it("escalates a repeatedly rejected cursor resume once with the real manual command", async () => { + const sessionId = "019faccc-6363-7555-8666-777788889999"; + stateMgr.writeState( + makeRecord({ + agent_id: "cmuxlayerCursor-auto-revive-exhausted", + state: "error", + surface_id: "surface:cursor-auto-revive-exhausted", + surface_provenance: "cmuxlayer_spawn", + parent_agent_id: "cmuxlayerClaude", + spawn_depth: 1, + cli: "cursor", + cli_session_id: sessionId, + auto_revive: true, + revive_attempts: MAX_RESPAWN_ATTEMPTS, + revive_last_attempt_at: "2026-08-13T07:00:00.000Z", + revive_next_attempt_at: "2026-08-13T07:00:00.000Z", + revive_last_outcome: "failed", + revive_last_error: "error: unknown option", + revive_previous_state: "working", + error: "Auto-revive attempt failed", + }), + ); + liveSurfaces = [makeSurface("surface:cursor-auto-revive-exhausted")]; + (mockClient.readScreen as ReturnType).mockResolvedValue({ + surface: "surface:cursor-auto-revive-exhausted", + text: [`% cursor agent --resume ${sessionId}`, "%"].join("\n"), + lines: 80, + scrollback_used: false, + }); + await engine.getRegistry().reconstitute(); + + await engine.runSweep(); + await engine.runSweep(); + + expect(mockClient.send).not.toHaveBeenCalled(); + expect( + engine.getAgentState("cmuxlayerCursor-auto-revive-exhausted"), + ).toMatchObject({ + state: "error", + revive_attempts: MAX_RESPAWN_ATTEMPTS, + revive_last_outcome: "unrecoverable", + }); + expect(readInbox("cmuxlayerClaude", { baseDir: TEST_DIR })).toEqual([ + expect.objectContaining({ + tag: "agent_cli_exit_unrecoverable", + task: expect.stringContaining( + `Manual fallback: cursor agent --resume ${sessionId}`, + ), + }), + ]); + }); + it("does not retry a failed auto-revive after the agent is intentionally stopped", async () => { const launchCommandSender = vi.fn().mockResolvedValue(undefined); engine.dispose(); @@ -13506,8 +13742,11 @@ describe("buildResumeCommand", () => { expect(buildRawResumeCommand("claude", "brainlayer", sessionId)).toBe( `MCP_CONNECTION_NONBLOCKING=1 CLAUDE_CODE_NO_FLICKER=1 claude --resume ${sessionId}`, ); + // `cursor agent` has no `--session`; the real flag is `--resume [chatId]`. + // Verified against `cursor agent --help` -- `--session` exits with + // "error: unknown option '--session'". expect(buildRawResumeCommand("cursor", "brainlayer", sessionId)).toBe( - `cursor agent --session ${sessionId}`, + `cursor agent --resume ${sessionId}`, ); expect(buildRawResumeCommand("gemini", "brainlayer", sessionId)).toBe( `MCP_CONNECTION_NONBLOCKING=1 CLAUDE_CODE_NO_FLICKER=1 gemini --resume ${sessionId}`,