diff --git a/docs/plans/2026-08-11-pr395-review-fixes.md b/docs/plans/2026-08-11-pr395-review-fixes.md new file mode 100644 index 00000000..3b93b0a8 --- /dev/null +++ b/docs/plans/2026-08-11-pr395-review-fixes.md @@ -0,0 +1,13 @@ +# PR 395 High-Severity Review Fixes + +## Goal + +Close the two high-severity review gaps without widening Phase 1 scope. + +## Design + +1. Bootstrap inbox state for every spawned role and append the concrete per-agent mailbox monitor/cursor contract to every delivered boot instruction. Preserve the caller's task text as the task summary. +2. Make the guarded `deliverAgentInput` route state the sole interactive-state authority for queued delivery. Represent a pre-mutation posture rejection as retryable, retain the durable queued receipt, and retry with exponential backoff capped at a fixed maximum. Mark a receipt terminal only when the target is gone or the submission outcome is uncertain. +3. Pin worker and workspace spawn boot delivery, posture disagreement retry, backoff, eventual submission, and target-gone resolution with focused tests. +4. Run focused tests, typecheck/build/full suite, compiled probes, push the signed commit, update PR 395, and inbox the lead. + diff --git a/src/agent-engine.ts b/src/agent-engine.ts index afc70da6..d63843ac 100644 --- a/src/agent-engine.ts +++ b/src/agent-engine.ts @@ -3,7 +3,16 @@ * These 7 functions are the engine that MCP tools (and later the 2-tool facade) drive. */ -import { existsSync, readFileSync, statSync } from "node:fs"; +import { randomUUID } from "node:crypto"; +import { + existsSync, + mkdirSync, + readFileSync, + renameSync, + statSync, + unlinkSync, + writeFileSync, +} from "node:fs"; import { homedir } from "node:os"; import { dirname, isAbsolute, join, resolve } from "node:path"; import { StateManager } from "./state-manager.js"; @@ -46,6 +55,7 @@ import { type AgentState, type CliType, type CloseForensicsEvent, + type DeliveryEventType, type PublicAgent, type WaitResult, } from "./agent-types.js"; @@ -152,6 +162,39 @@ import { type ProcessLiveness = "alive" | "gone" | "unknown"; +export type AgentDeliveryState = "submitted" | "queued" | "failed"; + +export interface AgentDeliveryReceipt { + delivery_id: string; + agent_id: string; + text: string; + press_enter: boolean; + source_event: DeliveryEventType; + delivery_state: AgentDeliveryState; + terminal: boolean; + created_at: string; + resolved_at: string | null; + retry_count: number; + submit_verified: boolean | null; + error: string | null; + /** Persisted before terminal mutation; a nonterminal value is never replayed after restart. */ + submission_started_at?: string | null; + /** Earliest wall-clock time at which a known pre-mutation rejection may retry. */ + next_attempt_at?: string | null; +} + +/** A known pre-mutation delivery rejection that is safe to retry. */ +export class RetryableDeliveryError extends Error { + constructor(message: string) { + super(message); + this.name = "RetryableDeliveryError"; + } +} + +type DeliverySubmitter = ( + receipt: AgentDeliveryReceipt, +) => Promise<{ retry_count: number; submit_verified: boolean | null }>; + export interface SpawnAgentParams { repo: string; model?: string; @@ -381,6 +424,8 @@ export interface AgentEngineOptions { fleetSidebarPublisher?: FleetSidebarPublisherLike; /** Render-only timeout for a working seat whose transcript/output stops advancing. */ fleetWorkingNoProgressTimeoutMs?: number; + /** Bound one queued terminal submission so lifecycle sweeps cannot hang forever. */ + deliverySubmitTimeoutMs?: number; } export type RolePlacementReconcileTrigger = "spawn" | "idle" | "boot"; @@ -904,6 +949,11 @@ export class AgentEngine { private fleetWorkingNoProgressTimeoutMs: number; private startupInitializePromise: Promise | null = null; private lifecycleMutationTail: Promise = Promise.resolve(); + private deliveryReceipts = new Map(); + private deliveryReceiptsPath: string; + private deliverySubmitter: DeliverySubmitter | null = null; + private deliveryDrainInFlight = false; + private deliverySubmitTimeoutMs: number; constructor( stateMgr: StateManager, registry: AgentRegistry, @@ -911,6 +961,15 @@ export class AgentEngine { opts?: AgentEngineOptions, ) { this.stateMgr = stateMgr; + this.deliveryReceiptsPath = join( + stateMgr.getBaseDir(), + "delivery-receipts.json", + ); + this.deliverySubmitTimeoutMs = Math.max( + 1, + opts?.deliverySubmitTimeoutMs ?? 30_000, + ); + this.loadDeliveryReceipts(); this.registry = registry; this.client = client; this.roleSurfaceIdsProvider = opts?.roleSurfaceIdsProvider; @@ -4463,6 +4522,236 @@ export class AgentEngine { async runSweep(): Promise { await this.runLifecycleMutation(() => this.runSweepOnce()); + await this.drainDeliveryQueue(); + } + + setDeliverySubmitter(submitter: DeliverySubmitter | null): void { + this.deliverySubmitter = submitter; + } + + private loadDeliveryReceipts(): void { + try { + const parsed: unknown = JSON.parse( + readFileSync(this.deliveryReceiptsPath, "utf8"), + ); + if (!Array.isArray(parsed)) return; + let repairedUncertainReceipt = false; + for (const candidate of parsed) { + if ( + candidate && + typeof candidate === "object" && + typeof (candidate as AgentDeliveryReceipt).delivery_id === "string" + ) { + const receipt: AgentDeliveryReceipt = { + submission_started_at: null, + next_attempt_at: null, + ...(candidate as AgentDeliveryReceipt), + }; + if ( + receipt.delivery_state === "queued" && + receipt.submission_started_at + ) { + receipt.delivery_state = "failed"; + receipt.terminal = true; + receipt.resolved_at = new Date().toISOString(); + receipt.error = + "Delivery outcome uncertain after process restart; refusing automatic replay"; + repairedUncertainReceipt = true; + } + this.deliveryReceipts.set(receipt.delivery_id, receipt); + } + } + if (repairedUncertainReceipt) { + try { + this.persistDeliveryReceipts(); + } catch { + // In-memory terminal state still prevents replay in this process. + } + } + } catch { + // Missing or corrupt legacy state must not prevent lifecycle startup. + } + } + + private persistDeliveryReceipts(): void { + mkdirSync(dirname(this.deliveryReceiptsPath), { recursive: true }); + const tempPath = `${this.deliveryReceiptsPath}.${process.pid}.${randomUUID()}.tmp`; + try { + writeFileSync( + tempPath, + `${JSON.stringify([...this.deliveryReceipts.values()], null, 2)}\n`, + "utf8", + ); + renameSync(tempPath, this.deliveryReceiptsPath); + } finally { + if (existsSync(tempPath)) unlinkSync(tempPath); + } + } + + queueDelivery(input: { + agent_id: string; + text: string; + press_enter: boolean; + source_event: DeliveryEventType; + }): AgentDeliveryReceipt { + const receipt: AgentDeliveryReceipt = { + delivery_id: randomUUID(), + ...input, + delivery_state: "queued", + terminal: false, + created_at: new Date().toISOString(), + resolved_at: null, + retry_count: 0, + submit_verified: null, + error: null, + submission_started_at: null, + next_attempt_at: null, + }; + this.deliveryReceipts.set(receipt.delivery_id, receipt); + try { + // Acceptance is not returned until the full replay payload is durable. + this.persistDeliveryReceipts(); + } catch (error) { + this.deliveryReceipts.delete(receipt.delivery_id); + throw error; + } + this.appendDeliveryReceiptEventBestEffort(receipt); + return { ...receipt }; + } + + resolveDelivery( + input: Omit & { + created_at?: string; + }, + opts?: { appendFailureEvent?: boolean }, + ): AgentDeliveryReceipt { + const receipt: AgentDeliveryReceipt = { + ...input, + created_at: input.created_at ?? new Date().toISOString(), + resolved_at: new Date().toISOString(), + }; + this.deliveryReceipts.set(receipt.delivery_id, receipt); + this.persistDeliveryReceipts(); + if (receipt.delivery_state === "failed" && opts?.appendFailureEvent) { + this.appendDeliveryReceiptEventBestEffort(receipt); + } + return { ...receipt }; + } + + getDeliveryReceipt(deliveryId: string): AgentDeliveryReceipt | null { + const receipt = this.deliveryReceipts.get(deliveryId); + return receipt ? { ...receipt } : null; + } + + async drainDeliveryQueue(): Promise { + if (this.deliveryDrainInFlight || !this.deliverySubmitter) return; + this.deliveryDrainInFlight = true; + try { + for (const receipt of this.deliveryReceipts.values()) { + if (receipt.delivery_state !== "queued") continue; + const agent = this.getAgentState(receipt.agent_id); + if (!agent) { + receipt.delivery_state = "failed"; + receipt.terminal = true; + receipt.resolved_at = new Date().toISOString(); + receipt.error = `Delivery target ${receipt.agent_id} is gone or no longer exists`; + this.persistDeliveryReceipts(); + this.appendDeliveryReceiptEventBestEffort(receipt); + continue; + } + if ( + receipt.next_attempt_at && + Date.parse(receipt.next_attempt_at) > Date.now() + ) { + continue; + } + try { + receipt.submission_started_at = new Date().toISOString(); + // This is the no-replay boundary. A crash after this write leaves an + // uncertain terminal receipt instead of re-sending terminal input. + this.persistDeliveryReceipts(); + let timeout: ReturnType | null = null; + const result = await Promise.race([ + this.deliverySubmitter(receipt), + new Promise((_resolve, reject) => { + timeout = setTimeout( + () => + reject( + new Error( + `Delivery submission timed out after ${this.deliverySubmitTimeoutMs}ms; outcome uncertain and will not be retried`, + ), + ), + this.deliverySubmitTimeoutMs, + ); + }), + ]).finally(() => { + if (timeout) clearTimeout(timeout); + }); + receipt.delivery_state = "submitted"; + receipt.terminal = true; + receipt.resolved_at = new Date().toISOString(); + receipt.retry_count += result.retry_count; + receipt.submit_verified = result.submit_verified; + receipt.error = null; + receipt.next_attempt_at = null; + } catch (error) { + if (error instanceof RetryableDeliveryError) { + receipt.submission_started_at = null; + receipt.retry_count += 1; + const backoffMs = Math.min( + 30_000, + 250 * 2 ** Math.min(receipt.retry_count - 1, 16), + ); + receipt.next_attempt_at = new Date( + Date.now() + backoffMs, + ).toISOString(); + receipt.error = error.message; + } else { + receipt.delivery_state = "failed"; + receipt.terminal = true; + receipt.resolved_at = new Date().toISOString(); + receipt.error = + error instanceof Error ? error.message : String(error); + } + } + this.persistDeliveryReceipts(); + // Successful delivery already emitted the correlated source event; + // failures have no such event and need an explicit terminal transition. + if (receipt.delivery_state === "failed") { + this.appendDeliveryReceiptEventBestEffort(receipt); + } + } + } finally { + this.deliveryDrainInFlight = false; + } + } + + private appendDeliveryReceiptEvent(receipt: AgentDeliveryReceipt): void { + const agent = this.getAgentState(receipt.agent_id); + this.stateMgr.getEventLog().appendDelivery({ + ts: receipt.resolved_at ?? receipt.created_at, + event_type: receipt.source_event, + source_agent: null, + target_surface: agent?.surface_id ?? "unknown", + target_agent: receipt.agent_id, + bytes: Buffer.byteLength(receipt.text), + press_enter: receipt.press_enter, + submit_verified: receipt.submit_verified, + retry_count: receipt.retry_count, + delivery_id: receipt.delivery_id, + delivery_state: receipt.delivery_state, + }); + } + + private appendDeliveryReceiptEventBestEffort( + receipt: AgentDeliveryReceipt, + ): void { + try { + this.appendDeliveryReceiptEvent(receipt); + } catch { + // Receipt persistence is authoritative; telemetry must not invalidate + // acceptance or tempt a caller to duplicate terminal input. + } } requestFleetSidebarRepublish(): void { diff --git a/src/agent-types.ts b/src/agent-types.ts index 0fc45dc7..faeca2d6 100644 --- a/src/agent-types.ts +++ b/src/agent-types.ts @@ -188,6 +188,11 @@ export interface DeliveryTelemetryEvent { press_enter: boolean | null; submit_verified: boolean | null; retry_count: number; + /** Stable receipt identity for agent-routed delivery state transitions. */ + delivery_id?: string; + /** Nonterminal acceptance or terminal resolution. */ + delivery_state?: "submitted" | "queued" | "failed"; + target_agent?: string; } export interface ControlHealthTelemetryEvent { diff --git a/src/inbox.ts b/src/inbox.ts index d15638a5..f9b77e72 100644 --- a/src/inbox.ts +++ b/src/inbox.ts @@ -20,13 +20,17 @@ // // send_input is KEPT as the fallback path — this channel is additive (belt-and-suspenders) until // proven in production. +import { randomUUID } from "node:crypto"; import { appendFileSync, existsSync, mkdirSync, readdirSync, readFileSync, + renameSync, + rmSync, unlinkSync, + writeFileSync, } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; @@ -112,6 +116,9 @@ export function inboxPath(agentId: string, opts?: InboxOpts): string { export function ackPath(agentId: string, opts?: InboxOpts): string { return join(agentDir(agentId, opts), "inbox.ack.jsonl"); } +export function inboxCursorPath(agentId: string, opts?: InboxOpts): string { + return join(agentDir(agentId, opts), "inbox.cursor"); +} export function heartbeatPath(agentId: string, opts?: InboxOpts): string { return join(agentDir(agentId, opts), "monitor.heartbeat"); } @@ -346,6 +353,75 @@ export function readAcks(agentId: string, opts?: InboxOpts): InboxAck[] { return readJsonl(ackPath(agentId, opts)); } +/** Last message the agent confirms it fully handled. The engine only reads it. */ +export function readInboxCursor( + agentId: string, + opts?: InboxOpts, +): string | null { + try { + const cursor = readFileSync(inboxCursorPath(agentId, opts), "utf8").trim(); + return cursor.length > 0 ? cursor : null; + } catch { + return null; + } +} + +/** + * Agent-side consumption helper. Call only after the message has been handled. + * The atomic rename prevents a resume from observing a partial watermark. + */ +export function writeInboxCursor( + agentId: string, + messageId: string, + opts?: InboxOpts, +): string { + ensureChannelDirForWrite(agentId, opts); + const path = inboxCursorPath(agentId, opts); + const lockPath = `${path}.lock`; + let lockAcquired = false; + let tempPath: string | null = null; + try { + try { + mkdirSync(lockPath); + lockAcquired = true; + } catch (error) { + if ( + error instanceof Error && + "code" in error && + error.code === "EEXIST" + ) { + throw new Error(`Inbox cursor is locked for ${agentId}; retry`); + } + throw error; + } + + // Read both the inbox and watermark only after acquiring the lock. This is + // the compare-and-set boundary across independently resumed agent processes. + const messages = readInbox(agentId, opts); + const nextIndex = messages.findIndex((message) => message.id === messageId); + if (nextIndex < 0) { + throw new Error(`Cannot advance inbox cursor to unknown message ${messageId}`); + } + const current = readInboxCursor(agentId, opts); + if (current) { + const currentIndex = messages.findIndex((message) => message.id === current); + if (currentIndex >= 0 && nextIndex < currentIndex) { + throw new Error( + `Cannot move inbox cursor backwards from ${current} to ${messageId}`, + ); + } + } + tempPath = `${path}.${process.pid}.${randomUUID()}.tmp`; + writeFileSync(tempPath, `${messageId}\n`, "utf8"); + renameSync(tempPath, path); + tempPath = null; + return messageId; + } finally { + if (tempPath) rmSync(tempPath, { force: true }); + if (lockAcquired) rmSync(lockPath, { recursive: true, force: true }); + } +} + /** Set of message ids that have been acked. */ export function ackedIds(agentId: string, opts?: InboxOpts): Set { return new Set(readAcks(agentId, opts).map((a) => a.ack_of)); @@ -360,8 +436,15 @@ export function replayUndelivered( agentId: string, opts?: InboxOpts, ): InboxMessage[] { + const messages = readInbox(agentId, opts); + const cursor = readInboxCursor(agentId, opts); + if (cursor) { + const cursorIndex = messages.findIndex((message) => message.id === cursor); + // An unknown/corrupt cursor cannot safely suppress anything: replay all. + return cursorIndex >= 0 ? messages.slice(cursorIndex + 1) : messages; + } const acked = ackedIds(agentId, opts); - return readInbox(agentId, opts).filter((m) => !acked.has(m.id)); + return messages.filter((m) => !acked.has(m.id)); } /** Append an ACK (deterministic delivery confirmation) and refresh the liveness heartbeat. */ diff --git a/src/index.ts b/src/index.ts index 9a5b01a3..41321375 100644 --- a/src/index.ts +++ b/src/index.ts @@ -19,6 +19,7 @@ import { runFleetSidebarCommand } from "./fleet-sidebar-cli.js"; import { RUNNING_VERSION } from "./version.js"; import { runDaemonFirstEntry } from "./entry.js"; import { isMainModule } from "./is-main.js"; +import { writeInboxCursor } from "./inbox.js"; const HELP_TEXT = `cmuxlayer — Terminal multiplexer MCP server for AI agent workspace orchestration. @@ -37,6 +38,9 @@ Usage: orc, golems, voicelayer, skillCreator, cmuxlayer, other. cmuxlayer fleet-sidebar state Print the persisted per-lane collapse preferences. + CMUX_INBOX_MSG_ID= cmuxlayer inbox-cursor + Advance the agent-owned inbox cursor after handling a + message. CMUXLAYER_INBOX_BASE_DIR overrides its base dir. Environment: CMUX_SOCKET_PATH Pin the MCP to a specific cmux instance's Unix socket @@ -78,6 +82,26 @@ async function main() { process.exitCode = result.ok ? 0 : 1; return; } + if (arg === "inbox-cursor") { + const agentId = process.argv[3]; + const messageId = process.env.CMUX_INBOX_MSG_ID; + if (!agentId || !messageId) { + process.stderr.write( + "Usage: CMUX_INBOX_MSG_ID= cmuxlayer inbox-cursor \n", + ); + process.exitCode = 2; + return; + } + writeInboxCursor( + agentId, + messageId, + process.env.CMUXLAYER_INBOX_BASE_DIR + ? { baseDir: process.env.CMUXLAYER_INBOX_BASE_DIR } + : undefined, + ); + process.stdout.write(`${messageId}\n`); + return; + } await runDaemonFirstEntry(); } diff --git a/src/server.ts b/src/server.ts index 6f71244f..0f39260e 100644 --- a/src/server.ts +++ b/src/server.ts @@ -29,6 +29,7 @@ import { resolveSpawnModelPolicy, } from "./model-policy.js"; import { StateManager } from "./state-manager.js"; +import { shellQuote } from "./agent-command.js"; import { createDefaultCloseForensicsRunner } from "./close-forensics.js"; import { currentTransportRetryCount, @@ -45,6 +46,7 @@ import { import { AgentEngine, AgentLaunchError, + RetryableDeliveryError, buildLaunchCommand, resolveSweepTiming, type AgentLifecycleEvent, @@ -110,6 +112,7 @@ import { import { dispatch, ensureInboxFile, + inboxCursorPath, inboxMonitorState, inboxPath, monitorAlive, @@ -2231,6 +2234,11 @@ type MonitorBootResult = { heartbeat_written: boolean; heartbeat_source: "server_boot"; monitor_command: string; + /** Agent-owned consumption watermark; the engine never writes this file. */ + cursor_path: string; + /** Run after handling with the message id supplied as CMUX_INBOX_MSG_ID. */ + cursor_update_command: string; + cursor_update_env: "CMUX_INBOX_MSG_ID"; error?: string; }; @@ -2512,6 +2520,7 @@ export type LifecycleAgentInputDeliverer = (args: { press_enter: boolean; allow_busy?: boolean; source_event: DeliveryEventType; + delivery_id?: string; }) => Promise; export interface CmuxServerContext { @@ -2890,6 +2899,12 @@ export function createServer(opts?: CreateServerOptions): McpServer { const inboxOpts: InboxOpts = inboxBaseDir ? { baseDir: inboxBaseDir } : {}; const ensureMonitorBoot = (agentId: string): MonitorBootResult => { let monitorCommand = ""; + const cursorPath = inboxCursorPath(agentId, inboxOpts); + const cursorUpdateCommand = `${ + inboxBaseDir + ? `CMUXLAYER_INBOX_BASE_DIR=${shellQuote(inboxBaseDir)} ` + : "" + }cmuxlayer inbox-cursor ${shellQuote(agentId)}`; try { monitorCommand = recommendedMonitorCommand(agentId, inboxOpts); ensureInboxFile(agentId, inboxOpts); @@ -2899,6 +2914,9 @@ export function createServer(opts?: CreateServerOptions): McpServer { heartbeat_written: true, heartbeat_source: "server_boot", monitor_command: monitorCommand, + cursor_path: cursorPath, + cursor_update_command: cursorUpdateCommand, + cursor_update_env: "CMUX_INBOX_MSG_ID", }; } catch (e) { return { @@ -2906,10 +2924,19 @@ export function createServer(opts?: CreateServerOptions): McpServer { heartbeat_written: false, heartbeat_source: "server_boot", monitor_command: monitorCommand, + cursor_path: cursorPath, + cursor_update_command: cursorUpdateCommand, + cursor_update_env: "CMUX_INBOX_MSG_ID", error: e instanceof Error ? e.message : String(e), }; } }; + const mailboxBootContract = ( + agentId: string, + monitorBoot: MonitorBootResult, + ): string => + `cmuxlayer mailbox contract for ${agentId}: monitor with ${monitorBoot.monitor_command}; ` + + `after each handled message run CMUX_INBOX_MSG_ID= ${monitorBoot.cursor_update_command}`; // Wired up by the agent-lifecycle block below (when enabled). Lets the // dispatch_to_agent nudge reuse the guarded relay path — stale-surface // resync + recycled-occupant identity checks — instead of raw keystrokes. @@ -4177,6 +4204,7 @@ export function createServer(opts?: CreateServerOptions): McpServer { onChunkDelivered?: (sentChunks: number) => void; source_event?: DeliveryEventType; source_agent?: string | null; + delivery_id?: string; verify_submit?: boolean; allow_recovery_enter_retry?: boolean; submit_verify_timeout_ms?: number; @@ -4304,6 +4332,15 @@ export function createServer(opts?: CreateServerOptions): McpServer { press_enter: opts.press_enter, submit_verified, retry_count, + ...(opts.delivery_id + ? { + delivery_id: opts.delivery_id, + delivery_state: + submit_verified === false + ? ("failed" as const) + : ("submitted" as const), + } + : {}), }); } @@ -4934,6 +4971,7 @@ export function createServer(opts?: CreateServerOptions): McpServer { cli?: CliType; prompt?: string; boot_prompt_path?: string | null; + injected_prompt?: string; timeout_ms?: number; onUpdateShellRelaunch?: () => Promise; resolveRoute?: () => Promise<{ surface: string; workspace?: string }>; @@ -4946,7 +4984,11 @@ export function createServer(opts?: CreateServerOptions): McpServer { }> => { const bootPromptPath = getBootPromptPath(opts.boot_prompt_path); assertBootPromptMode(opts.prompt, bootPromptPath); - if (!hasInlinePrompt(opts.prompt) && !bootPromptPath) { + if ( + !hasInlinePrompt(opts.prompt) && + !bootPromptPath && + !hasInlinePrompt(opts.injected_prompt) + ) { return { bytes: 0, retry_count: 0, @@ -4967,7 +5009,7 @@ export function createServer(opts?: CreateServerOptions): McpServer { const rawPrompt = bootPromptPath ? await readFile(bootPromptPath, "utf8") - : opts.prompt!; + : (opts.prompt ?? ""); let deliveryRoute = opts.resolveRoute ? await opts.resolveRoute() : readiness.route; @@ -5015,9 +5057,12 @@ export function createServer(opts?: CreateServerOptions): McpServer { !useFilePointer ? `boot_prompt_path is ${rawPrompt.length} characters; prefer a one-line file pointer for boot prompts over ${BOOT_PROMPT_PATH_WARNING_CHARS} characters` : null; - const deliveryText = useFilePointer + const callerDeliveryText = useFilePointer ? `Read and follow ${bootPromptPath}` : rawPrompt; + const deliveryText = [callerDeliveryText, opts.injected_prompt] + .filter((part): part is string => hasInlinePrompt(part)) + .join("\n\n"); const sanitizedText = sanitizeTerminalInput(deliveryText); const chunks = sanitizedText.length > SEND_INPUT_CHUNK_THRESHOLD @@ -5055,7 +5100,7 @@ export function createServer(opts?: CreateServerOptions): McpServer { ); return { ...delivery, - prompt_text: rawPrompt, + prompt_text: hasInlinePrompt(rawPrompt) ? rawPrompt : null, prompt_warning: promptWarning, }; } catch (error) { @@ -9285,6 +9330,7 @@ export function createServer(opts?: CreateServerOptions): McpServer { press_enter: boolean; allow_busy?: boolean; source_event: DeliveryEventType; + delivery_id?: string; }) => { await refreshManagedMetadataBestEffort(args.agent_id); let route = await engine.resolveAgentIoRoute(args.agent_id); @@ -9408,7 +9454,7 @@ export function createServer(opts?: CreateServerOptions): McpServer { !INTERACTIVE_AGENT_STATES.has(route.state) && !routeSurfaceAlive ) { - throw new Error( + throw new RetryableDeliveryError( `Agent "${args.agent_id}" is not in an interactive state (current: ${route.state}). ` + `Must be in: ${[...INTERACTIVE_AGENT_STATES].join(", ")}. ` + `Pass allow_busy: true to bypass this gate and deliver raw keystrokes regardless of state.`, @@ -9459,6 +9505,7 @@ export function createServer(opts?: CreateServerOptions): McpServer { stableSurfaceIdentity: deliveryRoute.surface_uuid, source_event: args.source_event, source_agent: args.agent_id, + delivery_id: args.delivery_id, // Verify every submitted agent relay — not just long ones. A short // relay (the common agent-to-agent case) to a frozen terminal must // be caught, never reported as ok. Busy sends do not retry Return: @@ -9490,6 +9537,16 @@ export function createServer(opts?: CreateServerOptions): McpServer { // Expose the guarded relay to dispatch_to_agent's nudge (registered above, // outside this lifecycle block). lifecycleAgentInputDeliverer = deliverAgentInput; + engine.setDeliverySubmitter(async (receipt) => + deliverAgentInput({ + agent_id: receipt.agent_id, + text: receipt.text, + press_enter: receipt.press_enter, + allow_busy: false, + source_event: receipt.source_event, + delivery_id: receipt.delivery_id, + }), + ); // Reconstitute and discover live surfaces before the first sidebar paint. // The engine initializer is idempotent because daemon connections share a @@ -9784,8 +9841,7 @@ export function createServer(opts?: CreateServerOptions): McpServer { effort: args.effort, cli: args.cli, prompt: spawnPrompt, - boot_prompt_pending: - hasInlinePrompt(args.prompt) || Boolean(bootPromptPath), + boot_prompt_pending: true, workspace: spawnWorkspace, cwd: worktree.prepared?.path, mcp_env: worktree.mcpEnv, @@ -9851,6 +9907,12 @@ export function createServer(opts?: CreateServerOptions): McpServer { result.surface_id, ); originalLaunchCommandsBySurface.delete(result.surface_id); + const monitorBoot = ensureMonitorBoot(result.agent_id); + const injectedBootPrompt = mailboxBootContract( + result.agent_id, + monitorBoot, + ); + const spawnedBinding = engine.getAgentState(result.agent_id); appendStaleBuildWarning(result); const placementWarnings = [ ...targetResolution.warnings, @@ -9867,7 +9929,7 @@ export function createServer(opts?: CreateServerOptions): McpServer { let bootPromptDelivery: Awaited> | undefined; try { - if (hasInlinePrompt(args.prompt) || bootPromptPath) { + { const deliveryWorkspace = spawnDeliveryWorkspace( result, spawnWorkspace, @@ -9875,11 +9937,14 @@ export function createServer(opts?: CreateServerOptions): McpServer { bootPromptDelivery = await deliverBootPrompt({ surface: result.surface_id, workspace: deliveryWorkspace, - resolveRoute: () => - resolveManagedDeliveryRoute(result.agent_id), + stableSurfaceIdentity: spawnedBinding?.surface_uuid, + resolveRoute: spawnedBinding?.surface_uuid + ? () => resolveManagedDeliveryRoute(result.agent_id) + : undefined, cli: args.cli, prompt: args.prompt, boot_prompt_path: bootPromptPath, + injected_prompt: injectedBootPrompt, timeout_ms: args.boot_prompt_timeout_ms, onUpdateShellRelaunch: () => relaunchSpawnAgentAfterUpdate({ @@ -9914,6 +9979,7 @@ export function createServer(opts?: CreateServerOptions): McpServer { const current = engine.getAgentState(result.agent_id); if ( current?.state === "booting" && + (hasInlinePrompt(args.prompt) || Boolean(bootPromptPath)) && bootPromptDelivery.submit_verified === true ) { const ready = stateMgr.transition(result.agent_id, "ready"); @@ -10006,8 +10072,7 @@ export function createServer(opts?: CreateServerOptions): McpServer { result.surface_id, spawnDeliveryWorkspace(result, spawnWorkspace), { - waitForReady: - !hasInlinePrompt(args.prompt) && !Boolean(bootPromptPath), + waitForReady: !bootPromptDelivery, }, ); if (focusRestoreWarning) { @@ -10026,10 +10091,6 @@ export function createServer(opts?: CreateServerOptions): McpServer { cli: args.cli, launcherName: launcherNameForCli(args.repo, args.cli), }); - const monitorBoot = - role === "orchestrator" - ? ensureMonitorBoot(result.agent_id) - : undefined; const topology = currentAgent ? await collectSurfaceTopology() : null; const health = currentAgent ? await evaluateServerAgentHealth( @@ -10570,7 +10631,7 @@ export function createServer(opts?: CreateServerOptions): McpServer { model: agent.model, cli: agent.cli, prompt: agent.prompt ?? "", - boot_prompt_pending: hasPrompt, + boot_prompt_pending: true, workspace, role: agent.role, auto_archive_on_done: false, @@ -10592,11 +10653,17 @@ export function createServer(opts?: CreateServerOptions): McpServer { result.surface_id, ); originalLaunchCommandsBySurface.delete(result.surface_id); + const monitorBoot = ensureMonitorBoot(result.agent_id); + const injectedBootPrompt = mailboxBootContract( + result.agent_id, + monitorBoot, + ); + const spawnedBinding = engine.getAgentState(result.agent_id); appendStaleBuildWarning(result); let bootPromptDelivery: Awaited> | undefined; - if (hasPrompt) { + { const deliveryWorkspace = spawnDeliveryWorkspace( result, workspace, @@ -10604,10 +10671,13 @@ export function createServer(opts?: CreateServerOptions): McpServer { bootPromptDelivery = await deliverBootPrompt({ surface: result.surface_id, workspace: deliveryWorkspace, - resolveRoute: () => - resolveManagedDeliveryRoute(result.agent_id), + stableSurfaceIdentity: spawnedBinding?.surface_uuid, + resolveRoute: spawnedBinding?.surface_uuid + ? () => resolveManagedDeliveryRoute(result.agent_id) + : undefined, cli: agent.cli, prompt: agent.prompt, + injected_prompt: injectedBootPrompt, timeout_ms: BOOT_PROMPT_TIMEOUT_MS, onUpdateShellRelaunch: () => relaunchSpawnAgentAfterUpdate({ @@ -10628,11 +10698,16 @@ export function createServer(opts?: CreateServerOptions): McpServer { task_summary: bootPromptDelivery.prompt_text ?? agent.prompt ?? "", boot_prompt_pending: false, + prompt_delivered: + hasPrompt && bootPromptDelivery.submit_verified === true, + submit_verified: hasPrompt + ? bootPromptDelivery.submit_verified + : null, }); registry.set(result.agent_id, updated); const current = engine.getAgentState(result.agent_id); - if (current?.state === "booting") { + if (current?.state === "booting" && hasPrompt) { const ready = stateMgr.transition(result.agent_id, "ready"); registry.set(result.agent_id, ready); result.state = "ready"; @@ -10650,10 +10725,6 @@ export function createServer(opts?: CreateServerOptions): McpServer { cli: agent.cli, launcherName: launcherNameForCli(agent.repo, agent.cli), }); - const monitorBoot = - role === "orchestrator" - ? ensureMonitorBoot(result.agent_id) - : undefined; const topology = currentAgent ? await collectSurfaceTopology() : null; @@ -10675,24 +10746,18 @@ export function createServer(opts?: CreateServerOptions): McpServer { role, health, monitor_boot: monitorBoot, - boot_prompt_delivered: hasPrompt - ? isBootPromptDelivered(bootPromptDelivery) - : undefined, - boot_prompt_submit_verified: hasPrompt - ? (bootPromptDelivery?.submit_verified ?? null) - : undefined, + boot_prompt_delivered: isBootPromptDelivered(bootPromptDelivery), + boot_prompt_submit_verified: + bootPromptDelivery?.submit_verified ?? null, }); leanSpawnedAgents.push( shapeSpawnResponse({ ...result, role, health, - boot_prompt_delivered: hasPrompt - ? isBootPromptDelivered(bootPromptDelivery) - : false, - boot_prompt_submit_verified: hasPrompt - ? (bootPromptDelivery?.submit_verified ?? null) - : null, + boot_prompt_delivered: isBootPromptDelivered(bootPromptDelivery), + boot_prompt_submit_verified: + bootPromptDelivery?.submit_verified ?? null, }), ); } @@ -10701,6 +10766,7 @@ export function createServer(opts?: CreateServerOptions): McpServer { focusRestoreLease, lastSurface, workspace, + { waitForReady: false }, ); // spawn_in_workspace builds its response from the per-agent objects, @@ -12046,7 +12112,7 @@ export function createServer(opts?: CreateServerOptions): McpServer { // 17. send_to server.tool( "send_to", - `${PANE_INPUT_BREAKAGE_GUIDANCE} Unified send path. mode=agent (default) routes by agent_id without exposing surface details; mode=surface writes text to a raw surface; mode=command atomically sends a command and Return; mode=key sends one normalized key. Raw-surface modes accept target or surface directly and deliberately do not require a healthy agent registry, preserving the fleet recovery escape hatch. Inline text is capped at ${SEND_INPUT_MAX_INLINE_CHARS} characters by default; use file-backed boot_prompt_path for launcher prompts or allow_long_inline:true only for deliberate raw sends. ${ZSH_BANG_INLINE_WARNING}`, + `${PANE_INPUT_BREAKAGE_GUIDANCE} Unified send path. mode=agent (default) routes by agent_id without exposing surface details; a busy agent returns a nonterminal queued delivery receipt that the lifecycle sweep drains when the agent becomes interactive. mode=surface writes text to a raw surface; mode=command atomically sends a command and Return; mode=key sends one normalized key. Raw-surface modes accept target or surface directly and deliberately do not require a healthy agent registry, preserving the fleet recovery escape hatch. Inline text is capped at ${SEND_INPUT_MAX_INLINE_CHARS} characters by default; use file-backed boot_prompt_path for launcher prompts or allow_long_inline:true only for deliberate raw sends. ${ZSH_BANG_INLINE_WARNING}`, { ...SendToArgsSchema.shape, text: SendToArgsSchema.shape.text.describe( @@ -12056,7 +12122,7 @@ export function createServer(opts?: CreateServerOptions): McpServer { "Press enter after sending text", ), allow_busy: SendToArgsSchema.shape.allow_busy.describe( - "If true, bypass the lifecycle-state gate so a working agent can receive an interjection. Queued-but-unsubmitted input returns an error and must not be retried blindly. Picker/menu and permission-prompt safety gates still refuse text; use mode=key for deliberate menu driving.", + "If true, bypass the lifecycle-state queue so a working agent receives an immediate interjection. Omit it to receive a nonterminal queued receipt that resolves through delivery events. Picker/menu and permission-prompt safety gates still refuse text; use mode=key for deliberate menu driving.", ), allow_long_inline: SendToArgsSchema.shape.allow_long_inline.describe( "Bypass the inline length and multi-paragraph safety guards for a deliberate raw send. Large allowed sends keep the existing chunked delivery behavior.", @@ -12064,6 +12130,7 @@ export function createServer(opts?: CreateServerOptions): McpServer { }, ANNOTATIONS.mutating, async (rawArgs) => { + let failedReceiptPayload: Record = {}; try { const parsedArgs = SendToArgsSchema.safeParse(rawArgs); if (!parsedArgs.success) { @@ -12161,16 +12228,97 @@ export function createServer(opts?: CreateServerOptions): McpServer { cli: targetAgent?.cli, allowLongInline: args.allow_long_inline, }); - const delivery = await deliverAgentInput({ + if (!args.allow_busy && targetAgent?.state === "working") { + const receipt = engine.queueDelivery({ + agent_id: agentId, + text: args.text, + press_enter: args.press_enter, + source_event: "send_to", + }); + const data = { + accepted: true, + agent_id: agentId, + delivery_id: receipt.delivery_id, + delivery_state: receipt.delivery_state, + terminal: receipt.terminal, + }; + return { + content: [ + { + type: "text", + text: `send_to accepted — delivery ${receipt.delivery_id} queued`, + }, + ], + structuredContent: data, + }; + } + const deliveryId = randomUUID(); + let delivery: Awaited>; + try { + delivery = await deliverAgentInput({ + agent_id: agentId, + text: args.text, + press_enter: args.press_enter, + allow_busy: args.allow_busy, + source_event: "send_to", + delivery_id: deliveryId, + }); + } catch (error) { + const failedReceipt = engine.resolveDelivery( + { + delivery_id: deliveryId, + agent_id: agentId, + text: args.text, + press_enter: args.press_enter, + source_event: "send_to", + delivery_state: "failed", + terminal: true, + retry_count: + error instanceof SubmitVerificationError + ? error.retry_count + : 0, + submit_verified: + error instanceof SubmitVerificationError ? false : null, + error: error instanceof Error ? error.message : String(error), + }, + { + // Submission-verification failures already emitted the source + // event immediately before throwing. Earlier failures did not. + appendFailureEvent: !(error instanceof SubmitVerificationError), + }, + ); + failedReceiptPayload = { + delivery_id: failedReceipt.delivery_id, + delivery_state: failedReceipt.delivery_state, + terminal: failedReceipt.terminal, + }; + throw error; + } + const receipt = engine.resolveDelivery({ + delivery_id: deliveryId, agent_id: agentId, text: args.text, press_enter: args.press_enter, - allow_busy: args.allow_busy, source_event: "send_to", + delivery_state: "submitted", + terminal: true, + retry_count: delivery.retry_count, + submit_verified: delivery.submit_verified, + error: null, }); + // Preserve the already-terminal receipt if optional evidence + // collection fails after the pane mutation has succeeded. + failedReceiptPayload = { + delivery_id: receipt.delivery_id, + delivery_state: receipt.delivery_state, + terminal: receipt.terminal, + }; const evidence = await collectDeliveryEvidence(agentId); const data = { agent_id: agentId, + delivery_id: receipt.delivery_id, + delivery_state: receipt.delivery_state, + terminal: receipt.terminal, retry_count: delivery.retry_count, submit_verified: delivery.submit_verified, ...evidence, @@ -12179,15 +12327,19 @@ export function createServer(opts?: CreateServerOptions): McpServer { } catch (e) { if (e instanceof DeliverySafetyGateError) { return err(e, { + ...failedReceiptPayload, error_code: e.error_code, submit_verified: e.submit_verified, screen: e.screen, }); } if (e instanceof SubmitVerificationError) { - return err(e, submitVerificationFailurePayload(e)); + return err(e, { + ...failedReceiptPayload, + ...submitVerificationFailurePayload(e), + }); } - return err(e); + return err(e, failedReceiptPayload); } }, ); diff --git a/tests/agent-engine.test.ts b/tests/agent-engine.test.ts index 7ced25b0..b32002af 100644 --- a/tests/agent-engine.test.ts +++ b/tests/agent-engine.test.ts @@ -18,6 +18,7 @@ import { join } from "node:path"; import { tmpdir } from "node:os"; import { AgentEngine, + RetryableDeliveryError, assertLauncherAvailable, buildLaunchCommand, buildResumeCommand, @@ -11128,6 +11129,211 @@ Session ID: ${sessionId}`, /not in an interactive state/, ); }); + + it("keeps a durably accepted queued receipt when telemetry logging fails", () => { + vi.spyOn(stateMgr.getEventLog(), "appendDelivery").mockImplementation( + () => { + throw new Error("telemetry unavailable"); + }, + ); + + const receipt = engine.queueDelivery({ + agent_id: "queued-telemetry-failure", + text: "persist me", + press_enter: true, + source_event: "send_to", + }); + + expect(receipt).toMatchObject({ + delivery_state: "queued", + terminal: false, + }); + expect(engine.getDeliveryReceipt(receipt.delivery_id)).toMatchObject({ + text: "persist me", + delivery_state: "queued", + }); + }); + + it("makes a hung queued submission terminal-uncertain without replay", async () => { + stateMgr.writeState( + makeRecord({ + agent_id: "hung-queued-delivery", + state: "idle", + surface_id: "surface:42", + }), + ); + liveSurfaces = [makeSurface("surface:42")]; + await engine.getRegistry().reconstitute(); + (engine as any).deliverySubmitTimeoutMs = 5; + engine.setDeliverySubmitter( + async () => await new Promise(() => {}), + ); + const receipt = engine.queueDelivery({ + agent_id: "hung-queued-delivery", + text: "submit once", + press_enter: true, + source_event: "send_to", + }); + + await engine.drainDeliveryQueue(); + + expect(engine.getDeliveryReceipt(receipt.delivery_id)).toMatchObject({ + delivery_state: "failed", + terminal: true, + error: expect.stringMatching(/timed out|uncertain/i), + }); + }); + + it("keeps an accepted receipt queued across a transient posture mismatch and retries after capped backoff", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-11T18:00:00.000Z")); + try { + stateMgr.writeState( + makeRecord({ + agent_id: "posture-race-delivery", + state: "idle", + surface_id: "surface:42", + }), + ); + liveSurfaces = [makeSurface("surface:42")]; + await engine.getRegistry().reconstitute(); + const submitter = vi + .fn() + .mockRejectedValueOnce( + new RetryableDeliveryError( + 'Agent "posture-race-delivery" is not in an interactive state (current: working)', + ), + ) + .mockResolvedValueOnce({ retry_count: 0, submit_verified: true }); + engine.setDeliverySubmitter(submitter); + const receipt = engine.queueDelivery({ + agent_id: "posture-race-delivery", + text: "accepted means eventual", + press_enter: true, + source_event: "send_to", + }); + + await engine.drainDeliveryQueue(); + + const deferred = engine.getDeliveryReceipt(receipt.delivery_id); + expect(deferred).toMatchObject({ + delivery_state: "queued", + terminal: false, + retry_count: 1, + submission_started_at: null, + error: expect.stringMatching(/not in an interactive state/), + }); + expect(deferred?.next_attempt_at).toBe("2026-08-11T18:00:00.250Z"); + await engine.drainDeliveryQueue(); + expect(submitter).toHaveBeenCalledTimes(1); + + vi.setSystemTime(new Date("2026-08-11T18:00:00.250Z")); + await engine.drainDeliveryQueue(); + + expect(submitter).toHaveBeenCalledTimes(2); + expect(engine.getDeliveryReceipt(receipt.delivery_id)).toMatchObject({ + delivery_state: "submitted", + terminal: true, + retry_count: 1, + submit_verified: true, + error: null, + }); + } finally { + vi.useRealTimers(); + } + }); + + it("terminal-fails a queued receipt only when its target is gone", async () => { + const submitter = vi.fn(); + engine.setDeliverySubmitter(submitter); + const receipt = engine.queueDelivery({ + agent_id: "gone-queued-delivery", + text: "cannot arrive", + press_enter: true, + source_event: "send_to", + }); + + await engine.drainDeliveryQueue(); + + expect(submitter).not.toHaveBeenCalled(); + expect(engine.getDeliveryReceipt(receipt.delivery_id)).toMatchObject({ + delivery_state: "failed", + terminal: true, + error: expect.stringMatching(/target.*gone|no longer exists/i), + }); + }); + + it("caps repeated retryable delivery backoff at thirty seconds", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-11T18:00:00.000Z")); + try { + stateMgr.writeState( + makeRecord({ + agent_id: "bounded-backoff-delivery", + state: "idle", + surface_id: "surface:42", + }), + ); + liveSurfaces = [makeSurface("surface:42")]; + await engine.getRegistry().reconstitute(); + engine.setDeliverySubmitter(async () => { + throw new RetryableDeliveryError("route posture changed"); + }); + const receipt = engine.queueDelivery({ + agent_id: "bounded-backoff-delivery", + text: "keep accepted", + press_enter: true, + source_event: "send_to", + }); + + let previousAttemptAt = Date.now(); + for (let attempt = 1; attempt <= 8; attempt += 1) { + await engine.drainDeliveryQueue(); + const deferred = engine.getDeliveryReceipt(receipt.delivery_id)!; + expect(deferred).toMatchObject({ + delivery_state: "queued", + terminal: false, + retry_count: attempt, + }); + const nextAttemptAt = Date.parse(deferred.next_attempt_at!); + const expectedDelay = Math.min(30_000, 250 * 2 ** (attempt - 1)); + expect(nextAttemptAt - previousAttemptAt).toBe(expectedDelay); + vi.setSystemTime(nextAttemptAt); + previousAttemptAt = nextAttemptAt; + } + } finally { + vi.useRealTimers(); + } + }); + + it("does not replay a queued receipt whose persisted submission had started", () => { + const receipt = engine.queueDelivery({ + agent_id: "crashed-queued-delivery", + text: "maybe landed", + press_enter: true, + source_event: "send_to", + }); + const receiptPath = join(TEST_DIR, "delivery-receipts.json"); + const persisted = JSON.parse(readFileSync(receiptPath, "utf8")); + persisted[0].submission_started_at = "2026-08-11T18:00:00.000Z"; + writeFileSync(receiptPath, `${JSON.stringify(persisted)}\n`, "utf8"); + + const restartedState = new StateManager(TEST_DIR); + const restarted = new AgentEngine( + restartedState, + new AgentRegistry(restartedState, async () => []), + mockClient, + ); + try { + expect(restarted.getDeliveryReceipt(receipt.delivery_id)).toMatchObject({ + delivery_state: "failed", + terminal: true, + error: expect.stringMatching(/uncertain|restart/i), + }); + } finally { + restarted.dispose(); + } + }); }); }); diff --git a/tests/inbox-nudge.test.ts b/tests/inbox-nudge.test.ts index 8eac9acc..a4aadb8f 100644 --- a/tests/inbox-nudge.test.ts +++ b/tests/inbox-nudge.test.ts @@ -61,6 +61,7 @@ function makeExec( primarySurfaceUuid?: string, ): ExecFn { let promptPending = false; + let pastePending = false; let currentScreenText = screenText; const surfaces: TestSurface[] = [ { @@ -157,11 +158,21 @@ function makeExec( } return { stdout: "{}", stderr: "" }; } + if (args.includes("set-buffer")) { + pastePending = String(args.at(-1) ?? "").trim().length > 0; + return { stdout: "{}", stderr: "" }; + } + if (args.includes("paste-buffer")) { + if (pastePending) promptPending = true; + pastePending = false; + return { stdout: "{}", stderr: "" }; + } if (args.includes("send")) { const text = String(args.at(-1) ?? ""); if ( text.trim() && - !/[A-Za-z0-9_.-]+(?:Claude|Codex|Cursor|Gemini|Kiro)\b/.test(text) + (text.includes("cmuxlayer mailbox contract") || + !/[A-Za-z0-9_.-]+(?:Claude|Codex|Cursor|Gemini|Kiro)\b/.test(text)) ) { promptPending = true; } diff --git a/tests/inbox.test.ts b/tests/inbox.test.ts index 52340459..ed57b9b3 100644 --- a/tests/inbox.test.ts +++ b/tests/inbox.test.ts @@ -2,6 +2,7 @@ import { afterAll, describe, expect, it } from "vitest"; import { appendFileSync, existsSync, + mkdirSync, mkdtempSync, readdirSync, rmSync, @@ -15,6 +16,7 @@ import { dispatch, dispatchOnce, inboxMonitorState, + inboxCursorPath, inboxPath, monitorAlive, pendingDispatches, @@ -25,7 +27,9 @@ import { recommendedCodexWatch, recommendedMonitorCommand, replayUndelivered, + readInboxCursor, surfacedLogPath, + writeInboxCursor, writeHeartbeat, } from "../src/inbox.js"; @@ -203,6 +207,33 @@ describe("inbox write-channel", () => { ]); }); + it("replays only messages strictly after the agent-written consumption cursor", () => { + dispatch("cursor-a1", { from: "orc", task: "t1", id: "m1" }, opts); + dispatch("cursor-a1", { from: "orc", task: "t2", id: "m2" }, opts); + dispatch("cursor-a1", { from: "orc", task: "t3", id: "m3" }, opts); + + writeInboxCursor("cursor-a1", "m1", opts); + + expect(readInboxCursor("cursor-a1", opts)).toBe("m1"); + expect(replayUndelivered("cursor-a1", opts).map((message) => message.id)).toEqual([ + "m2", + "m3", + ]); + expect(existsSync(inboxCursorPath("cursor-a1", opts))).toBe(true); + }); + + it("serializes cursor advancement with a per-agent cross-process lock", () => { + dispatch("cursor-lock", { from: "orc", task: "t1", id: "m1" }, opts); + mkdirSync(`${inboxCursorPath("cursor-lock", opts)}.lock`, { + recursive: true, + }); + + expect(() => writeInboxCursor("cursor-lock", "m1", opts)).toThrow( + /cursor.*locked/i, + ); + expect(readInboxCursor("cursor-lock", opts)).toBeNull(); + }); + it("FM#3 pendingDispatches flags un-acked messages older than the ack-timeout", () => { clock = 5_000; dispatch("a4", { from: "orc", task: "old", id: "old1" }, opts); diff --git a/tests/pointer-discipline.test.ts b/tests/pointer-discipline.test.ts index a59f233f..409a6aac 100644 --- a/tests/pointer-discipline.test.ts +++ b/tests/pointer-discipline.test.ts @@ -50,6 +50,7 @@ async function spawnReadyAgent(server: any) { model: "sonnet", cli: "claude", workspace: "workspace:1", + boot_prompt_timeout_ms: 100, }, {} as any, ); @@ -70,6 +71,7 @@ function makeLifecycleExec(initialReadyText: string | (() => string) = "codex> " let readyText = typeof initialReadyText === "function" ? initialReadyText() : initialReadyText; let promptPending = false; + let submissionObservationPending = false; return vi.fn().mockImplementation(async (_cmd, args: string[]) => { if (args.includes("send-key") && args.includes("return")) { @@ -77,13 +79,17 @@ function makeLifecycleExec(initialReadyText: string | (() => string) = "codex> " readyText = "gpt-5.5 xhigh - 99% left - ~/Gits/cmuxlayer\nWorking (1s - esc to interrupt)"; promptPending = false; + submissionObservationPending = true; } return { stdout: "{}", stderr: "" }; } if (args.includes("send")) { const text = String(args.at(-1) ?? ""); - if (text.trim() && !/Codex\b/.test(text)) { + if ( + text.trim() && + (text.includes("cmuxlayer mailbox contract") || !/Codex\b/.test(text)) + ) { promptPending = true; } return { stdout: "{}", stderr: "" }; @@ -95,13 +101,15 @@ function makeLifecycleExec(initialReadyText: string | (() => string) = "codex> " } if (args.includes("read-screen")) { + const observedText = submissionObservationPending + ? readyText + : typeof initialReadyText === "function" + ? initialReadyText() + : readyText; return { stdout: JSON.stringify({ surface: "surface:new", - text: - typeof initialReadyText === "function" - ? initialReadyText() - : readyText, + text: observedText, lines: 20, scrollback_used: false, }), @@ -110,6 +118,9 @@ function makeLifecycleExec(initialReadyText: string | (() => string) = "codex> " } if (args.includes("list-workspaces")) { + // Spawn response shaping enumerates topology after boot submission has + // been verified. Return subsequent reads to the caller-controlled screen. + submissionObservationPending = false; return { stdout: JSON.stringify({ workspaces: [ diff --git a/tests/server-agent-tools.test.ts b/tests/server-agent-tools.test.ts index 85bfde45..9732e50f 100644 --- a/tests/server-agent-tools.test.ts +++ b/tests/server-agent-tools.test.ts @@ -30,6 +30,7 @@ import { generateAgentId, type AgentRecord } from "../src/agent-types.js"; import type { ParsedScreenResult } from "../src/types.js"; import type { SeatManifest } from "../src/seat-manifest.js"; import { AgentRegistry } from "../src/agent-registry.js"; +import { AgentEngine } from "../src/agent-engine.js"; import { StateManager } from "../src/state-manager.js"; import { reconcileMonitorRegistry, @@ -99,11 +100,13 @@ function makeLifecycleExec(opts?: { let surfaceLive = true; let promptPending = false; let activeCli: "claude" | "codex" | "cursor" = "claude"; + let createdSurfaceCount = 0; + let currentSurface = "surface:new"; const listedSurface = () => surfaceLive ? { paneRef: "pane:1", - surfaceRef: "surface:new", + surfaceRef: currentSurface, title: "agent-pane", } : { @@ -123,6 +126,13 @@ function makeLifecycleExec(opts?: { return vi.fn().mockImplementation(async (_cmd, args) => { if (args.includes("new-split") || args.includes("new-surface")) { surfaceLive = true; + createdSurfaceCount += 1; + currentSurface = + createdSurfaceCount === 1 + ? "surface:new" + : `surface:new-${createdSurfaceCount}`; + readyText = "$ "; + promptPending = false; } if (args.includes("close-surface") && !opts?.closeKeepsSurface) { surfaceLive = false; @@ -135,7 +145,7 @@ function makeLifecycleExec(opts?: { } return { stdout: "{}", stderr: "" }; } - if (args.includes("send")) { + if (args.includes("send") || args.includes("set-buffer")) { const text = String(args[args.length - 1] ?? ""); if (text.includes("Codex")) { activeCli = "codex"; @@ -151,7 +161,9 @@ function makeLifecycleExec(opts?: { } if ( text.trim() && - !/[A-Za-z0-9_.-]+(?:Claude|Codex|Cursor|Gemini|Kiro)\b/.test(text) + !/^\s*[A-Za-z0-9_.-]+(?:Claude|Codex|Cursor|Gemini|Kiro)\b.*(?:^|\s)-s(?:\s|$)/.test( + text, + ) ) { promptPending = true; } @@ -220,7 +232,7 @@ function makeLifecycleExec(opts?: { if (args.includes("read-screen")) { return { stdout: JSON.stringify({ - surface: "surface:new", + surface: currentSurface, text: opts?.shellNeverReady ? "terminal initializing" : readyText, lines: 20, scrollback_used: false, @@ -236,7 +248,7 @@ function makeLifecycleExec(opts?: { return { stdout: JSON.stringify({ workspace, - surface: "surface:new", + surface: currentSurface, ...(opts?.surfaceUuid ? { surface_id: opts.surfaceUuid } : {}), pane: "pane:1", title: "", @@ -633,8 +645,8 @@ describe("lean spawn tool responses", () => { state: "booting", model: "codex", role: "worker", - boot_prompt_delivered: false, - boot_prompt_submit_verified: null, + boot_prompt_delivered: true, + boot_prompt_submit_verified: true, }); expect(parsed).not.toHaveProperty("health"); expect(parsed).not.toHaveProperty("model_policy"); @@ -659,6 +671,27 @@ describe("lean spawn tool responses", () => { ); }); + it("injects the agent-owned inbox cursor helper into orchestrator boot metadata", async () => { + const server = createLifecycleServer(makeLifecycleExec()); + const spawn = (server as any)._registeredTools["spawn_agent"]; + + const result = await spawn.handler( + { + repo: "cmuxlayer", + cli: "claude", + placement: "orchestrator", + verbose: true, + }, + {} as any, + ); + + expect(result.structuredContent.monitor_boot).toMatchObject({ + cursor_path: expect.stringMatching(/inbox\.cursor$/), + cursor_update_command: expect.stringContaining("inbox-cursor"), + cursor_update_env: "CMUX_INBOX_MSG_ID", + }); + }); + it("spawn_agent surfaces a real model coercion in lean mode", async () => { const server = createLifecycleServer(makeLifecycleExec()); const spawn = (server as any)._registeredTools["spawn_agent"]; @@ -1909,7 +1942,7 @@ describe("agent lifecycle tool handlers", () => { sendKey: vi.fn().mockResolvedValue(undefined), readScreen: vi.fn().mockResolvedValue({ surface: "surface:inherit", - text: "Codex\n>", + text: "OpenAI Codex\ncodex> ", lines: 1, scrollback_used: false, }), @@ -2017,7 +2050,7 @@ describe("agent lifecycle tool handlers", () => { sendKey: vi.fn().mockResolvedValue(undefined), readScreen: vi.fn().mockResolvedValue({ surface: "surface:caller", - text: "Codex\n>", + text: "OpenAI Codex\ncodex> ", lines: 1, scrollback_used: false, }), @@ -2603,15 +2636,13 @@ describe("agent lifecycle tool handlers", () => { "cmux", expect.arrayContaining(["send", "--surface", "surface:new"]), ); - expect(mockExec).toHaveBeenCalledWith( - "cmux", - expect.arrayContaining([ - "send", - "--surface", - "surface:new", - "fix prompt delivery", - ]), - ); + expect( + mockExec.mock.calls.some( + ([, args]) => + args.includes("set-buffer") && + String(args.at(-1) ?? "").includes("fix prompt delivery"), + ), + ).toBe(true); expect(mockExec).toHaveBeenCalledWith( "cmux", expect.arrayContaining([ @@ -2706,22 +2737,19 @@ describe("agent lifecycle tool handlers", () => { const parsed = parseToolResult(result); expect(parsed.ok).toBe(true); + expect( + mockExec.mock.calls.some( + ([, args]) => + args.includes("set-buffer") && + String(args.at(-1) ?? "").includes("UUID-bound boot prompt"), + ), + ).toBe(true); expect(mockExec).toHaveBeenCalledWith( "cmux", expect.arrayContaining([ - "send", + "paste-buffer", "--surface", "surface:moved", - "UUID-bound boot prompt", - ]), - ); - expect(mockExec).not.toHaveBeenCalledWith( - "cmux", - expect.arrayContaining([ - "send", - "--surface", - "surface:new", - "UUID-bound boot prompt", ]), ); }); @@ -2798,15 +2826,18 @@ describe("agent lifecycle tool handlers", () => { expect(parsed.workspace_id).toBe("workspace:1"); expect(parsed.actual_workspace_id).toBeUndefined(); - const promptSendCall = mockExec.mock.calls.find(([, args]) => { + const promptBufferCall = mockExec.mock.calls.find(([, args]) => { const argv = args as string[]; - return argv.includes("send") && argv.includes(prompt); + return ( + argv.includes("set-buffer") && + String(argv.at(-1) ?? "").includes(prompt) + ); }); - expect(promptSendCall).toBeDefined(); - const argv = promptSendCall![1] as string[]; - const workspaceIndex = argv.indexOf("--workspace"); - expect(workspaceIndex).toBeGreaterThanOrEqual(0); - expect(argv[workspaceIndex + 1]).toBe("workspace:1"); + expect(promptBufferCall).toBeDefined(); + expect(mockExec).toHaveBeenCalledWith( + "cmux", + expect.arrayContaining(["paste-buffer", "--workspace", "workspace:1"]), + ); }); it("spawn_agent delivers prompts to the resolved workspace when cmux returns an empty workspace", async () => { @@ -2827,15 +2858,19 @@ describe("agent lifecycle tool handlers", () => { const parsed = parseToolResult(result); expect(parsed.ok).toBe(true); - const promptSendCall = (exec as ReturnType).mock.calls.find( + const promptBufferCall = (exec as ReturnType).mock.calls.find( ([, args]) => { const argv = args as string[]; - return argv.includes("send") && argv.includes(prompt); + return ( + argv.includes("set-buffer") && + String(argv.at(-1) ?? "").includes(prompt) + ); }, ); - expect(promptSendCall).toBeDefined(); - expect(promptSendCall![1]).toEqual( - expect.arrayContaining(["--workspace", "workspace:1"]), + expect(promptBufferCall).toBeDefined(); + expect(exec).toHaveBeenCalledWith( + "cmux", + expect.arrayContaining(["paste-buffer", "--workspace", "workspace:1"]), ); }); @@ -2910,7 +2945,8 @@ describe("agent lifecycle tool handlers", () => { (chunk) => Buffer.byteLength(chunk, "utf-8") <= 16_000, ), ).toBe(true); - expect(chunks.join("")).toBe(prompt); + expect(chunks.join("")).toContain(prompt); + expect(chunks.join("")).toContain("cmuxlayer mailbox contract"); expect(chunks.join("")).toContain("\n\n"); }); @@ -2922,8 +2958,8 @@ describe("agent lifecycle tool handlers", () => { mockExec = vi.fn().mockImplementation(async (cmd, args) => { if ( !renamed && - args.includes("send") && - String(args.at(-1) ?? "") === "probe renamed state" + args.includes("set-buffer") && + String(args.at(-1) ?? "").includes("probe renamed state") ) { renamed = true; finalAgentId = renameOnlyAgentStateToSession(sessionId); @@ -3655,7 +3691,7 @@ describe("agent lifecycle tool handlers", () => { sendKey: vi.fn().mockResolvedValue(undefined), readScreen: vi.fn().mockResolvedValue({ surface: "surface:caller-worktree", - text: "Codex\n>", + text: "OpenAI Codex\ncodex> ", lines: 1, scrollback_used: false, }), @@ -3865,15 +3901,13 @@ describe("agent lifecycle tool handlers", () => { const parsed = result.structuredContent ?? JSON.parse(result.content[0].text); expect(parsed.ok).toBe(true); - expect(mockExec).toHaveBeenCalledWith( - "cmux", - expect.arrayContaining([ - "send", - "--surface", - "surface:new", - "file prompt body", - ]), - ); + expect( + mockExec.mock.calls.some( + ([, args]) => + args.includes("set-buffer") && + String(args.at(-1) ?? "").includes("file prompt body"), + ), + ).toBe(true); }); it("read_agent_output scans bounded tail lines by default", async () => { @@ -3978,9 +4012,9 @@ describe("agent lifecycle tool handlers", () => { stderr: "", }; } - if (args.includes("send")) { + if (args.includes("send") || args.includes("set-buffer")) { lastSentText = String(args.at(-1) ?? ""); - if (lastSentText === "file prompt body") { + if (lastSentText.includes("file prompt body")) { promptDelivered = true; } return { stdout: JSON.stringify({ ok: true }), stderr: "" }; @@ -3996,7 +4030,7 @@ describe("agent lifecycle tool handlers", () => { stdout: JSON.stringify({ surface: "surface:new", text: - lastSentText === "file prompt body" + lastSentText.includes("file prompt body") ? "gpt-5.5 xhigh · 99% left · ~/Gits/voicelayer\nWorking (1s • esc to interrupt)" : lastSentText === "" ? "$ " @@ -4058,17 +4092,13 @@ describe("agent lifecycle tool handlers", () => { "surface:new", ]), ); - expect(mockExec).toHaveBeenCalledWith( - "cmux", - expect.arrayContaining([ - "send", - "--workspace", - "workspace:voice", - "--surface", - "surface:new", - "file prompt body", - ]), - ); + expect( + mockExec.mock.calls.some( + ([, args]) => + args.includes("set-buffer") && + String(args.at(-1) ?? "").includes("file prompt body"), + ), + ).toBe(true); }, 10_000); it("spawn_agent treats launch submit verification as advisory when readiness appears with shell history", async () => { @@ -4130,9 +4160,9 @@ describe("agent lifecycle tool handlers", () => { stderr: "", }; } - if (args.includes("send")) { + if (args.includes("send") || args.includes("set-buffer")) { lastSentText = String(args.at(-1) ?? ""); - if (lastSentText === "file prompt body") { + if (lastSentText.includes("file prompt body")) { promptDelivered = true; } return { stdout: JSON.stringify({ ok: true }), stderr: "" }; @@ -4148,7 +4178,7 @@ describe("agent lifecycle tool handlers", () => { stdout: JSON.stringify({ surface: "surface:new", text: - lastSentText === "file prompt body" + lastSentText.includes("file prompt body") ? "gpt-5.5 xhigh · 99% left · ~/Gits/voicelayer\nWorking (1s • esc to interrupt)" : lastSentText === "" ? "$ " @@ -4953,7 +4983,7 @@ describe("agent lifecycle tool handlers", () => { {} as any, ); - const result = await list.handler({}, {} as any); + const result = await list.handler({ state: "working" }, {} as any); const parsed = result.structuredContent ?? JSON.parse(result.content[0].text); expect(parsed.ok).toBe(true); @@ -5769,7 +5799,7 @@ describe("agent lifecycle tool handlers", () => { {} as any, ); - const result = await list.handler({ state: "working" }, {} as any); + const result = await list.handler({}, {} as any); const parsed = result.structuredContent ?? JSON.parse(result.content[0].text); @@ -6994,6 +7024,48 @@ codex> expect(engine.getAgentState(agentId)?.state).toBe("idle"); }); + it("send_to returns a keyed terminal failed receipt when delivery fails", async () => { + let failReturn = false; + const base = makeLifecycleExec({ + surfaceUuid: "11111111-2222-4333-8444-555555555555", + }); + const exec: ExecFn = vi.fn().mockImplementation(async (cmd, args) => { + if (failReturn && args.includes("send-key") && args.includes("return")) { + throw new Error("Return delivery failed"); + } + return base(cmd, args); + }); + const server = createLifecycleServer(exec); + const spawn = (server as any)._registeredTools["spawn_agent"]; + const sendTo = (server as any)._registeredTools["send_to"]; + const spawnResult = await spawn.handler( + { repo: "test", model: "sonnet", cli: "claude" }, + {} as any, + ); + const agentId = parseToolResult(spawnResult).agent_id as string; + const engine = (server as any)._registeredTools["interact"]._engine; + const idle = engine.stateMgr.resetState(agentId, "idle", {}); + engine.getRegistry().set(agentId, idle); + failReturn = true; + + const result = await sendTo.handler( + { agent_id: agentId, text: "continue", press_enter: true }, + {} as any, + ); + const failed = parseToolResult(result); + + expect(result.isError).toBe(true); + expect(failed).toMatchObject({ + delivery_id: expect.any(String), + delivery_state: "failed", + terminal: true, + }); + expect(engine.getDeliveryReceipt(failed.delivery_id)).toMatchObject({ + delivery_state: "failed", + terminal: true, + }); + }); + it.each(["send_to", "send_to_agent"] as const)( "%s refuses routed delivery when the agent pane has fallen back to a bare shell", async (toolName) => { @@ -8123,7 +8195,7 @@ codex> expect(routeClient.client.send).not.toHaveBeenCalled(); }); - it("send_to without allow_busy still rejects working agents (backwards compat)", async () => { + it("send_to queues a busy composer and drains to a terminal submitted event", async () => { const server = createLifecycleServer(mockExec); const spawn = (server as any)._registeredTools["spawn_agent"]; const sendTo = (server as any)._registeredTools["send_to"]; @@ -8143,15 +8215,67 @@ codex> const engine = (server as any)._registeredTools["interact"]._engine; const registry = engine.getRegistry(); - const agent = registry.get(agentId); - registry.set(agentId, { ...agent, state: "working" }); + const working = engine.stateMgr.updateRecord(agentId, { state: "working" }); + registry.set(agentId, working); + mockExec.mockClear(); const result = await sendTo.handler( { agent_id: agentId, text: "hello", press_enter: true }, {} as any, ); - expect(result.isError).toBe(true); - expect(result.content[0].text).toMatch(/not in an interactive state/); + const queued = parseToolResult(result); + expect(result.isError).toBeFalsy(); + expect(queued).toMatchObject({ + agent_id: agentId, + delivery_id: expect.any(String), + delivery_state: "queued", + terminal: false, + }); + expect( + mockExec.mock.calls.filter(([, args]) => args.includes("send")), + ).toHaveLength(0); + + const restartedState = new StateManager(TEST_DIR); + const restartedRegistry = new AgentRegistry(restartedState, async () => []); + const restartedEngine = new AgentEngine( + restartedState, + restartedRegistry, + {} as any, + ); + expect(restartedEngine.getDeliveryReceipt(queued.delivery_id)).toMatchObject({ + delivery_id: queued.delivery_id, + text: "hello", + delivery_state: "queued", + terminal: false, + }); + + await engine.drainDeliveryQueue(); + expect(engine.getDeliveryReceipt(queued.delivery_id)).toMatchObject({ + delivery_state: "queued", + terminal: false, + }); + + const ready = engine.stateMgr.updateRecord(agentId, { state: "idle" }); + registry.set(agentId, ready); + await new Promise((resolve) => setTimeout(resolve, 275)); + await engine.drainDeliveryQueue(); + + const receipt = engine.getDeliveryReceipt(queued.delivery_id); + expect(receipt).toMatchObject({ + delivery_id: queued.delivery_id, + delivery_state: "submitted", + terminal: true, + }); + expect( + engine.stateMgr + .getEventLog() + .readEntries() + .filter((entry: any) => entry.delivery_id === queued.delivery_id) + .at(-1), + ).toMatchObject({ + delivery_state: "submitted", + delivery_id: queued.delivery_id, + }); }); it("send_to_agent with allow_busy=true delivers to agents in working state", async () => { @@ -8300,6 +8424,53 @@ codex> ).not.toContain("registry_screen_disagreement"); }); + it("send_to preserves a submitted receipt when post-delivery evidence throws", async () => { + const routeClient = makeUuidRouteClient([ + { + ref: "surface:evidence-error", + id: "11111111-2222-4333-8444-555555555555", + workspace_ref: "workspace:evidence-error", + }, + ]); + const record = makeServerAgentRecord({ + agent_id: "receipt-survives-evidence-error", + surface_id: "surface:evidence-error", + surface_uuid: "11111111-2222-4333-8444-555555555555", + workspace_id: "workspace:evidence-error", + state: "ready", + repo: "cmuxlayer", + cli: "codex", + }); + const server = await createUuidRouteServer(routeClient, record); + const engine = registeredTestTool(server, "interact")._engine; + const originalSend = routeClient.client.send.getMockImplementation(); + routeClient.client.send.mockImplementation( + async (surface: string, text: string) => { + await originalSend?.(surface, text); + vi.spyOn(engine, "getAgentState").mockImplementation(() => { + throw new Error("post-delivery topology unavailable"); + }); + }, + ); + + const result = await registeredTestTool(server, "send_to").handler( + { + agent_id: record.agent_id, + text: "delivered before evidence failed", + press_enter: false, + }, + {}, + ); + + expect(result.isError).toBe(true); + expect(parseToolResult(result)).toMatchObject({ + delivery_id: expect.any(String), + delivery_state: "submitted", + terminal: true, + error: expect.stringContaining("post-delivery topology unavailable"), + }); + }); + it("send_to omits evidence when a UUID-less row becomes foreign after delivery", async () => { const routeClient = makeUuidRouteClient([ { diff --git a/tests/server.test.ts b/tests/server.test.ts index 1037e95a..8d2a52de 100644 --- a/tests/server.test.ts +++ b/tests/server.test.ts @@ -1147,6 +1147,9 @@ describe("tool handler integration", () => { it("spawn_in_workspace tool handler creates, selects, then spawns agents", async () => { const calls: string[] = []; let surfaceIndex = 0; + const pendingBootContracts = new Set(); + const submittedBootContracts = new Set(); + const launchedSurfaces = new Set(); const mockClient = { createWorkspace: vi.fn().mockImplementation(async (title: string) => { calls.push(`create:${title}`); @@ -1182,14 +1185,37 @@ describe("tool handler integration", () => { }), newSurface: vi.fn(), focusSurface: vi.fn().mockResolvedValue(undefined), - send: vi.fn().mockResolvedValue(undefined), - sendKey: vi.fn().mockResolvedValue(undefined), - readScreen: vi.fn().mockResolvedValue({ - surface: "surface:1", - text: "Claude Code\n>", + send: vi.fn().mockImplementation(async (surface: string, text: string) => { + if (text.includes("cmuxlayer mailbox contract")) { + pendingBootContracts.add(surface); + } else { + launchedSurfaces.add(surface); + } + }), + pasteText: vi.fn().mockImplementation( + async (surface: string, text: string) => { + if (text.includes("cmuxlayer mailbox contract")) { + pendingBootContracts.add(surface); + } + }, + ), + sendKey: vi.fn().mockImplementation(async (surface: string, key: string) => { + if (key === "return" && pendingBootContracts.delete(surface)) { + submittedBootContracts.add(surface); + } + }), + readScreen: vi.fn().mockImplementation(async (surface: string) => ({ + surface, + text: submittedBootContracts.has(surface) + ? "Working (1s - esc to interrupt)" + : !launchedSurfaces.has(surface) + ? "$ " + : surface === "surface:2" + ? "OpenAI Codex\nmodel: gpt-5.4\n\n›" + : "Claude Code\n>", lines: 1, scrollback_used: false, - }), + })), log: vi.fn().mockResolvedValue(undefined), setStatus: vi.fn().mockResolvedValue(undefined), clearStatus: vi.fn().mockResolvedValue(undefined), @@ -1213,6 +1239,7 @@ describe("tool handler integration", () => { tool.handler( { workspace_title: "red-team", + verbose: true, agents: [ { repo: "brainlayer", @@ -1230,7 +1257,7 @@ describe("tool handler integration", () => { }, {} as any, ), - 1_000, + 100_000, ); const parsed = @@ -1238,19 +1265,22 @@ describe("tool handler integration", () => { expect(parsed.ok).toBe(true); expect(parsed.workspace).toBe("workspace:grid"); expect(parsed.agents).toHaveLength(2); - expect(parsed).not.toHaveProperty("retry_count"); + expect(parsed).toHaveProperty("retry_count", 0); expect(parsed.agents[0]).toMatchObject({ agent_id: expect.any(String), surface_id: "surface:1", - workspace_id: "workspace:grid", - state: "booting", - model: expect.any(String), role: "orchestrator", - boot_prompt_delivered: false, - boot_prompt_submit_verified: null, + boot_prompt_delivered: true, + boot_prompt_submit_verified: true, + }); + expect(parsed.agents[0]).toHaveProperty("health"); + expect(parsed.agents[0]).toHaveProperty("monitor_boot"); + expect(parsed.agents[1]).toMatchObject({ + role: "worker", + boot_prompt_delivered: true, + boot_prompt_submit_verified: true, + monitor_boot: expect.any(Object), }); - expect(parsed.agents[0]).not.toHaveProperty("health"); - expect(parsed.agents[0]).not.toHaveProperty("monitor_boot"); expect(calls.slice(0, 4)).toEqual([ "create:red-team", "select:workspace:grid", @@ -1259,32 +1289,6 @@ describe("tool handler integration", () => { ]); expect(calls).toContain("spawn:workspace:grid:surface:2"); - const verboseResult = await runWithFakeTimers( - () => - tool.handler( - { - workspace_title: "red-team", - reuse_workspace: "workspace:grid", - verbose: true, - agents: [ - { - repo: "cmuxlayer", - model: "gpt-5.4", - cli: "codex", - role: "worker", - }, - ], - }, - {} as any, - ), - 1_000, - ); - expect(verboseResult.structuredContent).toHaveProperty("retry_count", 0); - expect(verboseResult.structuredContent.agents[0]).toHaveProperty("health"); - expect(verboseResult.structuredContent.agents[0]).toHaveProperty( - "monitor_boot", - ); - await server.close(); rmSync(stateDir, { recursive: true, force: true }); }); @@ -8103,16 +8107,25 @@ describe("tool handler integration", () => { if (args.includes("send")) { const text = String(args.at(-1) ?? ""); sentTexts.push(text); - if (text.includes("cmuxlayerCodex")) { + if ( + text.includes("cmuxlayerCodex") && + !text.includes("cmuxlayer mailbox contract") + ) { if (launcherSends === 0) { delete process.env.REPOGOLEM_ALLOW_MODEL; } launcherSends += 1; - } else if (text === prompt) { + } else if (text.includes(prompt)) { promptSent = true; } return { stdout: "{}", stderr: "" }; } + if (args.includes("set-buffer")) { + const text = String(args.at(-1) ?? ""); + sentTexts.push(text); + if (text.includes(prompt)) promptSent = true; + return { stdout: "{}", stderr: "" }; + } if (args.includes("read-screen")) { let text = "$ "; if (launcherSends === 1) { @@ -8181,9 +8194,13 @@ describe("tool handler integration", () => { expect(parsed.boot_prompt_delivered).toBe(true); expect(launcherSends).toBe(2); expect( - sentTexts.filter((text) => text.includes("cmuxlayerCodex")), + sentTexts.filter( + (text) => + text.includes("cmuxlayerCodex") && + !text.includes("cmuxlayer mailbox contract"), + ), ).toEqual([fixture.launcher_command, fixture.launcher_command]); - expect(sentTexts.filter((text) => text === prompt)).toHaveLength(1); + expect(sentTexts.filter((text) => text.includes(prompt))).toHaveLength(1); expect(returnPresses).toBeGreaterThanOrEqual(3); expect( new StateManager(stateDir).readState(parsed.agent_id), @@ -8244,6 +8261,12 @@ describe("tool handler integration", () => { } return { stdout: "{}", stderr: "" }; } + if (args.includes("set-buffer")) { + const text = String(args.at(-1) ?? ""); + sentTexts.push(text); + if (text.includes(prompt)) promptSent = true; + return { stdout: "{}", stderr: "" }; + } if (args.includes("send-key")) { const key = String(args.at(-1) ?? ""); if (key === "ctrl-c") { @@ -8388,6 +8411,8 @@ describe("tool handler integration", () => { if (launcherSendAttempts === 1) { throw new Error("socket closed before receiving response"); } + } else if (text.includes("cmuxlayer mailbox contract")) { + composer += text; } return { stdout: "{}", stderr: "" }; } @@ -8403,6 +8428,8 @@ describe("tool handler integration", () => { ? fixture.screen : submitted === fixture.launcher_command ? "OpenAI Codex\nmodel: gpt-5.6-sol high\n\n›" + : submitted?.includes("cmuxlayer mailbox contract") + ? "OpenAI Codex\nWorking (1s - esc to interrupt)" : composer ? `etanheyman ~/Gits/brainlayer [main] $ ${composer}` : "etanheyman ~/Gits/brainlayer [main] $"; @@ -8440,7 +8467,8 @@ describe("tool handler integration", () => { 2_000, ); - expect(submittedCommands).toEqual([fixture.launcher_command]); + expect(submittedCommands[0]).toBe(fixture.launcher_command); + expect(submittedCommands[1]).toContain("cmuxlayer mailbox contract"); expect(submittedCommands).not.toContain(fixture.corrupted_command); expect(launcherSendAttempts).toBe(1); } finally { @@ -8554,6 +8582,8 @@ describe("tool handler integration", () => { launcherWriteBecameAmbiguous = true; throw new Error(fixture.replay.transport_error); } + } else if (text.includes("cmuxlayer mailbox contract")) { + composer += text; } return { stdout: "{}", stderr: "" }; } @@ -8564,7 +8594,9 @@ describe("tool handler integration", () => { } if (args.includes("read-screen")) { let text = fixture.replay.stale_probe_screen; - if (submittedCommands.length > 0) { + if (submittedCommands.at(-1)?.includes("cmuxlayer mailbox contract")) { + text = "OpenAI Codex\nWorking (1s - esc to interrupt)"; + } else if (submittedCommands.length > 0) { text = "OpenAI Codex\nmodel: gpt-5.6-sol xhigh\n\n›"; } else if (launcherWriteBecameAmbiguous) { ambiguityProbeReads += 1; @@ -8624,9 +8656,12 @@ describe("tool handler integration", () => { expect(ambiguityProbeReads).toBeGreaterThanOrEqual( probe === "delayed-visible" ? 2 : 1, ); - expect(submittedCommands).toEqual( - probe === "delayed-visible" ? [fixture.launcher_command] : [], - ); + if (probe === "delayed-visible") { + expect(submittedCommands[0]).toBe(fixture.launcher_command); + expect(submittedCommands[1]).toContain("cmuxlayer mailbox contract"); + } else { + expect(submittedCommands).toEqual([]); + } expect(submittedCommands).not.toContain( fixture.captured_corrupted_command, ); @@ -8733,13 +8768,22 @@ describe("tool handler integration", () => { if (args.includes("send")) { const text = String(args.at(-1) ?? ""); sentTexts.push(text); - if (text.includes("cmuxlayerCodex")) { + if ( + text.includes("cmuxlayerCodex") && + !text.includes("cmuxlayer mailbox contract") + ) { launcherSends += 1; - } else if (text === prompt) { + } else if (text.includes(prompt)) { promptSent = true; } return { stdout: "{}", stderr: "" }; } + if (args.includes("set-buffer")) { + const text = String(args.at(-1) ?? ""); + sentTexts.push(text); + if (text.includes(prompt)) promptSent = true; + return { stdout: "{}", stderr: "" }; + } if (args.includes("read-screen")) { let text = "$ "; if (launcherSends === 1 && !updateAccepted) { @@ -8820,7 +8864,7 @@ describe("tool handler integration", () => { expect(updateAccepted).toBe(true); expect(updateMenuKeys).toEqual(["return"]); expect(launcherSends).toBe(2); - expect(sentTexts.filter((text) => text === prompt)).toHaveLength(1); + expect(sentTexts.filter((text) => text.includes(prompt))).toHaveLength(1); } finally { if (previousAllowModel === undefined) { delete process.env.REPOGOLEM_ALLOW_MODEL; @@ -8913,13 +8957,22 @@ describe("tool handler integration", () => { if (args.includes("send")) { const text = String(args.at(-1) ?? ""); sentTexts.push(text); - if (text.includes("cmuxlayerCodex")) { + if ( + text.includes("cmuxlayerCodex") && + !text.includes("cmuxlayer mailbox contract") + ) { launcherSent = true; - } else if (text === prompt) { + } else if (text.includes(prompt)) { promptSent = true; } return { stdout: "{}", stderr: "" }; } + if (args.includes("set-buffer")) { + const text = String(args.at(-1) ?? ""); + sentTexts.push(text); + if (text.includes(prompt)) promptSent = true; + return { stdout: "{}", stderr: "" }; + } if (args.includes("read-screen")) { let text = "$ "; if (launcherSent && !updateMenuAccepted) { @@ -8988,12 +9041,12 @@ describe("tool handler integration", () => { result.structuredContent ?? JSON.parse(result.content[0].text); expect(parsed.ok).toBe(true); expect(parsed.boot_prompt_delivered).toBe(true); - // Poll the repaint, confirm readiness, then preflight once more at the - // shared text-delivery boundary before typing the boot prompt. + // Poll the repaint, confirm readiness, then perform the shared delivery + // preflight and verification for the combined caller + mailbox contract. expect(postDismissMenuReads).toBe(3); expect(sentKeys).not.toContain("down"); expect(sentKeys).toContain("return"); - expect(sentTexts.filter((text) => text === prompt)).toHaveLength(1); + expect(sentTexts.filter((text) => text.includes(prompt))).toHaveLength(1); } finally { if (previousAllowModel === undefined) { delete process.env.REPOGOLEM_ALLOW_MODEL; @@ -9075,7 +9128,7 @@ describe("tool handler integration", () => { } if (args.includes("send")) { const text = String(args.at(-1) ?? ""); - if (text.includes("cmuxlayerCodex")) { + if (text.startsWith("cmuxlayerCodex ")) { launcherSent = true; } return { stdout: "{}", stderr: "" }; @@ -9400,7 +9453,7 @@ describe("tool handler integration", () => { } if (args.includes("send")) { const text = String(args.at(-1) ?? ""); - if (text.includes("cmuxlayerCodex")) { + if (text.startsWith("cmuxlayerCodex ")) { launcherSends += 1; } else if (text === prompt) { promptSent = true; @@ -9630,7 +9683,7 @@ describe("tool handler integration", () => { } if (args.includes("send")) { const text = String(args.at(-1) ?? ""); - if (text.includes("cmuxlayerCodex")) { + if (text.startsWith("cmuxlayerCodex ")) { launcherSends += 1; } else if (text === prompt) { promptSent = true; diff --git a/tests/spawn-monitor-boot.test.ts b/tests/spawn-monitor-boot.test.ts index 061a4449..61f3100d 100644 --- a/tests/spawn-monitor-boot.test.ts +++ b/tests/spawn-monitor-boot.test.ts @@ -16,7 +16,20 @@ import { withTestSurfaceObserver } from "./helpers/test-surface-observer.js"; const STATE_DIR = join(tmpdir(), "cmuxlayer-spawn-monitor-boot-state"); function makeExec(): ExecFn { + let submitted = false; + let bootTextSent = false; + let activeCli: "claude" | "codex" = "claude"; return vi.fn().mockImplementation(async (_cmd, args) => { + if (args.includes("send-key") && args.includes("return")) { + submitted = bootTextSent; + return { stdout: "{}", stderr: "" }; + } + if (args.includes("send")) { + const text = String(args[args.length - 1] ?? ""); + if (/Codex/.test(text)) activeCli = "codex"; + if (/Claude/.test(text)) activeCli = "claude"; + if (text.includes("cmuxlayer mailbox contract")) bootTextSent = true; + } if (args.includes("list-workspaces")) { return { stdout: JSON.stringify({ @@ -75,7 +88,13 @@ function makeExec(): ExecFn { return { stdout: JSON.stringify({ surface: "surface:new", - text: "Claude Code\n>", + text: submitted + ? activeCli === "codex" + ? "gpt-5.5 xhigh · 99% left · ~/Gits/cmuxlayer\nWorking (1s • esc to interrupt)" + : "Claude Code\n✻ Working" + : activeCli === "codex" + ? "OpenAI Codex\ncodex> " + : "Claude Code\n>", lines: 20, scrollback_used: false, }), @@ -105,6 +124,16 @@ function sendCalls(exec: ExecFn): string[][] { .map(([, args]: [string, string[]]) => args); } +function deliveredInput(exec: ExecFn): string { + return (exec as ReturnType).mock.calls + .filter( + ([, args]: [string, string[]]) => + args.includes("send") || args.includes("set-buffer"), + ) + .flatMap(([, args]: [string, string[]]) => args) + .join("\n"); +} + describe("spawn monitor boot", () => { let inboxDir: string; let exec: ExecFn; @@ -152,6 +181,9 @@ describe("spawn monitor boot", () => { heartbeat_written: true, heartbeat_source: "server_boot", monitor_command: expect.stringContaining(parsed.agent_id), + cursor_path: expect.stringContaining(parsed.agent_id), + cursor_update_command: expect.stringContaining("inbox-cursor"), + cursor_update_env: "CMUX_INBOX_MSG_ID", }); expect(parsed.monitor_boot.monitor_command).toContain("tail -n0 -F"); expect(existsSync(inboxPath(parsed.agent_id, { baseDir: inboxDir }))).toBe( @@ -162,7 +194,7 @@ describe("spawn monitor boot", () => { ); }); - it("does not monitor-boot worker spawns", async () => { + it("injects the concrete mailbox and cursor contract into worker boots", async () => { const spawn = server._registeredTools["spawn_agent"]; const result = await spawn.handler( @@ -172,13 +204,23 @@ describe("spawn monitor boot", () => { cli: "codex", role: "worker", workspace: "workspace:1", + boot_prompt_timeout_ms: 500, + verbose: true, }, {} as any, ); const parsed = parseToolResult(result); expect(parsed.ok).toBe(true); - expect(parsed.monitor_boot).toBeUndefined(); + expect(parsed.monitor_boot).toMatchObject({ + status: "bootstrapped", + cursor_update_env: "CMUX_INBOX_MSG_ID", + }); + const deliveredText = deliveredInput(exec); + expect(deliveredText).toContain(parsed.agent_id); + expect(deliveredText).toContain(parsed.monitor_boot.monitor_command); + expect(deliveredText).toContain(parsed.monitor_boot.cursor_update_command); + expect(deliveredText).toContain("CMUX_INBOX_MSG_ID"); expect(monitorAlive(parsed.agent_id, 1_000, { baseDir: inboxDir })).toBe( false, ); @@ -221,6 +263,9 @@ describe("spawn monitor boot", () => { heartbeat_written: false, heartbeat_source: "server_boot", monitor_command: expect.stringContaining(parsed.agent_id), + cursor_path: expect.stringContaining(parsed.agent_id), + cursor_update_command: expect.stringContaining("inbox-cursor"), + cursor_update_env: "CMUX_INBOX_MSG_ID", error: expect.stringContaining("ENOTDIR"), }); } finally { diff --git a/tests/spawn-workspace.test.ts b/tests/spawn-workspace.test.ts index 403fce30..01dc8184 100644 --- a/tests/spawn-workspace.test.ts +++ b/tests/spawn-workspace.test.ts @@ -76,6 +76,9 @@ function repoLabelFromFixturePath(path: string): string { function makeWorkspaceClient() { let surfaceIndex = 0; const calls: string[] = []; + const activeCli = new Map(); + const submitted = new Set(); + const returnCount = new Map(); const client = { calls, createWorkspace: vi.fn().mockImplementation(async (title: string) => { @@ -88,16 +91,36 @@ function makeWorkspaceClient() { listWorkspaces: vi.fn().mockResolvedValue({ workspaces: [{ ref: "workspace:grid", title: "grid" }], }), - listPanes: vi.fn().mockResolvedValue({ + listPanes: vi.fn().mockImplementation(async () => ({ workspace_ref: "workspace:grid", window_ref: "window:1", - panes: [], - }), - listPaneSurfaces: vi.fn().mockResolvedValue({ - workspace_ref: "workspace:grid", - window_ref: "window:1", - pane_ref: "pane:1", - surfaces: [], + panes: Array.from({ length: surfaceIndex }, (_unused, index) => ({ + ref: `pane:${index + 1}`, + index, + focused: index === surfaceIndex - 1, + surface_count: 1, + surface_refs: [`surface:${index + 1}`], + selected_surface_ref: `surface:${index + 1}`, + })), + })), + listPaneSurfaces: vi.fn().mockImplementation(async (opts) => { + const pane = opts?.pane ?? "pane:1"; + const index = Number(pane.split(":").at(-1) ?? "1"); + return { + workspace_ref: "workspace:grid", + window_ref: "window:1", + pane_ref: pane, + surfaces: + index <= surfaceIndex + ? [{ + ref: `surface:${index}`, + title: "agent-pane", + type: "terminal", + index: 0, + selected: true, + }] + : [], + }; }), newSplit: vi.fn().mockImplementation(async (_direction, opts) => { surfaceIndex += 1; @@ -112,13 +135,35 @@ function makeWorkspaceClient() { }), newSurface: vi.fn(), focusSurface: vi.fn().mockResolvedValue(undefined), - send: vi.fn().mockResolvedValue(undefined), - sendKey: vi.fn().mockResolvedValue(undefined), - readScreen: vi.fn().mockResolvedValue({ - surface: "surface:1", - text: "Claude Code\n>", - lines: 1, - scrollback_used: false, + send: vi.fn().mockImplementation(async (surface: string, text: string) => { + if (/Codex/.test(text)) activeCli.set(surface, "codex"); + if (/Claude/.test(text)) activeCli.set(surface, "claude"); + }), + pasteText: vi.fn().mockImplementation(async (surface: string, text: string) => { + if (/Codex/.test(text)) activeCli.set(surface, "codex"); + if (/Claude/.test(text)) activeCli.set(surface, "claude"); + }), + sendKey: vi.fn().mockImplementation(async (surface: string, key: string) => { + if (key === "return") { + const count = (returnCount.get(surface) ?? 0) + 1; + returnCount.set(surface, count); + if (count >= 2) submitted.add(surface); + } + }), + readScreen: vi.fn().mockImplementation(async (surface: string) => { + const cli = activeCli.get(surface) ?? "claude"; + return { + surface, + text: submitted.has(surface) + ? cli === "codex" + ? "gpt-5.5 xhigh · 99% left · ~/Gits/cmuxlayer\nWorking (1s • esc to interrupt)" + : "Claude Code\n✻ Working" + : cli === "codex" + ? "OpenAI Codex\ncodex> " + : "Claude Code\nWhat can I help you with?\n>", + lines: 2, + scrollback_used: false, + }; }), log: vi.fn().mockResolvedValue(undefined), setStatus: vi.fn().mockResolvedValue(undefined), @@ -194,20 +239,23 @@ describe("workspace spawn tools", () => { surface_id: "surface:1", repo: "brainlayer", cli: "claude", - monitor_boot: { + monitor_boot: expect.objectContaining({ status: "bootstrapped", heartbeat_written: true, heartbeat_source: "server_boot", monitor_command: expect.any(String), - }, + }), }), expect.objectContaining({ surface_id: "surface:2", repo: "cmuxlayer", cli: "codex", + monitor_boot: expect.objectContaining({ + status: "bootstrapped", + cursor_update_env: "CMUX_INBOX_MSG_ID", + }), }), ]); - expect(parsed.agents[1].monitor_boot).toBeUndefined(); expect( existsSync(inboxPath(parsed.agents[0].agent_id, { baseDir: inboxDir })), ).toBe(true); @@ -224,11 +272,13 @@ describe("workspace spawn tools", () => { it("spawn_in_workspace reports every created identity when a later launch fails", async () => { const client = makeWorkspaceClient(); let launcherSends = 0; - client.send.mockImplementation(async () => { - launcherSends += 1; - if (launcherSends === 2) { + const normalSend = client.send.getMockImplementation()!; + client.send.mockImplementation(async (surface: string, text: string) => { + if (!text.includes("cmuxlayer mailbox contract")) launcherSends += 1; + if (launcherSends === 2 && !text.includes("cmuxlayer mailbox contract")) { throw new Error("second launcher send failed"); } + await normalSend(surface, text); }); const server = createServer({ client: client as any,