feat(rpc): structured value codec and payload accounting for the compact consumer (S03b-1b-I-a, part of #4272)

changes/unreleased/4385.md

@@ -0,0 +1,21 @@
+<!-- changes-log: packages/coding-agent/src/modes/rpc/changes.md -->
+## 2026-10-26 - Stage the bounded structured-value codec (S03b-1b-I-a, part of #4272)
+
+### What changed
+
+- `packages/coding-agent/src/modes/rpc/compact-values.ts` owns structured values in bounded field, array, and string pages, with shared immutable dependencies and cleanup of failed descriptor merges. It has no production caller in this increment; S03b-1b-I-b wires the settled index and removes the staging guard.
+- `packages/coding-agent/src/modes/rpc/compact-content-store.ts`, `compact-spool-storage.ts`, and `compact-storage-io.ts` account for structural payloads separately, coalesce their spill scheduling, and expose typed write-side limit failures.
+- `packages/coding-agent/src/modes/rpc/compact-text-roots.ts` owns shared dependency roots and grows the append tree only when the subtree is full.
+- Codec writes reject unsupported JSON values, cycles, nesting beyond 128 levels, more than 100,000 visited values, and a 16 MiB encoding budget with typed errors, releasing partial owners on failure. Scalars and strings whose JSON encoding fits 64 code units are inline; page traversal reads each structural leaf once.
+
+### Why
+
+- Bounded index descriptors need spillable values and keys before the index can depend on them. Failed storage must retain accepted content, and write failures must not be classified as reads.
+
+### Why an extension could not handle it
+
+- These modules own storage references, spool completion, and recovery reads below extension delivery.
+
+### Expected merge conflict zones
+
+- The compact store's leaf accounting, text-root construction, and spool operation types. All paths are fork-owned.

packages/coding-agent/src/modes/rpc/compact-content-store.ts

@@ -17,7 +17,24 @@ export const COMPACT_LEAF_CODE_UNITS = 32_768;
 
 export type LeafId = number;
 
+export class CompactWriteError extends Error {
+	readonly reason:
+		| "size_limit"
+		| "storage_unavailable"
+		| "depth_limit"
+		| "count_limit"
+		| "cycle"
+		| "unsupported_value";
+
+	constructor(reason: CompactWriteError["reason"]) {
+		super(`compact write refused (${reason})`);
+		this.name = "CompactWriteError";
+		this.reason = reason;
+	}
+}
+
 export interface CompactStorageStatus {
+	readonly structuredBytes: { readonly resident: number; readonly spilled: number; readonly pinned: number };
 	readonly residentPayloadBytes: number;
 	readonly pendingIoCopyBytes: number;
 	readonly pinnedOnlyBytes: number;
@@ -40,6 +57,7 @@ export interface CompactContentStoreOptions {
 }
 
 interface Leaf {
+	readonly value: boolean;
 	text: string | undefined;
 	readonly units: number;
 	live: number;
@@ -54,6 +72,9 @@ export class CompactContentStore extends CompactSpoolStorage {
 	private readonly leaves = new Map<LeafId, Leaf>();
 	/** Resident leaves in creation order, so each spill takes the oldest without rescanning committed ones. */
 	private readonly ramLeaves = new Set<LeafId>();
+	private readonly ramValues = new Set<LeafId>();
+	private valueQueued = false;
+	private readonly structuredBytes = { resident: 0, spilled: 0, pinned: 0 };
 	private nextLeafId = 1;
 	private residentPayloadBytes = 0;
 	private pinnedOnlyBytes = 0;
@@ -82,13 +103,14 @@ export class CompactContentStore extends CompactSpoolStorage {
 	}
 
 	/** Takes ownership of an immutable value with one live reference. */
-	createLeaf(text: string): LeafId {
+	createLeaf(text: string, value = false): LeafId {
 		this.assertReadable();
-		if (text.length > COMPACT_LEAF_CODE_UNITS) throw new CompactReadError("size_limit");
+		if (text.length > COMPACT_LEAF_CODE_UNITS) throw new CompactWriteError("size_limit");
 		const id = this.nextLeafId++;
-		this.leaves.set(id, { text, units: text.length, live: 1, pins: 0, state: "ram" });
-		this.ramLeaves.add(id);
-		this.residentPayloadBytes += text.length * 2;
+		this.leaves.set(id, { text, units: text.length, value, live: 1, pins: 0, state: "ram" });
+		(value ? this.ramValues : this.ramLeaves).add(id);
+		if (value) this.structuredBytes.resident += text.length * 2;
+		else this.residentPayloadBytes += text.length * 2;
 		this.scheduleSpill();
 		return id;
 	}
@@ -102,16 +124,22 @@ export class CompactContentStore extends CompactSpoolStorage {
 		const pinnedOnly = leaf.live === 0 && leaf.pins > 0;
 		leaf.live += live;
 		leaf.pins += pins;
-		if (pinnedOnly !== (leaf.live === 0 && leaf.pins > 0)) this.pinnedOnlyBytes += (pinnedOnly ? -2 : 2) * leaf.units;
+		if (pinnedOnly !== (leaf.live === 0 && leaf.pins > 0)) {
+			const bytes = (pinnedOnly ? -2 : 2) * leaf.units;
+			if (leaf.value) this.structuredBytes.pinned += bytes;
+			else this.pinnedOnlyBytes += bytes;
+		}
 		if (leaf.live > 0 || leaf.pins > 0) return;
 		this.leaves.delete(id);
-		this.ramLeaves.delete(id);
+		(leaf.value ? this.ramValues : this.ramLeaves).delete(id);
 		if (leaf.state === "committed" && leaf.path) {
-			this.spilledLiveBytes -= leaf.units * 2;
+			if (leaf.value) this.structuredBytes.spilled -= leaf.units * 2;
+			else this.spilledLiveBytes -= leaf.units * 2;
 			this.discard(leaf.path, leaf.units * 2);
 			return;
 		}
-		this.residentPayloadBytes -= leaf.units * 2;
+		if (leaf.value) this.structuredBytes.resident -= leaf.units * 2;
+		else this.residentPayloadBytes -= leaf.units * 2;
 	}
 
 	isCommitted(id: LeafId): boolean {
@@ -135,6 +163,7 @@ export class CompactContentStore extends CompactSpoolStorage {
 
 	status(): CompactStorageStatus {
 		return {
+			structuredBytes: { ...this.structuredBytes },
 			residentPayloadBytes: this.residentPayloadBytes,
 			pendingIoCopyBytes: this.pendingIoCopyBytes,
 			pinnedOnlyBytes: this.pinnedOnlyBytes,
@@ -150,6 +179,23 @@ export class CompactContentStore extends CompactSpoolStorage {
 	}
 
 	protected scheduleSpill(): void {
+		if (
+			!this.valueQueued &&
+			this.spillAllowed &&
+			this.ramValues.size &&
+			this.reclaimDebtBytes <= RECLAIM_WORK_ALLOWANCE_BYTES
+		) {
+			this.valueQueued = true;
+			this.enqueue("value_write", async () => {
+				const [id] = this.ramValues;
+				try {
+					if (id !== undefined) await this.flushLeaf(id);
+				} finally {
+					this.valueQueued = false;
+				}
+				this.scheduleSpill();
+			});
+		}
 		if (this.spillQueued || !this.spillAllowed || this.residentPayloadBytes <= this.spillAboveBytes) return;
 		if (this.reclaimDebtBytes > RECLAIM_WORK_ALLOWANCE_BYTES) return;
 		this.spillQueued = true;
@@ -159,22 +205,34 @@ export class CompactContentStore extends CompactSpoolStorage {
 	/** Spills one oldest resident leaf per job, so other storage work interleaves fairly. */
 	private async spillNext(): Promise<void> {
 		const [id] = this.ramLeaves;
-		const leaf = id === undefined ? undefined : this.leaves.get(id);
-		if (id === undefined || !leaf || leaf.text === undefined || !this.spillAllowed) {
-			this.spillQueued = false;
-			return;
+		if (id !== undefined) await this.flushLeaf(id);
+		this.spillQueued = false;
+		if (this.residentPayloadBytes > this.spillTargetBytes) {
+			this.spillQueued = this.spillAllowed && this.ramLeaves.size > 0;
+			if (this.spillQueued) this.enqueue("spill_write", () => this.spillNext());
 		}
+	}
+
+	/** Serial storage jobs can flush one newly built structural page without waiting on their own queue. */
+	async flushLeaf(id: LeafId): Promise<void> {
+		const leaf = this.leaves.get(id);
+		if (!leaf || leaf.state !== "ram" || leaf.text === undefined || !this.spillAllowed) return;
 		const bytes = leaf.units * 2;
 		leaf.state = "pending";
-		this.ramLeaves.delete(id);
+		const ram = leaf.value ? this.ramValues : this.ramLeaves;
+		ram.delete(id);
 		let path: string | undefined;
 		try {
-			path = await this.writeFile(Buffer.from(leaf.text, "utf16le"), "spill_write", bytes);
+			path = await this.writeFile(
+				Buffer.from(leaf.text, "utf16le"),
+				leaf.value ? "value_write" : "spill_write",
+				bytes,
+			);
 		} finally {
 			// A failed write, or a failure report that threw, keeps the leaf resident and spillable.
 			if (path === undefined) {
 				leaf.state = "ram";
-				if (this.leaves.get(id) === leaf) this.ramLeaves.add(id);
+				if (this.leaves.get(id) === leaf) ram.add(id);
 			}
 		}
 		const owned = this.leaves.get(id) === leaf;
@@ -184,14 +242,14 @@ export class CompactContentStore extends CompactSpoolStorage {
 			leaf.state = "committed";
 			leaf.path = path;
 			leaf.text = undefined;
-			this.residentPayloadBytes -= bytes;
-			this.spilledLiveBytes += bytes;
+			if (leaf.value) {
+				this.structuredBytes.resident -= bytes;
+				this.structuredBytes.spilled += bytes;
+			} else {
+				this.residentPayloadBytes -= bytes;
+				this.spilledLiveBytes += bytes;
+			}
 			this.committed(id);
 		}
-		this.spillQueued = false;
-		if (this.residentPayloadBytes > this.spillTargetBytes) {
-			this.spillQueued = this.spillAllowed;
-			if (this.spillQueued) this.enqueue("spill_write", () => this.spillNext());
-		}
 	}
 }

packages/coding-agent/src/modes/rpc/compact-spool-storage.ts

@@ -112,7 +112,8 @@ export abstract class CompactSpoolStorage {
 	): Promise<string | undefined> {
 		const path = await this.ensureDir(retainedInMemoryBytes);
 		if (path === undefined) return undefined;
-		const file = join(path, `${operation === "spill_write" ? "leaf" : "page"}-${this.nextFile++}.bin`);
+		const prefix = operation === "spill_write" ? "leaf" : operation === "value_write" ? "value" : "page";
+		const file = join(path, `${prefix}-${this.nextFile++}.bin`);
 		this.pendingIoCopyBytes += data.byteLength;
 		try {
 			await this.io.writeFile(file, data);
@@ -206,7 +207,8 @@ export abstract class CompactSpoolStorage {
 	}
 
 	private fail(operation: CompactStorageOperation, path: string, cause: unknown, retained: number): void {
-		if (operation === "spill_write" || operation === "spool_create") this.halted = true;
+		if (operation === "spill_write" || operation === "value_write" || operation === "spool_create")
+			this.halted = true;
 		this.record({ operation, path, cause: describeCause(cause), retainedInMemoryBytes: retained });
 	}
 

packages/coding-agent/src/modes/rpc/compact-storage-io.ts

@@ -8,6 +8,7 @@ import { readFile, unlink, writeFile } from "node:fs/promises";
 export type CompactStorageOperation =
 	| "spool_create"
 	| "spill_write"
+	| "value_write"
 	| "page_write"
 	| "leaf_read"
 	| "page_read"

packages/coding-agent/src/modes/rpc/compact-text-roots.ts

@@ -7,6 +7,7 @@ interface Node {
 	readonly right: LeafId;
 	readonly units: number;
 	readonly height: number;
+	readonly full: boolean;
 	live: number;
 	pins: number;
 }
@@ -27,6 +28,25 @@ export class CompactTextRoots {
 		return this.nodes.size;
 	}
 
+	/** One owned dependency root; callers retain their existing references. */
+	own(refs: readonly LeafId[]): LeafId {
+		if (refs.length === 0) return 0;
+		if (refs.length === 1) {
+			const [ref] = refs;
+			this.store.adjust(ref, 1, 0);
+			return ref;
+		}
+		const middle = Math.floor(refs.length / 2);
+		const left = this.own(refs.slice(0, middle));
+		const right = this.own(refs.slice(middle));
+		if (left === 0) return right;
+		if (right === 0) return left;
+		const root = this.join(left, right);
+		this.store.adjust(left, -1, 0);
+		this.store.adjust(right, -1, 0);
+		return root;
+	}
+
 	units(id: LeafId): number {
 		return id === 0 ? 0 : id > 0 ? this.store.leafUnits(id) : this.node(id).units;
 	}
@@ -129,7 +149,7 @@ export class CompactTextRoots {
 
 	private appendLeaf(root: LeafId, leaf: LeafId): LeafId {
 		const node = this.nodes.get(root);
-		if (!node || this.height(node.left) === this.height(node.right)) return this.join(root, leaf);
+		if (!node || node.full) return this.join(root, leaf);
 		const right = this.appendLeaf(node.right, leaf);
 		const result = this.join(node.left, right);
 		this.store.adjust(right, -1, 0);
@@ -147,6 +167,10 @@ export class CompactTextRoots {
 			right,
 			units: this.units(left) + this.units(right),
 			height: 1 + Math.max(this.height(left), this.height(right)),
+			full:
+				this.height(left) === this.height(right) &&
+				(this.nodes.get(left)?.full ?? true) &&
+				(this.nodes.get(right)?.full ?? true),
 			live: 1,
 			pins: 0,
 		});