Skip to content

Commit f179e6e

Browse files
fix(tui): drop tool-result image bytes from the transcript store unless image previews are on (#63577)
Co-authored-by: Aiden Cline <aidenpcline@gmail.com>
1 parent fd6a116 commit f179e6e

4 files changed

Lines changed: 324 additions & 18 deletions

File tree

‎packages/client/src/solid/data.ts‎

Lines changed: 70 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -84,9 +84,10 @@ export type CreateDataInput = {
8484
readonly onError?: (error: unknown) => void
8585
/**
8686
* Drops the payload of inline `data:` files in tool results before they enter the store. For clients that never
87-
* render tool-result files: a session full of screenshots otherwise keeps hundreds of MB of base64 alive.
87+
* render tool-result files: a session full of screenshots otherwise keeps hundreds of MB of base64 alive. A function
88+
* is read for each message and event, so a client can follow a setting that changes while it runs.
8889
*/
89-
readonly omitToolFileData?: boolean
90+
readonly omitToolFileData?: boolean | (() => boolean)
9091
}
9192

9293
const messagePageLimit = 20
@@ -182,8 +183,23 @@ function omitFileData(content: [ToolContent, ...ToolContent[]]): [ToolContent, .
182183
return [omit(content[0]), ...content.slice(1).map(omit)]
183184
}
184185

186+
const hasFileData = (part: Extract<SessionMessageInfo, { type: "assistant" }>["content"][number]) =>
187+
part.type === "tool" &&
188+
"content" in part.state &&
189+
!!part.state.content?.some((item) => item.type === "file" && item.uri.startsWith("data:"))
190+
191+
// A tool file whose `data:` payload `omitFileData` emptied.
192+
const isDroppedFile = (item: ToolContent) => item.type === "file" && item.uri === `data:${item.mime};base64,`
193+
194+
const hasDroppedFileData = (message: SessionMessageInfo) =>
195+
message.type === "assistant" &&
196+
message.content.some(
197+
(part) => part.type === "tool" && "content" in part.state && !!part.state.content?.some(isDroppedFile),
198+
)
199+
200+
// Returns the same message when there is nothing to drop, so the store keeps its objects.
185201
function omitMessageFileData(message: SessionMessageInfo): SessionMessageInfo {
186-
if (message.type !== "assistant") return message
202+
if (message.type !== "assistant" || !message.content.some(hasFileData)) return message
187203
return {
188204
...message,
189205
content: message.content.map((part) =>
@@ -268,16 +284,35 @@ export function createData(config: CreateDataInput) {
268284

269285
function refresh(load: () => Promise<unknown>) {
270286
if (disposed || (config.connection && config.connection.status() !== "connected")) return
271-
void load().catch((error) => {
272-
if (disposed || (config.connection && config.connection.status() !== "connected")) return
273-
if (isSessionNotFoundError(error)) {
274-
markDeleted(error.sessionID)
275-
return
276-
}
277-
if (isMessageNotFoundError(error)) return
278-
if (config.onError) return config.onError(error)
279-
console.error("Failed to refresh client data", error)
280-
})
287+
void load().catch(refreshFailed)
288+
}
289+
290+
// Copies back only the emptied tool files: live events may have moved the message on since the server read it.
291+
function restoreMessageFiles(sessionID: string, messageID: string) {
292+
return api()
293+
.session.message.get({ sessionID, messageID })
294+
.then((item) => {
295+
if (item.type !== "assistant" || omitFiles()) return
296+
message.editAssistant(sessionID, item.id, (assistant) => {
297+
assistant.content.forEach((part) => {
298+
if (part.type !== "tool" || !("content" in part.state) || !part.state.content?.some(isDroppedFile)) return
299+
const fetched = item.content.findLast((other) => other.type === "tool" && other.id === part.id)
300+
if (fetched?.type === "tool" && "content" in fetched.state && fetched.state.content)
301+
part.state.content = fetched.state.content
302+
})
303+
})
304+
})
305+
}
306+
307+
function refreshFailed(error: unknown) {
308+
if (disposed || (config.connection && config.connection.status() !== "connected")) return
309+
if (isSessionNotFoundError(error)) {
310+
markDeleted(error.sessionID)
311+
return
312+
}
313+
if (isMessageNotFoundError(error)) return
314+
if (config.onError) return config.onError(error)
315+
console.error("Failed to refresh client data", error)
281316
}
282317

283318
// Runs `load` once a burst of same-key events goes quiet, so N events cost one refetch.
@@ -319,7 +354,9 @@ export function createData(config: CreateDataInput) {
319354
Object.values(store.session.info).toSorted((a, b) => b.time.updated - a.time.updated),
320355
)
321356
const messageIndex = new Map<string, Map<string, number>>()
322-
const stored = (message: SessionMessageInfo) => (config.omitToolFileData ? omitMessageFileData(message) : message)
357+
const omitFiles = () =>
358+
typeof config.omitToolFileData === "function" ? config.omitToolFileData() : config.omitToolFileData === true
359+
const stored = (message: SessionMessageInfo) => (omitFiles() ? omitMessageFileData(message) : message)
323360
const sync = createSync()
324361
let activeUpdates: Map<string, DataSessionStatus | undefined> | undefined
325362
const pendingUpdates = new Map<string, Map<string, SessionInboxInfo | SessionInbox.Delivery | undefined>>()
@@ -1103,7 +1140,7 @@ export function createData(config: CreateDataInput) {
11031140
status: "completed",
11041141
input: tool.state.input,
11051142
metadata: event.data.metadata,
1106-
content: config.omitToolFileData ? omitFileData(event.data.content) : [...event.data.content],
1143+
content: omitFiles() ? omitFileData(event.data.content) : [...event.data.content],
11071144
}
11081145
tool.executed = event.data.executed || tool.executed === true
11091146
if (event.data.native) tool.native = { ...tool.native, ...event.data.native }
@@ -1119,7 +1156,7 @@ export function createData(config: CreateDataInput) {
11191156
input: typeof tool.state.input === "string" ? {} : tool.state.input,
11201157
metadata: event.data.metadata,
11211158
content:
1122-
config.omitToolFileData && event.data.content ? omitFileData(event.data.content) : event.data.content,
1159+
omitFiles() && event.data.content ? omitFileData(event.data.content) : event.data.content,
11231160
}
11241161
tool.executed = event.data.executed || tool.executed === true
11251162
if (event.data.native) tool.native = { ...tool.native, ...event.data.native }
@@ -1823,6 +1860,23 @@ export function createData(config: CreateDataInput) {
18231860
const position = messageIndex.get(sessionID)?.get(messageID)
18241861
return position === undefined ? undefined : messages?.[position]
18251862
},
1863+
/** Re-reads every loaded message whose tool file data was dropped, for a client that starts showing tool files. */
1864+
restoreFiles() {
1865+
const sessions = Object.entries(store.session.message).flatMap(([sessionID, messages]) => {
1866+
const ids = (messages ?? []).filter(hasDroppedFileData).map((item) => item.id)
1867+
return ids.length === 0 ? [] : [{ sessionID, ids }]
1868+
})
1869+
// Each session's messages are re-read one at a time, so many screenshots don't fan out at once.
1870+
return Promise.all(
1871+
sessions.map((session) =>
1872+
session.ids.reduce(
1873+
(previous, messageID) =>
1874+
previous.then(() => restoreMessageFiles(session.sessionID, messageID).catch(refreshFailed)),
1875+
Promise.resolve(),
1876+
),
1877+
),
1878+
)
1879+
},
18261880
sync(sessionID: string) {
18271881
return sync.run(`session.message:${sessionID}`, async () => {
18281882
const started = deletionEpoch

‎packages/client/test/solid-data.test.ts‎

Lines changed: 236 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { expect, test } from "bun:test"
22
import { getEventListeners } from "node:events"
3-
import { createRoot } from "solid-js"
3+
import { createRoot, createSignal } from "solid-js"
44
import { createData, type CreateDataInput } from "../src/solid"
55
import { OpenCode, type ModelInfo, type OpenCodeEvent, type Project, type SessionInfo } from "../src/promise"
66

@@ -330,6 +330,241 @@ test("omits inline tool-result file payloads from read and live tool results whe
330330
}
331331
})
332332

333+
test("reads a function omitToolFileData for each live tool result", async () => {
334+
const listeners = new Set<Parameters<CreateDataInput["event"]["listen"]>[0]>()
335+
const screenshot = `data:image/png;base64,${"A".repeat(4096)}`
336+
const running = (id: string) => ({
337+
type: "tool",
338+
id,
339+
name: "execute",
340+
time: { created: 1, ran: 1 },
341+
state: { status: "running", input: {}, metadata: {} },
342+
})
343+
const api = OpenCode.make({
344+
baseUrl: "http://opencode.local",
345+
fetch: async () =>
346+
Response.json({
347+
data: [
348+
{
349+
id: "msg_assistant",
350+
type: "assistant",
351+
agent: "build",
352+
model: { providerID: "provider", id: "model" },
353+
time: { created: 1 },
354+
content: [running("call_off"), running("call_on")],
355+
},
356+
],
357+
cursor: {},
358+
}),
359+
})
360+
const [omit, setOmit] = createSignal(true)
361+
const setup = createRoot((dispose) => ({
362+
data: createData({
363+
api: () => api,
364+
directory: "/project",
365+
omitToolFileData: omit,
366+
event: {
367+
on: () => () => {},
368+
listen(handler) {
369+
listeners.add(handler)
370+
return () => listeners.delete(handler)
371+
},
372+
},
373+
}),
374+
dispose,
375+
}))
376+
const succeed = (id: string, seq: number) => {
377+
const event: OpenCodeEvent = {
378+
id: `evt_${id}`,
379+
created: 2,
380+
type: "session.tool.success",
381+
durable: { aggregateID: "ses_toggle", seq, version: 2 },
382+
data: {
383+
sessionID: "ses_toggle",
384+
assistantMessageID: "msg_assistant",
385+
id,
386+
executed: true,
387+
content: [{ type: "file", uri: screenshot, mime: "image/png" }],
388+
},
389+
}
390+
listeners.forEach((listener) => listener({ name: event.type, details: event }))
391+
}
392+
try {
393+
await setup.data.session.message.sync("ses_toggle")
394+
succeed("call_off", 1)
395+
setOmit(false)
396+
succeed("call_on", 2)
397+
398+
expect(setup.data.session.message.get("ses_toggle", "msg_assistant")).toMatchObject({
399+
content: [
400+
{ state: { content: [{ type: "file", uri: "data:image/png;base64,", mime: "image/png" }] } },
401+
{ state: { content: [{ type: "file", uri: screenshot, mime: "image/png" }] } },
402+
],
403+
})
404+
} finally {
405+
setup.dispose()
406+
}
407+
})
408+
409+
test("restores dropped tool files from the server once they are shown again", async () => {
410+
const listeners = new Set<Parameters<CreateDataInput["event"]["listen"]>[0]>()
411+
const screenshot = `data:image/png;base64,${"A".repeat(4096)}`
412+
const assistant = (state: object) => ({
413+
id: "msg_assistant",
414+
type: "assistant",
415+
agent: "build",
416+
model: { providerID: "provider", id: "model" },
417+
time: { created: 1 },
418+
content: [{ type: "tool", id: "call_shot", name: "execute", time: { created: 1, ran: 1 }, state }],
419+
})
420+
const requests: string[] = []
421+
const api = OpenCode.make({
422+
baseUrl: "http://opencode.local",
423+
fetch: async (input) => {
424+
const url = new URL(input instanceof Request ? input.url : String(input))
425+
requests.push(url.pathname)
426+
if (url.pathname.endsWith("/message/msg_assistant"))
427+
return Response.json({
428+
data: assistant({
429+
status: "completed",
430+
input: {},
431+
content: [{ type: "file", uri: screenshot, mime: "image/png" }],
432+
}),
433+
})
434+
return Response.json({ data: [assistant({ status: "running", input: {}, metadata: {} })], cursor: {} })
435+
},
436+
})
437+
const [omit, setOmit] = createSignal(true)
438+
const setup = createRoot((dispose) => ({
439+
data: createData({
440+
api: () => api,
441+
directory: "/project",
442+
omitToolFileData: omit,
443+
event: {
444+
on: () => () => {},
445+
listen(handler) {
446+
listeners.add(handler)
447+
return () => listeners.delete(handler)
448+
},
449+
},
450+
}),
451+
dispose,
452+
}))
453+
try {
454+
await setup.data.session.message.sync("ses_restore")
455+
const success: OpenCodeEvent = {
456+
id: "evt_shot",
457+
created: 2,
458+
type: "session.tool.success",
459+
durable: { aggregateID: "ses_restore", seq: 1, version: 2 },
460+
data: {
461+
sessionID: "ses_restore",
462+
assistantMessageID: "msg_assistant",
463+
id: "call_shot",
464+
executed: true,
465+
content: [{ type: "file", uri: screenshot, mime: "image/png" }],
466+
},
467+
}
468+
listeners.forEach((listener) => listener({ name: success.type, details: success }))
469+
expect(setup.data.session.message.get("ses_restore", "msg_assistant")).toMatchObject({
470+
content: [{ state: { content: [{ uri: "data:image/png;base64," }] } }],
471+
})
472+
473+
setOmit(false)
474+
await setup.data.session.message.restoreFiles()
475+
476+
expect(requests.filter((path) => path.endsWith("/message/msg_assistant"))).toHaveLength(1)
477+
expect(setup.data.session.message.get("ses_restore", "msg_assistant")).toMatchObject({
478+
content: [{ state: { content: [{ type: "file", uri: screenshot, mime: "image/png" }] } }],
479+
})
480+
// Nothing is left to restore, so a second call reads nothing.
481+
await setup.data.session.message.restoreFiles()
482+
expect(requests.filter((path) => path.endsWith("/message/msg_assistant"))).toHaveLength(1)
483+
} finally {
484+
setup.dispose()
485+
}
486+
})
487+
488+
test("restoring dropped tool files keeps live progress made while the server read was in flight", async () => {
489+
const listeners = new Set<Parameters<CreateDataInput["event"]["listen"]>[0]>()
490+
const screenshot = `data:image/png;base64,${"A".repeat(4096)}`
491+
const tool = (id: string, state: object) => ({ type: "tool", id, name: "execute", time: { created: 1, ran: 1 }, state })
492+
const running = { status: "running", input: {}, metadata: {} }
493+
const assistant = (content: object[]) => ({
494+
id: "msg_assistant",
495+
type: "assistant",
496+
agent: "build",
497+
model: { providerID: "provider", id: "model" },
498+
time: { created: 1 },
499+
content,
500+
})
501+
const read = Promise.withResolvers<Response>()
502+
const api = OpenCode.make({
503+
baseUrl: "http://opencode.local",
504+
fetch: async (input) => {
505+
const url = new URL(input instanceof Request ? input.url : String(input))
506+
if (url.pathname.endsWith("/message/msg_assistant")) return read.promise
507+
return Response.json({ data: [assistant([tool("call_shot", running), tool("call_work", running)])], cursor: {} })
508+
},
509+
})
510+
const [omit, setOmit] = createSignal(true)
511+
const setup = createRoot((dispose) => ({
512+
data: createData({
513+
api: () => api,
514+
directory: "/project",
515+
omitToolFileData: omit,
516+
event: {
517+
on: () => () => {},
518+
listen(handler) {
519+
listeners.add(handler)
520+
return () => listeners.delete(handler)
521+
},
522+
},
523+
}),
524+
dispose,
525+
}))
526+
const succeed = (
527+
id: string,
528+
seq: number,
529+
content: Extract<OpenCodeEvent, { type: "session.tool.success" }>["data"]["content"],
530+
) => {
531+
const event: OpenCodeEvent = {
532+
id: `evt_${id}`,
533+
created: 2,
534+
type: "session.tool.success",
535+
durable: { aggregateID: "ses_inflight", seq, version: 2 },
536+
data: { sessionID: "ses_inflight", assistantMessageID: "msg_assistant", id, executed: true, content },
537+
}
538+
listeners.forEach((listener) => listener({ name: event.type, details: event }))
539+
}
540+
try {
541+
await setup.data.session.message.sync("ses_inflight")
542+
succeed("call_shot", 1, [{ type: "file", uri: screenshot, mime: "image/png" }])
543+
setOmit(false)
544+
const restoring = setup.data.session.message.restoreFiles()
545+
// The second tool finishes while the read is in flight; the server's snapshot predates it.
546+
succeed("call_work", 2, [{ type: "text", text: "done" }])
547+
read.resolve(
548+
Response.json({
549+
data: assistant([
550+
tool("call_shot", { status: "completed", input: {}, content: [{ type: "file", uri: screenshot, mime: "image/png" }] }),
551+
tool("call_work", running),
552+
]),
553+
}),
554+
)
555+
await restoring
556+
557+
expect(setup.data.session.message.get("ses_inflight", "msg_assistant")).toMatchObject({
558+
content: [
559+
{ id: "call_shot", state: { status: "completed", content: [{ type: "file", uri: screenshot }] } },
560+
{ id: "call_work", state: { status: "completed", content: [{ type: "text", text: "done" }] } },
561+
],
562+
})
563+
} finally {
564+
setup.dispose()
565+
}
566+
})
567+
333568
test("refreshes a loaded session's execution start when it starts running", async () => {
334569
const listeners = new Set<Parameters<CreateDataInput["event"]["listen"]>[0]>()
335570
const api = OpenCode.make({

0 commit comments

Comments
 (0)