Skip to content

Commit 38bd737

Browse files
authored
fix: keep text cuts from splitting characters (#63967)
1 parent 46744b6 commit 38bd737

14 files changed

Lines changed: 320 additions & 22 deletions

File tree

‎packages/core/src/ripgrep.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { ChildProcess } from "effect/process"
55
import { Entry, Match } from "@opencode/schema/filesystem"
66
import { makeLocationNode } from "@opencode/util/effect/app-node"
77
import { collectStream, waitForAbort } from "@opencode/util/process"
8+
import { TextBoundary } from "@opencode/util/text-boundary"
89
import { Environment } from "./environment/index.js"
910
import { NonNegativeInt, PositiveInt, RelativePath } from "./schema.js"
1011
import { RipgrepBinary } from "./ripgrep/binary.js"
@@ -252,7 +253,7 @@ const layer = Layer.effect(
252253
lines: {
253254
text:
254255
match.data.lines.text.length > MAX_LINE_CHARS
255-
? Buffer.from(match.data.lines.text.slice(0, MAX_LINE_CHARS), "utf8").toString() + "..."
256+
? TextBoundary.head(match.data.lines.text, MAX_LINE_CHARS) + "..."
256257
: match.data.lines.text,
257258
},
258259
path: { text: normalizePath(match.data.path.text) },

‎packages/core/src/session/title.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import { Database } from "../database/database.js"
88
import { Bus } from "../bus.js"
99
import { makeLocationNode } from "@opencode/util/effect/app-node"
1010
import { isExactRootFallback } from "@opencode/util/session-title-fallback"
11+
import { TextBoundary } from "@opencode/util/text-boundary"
1112
import { llmClient } from "../effect/app-node-platform.js"
1213
import { SessionContext } from "./context.js"
1314
import { SessionEvent } from "./event.js"
@@ -29,7 +30,8 @@ export interface Interface {
2930

3031
export class Service extends Context.Service<Service, Interface>()("@opencode/SessionTitle") {}
3132

32-
const truncate = (value: string) => (value.length <= MAX_LENGTH ? value : `${value.slice(0, MAX_LENGTH - 3)}...`)
33+
const truncate = (value: string) =>
34+
value.length <= MAX_LENGTH ? value : `${TextBoundary.head(value, MAX_LENGTH - 3)}...`
3335
// Framing the input as text to name keeps models from answering or carrying out the request instead.
3436
const messagePrompt = (text: string) =>
3537
`Here is the first message of the conversation:\n\n<message>\n${text}\n</message>\n\nReply with only the title.`
@@ -119,7 +121,7 @@ export const layer = Layer.effect(
119121
const text = !isUntitled(session)
120122
? yield* store.context(session.id).pipe(
121123
Effect.map((messages) => {
122-
const original = `Original request:\n${firstUser.text.slice(0, MAX_FIRST_MESSAGE_LENGTH)}`
124+
const original = `Original request:\n${TextBoundary.head(firstUser.text, MAX_FIRST_MESSAGE_LENGTH)}`
123125
const recent = messages
124126
.flatMap((message) => {
125127
if (message.type === "user" && message.id !== firstUser.id) return [`User: ${message.text.trim()}`]
@@ -134,7 +136,7 @@ export const layer = Layer.effect(
134136
if (!recent) return conversationPrompt(original)
135137
const prefix = `${original}\n\nRecent conversation:\n`
136138
const budget = MAX_CONTEXT_LENGTH - conversationPrompt("").length - prefix.length
137-
return conversationPrompt(`${prefix}${recent.slice(-budget)}`)
139+
return conversationPrompt(`${prefix}${TextBoundary.tail(recent, budget)}`)
138140
}),
139141
Effect.orElseSucceed(() => messagePrompt(firstUser.text)),
140142
)

‎packages/core/src/shell.ts‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import { AppProcess } from "@opencode/util/process"
2222
import { CrossSpawnSpawner } from "@opencode/util/cross-spawn-spawner"
2323
import { makeGlobalNode, makeLocationNode } from "@opencode/util/effect/app-node"
2424
import { FSUtil } from "@opencode/util/fs-util"
25+
import { TextBoundary } from "@opencode/util/text-boundary"
2526
import { Bus } from "./bus.js"
2627
import { Environment } from "./environment/index.js"
2728
import { FileRetention } from "./file-retention.js"
@@ -825,7 +826,7 @@ const layer = () =>
825826
// Read the byte before the window too, to tell whether the window starts on a line boundary.
826827
const before = Math.max(0, start - 1)
827828
const window = yield* readRange(info.file, before, size - before)
828-
const page = window.subarray(charBoundary(window, start - before)).toString("utf8")
829+
const page = TextBoundary.utf8Tail(window, size - start)
829830
const partial = start > 0 && window[0] !== 10
830831
const endsWithNewline = page.endsWith("\n")
831832
const lines = page.split("\n")
@@ -1142,13 +1143,6 @@ function spanCommand(command: string) {
11421143
return `${command.slice(0, SPAN_COMMAND_LIMIT).toWellFormed()}…`
11431144
}
11441145

1145-
// Moves past UTF-8 continuation bytes (at most 3) so a byte cut never splits a character.
1146-
function charBoundary(bytes: Buffer, offset: number) {
1147-
const limit = Math.min(bytes.length, offset + 3)
1148-
for (let index = offset; index < limit; index++) if ((bytes[index] & 0xc0) !== 0x80) return index
1149-
return limit
1150-
}
1151-
11521146
function formatSize(bytes: number) {
11531147
const kb = bytes / 1024
11541148
return kb > 1024 ? `${(kb / 1024).toFixed(1)} MB` : `${Math.round(kb)} KB`

‎packages/core/src/tool/read-filesystem.ts‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ export * as ReadToolFileSystem from "./read-filesystem.js"
33
import path from "path"
44
import { pathToFileURL } from "url"
55
import { makeLocationNode } from "@opencode/util/effect/app-node"
6+
import { TextBoundary } from "@opencode/util/text-boundary"
67
import { Context, Effect, Layer, Schema } from "effect"
78
import { lookup } from "mime-types"
89
import { Environment } from "../environment/index.js"
@@ -374,12 +375,8 @@ const list = (items: ReadonlyArray<Environment.DirEntry>, page: PageInput, sep =
374375
})
375376
}
376377

377-
const clampLine = (line: string) => {
378-
if (line.length <= MAX_LINE_LENGTH) return line
379-
const code = line.charCodeAt(MAX_LINE_LENGTH - 1)
380-
const end = code >= 0xd800 && code <= 0xdbff ? MAX_LINE_LENGTH - 1 : MAX_LINE_LENGTH
381-
return line.slice(0, end) + MAX_LINE_SUFFIX
382-
}
378+
const clampLine = (line: string) =>
379+
line.length <= MAX_LINE_LENGTH ? line : TextBoundary.head(line, MAX_LINE_LENGTH) + MAX_LINE_SUFFIX
383380

384381
const layer = Layer.effect(
385382
Service,

‎packages/core/test/ripgrep.test.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,28 @@ describe("Ripgrep", () => {
184184
}),
185185
),
186186
)
187+
188+
it.live("ends oversized line previews on a whole character", () =>
189+
withTempDir((tmp) =>
190+
Effect.gen(function* () {
191+
yield* Effect.promise(() =>
192+
fs.writeFile(
193+
path.join(tmp.path, "emoji.txt"),
194+
[1_998, 1_999, 2_000].map((count) => `${"x".repeat(count)}😀tail\n`).join(""),
195+
),
196+
)
197+
198+
const ripgrep = yield* Ripgrep.Service
199+
const matches = yield* ripgrep.grep({ cwd: tmp.path, pattern: "tail", limit: 10 })
200+
201+
expect(matches.map((match) => match.text)).toEqual([
202+
`${"x".repeat(1_998)}😀...`,
203+
`${"x".repeat(1_999)}...`,
204+
`${"x".repeat(2_000)}...`,
205+
])
206+
}),
207+
),
208+
)
187209
})
188210

189211
describe("RipgrepBinary.verify", () => {

‎packages/core/test/session-title.test.ts‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -634,6 +634,52 @@ it.effect("bounds regeneration context while preserving the original request and
634634
}),
635635
)
636636

637+
it.effect("keeps the clipped regeneration context well formed", () =>
638+
Effect.gen(function* () {
639+
yield* enableTitleAgent
640+
const sessionID = Session.ID.make("ses_title_regenerate_emoji")
641+
yield* insertSession(sessionID, "Original title")
642+
// The leading x puts both cuts, the original request's head and the recent conversation's tail, inside an emoji.
643+
yield* prompt(sessionID, `x${"😀".repeat(1_500)}`)
644+
yield* prompt(sessionID, `x${"😀".repeat(5_000)}`)
645+
646+
const title = yield* SessionTitle.Service
647+
yield* title.generate(sessionID)
648+
649+
const content = requests[0]?.messages[0]?.content[0]
650+
expect(content?.type).toBe("text")
651+
if (content?.type !== "text") return
652+
expect(content.text.length).toBeLessThanOrEqual(8_000)
653+
expect(content.text.length).toBeGreaterThan(7_900)
654+
expect(content.text.isWellFormed()).toBe(true)
655+
}),
656+
)
657+
658+
for (const [prefix, stored] of [
659+
[95, `${"a".repeat(95)}😀...`],
660+
[96, `${"a".repeat(96)}...`],
661+
[97, `${"a".repeat(97)}...`],
662+
] as const)
663+
it.effect(`clips a long generated title without splitting an emoji after ${prefix} letters`, () =>
664+
Effect.gen(function* () {
665+
yield* enableTitleAgent
666+
titleStream = () =>
667+
Stream.make(
668+
LLMEvent.textDelta({ id: "title", text: `${"a".repeat(prefix)}😀${"b".repeat(20)}` }),
669+
LLMEvent.finish({ reason: { normalized: "stop" } }),
670+
)
671+
const sessionID = Session.ID.make(`ses_title_emoji_${prefix}`)
672+
yield* insertSession(sessionID)
673+
yield* prompt(sessionID, "Name this session")
674+
675+
const title = yield* SessionTitle.Service
676+
yield* title.generate(sessionID)
677+
678+
const store = yield* SessionStore.Service
679+
expect((yield* store.get(sessionID))?.title).toBe(stored)
680+
}),
681+
)
682+
637683
it.effect("preserves the existing title when regeneration fails", () =>
638684
Effect.gen(function* () {
639685
yield* enableTitleAgent

‎packages/session-ui/src/tools/shell-output-machine.test.ts‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,31 @@ describe("transitionShellFollower", () => {
173173
expect(stopped.effects).toEqual([{ type: "read", attempt: 3, cursor: 1 }])
174174
})
175175

176+
test("keeps the retained tail on a whole character when the window cuts an emoji", () => {
177+
// Each page is valid text; only the rolling 65,536-unit window would start on the second half of the emoji.
178+
const first = fresh(false)
179+
180+
if (first.status !== "reading") throw new Error(`expected a read, got ${first.status}`)
181+
182+
const next = transitionShellFollower(first, {
183+
type: "read.succeeded",
184+
attempt: first.attempt,
185+
page: { output: "😀", cursor: 4, size: 65_539 },
186+
}).state
187+
188+
if (next.status !== "reading") throw new Error(`expected a read, got ${next.status}`)
189+
190+
const done = transitionShellFollower(next, {
191+
type: "read.succeeded",
192+
attempt: next.attempt,
193+
page: { output: "x".repeat(65_535), cursor: 65_539, size: 65_539 },
194+
})
195+
196+
expect(done.state).toMatchObject({ status: "complete", output: "x".repeat(65_535) })
197+
expect(done.effects).toContainEqual({ type: "emit", output: "x".repeat(65_535) })
198+
expect(done.effects).toContainEqual({ type: "cache", cursor: 65_539, output: "x".repeat(65_535), complete: true })
199+
})
200+
176201
test("a missing shell is remembered and never read again", () => {
177202
const missing = transitionShellFollower(fresh(true), { type: "read.missing", attempt: 1 })
178203

‎packages/session-ui/src/tools/shell-output-machine.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import { TextBoundary } from "@opencode/util/text-boundary"
2+
13
// Same page size as the TUI shell output viewer; only this much recent output is retained and rendered.
24
export const SHELL_OUTPUT_TAIL_BYTES = 64 * 1024
35

@@ -144,7 +146,7 @@ export function transitionShellFollower(state: ShellFollowerState, event: ShellF
144146
const page = event.page
145147
const advanced = page.cursor > state.cursor
146148
const cursor = Math.max(state.cursor, page.cursor)
147-
const output = (state.output + page.output).slice(-SHELL_OUTPUT_TAIL_BYTES)
149+
const output = TextBoundary.tail(state.output + page.output, SHELL_OUTPUT_TAIL_BYTES)
148150
const emit = { type: "emit", output } as const
149151
const attempt = state.attempt + 1
150152

‎packages/tui/src/component/dialog-shell-output.tsx‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { useTerminalDimensions } from "@opentui/solid"
33
import { isShellNotFoundError, type LocationRef, type ShellInfo } from "@opencode/client"
44
import { createEffect, createMemo, createSignal, onCleanup, Show, untrack } from "solid-js"
55
import stripAnsi from "strip-ansi"
6+
import { TextBoundary } from "@opencode/util/text-boundary"
67
import { useNewDesigns } from "../config"
78
import { useClient } from "../context/client"
89
import { useClipboard } from "../context/clipboard"
@@ -61,7 +62,7 @@ export function DialogShellOutput(props: { shell: ShellInfo; location: LocationR
6162
setOutput((previous) => {
6263
const next = (previous ?? "") + page.data.output
6364
if (next.length > PAGE_BYTES) setOmitted(true)
64-
return next.slice(-PAGE_BYTES)
65+
return TextBoundary.tail(next, PAGE_BYTES)
6566
})
6667
setError("")
6768
return cursor < page.data.size

‎packages/tui/src/routes/session/index.tsx‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@ import { stringWidth } from "../../util/string-width"
120120
import { useArgs } from "../../context/args"
121121
import { withTimestampedFallback } from "@opencode/util/session-title-fallback"
122122
import { SessionTurn } from "@opencode/util/session-turn"
123+
import { TextBoundary } from "@opencode/util/text-boundary"
123124
import { useSessionTabs, type ScrollAnchor } from "../../context/session-tabs"
124125
import { createSingleFlight } from "../../util/single-flight"
125126
import { createDelayedPresence } from "../../util/delayed-presence"
@@ -3069,7 +3070,7 @@ function ShellDisplay(props: {
30693070
const next = stripAnsi(output + response.data.output)
30703071
if (next.length <= SHELL_DISPLAY_LIMIT) return next
30713072
setOutputTruncated(true)
3072-
return next.slice(-SHELL_DISPLAY_LIMIT)
3073+
return TextBoundary.tail(next, SHELL_DISPLAY_LIMIT)
30733074
})
30743075
if (response.data.cursor <= cursor) break
30753076
cursor = response.data.cursor

0 commit comments

Comments
 (0)