Skip to content

Commit ae8d59f

Browse files
authored
fix(core): fit compaction output limit to the request actually sent (#55613)
1 parent 7ced591 commit ae8d59f

3 files changed

Lines changed: 68 additions & 3 deletions

File tree

‎packages/core/src/session/compaction.ts‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -288,11 +288,17 @@ export const layer = Layer.effect(
288288
// template gets one reminder before it counts as a failure.
289289
const send = (request: LLMRequest) =>
290290
Effect.gen(function* () {
291-
const prompted = LLMRequest.update(request, { messages: [...request.messages, Message.user(prompt)] })
291+
const prompted = fitOutput(
292+
LLMRequest.update(request, { messages: [...request.messages, Message.user(prompt)] }),
293+
context.model.limit,
294+
)
292295
const reply = yield* stream(context, prompted, prepared.options)
293296
if (filled(reply.text)) return { ...reply, recent: split.recent }
294297

295-
const nudged = LLMRequest.update(prompted, { messages: [...prompted.messages, Message.user(NUDGE)] })
298+
const nudged = fitOutput(
299+
LLMRequest.update(prompted, { messages: [...prompted.messages, Message.user(NUDGE)] }),
300+
context.model.limit,
301+
)
296302
const retry = yield* stream(context, nudged, prepared.options)
297303
if (filled(retry.text)) return { ...retry, recent: split.recent }
298304
return yield* Effect.fail<Failure>({
@@ -922,6 +928,25 @@ const hasMeasuredPrompt = (message: SessionMessage.Info, model: SessionContext.L
922928
message.tokens !== undefined &&
923929
message.tokens.input + message.tokens.cache.read + message.tokens.cache.write > 0
924930

931+
/**
932+
* `prepare` sizes the output limit for a request of `budget` tokens, but `deliver` can send a larger one when it
933+
* cannot shrink the conversation further. Lower the limit to the room the request being sent leaves in the window,
934+
* so the input and the requested output never exceed the window by our own estimate.
935+
*
936+
* Like `budget` in `prepare`, the estimate counts as measured, without the margin primary requests add for estimates:
937+
* `deliver` aims at about 90% of the window, so that margin would cut every summary near its target to the minimum.
938+
*/
939+
const fitOutput = (request: LLMRequest, limit: SessionContext.Loaded["model"]["limit"]) => {
940+
const current = request.generation?.maxTokens
941+
if (current === undefined) return request
942+
const maxTokens = Math.min(
943+
current,
944+
SessionModelRequest.outputLimit(limit, "compaction", { measured: estimateRequest(request), estimated: 0 }),
945+
)
946+
if (maxTokens === current) return request
947+
return LLMRequest.update(request, { generation: { ...request.generation, maxTokens } })
948+
}
949+
925950
const estimateRequest = (request: Pick<LLMRequest, "system" | "tools" | "messages">) =>
926951
request.system.reduce((sum, part) => sum + Token.estimate(part.text), 0) +
927952
request.tools.reduce((sum, tool) => sum + estimateTool(tool), 0) +

‎packages/core/src/session/model-request.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ export interface Input {
8585
}
8686

8787
/** The default output limit: the catalog limit, fitted to the room the prompt leaves in the context window. */
88-
const outputLimit = (
88+
export const outputLimit = (
8989
limit: Model.Info["limit"],
9090
kind: "primary" | "compaction",
9191
inputTokens?: Input["inputTokens"],

‎packages/core/test/session-compaction.test.ts‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -498,6 +498,46 @@ it.effect("manual compaction summarizes short context instead of no-op", () =>
498498
}),
499499
)
500500

501+
it.effect("compaction lowers the output limit when it sends a request it could not shrink", () =>
502+
Effect.gen(function* () {
503+
requests = []
504+
const session = yield* insertSession(Session.ID.make("ses_unshrinkable_compaction"))
505+
// One exchange estimated at 250k tokens, past the 200k window, so no older exchange can be dropped.
506+
const messages = [
507+
SessionMessage.User.make({
508+
id: SessionMessage.ID.create(),
509+
type: "user",
510+
text: "x".repeat(1_000_000),
511+
time: { created: DateTime.makeUnsafe(0) },
512+
}),
513+
]
514+
515+
expect(yield* compactManually(session, messages)).toEqual({ status: "completed" })
516+
expect(requests).toHaveLength(1)
517+
expect(requests[0]?.generation).toEqual(GenerationOptions.make({ maxTokens: 1_024 }))
518+
}),
519+
)
520+
521+
it.effect("compaction keeps the output limit for a request that fits its target", () =>
522+
Effect.gen(function* () {
523+
requests = []
524+
const session = yield* insertSession(Session.ID.make("ses_fitting_compaction"))
525+
// About 170k estimated tokens, under the 180k target, leaving 30k of the 200k window.
526+
const messages = [
527+
SessionMessage.User.make({
528+
id: SessionMessage.ID.create(),
529+
type: "user",
530+
text: "x".repeat(680_000),
531+
time: { created: DateTime.makeUnsafe(0) },
532+
}),
533+
]
534+
535+
expect(yield* compactManually(session, messages)).toEqual({ status: "completed" })
536+
expect(requests).toHaveLength(1)
537+
expect(requests[0]?.generation).toEqual(GenerationOptions.make({ maxTokens: 20_000 }))
538+
}),
539+
)
540+
501541
it.effect("compaction hooks can supply the summary instead of the model", () =>
502542
Effect.gen(function* () {
503543
requests = []

0 commit comments

Comments
 (0)