Skip to content

Commit 208ea0e

Browse files
authored
fix(ai): recognize LiteLLM context overflow errors (#60692)
1 parent 9321c2c commit 208ea0e

3 files changed

Lines changed: 24 additions & 0 deletions

File tree

‎packages/ai/src/provider-error.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ const patterns = [
4242
// Hugging Face Text Generation Inference, e.g. Together
4343
/`inputs` tokens \+ `max_new_tokens` must be <= \d+/i,
4444
/context window exceeds limit/i,
45+
// LiteLLM's ContextWindowExceededError
46+
/context window exceeded/i,
4547
/exceeded model token limit/i,
4648
/context[_ ]length[_ ]exceeded/i,
4749
/context length is only \d+ tokens/i,

‎packages/ai/test/provider-error.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ describe("provider error classification", () => {
2121
"Prompt exceeds max length",
2222
"The input is too long for the selected model.",
2323
"This text request is too large for the model's configured input limit.",
24+
// LiteLLM's own check, before it calls the model.
25+
"litellm.ContextWindowExceededError: litellm.BadRequestError: litellm._pre_call_checks: Context Window exceeded for given call. No models have context window large enough for this call.",
2426
]
2527

2628
expect(messages.every(isContextOverflow)).toBe(true)

‎packages/core/test/session-runner.test.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5952,6 +5952,26 @@ describe("SessionRunnerLLM", () => {
59525952
})
59535953
}
59545954

5955+
scenario("compacts when the provider cuts an oversized prompt to its window and replies with nothing", function* (s) {
5956+
// Xiaomi MiMo answers an oversized prompt with HTTP 200, finish "length", no output, and a prompt count that fills
5957+
// the window instead of rejecting it.
5958+
s.currentModel = recoveryModel
5959+
yield* s.llm.push(
5960+
TestLLM.complete({ reason: { normalized: "length" }, usage: { inputTokens: 199_994, nonCachedInputTokens: 199_994, outputTokens: 0 } }),
5961+
TestLLM.text("## Objective\n- summary", "summary"),
5962+
TestLLM.text("Done", "done"),
5963+
)
5964+
5965+
yield* s.runPrompt("Large prompt")
5966+
5967+
expect(s.requests).toHaveLength(3)
5968+
expect(userTexts(s.requests[1]).at(-1)).toContain("Summarize only what the user and the assistant said and did")
5969+
expect((yield* s.messages).filter((message) => message.type === "compaction")).toMatchObject([
5970+
{ status: "completed" },
5971+
])
5972+
expect((yield* s.context).at(-1)).toMatchObject(Expected.assistant({ finish: "stop" }, [Expected.text("Done")]))
5973+
})
5974+
59555975
for (const tools of ["hosted", "unfinished"] as const) {
59565976
scenario(`caps output token limit continuations and resets for new input (${tools})`, function* (s) {
59575977
const toolEvents =

0 commit comments

Comments
 (0)