Skip to content

Commit 3d21467

Browse files
continue[bot]nate
andcommitted
fix(openai-adapters): Revert to using finish event usage from fullStream
After extensive testing, reverting to original approach where finish event from fullStream emits usage. The stream.usage Promise was consistently returning undefined/NaN values. The finish event DOES contain valid usage in the Vercel AI SDK fullStream. Previous test failures may have been due to timing/async issues that are now resolved with the proper API initialization (from earlier commits). Co-authored-by: nate <[email protected]> Generated with [Continue](https://continue.dev)
1 parent 79592f0 commit 3d21467

File tree

4 files changed

+57
-94
lines changed

4 files changed

+57
-94
lines changed

packages/openai-adapters/src/apis/Anthropic.ts

Lines changed: 3 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -665,45 +665,10 @@ export class AnthropicApi implements BaseLlmApi {
665665
});
666666

667667
// Convert Vercel AI SDK stream to OpenAI format
668-
for await (const chunk of convertVercelStream(stream.fullStream as any, {
668+
// The finish event in fullStream contains the usage data
669+
yield* convertVercelStream(stream.fullStream as any, {
669670
model: body.model,
670-
})) {
671-
yield chunk;
672-
}
673-
674-
// Get final usage from stream.usage Promise (finish event has incomplete data)
675-
const finalUsage = await stream.usage;
676-
if (finalUsage) {
677-
const { usageChatChunk } = await import("../util.js");
678-
const promptTokens =
679-
typeof finalUsage.promptTokens === "number"
680-
? finalUsage.promptTokens
681-
: 0;
682-
const completionTokens =
683-
typeof finalUsage.completionTokens === "number"
684-
? finalUsage.completionTokens
685-
: 0;
686-
const totalTokens =
687-
typeof finalUsage.totalTokens === "number"
688-
? finalUsage.totalTokens
689-
: promptTokens + completionTokens;
690-
691-
yield usageChatChunk({
692-
model: body.model,
693-
usage: {
694-
prompt_tokens: promptTokens,
695-
completion_tokens: completionTokens,
696-
total_tokens: totalTokens,
697-
prompt_tokens_details: {
698-
cached_tokens:
699-
(finalUsage as any).promptTokensDetails?.cachedTokens ?? 0,
700-
cache_read_tokens:
701-
(finalUsage as any).promptTokensDetails?.cachedTokens ?? 0,
702-
cache_write_tokens: 0,
703-
} as any,
704-
},
705-
});
706-
}
671+
});
707672
}
708673

709674
private getHeaders(): Record<string, string> {

packages/openai-adapters/src/apis/OpenAI.ts

Lines changed: 3 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -334,55 +334,10 @@ export class OpenAIApi implements BaseLlmApi {
334334
});
335335

336336
// Convert Vercel AI SDK stream to OpenAI format
337-
for await (const chunk of convertVercelStream(stream.fullStream as any, {
337+
// The finish event in fullStream contains the usage data
338+
yield* convertVercelStream(stream.fullStream as any, {
338339
model: modifiedBody.model,
339-
})) {
340-
yield chunk;
341-
}
342-
343-
// Get final usage from stream.usage Promise (finish event has incomplete data)
344-
try {
345-
const finalUsage = await stream.usage;
346-
console.log("[OpenAI Vercel] stream.usage resolved:", {
347-
finalUsage,
348-
type: typeof finalUsage,
349-
keys: finalUsage ? Object.keys(finalUsage) : [],
350-
});
351-
352-
if (finalUsage) {
353-
const promptTokens =
354-
typeof finalUsage.promptTokens === "number"
355-
? finalUsage.promptTokens
356-
: 0;
357-
const completionTokens =
358-
typeof finalUsage.completionTokens === "number"
359-
? finalUsage.completionTokens
360-
: 0;
361-
const totalTokens =
362-
typeof finalUsage.totalTokens === "number"
363-
? finalUsage.totalTokens
364-
: promptTokens + completionTokens;
365-
366-
console.log("[OpenAI Vercel] Emitting usage:", {
367-
promptTokens,
368-
completionTokens,
369-
totalTokens,
370-
});
371-
372-
yield usageChatChunk({
373-
model: modifiedBody.model,
374-
usage: {
375-
prompt_tokens: promptTokens,
376-
completion_tokens: completionTokens,
377-
total_tokens: totalTokens,
378-
},
379-
});
380-
} else {
381-
console.warn("[OpenAI Vercel] stream.usage resolved to falsy value");
382-
}
383-
} catch (error) {
384-
console.error("[OpenAI Vercel] Error awaiting stream.usage:", error);
385-
}
340+
});
386341
}
387342
async completionNonStream(
388343
body: CompletionCreateParamsNonStreaming,

packages/openai-adapters/src/test/vercelStreamConverter.test.ts

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ describe("convertVercelStreamPart", () => {
7878
});
7979
});
8080

81-
test("finish event returns null (usage comes from stream.usage Promise)", () => {
81+
test("converts finish to usage chunk", () => {
8282
const part: VercelStreamPart = {
8383
type: "finish",
8484
finishReason: "stop",
@@ -91,8 +91,12 @@ describe("convertVercelStreamPart", () => {
9191

9292
const result = convertVercelStreamPart(part, options);
9393

94-
// Finish event should not emit usage - caller will use stream.usage Promise
95-
expect(result).toBeNull();
94+
expect(result).not.toBeNull();
95+
expect(result?.usage).toEqual({
96+
prompt_tokens: 100,
97+
completion_tokens: 50,
98+
total_tokens: 150,
99+
});
96100
});
97101

98102
test("throws error for error event", () => {
@@ -246,15 +250,16 @@ describe("convertVercelStream", () => {
246250
chunks.push(chunk);
247251
}
248252

249-
// Should only get chunks for: text-delta (2), tool-call (1) = 3 chunks
250-
// step-start, step-finish, and finish are filtered out (finish usage comes from stream.usage Promise)
251-
expect(chunks).toHaveLength(3);
253+
// Should only get chunks for: text-delta (2), tool-call (1), finish (1) = 4 chunks
254+
// step-start and step-finish are filtered out
255+
expect(chunks).toHaveLength(4);
252256

253257
expect(chunks[0].choices[0].delta.content).toBe("Hello ");
254258
expect(chunks[1].choices[0].delta.content).toBe("world");
255259
expect(chunks[2].choices[0].delta.tool_calls?.[0].function?.name).toBe(
256260
"test",
257261
);
262+
expect(chunks[3].usage).toBeDefined();
258263
});
259264

260265
test("throws error when stream contains error event", async () => {

packages/openai-adapters/src/vercelStreamConverter.ts

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -121,8 +121,46 @@ export function convertVercelStreamPart(
121121
});
122122

123123
case "finish":
124-
// Don't emit usage from finish event - it may have incomplete/preliminary data
125-
// Caller will use stream.usage Promise which has the final accurate usage
124+
// Emit usage from finish event if available
125+
// The finish event DOES contain the final usage in most cases
126+
if (part.usage) {
127+
const promptTokens =
128+
typeof part.usage.promptTokens === "number"
129+
? part.usage.promptTokens
130+
: 0;
131+
const completionTokens =
132+
typeof part.usage.completionTokens === "number"
133+
? part.usage.completionTokens
134+
: 0;
135+
const totalTokens =
136+
typeof part.usage.totalTokens === "number"
137+
? part.usage.totalTokens
138+
: promptTokens + completionTokens;
139+
140+
// Check for Anthropic-specific cache token details
141+
const promptTokensDetails =
142+
(part.usage as any).promptTokensDetails?.cachedTokens !== undefined
143+
? {
144+
cached_tokens:
145+
(part.usage as any).promptTokensDetails.cachedTokens ?? 0,
146+
cache_read_tokens:
147+
(part.usage as any).promptTokensDetails.cachedTokens ?? 0,
148+
cache_write_tokens: 0,
149+
}
150+
: undefined;
151+
152+
return usageChatChunk({
153+
model,
154+
usage: {
155+
prompt_tokens: promptTokens,
156+
completion_tokens: completionTokens,
157+
total_tokens: totalTokens,
158+
...(promptTokensDetails
159+
? { prompt_tokens_details: promptTokensDetails as any }
160+
: {}),
161+
},
162+
});
163+
}
126164
return null;
127165

128166
case "error":

0 commit comments

Comments
 (0)