diff --git a/.changeset/openai-pdf-document-input.md b/.changeset/openai-pdf-document-input.md new file mode 100644 index 000000000..7fe17da1c --- /dev/null +++ b/.changeset/openai-pdf-document-input.md @@ -0,0 +1,26 @@ +--- +'@tanstack/openai-base': minor +'@tanstack/ai-openai': minor +--- + +feat(ai-openai): support PDF `document` content parts in the Responses adapter. + +`openaiText`'s Responses adapter now accepts PDF `document` content parts, for any model whose `model-meta` entry declares the `document` input modality. Base64 data sources are sent as `input_file` with a `file_data` data URL and a `filename` (from `metadata.filename`, defaulting to `document.pdf`). URL sources are sent as `input_file` with `file_url`. + +```ts +const adapter = openaiText('gpt-5.5') + +const message = { + role: 'user', + content: [ + { type: 'text', content: 'Summarize this document' }, + { + type: 'document', + source: { type: 'data', value: pdfBase64, mimeType: 'application/pdf' }, + metadata: { filename: 'report.pdf' }, + }, + ], +} +``` + +Non-PDF MIME types are rejected before the request is sent — including pre-wrapped `data:` URLs whose media type disagrees with `mimeType` — so callers get an actionable message instead of an opaque provider `400`. Inline base64 payloads are also checked for the `%PDF` magic bytes, so non-PDF data mislabeled (or sent without) a PDF `mimeType` is rejected locally. `OpenAIDocumentMetadata` gains `filename` and `detail`. The Chat Completions adapter throws a document-specific error pointing here, since documents are Responses-only. diff --git a/docs/advanced/multimodal-content.md b/docs/advanced/multimodal-content.md index 0bea1a795..baa86b534 100644 --- a/docs/advanced/multimodal-content.md +++ b/docs/advanced/multimodal-content.md @@ -94,11 +94,12 @@ const response = await chat({ ### OpenAI -OpenAI supports images and audio in their vision and audio models: +OpenAI supports images, audio, and PDF documents in their vision, audio, and +document-capable models: ```typescript import { openaiText } from '@tanstack/ai-openai' -import { imageBase64 } from './data' +import { imageBase64, pdfBase64 } from './data' const adapter = openaiText('gpt-5.5') @@ -116,10 +117,30 @@ const message = { } ``` +```typescript +import { pdfBase64 } from './data' + +// PDF document via base64 data (the API requires a filename alongside +// inline data; defaults to "document.pdf" when omitted) +const documentMessage = { + role: 'user', + content: [ + { type: 'text', content: 'Summarize this document' }, + { + type: 'document', + source: { type: 'data', value: pdfBase64, mimeType: 'application/pdf' }, + metadata: { filename: 'report.pdf' } + } + ] +} +``` + **Supported modalities by model:** -- `gpt-5.2`, `gpt-5-mini`: text, image +- `gpt-5.5`, `gpt-5.2`, `gpt-5-mini` (among others): text, image, PDF document - `gpt-4o-audio`: text, audio +Check each model's `supports.input` in `@tanstack/ai-openai`'s `model-meta.ts` for the authoritative per-model list. + ### Anthropic Anthropic's Claude models support images and PDF documents: diff --git a/docs/config.json b/docs/config.json index 592302cbc..6cb93a46f 100644 --- a/docs/config.json +++ b/docs/config.json @@ -477,7 +477,8 @@ { "label": "Multimodal Content", "to": "advanced/multimodal-content", - "addedAt": "2026-04-15" + "addedAt": "2026-04-15", + "updatedAt": "2026-07-21" }, { "label": "Per-Model Type Safety", diff --git a/packages/ai-openai/src/message-types.ts b/packages/ai-openai/src/message-types.ts index 895c6e397..19eea009f 100644 --- a/packages/ai-openai/src/message-types.ts +++ b/packages/ai-openai/src/message-types.ts @@ -44,9 +44,32 @@ export interface OpenAIVideoMetadata {} /** * Metadata for OpenAI document content parts. - * Note: Direct document support may vary; PDFs often need to be converted to images. + * + * Documents are supported by the Responses adapter only, which sends them + * as `input_file`; the Chat Completions adapter rejects document parts. + * This adapter currently supports only `application/pdf` documents. Inline + * (base64) data is verified locally — both the declared/embedded MIME type + * and the `%PDF` content header, so mislabeled non-PDF data is rejected + * before the request. URL-sourced documents are sent to OpenAI unvalidated, + * so a non-PDF URL fails server-side instead. + * + * @see https://developers.openai.com/api/docs/guides/pdf-files */ -export interface OpenAIDocumentMetadata {} +export interface OpenAIDocumentMetadata { + /** + * Filename sent alongside inline (base64) PDF data. + * @default 'document.pdf' + */ + filename?: string + /** + * Rendering quality for the file's page images. Omitted by default so the + * API applies its own default ('auto'), which resolves to 'low' or 'high' + * depending on the model. + * + * @see https://developers.openai.com/api/docs/guides/pdf-files + */ + detail?: 'auto' | 'low' | 'high' +} /** * Metadata for OpenAI text content parts. diff --git a/packages/ai-openai/src/model-meta.ts b/packages/ai-openai/src/model-meta.ts index b28d0c604..5ed050144 100644 --- a/packages/ai-openai/src/model-meta.ts +++ b/packages/ai-openai/src/model-meta.ts @@ -11,7 +11,7 @@ import type { interface ModelMeta { name: string supports: { - input: Array<'text' | 'image' | 'audio' | 'video'> + input: Array<'text' | 'image' | 'audio' | 'video' | 'document'> output: Array<'text' | 'image' | 'audio' | 'video'> endpoints: Array< | 'chat' @@ -74,7 +74,7 @@ const GPT5_2 = { max_output_tokens: 128_000, knowledge_cutoff: '2025-08-31', supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'chat-completions'], features: [ @@ -119,7 +119,7 @@ const GPT5_2_PRO = { max_output_tokens: 128_000, knowledge_cutoff: '2025-08-31', supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'chat-completions'], features: ['streaming', 'function_calling'], @@ -158,7 +158,7 @@ const GPT5_2_CHAT = { max_output_tokens: 16_384, knowledge_cutoff: '2025-08-31', supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'chat-completions'], features: ['streaming', 'function_calling', 'structured_outputs'], @@ -194,7 +194,7 @@ const GPT5_1 = { max_output_tokens: 128_000, knowledge_cutoff: '2024-09-30', supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text', 'image'], endpoints: ['chat', 'chat-completions'], features: [ @@ -240,7 +240,7 @@ const GPT5_1_CODEX = { max_output_tokens: 128_000, knowledge_cutoff: '2024-09-30', supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text', 'image'], endpoints: ['chat'], features: ['streaming', 'function_calling', 'structured_outputs'], @@ -277,7 +277,7 @@ const GPT5 = { max_output_tokens: 128_000, knowledge_cutoff: '2024-09-30', supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'chat-completions', 'batch'], features: [ @@ -323,7 +323,7 @@ const GPT5_MINI = { max_output_tokens: 128_000, knowledge_cutoff: '2024-05-31', supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'chat-completions', 'batch'], features: ['streaming', 'structured_outputs', 'function_calling'], @@ -373,7 +373,7 @@ const GPT5_NANO = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'chat-completions', 'batch'], features: ['streaming', 'structured_outputs', 'function_calling'], @@ -413,7 +413,7 @@ const GPT5_PRO = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'batch'], features: ['streaming', 'structured_outputs', 'function_calling'], @@ -454,7 +454,7 @@ const GPT5_CODEX = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text', 'image'], endpoints: ['chat'], features: ['streaming', 'structured_outputs', 'function_calling'], @@ -686,7 +686,7 @@ const O3_PRO = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'batch'], features: ['function_calling', 'structured_outputs'], @@ -839,7 +839,7 @@ const O3 = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'batch', 'chat-completions'], features: ['function_calling', 'structured_outputs', 'streaming'], @@ -880,7 +880,7 @@ const O4_MINI = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'batch', 'chat-completions', 'fine-tuning'], features: [ @@ -925,7 +925,7 @@ const GPT4_1 = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: [ 'chat', @@ -978,7 +978,7 @@ const GPT4_1_MINI = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: [ 'chat', @@ -1029,7 +1029,7 @@ const GPT4_1_NANO = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: [ 'chat', @@ -1080,7 +1080,7 @@ const O1_PRO = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'batch'], features: ['function_calling', 'structured_outputs'], @@ -1283,7 +1283,7 @@ const O1 = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'batch', 'chat-completions', 'assistants'], features: ['function_calling', 'structured_outputs', 'streaming'], @@ -1333,7 +1333,7 @@ const GPT_4O = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: [ 'chat', @@ -1410,7 +1410,7 @@ const GPT_4O_MINI = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: [ 'chat', @@ -2048,7 +2048,7 @@ const GPT_5_5 = { context_window: 1_050_000, max_output_tokens: 128_000, supports: { - input: ['image', 'text'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'chat-completions'], features: [ @@ -2093,7 +2093,7 @@ const GPT_5_5_PRO = { context_window: 1_050_000, max_output_tokens: 128_000, supports: { - input: ['image', 'text'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'chat-completions'], features: [ diff --git a/packages/ai-openai/tests/model-meta.test.ts b/packages/ai-openai/tests/model-meta.test.ts index 77203fd0f..c6f9ec09d 100644 --- a/packages/ai-openai/tests/model-meta.test.ts +++ b/packages/ai-openai/tests/model-meta.test.ts @@ -840,223 +840,223 @@ type MessageWithContent = { role: 'user'; content: Array } describe('OpenAI Model Input Modality Type Assertions', () => { // ===== Models with text + image input ===== - describe('gpt-5.1 (text + image)', () => { + describe('gpt-5.1 (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['gpt-5.1'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('gpt-5.1-codex (text + image)', () => { + describe('gpt-5.1-codex (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['gpt-5.1-codex'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('gpt-5 (text + image)', () => { + describe('gpt-5 (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['gpt-5'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('gpt-5-mini (text + image)', () => { + describe('gpt-5-mini (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['gpt-5-mini'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('gpt-5-nano (text + image)', () => { + describe('gpt-5-nano (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['gpt-5-nano'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('gpt-5-pro (text + image)', () => { + describe('gpt-5-pro (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['gpt-5-pro'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('gpt-5-codex (text + image)', () => { + describe('gpt-5-codex (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['gpt-5-codex'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('gpt-4.1 (text + image)', () => { + describe('gpt-4.1 (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['gpt-4.1'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('gpt-4.1-mini (text + image)', () => { + describe('gpt-4.1-mini (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['gpt-4.1-mini'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('gpt-4.1-nano (text + image)', () => { + describe('gpt-4.1-nano (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['gpt-4.1-nano'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) @@ -1104,47 +1104,47 @@ describe('OpenAI Model Input Modality Type Assertions', () => { }) }) - describe('o3 (text + image)', () => { + describe('o3 (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['o3'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('o3-pro (text + image)', () => { + describe('o3-pro (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['o3-pro'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) @@ -1192,69 +1192,69 @@ describe('OpenAI Model Input Modality Type Assertions', () => { }) }) - describe('o4-mini (text + image)', () => { + describe('o4-mini (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['o4-mini'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('o1 (text + image)', () => { + describe('o1 (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['o1'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('o1-pro (text + image)', () => { + describe('o1-pro (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['o1-pro'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) diff --git a/packages/openai-base/src/adapters/chat-completions-text.ts b/packages/openai-base/src/adapters/chat-completions-text.ts index 72ea19ca7..abf1d42e4 100644 --- a/packages/openai-base/src/adapters/chat-completions-text.ts +++ b/packages/openai-base/src/adapters/chat-completions-text.ts @@ -1332,6 +1332,17 @@ export abstract class OpenAIBaseChatCompletionsTextAdapter< } } + if (part.type === 'document') { + // Documents (PDF) are implemented on the Responses adapter, which maps + // them to `input_file`. Model modality arrays are not endpoint-scoped, + // so a document part can type-check for a model this adapter serves — + // point callers at the supported path instead of a generic error. + throw new Error( + `${this.name} does not support document parts on the Chat Completions ` + + `API; use the Responses adapter, which sends them as input_file.`, + ) + } + // Unsupported content type — subclasses can override to handle more types return null } diff --git a/packages/openai-base/src/adapters/responses-text.ts b/packages/openai-base/src/adapters/responses-text.ts index d48018209..06f9f3868 100644 --- a/packages/openai-base/src/adapters/responses-text.ts +++ b/packages/openai-base/src/adapters/responses-text.ts @@ -31,6 +31,10 @@ import type { TextOptions, } from '@tanstack/ai' +// Base64 encoding of the '%PDF' file header — every PDF payload starts with +// these bytes, so inline document data must begin with this prefix. +const PDF_BASE64_MAGIC = 'JVBERi' + /** * Shared implementation of the OpenAI Responses API. Holds the stream-event * accumulator + AG-UI lifecycle and calls the OpenAI SDK directly. Subclasses @@ -1766,7 +1770,7 @@ export abstract class OpenAIBaseResponsesTextAdapter< throw new Error( `User message for ${this.name} has no content parts. ` + `Empty user messages would produce a paid request with no input; ` + - `provide at least one text/image/audio part or omit the message.`, + `provide at least one text/image/audio/document part or omit the message.`, ) } @@ -1782,7 +1786,7 @@ export abstract class OpenAIBaseResponsesTextAdapter< /** * Converts a ContentPart to Responses API input content item. - * Handles text, image, and audio content parts. + * Handles text, image, audio, and document (PDF) content parts. * Override this in subclasses for additional content types or provider-specific metadata. */ protected convertContentPartToInput(part: ContentPart): ResponseInputContent { @@ -1826,7 +1830,7 @@ export abstract class OpenAIBaseResponsesTextAdapter< } } // Wrap raw base64 in a data URL — `input_file` rejects bare base64 - // payloads (matches the image branch above which already does this). + // payloads (matches the image branch above). // Default the MIME if missing so we never interpolate `undefined`. const audioValue = part.source.value const audioMime = part.source.mimeType || 'application/octet-stream' @@ -1839,12 +1843,85 @@ export abstract class OpenAIBaseResponsesTextAdapter< } } + case 'document': { + const documentMetadata = part.metadata as + | { filename?: string; detail?: 'auto' | 'low' | 'high' } + | undefined + // Spread `detail` only when provided so the API applies its own + // default ('auto'). The Responses API accepts 'auto' | 'low' | 'high', + // but the pinned OpenAI SDK's `ResponseInputFile.detail` type still + // lists only 'low' | 'high' — cast so 'auto' (a valid API value) can + // pass through without a type error. + const documentDetail = + documentMetadata?.detail !== undefined + ? { detail: documentMetadata.detail as 'low' | 'high' } + : {} + if (part.source.type === 'url') { + // The Responses API fetches the PDF itself; filename and MIME + // type are inferred from the response. + return { + type: 'input_file', + file_url: part.source.value, + ...documentDetail, + } + } + // This adapter supports only PDF documents; anything else is + // rejected. MIME types are case-insensitive and can carry + // ;parameters (RFC 2045), so the type is normalized before comparing. + const documentValue = part.source.value + const documentMime = ((part.source.mimeType || 'application/pdf').split(';')[0] ?? '') + .trim() + .toLowerCase() + if (documentMime !== 'application/pdf') { + throw new Error( + `${this.name} document parts only support application/pdf ` + + `(received ${documentMime})`, + ) + } + // A pre-wrapped data URL carries its own media type — validate it too. + if ( + documentValue.startsWith('data:') && + !/^data:application\/pdf[;,]/i.test(documentValue) + ) { + throw new Error( + `${this.name} document parts only support application/pdf ` + + `(received data URL with non-PDF media type)`, + ) + } + // Sniff the payload so a non-PDF labeled (or unlabeled) as PDF is + // caught locally instead of by an opaque provider 400. Only base64 + // payloads are checked; a rare `data:` URL without `;base64` is left + // to the server. + const documentBase64 = documentValue.startsWith('data:') + ? /;base64,/i.test(documentValue) + ? documentValue.slice(documentValue.indexOf(',') + 1) + : '' + : documentValue + if (documentBase64 && !documentBase64.startsWith(PDF_BASE64_MAGIC)) { + throw new Error( + `${this.name} document parts only support application/pdf ` + + `(inline data does not start with the %PDF header)`, + ) + } + // Wrap raw base64 in a data URL — `input_file` rejects bare base64 + // payloads (matches the image and audio branches above). + const documentFileData = documentValue.startsWith('data:') + ? documentValue + : `data:${documentMime};base64,${documentValue}` + return { + type: 'input_file', + // The Responses API requires a filename alongside PDF `file_data`. + filename: documentMetadata?.filename || 'document.pdf', + file_data: documentFileData, + ...documentDetail, + } + } + case 'video': - case 'document': default: - // OpenAI Responses API doesn't accept native video/document parts on - // this path — surface as explicit unsupported error so callers see - // the same message regardless of which content type leaked through. + // OpenAI Responses API doesn't accept native video parts on this + // path — surface as explicit unsupported error so callers see the + // same message regardless of which content type leaked through. throw new Error(`Unsupported content part type: ${part.type}`) } } diff --git a/packages/openai-base/tests/chat-completions-text.test.ts b/packages/openai-base/tests/chat-completions-text.test.ts index e92e34f1d..3b58d7d77 100644 --- a/packages/openai-base/tests/chat-completions-text.test.ts +++ b/packages/openai-base/tests/chat-completions-text.test.ts @@ -620,6 +620,44 @@ describe('OpenAIBaseChatCompletionsTextAdapter', () => { }) describe('error handling', () => { + it('points document parts at the Responses adapter', async () => { + mockCreate = vi.fn() + const adapter = new TestChatCompletionsAdapter(testConfig, 'test-model') + const chunks: Array = [] + + for await (const chunk of adapter.chatStream({ + logger: testLogger, + model: 'test-model', + messages: [ + { + role: 'user', + content: [ + { type: 'text', content: 'summarize this' }, + { + type: 'document', + source: { + type: 'data', + value: 'JVBERi0xLjQK', + mimeType: 'application/pdf', + }, + }, + ], + }, + ], + })) { + chunks.push(chunk) + } + + const runErrorChunk = chunks.find((c) => c.type === 'RUN_ERROR') + expect(runErrorChunk).toBeDefined() + if (runErrorChunk?.type === 'RUN_ERROR') { + expect(runErrorChunk.message).toMatch(/Responses adapter/) + expect(runErrorChunk.message).toMatch(/document/) + } + // No request should have been attempted. + expect(mockCreate).not.toHaveBeenCalled() + }) + it('emits RUN_ERROR on stream error', async () => { const streamChunks = [ { diff --git a/packages/openai-base/tests/responses-text.test.ts b/packages/openai-base/tests/responses-text.test.ts index 884954267..4ffc0db6f 100644 --- a/packages/openai-base/tests/responses-text.test.ts +++ b/packages/openai-base/tests/responses-text.test.ts @@ -2239,6 +2239,421 @@ describe('OpenAIBaseResponsesTextAdapter', () => { }) }) + describe('document content parts (PDF input)', () => { + // A minimal one-page PDF. + const TINY_PDF_BASE64 = Buffer.from( + '%PDF-1.4\n1 0 obj<>endobj\n' + + '2 0 obj<>endobj\n' + + '3 0 obj<>endobj\n' + + 'trailer<>\n%%EOF', + ).toString('base64') + + // A payload that is not a PDF (no '%PDF' header). + const NOT_PDF_BASE64 = Buffer.from('hello, not a pdf').toString('base64') + + const minimalStreamChunks = [ + { + type: 'response.created', + response: { id: 'resp-doc-1', model: 'test-model', status: 'in_progress' }, + }, + { + type: 'response.completed', + response: { + id: 'resp-doc-1', + model: 'test-model', + status: 'completed', + output: [], + usage: { input_tokens: 10, output_tokens: 1, total_tokens: 11 }, + }, + }, + ] + + async function runChat(content: Array): Promise> { + setupMockResponsesClient(minimalStreamChunks) + const adapter = new TestResponsesAdapter(testConfig, 'test-model') + const chunks: Array = [] + for await (const chunk of adapter.chatStream({ + logger: testLogger, + model: 'test-model', + messages: [{ role: 'user', content }], + })) { + chunks.push(chunk) + } + return chunks + } + + it('converts a base64 data source to input_file with a default filename', async () => { + await runChat([ + { + type: 'document', + source: { + type: 'data', + value: TINY_PDF_BASE64, + mimeType: 'application/pdf', + }, + }, + ]) + + const [payload] = mockResponsesCreate.mock.calls[0]! + expect(payload.input[0].content).toEqual([ + { + type: 'input_file', + filename: 'document.pdf', + file_data: `data:application/pdf;base64,${TINY_PDF_BASE64}`, + }, + ]) + }) + + it('uses metadata.filename when provided', async () => { + await runChat([ + { + type: 'document', + source: { + type: 'data', + value: TINY_PDF_BASE64, + mimeType: 'application/pdf', + }, + metadata: { filename: 'report.pdf' }, + }, + ]) + + const [payload] = mockResponsesCreate.mock.calls[0]! + expect(payload.input[0].content[0].filename).toBe('report.pdf') + }) + + it('passes an existing data URL through without double-wrapping', async () => { + const dataUrl = `data:application/pdf;base64,${TINY_PDF_BASE64}` + await runChat([ + { + type: 'document', + source: { type: 'data', value: dataUrl, mimeType: 'application/pdf' }, + }, + ]) + + const [payload] = mockResponsesCreate.mock.calls[0]! + expect(payload.input[0].content[0].file_data).toBe(dataUrl) + }) + + it('converts a URL source to input_file with file_url only', async () => { + await runChat([ + { + type: 'document', + source: { type: 'url', value: 'https://example.com/doc.pdf' }, + }, + ]) + + const [payload] = mockResponsesCreate.mock.calls[0]! + const part = payload.input[0].content[0] + expect(part).toEqual({ + type: 'input_file', + file_url: 'https://example.com/doc.pdf', + }) + expect(part).not.toHaveProperty('file_data') + expect(part).not.toHaveProperty('filename') + }) + + it('emits RUN_ERROR for a non-PDF document MIME type', async () => { + const chunks = await runChat([ + { + type: 'document', + source: { + type: 'data', + value: TINY_PDF_BASE64, + mimeType: 'application/msword', + }, + }, + ]) + + const errorChunk = chunks.find((c) => c.type === 'RUN_ERROR') + expect(errorChunk).toBeDefined() + if (errorChunk?.type === 'RUN_ERROR') { + expect(errorChunk.message).toMatch(/only support application\/pdf/) + expect(errorChunk.message).toMatch(/application\/msword/) + } + }) + + it('emits RUN_ERROR for a data URL with a non-PDF media type', async () => { + const chunks = await runChat([ + { + type: 'document', + source: { + type: 'data', + value: `data:image/png;base64,${TINY_PDF_BASE64}`, + mimeType: 'application/pdf', + }, + }, + ]) + + const errorChunk = chunks.find((c) => c.type === 'RUN_ERROR') + expect(errorChunk).toBeDefined() + if (errorChunk?.type === 'RUN_ERROR') { + expect(errorChunk.message).toMatch(/only support application\/pdf/) + expect(errorChunk.message).toMatch(/non-PDF media type/) + } + }) + + it('rejects a data URL whose media type merely extends application/pdf', async () => { + const chunks = await runChat([ + { + type: 'document', + source: { + type: 'data', + value: `data:application/pdf+xml;base64,${TINY_PDF_BASE64}`, + mimeType: 'application/pdf', + }, + }, + ]) + + const errorChunk = chunks.find((c) => c.type === 'RUN_ERROR') + expect(errorChunk).toBeDefined() + if (errorChunk?.type === 'RUN_ERROR') { + expect(errorChunk.message).toMatch(/non-PDF media type/) + } + }) + + it('accepts case-insensitive PDF media types (RFC 2045)', async () => { + const dataUrl = `data:APPLICATION/PDF;base64,${TINY_PDF_BASE64}` + await runChat([ + { + type: 'document', + source: { type: 'data', value: dataUrl, mimeType: 'Application/PDF' }, + }, + ]) + + const [payload] = mockResponsesCreate.mock.calls[0]! + expect(payload.input[0].content[0].file_data).toBe(dataUrl) + }) + + it('passes metadata.detail through on both source shapes', async () => { + await runChat([ + { + type: 'document', + source: { + type: 'data', + value: TINY_PDF_BASE64, + mimeType: 'application/pdf', + }, + metadata: { detail: 'high' }, + }, + { + type: 'document', + source: { type: 'url', value: 'https://example.com/doc.pdf' }, + metadata: { detail: 'low' }, + }, + ]) + + const [payload] = mockResponsesCreate.mock.calls[0]! + expect(payload.input[0].content[0].detail).toBe('high') + expect(payload.input[0].content[1]).toEqual({ + type: 'input_file', + file_url: 'https://example.com/doc.pdf', + detail: 'low', + }) + }) + + it("passes detail: 'auto' through to the payload", async () => { + await runChat([ + { + type: 'document', + source: { + type: 'data', + value: TINY_PDF_BASE64, + mimeType: 'application/pdf', + }, + metadata: { detail: 'auto' }, + }, + ]) + + const [payload] = mockResponsesCreate.mock.calls[0]! + expect(payload.input[0].content[0].detail).toBe('auto') + }) + + it('omits detail when metadata does not provide it', async () => { + await runChat([ + { + type: 'document', + source: { + type: 'data', + value: TINY_PDF_BASE64, + mimeType: 'application/pdf', + }, + }, + ]) + + const [payload] = mockResponsesCreate.mock.calls[0]! + expect(payload.input[0].content[0]).not.toHaveProperty('detail') + }) + + it('still rejects video content parts', async () => { + const chunks = await runChat([ + { + type: 'video', + source: { type: 'url', value: 'https://example.com/clip.mp4' }, + }, + ]) + + const errorChunk = chunks.find((c) => c.type === 'RUN_ERROR') + expect(errorChunk).toBeDefined() + if (errorChunk?.type === 'RUN_ERROR') { + expect(errorChunk.message).toMatch( + /Unsupported content part type: video/, + ) + } + }) + + it('keeps text and document parts in order within one message', async () => { + await runChat([ + { type: 'text', content: 'summarize this' }, + { + type: 'document', + source: { + type: 'data', + value: TINY_PDF_BASE64, + mimeType: 'application/pdf', + }, + }, + ]) + + const [payload] = mockResponsesCreate.mock.calls[0]! + expect(payload.input[0].content).toEqual([ + { type: 'input_text', text: 'summarize this' }, + { + type: 'input_file', + filename: 'document.pdf', + file_data: `data:application/pdf;base64,${TINY_PDF_BASE64}`, + }, + ]) + }) + + it('converts a document part inside a tool result', async () => { + setupMockResponsesClient(minimalStreamChunks) + const adapter = new TestResponsesAdapter(testConfig, 'test-model') + const chunks: Array = [] + for await (const chunk of adapter.chatStream({ + logger: testLogger, + model: 'test-model', + messages: [ + { role: 'user', content: 'fetch the doc' }, + { + role: 'assistant', + content: '', + toolCalls: [ + { + id: 'call_doc', + type: 'function', + function: { name: 'fetch_doc', arguments: '{}' }, + }, + ], + }, + { + role: 'tool', + toolCallId: 'call_doc', + content: [ + { type: 'text', content: 'here it is' }, + { + type: 'document', + source: { + type: 'data', + value: TINY_PDF_BASE64, + mimeType: 'application/pdf', + }, + }, + ], + }, + ], + })) { + chunks.push(chunk) + } + + const [payload] = mockResponsesCreate.mock.calls[0]! + const out = payload.input.find( + (i: any) => i.type === 'function_call_output', + ) + expect(out.output).toEqual([ + { type: 'input_text', text: 'here it is' }, + { + type: 'input_file', + filename: 'document.pdf', + file_data: `data:application/pdf;base64,${TINY_PDF_BASE64}`, + }, + ]) + }) + + it('emits RUN_ERROR for raw base64 without a mimeType that is not a PDF', async () => { + const chunks = await runChat([ + { + type: 'document', + source: { type: 'data', value: NOT_PDF_BASE64 }, + }, + ]) + + const errorChunk = chunks.find((c) => c.type === 'RUN_ERROR') + expect(errorChunk).toBeDefined() + if (errorChunk?.type === 'RUN_ERROR') { + expect(errorChunk.message).toMatch(/only support application\/pdf/) + expect(errorChunk.message).toMatch(/%PDF/) + } + }) + + it('accepts raw base64 PDF data without a mimeType', async () => { + await runChat([ + { + type: 'document', + source: { type: 'data', value: TINY_PDF_BASE64 }, + }, + ]) + + const [payload] = mockResponsesCreate.mock.calls[0]! + expect(payload.input[0].content).toEqual([ + { + type: 'input_file', + filename: 'document.pdf', + file_data: `data:application/pdf;base64,${TINY_PDF_BASE64}`, + }, + ]) + }) + + it('emits RUN_ERROR for non-PDF data mislabeled as application/pdf', async () => { + const chunks = await runChat([ + { + type: 'document', + source: { + type: 'data', + value: NOT_PDF_BASE64, + mimeType: 'application/pdf', + }, + }, + ]) + + const errorChunk = chunks.find((c) => c.type === 'RUN_ERROR') + expect(errorChunk).toBeDefined() + if (errorChunk?.type === 'RUN_ERROR') { + expect(errorChunk.message).toMatch(/only support application\/pdf/) + expect(errorChunk.message).toMatch(/%PDF/) + } + }) + + it('emits RUN_ERROR for a PDF data URL whose payload is not a PDF', async () => { + const chunks = await runChat([ + { + type: 'document', + source: { + type: 'data', + value: `data:application/pdf;base64,${NOT_PDF_BASE64}`, + mimeType: 'application/pdf', + }, + }, + ]) + + const errorChunk = chunks.find((c) => c.type === 'RUN_ERROR') + expect(errorChunk).toBeDefined() + if (errorChunk?.type === 'RUN_ERROR') { + expect(errorChunk.message).toMatch(/only support application\/pdf/) + expect(errorChunk.message).toMatch(/%PDF/) + } + }) + }) + describe('subclassing', () => { it('allows subclassing with custom name', () => { class MyProviderAdapter extends OpenAIBaseResponsesTextAdapter { diff --git a/testing/e2e/README.md b/testing/e2e/README.md index 8c005b672..60a57bcae 100644 --- a/testing/e2e/README.md +++ b/testing/e2e/README.md @@ -28,6 +28,7 @@ Each test iterates over supported providers using `providersFor('feature')`: | agentic-structured | 7 | `tests/agentic-structured.spec.ts` | | reasoning | 3 | `tests/reasoning.spec.ts` | | multimodal-image | 5 | `tests/multimodal-image.spec.ts` | +| multimodal-document | 1 | `tests/multimodal-document.spec.ts` | | multimodal-structured | 5 | `tests/multimodal-structured.spec.ts` | | summarize | 6 | `tests/summarize.spec.ts` | | summarize-stream | 6 | `tests/summarize-stream.spec.ts` | diff --git a/testing/e2e/fixtures/multimodal-document/basic.json b/testing/e2e/fixtures/multimodal-document/basic.json new file mode 100644 index 000000000..13d5968dc --- /dev/null +++ b/testing/e2e/fixtures/multimodal-document/basic.json @@ -0,0 +1,12 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "[mmdocument] summarize this pdf" + }, + "response": { + "content": "The PDF contains a single blank page with no text content." + } + } + ] +} diff --git a/testing/e2e/src/components/ChatUI.tsx b/testing/e2e/src/components/ChatUI.tsx index 6716d11eb..e4e4a7c0d 100644 --- a/testing/e2e/src/components/ChatUI.tsx +++ b/testing/e2e/src/components/ChatUI.tsx @@ -1,52 +1,56 @@ -import { useEffect, useRef, useState } from 'react' -import ReactMarkdown from 'react-markdown' -import rehypeRaw from 'rehype-raw' -import rehypeSanitize from 'rehype-sanitize' -import remarkGfm from 'remark-gfm' -import type { UIMessage } from '@tanstack/ai-react' +import { useEffect, useRef, useState } from "react"; +import ReactMarkdown from "react-markdown"; +import rehypeRaw from "rehype-raw"; +import rehypeSanitize from "rehype-sanitize"; +import remarkGfm from "remark-gfm"; +import type { UIMessage } from "@tanstack/ai-react"; import type { AnyClientTool, BoundInterrupts, QueuedMessage, -} from '@tanstack/ai-client' -import { ToolCallDisplay } from '@/components/ToolCallDisplay' -import { ApprovalPrompt } from '@/components/ApprovalPrompt' +} from "@tanstack/ai-client"; +import { ToolCallDisplay } from "@/components/ToolCallDisplay"; +import { ApprovalPrompt } from "@/components/ApprovalPrompt"; interface ChatUIProps< TTools extends ReadonlyArray = ReadonlyArray, > { - messages: Array - isLoading: boolean - onSendMessage: (text: string) => void - onSendMessageWithImage?: (text: string, file: File) => void + messages: Array; + isLoading: boolean; + onSendMessage: (text: string) => void; + onSendMessageWithImage?: (text: string, file: File) => void; + /** Sends the prompt plus an attached document content part. */ + onSendMessageWithDocument?: (text: string, file: File) => void; /** * Bound AG-UI interrupts from `useChat({ tools })` — * `BoundInterrupts` (library type, not a harness DTO). */ - interrupts?: BoundInterrupts + interrupts?: BoundInterrupts; /** @deprecated Prefer `interrupts` + resolveInterrupt. */ addToolApprovalResponse?: (response: { - id: string - approved: boolean - }) => Promise - showImageInput?: boolean - onStop?: () => void + id: string; + approved: boolean; + }) => Promise; + showImageInput?: boolean; + /** Renders the PDF file input (multimodal-document feature only). */ + showDocumentInput?: boolean; + onStop?: () => void; /** When the streaming structured-output CUSTOM event lands, the page * exposes the parsed object here so e2e tests can assert that the event * reached the client (not just that the JSON text was rendered). */ - structuredObject?: unknown + structuredObject?: unknown; /** Number of TEXT_MESSAGE_CONTENT chunks observed. Used by streaming e2e * tests to verify the response actually streamed in multiple deltas. */ - contentDeltaCount?: number + contentDeltaCount?: number; /** Messages sent while a stream was already in flight — held here by * `useChat` and auto-sent FIFO once the run settles. Rendered in a * region separate from `messages` so e2e tests can assert queued state * distinctly from the delivered conversation. */ - queue?: Array + queue?: Array; /** Remove a queued message before it drains. */ - cancelQueued?: (id: string) => void + cancelQueued?: (id: string) => void; /** Block new input while pending interrupts await resolution. */ - hasPendingInterrupt?: boolean + hasPendingInterrupt?: boolean; } export function ChatUI< @@ -56,9 +60,11 @@ export function ChatUI< isLoading, onSendMessage, onSendMessageWithImage, + onSendMessageWithDocument, interrupts = [], addToolApprovalResponse, showImageInput, + showDocumentInput, onStop, structuredObject, contentDeltaCount, @@ -66,22 +72,22 @@ export function ChatUI< cancelQueued, hasPendingInterrupt = false, }: ChatUIProps) { - const [input, setInput] = useState('') - const messagesRef = useRef(null) - const inputRef = useRef(null) + const [input, setInput] = useState(""); + const messagesRef = useRef(null); + const inputRef = useRef(null); useEffect(() => { if (messagesRef.current) { - messagesRef.current.scrollTop = messagesRef.current.scrollHeight + messagesRef.current.scrollTop = messagesRef.current.scrollHeight; } - }, [messages]) + }, [messages]); const handleSubmit = () => { - if (hasPendingInterrupt) return - if (!input.trim()) return - onSendMessage(input.trim()) - setInput('') - } + if (hasPendingInterrupt) return; + if (!input.trim()) return; + onSendMessage(input.trim()); + setInput(""); + }; return (
@@ -107,22 +113,22 @@ export function ChatUI< {interrupts // Only actionable pauses — staged/error are not clickable Approve // prompts; submitting is already omitted from the public list. - .filter((interrupt) => interrupt.status === 'pending') + .filter((interrupt) => interrupt.status === "pending") .map((interrupt) => { // Tool-approval interrupts expose `toolName` / `originalArgs`. // Structural narrow (not only `kind ===`) so this stays valid when // `TTools` defaults to a tools array whose `ChatInterrupt` union is // generic-only at the type level but still carries approval at runtime. if ( - !('toolName' in interrupt) || - !('originalArgs' in interrupt) || + !("toolName" in interrupt) || + !("originalArgs" in interrupt) || // `unbound` pauses carry no resolver — they belong to another // producer on the stream. - !('resolveInterrupt' in interrupt) + !("resolveInterrupt" in interrupt) ) { - return null + return null; } - const toolName = String(interrupt.toolName) + const toolName = String(interrupt.toolName); return (
- Tool {toolName}{' '} + Tool {toolName}{" "} requires approval
@@ -153,22 +159,22 @@ export function ChatUI<
- ) + ); })} {messages.map((message) => (
{message.parts.map((part, i) => { - if (part.type === 'text') { + if (part.type === "text") { return (
- ) + ); } - if (part.type === 'image') { - const imgPart = part as any + if (part.type === "image") { + const imgPart = part as any; const src = - imgPart.source?.type === 'data' + imgPart.source?.type === "data" ? `data:${imgPart.source.mimeType};base64,${imgPart.source.value}` - : imgPart.source?.type === 'url' + : imgPart.source?.type === "url" ? imgPart.source.value - : undefined + : undefined; return src ? ( - ) : null + ) : null; } - if (part.type === 'thinking') { + if (part.type === "thinking") { return (
{part.content}
- ) + ); } // Prefer bound `interrupts` UI above. Legacy message-part prompts // remain only when no interrupt list was provided (compat path). if ( interrupts.length === 0 && - part.type === 'tool-call' && - (part as any).state === 'approval-requested' && + part.type === "tool-call" && + (part as any).state === "approval-requested" && addToolApprovalResponse ) { return ( @@ -227,12 +233,12 @@ export function ChatUI< part={part} onRespond={addToolApprovalResponse} /> - ) + ); } - if (part.type === 'tool-call') { - return + if (part.type === "tool-call") { + return ; } - if (part.type === 'tool-result') { + if (part.type === "tool-result") { return (
Result: {(part as any).content}
- ) + ); } - if (part.type === 'structured-output') { + if (part.type === "structured-output") { // Render the streamed JSON so the assistant message has // visible content for selectors (e.g. `getLastAssistantMessage`). // Previously this content arrived as a `text` part — the new // routing puts it on a `structured-output` part instead. - const sop = part as any + const sop = part as any; const text = sop.raw || - (sop.data !== undefined ? JSON.stringify(sop.data) : '') - if (text === '') return null + (sop.data !== undefined ? JSON.stringify(sop.data) : ""); + if (text === "") return null; return (
{text}
- ) + ); } - return null + return null; })}
))} @@ -290,7 +296,7 @@ export function ChatUI< className="flex items-center justify-between gap-2 text-xs text-gray-400 bg-gray-800/40 rounded px-2 py-1" > - {typeof queued.content === 'string' + {typeof queued.content === "string" ? queued.content : JSON.stringify(queued.content)} @@ -317,15 +323,33 @@ export function ChatUI< data-testid="image-attachment-input" className="text-xs text-gray-400" onChange={(e) => { - const file = e.target.files?.[0] + const file = e.target.files?.[0]; // Read the prompt from the live input DOM value rather than the // `input` React state. Attaching a file auto-sends, and under // load a controlled input's state can lag the committed DOM // value — reading state here would send an empty/partial prompt. - const text = (inputRef.current?.value ?? input).trim() + const text = (inputRef.current?.value ?? input).trim(); if (file && text && onSendMessageWithImage) { - onSendMessageWithImage(text, file) - setInput('') + onSendMessageWithImage(text, file); + setInput(""); + } + }} + /> + )} + {showDocumentInput && ( + { + const file = e.target.files?.[0]; + // Same DOM-value read as the image input above. + const text = (inputRef.current?.value ?? input).trim(); + if (file && text && onSendMessageWithDocument) { + onSendMessageWithDocument(text, file); + setInput(""); + e.target.value = ""; } }} /> @@ -337,9 +361,9 @@ export function ChatUI< value={input} onChange={(e) => setInput(e.target.value)} onKeyDown={(e) => { - if (e.key === 'Enter' && !e.shiftKey) { - e.preventDefault() - handleSubmit() + if (e.key === "Enter" && !e.shiftKey) { + e.preventDefault(); + handleSubmit(); } }} placeholder="Type a message..." @@ -369,5 +393,5 @@ export function ChatUI< )}
- ) + ); } diff --git a/testing/e2e/src/lib/feature-support.ts b/testing/e2e/src/lib/feature-support.ts index cd3e2203d..07f22c4fd 100644 --- a/testing/e2e/src/lib/feature-support.ts +++ b/testing/e2e/src/lib/feature-support.ts @@ -188,6 +188,9 @@ export const matrix: Record> = { 'grok', 'openrouter', ]), + // OpenAI only: this feature exercises the Responses adapter's PDF + // `input_file` conversion (base64 `file_data` + filename). + 'multimodal-document': new Set(['openai']), // Bedrock excluded: same text-only default e2e model as multimodal-image above. 'multimodal-structured': new Set([ 'openai', diff --git a/testing/e2e/src/lib/features.ts b/testing/e2e/src/lib/features.ts index f6a0b57d1..9c96948c2 100644 --- a/testing/e2e/src/lib/features.ts +++ b/testing/e2e/src/lib/features.ts @@ -88,6 +88,10 @@ export const featureConfigs: Record = { tools: [], modelOptions: {}, }, + 'multimodal-document': { + tools: [], + modelOptions: {}, + }, 'multimodal-structured': { tools: [], modelOptions: {}, diff --git a/testing/e2e/src/lib/types.ts b/testing/e2e/src/lib/types.ts index 0e362b5b1..9017f91f4 100644 --- a/testing/e2e/src/lib/types.ts +++ b/testing/e2e/src/lib/types.ts @@ -30,6 +30,7 @@ export type Feature = | 'agentic-structured' | 'agentic-structured-stream' | 'multimodal-image' + | 'multimodal-document' | 'multimodal-structured' | 'summarize' | 'summarize-stream' @@ -76,6 +77,7 @@ export const ALL_FEATURES: Feature[] = [ 'agentic-structured', 'agentic-structured-stream', 'multimodal-image', + 'multimodal-document', 'multimodal-structured', 'summarize', 'summarize-stream', diff --git a/testing/e2e/src/routes/$provider/$feature.tsx b/testing/e2e/src/routes/$provider/$feature.tsx index 0d9ef1105..515ba6440 100644 --- a/testing/e2e/src/routes/$provider/$feature.tsx +++ b/testing/e2e/src/routes/$provider/$feature.tsx @@ -1,108 +1,108 @@ -import { useState } from 'react' -import { createFileRoute } from '@tanstack/react-router' -import { uiMessagesToWire } from '@tanstack/ai' -import { fetchServerSentEvents, useChat } from '@tanstack/ai-react' -import { clientTools } from '@tanstack/ai-client' -import type { ChatClientPersistence, UIMessage } from '@tanstack/ai-client' -import type { GeminiInteractionsCustomEventValue } from '@tanstack/ai-gemini/experimental' -import type { Feature, Mode, Provider } from '@/lib/types' -import { ALL_FEATURES, ALL_PROVIDERS } from '@/lib/types' -import { isSupported } from '@/lib/feature-support' -import { addToCartToolDef } from '@/lib/tools' -import { NotSupported } from '@/components/NotSupported' -import { ChatUI } from '@/components/ChatUI' -import { ImageGenUI } from '@/components/ImageGenUI' -import { TTSUI } from '@/components/TTSUI' -import { TranscriptionUI } from '@/components/TranscriptionUI' -import { VideoGenUI } from '@/components/VideoGenUI' -import { AudioGenUI } from '@/components/AudioGenUI' +import { useState } from "react"; +import { createFileRoute } from "@tanstack/react-router"; +import { uiMessagesToWire } from "@tanstack/ai"; +import { fetchServerSentEvents, useChat } from "@tanstack/ai-react"; +import { clientTools } from "@tanstack/ai-client"; +import type { ChatClientPersistence, UIMessage } from "@tanstack/ai-client"; +import type { GeminiInteractionsCustomEventValue } from "@tanstack/ai-gemini/experimental"; +import type { Feature, Mode, Provider } from "@/lib/types"; +import { ALL_FEATURES, ALL_PROVIDERS } from "@/lib/types"; +import { isSupported } from "@/lib/feature-support"; +import { addToCartToolDef } from "@/lib/tools"; +import { NotSupported } from "@/components/NotSupported"; +import { ChatUI } from "@/components/ChatUI"; +import { ImageGenUI } from "@/components/ImageGenUI"; +import { TTSUI } from "@/components/TTSUI"; +import { TranscriptionUI } from "@/components/TranscriptionUI"; +import { VideoGenUI } from "@/components/VideoGenUI"; +import { AudioGenUI } from "@/components/AudioGenUI"; -const VALID_MODES = new Set(['sse', 'http-stream', 'fetcher']) +const VALID_MODES = new Set(["sse", "http-stream", "fetcher"]); -export const Route = createFileRoute('/$provider/$feature')({ +export const Route = createFileRoute("/$provider/$feature")({ component: FeaturePage, validateSearch: (search: Record) => { const port = - typeof search.aimockPort === 'number' + typeof search.aimockPort === "number" ? search.aimockPort - : typeof search.aimockPort === 'string' + : typeof search.aimockPort === "string" ? parseInt(search.aimockPort, 10) - : undefined - const rawMode = typeof search.mode === 'string' ? search.mode : undefined + : undefined; + const rawMode = typeof search.mode === "string" ? search.mode : undefined; return { - testId: typeof search.testId === 'string' ? search.testId : undefined, + testId: typeof search.testId === "string" ? search.testId : undefined, aimockPort: port != null && !isNaN(port) ? port : undefined, mode: rawMode && VALID_MODES.has(rawMode as Mode) ? (rawMode as Mode) : undefined, persistence: - search.persistence === 'localStorage' ? 'localStorage' : undefined, + search.persistence === "localStorage" ? "localStorage" : undefined, serverPersistence: search.serverPersistence === true || search.serverPersistence === 1 || - search.serverPersistence === '1', - } + search.serverPersistence === "1", + }; }, -}) +}); const MEDIA_FEATURES = new Set([ - 'image-gen', - 'image-to-image', - 'tts', - 'transcription', - 'transcription-diarization', - 'video-gen', - 'image-to-video', - 'interactions-video', - 'audio-gen', - 'sound-effects', -]) + "image-gen", + "image-to-image", + "tts", + "transcription", + "transcription-diarization", + "video-gen", + "image-to-video", + "interactions-video", + "audio-gen", + "sound-effects", +]); const addToCartClient = addToCartToolDef.client((args) => ({ success: true, - cartId: 'CART_' + Date.now(), + cartId: "CART_" + Date.now(), guitarId: args.guitarId, quantity: args.quantity, -})) +})); -type StoredUIMessage = Omit & { - createdAt?: Date | string -} +type StoredUIMessage = Omit & { + createdAt?: Date | string; +}; function serializeJson(value: unknown): string { - const serialized = JSON.stringify(value) + const serialized = JSON.stringify(value); if (serialized === undefined) { - throw new TypeError('The persistence value is not JSON serializable') + throw new TypeError("The persistence value is not JSON serializable"); } - return serialized + return serialized; } const isProvider = (s: string): s is Provider => - (ALL_PROVIDERS as ReadonlyArray).includes(s) + (ALL_PROVIDERS as ReadonlyArray).includes(s); const isFeature = (s: string): s is Feature => - (ALL_FEATURES as ReadonlyArray).includes(s) + (ALL_FEATURES as ReadonlyArray).includes(s); const isRecord = (value: unknown): value is Record => - value !== null && typeof value === 'object' && !Array.isArray(value) + value !== null && typeof value === "object" && !Array.isArray(value); function isStoredUIMessage(value: unknown): value is StoredUIMessage { return ( isRecord(value) && - typeof value.id === 'string' && - (value.role === 'system' || - value.role === 'user' || - value.role === 'assistant') && + typeof value.id === "string" && + (value.role === "system" || + value.role === "user" || + value.role === "assistant") && Array.isArray(value.parts) && (value.createdAt === undefined || value.createdAt instanceof Date || - typeof value.createdAt === 'string') - ) + typeof value.createdAt === "string") + ); } function deserializeMessages(raw: string): Array { - const parsed: unknown = JSON.parse(raw) + const parsed: unknown = JSON.parse(raw); if (!Array.isArray(parsed) || !parsed.every(isStoredUIMessage)) { - throw new TypeError('Stored messages are invalid') + throw new TypeError("Stored messages are invalid"); } return parsed.map(({ createdAt, ...message }) => ({ ...message, @@ -112,37 +112,37 @@ function deserializeMessages(raw: string): Array { createdAt instanceof Date ? createdAt : new Date(createdAt), } : {}), - })) + })); } /** Simple localStorage message adapter (no @tanstack/ai-client storage helpers). */ const messagePersistence: ChatClientPersistence = { getItem(id) { try { - const raw = localStorage.getItem(id) - return raw === null ? null : deserializeMessages(raw) + const raw = localStorage.getItem(id); + return raw === null ? null : deserializeMessages(raw); } catch { - return null + return null; } }, setItem(id, messages) { - localStorage.setItem(id, serializeJson(messages)) + localStorage.setItem(id, serializeJson(messages)); }, removeItem(id) { - localStorage.removeItem(id) + localStorage.removeItem(id); }, -} +}; function FeaturePage() { - const { provider, feature } = Route.useParams() - const { testId, aimockPort, mode } = Route.useSearch() + const { provider, feature } = Route.useParams(); + const { testId, aimockPort, mode } = Route.useSearch(); if ( !isProvider(provider) || !isFeature(feature) || !isSupported(provider, feature) ) { - return + return ; } if (MEDIA_FEATURES.has(feature)) { @@ -150,14 +150,14 @@ function FeaturePage() { - ) + ); } - return + return ; } function MediaFeature({ @@ -167,14 +167,14 @@ function MediaFeature({ testId, aimockPort, }: { - provider: Provider - feature: Feature - mode: Mode - testId?: string - aimockPort?: number + provider: Provider; + feature: Feature; + mode: Mode; + testId?: string; + aimockPort?: number; }) { switch (feature) { - case 'image-gen': + case "image-gen": return ( - ) - case 'image-to-image': + ); + case "image-to-image": return ( - ) - case 'tts': + ); + case "tts": return ( - ) - case 'transcription': - case 'transcription-diarization': + ); + case "transcription": + case "transcription-diarization": return ( - ) - case 'video-gen': + ); + case "video-gen": return ( - ) - case 'image-to-video': + ); + case "image-to-video": return ( - ) - case 'interactions-video': + ); + case "interactions-video": return ( - ) - case 'audio-gen': - case 'sound-effects': + ); + case "audio-gen": + case "sound-effects": return ( - ) + ); default: - return + return ; } } @@ -263,50 +263,51 @@ function ChatFeature({ feature, mode, }: { - provider: Provider - feature: Feature - mode?: Mode + provider: Provider; + feature: Feature; + mode?: Mode; }) { - const needsApproval = feature === 'tool-approval' + const needsApproval = feature === "tool-approval"; const showImageInput = - feature === 'multimodal-image' || feature === 'multimodal-structured' + feature === "multimodal-image" || feature === "multimodal-structured"; + const showDocumentInput = feature === "multimodal-document"; // Stable tools tuple so `useChat` / `BoundInterrupts` keep approval typing // (and ChatUI can accept `interrupts` without casts). - const approvalTools = clientTools(addToCartClient) - const tools = needsApproval ? approvalTools : undefined + const approvalTools = clientTools(addToCartClient); + const tools = needsApproval ? approvalTools : undefined; const { testId, aimockPort, persistence, serverPersistence } = - Route.useSearch() - const persistenceEnabled = persistence === 'localStorage' - const serverPersistenceEnabled = serverPersistence === true - const baseChatId = `e2e-chat-${testId ?? `${provider}-${feature}`}` + Route.useSearch(); + const persistenceEnabled = persistence === "localStorage"; + const serverPersistenceEnabled = serverPersistence === true; + const baseChatId = `e2e-chat-${testId ?? `${provider}-${feature}`}`; // When persistence is on, expose a tiny thread switcher so e2e can verify that // changing the `id` in place swaps to that id's own persisted history (the // render-from-getMessages + activeClientRef path), keyed per thread. Start on // thread "a" (not null) so the page loads already on a thread id — switching // is then a pure in-place id swap with no initial null→thread transition. const [activeThread, setActiveThread] = useState( - persistenceEnabled ? 'a' : null, - ) - const chatId = activeThread ? `${baseChatId}:${activeThread}` : baseChatId + persistenceEnabled ? "a" : null, + ); + const chatId = activeThread ? `${baseChatId}:${activeThread}` : baseChatId; - const [structuredObject, setStructuredObject] = useState(null) - const [contentDeltaCount, setContentDeltaCount] = useState(0) + const [structuredObject, setStructuredObject] = useState(null); + const [contentDeltaCount, setContentDeltaCount] = useState(0); const [interactionId, setInteractionId] = useState( undefined, - ) + ); const transport = - mode === 'fetcher' + mode === "fetcher" ? { fetcher: async ( input: { - messages: Array - data?: unknown - threadId: string - runId: string - resume?: Array + messages: Array; + data?: unknown; + threadId: string; + runId: string; + resume?: Array; }, options: { signal: AbortSignal }, ) => @@ -316,14 +317,14 @@ function ChatFeature({ // `useChat({ body })` already flowed provider/feature/testId/ // aimockPort into `input.data`, so it forwards as // `forwardedProps`. - fetch('/api/chat', { - method: 'POST', + fetch("/api/chat", { + method: "POST", headers: { - 'Content-Type': 'application/json', + "Content-Type": "application/json", // Sentinel header so e2e tests can positively assert the // fetcher path executed (and didn't silently fall back to // the connection adapter). - 'x-tanstack-ai-transport': 'fetcher', + "x-tanstack-ai-transport": "fetcher", }, body: JSON.stringify({ threadId: input.threadId, @@ -338,7 +339,7 @@ function ChatFeature({ signal: options.signal, }), } - : { connection: fetchServerSentEvents('/api/chat') } + : { connection: fetchServerSentEvents("/api/chat") }; const { messages, @@ -367,22 +368,22 @@ function ChatFeature({ // on this branch (durable resume adapters live on feat/persistence). persistence: persistenceEnabled ? messagePersistence : undefined, onCustomEvent: (eventType, data) => { - if (eventType === 'structured-output.complete') { - const value = data as { object: unknown; raw: string } | undefined - setStructuredObject(value?.object ?? null) - } else if (eventType === 'gemini.interactionId') { + if (eventType === "structured-output.complete") { + const value = data as { object: unknown; raw: string } | undefined; + setStructuredObject(value?.object ?? null); + } else if (eventType === "gemini.interactionId") { const value = data as - | GeminiInteractionsCustomEventValue<'gemini.interactionId'> - | undefined - if (value?.interactionId) setInteractionId(value.interactionId) + | GeminiInteractionsCustomEventValue<"gemini.interactionId"> + | undefined; + if (value?.interactionId) setInteractionId(value.interactionId); } }, onChunk: (chunk) => { - if (chunk.type === 'TEXT_MESSAGE_CONTENT') { - setContentDeltaCount((n) => n + 1) + if (chunk.type === "TEXT_MESSAGE_CONTENT") { + setContentDeltaCount((n) => n + 1); } }, - }) + }); return ( <> @@ -409,14 +410,14 @@ function ChatFeature({ @@ -433,37 +434,63 @@ function ChatFeature({ queue={queue} cancelQueued={cancelQueued} onSendMessage={(text) => { - sendMessage(text) + sendMessage(text); }} onSendMessageWithImage={ showImageInput ? (text, file) => { - const reader = new FileReader() + const reader = new FileReader(); + reader.onload = () => { + const base64 = (reader.result as string).split(",")[1]; + sendMessage({ + content: [ + { type: "text", content: text }, + { + type: "image", + source: { + type: "data", + value: base64, + mimeType: file.type, + }, + }, + ], + }); + }; + reader.readAsDataURL(file); + } + : undefined + } + onSendMessageWithDocument={ + showDocumentInput + ? (text, file) => { + const reader = new FileReader(); reader.onload = () => { - const base64 = (reader.result as string).split(',')[1] + const base64 = (reader.result as string).split(",")[1]; sendMessage({ content: [ - { type: 'text', content: text }, + { type: "text", content: text }, { - type: 'image', + type: "document", source: { - type: 'data', + type: "data", value: base64, mimeType: file.type, }, + metadata: { filename: file.name }, }, ], - }) - } - reader.readAsDataURL(file) + }); + }; + reader.readAsDataURL(file); } : undefined } interrupts={needsApproval ? interrupts : undefined} - hasPendingInterrupt={interrupts.some((i) => i.status === 'pending')} + hasPendingInterrupt={interrupts.some((i) => i.status === "pending")} showImageInput={showImageInput} + showDocumentInput={showDocumentInput} onStop={stop} /> - ) + ); } diff --git a/testing/e2e/test-assets/tiny.pdf b/testing/e2e/test-assets/tiny.pdf new file mode 100644 index 000000000..0909dcbe3 Binary files /dev/null and b/testing/e2e/test-assets/tiny.pdf differ diff --git a/testing/e2e/tests/helpers.ts b/testing/e2e/tests/helpers.ts index ff4d0ebcd..de8ddfb72 100644 --- a/testing/e2e/tests/helpers.ts +++ b/testing/e2e/tests/helpers.ts @@ -31,6 +31,8 @@ export async function sendMessage(page: Page, text: string) { }) } +/** Types a prompt and attaches an image; attaching auto-sends. Retries until + * the user bubble renders — the inline comment below explains why. */ export async function sendMessageWithImage( page: Page, text: string, @@ -64,6 +66,30 @@ export async function sendMessageWithImage( }).toPass({ timeout: 15_000, intervals: [250, 500, 1000] }) } +/** Types a prompt and attaches a PDF; attaching auto-sends. Retries until the + * user bubble renders (same fragile-controlled-input problem as + * sendMessageWithImage — see the comment there). */ +export async function sendMessageWithDocument( + page: Page, + text: string, + documentPath: string, +) { + const input = page.getByTestId('chat-input') + const fileInput = page.getByTestId('document-attachment-input') + const userMessages = page.getByTestId('user-message') + + // Same retry-to-observable-outcome pattern as sendMessageWithImage above. + await expect(async () => { + await input.click() + await input.fill('') + await input.pressSequentially(text, { delay: 15 }) + expect(await input.inputValue()).toBe(text) + await fileInput.setInputFiles([]) + await fileInput.setInputFiles(documentPath) + await expect(userMessages.first()).toBeVisible({ timeout: 2_000 }) + }).toPass({ timeout: 15_000, intervals: [250, 500, 1000] }) +} + export async function waitForResponse(page: Page, timeout = 15_000) { try { await page diff --git a/testing/e2e/tests/multimodal-document.spec.ts b/testing/e2e/tests/multimodal-document.spec.ts new file mode 100644 index 000000000..39fc6c6b2 --- /dev/null +++ b/testing/e2e/tests/multimodal-document.spec.ts @@ -0,0 +1,38 @@ +import { test, expect } from './fixtures' +import { + sendMessageWithDocument, + waitForResponse, + getLastAssistantMessage, + featureUrl, +} from './helpers' +import { providersFor } from './test-matrix' +import { fileURLToPath } from 'url' +import path from 'path' + +const __filename = fileURLToPath(import.meta.url) +const __dirname = path.dirname(__filename) +const testDocumentPath = path.resolve(__dirname, '../test-assets/tiny.pdf') + +for (const provider of providersFor('multimodal-document')) { + test.describe(`${provider} — multimodal-document`, () => { + test('answers a question about an uploaded PDF', async ({ + page, + testId, + aimockPort, + }) => { + await page.goto( + featureUrl(provider, 'multimodal-document', testId, aimockPort), + ) + + await sendMessageWithDocument( + page, + '[mmdocument] summarize this pdf', + testDocumentPath, + ) + await waitForResponse(page) + + const response = await getLastAssistantMessage(page) + expect(response).toContain('blank page') + }) + }) +}