From 260c0fc150cb912fef7d3f794d028b4cccc4e13f Mon Sep 17 00:00:00 2001 From: daniel Date: Mon, 20 Jul 2026 12:51:31 -0700 Subject: [PATCH 1/4] feat(openai): support PDF document content parts in the Responses adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenAI chat models that declare the `document` input modality now accept PDF `document` content parts through `openaiText` on the Responses adapter. Base64 data sources are sent as `input_file` with a `file_data` data URL and a `filename` (from `metadata.filename`, defaulting to `document.pdf`); URL sources are sent as `input_file` with `file_url`. The adapter rejects non-PDF document MIME types with a clear error before the request goes out — including pre-wrapped `data:` URLs whose media type disagrees with `mimeType` — so callers get an actionable message instead of an opaque provider 400. `OpenAIDocumentMetadata` gains `filename` and `detail`, and the document-capable chat models declare the `document` input modality. The Chat Completions adapter throws a document-specific error pointing at the Responses adapter, since documents are only supported there. Adds unit coverage for the Responses and Chat Completions adapters, and an e2e fixture/spec/page for the multimodal-document feature. --- packages/ai-openai/src/message-types.ts | 22 +- packages/ai-openai/src/model-meta.ts | 46 +-- packages/ai-openai/tests/model-meta.test.ts | 180 +++++----- .../src/adapters/chat-completions-text.ts | 11 + .../src/adapters/responses-text.ts | 69 +++- .../tests/chat-completions-text.test.ts | 38 +++ .../openai-base/tests/responses-text.test.ts | 321 ++++++++++++++++++ testing/e2e/README.md | 1 + .../fixtures/multimodal-document/basic.json | 12 + testing/e2e/src/components/ChatUI.tsx | 21 ++ testing/e2e/src/lib/feature-support.ts | 3 + testing/e2e/src/lib/features.ts | 4 + testing/e2e/src/lib/types.ts | 2 + testing/e2e/src/routes/$provider/$feature.tsx | 27 ++ testing/e2e/test-assets/tiny.pdf | Bin 0 -> 192 bytes testing/e2e/tests/helpers.ts | 21 ++ testing/e2e/tests/multimodal-document.spec.ts | 38 +++ 17 files changed, 694 insertions(+), 122 deletions(-) create mode 100644 testing/e2e/fixtures/multimodal-document/basic.json create mode 100644 testing/e2e/test-assets/tiny.pdf create mode 100644 testing/e2e/tests/multimodal-document.spec.ts diff --git a/packages/ai-openai/src/message-types.ts b/packages/ai-openai/src/message-types.ts index 895c6e397..57c9b7e04 100644 --- a/packages/ai-openai/src/message-types.ts +++ b/packages/ai-openai/src/message-types.ts @@ -44,9 +44,27 @@ export interface OpenAIVideoMetadata {} /** * Metadata for OpenAI document content parts. - * Note: Direct document support may vary; PDFs often need to be converted to images. + * + * Documents are supported by the Responses adapter only, which sends them + * as `input_file`; the Chat Completions adapter rejects document parts. + * This adapter currently supports only `application/pdf` documents. That is + * enforced locally for inline (base64) data; URL-sourced documents are sent + * to OpenAI unvalidated, so a non-PDF URL fails server-side instead. + * + * @see https://developers.openai.com/api/docs/guides/pdf-files */ -export interface OpenAIDocumentMetadata {} +export interface OpenAIDocumentMetadata { + /** + * Filename sent alongside inline (base64) PDF data. + * @default 'document.pdf' + */ + filename?: string + /** + * Rendering quality for the file's page images. Omitted by default so the + * API applies its own default ('low'). + */ + detail?: 'low' | 'high' +} /** * Metadata for OpenAI text content parts. diff --git a/packages/ai-openai/src/model-meta.ts b/packages/ai-openai/src/model-meta.ts index b28d0c604..5ed050144 100644 --- a/packages/ai-openai/src/model-meta.ts +++ b/packages/ai-openai/src/model-meta.ts @@ -11,7 +11,7 @@ import type { interface ModelMeta { name: string supports: { - input: Array<'text' | 'image' | 'audio' | 'video'> + input: Array<'text' | 'image' | 'audio' | 'video' | 'document'> output: Array<'text' | 'image' | 'audio' | 'video'> endpoints: Array< | 'chat' @@ -74,7 +74,7 @@ const GPT5_2 = { max_output_tokens: 128_000, knowledge_cutoff: '2025-08-31', supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'chat-completions'], features: [ @@ -119,7 +119,7 @@ const GPT5_2_PRO = { max_output_tokens: 128_000, knowledge_cutoff: '2025-08-31', supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'chat-completions'], features: ['streaming', 'function_calling'], @@ -158,7 +158,7 @@ const GPT5_2_CHAT = { max_output_tokens: 16_384, knowledge_cutoff: '2025-08-31', supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'chat-completions'], features: ['streaming', 'function_calling', 'structured_outputs'], @@ -194,7 +194,7 @@ const GPT5_1 = { max_output_tokens: 128_000, knowledge_cutoff: '2024-09-30', supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text', 'image'], endpoints: ['chat', 'chat-completions'], features: [ @@ -240,7 +240,7 @@ const GPT5_1_CODEX = { max_output_tokens: 128_000, knowledge_cutoff: '2024-09-30', supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text', 'image'], endpoints: ['chat'], features: ['streaming', 'function_calling', 'structured_outputs'], @@ -277,7 +277,7 @@ const GPT5 = { max_output_tokens: 128_000, knowledge_cutoff: '2024-09-30', supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'chat-completions', 'batch'], features: [ @@ -323,7 +323,7 @@ const GPT5_MINI = { max_output_tokens: 128_000, knowledge_cutoff: '2024-05-31', supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'chat-completions', 'batch'], features: ['streaming', 'structured_outputs', 'function_calling'], @@ -373,7 +373,7 @@ const GPT5_NANO = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'chat-completions', 'batch'], features: ['streaming', 'structured_outputs', 'function_calling'], @@ -413,7 +413,7 @@ const GPT5_PRO = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'batch'], features: ['streaming', 'structured_outputs', 'function_calling'], @@ -454,7 +454,7 @@ const GPT5_CODEX = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text', 'image'], endpoints: ['chat'], features: ['streaming', 'structured_outputs', 'function_calling'], @@ -686,7 +686,7 @@ const O3_PRO = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'batch'], features: ['function_calling', 'structured_outputs'], @@ -839,7 +839,7 @@ const O3 = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'batch', 'chat-completions'], features: ['function_calling', 'structured_outputs', 'streaming'], @@ -880,7 +880,7 @@ const O4_MINI = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'batch', 'chat-completions', 'fine-tuning'], features: [ @@ -925,7 +925,7 @@ const GPT4_1 = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: [ 'chat', @@ -978,7 +978,7 @@ const GPT4_1_MINI = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: [ 'chat', @@ -1029,7 +1029,7 @@ const GPT4_1_NANO = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: [ 'chat', @@ -1080,7 +1080,7 @@ const O1_PRO = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'batch'], features: ['function_calling', 'structured_outputs'], @@ -1283,7 +1283,7 @@ const O1 = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'batch', 'chat-completions', 'assistants'], features: ['function_calling', 'structured_outputs', 'streaming'], @@ -1333,7 +1333,7 @@ const GPT_4O = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: [ 'chat', @@ -1410,7 +1410,7 @@ const GPT_4O_MINI = { }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: [ 'chat', @@ -2048,7 +2048,7 @@ const GPT_5_5 = { context_window: 1_050_000, max_output_tokens: 128_000, supports: { - input: ['image', 'text'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'chat-completions'], features: [ @@ -2093,7 +2093,7 @@ const GPT_5_5_PRO = { context_window: 1_050_000, max_output_tokens: 128_000, supports: { - input: ['image', 'text'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat', 'chat-completions'], features: [ diff --git a/packages/ai-openai/tests/model-meta.test.ts b/packages/ai-openai/tests/model-meta.test.ts index 77203fd0f..c6f9ec09d 100644 --- a/packages/ai-openai/tests/model-meta.test.ts +++ b/packages/ai-openai/tests/model-meta.test.ts @@ -840,223 +840,223 @@ type MessageWithContent = { role: 'user'; content: Array } describe('OpenAI Model Input Modality Type Assertions', () => { // ===== Models with text + image input ===== - describe('gpt-5.1 (text + image)', () => { + describe('gpt-5.1 (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['gpt-5.1'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('gpt-5.1-codex (text + image)', () => { + describe('gpt-5.1-codex (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['gpt-5.1-codex'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('gpt-5 (text + image)', () => { + describe('gpt-5 (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['gpt-5'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('gpt-5-mini (text + image)', () => { + describe('gpt-5-mini (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['gpt-5-mini'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('gpt-5-nano (text + image)', () => { + describe('gpt-5-nano (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['gpt-5-nano'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('gpt-5-pro (text + image)', () => { + describe('gpt-5-pro (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['gpt-5-pro'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('gpt-5-codex (text + image)', () => { + describe('gpt-5-codex (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['gpt-5-codex'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('gpt-4.1 (text + image)', () => { + describe('gpt-4.1 (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['gpt-4.1'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('gpt-4.1-mini (text + image)', () => { + describe('gpt-4.1-mini (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['gpt-4.1-mini'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('gpt-4.1-nano (text + image)', () => { + describe('gpt-4.1-nano (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['gpt-4.1-nano'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) @@ -1104,47 +1104,47 @@ describe('OpenAI Model Input Modality Type Assertions', () => { }) }) - describe('o3 (text + image)', () => { + describe('o3 (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['o3'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('o3-pro (text + image)', () => { + describe('o3-pro (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['o3-pro'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) @@ -1192,69 +1192,69 @@ describe('OpenAI Model Input Modality Type Assertions', () => { }) }) - describe('o4-mini (text + image)', () => { + describe('o4-mini (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['o4-mini'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('o1 (text + image)', () => { + describe('o1 (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['o1'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) - describe('o1-pro (text + image)', () => { + describe('o1-pro (text + image + document)', () => { type Modalities = OpenAIModelInputModalitiesByName['o1-pro'] type Message = ConstrainedModelMessage> - it('should allow TextPart and ImagePart', () => { + it('should allow TextPart, ImagePart, and DocumentPart', () => { expectTypeOf>().toExtend() expectTypeOf>().toExtend() + expectTypeOf< + MessageWithContent + >().toExtend() }) - it('should NOT allow AudioPart, VideoPart, or DocumentPart', () => { + it('should NOT allow AudioPart or VideoPart', () => { expectTypeOf< MessageWithContent >().not.toExtend() expectTypeOf< MessageWithContent >().not.toExtend() - expectTypeOf< - MessageWithContent - >().not.toExtend() }) }) diff --git a/packages/openai-base/src/adapters/chat-completions-text.ts b/packages/openai-base/src/adapters/chat-completions-text.ts index 72ea19ca7..abf1d42e4 100644 --- a/packages/openai-base/src/adapters/chat-completions-text.ts +++ b/packages/openai-base/src/adapters/chat-completions-text.ts @@ -1332,6 +1332,17 @@ export abstract class OpenAIBaseChatCompletionsTextAdapter< } } + if (part.type === 'document') { + // Documents (PDF) are implemented on the Responses adapter, which maps + // them to `input_file`. Model modality arrays are not endpoint-scoped, + // so a document part can type-check for a model this adapter serves — + // point callers at the supported path instead of a generic error. + throw new Error( + `${this.name} does not support document parts on the Chat Completions ` + + `API; use the Responses adapter, which sends them as input_file.`, + ) + } + // Unsupported content type — subclasses can override to handle more types return null } diff --git a/packages/openai-base/src/adapters/responses-text.ts b/packages/openai-base/src/adapters/responses-text.ts index d48018209..bf2a34c1e 100644 --- a/packages/openai-base/src/adapters/responses-text.ts +++ b/packages/openai-base/src/adapters/responses-text.ts @@ -1766,7 +1766,7 @@ export abstract class OpenAIBaseResponsesTextAdapter< throw new Error( `User message for ${this.name} has no content parts. ` + `Empty user messages would produce a paid request with no input; ` + - `provide at least one text/image/audio part or omit the message.`, + `provide at least one text/image/audio/document part or omit the message.`, ) } @@ -1782,7 +1782,7 @@ export abstract class OpenAIBaseResponsesTextAdapter< /** * Converts a ContentPart to Responses API input content item. - * Handles text, image, and audio content parts. + * Handles text, image, audio, and document (PDF) content parts. * Override this in subclasses for additional content types or provider-specific metadata. */ protected convertContentPartToInput(part: ContentPart): ResponseInputContent { @@ -1826,7 +1826,7 @@ export abstract class OpenAIBaseResponsesTextAdapter< } } // Wrap raw base64 in a data URL — `input_file` rejects bare base64 - // payloads (matches the image branch above which already does this). + // payloads (matches the image branch above). // Default the MIME if missing so we never interpolate `undefined`. const audioValue = part.source.value const audioMime = part.source.mimeType || 'application/octet-stream' @@ -1839,12 +1839,67 @@ export abstract class OpenAIBaseResponsesTextAdapter< } } + case 'document': { + const documentMetadata = part.metadata as + | { filename?: string; detail?: 'low' | 'high' } + | undefined + // `detail` is optional with no 'auto' member (unlike input_image) — + // spread only when provided so the API applies its own default. + const documentDetail = + documentMetadata?.detail !== undefined + ? { detail: documentMetadata.detail } + : {} + if (part.source.type === 'url') { + // The Responses API fetches the PDF itself; filename and MIME + // type are inferred from the response. + return { + type: 'input_file', + file_url: part.source.value, + ...documentDetail, + } + } + // This adapter supports only PDF documents; anything else is + // rejected. MIME types are case-insensitive and can carry + // ;parameters (RFC 2045), so the type is normalized before comparing. + const documentValue = part.source.value + const documentMime = ((part.source.mimeType || 'application/pdf').split(';')[0] ?? '') + .trim() + .toLowerCase() + if (documentMime !== 'application/pdf') { + throw new Error( + `${this.name} document parts only support application/pdf ` + + `(received ${documentMime})`, + ) + } + // A pre-wrapped data URL carries its own media type — validate it too. + if ( + documentValue.startsWith('data:') && + !/^data:application\/pdf[;,]/i.test(documentValue) + ) { + throw new Error( + `${this.name} document parts only support application/pdf ` + + `(received data URL with non-PDF media type)`, + ) + } + // Wrap raw base64 in a data URL — `input_file` rejects bare base64 + // payloads (matches the image and audio branches above). + const documentFileData = documentValue.startsWith('data:') + ? documentValue + : `data:${documentMime};base64,${documentValue}` + return { + type: 'input_file', + // The Responses API requires a filename alongside PDF `file_data`. + filename: documentMetadata?.filename || 'document.pdf', + file_data: documentFileData, + ...documentDetail, + } + } + case 'video': - case 'document': default: - // OpenAI Responses API doesn't accept native video/document parts on - // this path — surface as explicit unsupported error so callers see - // the same message regardless of which content type leaked through. + // OpenAI Responses API doesn't accept native video parts on this + // path — surface as explicit unsupported error so callers see the + // same message regardless of which content type leaked through. throw new Error(`Unsupported content part type: ${part.type}`) } } diff --git a/packages/openai-base/tests/chat-completions-text.test.ts b/packages/openai-base/tests/chat-completions-text.test.ts index e92e34f1d..3b58d7d77 100644 --- a/packages/openai-base/tests/chat-completions-text.test.ts +++ b/packages/openai-base/tests/chat-completions-text.test.ts @@ -620,6 +620,44 @@ describe('OpenAIBaseChatCompletionsTextAdapter', () => { }) describe('error handling', () => { + it('points document parts at the Responses adapter', async () => { + mockCreate = vi.fn() + const adapter = new TestChatCompletionsAdapter(testConfig, 'test-model') + const chunks: Array = [] + + for await (const chunk of adapter.chatStream({ + logger: testLogger, + model: 'test-model', + messages: [ + { + role: 'user', + content: [ + { type: 'text', content: 'summarize this' }, + { + type: 'document', + source: { + type: 'data', + value: 'JVBERi0xLjQK', + mimeType: 'application/pdf', + }, + }, + ], + }, + ], + })) { + chunks.push(chunk) + } + + const runErrorChunk = chunks.find((c) => c.type === 'RUN_ERROR') + expect(runErrorChunk).toBeDefined() + if (runErrorChunk?.type === 'RUN_ERROR') { + expect(runErrorChunk.message).toMatch(/Responses adapter/) + expect(runErrorChunk.message).toMatch(/document/) + } + // No request should have been attempted. + expect(mockCreate).not.toHaveBeenCalled() + }) + it('emits RUN_ERROR on stream error', async () => { const streamChunks = [ { diff --git a/packages/openai-base/tests/responses-text.test.ts b/packages/openai-base/tests/responses-text.test.ts index 884954267..ccad68416 100644 --- a/packages/openai-base/tests/responses-text.test.ts +++ b/packages/openai-base/tests/responses-text.test.ts @@ -2239,6 +2239,327 @@ describe('OpenAIBaseResponsesTextAdapter', () => { }) }) + describe('document content parts (PDF input)', () => { + // A minimal one-page PDF. + const TINY_PDF_BASE64 = Buffer.from( + '%PDF-1.4\n1 0 obj<>endobj\n' + + '2 0 obj<>endobj\n' + + '3 0 obj<>endobj\n' + + 'trailer<>\n%%EOF', + ).toString('base64') + + const minimalStreamChunks = [ + { + type: 'response.created', + response: { id: 'resp-doc-1', model: 'test-model', status: 'in_progress' }, + }, + { + type: 'response.completed', + response: { + id: 'resp-doc-1', + model: 'test-model', + status: 'completed', + output: [], + usage: { input_tokens: 10, output_tokens: 1, total_tokens: 11 }, + }, + }, + ] + + async function runChat(content: Array): Promise> { + setupMockResponsesClient(minimalStreamChunks) + const adapter = new TestResponsesAdapter(testConfig, 'test-model') + const chunks: Array = [] + for await (const chunk of adapter.chatStream({ + logger: testLogger, + model: 'test-model', + messages: [{ role: 'user', content }], + })) { + chunks.push(chunk) + } + return chunks + } + + it('converts a base64 data source to input_file with a default filename', async () => { + await runChat([ + { + type: 'document', + source: { + type: 'data', + value: TINY_PDF_BASE64, + mimeType: 'application/pdf', + }, + }, + ]) + + const [payload] = mockResponsesCreate.mock.calls[0]! + expect(payload.input[0].content).toEqual([ + { + type: 'input_file', + filename: 'document.pdf', + file_data: `data:application/pdf;base64,${TINY_PDF_BASE64}`, + }, + ]) + }) + + it('uses metadata.filename when provided', async () => { + await runChat([ + { + type: 'document', + source: { + type: 'data', + value: TINY_PDF_BASE64, + mimeType: 'application/pdf', + }, + metadata: { filename: 'report.pdf' }, + }, + ]) + + const [payload] = mockResponsesCreate.mock.calls[0]! + expect(payload.input[0].content[0].filename).toBe('report.pdf') + }) + + it('passes an existing data URL through without double-wrapping', async () => { + const dataUrl = `data:application/pdf;base64,${TINY_PDF_BASE64}` + await runChat([ + { + type: 'document', + source: { type: 'data', value: dataUrl, mimeType: 'application/pdf' }, + }, + ]) + + const [payload] = mockResponsesCreate.mock.calls[0]! + expect(payload.input[0].content[0].file_data).toBe(dataUrl) + }) + + it('converts a URL source to input_file with file_url only', async () => { + await runChat([ + { + type: 'document', + source: { type: 'url', value: 'https://example.com/doc.pdf' }, + }, + ]) + + const [payload] = mockResponsesCreate.mock.calls[0]! + const part = payload.input[0].content[0] + expect(part).toEqual({ + type: 'input_file', + file_url: 'https://example.com/doc.pdf', + }) + expect(part).not.toHaveProperty('file_data') + expect(part).not.toHaveProperty('filename') + }) + + it('emits RUN_ERROR for a non-PDF document MIME type', async () => { + const chunks = await runChat([ + { + type: 'document', + source: { + type: 'data', + value: TINY_PDF_BASE64, + mimeType: 'application/msword', + }, + }, + ]) + + const errorChunk = chunks.find((c) => c.type === 'RUN_ERROR') + expect(errorChunk).toBeDefined() + if (errorChunk?.type === 'RUN_ERROR') { + expect(errorChunk.message).toMatch(/only support application\/pdf/) + expect(errorChunk.message).toMatch(/application\/msword/) + } + }) + + it('emits RUN_ERROR for a data URL with a non-PDF media type', async () => { + const chunks = await runChat([ + { + type: 'document', + source: { + type: 'data', + value: `data:image/png;base64,${TINY_PDF_BASE64}`, + mimeType: 'application/pdf', + }, + }, + ]) + + const errorChunk = chunks.find((c) => c.type === 'RUN_ERROR') + expect(errorChunk).toBeDefined() + if (errorChunk?.type === 'RUN_ERROR') { + expect(errorChunk.message).toMatch(/only support application\/pdf/) + expect(errorChunk.message).toMatch(/non-PDF media type/) + } + }) + + it('rejects a data URL whose media type merely extends application/pdf', async () => { + const chunks = await runChat([ + { + type: 'document', + source: { + type: 'data', + value: `data:application/pdf+xml;base64,${TINY_PDF_BASE64}`, + mimeType: 'application/pdf', + }, + }, + ]) + + const errorChunk = chunks.find((c) => c.type === 'RUN_ERROR') + expect(errorChunk).toBeDefined() + if (errorChunk?.type === 'RUN_ERROR') { + expect(errorChunk.message).toMatch(/non-PDF media type/) + } + }) + + it('accepts case-insensitive PDF media types (RFC 2045)', async () => { + const dataUrl = `data:APPLICATION/PDF;base64,${TINY_PDF_BASE64}` + await runChat([ + { + type: 'document', + source: { type: 'data', value: dataUrl, mimeType: 'Application/PDF' }, + }, + ]) + + const [payload] = mockResponsesCreate.mock.calls[0]! + expect(payload.input[0].content[0].file_data).toBe(dataUrl) + }) + + it('passes metadata.detail through on both source shapes', async () => { + await runChat([ + { + type: 'document', + source: { + type: 'data', + value: TINY_PDF_BASE64, + mimeType: 'application/pdf', + }, + metadata: { detail: 'high' }, + }, + { + type: 'document', + source: { type: 'url', value: 'https://example.com/doc.pdf' }, + metadata: { detail: 'low' }, + }, + ]) + + const [payload] = mockResponsesCreate.mock.calls[0]! + expect(payload.input[0].content[0].detail).toBe('high') + expect(payload.input[0].content[1]).toEqual({ + type: 'input_file', + file_url: 'https://example.com/doc.pdf', + detail: 'low', + }) + }) + + it('omits detail when metadata does not provide it', async () => { + await runChat([ + { + type: 'document', + source: { + type: 'data', + value: TINY_PDF_BASE64, + mimeType: 'application/pdf', + }, + }, + ]) + + const [payload] = mockResponsesCreate.mock.calls[0]! + expect(payload.input[0].content[0]).not.toHaveProperty('detail') + }) + + it('still rejects video content parts', async () => { + const chunks = await runChat([ + { + type: 'video', + source: { type: 'url', value: 'https://example.com/clip.mp4' }, + }, + ]) + + const errorChunk = chunks.find((c) => c.type === 'RUN_ERROR') + expect(errorChunk).toBeDefined() + if (errorChunk?.type === 'RUN_ERROR') { + expect(errorChunk.message).toMatch( + /Unsupported content part type: video/, + ) + } + }) + + it('keeps text and document parts in order within one message', async () => { + await runChat([ + { type: 'text', content: 'summarize this' }, + { + type: 'document', + source: { + type: 'data', + value: TINY_PDF_BASE64, + mimeType: 'application/pdf', + }, + }, + ]) + + const [payload] = mockResponsesCreate.mock.calls[0]! + expect(payload.input[0].content).toEqual([ + { type: 'input_text', text: 'summarize this' }, + { + type: 'input_file', + filename: 'document.pdf', + file_data: `data:application/pdf;base64,${TINY_PDF_BASE64}`, + }, + ]) + }) + + it('converts a document part inside a tool result', async () => { + setupMockResponsesClient(minimalStreamChunks) + const adapter = new TestResponsesAdapter(testConfig, 'test-model') + const chunks: Array = [] + for await (const chunk of adapter.chatStream({ + logger: testLogger, + model: 'test-model', + messages: [ + { role: 'user', content: 'fetch the doc' }, + { + role: 'assistant', + content: '', + toolCalls: [ + { + id: 'call_doc', + type: 'function', + function: { name: 'fetch_doc', arguments: '{}' }, + }, + ], + }, + { + role: 'tool', + toolCallId: 'call_doc', + content: [ + { type: 'text', content: 'here it is' }, + { + type: 'document', + source: { + type: 'data', + value: TINY_PDF_BASE64, + mimeType: 'application/pdf', + }, + }, + ], + }, + ], + })) { + chunks.push(chunk) + } + + const [payload] = mockResponsesCreate.mock.calls[0]! + const out = payload.input.find( + (i: any) => i.type === 'function_call_output', + ) + expect(out.output).toEqual([ + { type: 'input_text', text: 'here it is' }, + { + type: 'input_file', + filename: 'document.pdf', + file_data: `data:application/pdf;base64,${TINY_PDF_BASE64}`, + }, + ]) + }) + }) + describe('subclassing', () => { it('allows subclassing with custom name', () => { class MyProviderAdapter extends OpenAIBaseResponsesTextAdapter { diff --git a/testing/e2e/README.md b/testing/e2e/README.md index cc06de6ca..0176ea3da 100644 --- a/testing/e2e/README.md +++ b/testing/e2e/README.md @@ -28,6 +28,7 @@ Each test iterates over supported providers using `providersFor('feature')`: | agentic-structured | 7 | `tests/agentic-structured.spec.ts` | | reasoning | 3 | `tests/reasoning.spec.ts` | | multimodal-image | 5 | `tests/multimodal-image.spec.ts` | +| multimodal-document | 1 | `tests/multimodal-document.spec.ts` | | multimodal-structured | 5 | `tests/multimodal-structured.spec.ts` | | summarize | 6 | `tests/summarize.spec.ts` | | summarize-stream | 6 | `tests/summarize-stream.spec.ts` | diff --git a/testing/e2e/fixtures/multimodal-document/basic.json b/testing/e2e/fixtures/multimodal-document/basic.json new file mode 100644 index 000000000..13d5968dc --- /dev/null +++ b/testing/e2e/fixtures/multimodal-document/basic.json @@ -0,0 +1,12 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "[mmdocument] summarize this pdf" + }, + "response": { + "content": "The PDF contains a single blank page with no text content." + } + } + ] +} diff --git a/testing/e2e/src/components/ChatUI.tsx b/testing/e2e/src/components/ChatUI.tsx index dd5c38591..01b8bc807 100644 --- a/testing/e2e/src/components/ChatUI.tsx +++ b/testing/e2e/src/components/ChatUI.tsx @@ -13,11 +13,13 @@ interface ChatUIProps { isLoading: boolean onSendMessage: (text: string) => void onSendMessageWithImage?: (text: string, file: File) => void + onSendMessageWithDocument?: (text: string, file: File) => void addToolApprovalResponse?: (response: { id: string approved: boolean }) => Promise showImageInput?: boolean + showDocumentInput?: boolean onStop?: () => void /** When the streaming structured-output CUSTOM event lands, the page * exposes the parsed object here so e2e tests can assert that the event @@ -40,8 +42,10 @@ export function ChatUI({ isLoading, onSendMessage, onSendMessageWithImage, + onSendMessageWithDocument, addToolApprovalResponse, showImageInput, + showDocumentInput, onStop, structuredObject, contentDeltaCount, @@ -257,6 +261,23 @@ export function ChatUI({ }} /> )} + {showDocumentInput && ( + { + const file = e.target.files?.[0] + // Same DOM-value read as the image input above. + const text = (inputRef.current?.value ?? input).trim() + if (file && text && onSendMessageWithDocument) { + onSendMessageWithDocument(text, file) + setInput('') + } + }} + /> + )} > = { 'grok', 'openrouter', ]), + // OpenAI only: this feature exercises the Responses adapter's PDF + // `input_file` conversion (base64 `file_data` + filename). + 'multimodal-document': new Set(['openai']), // Bedrock excluded: same text-only default e2e model as multimodal-image above. 'multimodal-structured': new Set([ 'openai', diff --git a/testing/e2e/src/lib/features.ts b/testing/e2e/src/lib/features.ts index f6a0b57d1..9c96948c2 100644 --- a/testing/e2e/src/lib/features.ts +++ b/testing/e2e/src/lib/features.ts @@ -88,6 +88,10 @@ export const featureConfigs: Record = { tools: [], modelOptions: {}, }, + 'multimodal-document': { + tools: [], + modelOptions: {}, + }, 'multimodal-structured': { tools: [], modelOptions: {}, diff --git a/testing/e2e/src/lib/types.ts b/testing/e2e/src/lib/types.ts index 0e362b5b1..9017f91f4 100644 --- a/testing/e2e/src/lib/types.ts +++ b/testing/e2e/src/lib/types.ts @@ -30,6 +30,7 @@ export type Feature = | 'agentic-structured' | 'agentic-structured-stream' | 'multimodal-image' + | 'multimodal-document' | 'multimodal-structured' | 'summarize' | 'summarize-stream' @@ -76,6 +77,7 @@ export const ALL_FEATURES: Feature[] = [ 'agentic-structured', 'agentic-structured-stream', 'multimodal-image', + 'multimodal-document', 'multimodal-structured', 'summarize', 'summarize-stream', diff --git a/testing/e2e/src/routes/$provider/$feature.tsx b/testing/e2e/src/routes/$provider/$feature.tsx index 85af30d0a..cae74a205 100644 --- a/testing/e2e/src/routes/$provider/$feature.tsx +++ b/testing/e2e/src/routes/$provider/$feature.tsx @@ -223,6 +223,7 @@ function ChatFeature({ const needsApproval = feature === 'tool-approval' const showImageInput = feature === 'multimodal-image' || feature === 'multimodal-structured' + const showDocumentInput = feature === 'multimodal-document' const tools = needsApproval ? clientTools(addToCartClient) : undefined @@ -388,10 +389,36 @@ function ChatFeature({ } : undefined } + onSendMessageWithDocument={ + showDocumentInput + ? (text, file) => { + const reader = new FileReader() + reader.onload = () => { + const base64 = (reader.result as string).split(',')[1] + sendMessage({ + content: [ + { type: 'text', content: text }, + { + type: 'document', + source: { + type: 'data', + value: base64, + mimeType: file.type, + }, + metadata: { filename: file.name }, + }, + ], + }) + } + reader.readAsDataURL(file) + } + : undefined + } addToolApprovalResponse={ needsApproval ? addToolApprovalResponse : undefined } showImageInput={showImageInput} + showDocumentInput={showDocumentInput} onStop={stop} /> diff --git a/testing/e2e/test-assets/tiny.pdf b/testing/e2e/test-assets/tiny.pdf new file mode 100644 index 0000000000000000000000000000000000000000..0909dcbe357bee6592f36c82ba03bc55cbd59e1a GIT binary patch literal 192 zcmY!laB+gRTQ)d& literal 0 HcmV?d00001 diff --git a/testing/e2e/tests/helpers.ts b/testing/e2e/tests/helpers.ts index ff4d0ebcd..55dd40c93 100644 --- a/testing/e2e/tests/helpers.ts +++ b/testing/e2e/tests/helpers.ts @@ -64,6 +64,27 @@ export async function sendMessageWithImage( }).toPass({ timeout: 15_000, intervals: [250, 500, 1000] }) } +export async function sendMessageWithDocument( + page: Page, + text: string, + documentPath: string, +) { + const input = page.getByTestId('chat-input') + const fileInput = page.getByTestId('document-attachment-input') + const userMessages = page.getByTestId('user-message') + + // Same retry-to-observable-outcome pattern as sendMessageWithImage above. + await expect(async () => { + await input.click() + await input.fill('') + await input.pressSequentially(text, { delay: 15 }) + expect(await input.inputValue()).toBe(text) + await fileInput.setInputFiles([]) + await fileInput.setInputFiles(documentPath) + await expect(userMessages.first()).toBeVisible({ timeout: 2_000 }) + }).toPass({ timeout: 15_000, intervals: [250, 500, 1000] }) +} + export async function waitForResponse(page: Page, timeout = 15_000) { try { await page diff --git a/testing/e2e/tests/multimodal-document.spec.ts b/testing/e2e/tests/multimodal-document.spec.ts new file mode 100644 index 000000000..39fc6c6b2 --- /dev/null +++ b/testing/e2e/tests/multimodal-document.spec.ts @@ -0,0 +1,38 @@ +import { test, expect } from './fixtures' +import { + sendMessageWithDocument, + waitForResponse, + getLastAssistantMessage, + featureUrl, +} from './helpers' +import { providersFor } from './test-matrix' +import { fileURLToPath } from 'url' +import path from 'path' + +const __filename = fileURLToPath(import.meta.url) +const __dirname = path.dirname(__filename) +const testDocumentPath = path.resolve(__dirname, '../test-assets/tiny.pdf') + +for (const provider of providersFor('multimodal-document')) { + test.describe(`${provider} — multimodal-document`, () => { + test('answers a question about an uploaded PDF', async ({ + page, + testId, + aimockPort, + }) => { + await page.goto( + featureUrl(provider, 'multimodal-document', testId, aimockPort), + ) + + await sendMessageWithDocument( + page, + '[mmdocument] summarize this pdf', + testDocumentPath, + ) + await waitForResponse(page) + + const response = await getLastAssistantMessage(page) + expect(response).toContain('blank page') + }) + }) +} From b20a655dfced4c8910a6787228dbd256c211b00a Mon Sep 17 00:00:00 2001 From: daniel Date: Mon, 20 Jul 2026 17:09:24 -0700 Subject: [PATCH 2/4] docs(openai): document the PDF document-input support Add a changeset (minor bump for `@tanstack/openai-base` and `@tanstack/ai-openai`) and a multimodal-content docs example for the new `document` content part support on the Responses adapter. The example uses `gpt-5.5`, one of the models that declares the `document` input modality; the "Supported modalities by model" list is expanded to include it and points readers at `model-meta.ts` for the full, authoritative list. --- .changeset/openai-pdf-document-input.md | 26 ++++++++++++++++++++++++ docs/advanced/multimodal-content.md | 27 ++++++++++++++++++++++--- docs/config.json | 3 ++- 3 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 .changeset/openai-pdf-document-input.md diff --git a/.changeset/openai-pdf-document-input.md b/.changeset/openai-pdf-document-input.md new file mode 100644 index 000000000..18bf050d5 --- /dev/null +++ b/.changeset/openai-pdf-document-input.md @@ -0,0 +1,26 @@ +--- +'@tanstack/openai-base': minor +'@tanstack/ai-openai': minor +--- + +feat(ai-openai): support PDF `document` content parts in the Responses adapter. + +`openaiText`'s Responses adapter now accepts PDF `document` content parts, for any model whose `model-meta` entry declares the `document` input modality. Base64 data sources are sent as `input_file` with a `file_data` data URL and a `filename` (from `metadata.filename`, defaulting to `document.pdf`). URL sources are sent as `input_file` with `file_url`. + +```ts +const adapter = openaiText('gpt-5.5') + +const message = { + role: 'user', + content: [ + { type: 'text', content: 'Summarize this document' }, + { + type: 'document', + source: { type: 'data', value: pdfBase64, mimeType: 'application/pdf' }, + metadata: { filename: 'report.pdf' }, + }, + ], +} +``` + +Non-PDF MIME types are rejected before the request is sent — including pre-wrapped `data:` URLs whose media type disagrees with `mimeType` — so callers get an actionable message instead of an opaque provider `400`. `OpenAIDocumentMetadata` gains `filename` and `detail`. The Chat Completions adapter throws a document-specific error pointing here, since documents are Responses-only. diff --git a/docs/advanced/multimodal-content.md b/docs/advanced/multimodal-content.md index 0bea1a795..baa86b534 100644 --- a/docs/advanced/multimodal-content.md +++ b/docs/advanced/multimodal-content.md @@ -94,11 +94,12 @@ const response = await chat({ ### OpenAI -OpenAI supports images and audio in their vision and audio models: +OpenAI supports images, audio, and PDF documents in their vision, audio, and +document-capable models: ```typescript import { openaiText } from '@tanstack/ai-openai' -import { imageBase64 } from './data' +import { imageBase64, pdfBase64 } from './data' const adapter = openaiText('gpt-5.5') @@ -116,10 +117,30 @@ const message = { } ``` +```typescript +import { pdfBase64 } from './data' + +// PDF document via base64 data (the API requires a filename alongside +// inline data; defaults to "document.pdf" when omitted) +const documentMessage = { + role: 'user', + content: [ + { type: 'text', content: 'Summarize this document' }, + { + type: 'document', + source: { type: 'data', value: pdfBase64, mimeType: 'application/pdf' }, + metadata: { filename: 'report.pdf' } + } + ] +} +``` + **Supported modalities by model:** -- `gpt-5.2`, `gpt-5-mini`: text, image +- `gpt-5.5`, `gpt-5.2`, `gpt-5-mini` (among others): text, image, PDF document - `gpt-4o-audio`: text, audio +Check each model's `supports.input` in `@tanstack/ai-openai`'s `model-meta.ts` for the authoritative per-model list. + ### Anthropic Anthropic's Claude models support images and PDF documents: diff --git a/docs/config.json b/docs/config.json index 55a365282..777712add 100644 --- a/docs/config.json +++ b/docs/config.json @@ -439,7 +439,8 @@ { "label": "Multimodal Content", "to": "advanced/multimodal-content", - "addedAt": "2026-04-15" + "addedAt": "2026-04-15", + "updatedAt": "2026-07-21" }, { "label": "Per-Model Type Safety", From e5d210eba0d335935475ee67637ff6076ddc00c8 Mon Sep 17 00:00:00 2001 From: daniel Date: Tue, 21 Jul 2026 11:02:44 -0700 Subject: [PATCH 3/4] fix(openai): verify inline document data is a real PDF and allow detail 'auto' Addresses CodeRabbit review on #977. Inline (base64) document data must now start with the '%PDF' file header, so non-PDF data sent without a mimeType, or mislabeled as application/pdf, is rejected locally instead of failing as an opaque provider 400. URL sources are unchanged (still validated server-side). Adds regression tests for the missing-mimeType, mislabeled, and non-PDF-data-URL cases. Widens OpenAIDocumentMetadata.detail to 'auto' | 'low' | 'high' to match the API, and casts at the payload boundary because the pinned OpenAI SDK type still lists only 'low' | 'high'. Fixes the doc comments that claimed the default was 'low' (the actual default, 'auto', is model-dependent). Adds docstrings to the e2e document/image send helpers and the ChatUI props to cover the docstring pre-merge check. --- .changeset/openai-pdf-document-input.md | 2 +- packages/ai-openai/src/message-types.ts | 15 ++- .../src/adapters/responses-text.ts | 30 +++++- .../openai-base/tests/responses-text.test.ts | 94 +++++++++++++++++++ testing/e2e/src/components/ChatUI.tsx | 4 + testing/e2e/tests/helpers.ts | 5 + 6 files changed, 140 insertions(+), 10 deletions(-) diff --git a/.changeset/openai-pdf-document-input.md b/.changeset/openai-pdf-document-input.md index 18bf050d5..7fe17da1c 100644 --- a/.changeset/openai-pdf-document-input.md +++ b/.changeset/openai-pdf-document-input.md @@ -23,4 +23,4 @@ const message = { } ``` -Non-PDF MIME types are rejected before the request is sent — including pre-wrapped `data:` URLs whose media type disagrees with `mimeType` — so callers get an actionable message instead of an opaque provider `400`. `OpenAIDocumentMetadata` gains `filename` and `detail`. The Chat Completions adapter throws a document-specific error pointing here, since documents are Responses-only. +Non-PDF MIME types are rejected before the request is sent — including pre-wrapped `data:` URLs whose media type disagrees with `mimeType` — so callers get an actionable message instead of an opaque provider `400`. Inline base64 payloads are also checked for the `%PDF` magic bytes, so non-PDF data mislabeled (or sent without) a PDF `mimeType` is rejected locally. `OpenAIDocumentMetadata` gains `filename` and `detail`. The Chat Completions adapter throws a document-specific error pointing here, since documents are Responses-only. diff --git a/packages/ai-openai/src/message-types.ts b/packages/ai-openai/src/message-types.ts index 57c9b7e04..19eea009f 100644 --- a/packages/ai-openai/src/message-types.ts +++ b/packages/ai-openai/src/message-types.ts @@ -47,9 +47,11 @@ export interface OpenAIVideoMetadata {} * * Documents are supported by the Responses adapter only, which sends them * as `input_file`; the Chat Completions adapter rejects document parts. - * This adapter currently supports only `application/pdf` documents. That is - * enforced locally for inline (base64) data; URL-sourced documents are sent - * to OpenAI unvalidated, so a non-PDF URL fails server-side instead. + * This adapter currently supports only `application/pdf` documents. Inline + * (base64) data is verified locally — both the declared/embedded MIME type + * and the `%PDF` content header, so mislabeled non-PDF data is rejected + * before the request. URL-sourced documents are sent to OpenAI unvalidated, + * so a non-PDF URL fails server-side instead. * * @see https://developers.openai.com/api/docs/guides/pdf-files */ @@ -61,9 +63,12 @@ export interface OpenAIDocumentMetadata { filename?: string /** * Rendering quality for the file's page images. Omitted by default so the - * API applies its own default ('low'). + * API applies its own default ('auto'), which resolves to 'low' or 'high' + * depending on the model. + * + * @see https://developers.openai.com/api/docs/guides/pdf-files */ - detail?: 'low' | 'high' + detail?: 'auto' | 'low' | 'high' } /** diff --git a/packages/openai-base/src/adapters/responses-text.ts b/packages/openai-base/src/adapters/responses-text.ts index bf2a34c1e..06f9f3868 100644 --- a/packages/openai-base/src/adapters/responses-text.ts +++ b/packages/openai-base/src/adapters/responses-text.ts @@ -31,6 +31,10 @@ import type { TextOptions, } from '@tanstack/ai' +// Base64 encoding of the '%PDF' file header — every PDF payload starts with +// these bytes, so inline document data must begin with this prefix. +const PDF_BASE64_MAGIC = 'JVBERi' + /** * Shared implementation of the OpenAI Responses API. Holds the stream-event * accumulator + AG-UI lifecycle and calls the OpenAI SDK directly. Subclasses @@ -1841,13 +1845,16 @@ export abstract class OpenAIBaseResponsesTextAdapter< case 'document': { const documentMetadata = part.metadata as - | { filename?: string; detail?: 'low' | 'high' } + | { filename?: string; detail?: 'auto' | 'low' | 'high' } | undefined - // `detail` is optional with no 'auto' member (unlike input_image) — - // spread only when provided so the API applies its own default. + // Spread `detail` only when provided so the API applies its own + // default ('auto'). The Responses API accepts 'auto' | 'low' | 'high', + // but the pinned OpenAI SDK's `ResponseInputFile.detail` type still + // lists only 'low' | 'high' — cast so 'auto' (a valid API value) can + // pass through without a type error. const documentDetail = documentMetadata?.detail !== undefined - ? { detail: documentMetadata.detail } + ? { detail: documentMetadata.detail as 'low' | 'high' } : {} if (part.source.type === 'url') { // The Responses API fetches the PDF itself; filename and MIME @@ -1881,6 +1888,21 @@ export abstract class OpenAIBaseResponsesTextAdapter< `(received data URL with non-PDF media type)`, ) } + // Sniff the payload so a non-PDF labeled (or unlabeled) as PDF is + // caught locally instead of by an opaque provider 400. Only base64 + // payloads are checked; a rare `data:` URL without `;base64` is left + // to the server. + const documentBase64 = documentValue.startsWith('data:') + ? /;base64,/i.test(documentValue) + ? documentValue.slice(documentValue.indexOf(',') + 1) + : '' + : documentValue + if (documentBase64 && !documentBase64.startsWith(PDF_BASE64_MAGIC)) { + throw new Error( + `${this.name} document parts only support application/pdf ` + + `(inline data does not start with the %PDF header)`, + ) + } // Wrap raw base64 in a data URL — `input_file` rejects bare base64 // payloads (matches the image and audio branches above). const documentFileData = documentValue.startsWith('data:') diff --git a/packages/openai-base/tests/responses-text.test.ts b/packages/openai-base/tests/responses-text.test.ts index ccad68416..4ffc0db6f 100644 --- a/packages/openai-base/tests/responses-text.test.ts +++ b/packages/openai-base/tests/responses-text.test.ts @@ -2248,6 +2248,9 @@ describe('OpenAIBaseResponsesTextAdapter', () => { 'trailer<>\n%%EOF', ).toString('base64') + // A payload that is not a PDF (no '%PDF' header). + const NOT_PDF_BASE64 = Buffer.from('hello, not a pdf').toString('base64') + const minimalStreamChunks = [ { type: 'response.created', @@ -2448,6 +2451,23 @@ describe('OpenAIBaseResponsesTextAdapter', () => { }) }) + it("passes detail: 'auto' through to the payload", async () => { + await runChat([ + { + type: 'document', + source: { + type: 'data', + value: TINY_PDF_BASE64, + mimeType: 'application/pdf', + }, + metadata: { detail: 'auto' }, + }, + ]) + + const [payload] = mockResponsesCreate.mock.calls[0]! + expect(payload.input[0].content[0].detail).toBe('auto') + }) + it('omits detail when metadata does not provide it', async () => { await runChat([ { @@ -2558,6 +2578,80 @@ describe('OpenAIBaseResponsesTextAdapter', () => { }, ]) }) + + it('emits RUN_ERROR for raw base64 without a mimeType that is not a PDF', async () => { + const chunks = await runChat([ + { + type: 'document', + source: { type: 'data', value: NOT_PDF_BASE64 }, + }, + ]) + + const errorChunk = chunks.find((c) => c.type === 'RUN_ERROR') + expect(errorChunk).toBeDefined() + if (errorChunk?.type === 'RUN_ERROR') { + expect(errorChunk.message).toMatch(/only support application\/pdf/) + expect(errorChunk.message).toMatch(/%PDF/) + } + }) + + it('accepts raw base64 PDF data without a mimeType', async () => { + await runChat([ + { + type: 'document', + source: { type: 'data', value: TINY_PDF_BASE64 }, + }, + ]) + + const [payload] = mockResponsesCreate.mock.calls[0]! + expect(payload.input[0].content).toEqual([ + { + type: 'input_file', + filename: 'document.pdf', + file_data: `data:application/pdf;base64,${TINY_PDF_BASE64}`, + }, + ]) + }) + + it('emits RUN_ERROR for non-PDF data mislabeled as application/pdf', async () => { + const chunks = await runChat([ + { + type: 'document', + source: { + type: 'data', + value: NOT_PDF_BASE64, + mimeType: 'application/pdf', + }, + }, + ]) + + const errorChunk = chunks.find((c) => c.type === 'RUN_ERROR') + expect(errorChunk).toBeDefined() + if (errorChunk?.type === 'RUN_ERROR') { + expect(errorChunk.message).toMatch(/only support application\/pdf/) + expect(errorChunk.message).toMatch(/%PDF/) + } + }) + + it('emits RUN_ERROR for a PDF data URL whose payload is not a PDF', async () => { + const chunks = await runChat([ + { + type: 'document', + source: { + type: 'data', + value: `data:application/pdf;base64,${NOT_PDF_BASE64}`, + mimeType: 'application/pdf', + }, + }, + ]) + + const errorChunk = chunks.find((c) => c.type === 'RUN_ERROR') + expect(errorChunk).toBeDefined() + if (errorChunk?.type === 'RUN_ERROR') { + expect(errorChunk.message).toMatch(/only support application\/pdf/) + expect(errorChunk.message).toMatch(/%PDF/) + } + }) }) describe('subclassing', () => { diff --git a/testing/e2e/src/components/ChatUI.tsx b/testing/e2e/src/components/ChatUI.tsx index 01b8bc807..d1bee7835 100644 --- a/testing/e2e/src/components/ChatUI.tsx +++ b/testing/e2e/src/components/ChatUI.tsx @@ -12,13 +12,17 @@ interface ChatUIProps { messages: Array isLoading: boolean onSendMessage: (text: string) => void + /** Sends the typed prompt plus an attached image as an image content part. */ onSendMessageWithImage?: (text: string, file: File) => void + /** Sends the typed prompt plus an attached PDF as a document content part. */ onSendMessageWithDocument?: (text: string, file: File) => void addToolApprovalResponse?: (response: { id: string approved: boolean }) => Promise + /** Renders the image file input (multimodal image features only). */ showImageInput?: boolean + /** Renders the PDF file input (multimodal-document feature only). */ showDocumentInput?: boolean onStop?: () => void /** When the streaming structured-output CUSTOM event lands, the page diff --git a/testing/e2e/tests/helpers.ts b/testing/e2e/tests/helpers.ts index 55dd40c93..de8ddfb72 100644 --- a/testing/e2e/tests/helpers.ts +++ b/testing/e2e/tests/helpers.ts @@ -31,6 +31,8 @@ export async function sendMessage(page: Page, text: string) { }) } +/** Types a prompt and attaches an image; attaching auto-sends. Retries until + * the user bubble renders — the inline comment below explains why. */ export async function sendMessageWithImage( page: Page, text: string, @@ -64,6 +66,9 @@ export async function sendMessageWithImage( }).toPass({ timeout: 15_000, intervals: [250, 500, 1000] }) } +/** Types a prompt and attaches a PDF; attaching auto-sends. Retries until the + * user bubble renders (same fragile-controlled-input problem as + * sendMessageWithImage — see the comment there). */ export async function sendMessageWithDocument( page: Page, text: string, From 1c39896c2161f7c88c08582ccac9eabf15b62952 Mon Sep 17 00:00:00 2001 From: daniel Date: Tue, 21 Jul 2026 11:16:50 -0700 Subject: [PATCH 4/4] fix(e2e): clear the document file input after sending --- testing/e2e/src/components/ChatUI.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/testing/e2e/src/components/ChatUI.tsx b/testing/e2e/src/components/ChatUI.tsx index d1bee7835..226a0f8c4 100644 --- a/testing/e2e/src/components/ChatUI.tsx +++ b/testing/e2e/src/components/ChatUI.tsx @@ -278,6 +278,7 @@ export function ChatUI({ if (file && text && onSendMessageWithDocument) { onSendMessageWithDocument(text, file) setInput('') + e.target.value = '' } }} />