Skip to content

Commit 260c0fc

Browse files
dangreenberggitdangreenberggit
committed
feat(openai): support PDF document content parts in the Responses adapter
OpenAI chat models that declare the `document` input modality now accept PDF `document` content parts through `openaiText` on the Responses adapter. Base64 data sources are sent as `input_file` with a `file_data` data URL and a `filename` (from `metadata.filename`, defaulting to `document.pdf`); URL sources are sent as `input_file` with `file_url`. The adapter rejects non-PDF document MIME types with a clear error before the request goes out — including pre-wrapped `data:` URLs whose media type disagrees with `mimeType` — so callers get an actionable message instead of an opaque provider 400. `OpenAIDocumentMetadata` gains `filename` and `detail`, and the document-capable chat models declare the `document` input modality. The Chat Completions adapter throws a document-specific error pointing at the Responses adapter, since documents are only supported there. Adds unit coverage for the Responses and Chat Completions adapters, and an e2e fixture/spec/page for the multimodal-document feature.
1 parent bcb1b93 commit 260c0fc

17 files changed

Lines changed: 694 additions & 122 deletions

File tree

‎packages/ai-openai/src/message-types.ts‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,27 @@ export interface OpenAIVideoMetadata {}
4444

4545
/**
4646
* Metadata for OpenAI document content parts.
47-
* Note: Direct document support may vary; PDFs often need to be converted to images.
47+
*
48+
* Documents are supported by the Responses adapter only, which sends them
49+
* as `input_file`; the Chat Completions adapter rejects document parts.
50+
* This adapter currently supports only `application/pdf` documents. That is
51+
* enforced locally for inline (base64) data; URL-sourced documents are sent
52+
* to OpenAI unvalidated, so a non-PDF URL fails server-side instead.
53+
*
54+
* @see https://developers.openai.com/api/docs/guides/pdf-files
4855
*/
49-
export interface OpenAIDocumentMetadata {}
56+
export interface OpenAIDocumentMetadata {
57+
/**
58+
* Filename sent alongside inline (base64) PDF data.
59+
* @default 'document.pdf'
60+
*/
61+
filename?: string
62+
/**
63+
* Rendering quality for the file's page images. Omitted by default so the
64+
* API applies its own default ('low').
65+
*/
66+
detail?: 'low' | 'high'
67+
}
5068

5169
/**
5270
* Metadata for OpenAI text content parts.

‎packages/ai-openai/src/model-meta.ts‎

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import type {
1111
interface ModelMeta<TProviderOptions = unknown> {
1212
name: string
1313
supports: {
14-
input: Array<'text' | 'image' | 'audio' | 'video'>
14+
input: Array<'text' | 'image' | 'audio' | 'video' | 'document'>
1515
output: Array<'text' | 'image' | 'audio' | 'video'>
1616
endpoints: Array<
1717
| 'chat'
@@ -74,7 +74,7 @@ const GPT5_2 = {
7474
max_output_tokens: 128_000,
7575
knowledge_cutoff: '2025-08-31',
7676
supports: {
77-
input: ['text', 'image'],
77+
input: ['text', 'image', 'document'],
7878
output: ['text'],
7979
endpoints: ['chat', 'chat-completions'],
8080
features: [
@@ -119,7 +119,7 @@ const GPT5_2_PRO = {
119119
max_output_tokens: 128_000,
120120
knowledge_cutoff: '2025-08-31',
121121
supports: {
122-
input: ['text', 'image'],
122+
input: ['text', 'image', 'document'],
123123
output: ['text'],
124124
endpoints: ['chat', 'chat-completions'],
125125
features: ['streaming', 'function_calling'],
@@ -158,7 +158,7 @@ const GPT5_2_CHAT = {
158158
max_output_tokens: 16_384,
159159
knowledge_cutoff: '2025-08-31',
160160
supports: {
161-
input: ['text', 'image'],
161+
input: ['text', 'image', 'document'],
162162
output: ['text'],
163163
endpoints: ['chat', 'chat-completions'],
164164
features: ['streaming', 'function_calling', 'structured_outputs'],
@@ -194,7 +194,7 @@ const GPT5_1 = {
194194
max_output_tokens: 128_000,
195195
knowledge_cutoff: '2024-09-30',
196196
supports: {
197-
input: ['text', 'image'],
197+
input: ['text', 'image', 'document'],
198198
output: ['text', 'image'],
199199
endpoints: ['chat', 'chat-completions'],
200200
features: [
@@ -240,7 +240,7 @@ const GPT5_1_CODEX = {
240240
max_output_tokens: 128_000,
241241
knowledge_cutoff: '2024-09-30',
242242
supports: {
243-
input: ['text', 'image'],
243+
input: ['text', 'image', 'document'],
244244
output: ['text', 'image'],
245245
endpoints: ['chat'],
246246
features: ['streaming', 'function_calling', 'structured_outputs'],
@@ -277,7 +277,7 @@ const GPT5 = {
277277
max_output_tokens: 128_000,
278278
knowledge_cutoff: '2024-09-30',
279279
supports: {
280-
input: ['text', 'image'],
280+
input: ['text', 'image', 'document'],
281281
output: ['text'],
282282
endpoints: ['chat', 'chat-completions', 'batch'],
283283
features: [
@@ -323,7 +323,7 @@ const GPT5_MINI = {
323323
max_output_tokens: 128_000,
324324
knowledge_cutoff: '2024-05-31',
325325
supports: {
326-
input: ['text', 'image'],
326+
input: ['text', 'image', 'document'],
327327
output: ['text'],
328328
endpoints: ['chat', 'chat-completions', 'batch'],
329329
features: ['streaming', 'structured_outputs', 'function_calling'],
@@ -373,7 +373,7 @@ const GPT5_NANO = {
373373
},
374374
},
375375
supports: {
376-
input: ['text', 'image'],
376+
input: ['text', 'image', 'document'],
377377
output: ['text'],
378378
endpoints: ['chat', 'chat-completions', 'batch'],
379379
features: ['streaming', 'structured_outputs', 'function_calling'],
@@ -413,7 +413,7 @@ const GPT5_PRO = {
413413
},
414414
},
415415
supports: {
416-
input: ['text', 'image'],
416+
input: ['text', 'image', 'document'],
417417
output: ['text'],
418418
endpoints: ['chat', 'batch'],
419419
features: ['streaming', 'structured_outputs', 'function_calling'],
@@ -454,7 +454,7 @@ const GPT5_CODEX = {
454454
},
455455
},
456456
supports: {
457-
input: ['text', 'image'],
457+
input: ['text', 'image', 'document'],
458458
output: ['text', 'image'],
459459
endpoints: ['chat'],
460460
features: ['streaming', 'structured_outputs', 'function_calling'],
@@ -686,7 +686,7 @@ const O3_PRO = {
686686
},
687687
},
688688
supports: {
689-
input: ['text', 'image'],
689+
input: ['text', 'image', 'document'],
690690
output: ['text'],
691691
endpoints: ['chat', 'batch'],
692692
features: ['function_calling', 'structured_outputs'],
@@ -839,7 +839,7 @@ const O3 = {
839839
},
840840
},
841841
supports: {
842-
input: ['text', 'image'],
842+
input: ['text', 'image', 'document'],
843843
output: ['text'],
844844
endpoints: ['chat', 'batch', 'chat-completions'],
845845
features: ['function_calling', 'structured_outputs', 'streaming'],
@@ -880,7 +880,7 @@ const O4_MINI = {
880880
},
881881
},
882882
supports: {
883-
input: ['text', 'image'],
883+
input: ['text', 'image', 'document'],
884884
output: ['text'],
885885
endpoints: ['chat', 'batch', 'chat-completions', 'fine-tuning'],
886886
features: [
@@ -925,7 +925,7 @@ const GPT4_1 = {
925925
},
926926
},
927927
supports: {
928-
input: ['text', 'image'],
928+
input: ['text', 'image', 'document'],
929929
output: ['text'],
930930
endpoints: [
931931
'chat',
@@ -978,7 +978,7 @@ const GPT4_1_MINI = {
978978
},
979979
},
980980
supports: {
981-
input: ['text', 'image'],
981+
input: ['text', 'image', 'document'],
982982
output: ['text'],
983983
endpoints: [
984984
'chat',
@@ -1029,7 +1029,7 @@ const GPT4_1_NANO = {
10291029
},
10301030
},
10311031
supports: {
1032-
input: ['text', 'image'],
1032+
input: ['text', 'image', 'document'],
10331033
output: ['text'],
10341034
endpoints: [
10351035
'chat',
@@ -1080,7 +1080,7 @@ const O1_PRO = {
10801080
},
10811081
},
10821082
supports: {
1083-
input: ['text', 'image'],
1083+
input: ['text', 'image', 'document'],
10841084
output: ['text'],
10851085
endpoints: ['chat', 'batch'],
10861086
features: ['function_calling', 'structured_outputs'],
@@ -1283,7 +1283,7 @@ const O1 = {
12831283
},
12841284
},
12851285
supports: {
1286-
input: ['text', 'image'],
1286+
input: ['text', 'image', 'document'],
12871287
output: ['text'],
12881288
endpoints: ['chat', 'batch', 'chat-completions', 'assistants'],
12891289
features: ['function_calling', 'structured_outputs', 'streaming'],
@@ -1333,7 +1333,7 @@ const GPT_4O = {
13331333
},
13341334
},
13351335
supports: {
1336-
input: ['text', 'image'],
1336+
input: ['text', 'image', 'document'],
13371337
output: ['text'],
13381338
endpoints: [
13391339
'chat',
@@ -1410,7 +1410,7 @@ const GPT_4O_MINI = {
14101410
},
14111411
},
14121412
supports: {
1413-
input: ['text', 'image'],
1413+
input: ['text', 'image', 'document'],
14141414
output: ['text'],
14151415
endpoints: [
14161416
'chat',
@@ -2048,7 +2048,7 @@ const GPT_5_5 = {
20482048
context_window: 1_050_000,
20492049
max_output_tokens: 128_000,
20502050
supports: {
2051-
input: ['image', 'text'],
2051+
input: ['text', 'image', 'document'],
20522052
output: ['text'],
20532053
endpoints: ['chat', 'chat-completions'],
20542054
features: [
@@ -2093,7 +2093,7 @@ const GPT_5_5_PRO = {
20932093
context_window: 1_050_000,
20942094
max_output_tokens: 128_000,
20952095
supports: {
2096-
input: ['image', 'text'],
2096+
input: ['text', 'image', 'document'],
20972097
output: ['text'],
20982098
endpoints: ['chat', 'chat-completions'],
20992099
features: [

0 commit comments

Comments
 (0)