Skip to content

fix: validate image-only attachments for OpenAI/xAI #389

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mikehostetler merged 1 commit into from
Jan 31, 2026
Merged

fix: validate image-only attachments for OpenAI/xAI #389

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions guides/openai.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@ Access GPT models including standard chat models and reasoning models (o1, o3, G
OPENAI_API_KEY=sk-...
```

## Attachments

OpenAI Chat Completions API only supports image attachments (JPEG, PNG, GIF, WebP).
For document support (PDFs, etc.), use Anthropic or Google providers.

## Dual API Architecture

OpenAI provider automatically routes between two APIs based on model metadata:
Expand Down
5 changes: 5 additions & 0 deletions guides/xai.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@ Access Grok models with real-time web search and reasoning capabilities.
XAI_API_KEY=xai-...
```

## Attachments

xAI Chat Completions API only supports image attachments (JPEG, PNG, GIF, WebP).
For document support (PDFs, etc.), use Anthropic or Google providers.

## Provider Options

Passed via `:provider_options` keyword:
Expand Down
36 changes: 36 additions & 0 deletions lib/req_llm/provider/defaults.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -739,6 +739,42 @@ defmodule ReqLLM.Provider.Defaults do

defp merge_content_metadata(base, _), do: base

@image_mimes ~w(image/jpeg image/png image/gif image/webp)

@doc """
Validates that a context contains only image file attachments.

Returns `:ok` if all file attachments are images (JPEG, PNG, GIF, WebP),
or `{:error, reason}` with a descriptive message if non-image files are found.

This is used by providers like OpenAI and xAI that only support image attachments
via their Chat Completions API.
"""
@spec validate_image_only_attachments(ReqLLM.Context.t()) :: :ok | {:error, String.t()}
def validate_image_only_attachments(%ReqLLM.Context{messages: messages}) do
non_image_parts =
messages
|> Enum.flat_map(fn msg -> msg.content || [] end)
|> Enum.filter(fn part ->
part.type == :file and part.media_type not in @image_mimes
end)

case non_image_parts do
[] ->
:ok

parts ->
mimes = parts |> Enum.map(& &1.media_type) |> Enum.uniq() |> Enum.join(", ")

{:error,
"This provider only supports image attachments (JPEG, PNG, GIF, WebP). " <>
"Found unsupported file types: #{mimes}. " <>
"Consider using Anthropic or Google for document support."}
end
end

def validate_image_only_attachments(_), do: :ok

@doc """
Decodes OpenAI-format response body to ReqLLM.Response.

Expand Down
11 changes: 11 additions & 0 deletions lib/req_llm/providers/openai.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -295,6 +295,7 @@ defmodule ReqLLM.Providers.OpenAI do
def prepare_request(:chat, model_spec, prompt, opts) do
with {:ok, model} <- ReqLLM.model(model_spec),
{:ok, context} <- ReqLLM.Context.normalize(prompt, opts),
:ok <- validate_attachments(context),
opts_with_context = Keyword.put(opts, :context, context),
http_opts = Keyword.get(opts, :req_http_options, []),
{:ok, processed_opts} <-
Expand Down Expand Up @@ -653,4 +654,14 @@ defmodule ReqLLM.Providers.OpenAI do
end

defp enforce_strict_schema_requirements(schema), do: schema

defp validate_attachments(context) do
case ReqLLM.Provider.Defaults.validate_image_only_attachments(context) do
:ok ->
:ok

{:error, message} ->
{:error, ReqLLM.Error.Invalid.Parameter.exception(parameter: message)}
end
end
end
11 changes: 11 additions & 0 deletions lib/req_llm/providers/xai.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,7 @@ defmodule ReqLLM.Providers.XAI do
defp prepare_chat_request(model_spec, prompt, opts) do
with {:ok, model} <- ReqLLM.model(model_spec),
{:ok, context} <- ReqLLM.Context.normalize(prompt, opts),
:ok <- validate_attachments(context),
opts_with_context = Keyword.put(opts, :context, context),
http_opts = Keyword.get(opts, :req_http_options, []),
{:ok, processed_opts} <-
Expand Down Expand Up @@ -1114,4 +1115,14 @@ defmodule ReqLLM.Providers.XAI do
end

defp maybe_add_additional_properties(schema, false), do: schema

defp validate_attachments(context) do
case ReqLLM.Provider.Defaults.validate_image_only_attachments(context) do
:ok ->
:ok

{:error, message} ->
{:error, ReqLLM.Error.Invalid.Parameter.exception(parameter: message)}
end
end
end
63 changes: 63 additions & 0 deletions test/providers/openai_test.exs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1120,6 +1120,69 @@ defmodule ReqLLM.Providers.OpenAITest do
end
end

describe "attachment validation" do
test "accepts image attachments" do
{:ok, model} = ReqLLM.model("openai:gpt-4o")

image_part = ReqLLM.Message.ContentPart.file(<<1, 2, 3>>, "image.png", "image/png")
message = %ReqLLM.Message{role: :user, content: [image_part]}
context = %ReqLLM.Context{messages: [message]}

{:ok, _request} = OpenAI.prepare_request(:chat, model, context, [])
end

test "accepts jpeg, gif, and webp attachments" do
{:ok, model} = ReqLLM.model("openai:gpt-4o")

for mime <- ~w(image/jpeg image/gif image/webp) do
part = ReqLLM.Message.ContentPart.file(<<1, 2, 3>>, "image", mime)
message = %ReqLLM.Message{role: :user, content: [part]}
context = %ReqLLM.Context{messages: [message]}

assert {:ok, _request} = OpenAI.prepare_request(:chat, model, context, [])
end
end

test "rejects PDF attachments with clear error" do
{:ok, model} = ReqLLM.model("openai:gpt-4o")

pdf_part = ReqLLM.Message.ContentPart.file(<<1, 2, 3>>, "doc.pdf", "application/pdf")
message = %ReqLLM.Message{role: :user, content: [pdf_part]}
context = %ReqLLM.Context{messages: [message]}

{:error, error} = OpenAI.prepare_request(:chat, model, context, [])

assert %ReqLLM.Error.Invalid.Parameter{} = error
assert error.parameter =~ "only supports image attachments"
assert error.parameter =~ "application/pdf"
assert error.parameter =~ "Anthropic or Google"
end

test "rejects text file attachments" do
{:ok, model} = ReqLLM.model("openai:gpt-4o")

text_part = ReqLLM.Message.ContentPart.file("content", "file.txt", "text/plain")
message = %ReqLLM.Message{role: :user, content: [text_part]}
context = %ReqLLM.Context{messages: [message]}

{:error, error} = OpenAI.prepare_request(:chat, model, context, [])

assert %ReqLLM.Error.Invalid.Parameter{} = error
assert error.parameter =~ "text/plain"
end

test "allows mixed text and image content" do
{:ok, model} = ReqLLM.model("openai:gpt-4o")

text_part = ReqLLM.Message.ContentPart.text("Describe this image")
image_part = ReqLLM.Message.ContentPart.file(<<1, 2, 3>>, "image.png", "image/png")
message = %ReqLLM.Message{role: :user, content: [text_part, image_part]}
context = %ReqLLM.Context{messages: [message]}

{:ok, _request} = OpenAI.prepare_request(:chat, model, context, [])
end
end

describe "ResponsesAPI tool encoding" do
test "passes through built-in web_search tool definitions" do
{:ok, model} = ReqLLM.model("openai:gpt-5-nano")
Expand Down
63 changes: 63 additions & 0 deletions test/providers/xai_test.exs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -668,6 +668,69 @@ defmodule ReqLLM.Providers.XAITest do
end
end

describe "attachment validation" do
test "accepts image attachments" do
{:ok, model} = ReqLLM.model("xai:grok-3")

image_part = ReqLLM.Message.ContentPart.file(<<1, 2, 3>>, "image.png", "image/png")
message = %ReqLLM.Message{role: :user, content: [image_part]}
context = %ReqLLM.Context{messages: [message]}

{:ok, _request} = XAI.prepare_request(:chat, model, context, [])
end

test "accepts jpeg, gif, and webp attachments" do
{:ok, model} = ReqLLM.model("xai:grok-3")

for mime <- ~w(image/jpeg image/gif image/webp) do
part = ReqLLM.Message.ContentPart.file(<<1, 2, 3>>, "image", mime)
message = %ReqLLM.Message{role: :user, content: [part]}
context = %ReqLLM.Context{messages: [message]}

assert {:ok, _request} = XAI.prepare_request(:chat, model, context, [])
end
end

test "rejects PDF attachments with clear error" do
{:ok, model} = ReqLLM.model("xai:grok-3")

pdf_part = ReqLLM.Message.ContentPart.file(<<1, 2, 3>>, "doc.pdf", "application/pdf")
message = %ReqLLM.Message{role: :user, content: [pdf_part]}
context = %ReqLLM.Context{messages: [message]}

{:error, error} = XAI.prepare_request(:chat, model, context, [])

assert %ReqLLM.Error.Invalid.Parameter{} = error
assert error.parameter =~ "only supports image attachments"
assert error.parameter =~ "application/pdf"
assert error.parameter =~ "Anthropic or Google"
end

test "rejects text file attachments" do
{:ok, model} = ReqLLM.model("xai:grok-3")

text_part = ReqLLM.Message.ContentPart.file("content", "file.txt", "text/plain")
message = %ReqLLM.Message{role: :user, content: [text_part]}
context = %ReqLLM.Context{messages: [message]}

{:error, error} = XAI.prepare_request(:chat, model, context, [])

assert %ReqLLM.Error.Invalid.Parameter{} = error
assert error.parameter =~ "text/plain"
end

test "allows mixed text and image content" do
{:ok, model} = ReqLLM.model("xai:grok-3")

text_part = ReqLLM.Message.ContentPart.text("Describe this image")
image_part = ReqLLM.Message.ContentPart.file(<<1, 2, 3>>, "image.png", "image/png")
message = %ReqLLM.Message{role: :user, content: [text_part, image_part]}
context = %ReqLLM.Context{messages: [message]}

{:ok, _request} = XAI.prepare_request(:chat, model, context, [])
end
end

describe "context validation" do
test "multiple system messages should fail" do
invalid_context =
Expand Down