From 70ca9c9ce59bbb9ad22709608c344deded76e169 Mon Sep 17 00:00:00 2001 From: Wen-Tien Chang Date: Wed, 10 Dec 2025 21:00:42 +0800 Subject: [PATCH 1/3] fix: Preserve non-text tool outputs in LiteLLM and chatcmpl converter --- src/agents/models/chatcmpl_converter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/agents/models/chatcmpl_converter.py b/src/agents/models/chatcmpl_converter.py index abdfa30471..0ae11b58e8 100644 --- a/src/agents/models/chatcmpl_converter.py +++ b/src/agents/models/chatcmpl_converter.py @@ -544,8 +544,9 @@ def ensure_assistant_message() -> ChatCompletionAssistantMessageParam: msg: ChatCompletionToolMessageParam = { "role": "tool", "tool_call_id": func_output["call_id"], - "content": cls.extract_text_content(output_content), + "content": cls.extract_all_content(output_content), } + result.append(msg) # 6) item reference => handle or raise From 9ba59f3280ded3d17bb0d5ff16db5dac735c2407 Mon Sep 17 00:00:00 2001 From: Wen-Tien Chang Date: Sat, 20 Dec 2025 18:04:12 +0800 Subject: [PATCH 2/3] feat: Support media content in tool message outputs --- src/agents/models/chatcmpl_converter.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/agents/models/chatcmpl_converter.py b/src/agents/models/chatcmpl_converter.py index 0ae11b58e8..34a380f108 100644 --- a/src/agents/models/chatcmpl_converter.py +++ b/src/agents/models/chatcmpl_converter.py @@ -17,7 +17,6 @@ ChatCompletionMessageParam, ChatCompletionSystemMessageParam, ChatCompletionToolChoiceOptionParam, - ChatCompletionToolMessageParam, ChatCompletionUserMessageParam, ) from openai.types.chat.chat_completion_content_part_param import File, FileFile @@ -42,6 +41,7 @@ ) from openai.types.responses.response_input_param import FunctionCallOutput, ItemReference, Message from openai.types.responses.response_reasoning_item import Content, Summary +from typing_extensions import Required, TypedDict from ..agent_output import AgentOutputSchemaBase from ..exceptions import AgentsException, UserError @@ -54,6 +54,18 @@ ResponseInputContentWithAudioParam = Union[ResponseInputContentParam, ResponseInputAudioParam] +class ExtendedChatCompletionToolMessageParam(TypedDict, total=False): + """Extended tool message that supports media content (images, audio, files). + + Some chatcmpl providers support media in tool outputs beyond what OpenAI's + ChatCompletionToolMessageParam allows (which only accepts text content). + """ + + role: Required[Literal["tool"]] + tool_call_id: Required[str] + content: Required[str | list[ChatCompletionContentPartParam]] + + class Converter: @classmethod def convert_tool_choice( @@ -541,13 +553,13 @@ def ensure_assistant_message() -> ChatCompletionAssistantMessageParam: output_content = cast( Union[str, Iterable[ResponseInputContentWithAudioParam]], func_output["output"] ) - msg: ChatCompletionToolMessageParam = { + msg: ExtendedChatCompletionToolMessageParam = { "role": "tool", "tool_call_id": func_output["call_id"], "content": cls.extract_all_content(output_content), } - result.append(msg) + result.append(cast(ChatCompletionMessageParam, msg)) # 6) item reference => handle or raise elif item_ref := cls.maybe_item_reference(item): From bb7280f7e7c634888bff603dff01dd50a13b8676 Mon Sep 17 00:00:00 2001 From: Wen-Tien Chang Date: Mon, 22 Dec 2025 19:51:26 +0800 Subject: [PATCH 3/3] feat: Add preserve_tool_output_all_content parameter to items_to_messages --- src/agents/extensions/models/litellm_model.py | 4 ++- src/agents/models/chatcmpl_converter.py | 32 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/agents/extensions/models/litellm_model.py b/src/agents/extensions/models/litellm_model.py index 439146c6c1..abde2bd9ed 100644 --- a/src/agents/extensions/models/litellm_model.py +++ b/src/agents/extensions/models/litellm_model.py @@ -280,7 +280,9 @@ async def _fetch_response( ) converted_messages = Converter.items_to_messages( - input, preserve_thinking_blocks=preserve_thinking_blocks + input, + preserve_thinking_blocks=preserve_thinking_blocks, + preserve_tool_output_all_content=True, ) # Fix for interleaved thinking bug: reorder messages to ensure tool_use comes before tool_result # noqa: E501 diff --git a/src/agents/models/chatcmpl_converter.py b/src/agents/models/chatcmpl_converter.py index 34a380f108..c1f88d437b 100644 --- a/src/agents/models/chatcmpl_converter.py +++ b/src/agents/models/chatcmpl_converter.py @@ -17,6 +17,7 @@ ChatCompletionMessageParam, ChatCompletionSystemMessageParam, ChatCompletionToolChoiceOptionParam, + ChatCompletionToolMessageParam, ChatCompletionUserMessageParam, ) from openai.types.chat.chat_completion_content_part_param import File, FileFile @@ -41,7 +42,6 @@ ) from openai.types.responses.response_input_param import FunctionCallOutput, ItemReference, Message from openai.types.responses.response_reasoning_item import Content, Summary -from typing_extensions import Required, TypedDict from ..agent_output import AgentOutputSchemaBase from ..exceptions import AgentsException, UserError @@ -54,18 +54,6 @@ ResponseInputContentWithAudioParam = Union[ResponseInputContentParam, ResponseInputAudioParam] -class ExtendedChatCompletionToolMessageParam(TypedDict, total=False): - """Extended tool message that supports media content (images, audio, files). - - Some chatcmpl providers support media in tool outputs beyond what OpenAI's - ChatCompletionToolMessageParam allows (which only accepts text content). - """ - - role: Required[Literal["tool"]] - tool_call_id: Required[str] - content: Required[str | list[ChatCompletionContentPartParam]] - - class Converter: @classmethod def convert_tool_choice( @@ -352,6 +340,7 @@ def items_to_messages( cls, items: str | Iterable[TResponseInputItem], preserve_thinking_blocks: bool = False, + preserve_tool_output_all_content: bool = False, ) -> list[ChatCompletionMessageParam]: """ Convert a sequence of 'Item' objects into a list of ChatCompletionMessageParam. @@ -362,6 +351,12 @@ def items_to_messages( for reasoning models like Claude 4 Sonnet/Opus which support interleaved thinking. When True, thinking blocks are reconstructed and included in assistant messages with tool calls. + preserve_tool_output_all_content: Whether to preserve non-text content (like images) + in tool outputs. When False (default), only text content is extracted. + OpenAI Chat Completions API doesn't support non-text content in tool results. + When True, all content types including images are preserved. This is useful + for model providers (e.g. Anthropic via LiteLLM) that support processing + non-text content in tool results. Rules: - EasyInputMessage or InputMessage (role=user) => ChatCompletionUserMessageParam @@ -553,13 +548,16 @@ def ensure_assistant_message() -> ChatCompletionAssistantMessageParam: output_content = cast( Union[str, Iterable[ResponseInputContentWithAudioParam]], func_output["output"] ) - msg: ExtendedChatCompletionToolMessageParam = { + if preserve_tool_output_all_content: + tool_result_content = cls.extract_all_content(output_content) + else: + tool_result_content = cls.extract_text_content(output_content) # type: ignore[assignment] + msg: ChatCompletionToolMessageParam = { "role": "tool", "tool_call_id": func_output["call_id"], - "content": cls.extract_all_content(output_content), + "content": tool_result_content, # type: ignore[typeddict-item] } - - result.append(cast(ChatCompletionMessageParam, msg)) + result.append(msg) # 6) item reference => handle or raise elif item_ref := cls.maybe_item_reference(item):