From 6e33d587143b6b71951265febacd5c710da8ee82 Mon Sep 17 00:00:00 2001 From: Lin-Nikaido Date: Fri, 5 Sep 2025 17:42:27 +0900 Subject: [PATCH 1/6] fix: Cannot send attach file to OpenAI throw LiteLlm --- src/google/adk/models/lite_llm.py | 70 ++++++++-- tests/unittests/models/test_litellm.py | 175 +++++++++++++++++++++---- 2 files changed, 208 insertions(+), 37 deletions(-) diff --git a/src/google/adk/models/lite_llm.py b/src/google/adk/models/lite_llm.py index d84df9abbc..33a008aa53 100644 --- a/src/google/adk/models/lite_llm.py +++ b/src/google/adk/models/lite_llm.py @@ -154,8 +154,8 @@ def _safe_json_serialize(obj) -> str: return str(obj) -def _content_to_message_param( - content: types.Content, +async def _content_to_message_param( + content: types.Content, custom_llm_provider: str = None ) -> Union[Message, list[Message]]: """Converts a types.Content to a litellm Message or list of Messages. @@ -184,7 +184,9 @@ def _content_to_message_param( # Handle user or assistant messages role = _to_litellm_role(content.role) - message_content = _get_content(content.parts) or None + message_content = ( + await _get_content(content.parts, custom_llm_provider) or None + ) if role == "user": return ChatCompletionUserMessage(role="user", content=message_content) @@ -223,8 +225,8 @@ def _content_to_message_param( ) -def _get_content( - parts: Iterable[types.Part], +async def _get_content( + parts: Iterable[types.Part], custom_llm_provider: str = None ) -> Union[OpenAIMessageContent, str]: """Converts a list of parts to litellm content. @@ -251,6 +253,14 @@ def _get_content( ): base64_string = base64.b64encode(part.inline_data.data).decode("utf-8") data_uri = f"data:{part.inline_data.mime_type};base64,{base64_string}" + if custom_llm_provider in ["openai", "azure"]: + open_ai_file_object = await litellm.acreate_file( + file=part.inline_data.data, + purpose="assistants", + custom_llm_provider=custom_llm_provider, # type: ignore + ) + else: + open_ai_file_object = None if part.inline_data.mime_type.startswith("image"): # Use full MIME type (e.g., "image/png") for providers that validate it @@ -273,12 +283,32 @@ def _get_content( "type": "audio_url", "audio_url": {"url": data_uri, "format": format_type}, }) - elif part.inline_data.mime_type == "application/pdf": + elif ( + part.inline_data.mime_type.startswith("text/") + or part.inline_data.mime_type == "application/pdf" + or part.inline_data.mime_type == "application/msword" + or part.inline_data.mime_type == "application/json" + or part.inline_data.mime_type == "application/x-sh" + or part.inline_data.mime_type == "application/typescript" + or part.inline_data.mime_type + == "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + or part.inline_data.mime_type + == "application/vnd.openxmlformats-officedocument.presentationml.presentation" + ): format_type = part.inline_data.mime_type - content_objects.append({ - "type": "file", - "file": {"file_data": data_uri, "format": format_type}, - }) + if open_ai_file_object: + content_objects.append({ + "type": "file", + "file": { + "file_id": open_ai_file_object.id, + "format": format_type, + }, + }) + else: + content_objects.append({ + "type": "file", + "file": {"file_data": data_uri, "format": format_type}, + }) else: raise ValueError("LiteLlm(BaseLlm) does not support this content part.") @@ -521,7 +551,7 @@ def _message_to_generate_content_response( ) -def _get_completion_inputs( +async def _get_completion_inputs( llm_request: LlmRequest, ) -> Tuple[ List[Message], @@ -537,10 +567,24 @@ def _get_completion_inputs( Returns: The litellm inputs (message list, tool dictionary, response format and generation params). """ + # 0. check custom_llm_provider + if llm_request.model is None: + custom_llm_provider = "UNK" + elif "gemini" in llm_request.model: + custom_llm_provider = "vertex_ai" + elif "azure" in llm_request.model: + custom_llm_provider = "azure" + elif "openai" in llm_request.model: + custom_llm_provider = "azure" + else: + custom_llm_provider = "UNK" + # 1. Construct messages messages: List[Message] = [] for content in llm_request.contents or []: - message_param_or_list = _content_to_message_param(content) + message_param_or_list = await _content_to_message_param( + content, custom_llm_provider + ) if isinstance(message_param_or_list, list): messages.extend(message_param_or_list) elif message_param_or_list: # Ensure it's not None before appending @@ -800,7 +844,7 @@ async def generate_content_async( logger.debug(_build_request_log(llm_request)) messages, tools, response_format, generation_params = ( - _get_completion_inputs(llm_request) + await _get_completion_inputs(llm_request) ) if "functions" in self._additional_args: diff --git a/tests/unittests/models/test_litellm.py b/tests/unittests/models/test_litellm.py index a7152f5562..d0ebfdfd5a 100644 --- a/tests/unittests/models/test_litellm.py +++ b/tests/unittests/models/test_litellm.py @@ -410,6 +410,13 @@ def lite_llm_instance(mock_client): return LiteLlm(model="test_model", llm_client=mock_client) +@pytest.fixture +def openai_instance(mock_client, model: str = None): + if model is None: + model = "openai/gpt-5" + return LiteLlm(model=model, llm_client=mock_client) + + class MockLLMClient(LiteLLMClient): def __init__(self, acompletion_mock, completion_mock): @@ -879,16 +886,18 @@ async def test_generate_content_async_with_usage_metadata( mock_acompletion.assert_called_once() -def test_content_to_message_param_user_message(): +@pytest.mark.asyncio +async def test_content_to_message_param_user_message(): content = types.Content( role="user", parts=[types.Part.from_text(text="Test prompt")] ) - message = _content_to_message_param(content) + message = await _content_to_message_param(content) assert message["role"] == "user" assert message["content"] == "Test prompt" -def test_content_to_message_param_multi_part_function_response(): +@pytest.mark.asyncio +async def test_content_to_message_param_multi_part_function_response(): part1 = types.Part.from_function_response( name="function_one", response={"result": "result_one"}, @@ -905,7 +914,7 @@ def test_content_to_message_param_multi_part_function_response(): role="tool", parts=[part1, part2], ) - messages = _content_to_message_param(content) + messages = await _content_to_message_param(content) assert isinstance(messages, list) assert len(messages) == 2 @@ -918,16 +927,18 @@ def test_content_to_message_param_multi_part_function_response(): assert messages[1]["content"] == '{"value": 123}' -def test_content_to_message_param_assistant_message(): +@pytest.mark.asyncio +async def test_content_to_message_param_assistant_message(): content = types.Content( role="assistant", parts=[types.Part.from_text(text="Test response")] ) - message = _content_to_message_param(content) + message = await _content_to_message_param(content) assert message["role"] == "assistant" assert message["content"] == "Test response" -def test_content_to_message_param_function_call(): +@pytest.mark.asyncio +async def test_content_to_message_param_function_call(): content = types.Content( role="assistant", parts=[ @@ -938,7 +949,7 @@ def test_content_to_message_param_function_call(): ], ) content.parts[1].function_call.id = "test_tool_call_id" - message = _content_to_message_param(content) + message = await _content_to_message_param(content) assert message["role"] == "assistant" assert message["content"] == "test response" @@ -949,7 +960,8 @@ def test_content_to_message_param_function_call(): assert tool_call["function"]["arguments"] == '{"test_arg": "test_value"}' -def test_content_to_message_param_multipart_content(): +@pytest.mark.asyncio +async def test_content_to_message_param_multipart_content(): """Test handling of multipart content where final_content is a list with text objects.""" content = types.Content( role="assistant", @@ -958,7 +970,7 @@ def test_content_to_message_param_multipart_content(): types.Part.from_bytes(data=b"test_image_data", mime_type="image/png"), ], ) - message = _content_to_message_param(content) + message = await _content_to_message_param(content) assert message["role"] == "assistant" # When content is a list and the first element is a text object with type "text", # it should extract the text (for providers like ollama_chat that don't handle lists well) @@ -967,7 +979,8 @@ def test_content_to_message_param_multipart_content(): assert message["tool_calls"] is None -def test_content_to_message_param_single_text_object_in_list(): +@pytest.mark.asyncio +async def test_content_to_message_param_single_text_object_in_list(): """Test extraction of text from single text object in list (for ollama_chat compatibility).""" from unittest.mock import patch @@ -979,7 +992,7 @@ def test_content_to_message_param_single_text_object_in_list(): role="assistant", parts=[types.Part.from_text(text="single text")], ) - message = _content_to_message_param(content) + message = await _content_to_message_param(content) assert message["role"] == "assistant" # Should extract the text from the single text object assert message["content"] == "single text" @@ -1021,17 +1034,19 @@ def test_message_to_generate_content_response_tool_call(): assert response.content.parts[0].function_call.id == "test_tool_call_id" -def test_get_content_text(): +@pytest.mark.asyncio +async def test_get_content_text(): parts = [types.Part.from_text(text="Test text")] - content = _get_content(parts) + content = await _get_content(parts) assert content == "Test text" -def test_get_content_image(): +@pytest.mark.asyncio +async def test_get_content_image(): parts = [ types.Part.from_bytes(data=b"test_image_data", mime_type="image/png") ] - content = _get_content(parts) + content = await _get_content(parts) assert content[0]["type"] == "image_url" assert ( content[0]["image_url"]["url"] @@ -1040,11 +1055,12 @@ def test_get_content_image(): assert content[0]["image_url"]["format"] == "image/png" -def test_get_content_video(): +@pytest.mark.asyncio +async def test_get_content_video(): parts = [ types.Part.from_bytes(data=b"test_video_data", mime_type="video/mp4") ] - content = _get_content(parts) + content = await _get_content(parts) assert content[0]["type"] == "video_url" assert ( content[0]["video_url"]["url"] @@ -1053,11 +1069,12 @@ def test_get_content_video(): assert content[0]["video_url"]["format"] == "video/mp4" -def test_get_content_pdf(): +@pytest.mark.asyncio +async def test_get_content_pdf(): parts = [ types.Part.from_bytes(data=b"test_pdf_data", mime_type="application/pdf") ] - content = _get_content(parts) + content = await _get_content(parts) assert content[0]["type"] == "file" assert ( content[0]["file"]["file_data"] @@ -1066,11 +1083,12 @@ def test_get_content_pdf(): assert content[0]["file"]["format"] == "application/pdf" -def test_get_content_audio(): +@pytest.mark.asyncio +async def test_get_content_audio(): parts = [ types.Part.from_bytes(data=b"test_audio_data", mime_type="audio/mpeg") ] - content = _get_content(parts) + content = await _get_content(parts) assert content[0]["type"] == "audio_url" assert ( content[0]["audio_url"]["url"] @@ -1546,7 +1564,7 @@ async def test_generate_content_async_non_compliant_multiple_function_calls( @pytest.mark.asyncio -def test_get_completion_inputs_generation_params(): +async def test_get_completion_inputs_generation_params(): # Test that generation_params are extracted and mapped correctly req = LlmRequest( contents=[ @@ -1564,7 +1582,7 @@ def test_get_completion_inputs_generation_params(): ) from google.adk.models.lite_llm import _get_completion_inputs - _, _, _, generation_params = _get_completion_inputs(req) + _, _, _, generation_params = await _get_completion_inputs(req) assert generation_params["temperature"] == 0.33 assert generation_params["max_completion_tokens"] == 123 assert generation_params["top_p"] == 0.88 @@ -1623,3 +1641,112 @@ def test_non_gemini_litellm_no_warning(): # Test with non-Gemini model LiteLlm(model="openai/gpt-4o") assert len(w) == 0 + + +@pytest.mark.asyncio +async def test_get_file_id_from_litellm_openai( + mocker, +): + """Test for request with attach file as file_id for OpenAI""" + from google.adk.models.lite_llm import _get_completion_inputs + + mock_return = mocker.MagicMock() + mock_return.id = "test_file_id" + acreate_file_mock = AsyncMock(return_value=mock_return) + mocker.patch( + "google.adk.models.lite_llm.litellm.acreate_file", + new=acreate_file_mock, + ) + + data_part = types.Part.from_bytes( + data=b"test_pdf_data", mime_type="application/pdf" + ) + data_part.inline_data.display_name = "test_file.pdf" + + llm_request = LlmRequest( + model="openai/gpt-4o", + contents=[ + types.Content( + role="user", + parts=[ + types.Part.from_text(text="Test attach PDF file"), + data_part, + ], + ) + ], + config=types.GenerateContentConfig( + tools=[], + ), + ) + messages, tools, response_format, generation_params = ( + await _get_completion_inputs(llm_request) + ) + assert messages + assert messages == [{ + "role": "user", + "content": [ + {"type": "text", "text": "Test attach PDF file"}, + { + "type": "file", + "file": {"file_id": "test_file_id", "format": "application/pdf"}, + }, + ], + }] + + +@pytest.mark.asyncio +async def test_get_file_id_from_litellm_gemini( + mocker, +): + """Test for request with attach file **NOT** as file_id for gemini (or other than openai or azure)""" + + from google.adk.models.lite_llm import _get_completion_inputs + + mock_return = mocker.MagicMock() + mock_return.id = "test_file_id" + acreate_file_mock = AsyncMock(return_value=mock_return) + mocker.patch( + "google.adk.models.lite_llm.litellm.acreate_file", + new=acreate_file_mock, + ) + + data_part = types.Part.from_bytes( + data=b"test_pdf_data", mime_type="application/pdf" + ) + data_part.inline_data.display_name = "test_file.pdf" + + llm_request = LlmRequest( + model="gemini", + contents=[ + types.Content( + role="user", + parts=[ + types.Part.from_text(text="Test attach PDF file"), + data_part, + ], + ) + ], + config=types.GenerateContentConfig( + tools=[], + ), + ) + + messages, tools, response_format, generation_params = ( + await _get_completion_inputs(llm_request) + ) + assert messages + assert messages == [{ + "role": "user", + "content": [ + {"type": "text", "text": "Test attach PDF file"}, + { + "type": "file", + "file": { + "file_data": ( + "data:application/pdf;base64,dGVzdF9wZGZfZGF0YQ==" + ), + "format": "application/pdf", + }, + }, + ], + }] From d96a8374544c536003630a63740b58ac9765bf34 Mon Sep 17 00:00:00 2001 From: Lin Nikaido <153617286+Lin-Nikaido@users.noreply.github.com> Date: Fri, 5 Sep 2025 18:24:48 +0900 Subject: [PATCH 2/6] Update src/google/adk/models/lite_llm.py fix typo Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- src/google/adk/models/lite_llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/google/adk/models/lite_llm.py b/src/google/adk/models/lite_llm.py index 33a008aa53..d5f3c5c3d5 100644 --- a/src/google/adk/models/lite_llm.py +++ b/src/google/adk/models/lite_llm.py @@ -575,7 +575,7 @@ async def _get_completion_inputs( elif "azure" in llm_request.model: custom_llm_provider = "azure" elif "openai" in llm_request.model: - custom_llm_provider = "azure" + custom_llm_provider = "openai" else: custom_llm_provider = "UNK" From a19036d9cd9a074456ac0814542150cb63cc7845 Mon Sep 17 00:00:00 2001 From: Lin Nikaido <153617286+Lin-Nikaido@users.noreply.github.com> Date: Fri, 5 Sep 2025 18:25:07 +0900 Subject: [PATCH 3/6] Update src/google/adk/models/lite_llm.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- src/google/adk/models/lite_llm.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/google/adk/models/lite_llm.py b/src/google/adk/models/lite_llm.py index d5f3c5c3d5..9a68d7560c 100644 --- a/src/google/adk/models/lite_llm.py +++ b/src/google/adk/models/lite_llm.py @@ -285,15 +285,16 @@ async def _get_content( }) elif ( part.inline_data.mime_type.startswith("text/") - or part.inline_data.mime_type == "application/pdf" - or part.inline_data.mime_type == "application/msword" - or part.inline_data.mime_type == "application/json" - or part.inline_data.mime_type == "application/x-sh" - or part.inline_data.mime_type == "application/typescript" or part.inline_data.mime_type - == "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - or part.inline_data.mime_type - == "application/vnd.openxmlformats-officedocument.presentationml.presentation" + in { + "application/pdf", + "application/msword", + "application/json", + "application/x-sh", + "application/typescript", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + } ): format_type = part.inline_data.mime_type if open_ai_file_object: From 375014e2f6b2be092e0830c029b21aa175dab2b8 Mon Sep 17 00:00:00 2001 From: Lin-Nikaido Date: Mon, 8 Sep 2025 15:08:35 +0900 Subject: [PATCH 4/6] fix: typo --- src/google/adk/models/lite_llm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/google/adk/models/lite_llm.py b/src/google/adk/models/lite_llm.py index 9a68d7560c..66ef5d9f76 100644 --- a/src/google/adk/models/lite_llm.py +++ b/src/google/adk/models/lite_llm.py @@ -302,7 +302,6 @@ async def _get_content( "type": "file", "file": { "file_id": open_ai_file_object.id, - "format": format_type, }, }) else: From db0e3b8c25c5f3ecf36b11ca4af2fcd6009e82fe Mon Sep 17 00:00:00 2001 From: Lin Nikaido <153617286+Lin-Nikaido@users.noreply.github.com> Date: Mon, 8 Sep 2025 18:10:29 +0900 Subject: [PATCH 5/6] fix typo --- src/google/adk/models/lite_llm.py | 2 +- tests/unittests/models/test_litellm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/google/adk/models/lite_llm.py b/src/google/adk/models/lite_llm.py index 66ef5d9f76..464065f5ea 100644 --- a/src/google/adk/models/lite_llm.py +++ b/src/google/adk/models/lite_llm.py @@ -307,7 +307,7 @@ async def _get_content( else: content_objects.append({ "type": "file", - "file": {"file_data": data_uri, "format": format_type}, + "file": {"file_data": data_uri}, }) else: raise ValueError("LiteLlm(BaseLlm) does not support this content part.") diff --git a/tests/unittests/models/test_litellm.py b/tests/unittests/models/test_litellm.py index d0ebfdfd5a..ba38cd2ea9 100644 --- a/tests/unittests/models/test_litellm.py +++ b/tests/unittests/models/test_litellm.py @@ -1688,7 +1688,7 @@ async def test_get_file_id_from_litellm_openai( {"type": "text", "text": "Test attach PDF file"}, { "type": "file", - "file": {"file_id": "test_file_id", "format": "application/pdf"}, + "file": {"file_id": "test_file_id"}, }, ], }] From 1c136504eb0cc2dfe7f3931c31b69f17d528f497 Mon Sep 17 00:00:00 2001 From: Lin Nikaido <153617286+Lin-Nikaido@users.noreply.github.com> Date: Mon, 8 Sep 2025 18:11:47 +0900 Subject: [PATCH 6/6] fix: typo --- src/google/adk/models/lite_llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/google/adk/models/lite_llm.py b/src/google/adk/models/lite_llm.py index 464065f5ea..66ef5d9f76 100644 --- a/src/google/adk/models/lite_llm.py +++ b/src/google/adk/models/lite_llm.py @@ -307,7 +307,7 @@ async def _get_content( else: content_objects.append({ "type": "file", - "file": {"file_data": data_uri}, + "file": {"file_data": data_uri, "format": format_type}, }) else: raise ValueError("LiteLlm(BaseLlm) does not support this content part.")