From c56c4f90471e4f2f7f2694e6bee41214449d6b4f Mon Sep 17 00:00:00 2001 From: Gal Hadar Date: Wed, 15 Oct 2025 13:41:07 +0300 Subject: [PATCH 1/2] Add Tavily as search backend option --- README.md | 16 +++-- gpt-oss-mcp-server/browser_server.py | 6 +- gpt-oss-mcp-server/reference-system-prompt.py | 4 +- gpt_oss/chat.py | 4 +- gpt_oss/responses_api/api_server.py | 6 +- gpt_oss/tools/simple_browser/__init__.py | 3 +- gpt_oss/tools/simple_browser/backend.py | 71 +++++++++++++++++++ .../tools/simple_browser/test_backend.py | 42 ++++++++++- 8 files changed, 134 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 0104cec4..a02f4aeb 100644 --- a/README.md +++ b/README.md @@ -426,7 +426,7 @@ codex -p oss ### Browser > [!WARNING] -> This implementation is purely for educational purposes and should not be used in production. You should implement your own equivalent of the [`YouComBackend`](gpt_oss/tools/simple_browser/backend.py) class with your own browsing environment. Currently we have available `YouComBackend` and `ExaBackend`. +> This implementation is purely for educational purposes and should not be used in production. You should implement your own equivalent of the [`TavilyBackend`](gpt_oss/tools/simple_browser/backend.py) class with your own browsing environment. Currently we have available `TavilyBackend`, `YouComBackend`, and `ExaBackend`. Both gpt-oss models were trained with the capability to browse using the `browser` tool that exposes the following three methods: @@ -441,17 +441,21 @@ To enable the browser tool, you'll have to place the definition into the `system ```python import datetime from gpt_oss.tools.simple_browser import SimpleBrowserTool -from gpt_oss.tools.simple_browser.backend import YouComBackend +from gpt_oss.tools.simple_browser.backend import TavilyBackend from openai_harmony import SystemContent, Message, Conversation, Role, load_harmony_encoding, HarmonyEncodingName encoding = load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS) # Depending on the choice of the browser backend you need corresponding env variables setup -# In case you use You.com backend requires you to have set the YDC_API_KEY environment variable, -# while for Exa you might need EXA_API_KEY environment variable set -backend = YouComBackend( - source="web", +# - Tavily backend requires TAVILY_API_KEY environment variable +# - You.com backend requires YDC_API_KEY environment variable +# - Exa backend requires EXA_API_KEY environment variable +backend = TavilyBackend( + source="web", ) +# backend = YouComBackend( +# source="web", +# ) # backend = ExaBackend( # source="web", # ) diff --git a/gpt-oss-mcp-server/browser_server.py b/gpt-oss-mcp-server/browser_server.py index b37a63a6..63b16a70 100644 --- a/gpt-oss-mcp-server/browser_server.py +++ b/gpt-oss-mcp-server/browser_server.py @@ -6,7 +6,7 @@ from mcp.server.fastmcp import Context, FastMCP from gpt_oss.tools.simple_browser import SimpleBrowserTool -from gpt_oss.tools.simple_browser.backend import YouComBackend, ExaBackend +from gpt_oss.tools.simple_browser.backend import YouComBackend, ExaBackend, TavilyBackend @dataclass class AppContext: @@ -15,7 +15,9 @@ class AppContext: def create_or_get_browser(self, session_id: str) -> SimpleBrowserTool: if session_id not in self.browsers: tool_backend = os.getenv("BROWSER_BACKEND", "exa") - if tool_backend == "youcom": + if tool_backend == "tavily": + backend = TavilyBackend(source="web") + elif tool_backend == "youcom": backend = YouComBackend(source="web") elif tool_backend == "exa": backend = ExaBackend(source="web") diff --git a/gpt-oss-mcp-server/reference-system-prompt.py b/gpt-oss-mcp-server/reference-system-prompt.py index 6ddbf7c9..28f8462e 100644 --- a/gpt-oss-mcp-server/reference-system-prompt.py +++ b/gpt-oss-mcp-server/reference-system-prompt.py @@ -1,7 +1,7 @@ import datetime from gpt_oss.tools.simple_browser import SimpleBrowserTool -from gpt_oss.tools.simple_browser.backend import YouComBackend +from gpt_oss.tools.simple_browser.backend import TavilyBackend from gpt_oss.tools.python_docker.docker_tool import PythonTool from gpt_oss.tokenizer import tokenizer @@ -22,7 +22,7 @@ ReasoningEffort.LOW).with_conversation_start_date( datetime.datetime.now().strftime("%Y-%m-%d"))) -backend = YouComBackend(source="web") +backend = TavilyBackend(source="web") browser_tool = SimpleBrowserTool(backend=backend) system_message_content = system_message_content.with_tools( browser_tool.tool_config) diff --git a/gpt_oss/chat.py b/gpt_oss/chat.py index 4856a397..9c807449 100644 --- a/gpt_oss/chat.py +++ b/gpt_oss/chat.py @@ -19,7 +19,7 @@ from gpt_oss.tools import apply_patch from gpt_oss.tools.simple_browser import SimpleBrowserTool -from gpt_oss.tools.simple_browser.backend import YouComBackend +from gpt_oss.tools.simple_browser.backend import TavilyBackend from gpt_oss.tools.python_docker.docker_tool import PythonTool from openai_harmony import ( @@ -85,7 +85,7 @@ def main(args): ) if args.browser: - backend = YouComBackend( + backend = TavilyBackend( source="web", ) browser_tool = SimpleBrowserTool(backend=backend) diff --git a/gpt_oss/responses_api/api_server.py b/gpt_oss/responses_api/api_server.py index 009fa8d8..34aeea7e 100644 --- a/gpt_oss/responses_api/api_server.py +++ b/gpt_oss/responses_api/api_server.py @@ -23,7 +23,7 @@ from gpt_oss.tools.python_docker.docker_tool import PythonTool from gpt_oss.tools.simple_browser import SimpleBrowserTool -from gpt_oss.tools.simple_browser.backend import YouComBackend, ExaBackend +from gpt_oss.tools.simple_browser.backend import YouComBackend, ExaBackend, TavilyBackend from .events import ( ResponseCodeInterpreterCallCodeDelta, @@ -1148,7 +1148,9 @@ async def generate(body: ResponsesRequest, request: Request): if use_browser_tool: tool_backend = os.getenv("BROWSER_BACKEND", "exa") - if tool_backend == "youcom": + if tool_backend == "tavily": + backend = TavilyBackend(source="web") + elif tool_backend == "youcom": backend = YouComBackend(source="web") elif tool_backend == "exa": backend = ExaBackend(source="web") diff --git a/gpt_oss/tools/simple_browser/__init__.py b/gpt_oss/tools/simple_browser/__init__.py index da3ff280..b7483e06 100644 --- a/gpt_oss/tools/simple_browser/__init__.py +++ b/gpt_oss/tools/simple_browser/__init__.py @@ -1,8 +1,9 @@ from .simple_browser_tool import SimpleBrowserTool -from .backend import ExaBackend, YouComBackend +from .backend import ExaBackend, YouComBackend, TavilyBackend __all__ = [ "SimpleBrowserTool", "ExaBackend", "YouComBackend", + "TavilyBackend", ] diff --git a/gpt_oss/tools/simple_browser/backend.py b/gpt_oss/tools/simple_browser/backend.py index 33daf8d6..2f513d27 100644 --- a/gpt_oss/tools/simple_browser/backend.py +++ b/gpt_oss/tools/simple_browser/backend.py @@ -250,3 +250,74 @@ async def fetch(self, url: str, session: ClientSession) -> PageContents: session=session, ) +@chz.chz(typecheck=True) +class TavilyBackend(Backend): + """Backend that uses the Tavily Search API.""" + + source: str = chz.field(doc="Description of the backend source") + + BASE_URL: str = "https://api.tavily.com" + + def _get_api_key(self) -> str: + key = os.environ.get("TAVILY_API_KEY") + if not key: + raise BackendError("Tavily API key not provided") + return key + + + async def search( + self, query: str, topn: int, session: ClientSession + ) -> PageContents: + data = await self._post( + session, + "/search", + {"query": query, "max_results": topn}, + ) + # make a simple HTML page to work with browser format + titles_and_urls = [] + if "results" in data: + titles_and_urls = [ + (result["title"], result["url"], result.get("content", "")) + for result in data["results"] + ] + html_page = f""" + +

Search Results

+ + +""" + + return process_html( + html=html_page, + url="", + title=query, + display_urls=True, + session=session, + ) + + async def fetch(self, url: str, session: ClientSession) -> PageContents: + is_view_source = url.startswith(VIEW_SOURCE_PREFIX) + if is_view_source: + url = url[len(VIEW_SOURCE_PREFIX) :] + + # Use Tavily's extract functionality to fetch webpage content + data = await self._post( + session, + "/extract", + {"urls": [url], "format": "html_tags"}, + ) + + if not data or "results" not in data or not data["results"]: + raise BackendError(f"No contents returned for {url}") + + result = data["results"][0] + + return process_html( + html=result.get("raw_content", ""), + url=url, + title=result.get("title", ""), + display_urls=True, + session=session, + ) diff --git a/tests/gpt_oss/tools/simple_browser/test_backend.py b/tests/gpt_oss/tools/simple_browser/test_backend.py index ab0dc780..15f14b2c 100644 --- a/tests/gpt_oss/tools/simple_browser/test_backend.py +++ b/tests/gpt_oss/tools/simple_browser/test_backend.py @@ -3,7 +3,7 @@ from unittest import mock from aiohttp import ClientSession -from gpt_oss.tools.simple_browser.backend import YouComBackend +from gpt_oss.tools.simple_browser.backend import YouComBackend, TavilyBackend class MockAiohttpResponse: """Mocks responses for get/post requests from async libraries.""" @@ -22,7 +22,7 @@ async def __aenter__(self): return self def mock_os_environ_get(name: str, default: Any = "test_api_key"): - assert name in ["YDC_API_KEY"] + assert name in ["YDC_API_KEY", "TAVILY_API_KEY"] return default def test_youcom_backend(): @@ -67,4 +67,40 @@ async def test_youcom_backend_fetch(mock_session_get): assert result.text == "\nURL: https://www.example.com/fetch1\nFetch Result 1 text" - \ No newline at end of file +def test_tavily_backend(): + backend = TavilyBackend(source="web") + assert backend.source == "web" + +@pytest.mark.asyncio +@mock.patch("aiohttp.ClientSession.post") +async def test_tavily_backend_search(mock_session_post): + backend = TavilyBackend(source="web") + api_response = { + "results": [ + {"title": "Result 1", "url": "https://www.example.com/1", "content": "Content snippet 1"}, + {"title": "Result 2", "url": "https://www.example.com/2", "content": "Content snippet 2"}, + {"title": "Result 3", "url": "https://www.example.com/3", "content": "Content snippet 3"}, + ] + } + with mock.patch("os.environ.get", wraps=mock_os_environ_get): + mock_session_post.return_value = MockAiohttpResponse(api_response, 200) + async with ClientSession() as session: + result = await backend.search(query="test query", topn=10, session=session) + assert result.title == "test query" + assert result.urls == {"0": "https://www.example.com/1", "1": "https://www.example.com/2", "2": "https://www.example.com/3"} + +@pytest.mark.asyncio +@mock.patch("aiohttp.ClientSession.post") +async def test_tavily_backend_fetch(mock_session_post): + backend = TavilyBackend(source="web") + api_response = { + "results": [ + {"title": "Page Title", "url": "https://www.example.com/page", "raw_content": "This is the page content"}, + ] + } + with mock.patch("os.environ.get", wraps=mock_os_environ_get): + mock_session_post.return_value = MockAiohttpResponse(api_response, 200) + async with ClientSession() as session: + result = await backend.fetch(url="https://www.example.com/page", session=session) + assert result.title == "Page Title" + assert "This is the page content" in result.text From d53e92d093a045e4c56bc6bdfc342545b93010b5 Mon Sep 17 00:00:00 2001 From: Gal Hadar Date: Wed, 15 Oct 2025 20:31:14 +0300 Subject: [PATCH 2/2] add Tavily headers to post request --- gpt_oss/tools/simple_browser/backend.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/gpt_oss/tools/simple_browser/backend.py b/gpt_oss/tools/simple_browser/backend.py index 2f513d27..ebf9c9db 100644 --- a/gpt_oss/tools/simple_browser/backend.py +++ b/gpt_oss/tools/simple_browser/backend.py @@ -264,6 +264,17 @@ def _get_api_key(self) -> str: raise BackendError("Tavily API key not provided") return key + async def _post(self, session: ClientSession, endpoint: str, payload: dict) -> dict: + headers = { + "Authorization": f"Bearer {self._get_api_key()}", + "Content-Type": "application/json" + } + async with session.post(f"{self.BASE_URL}{endpoint}", json=payload, headers=headers) as resp: + if resp.status != 200: + raise BackendError( + f"{self.__class__.__name__} error {resp.status}: {await resp.text()}" + ) + return await resp.json() async def search( self, query: str, topn: int, session: ClientSession