diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py index 4a359eded38c..2c71a2558df0 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py @@ -13,7 +13,7 @@ Sequence, ) -from autogen_core import CancellationToken, FunctionCall +from autogen_core import CancellationToken, Component, ComponentModel, FunctionCall from autogen_core.memory import Memory from autogen_core.model_context import ( ChatCompletionContext, @@ -28,6 +28,8 @@ UserMessage, ) from autogen_core.tools import FunctionTool, Tool +from pydantic import BaseModel +from typing_extensions import Self from .. import EVENT_LOGGER_NAME from ..base import Handoff as HandoffBase @@ -49,7 +51,21 @@ event_logger = logging.getLogger(EVENT_LOGGER_NAME) -class AssistantAgent(BaseChatAgent): +class AssistantAgentConfig(BaseModel): + """The declarative configuration for the assistant agent.""" + + name: str + model_client: ComponentModel + # tools: List[Any] | None = None # TBD + handoffs: List[HandoffBase | str] | None = None + model_context: ComponentModel | None = None + description: str + system_message: str | None = None + reflect_on_tool_use: bool + tool_call_summary_format: str + + +class AssistantAgent(BaseChatAgent, Component[AssistantAgentConfig]): """An agent that provides assistance with tool use. The :meth:`on_messages` returns a :class:`~autogen_agentchat.base.Response` @@ -229,6 +245,9 @@ async def main() -> None: See `o1 beta limitations `_ for more details. """ + component_config_schema = AssistantAgentConfig + component_provider_override = "autogen_agentchat.agents.AssistantAgent" + def __init__( self, name: str, @@ -462,3 +481,40 @@ async def load_state(self, state: Mapping[str, Any]) -> None: assistant_agent_state = AssistantAgentState.model_validate(state) # Load the model context state. await self._model_context.load_state(assistant_agent_state.llm_context) + + def _to_config(self) -> AssistantAgentConfig: + """Convert the assistant agent to a declarative config.""" + + # raise an error if tools is not empty until it is implemented + # TBD : Implement serializing tools and remove this check. + if self._tools and len(self._tools) > 0: + raise NotImplementedError("Serializing tools is not implemented yet.") + + return AssistantAgentConfig( + name=self.name, + model_client=self._model_client.dump_component(), + # tools=[], # TBD + handoffs=list(self._handoffs.values()), + model_context=self._model_context.dump_component(), + description=self.description, + system_message=self._system_messages[0].content + if self._system_messages and isinstance(self._system_messages[0].content, str) + else None, + reflect_on_tool_use=self._reflect_on_tool_use, + tool_call_summary_format=self._tool_call_summary_format, + ) + + @classmethod + def _from_config(cls, config: AssistantAgentConfig) -> Self: + """Create an assistant agent from a declarative config.""" + return cls( + name=config.name, + model_client=ChatCompletionClient.load_component(config.model_client), + # tools=[], # TBD + handoffs=config.handoffs, + model_context=None, + description=config.description, + system_message=config.system_message, + reflect_on_tool_use=config.reflect_on_tool_use, + tool_call_summary_format=config.tool_call_summary_format, + ) diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_base_chat_agent.py b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_base_chat_agent.py index 42b7cb78a007..97b9de76242c 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_base_chat_agent.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_base_chat_agent.py @@ -1,7 +1,8 @@ from abc import ABC, abstractmethod from typing import Any, AsyncGenerator, List, Mapping, Sequence -from autogen_core import CancellationToken +from autogen_core import CancellationToken, ComponentBase +from pydantic import BaseModel from ..base import ChatAgent, Response, TaskResult from ..messages import ( @@ -13,7 +14,7 @@ from ..state import BaseState -class BaseChatAgent(ChatAgent, ABC): +class BaseChatAgent(ChatAgent, ABC, ComponentBase[BaseModel]): """Base class for a chat agent. This abstract class provides a base implementation for a :class:`ChatAgent`. @@ -35,6 +36,8 @@ class BaseChatAgent(ChatAgent, ABC): This design principle must be followed when creating a new agent. """ + component_type = "agent" + def __init__(self, name: str, description: str) -> None: self._name = name if self._name.isidentifier() is False: diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_user_proxy_agent.py b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_user_proxy_agent.py index e53d3b09acfa..30c444a9a36f 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_user_proxy_agent.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_user_proxy_agent.py @@ -5,7 +5,9 @@ from inspect import iscoroutinefunction from typing import Any, AsyncGenerator, Awaitable, Callable, ClassVar, Generator, Optional, Sequence, Union, cast -from autogen_core import CancellationToken +from autogen_core import CancellationToken, Component +from pydantic import BaseModel +from typing_extensions import Self from ..base import Response from ..messages import AgentEvent, ChatMessage, HandoffMessage, TextMessage, UserInputRequestedEvent @@ -24,7 +26,15 @@ async def cancellable_input(prompt: str, cancellation_token: Optional[Cancellati return await task -class UserProxyAgent(BaseChatAgent): +class UserProxyAgentConfig(BaseModel): + """Declarative configuration for the UserProxyAgent.""" + + name: str + description: str = "A human user" + input_func: str | None = None + + +class UserProxyAgent(BaseChatAgent, Component[UserProxyAgentConfig]): """An agent that can represent a human user through an input function. This agent can be used to represent a human user in a chat system by providing a custom input function. @@ -109,6 +119,10 @@ async def cancellable_user_agent(): print(f"BaseException: {e}") """ + component_type = "agent" + component_provider_override = "autogen_agentchat.agents.UserProxyAgent" + component_config_schema = UserProxyAgentConfig + class InputRequestContext: def __init__(self) -> None: raise RuntimeError( @@ -218,3 +232,11 @@ async def on_messages_stream( async def on_reset(self, cancellation_token: Optional[CancellationToken] = None) -> None: """Reset agent state.""" pass + + def _to_config(self) -> UserProxyAgentConfig: + # TODO: Add ability to serialie input_func + return UserProxyAgentConfig(name=self.name, description=self.description, input_func=None) + + @classmethod + def _from_config(cls, config: UserProxyAgentConfig) -> Self: + return cls(name=config.name, description=config.description, input_func=None) diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/base/_termination.py b/python/packages/autogen-agentchat/src/autogen_agentchat/base/_termination.py index dcefa5a04111..d8a3adb96818 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/base/_termination.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/base/_termination.py @@ -48,7 +48,6 @@ async def main() -> None: """ component_type = "termination" - # component_config_schema = BaseModel # type: ignore @property @abstractmethod diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/conditions/_terminations.py b/python/packages/autogen-agentchat/src/autogen_agentchat/conditions/_terminations.py index d824815aeb1f..e33f2fcb70a7 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/conditions/_terminations.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/conditions/_terminations.py @@ -16,7 +16,6 @@ class StopMessageTerminationConfig(BaseModel): class StopMessageTermination(TerminationCondition, Component[StopMessageTerminationConfig]): """Terminate the conversation if a StopMessage is received.""" - component_type = "termination" component_config_schema = StopMessageTerminationConfig component_provider_override = "autogen_agentchat.conditions.StopMessageTermination" @@ -58,7 +57,6 @@ class MaxMessageTermination(TerminationCondition, Component[MaxMessageTerminatio max_messages: The maximum number of messages allowed in the conversation. """ - component_type = "termination" component_config_schema = MaxMessageTerminationConfig component_provider_override = "autogen_agentchat.conditions.MaxMessageTermination" @@ -104,7 +102,6 @@ class TextMentionTermination(TerminationCondition, Component[TextMentionTerminat text: The text to look for in the messages. """ - component_type = "termination" component_config_schema = TextMentionTerminationConfig component_provider_override = "autogen_agentchat.conditions.TextMentionTermination" @@ -159,7 +156,6 @@ class TokenUsageTermination(TerminationCondition, Component[TokenUsageTerminatio ValueError: If none of max_total_token, max_prompt_token, or max_completion_token is provided. """ - component_type = "termination" component_config_schema = TokenUsageTerminationConfig component_provider_override = "autogen_agentchat.conditions.TokenUsageTermination" @@ -234,7 +230,6 @@ class HandoffTermination(TerminationCondition, Component[HandoffTerminationConfi target (str): The target of the handoff message. """ - component_type = "termination" component_config_schema = HandoffTerminationConfig component_provider_override = "autogen_agentchat.conditions.HandoffTermination" @@ -279,7 +274,6 @@ class TimeoutTermination(TerminationCondition, Component[TimeoutTerminationConfi timeout_seconds: The maximum duration in seconds before terminating the conversation. """ - component_type = "termination" component_config_schema = TimeoutTerminationConfig component_provider_override = "autogen_agentchat.conditions.TimeoutTermination" @@ -339,7 +333,6 @@ class ExternalTermination(TerminationCondition, Component[ExternalTerminationCon """ - component_type = "termination" component_config_schema = ExternalTerminationConfig component_provider_override = "autogen_agentchat.conditions.ExternalTermination" @@ -389,7 +382,6 @@ class SourceMatchTermination(TerminationCondition, Component[SourceMatchTerminat TerminatedException: If the termination condition has already been reached. """ - component_type = "termination" component_config_schema = SourceMatchTerminationConfig component_provider_override = "autogen_agentchat.conditions.SourceMatchTermination" diff --git a/python/packages/autogen-agentchat/tests/test_assistant_agent.py b/python/packages/autogen-agentchat/tests/test_assistant_agent.py index 930b4f8f7959..e477ed3f1245 100644 --- a/python/packages/autogen-agentchat/tests/test_assistant_agent.py +++ b/python/packages/autogen-agentchat/tests/test_assistant_agent.py @@ -592,3 +592,51 @@ class BadMemory: assert not isinstance(BadMemory(), Memory) assert isinstance(ListMemory(), Memory) + + +@pytest.mark.asyncio +async def test_assistant_agent_declarative(monkeypatch: pytest.MonkeyPatch) -> None: + model = "gpt-4o-2024-05-13" + chat_completions = [ + ChatCompletion( + id="id1", + choices=[ + Choice( + finish_reason="stop", + index=0, + message=ChatCompletionMessage(content="Response to message 3", role="assistant"), + ) + ], + created=0, + model=model, + object="chat.completion", + usage=CompletionUsage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ), + ] + mock = _MockChatCompletion(chat_completions) + monkeypatch.setattr(AsyncCompletions, "create", mock.mock_create) + model_context = BufferedChatCompletionContext(buffer_size=2) + agent = AssistantAgent( + "test_agent", + model_client=OpenAIChatCompletionClient(model=model, api_key=""), + model_context=model_context, + ) + + agent_config = agent.dump_component() + assert agent_config.provider == "autogen_agentchat.agents.AssistantAgent" + + agent2 = AssistantAgent.load_component(agent_config) + assert agent2.name == agent.name + + agent3 = AssistantAgent( + "test_agent", + model_client=OpenAIChatCompletionClient(model=model, api_key=""), + model_context=model_context, + tools=[ + _pass_function, + _fail_function, + FunctionTool(_echo_function, description="Echo"), + ], + ) + with pytest.raises(NotImplementedError): + agent3.dump_component() diff --git a/python/packages/autogen-agentchat/tests/test_declarative_components.py b/python/packages/autogen-agentchat/tests/test_declarative_components.py index 35cf54f86416..4d7ba3f38bfb 100644 --- a/python/packages/autogen-agentchat/tests/test_declarative_components.py +++ b/python/packages/autogen-agentchat/tests/test_declarative_components.py @@ -11,6 +11,11 @@ TokenUsageTermination, ) from autogen_core import ComponentLoader, ComponentModel +from autogen_core.model_context import ( + BufferedChatCompletionContext, + HeadAndTailChatCompletionContext, + UnboundedChatCompletionContext, +) @pytest.mark.asyncio @@ -92,3 +97,35 @@ async def test_termination_declarative() -> None: # Test loading complex composition loaded_composite = ComponentLoader.load_component(composite_config) assert isinstance(loaded_composite, AndTerminationCondition) + + +@pytest.mark.asyncio +async def test_chat_completion_context_declarative() -> None: + unbounded_context = UnboundedChatCompletionContext() + buffered_context = BufferedChatCompletionContext(buffer_size=5) + head_tail_context = HeadAndTailChatCompletionContext(head_size=3, tail_size=2) + + # Test serialization + unbounded_config = unbounded_context.dump_component() + assert unbounded_config.provider == "autogen_core.model_context.UnboundedChatCompletionContext" + + buffered_config = buffered_context.dump_component() + assert buffered_config.provider == "autogen_core.model_context.BufferedChatCompletionContext" + assert buffered_config.config["buffer_size"] == 5 + + head_tail_config = head_tail_context.dump_component() + assert head_tail_config.provider == "autogen_core.model_context.HeadAndTailChatCompletionContext" + assert head_tail_config.config["head_size"] == 3 + assert head_tail_config.config["tail_size"] == 2 + + # Test deserialization + loaded_unbounded = ComponentLoader.load_component(unbounded_config, UnboundedChatCompletionContext) + assert isinstance(loaded_unbounded, UnboundedChatCompletionContext) + + loaded_buffered = ComponentLoader.load_component(buffered_config, BufferedChatCompletionContext) + + assert isinstance(loaded_buffered, BufferedChatCompletionContext) + + loaded_head_tail = ComponentLoader.load_component(head_tail_config, HeadAndTailChatCompletionContext) + + assert isinstance(loaded_head_tail, HeadAndTailChatCompletionContext) diff --git a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/index.md b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/index.md index 5546417eb6d2..e83288338dc7 100644 --- a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/index.md +++ b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/index.md @@ -66,6 +66,18 @@ Sample code and use cases How to migrate from AutoGen 0.2.x to 0.4.x. ::: + +:::{grid-item-card} {fas}`save;pst-color-primary` Serialize Components +:link: ./serialize-components.html + +Serialize and deserialize components +::: + +:::{grid-item-card} {fas}`brain;pst-color-primary` Memory +:link: ./memory.html + +Add memory capabilities to your agents +::: :::: ```{toctree} @@ -91,8 +103,7 @@ tutorial/human-in-the-loop tutorial/termination tutorial/custom-agents tutorial/state -tutorial/declarative -tutorial/memory + ``` ```{toctree} @@ -103,6 +114,8 @@ tutorial/memory selector-group-chat swarm magentic-one +memory +serialize-components ``` ```{toctree} diff --git a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/memory.ipynb b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/memory.ipynb similarity index 100% rename from python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/memory.ipynb rename to python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/memory.ipynb diff --git a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/serialize-components.ipynb b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/serialize-components.ipynb new file mode 100644 index 000000000000..5a3855f48080 --- /dev/null +++ b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/serialize-components.ipynb @@ -0,0 +1,171 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Serializing Components \n", + "\n", + "AutoGen provides a {py:class}`~autogen_core.Component` configuration class that defines behaviours for to serialize/deserialize component into declarative specifications. This is useful for debugging, visualizing, and even for sharing your work with others. In this notebook, we will demonstrate how to serialize multiple components to a declarative specification like a JSON file. \n", + "\n", + "\n", + "```{note}\n", + "This is work in progress\n", + "``` \n", + "\n", + "We will be implementing declarative support for the following components:\n", + "\n", + "- Termination conditions ✔️\n", + "- Tools \n", + "- Agents \n", + "- Teams \n", + "\n", + "\n", + "### Termination Condition Example \n", + "\n", + "In the example below, we will define termination conditions (a part of an agent team) in python, export this to a dictionary/json and also demonstrate how the termination condition object can be loaded from the dictionary/json. \n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Config: {\"provider\":\"autogen_agentchat.base.OrTerminationCondition\",\"component_type\":\"termination\",\"version\":1,\"component_version\":1,\"description\":null,\"config\":{\"conditions\":[{\"provider\":\"autogen_agentchat.conditions.MaxMessageTermination\",\"component_type\":\"termination\",\"version\":1,\"component_version\":1,\"config\":{\"max_messages\":5}},{\"provider\":\"autogen_agentchat.conditions.StopMessageTermination\",\"component_type\":\"termination\",\"version\":1,\"component_version\":1,\"config\":{}}]}}\n" + ] + } + ], + "source": [ + "from autogen_agentchat.conditions import MaxMessageTermination, StopMessageTermination\n", + "\n", + "max_termination = MaxMessageTermination(5)\n", + "stop_termination = StopMessageTermination()\n", + "\n", + "or_termination = max_termination | stop_termination\n", + "\n", + "or_term_config = or_termination.dump_component()\n", + "print(\"Config: \", or_term_config.model_dump_json())\n", + "\n", + "new_or_termination = or_termination.load_component(or_term_config)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Agent Example \n", + "\n", + "In the example below, we will define an agent in python, export this to a dictionary/json and also demonstrate how the agent object can be loaded from the dictionary/json." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from autogen_agentchat.agents import AssistantAgent, UserProxyAgent\n", + "from autogen_ext.models.openai import OpenAIChatCompletionClient\n", + "\n", + "# Create an agent that uses the OpenAI GPT-4o model.\n", + "model_client = OpenAIChatCompletionClient(\n", + " model=\"gpt-4o\",\n", + " # api_key=\"YOUR_API_KEY\",\n", + ")\n", + "agent = AssistantAgent(\n", + " name=\"assistant\",\n", + " model_client=model_client,\n", + " handoffs=[\"flights_refunder\", \"user\"],\n", + " # tools=[], # serializing tools is not yet supported\n", + " system_message=\"Use tools to solve tasks.\",\n", + ")\n", + "user_proxy = UserProxyAgent(name=\"user\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\"provider\":\"autogen_agentchat.agents.UserProxyAgent\",\"component_type\":\"agent\",\"version\":1,\"component_version\":1,\"description\":null,\"config\":{\"name\":\"user\",\"description\":\"A human user\"}}\n" + ] + } + ], + "source": [ + "user_proxy_config = user_proxy.dump_component() # dump component\n", + "print(user_proxy_config.model_dump_json())\n", + "up_new = user_proxy.load_component(user_proxy_config) # load component" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\"provider\":\"autogen_agentchat.agents.AssistantAgent\",\"component_type\":\"agent\",\"version\":1,\"component_version\":1,\"description\":null,\"config\":{\"name\":\"assistant\",\"model_client\":{\"provider\":\"autogen_ext.models.openai.OpenAIChatCompletionClient\",\"component_type\":\"model\",\"version\":1,\"component_version\":1,\"config\":{\"model\":\"gpt-4o\"}},\"handoffs\":[{\"target\":\"flights_refunder\",\"description\":\"Handoff to flights_refunder.\",\"name\":\"transfer_to_flights_refunder\",\"message\":\"Transferred to flights_refunder, adopting the role of flights_refunder immediately.\"},{\"target\":\"user\",\"description\":\"Handoff to user.\",\"name\":\"transfer_to_user\",\"message\":\"Transferred to user, adopting the role of user immediately.\"}],\"model_context\":{\"provider\":\"autogen_core.model_context.UnboundedChatCompletionContext\",\"component_type\":\"chat_completion_context\",\"version\":1,\"component_version\":1,\"config\":{}},\"description\":\"An agent that provides assistance with ability to use tools.\",\"system_message\":\"Use tools to solve tasks.\",\"reflect_on_tool_use\":false,\"tool_call_summary_format\":\"{result}\"}}\n" + ] + } + ], + "source": [ + "agent_config = agent.dump_component() # dump component\n", + "print(agent_config.model_dump_json())\n", + "agent_new = agent.load_component(agent_config) # load component" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "A similar approach can be used to serialize the `MultiModalWebSurfer` agent.\n", + "\n", + "```python\n", + "from autogen_ext.agents.web_surfer import MultimodalWebSurfer\n", + "\n", + "agent = MultimodalWebSurfer(\n", + " name=\"web_surfer\",\n", + " model_client=model_client,\n", + " headless=False,\n", + ")\n", + "\n", + "web_surfer_config = agent.dump_component() # dump component\n", + "print(web_surfer_config.model_dump_json())\n", + "\n", + "```" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/declarative.ipynb b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/declarative.ipynb deleted file mode 100644 index 274135c1155c..000000000000 --- a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/declarative.ipynb +++ /dev/null @@ -1,119 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Declarative Components \n", - "\n", - "AutoGen provides a declarative {py:class}`~autogen_core.Component` configuration class that defines behaviours for declarative import/export. This is useful for debugging, visualizing, and even for sharing your work with others. In this notebook, we will demonstrate how to export a declarative representation of a multiagent team in the form of a JSON file. \n", - "\n", - "\n", - "```{note}\n", - "This is work in progress\n", - "``` \n", - "\n", - "We will be implementing declarative support for the following components:\n", - "\n", - "- Termination conditions ✔️\n", - "- Tools \n", - "- Agents \n", - "- Teams \n", - "\n", - "\n", - "### Termination Condition Example \n", - "\n", - "In the example below, we will define termination conditions (a part of an agent team) in python, export this to a dictionary/json and also demonstrate how the termination condition object can be loaded from the dictionary/json. \n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from autogen_agentchat.conditions import MaxMessageTermination, StopMessageTermination\n", - "\n", - "max_termination = MaxMessageTermination(5)\n", - "stop_termination = StopMessageTermination()" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "provider='autogen_agentchat.conditions.MaxMessageTermination' component_type='termination' version=1 component_version=1 description=None config={'max_messages': 5}\n" - ] - } - ], - "source": [ - "print(max_termination.dump_component())" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'provider': 'autogen_agentchat.conditions.MaxMessageTermination', 'component_type': 'termination', 'version': 1, 'component_version': 1, 'description': None, 'config': {'max_messages': 5}}\n" - ] - } - ], - "source": [ - "print(max_termination.dump_component().model_dump())" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "ComponentModel(provider='autogen_agentchat.base.OrTerminationCondition', component_type='termination', version=1, component_version=1, description=None, config={'conditions': [{'provider': 'autogen_agentchat.conditions.MaxMessageTermination', 'component_type': 'termination', 'version': 1, 'component_version': 1, 'config': {'max_messages': 5}}, {'provider': 'autogen_agentchat.conditions.StopMessageTermination', 'component_type': 'termination', 'version': 1, 'component_version': 1, 'config': {}}]})" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "or_termination = max_termination | stop_termination\n", - "or_termination.dump_component()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/python/packages/autogen-core/src/autogen_core/model_context/_buffered_chat_completion_context.py b/python/packages/autogen-core/src/autogen_core/model_context/_buffered_chat_completion_context.py index f66197246e91..dcece60b1cd7 100644 --- a/python/packages/autogen-core/src/autogen_core/model_context/_buffered_chat_completion_context.py +++ b/python/packages/autogen-core/src/autogen_core/model_context/_buffered_chat_completion_context.py @@ -1,10 +1,19 @@ from typing import List +from pydantic import BaseModel +from typing_extensions import Self + +from .._component_config import Component from ..models import FunctionExecutionResultMessage, LLMMessage from ._chat_completion_context import ChatCompletionContext -class BufferedChatCompletionContext(ChatCompletionContext): +class BufferedChatCompletionContextConfig(BaseModel): + buffer_size: int + initial_messages: List[LLMMessage] | None = None + + +class BufferedChatCompletionContext(ChatCompletionContext, Component[BufferedChatCompletionContextConfig]): """A buffered chat completion context that keeps a view of the last n messages, where n is the buffer size. The buffer size is set at initialization. @@ -13,6 +22,9 @@ class BufferedChatCompletionContext(ChatCompletionContext): initial_messages (List[LLMMessage] | None): The initial messages. """ + component_config_schema = BufferedChatCompletionContextConfig + component_provider_override = "autogen_core.model_context.BufferedChatCompletionContext" + def __init__(self, buffer_size: int, initial_messages: List[LLMMessage] | None = None) -> None: super().__init__(initial_messages) if buffer_size <= 0: @@ -27,3 +39,10 @@ async def get_messages(self) -> List[LLMMessage]: # Remove the first message from the list. messages = messages[1:] return messages + + def _to_config(self) -> BufferedChatCompletionContextConfig: + return BufferedChatCompletionContextConfig(buffer_size=self._buffer_size, initial_messages=self._messages) + + @classmethod + def _from_config(cls, config: BufferedChatCompletionContextConfig) -> Self: + return cls(**config.model_dump()) diff --git a/python/packages/autogen-core/src/autogen_core/model_context/_chat_completion_context.py b/python/packages/autogen-core/src/autogen_core/model_context/_chat_completion_context.py index 33b1dac7fa18..d2b82ec1fb31 100644 --- a/python/packages/autogen-core/src/autogen_core/model_context/_chat_completion_context.py +++ b/python/packages/autogen-core/src/autogen_core/model_context/_chat_completion_context.py @@ -3,10 +3,11 @@ from pydantic import BaseModel, Field +from .._component_config import ComponentBase from ..models import LLMMessage -class ChatCompletionContext(ABC): +class ChatCompletionContext(ABC, ComponentBase[BaseModel]): """An abstract base class for defining the interface of a chat completion context. A chat completion context lets agents store and retrieve LLM messages. It can be implemented with different recall strategies. @@ -15,6 +16,8 @@ class ChatCompletionContext(ABC): initial_messages (List[LLMMessage] | None): The initial messages. """ + component_type = "chat_completion_context" + def __init__(self, initial_messages: List[LLMMessage] | None = None) -> None: self._messages: List[LLMMessage] = initial_messages or [] diff --git a/python/packages/autogen-core/src/autogen_core/model_context/_head_and_tail_chat_completion_context.py b/python/packages/autogen-core/src/autogen_core/model_context/_head_and_tail_chat_completion_context.py index 2518f456b632..a37d5927b19f 100644 --- a/python/packages/autogen-core/src/autogen_core/model_context/_head_and_tail_chat_completion_context.py +++ b/python/packages/autogen-core/src/autogen_core/model_context/_head_and_tail_chat_completion_context.py @@ -1,11 +1,21 @@ from typing import List +from pydantic import BaseModel +from typing_extensions import Self + +from .._component_config import Component from .._types import FunctionCall from ..models import AssistantMessage, FunctionExecutionResultMessage, LLMMessage, UserMessage from ._chat_completion_context import ChatCompletionContext -class HeadAndTailChatCompletionContext(ChatCompletionContext): +class HeadAndTailChatCompletionContextConfig(BaseModel): + head_size: int + tail_size: int + initial_messages: List[LLMMessage] | None = None + + +class HeadAndTailChatCompletionContext(ChatCompletionContext, Component[HeadAndTailChatCompletionContextConfig]): """A chat completion context that keeps a view of the first n and last m messages, where n is the head size and m is the tail size. The head and tail sizes are set at initialization. @@ -16,6 +26,9 @@ class HeadAndTailChatCompletionContext(ChatCompletionContext): initial_messages (List[LLMMessage] | None): The initial messages. """ + component_config_schema = HeadAndTailChatCompletionContextConfig + component_provider_override = "autogen_core.model_context.HeadAndTailChatCompletionContext" + def __init__(self, head_size: int, tail_size: int, initial_messages: List[LLMMessage] | None = None) -> None: super().__init__(initial_messages) if head_size <= 0: @@ -52,3 +65,12 @@ async def get_messages(self) -> List[LLMMessage]: placeholder_messages = [UserMessage(content=f"Skipped {num_skipped} messages.", source="System")] return head_messages + placeholder_messages + tail_messages + + def _to_config(self) -> HeadAndTailChatCompletionContextConfig: + return HeadAndTailChatCompletionContextConfig( + head_size=self._head_size, tail_size=self._tail_size, initial_messages=self._messages + ) + + @classmethod + def _from_config(cls, config: HeadAndTailChatCompletionContextConfig) -> Self: + return cls(head_size=config.head_size, tail_size=config.tail_size, initial_messages=config.initial_messages) diff --git a/python/packages/autogen-core/src/autogen_core/model_context/_unbounded_chat_completion_context.py b/python/packages/autogen-core/src/autogen_core/model_context/_unbounded_chat_completion_context.py index dff45bfc92d8..4bc26db46ae6 100644 --- a/python/packages/autogen-core/src/autogen_core/model_context/_unbounded_chat_completion_context.py +++ b/python/packages/autogen-core/src/autogen_core/model_context/_unbounded_chat_completion_context.py @@ -1,12 +1,30 @@ from typing import List +from pydantic import BaseModel +from typing_extensions import Self + +from .._component_config import Component from ..models import LLMMessage from ._chat_completion_context import ChatCompletionContext -class UnboundedChatCompletionContext(ChatCompletionContext): +class UnboundedChatCompletionContextConfig(BaseModel): + pass + + +class UnboundedChatCompletionContext(ChatCompletionContext, Component[UnboundedChatCompletionContextConfig]): """An unbounded chat completion context that keeps a view of the all the messages.""" + component_config_schema = UnboundedChatCompletionContextConfig + component_provider_override = "autogen_core.model_context.UnboundedChatCompletionContext" + async def get_messages(self) -> List[LLMMessage]: """Get at most `buffer_size` recent messages.""" return self._messages + + def _to_config(self) -> UnboundedChatCompletionContextConfig: + return UnboundedChatCompletionContextConfig() + + @classmethod + def _from_config(cls, config: UnboundedChatCompletionContextConfig) -> Self: + return cls() diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py index d266a2086529..f90dc01cdda2 100644 --- a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py +++ b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py @@ -24,7 +24,7 @@ from autogen_agentchat.agents import BaseChatAgent from autogen_agentchat.base import Response from autogen_agentchat.messages import AgentEvent, ChatMessage, MultiModalMessage, TextMessage -from autogen_core import EVENT_LOGGER_NAME, CancellationToken, FunctionCall +from autogen_core import EVENT_LOGGER_NAME, CancellationToken, Component, ComponentModel, FunctionCall from autogen_core import Image as AGImage from autogen_core.models import ( AssistantMessage, @@ -36,6 +36,8 @@ ) from PIL import Image from playwright.async_api import BrowserContext, Download, Page, Playwright, async_playwright +from pydantic import BaseModel +from typing_extensions import Self from ._events import WebSurferEvent from ._prompts import WEB_SURFER_OCR_PROMPT, WEB_SURFER_QA_PROMPT, WEB_SURFER_QA_SYSTEM_MESSAGE, WEB_SURFER_TOOL_PROMPT @@ -58,7 +60,23 @@ from .playwright_controller import PlaywrightController -class MultimodalWebSurfer(BaseChatAgent): +class MultimodalWebSurferConfig(BaseModel): + name: str + model_client: ComponentModel + downloads_folder: str | None = None + description: str | None = None + debug_dir: str | None = None + headless: bool = True + start_page: str | None = "https://www.bing.com/" + animate_actions: bool = False + to_save_screenshots: bool = False + use_ocr: bool = False + browser_channel: str | None = None + browser_data_dir: str | None = None + to_resize_viewport: bool = True + + +class MultimodalWebSurfer(BaseChatAgent, Component[MultimodalWebSurferConfig]): """ MultimodalWebSurfer is a multimodal agent that acts as a web surfer that can search the web and visit web pages. @@ -144,6 +162,10 @@ async def main() -> None: asyncio.run(main()) """ + component_type = "agent" + component_config_schema = MultimodalWebSurferConfig + component_provider_override = "autogen_ext.agents.web_surfer.MultimodalWebSurfer" + DEFAULT_DESCRIPTION = """ A helpful assistant with access to a web browser. Ask them to perform web searches, open pages, and interact with content (e.g., clicking links, scrolling the viewport, etc., filling in form fields, etc.). @@ -242,7 +264,8 @@ def _download_handler(download: Download) -> None: TOOL_SLEEP, TOOL_HOVER, ] - self.n_lines_page_text = 50 # Number of lines of text to extract from the page in the absence of OCR + # Number of lines of text to extract from the page in the absence of OCR + self.n_lines_page_text = 50 self.did_lazy_init = False # flag to check if we have initialized the browser async def _lazy_init( @@ -317,7 +340,8 @@ async def _set_debug_dir(self, debug_dir: str | None) -> None: if self.to_save_screenshots: current_timestamp = "_" + int(time.time()).__str__() screenshot_png_name = "screenshot" + current_timestamp + ".png" - await self._page.screenshot(path=os.path.join(self.debug_dir, screenshot_png_name)) + + await self._page.screenshot(path=os.path.join(self.debug_dir, screenshot_png_name)) # type: ignore self.logger.info( WebSurferEvent( source=self.name, @@ -346,6 +370,7 @@ async def on_reset(self, cancellation_token: CancellationToken) -> None: if self.to_save_screenshots: current_timestamp = "_" + int(time.time()).__str__() screenshot_png_name = "screenshot" + current_timestamp + ".png" + await self._page.screenshot(path=os.path.join(self.debug_dir, screenshot_png_name)) # type: ignore self.logger.info( WebSurferEvent( @@ -704,6 +729,7 @@ async def _execute_tool( if self.to_save_screenshots: current_timestamp = "_" + int(time.time()).__str__() screenshot_png_name = "screenshot" + current_timestamp + ".png" + async with aiofiles.open(os.path.join(self.debug_dir, screenshot_png_name), "wb") as file: # type: ignore await file.write(new_screenshot) # type: ignore self.logger.info( @@ -861,3 +887,38 @@ async def _summarize_page( scaled_screenshot.close() assert isinstance(response.content, str) return response.content + + def _to_config(self) -> MultimodalWebSurferConfig: + return MultimodalWebSurferConfig( + name=self.name, + model_client=self._model_client.dump_component(), + downloads_folder=self.downloads_folder, + description=self.description, + debug_dir=self.debug_dir, + headless=self.headless, + start_page=self.start_page, + animate_actions=self.animate_actions, + to_save_screenshots=self.to_save_screenshots, + use_ocr=self.use_ocr, + browser_channel=self.browser_channel, + browser_data_dir=self.browser_data_dir, + to_resize_viewport=self.to_resize_viewport, + ) + + @classmethod + def _from_config(cls, config: MultimodalWebSurferConfig) -> Self: + return cls( + name=config.name, + model_client=ChatCompletionClient.load_component(config.model_client), + downloads_folder=config.downloads_folder, + description=config.description or cls.DEFAULT_DESCRIPTION, + debug_dir=config.debug_dir, + headless=config.headless, + start_page=config.start_page or cls.DEFAULT_START_PAGE, + animate_actions=config.animate_actions, + to_save_screenshots=config.to_save_screenshots, + use_ocr=config.use_ocr, + browser_channel=config.browser_channel, + browser_data_dir=config.browser_data_dir, + to_resize_viewport=config.to_resize_viewport, + ) diff --git a/python/packages/autogen-ext/tests/test_websurfer_agent.py b/python/packages/autogen-ext/tests/test_websurfer_agent.py index d8a36e4d9549..a2aa33a10931 100644 --- a/python/packages/autogen-ext/tests/test_websurfer_agent.py +++ b/python/packages/autogen-ext/tests/test_websurfer_agent.py @@ -145,3 +145,38 @@ async def test_run_websurfer(monkeypatch: pytest.MonkeyPatch) -> None: ) # type: ignore url_after_sleep = agent._page.url # type: ignore assert url_after_no_tool == url_after_sleep + + +@pytest.mark.asyncio +async def test_run_websurfer_declarative(monkeypatch: pytest.MonkeyPatch) -> None: + model = "gpt-4o-2024-05-13" + chat_completions = [ + ChatCompletion( + id="id1", + choices=[ + Choice( + finish_reason="stop", + index=0, + message=ChatCompletionMessage(content="Response to message 3", role="assistant"), + ) + ], + created=0, + model=model, + object="chat.completion", + usage=CompletionUsage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ), + ] + mock = _MockChatCompletion(chat_completions) + monkeypatch.setattr(AsyncCompletions, "create", mock.mock_create) + + agent = MultimodalWebSurfer( + "WebSurfer", model_client=OpenAIChatCompletionClient(model=model, api_key=""), use_ocr=False + ) + + agent_config = agent.dump_component() + assert agent_config.provider == "autogen_ext.agents.web_surfer.MultimodalWebSurfer" + assert agent_config.config["name"] == "WebSurfer" + + loaded_agent = MultimodalWebSurfer.load_component(agent_config) + assert isinstance(loaded_agent, MultimodalWebSurfer) + assert loaded_agent.name == "WebSurfer"