From 9014f6b9722fb6ea877916b2a0ac2e6e5fc62777 Mon Sep 17 00:00:00 2001 From: pulinduvidmal Date: Fri, 19 Dec 2025 10:57:46 +0530 Subject: [PATCH 1/9] feat: add file support for messenger integration --- .../integration/messenger/messenger_chat.py | 421 +++++++++++++++++- examples/api/a2a/multi/uv.lock | 8 +- examples/api/gmail/uv.lock | 8 +- examples/api/hooks/uv.lock | 8 +- examples/api/instagram/uv.lock | 8 +- examples/api/mcp/multi/uv.lock | 8 +- examples/api/messenger/uv.lock | 80 ++-- examples/api/openai/uv.lock | 8 +- examples/api/slack/uv.lock | 8 +- examples/api/telegram/uv.lock | 8 +- examples/api/whatsapp/uv.lock | 8 +- .../aws-containerized/openai-dynamodb/uv.lock | 8 +- examples/aws-serverless/openai/uv.lock | 8 +- examples/cli/multi/uv.lock | 8 +- examples/cli/openai-dynamic/uv.lock | 8 +- examples/cli/openai/uv.lock | 2 +- examples/containerized/openai/uv.lock | 8 +- 17 files changed, 509 insertions(+), 106 deletions(-) diff --git a/ak-py/src/agentkernel/integration/messenger/messenger_chat.py b/ak-py/src/agentkernel/integration/messenger/messenger_chat.py index 1cc6d883..a9cf6dc0 100644 --- a/ak-py/src/agentkernel/integration/messenger/messenger_chat.py +++ b/ak-py/src/agentkernel/integration/messenger/messenger_chat.py @@ -1,3 +1,4 @@ +import base64 import hashlib import hmac import logging @@ -7,7 +8,13 @@ from fastapi import APIRouter, HTTPException, Request from ...api import RESTRequestHandler -from ...core import AgentService, Config +from ...core import ( + AgentRequestFile, + AgentRequestImage, + AgentRequestText, + AgentService, + Config, +) class AgentMessengerRequestHandler(RESTRequestHandler): @@ -144,18 +151,22 @@ async def _handle_message(self, messaging_event: dict): message = messaging_event.get("message", {}) message_id = message.get("mid") message_text = message.get("text") + attachments = message.get("attachments", []) if not sender_id or not message_id: self._log.warning("Message missing required fields (sender/mid)") return - # Skip messages with attachments that don't have text - if not message_text: - self._log.warning("Message has no text content") + # Allow message if it has text OR attachments + if not message_text and not attachments: + self._log.warning("Message has no text or attachments") return - self._log.debug(f"Processing message {message_id} from {sender_id}: {message_text}") - await self._process_agent_message(sender_id, message_text) + if message_text: + self._log.debug(f"Processing message {message_id} from {sender_id}: {message_text}") + if attachments: + self._log.debug(f"Processing message {message_id} from {sender_id} with {len(attachments)} attachment(s)") + await self._process_agent_message(sender_id, message_text or "", attachments) async def _handle_postback(self, messaging_event: dict): """ @@ -184,7 +195,10 @@ async def _handle_postback(self, messaging_event: dict): self._log.debug(f"Processing postback from {sender_id}: {message_text}") await self._process_agent_message(sender_id, message_text) - async def _process_agent_message(self, sender_id: str, message_text: str): + async def _process_agent_message(self, sender_id: str, message_text: str, attachments: list = None): + if attachments is None: + attachments = [] + service = AgentService() session_id = sender_id # Use sender_id as session_id to maintain conversation context try: @@ -202,8 +216,66 @@ async def _process_agent_message(self, sender_id: str, message_text: str): await self._send_typing_indicator(sender_id, False) return - # Run the agent - result = await service.run(message_text) + # Extract and download attachments + processed_attachments = await self._extract_attachments(attachments) + + # Get session to store/retrieve attachment history + session = service.session + + # Store current attachment metadata in session + if processed_attachments: + self._store_attachment_metadata(session, processed_attachments) + + # Build request list: start with text only if it's not empty + requests = [] + if message_text.strip(): # Only add text if it's not empty + requests.append(AgentRequestText(text=message_text)) + requests.extend(processed_attachments) + + # Add conversation context if available + conversation_context = self._get_conversation_context(session) + if conversation_context: + requests.insert(0, AgentRequestText(text=conversation_context)) + self._log.info(f"[AGENT_CONTEXT] Added conversation history to context") + + # Add previous attachments to the request if available + previous_attachments = self._get_previous_attachments(session, processed_attachments) + self._log.info(f"[DEBUG] previous_attachments returned: {len(previous_attachments)} items") + if previous_attachments: + for att in previous_attachments: + self._log.info( + f"[DEBUG] Previous attachment: name={getattr(att, 'name', '?')}, has_data={bool(getattr(att, 'image_data' if 'Image' in type(att).__name__ else 'file_data', None))}" + ) + requests.extend(previous_attachments) + self._log.info(f"[AGENT_CONTEXT] Added {len(previous_attachments)} previous attachment(s) to context") + else: + self._log.info(f"[DEBUG] No previous attachments to add") + + # Log request summary + self._log.info( + f"[AGENT_INPUT] Total requests: {len(requests)} (text + {len(processed_attachments)} attachment(s) + context)" + ) + + # Store user message in conversation history for context (before agent runs) + if message_text.strip(): + self._store_user_message(session, message_text) + elif processed_attachments: + # Even if no text, store that user sent attachment(s) for context + attachment_names = [getattr(att, "name", "file") for att in processed_attachments] + self._store_user_message(session, f"[sent image/file: {', '.join(attachment_names)}]") + + # Run the agent - always use run_multi if there are requests + if len(requests) > 0: + self._log.info( + f"[AGENT_CALL] Running agent with {len(requests)} request(s) (text + {len(processed_attachments)} attachment(s))" + ) + result = await service.run_multi(requests) + else: + # No requests at all - nothing to process + self._log.warning("No text or attachments to process") + await self._send_message(sender_id, "Please send a message or attachment.") + await self._send_typing_indicator(sender_id, False) + return if hasattr(result, "raw"): response_text = str(result.raw) @@ -212,6 +284,9 @@ async def _process_agent_message(self, sender_id: str, message_text: str): self._log.debug(f"Agent response: {response_text}") + # Store agent response in session for context + self._store_agent_response(session, response_text) + # Turn off typing indicator and send the response await self._send_typing_indicator(sender_id, False) await self._send_message(sender_id, response_text) @@ -302,3 +377,331 @@ async def _mark_seen(self, recipient_id: str): self._log.debug(f"Message marked as seen: {recipient_id}") except Exception as e: self._log.warning(f"Failed to mark message as seen: {e}") + + async def _extract_attachments(self, attachments: list) -> list: + """ + Extract and download attachments from Messenger message. + Messenger format: [{"type": "image", "payload": {"url": "..."}}] + + :param attachments: List of attachment objects from Messenger webhook + :return: List of AgentRequestImage or AgentRequestFile objects + """ + processed_attachments = [] + + if not attachments: + return processed_attachments + + try: + async with httpx.AsyncClient() as client: + for attachment in attachments: + attachment_type = attachment.get("type", "") + payload = attachment.get("payload", {}) + url = payload.get("url") + + if not url: + self._log.warning("Attachment missing URL") + continue + + try: + # Download attachment + response = await client.get(url) + response.raise_for_status() + + # Get MIME type from Content-Type header + mime_type = response.headers.get("content-type", "application/octet-stream").split(";")[0] + + # Convert to base64 + file_data = base64.b64encode(response.content).decode("utf-8") + + # Extract filename from URL or use default + filename = url.split("/")[-1].split("?")[0] or f"attachment_{len(processed_attachments)}" + + # Create appropriate request object + if attachment_type == "image" or mime_type.startswith("image/"): + request = AgentRequestImage(image_data=file_data, name=filename, mime_type=mime_type) + processed_attachments.append(request) + self._log.debug(f"Extracted image: {filename} ({mime_type})") + + elif attachment_type == "file" or mime_type in [ + "application/pdf", + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.ms-excel", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ]: + request = AgentRequestFile(file_data=file_data, name=filename, mime_type=mime_type) + processed_attachments.append(request) + self._log.debug(f"Extracted file: {filename} ({mime_type})") + + else: + self._log.debug(f"Skipping unsupported attachment type: {mime_type}") + + except Exception as e: + self._log.warning(f"Error downloading attachment from {url}: {e}") + continue + + except Exception as e: + self._log.warning(f"Error processing attachments: {e}\n{traceback.format_exc()}") + + if processed_attachments: + self._log.info(f"Extracted {len(processed_attachments)} attachment(s)") + + return processed_attachments + + def _store_attachment_metadata(self, session, processed_attachments: list): + """ + Store attachment data and metadata in session for future reference. + Stores the actual base64-encoded data so images/files can be re-used. + + :param session: Agent session object + :param processed_attachments: List of AgentRequestImage or AgentRequestFile objects + """ + try: + import json + from datetime import datetime + + # Get existing attachment history + history = session.get_data(key="attachment_history") + if not history: + history = [] + else: + # Parse if it's a JSON string + if isinstance(history, str): + history = json.loads(history) + + # Add new attachments with full data + for attachment in processed_attachments: + # Extract base64 data based on type + if hasattr(attachment, "image_data"): + file_data = attachment.image_data + elif hasattr(attachment, "file_data"): + file_data = attachment.file_data + else: + file_data = None + + attachment_info = { + "name": getattr(attachment, "name", "unknown"), + "mime_type": getattr(attachment, "mime_type", "unknown"), + "type": type(attachment).__name__, # AgentRequestImage or AgentRequestFile + "data": file_data, # Store the actual base64 data + "timestamp": datetime.now().isoformat(), + } + history.append(attachment_info) + self._log.debug( + f"Stored attachment in session: {attachment_info['name']} ({attachment_info['mime_type']})" + ) + + # Keep only last 10 attachments to avoid session bloat (full data takes more space) + if len(history) > 10: + history = history[-10:] + + # Store back to session + session.set_data(key="attachment_history", data=json.dumps(history)) + self._log.info( + f"[DEBUG] Stored {len(processed_attachments)} attachment(s) in session. Total history: {len(history)}" + ) + self._log.info(f"[DEBUG] Attachment history keys: {[h['name'] for h in history]}") + + except Exception as e: + self._log.warning(f"Error storing attachment data: {e}") + + def _get_previous_attachments(self, session, current_attachments: list) -> list: + """ + Retrieve previous attachments from session and recreate request objects. + Only returns attachments NOT in the current message (to avoid duplicates). + + :param session: Agent session object + :param current_attachments: List of current attachments (to skip duplicates) + :return: List of AgentRequestImage or AgentRequestFile objects from history + """ + try: + import json + + history = session.get_data(key="attachment_history") + self._log.info( + f"[DEBUG] Retrieved history from session: {type(history)}, length: {len(history) if history else 0}" + ) + + if not history: + self._log.info(f"[DEBUG] History is empty or None") + return [] + + # Parse if it's a JSON string + if isinstance(history, str): + history = json.loads(history) + self._log.info(f"[DEBUG] Parsed JSON history, items: {len(history)}") + + if not history: + return [] + + # Get names of current attachments to skip duplicates + current_names = {getattr(att, "name", "") for att in current_attachments} + self._log.info(f"[DEBUG] Current attachment names: {current_names}") + + # Recreate request objects from history (excluding current) + previous_attachments = [] + for attachment_info in history: + att_name = attachment_info.get("name", "unknown") + self._log.info( + f"[DEBUG] Checking history item: {att_name}, in current_names? {att_name in current_names}" + ) + + # Skip if it's in the current message + if att_name in current_names: + self._log.info(f"[DEBUG] Skipping {att_name} (duplicate)") + continue + + attachment_type = attachment_info.get("type", "AgentRequestFile") + mime_type = attachment_info.get("mime_type", "unknown") + file_data = attachment_info.get("data") + name = attachment_info.get("name", "unknown") + + if not file_data: + self._log.warning(f"[DEBUG] No file_data for {name}") + continue + + # Recreate the appropriate request object + try: + if attachment_type == "AgentRequestImage": + request = AgentRequestImage(image_data=file_data, name=name, mime_type=mime_type) + previous_attachments.append(request) + self._log.info(f"[DEBUG] Recreated image: {name} ({len(file_data)} bytes)") + elif attachment_type == "AgentRequestFile": + request = AgentRequestFile(file_data=file_data, name=name, mime_type=mime_type) + previous_attachments.append(request) + self._log.info(f"[DEBUG] Recreated file: {name} ({len(file_data)} bytes)") + else: + self._log.warning(f"[DEBUG] Unknown attachment type: {attachment_type}") + except Exception as e: + self._log.warning(f"Error recreating attachment {name}: {e}") + continue + + self._log.info(f"[DEBUG] Total previous attachments recreated: {len(previous_attachments)}") + return previous_attachments + + except Exception as e: + self._log.warning(f"Error retrieving previous attachments: {e}\n{traceback.format_exc()}") + return [] + + def _store_agent_response(self, session, response_text: str): + """ + Store agent response in session for conversation context. + Helps agent understand what it already said in previous turns. + + :param session: Agent session object + :param response_text: Agent's response text + """ + try: + import json + from datetime import datetime + + # Get existing conversation history + history = session.get_data(key="conversation_history") + if not history: + history = [] + else: + # Parse if it's a JSON string + if isinstance(history, str): + history = json.loads(history) + + # Add agent response + history.append( + { + "role": "agent", + "content": response_text, + "timestamp": datetime.now().isoformat(), + } + ) + + # Keep only last 20 exchanges to avoid session bloat + if len(history) > 20: + history = history[-20:] + + # Store back to session + session.set_data(key="conversation_history", data=json.dumps(history)) + self._log.info(f"[DEBUG] Stored agent response in conversation history. Total: {len(history)}") + + except Exception as e: + self._log.warning(f"Error storing agent response: {e}") + + def _store_user_message(self, session, message_text: str): + """ + Store user message in session for conversation context. + Helps agent understand the full conversation flow and context. + + :param session: Agent session object + :param message_text: User's message text + """ + try: + import json + from datetime import datetime + + # Get existing conversation history + history = session.get_data(key="conversation_history") + if not history: + history = [] + else: + # Parse if it's a JSON string + if isinstance(history, str): + history = json.loads(history) + + # Add user message + history.append( + { + "role": "user", + "content": message_text, + "timestamp": datetime.now().isoformat(), + } + ) + + # Keep only last 20 exchanges to avoid session bloat + if len(history) > 20: + history = history[-20:] + + # Store back to session + session.set_data(key="conversation_history", data=json.dumps(history)) + self._log.info(f"[DEBUG] Stored user message in conversation history. Total: {len(history)}") + + except Exception as e: + self._log.warning(f"Error storing user message: {e}") + + def _get_conversation_context(self, session) -> str: + """ + Retrieve conversation history and format as context for agent. + Helps agent understand what was already discussed. + + :param session: Agent session object + :return: Formatted conversation context string + """ + try: + import json + + history = session.get_data(key="conversation_history") + if not history: + return "" + + # Parse if it's a JSON string + if isinstance(history, str): + history = json.loads(history) + + if not history: + return "" + + # Format as readable context + context_lines = ["[Previous conversation context:"] + for item in history[-10:]: # Last 10 messages to cover multi-turn conversations + role = item.get("role", "unknown") + content = item.get("content", "") + # Truncate long responses to 200 chars + if len(content) > 200: + content = content[:200] + "..." + context_lines.append(f" {role}: {content}") + context_lines.append("]") + + result = "\n".join(context_lines) + self._log.info(f"[DEBUG] Retrieved conversation context ({len(history)} items)") + return result + + except Exception as e: + self._log.warning(f"Error retrieving conversation context: {e}") + return "" diff --git a/examples/api/a2a/multi/uv.lock b/examples/api/a2a/multi/uv.lock index 663af8b6..4a787600 100644 --- a/examples/api/a2a/multi/uv.lock +++ b/examples/api/a2a/multi/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.13'", @@ -2087,7 +2087,7 @@ wheels = [ [[package]] name = "openai-agents" -version = "0.6.4" +version = "0.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "griffe" }, @@ -2098,9 +2098,9 @@ dependencies = [ { name = "types-requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/36/826ce8ad497904a1becdedef80326c2fe932c754646b77405a8dc4cd49f7/openai_agents-0.6.4.tar.gz", hash = "sha256:07836865ed9c37946523d44b2d87ad375673b6558e783fa086db004a892331ec", size = 2022961, upload-time = "2025-12-19T06:42:55.356Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/0b/1bfc1f47708ce5500ad6b05ba8a0a789232ee6f5b9dd68938131c4674533/openai_agents-0.6.3.tar.gz", hash = "sha256:436479f201910cfc466893854b47d0f3acbf7b3bdafa95eedb590ed0d40393ef", size = 2016166, upload-time = "2025-12-11T18:07:47.823Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/ed/f2282debf62c52241959b111d2e35105b43b2ea6ff060833734405bb4d0f/openai_agents-0.6.4-py3-none-any.whl", hash = "sha256:d14635c1fa0ee39e79b81e5cab2f22dd5024772d3dbc0770d8307fd2548b3951", size = 241982, upload-time = "2025-12-19T06:42:53.92Z" }, + { url = "https://files.pythonhosted.org/packages/3e/06/d4bf0a8403ebc7d6b0fb2b45e41d6da6996b20f1dde1debffdac1b5ccb63/openai_agents-0.6.3-py3-none-any.whl", hash = "sha256:ada8b598f4db787939a62c8a291d07cbe68dae2d635955c44a0a0300746ee84f", size = 239015, upload-time = "2025-12-11T18:07:46.275Z" }, ] [[package]] diff --git a/examples/api/gmail/uv.lock b/examples/api/gmail/uv.lock index 9ea2d7e4..cb52e44a 100644 --- a/examples/api/gmail/uv.lock +++ b/examples/api/gmail/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14'", @@ -2000,7 +2000,7 @@ wheels = [ [[package]] name = "openai-agents" -version = "0.6.4" +version = "0.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "griffe" }, @@ -2011,9 +2011,9 @@ dependencies = [ { name = "types-requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/36/826ce8ad497904a1becdedef80326c2fe932c754646b77405a8dc4cd49f7/openai_agents-0.6.4.tar.gz", hash = "sha256:07836865ed9c37946523d44b2d87ad375673b6558e783fa086db004a892331ec", size = 2022961, upload-time = "2025-12-19T06:42:55.356Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/0b/1bfc1f47708ce5500ad6b05ba8a0a789232ee6f5b9dd68938131c4674533/openai_agents-0.6.3.tar.gz", hash = "sha256:436479f201910cfc466893854b47d0f3acbf7b3bdafa95eedb590ed0d40393ef", size = 2016166, upload-time = "2025-12-11T18:07:47.823Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/ed/f2282debf62c52241959b111d2e35105b43b2ea6ff060833734405bb4d0f/openai_agents-0.6.4-py3-none-any.whl", hash = "sha256:d14635c1fa0ee39e79b81e5cab2f22dd5024772d3dbc0770d8307fd2548b3951", size = 241982, upload-time = "2025-12-19T06:42:53.92Z" }, + { url = "https://files.pythonhosted.org/packages/3e/06/d4bf0a8403ebc7d6b0fb2b45e41d6da6996b20f1dde1debffdac1b5ccb63/openai_agents-0.6.3-py3-none-any.whl", hash = "sha256:ada8b598f4db787939a62c8a291d07cbe68dae2d635955c44a0a0300746ee84f", size = 239015, upload-time = "2025-12-11T18:07:46.275Z" }, ] [[package]] diff --git a/examples/api/hooks/uv.lock b/examples/api/hooks/uv.lock index b33429c6..62c1dbca 100644 --- a/examples/api/hooks/uv.lock +++ b/examples/api/hooks/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14'", @@ -1801,7 +1801,7 @@ wheels = [ [[package]] name = "openai-agents" -version = "0.6.4" +version = "0.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "griffe" }, @@ -1812,9 +1812,9 @@ dependencies = [ { name = "types-requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/36/826ce8ad497904a1becdedef80326c2fe932c754646b77405a8dc4cd49f7/openai_agents-0.6.4.tar.gz", hash = "sha256:07836865ed9c37946523d44b2d87ad375673b6558e783fa086db004a892331ec", size = 2022961, upload-time = "2025-12-19T06:42:55.356Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/0b/1bfc1f47708ce5500ad6b05ba8a0a789232ee6f5b9dd68938131c4674533/openai_agents-0.6.3.tar.gz", hash = "sha256:436479f201910cfc466893854b47d0f3acbf7b3bdafa95eedb590ed0d40393ef", size = 2016166, upload-time = "2025-12-11T18:07:47.823Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/ed/f2282debf62c52241959b111d2e35105b43b2ea6ff060833734405bb4d0f/openai_agents-0.6.4-py3-none-any.whl", hash = "sha256:d14635c1fa0ee39e79b81e5cab2f22dd5024772d3dbc0770d8307fd2548b3951", size = 241982, upload-time = "2025-12-19T06:42:53.92Z" }, + { url = "https://files.pythonhosted.org/packages/3e/06/d4bf0a8403ebc7d6b0fb2b45e41d6da6996b20f1dde1debffdac1b5ccb63/openai_agents-0.6.3-py3-none-any.whl", hash = "sha256:ada8b598f4db787939a62c8a291d07cbe68dae2d635955c44a0a0300746ee84f", size = 239015, upload-time = "2025-12-11T18:07:46.275Z" }, ] [[package]] diff --git a/examples/api/instagram/uv.lock b/examples/api/instagram/uv.lock index 10796028..e7c90fdb 100644 --- a/examples/api/instagram/uv.lock +++ b/examples/api/instagram/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14'", @@ -1883,7 +1883,7 @@ wheels = [ [[package]] name = "openai-agents" -version = "0.6.4" +version = "0.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "griffe" }, @@ -1894,9 +1894,9 @@ dependencies = [ { name = "types-requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/36/826ce8ad497904a1becdedef80326c2fe932c754646b77405a8dc4cd49f7/openai_agents-0.6.4.tar.gz", hash = "sha256:07836865ed9c37946523d44b2d87ad375673b6558e783fa086db004a892331ec", size = 2022961, upload-time = "2025-12-19T06:42:55.356Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/0b/1bfc1f47708ce5500ad6b05ba8a0a789232ee6f5b9dd68938131c4674533/openai_agents-0.6.3.tar.gz", hash = "sha256:436479f201910cfc466893854b47d0f3acbf7b3bdafa95eedb590ed0d40393ef", size = 2016166, upload-time = "2025-12-11T18:07:47.823Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/ed/f2282debf62c52241959b111d2e35105b43b2ea6ff060833734405bb4d0f/openai_agents-0.6.4-py3-none-any.whl", hash = "sha256:d14635c1fa0ee39e79b81e5cab2f22dd5024772d3dbc0770d8307fd2548b3951", size = 241982, upload-time = "2025-12-19T06:42:53.92Z" }, + { url = "https://files.pythonhosted.org/packages/3e/06/d4bf0a8403ebc7d6b0fb2b45e41d6da6996b20f1dde1debffdac1b5ccb63/openai_agents-0.6.3-py3-none-any.whl", hash = "sha256:ada8b598f4db787939a62c8a291d07cbe68dae2d635955c44a0a0300746ee84f", size = 239015, upload-time = "2025-12-11T18:07:46.275Z" }, ] [[package]] diff --git a/examples/api/mcp/multi/uv.lock b/examples/api/mcp/multi/uv.lock index b6398339..8a3249aa 100644 --- a/examples/api/mcp/multi/uv.lock +++ b/examples/api/mcp/multi/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.13'", @@ -2222,7 +2222,7 @@ wheels = [ [[package]] name = "openai-agents" -version = "0.6.4" +version = "0.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "griffe" }, @@ -2233,9 +2233,9 @@ dependencies = [ { name = "types-requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/36/826ce8ad497904a1becdedef80326c2fe932c754646b77405a8dc4cd49f7/openai_agents-0.6.4.tar.gz", hash = "sha256:07836865ed9c37946523d44b2d87ad375673b6558e783fa086db004a892331ec", size = 2022961, upload-time = "2025-12-19T06:42:55.356Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/0b/1bfc1f47708ce5500ad6b05ba8a0a789232ee6f5b9dd68938131c4674533/openai_agents-0.6.3.tar.gz", hash = "sha256:436479f201910cfc466893854b47d0f3acbf7b3bdafa95eedb590ed0d40393ef", size = 2016166, upload-time = "2025-12-11T18:07:47.823Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/ed/f2282debf62c52241959b111d2e35105b43b2ea6ff060833734405bb4d0f/openai_agents-0.6.4-py3-none-any.whl", hash = "sha256:d14635c1fa0ee39e79b81e5cab2f22dd5024772d3dbc0770d8307fd2548b3951", size = 241982, upload-time = "2025-12-19T06:42:53.92Z" }, + { url = "https://files.pythonhosted.org/packages/3e/06/d4bf0a8403ebc7d6b0fb2b45e41d6da6996b20f1dde1debffdac1b5ccb63/openai_agents-0.6.3-py3-none-any.whl", hash = "sha256:ada8b598f4db787939a62c8a291d07cbe68dae2d635955c44a0a0300746ee84f", size = 239015, upload-time = "2025-12-11T18:07:46.275Z" }, ] [[package]] diff --git a/examples/api/messenger/uv.lock b/examples/api/messenger/uv.lock index e7357fc7..f6db7a62 100644 --- a/examples/api/messenger/uv.lock +++ b/examples/api/messenger/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14'", @@ -10,7 +10,7 @@ resolution-markers = [ [[package]] name = "agentkernel" version = "0.2.9" -source = { registry = "https://pypi.org/simple" } +source = { registry = "../../../ak-py/dist" } dependencies = [ { name = "deprecated" }, { name = "pydantic" }, @@ -18,9 +18,9 @@ dependencies = [ { name = "pyyaml" }, { name = "singleton-type" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/75/a4/c3b7b1663131bd275b99bb91d77d66983f598c9e0b686f57558ab9b6e1e2/agentkernel-0.2.9.tar.gz", hash = "sha256:976ccc343039f35ab4661e97f42f935ec88627ba1763980c68f983523ef09fdc", size = 76953, upload-time = "2025-12-18T12:28:27.293Z" } +sdist = { path = "agentkernel-0.2.9.tar.gz" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e6/6b/ea9652f7d83c29fdff8322bcb4a0ac8fa6c55214f9e0334a947a86ac2d95/agentkernel-0.2.9-py3-none-any.whl", hash = "sha256:307d6299baf7f9f579ab100492ddc2a66facbb8fc3b2960937b007f40bb66efa", size = 121841, upload-time = "2025-12-18T12:28:26.002Z" }, + { path = "agentkernel-0.2.9-py3-none-any.whl" }, ] [package.optional-dependencies] @@ -1299,7 +1299,7 @@ wheels = [ [[package]] name = "langchain-core" -version = "1.2.3" +version = "1.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jsonpatch" }, @@ -1311,23 +1311,23 @@ dependencies = [ { name = "typing-extensions" }, { name = "uuid-utils" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/70/ea/8380184b287da43d3d2556475b985cf3e27569e9d8bbe33195600a98cabb/langchain_core-1.2.3.tar.gz", hash = "sha256:61f5197aa101cd5605879ef37f2b0ac56c079974d94d347849b8d4fe18949746", size = 803567, upload-time = "2025-12-18T20:13:10.574Z" } +sdist = { url = "https://files.pythonhosted.org/packages/70/45/3d63fd7dc873abd9a0b1960775554dcc2a45dd4905937ec0b3d101dd5f10/langchain_core-1.2.2.tar.gz", hash = "sha256:3f9c28ec6d0fe47636d28b19799794458d55da81f37309832b2b9d11c93c5e95", size = 803123, upload-time = "2025-12-16T20:25:53.788Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f0/57/cfc1d12e273d33d16bab7ce9a135244e6f5677a92a5a99e69a61b22b7d93/langchain_core-1.2.3-py3-none-any.whl", hash = "sha256:c3501cf0219daf67a0ae23f6d6bdf3b41ab695efd8f0f3070a566e368b8c3dc7", size = 476384, upload-time = "2025-12-18T20:13:08.998Z" }, + { url = "https://files.pythonhosted.org/packages/59/97/57497c8b26829e38c8dd4abe972d75e38fc3904324a3042bb01d9e0753b8/langchain_core-1.2.2-py3-none-any.whl", hash = "sha256:3a83dc14217de5cba11b1a0bd43c48702401bbd18dc25cac2ffab5ac83a61cd0", size = 476125, upload-time = "2025-12-16T20:25:52.581Z" }, ] [[package]] name = "langchain-openai" -version = "1.1.6" +version = "1.1.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, { name = "openai" }, { name = "tiktoken" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ae/67/228dc28b4498ea16422577013b5bb4ba35a1b99f8be975d6747c7a9f7e6a/langchain_openai-1.1.6.tar.gz", hash = "sha256:e306612654330ae36fb6bbe36db91c98534312afade19e140c3061fe4208dac8", size = 1038310, upload-time = "2025-12-18T17:58:52.84Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/1c/008a6dd7b3523121be1a4f24701b099ae79193dab9b329dfb787bece08bf/langchain_openai-1.1.5.tar.gz", hash = "sha256:a8ca5f3919bd948867c7d427a575b34f7c141110ef7cbc14ea7bbc46363871de", size = 1038129, upload-time = "2025-12-17T19:14:36.392Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/db/5b/1f6521df83c1a8e8d3f52351883b59683e179c0aa1bec75d0a77a394c9e7/langchain_openai-1.1.6-py3-none-any.whl", hash = "sha256:c42d04a67a85cee1d994afe400800d2b09ebf714721345f0b651eb06a02c3948", size = 84701, upload-time = "2025-12-18T17:58:51.527Z" }, + { url = "https://files.pythonhosted.org/packages/e8/c5/22b690a27ba6b1ca6876270473aab1610cb8767314e5038cb6b826d9b69b/langchain_openai-1.1.5-py3-none-any.whl", hash = "sha256:d3a3b0c39e1513bbb9e5d4526c194909a00c5733195dbe90bfea6619b00420ca", size = 84569, upload-time = "2025-12-17T19:14:35.529Z" }, ] [[package]] @@ -1387,15 +1387,15 @@ wheels = [ [[package]] name = "langgraph-sdk" -version = "0.3.1" +version = "0.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, { name = "orjson" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a9/d3/b6be0b0aba2a53a8920a2b0b4328a83121ec03eea9952e576d06a4182f6f/langgraph_sdk-0.3.1.tar.gz", hash = "sha256:f6dadfd2444eeff3e01405a9005c95fb3a028d4bd954ebec80ea6150084f92bb", size = 130312, upload-time = "2025-12-18T22:11:47.42Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2b/1b/f328afb4f24f6e18333ff357d9580a3bb5b133ff2c7aae34fef7f5b87f31/langgraph_sdk-0.3.0.tar.gz", hash = "sha256:4145bc3c34feae227ae918341f66d3ba7d1499722c1ef4a8aae5ea828897d1d4", size = 130366, upload-time = "2025-12-12T22:19:30.323Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ab/fe/0c1c9c01a154eba62b20b02fabe811fd94a2b810061ae9e4d8462b8cf85a/langgraph_sdk-0.3.1-py3-none-any.whl", hash = "sha256:0b856923bfd20bf3441ce9d03bef488aa333fb610e972618799a9d584436acad", size = 66517, upload-time = "2025-12-18T22:11:46.625Z" }, + { url = "https://files.pythonhosted.org/packages/69/48/ee4d7afb3c3d38bd2ebe51a4d37f1ed7f1058dd242f35994b562203067aa/langgraph_sdk-0.3.0-py3-none-any.whl", hash = "sha256:c1ade483fba17ae354ee920e4779042b18d5aba875f2a858ba569f62f628f26f", size = 66489, upload-time = "2025-12-12T22:19:29.228Z" }, ] [[package]] @@ -1812,7 +1812,7 @@ wheels = [ [[package]] name = "openai" -version = "2.14.0" +version = "2.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -1824,14 +1824,14 @@ dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d8/b1/12fe1c196bea326261718eb037307c1c1fe1dedc2d2d4de777df822e6238/openai-2.14.0.tar.gz", hash = "sha256:419357bedde9402d23bf8f2ee372fca1985a73348debba94bddff06f19459952", size = 626938, upload-time = "2025-12-19T03:28:45.742Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/39/8e347e9fda125324d253084bb1b82407e5e3c7777a03dc398f79b2d95626/openai-2.13.0.tar.gz", hash = "sha256:9ff633b07a19469ec476b1e2b5b26c5ef700886524a7a72f65e6f0b5203142d5", size = 626583, upload-time = "2025-12-16T18:19:44.387Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/27/4b/7c1a00c2c3fbd004253937f7520f692a9650767aa73894d7a34f0d65d3f4/openai-2.14.0-py3-none-any.whl", hash = "sha256:7ea40aca4ffc4c4a776e77679021b47eec1160e341f42ae086ba949c9dcc9183", size = 1067558, upload-time = "2025-12-19T03:28:43.727Z" }, + { url = "https://files.pythonhosted.org/packages/bb/d5/eb52edff49d3d5ea116e225538c118699ddeb7c29fa17ec28af14bc10033/openai-2.13.0-py3-none-any.whl", hash = "sha256:746521065fed68df2f9c2d85613bb50844343ea81f60009b60e6a600c9352c79", size = 1066837, upload-time = "2025-12-16T18:19:43.124Z" }, ] [[package]] name = "openai-agents" -version = "0.6.4" +version = "0.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "griffe" }, @@ -1842,9 +1842,9 @@ dependencies = [ { name = "types-requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/36/826ce8ad497904a1becdedef80326c2fe932c754646b77405a8dc4cd49f7/openai_agents-0.6.4.tar.gz", hash = "sha256:07836865ed9c37946523d44b2d87ad375673b6558e783fa086db004a892331ec", size = 2022961, upload-time = "2025-12-19T06:42:55.356Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/0b/1bfc1f47708ce5500ad6b05ba8a0a789232ee6f5b9dd68938131c4674533/openai_agents-0.6.3.tar.gz", hash = "sha256:436479f201910cfc466893854b47d0f3acbf7b3bdafa95eedb590ed0d40393ef", size = 2016166, upload-time = "2025-12-11T18:07:47.823Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/ed/f2282debf62c52241959b111d2e35105b43b2ea6ff060833734405bb4d0f/openai_agents-0.6.4-py3-none-any.whl", hash = "sha256:d14635c1fa0ee39e79b81e5cab2f22dd5024772d3dbc0770d8307fd2548b3951", size = 241982, upload-time = "2025-12-19T06:42:53.92Z" }, + { url = "https://files.pythonhosted.org/packages/3e/06/d4bf0a8403ebc7d6b0fb2b45e41d6da6996b20f1dde1debffdac1b5ccb63/openai_agents-0.6.3-py3-none-any.whl", hash = "sha256:ada8b598f4db787939a62c8a291d07cbe68dae2d635955c44a0a0300746ee84f", size = 239015, upload-time = "2025-12-11T18:07:46.275Z" }, ] [[package]] @@ -3233,27 +3233,27 @@ wheels = [ [[package]] name = "ty" -version = "0.0.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/48/d9/97d5808e851f790e58f8a54efb5c7b9f404640baf9e295f424846040b316/ty-0.0.4.tar.gz", hash = "sha256:2ea47a0089d74730658ec4e988c8ef476a1e9bd92df3e56709c4003c2895ff3b", size = 4780289, upload-time = "2025-12-19T00:13:53.12Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/94/b32a962243cc8a16e8dc74cf1fe75e8bb013d0e13e71bb540e2c86214b61/ty-0.0.4-py3-none-linux_armv6l.whl", hash = "sha256:5225da65a8d1defeb21ee9d74298b1b97c6cbab36e235a310c1430d9079e4b6a", size = 9762399, upload-time = "2025-12-19T00:14:11.261Z" }, - { url = "https://files.pythonhosted.org/packages/d1/d2/7c76e0c22ddfc2fcd4a3458a65f87ce074070eb1c68c07ee475cc2b6ea68/ty-0.0.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:f87770d7988f470b795a2043185082fa959dbe1979a11b4bfe20f1214d37bd6e", size = 9590410, upload-time = "2025-12-19T00:13:55.759Z" }, - { url = "https://files.pythonhosted.org/packages/a5/84/de4b1fc85669faca3622071d5a3f3ec7bfb239971f368c28fae461d3398a/ty-0.0.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:ecf68b8ea48674a289d733b4786aecc259242a2d9a920b3ec8583db18c67496a", size = 9131113, upload-time = "2025-12-19T00:14:08.593Z" }, - { url = "https://files.pythonhosted.org/packages/a7/ff/b5bf385b6983be56a470856bbcbac1b7e816bcd765a7e9d39ab2399e387d/ty-0.0.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efc396d76a57e527393cae4ee8faf23b93be3df9e93202f39925721a7a2bb7b8", size = 9599152, upload-time = "2025-12-19T00:13:40.484Z" }, - { url = "https://files.pythonhosted.org/packages/36/d6/9880ba106f2f20d13e6a5dca5d5ca44bfb3782936ee67ff635f89a2959c0/ty-0.0.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c893b968d2f9964a4d4db9992c9ba66b01f411b1f48dffcde08622e19cd6ab97", size = 9585368, upload-time = "2025-12-19T00:14:00.994Z" }, - { url = "https://files.pythonhosted.org/packages/3f/53/503cfc18bc4c7c4e02f89dd43debc41a6e343b41eb43df658dfb493a386d/ty-0.0.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:526c925b80d68a53c165044d2370fcfc0def1f119f7b7e483ee61d24da6fb891", size = 9998412, upload-time = "2025-12-19T00:14:18.653Z" }, - { url = "https://files.pythonhosted.org/packages/1d/bd/dd2d3e29834da5add2eda0ab5b433171ce9ce9a248c364d2e237f82073d7/ty-0.0.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:857f605a7fa366b6c6e6f38abc311d0606be513c2bee8977b5c8fd4bde1a82d5", size = 10853890, upload-time = "2025-12-19T00:13:50.891Z" }, - { url = "https://files.pythonhosted.org/packages/07/fe/28ba3be1672e6b8df46e43de66a02dc076ffba7853d391a5466421886225/ty-0.0.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b4cc981aa3ebdac2c233421b1e58c80b0df6a8e6e6fa8b9e69fbdfd2f82768af", size = 10587263, upload-time = "2025-12-19T00:14:21.577Z" }, - { url = "https://files.pythonhosted.org/packages/26/9c/bb598772043f686afe5bc26cb386020709c1a0bcc164bc22ad9da2b4f55d/ty-0.0.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b03b2708b0bf67c76424a860f848aebaa4772c05529170c3761bfcaea93ec199", size = 10401204, upload-time = "2025-12-19T00:13:43.453Z" }, - { url = "https://files.pythonhosted.org/packages/ac/18/71765e9d63669bf09461c3fea84a7a63232ccb0e83b84676f07b987fc217/ty-0.0.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:469890e885544beb129c21e2f8f15321f0573d094aec13da68593c5f86389ff9", size = 10129713, upload-time = "2025-12-19T00:14:13.725Z" }, - { url = "https://files.pythonhosted.org/packages/c3/2d/c03eba570aa85e9c361de5ed36d60b9ab139e93ee91057f455ab4af48e54/ty-0.0.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:abfd928d09567e12068aeca875e920def3badf1978896f474aa4b85b552703c4", size = 9586203, upload-time = "2025-12-19T00:14:03.423Z" }, - { url = "https://files.pythonhosted.org/packages/61/f1/8c3c82a8df69bd4417c77be4f895d043db26dd47bfcc90b33dc109cd0096/ty-0.0.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:44b8e94f9d64df12eae4cf8031c5ca9a4c610b57092b26ad3d68d91bcc7af122", size = 9608230, upload-time = "2025-12-19T00:13:58.252Z" }, - { url = "https://files.pythonhosted.org/packages/51/0c/d8ba3a85c089c246ef6bd49d0f0b40bc0f9209bb819e8c02ccbea5cb4d57/ty-0.0.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9d6a439813e21a06769daf858105818c385d88018929d4a56970d4ddd5cd3df2", size = 9725125, upload-time = "2025-12-19T00:14:05.996Z" }, - { url = "https://files.pythonhosted.org/packages/4d/38/e30f64ad1e40905c766576ec70cffc69163591a5842ce14652672f6ab394/ty-0.0.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:c3cfcf26cfe6c828e91d7a529cc2dda37bc3b51ba06909c9be07002a6584af52", size = 10237174, upload-time = "2025-12-19T00:14:23.858Z" }, - { url = "https://files.pythonhosted.org/packages/cb/d7/8d650aa0be8936dd3ed74e2b0655230e2904caa6077c30c16a089b523cff/ty-0.0.4-py3-none-win32.whl", hash = "sha256:58bbf70dd27af6b00dedbdebeec92d5993aa238664f96fa5c0064930f7a0d30b", size = 9188434, upload-time = "2025-12-19T00:13:45.875Z" }, - { url = "https://files.pythonhosted.org/packages/82/d7/9fc0c81cf0b0d281ac9c18bfbdb4d6bae2173503ba79e40b210ab41c2c8b/ty-0.0.4-py3-none-win_amd64.whl", hash = "sha256:7c2db0f96218f08c140bd9d3fcbb1b3c8c5c4f0c9b0a5624487f0a2bf4b76163", size = 10019313, upload-time = "2025-12-19T00:14:15.968Z" }, - { url = "https://files.pythonhosted.org/packages/5f/b8/3e3246738eed1cd695c5964a401f3b9c757d20ac21fdae06281af9f40ef6/ty-0.0.4-py3-none-win_arm64.whl", hash = "sha256:69f14fc98e4a847afa9f8c5d5234d008820dbc09c7dcdb3ac1ba16628f5132df", size = 9561857, upload-time = "2025-12-19T00:13:48.382Z" }, +version = "0.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bb/cd/aee86c0da3240960d6b7e807f3a41c89bae741495d81ca303200b0103dc9/ty-0.0.3.tar.gz", hash = "sha256:831259e22d3855436701472d4c0da200cd45041bc677eae79415d684f541de8a", size = 4769098, upload-time = "2025-12-18T02:16:49.773Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/ef/2d0d18e8fe6b673d3e1ea642f18404d7edfa9d08310f7203e8f0e7dc862e/ty-0.0.3-py3-none-linux_armv6l.whl", hash = "sha256:cd035bb75acecb78ac1ba8c4cc696f57a586e29d36e84bd691bc3b5b8362794c", size = 9763890, upload-time = "2025-12-18T02:16:56.879Z" }, + { url = "https://files.pythonhosted.org/packages/bb/67/0ae31574619a7264df8cf8e641f246992db22ac1720c2a72953aa31cbe61/ty-0.0.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:7708eaf73485e263efc7ef339f8e4487d3f5885779edbeec504fd72e4521c376", size = 9558276, upload-time = "2025-12-18T02:16:45.453Z" }, + { url = "https://files.pythonhosted.org/packages/d7/f7/3b9c033e80910972fca3783e4a52ba9cb7cd5c8b6828a87986646d64082b/ty-0.0.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:3113a633f46ec789f6df675b7afc5d3ab20c247c92ae4dbb9aa5b704768c18b2", size = 9094451, upload-time = "2025-12-18T02:17:01.155Z" }, + { url = "https://files.pythonhosted.org/packages/9a/29/9a90ed6bef00142a088965100b5e0a5d11805b9729c151ca598331bbd92b/ty-0.0.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a451f3f73a04bf18e551b1ebebb79b20fac5f09740a353f7e07b5f607b217c4f", size = 9568049, upload-time = "2025-12-18T02:16:28.643Z" }, + { url = "https://files.pythonhosted.org/packages/2f/ab/8daeb12912c2de8a3154db652931f4ad0d27c555faebcaf34af08bcfd0d2/ty-0.0.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9f6e926b6de0becf0452e1afad75cb71f889a4777cd14269e5447d46c01b2770", size = 9547711, upload-time = "2025-12-18T02:16:54.464Z" }, + { url = "https://files.pythonhosted.org/packages/91/54/f5c1f293f647beda717fee2448cc927ac0d05f66bebe18647680a67e1d67/ty-0.0.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:160e7974150f9f359c31d5808214676d1baa05321ab5a7b29fb09f4906dbdb38", size = 9983225, upload-time = "2025-12-18T02:17:05.672Z" }, + { url = "https://files.pythonhosted.org/packages/95/34/065962cfa2e87c10db839512229940a366b8ca1caffa2254a277b1694e5a/ty-0.0.3-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:726576df31d4e76934ffc64f2939d4a9bc195c7427452c8c159261ad00bd1b5e", size = 10851148, upload-time = "2025-12-18T02:16:38.354Z" }, + { url = "https://files.pythonhosted.org/packages/54/27/e2a8cbfc33999eef882ccd1b816ed615293f96e96f6df60cd12f84b69ca2/ty-0.0.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5014cf4744c94d9ea7b43314199ddaf52564a80b3d006e4ba0fe982bc42f4e8b", size = 10564441, upload-time = "2025-12-18T02:17:03.584Z" }, + { url = "https://files.pythonhosted.org/packages/91/6d/dcce3e222e59477c1f2b3a012cc76428d7032248138cd5544ad7f1cda7bd/ty-0.0.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2a9a51dc040f2718725f34ae6ef51fe8f8bd689e21bd3e82f4e71767034928de", size = 10358651, upload-time = "2025-12-18T02:16:26.091Z" }, + { url = "https://files.pythonhosted.org/packages/53/36/b6d0154b83a5997d607bf1238200271c17223f68aab2c778ded5424f9c1e/ty-0.0.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e0e6188eddd3a228c449261bb398e8621d33b92c1fc03599afdfad4388327a48", size = 10120457, upload-time = "2025-12-18T02:16:51.864Z" }, + { url = "https://files.pythonhosted.org/packages/cc/46/05dc826674ee1a451406e4c253c71700a6f707bae88b706a4c9e9bba6919/ty-0.0.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:5cc55e08d5d18edf1c5051af02456bd359716f07aae0a305e4cefe7735188540", size = 9551642, upload-time = "2025-12-18T02:16:33.518Z" }, + { url = "https://files.pythonhosted.org/packages/64/8a/f90b60d103fd5ec04ecbac091a64e607e6cd37cec6e718bba17cb2022644/ty-0.0.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:34b2d589412a81d1fd6d7fe461353068496c2bf1f7113742bd6d88d1d57ec3ad", size = 9572234, upload-time = "2025-12-18T02:16:31.013Z" }, + { url = "https://files.pythonhosted.org/packages/e8/72/5d3c6d34562d019ba7f3102b2a6d0c8e9e24ef39e70f09645c36a66765b7/ty-0.0.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:8a065eb2959f141fe4adafc14d57463cfa34f6cc4844a4ed56b2dce1a53a419a", size = 9701682, upload-time = "2025-12-18T02:16:41.379Z" }, + { url = "https://files.pythonhosted.org/packages/ef/44/bda434f788b320c9550a48c549e4a8c507e3d8a6ccb04ba5bd098307ba1e/ty-0.0.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e7177421f830a493f98d22f86d940b5a38788866e6062f680881f19be35ba3bb", size = 10213714, upload-time = "2025-12-18T02:16:35.648Z" }, + { url = "https://files.pythonhosted.org/packages/53/a6/b76a787938026c3d209131e5773de32cf6fc41210e0dd97874aafa20f394/ty-0.0.3-py3-none-win32.whl", hash = "sha256:e3e590bf5f33cb118a53c6d5242eedf7924d45517a5ee676c7a16be3a1389d2f", size = 9160441, upload-time = "2025-12-18T02:16:43.404Z" }, + { url = "https://files.pythonhosted.org/packages/fe/db/da60eb8252768323aee0ce69a08b95011088c003f80204b12380fe562fd2/ty-0.0.3-py3-none-win_amd64.whl", hash = "sha256:5af25b1fed8a536ce8072a9ae6a70cd2b559aa5294d43f57071fbdcd31dd2b0e", size = 10034265, upload-time = "2025-12-18T02:16:47.602Z" }, + { url = "https://files.pythonhosted.org/packages/5f/9c/9045cebdfc394c6f8c1e73a99d3aeda1bc639aace392e8ff4d695f1fab73/ty-0.0.3-py3-none-win_arm64.whl", hash = "sha256:29078b3100351a8b37339771615f13b8e4a4ff52b344d33f774f8d1a665a0ca5", size = 9513095, upload-time = "2025-12-18T02:16:59.073Z" }, ] [[package]] diff --git a/examples/api/openai/uv.lock b/examples/api/openai/uv.lock index 9f1bd857..3781b886 100644 --- a/examples/api/openai/uv.lock +++ b/examples/api/openai/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14'", @@ -1663,7 +1663,7 @@ wheels = [ [[package]] name = "openai-agents" -version = "0.6.4" +version = "0.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "griffe" }, @@ -1674,9 +1674,9 @@ dependencies = [ { name = "types-requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/36/826ce8ad497904a1becdedef80326c2fe932c754646b77405a8dc4cd49f7/openai_agents-0.6.4.tar.gz", hash = "sha256:07836865ed9c37946523d44b2d87ad375673b6558e783fa086db004a892331ec", size = 2022961, upload-time = "2025-12-19T06:42:55.356Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/0b/1bfc1f47708ce5500ad6b05ba8a0a789232ee6f5b9dd68938131c4674533/openai_agents-0.6.3.tar.gz", hash = "sha256:436479f201910cfc466893854b47d0f3acbf7b3bdafa95eedb590ed0d40393ef", size = 2016166, upload-time = "2025-12-11T18:07:47.823Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/ed/f2282debf62c52241959b111d2e35105b43b2ea6ff060833734405bb4d0f/openai_agents-0.6.4-py3-none-any.whl", hash = "sha256:d14635c1fa0ee39e79b81e5cab2f22dd5024772d3dbc0770d8307fd2548b3951", size = 241982, upload-time = "2025-12-19T06:42:53.92Z" }, + { url = "https://files.pythonhosted.org/packages/3e/06/d4bf0a8403ebc7d6b0fb2b45e41d6da6996b20f1dde1debffdac1b5ccb63/openai_agents-0.6.3-py3-none-any.whl", hash = "sha256:ada8b598f4db787939a62c8a291d07cbe68dae2d635955c44a0a0300746ee84f", size = 239015, upload-time = "2025-12-11T18:07:46.275Z" }, ] [[package]] diff --git a/examples/api/slack/uv.lock b/examples/api/slack/uv.lock index 62b35728..8afe9513 100644 --- a/examples/api/slack/uv.lock +++ b/examples/api/slack/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14'", @@ -1715,7 +1715,7 @@ wheels = [ [[package]] name = "openai-agents" -version = "0.6.4" +version = "0.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "griffe" }, @@ -1726,9 +1726,9 @@ dependencies = [ { name = "types-requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/36/826ce8ad497904a1becdedef80326c2fe932c754646b77405a8dc4cd49f7/openai_agents-0.6.4.tar.gz", hash = "sha256:07836865ed9c37946523d44b2d87ad375673b6558e783fa086db004a892331ec", size = 2022961, upload-time = "2025-12-19T06:42:55.356Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/0b/1bfc1f47708ce5500ad6b05ba8a0a789232ee6f5b9dd68938131c4674533/openai_agents-0.6.3.tar.gz", hash = "sha256:436479f201910cfc466893854b47d0f3acbf7b3bdafa95eedb590ed0d40393ef", size = 2016166, upload-time = "2025-12-11T18:07:47.823Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/ed/f2282debf62c52241959b111d2e35105b43b2ea6ff060833734405bb4d0f/openai_agents-0.6.4-py3-none-any.whl", hash = "sha256:d14635c1fa0ee39e79b81e5cab2f22dd5024772d3dbc0770d8307fd2548b3951", size = 241982, upload-time = "2025-12-19T06:42:53.92Z" }, + { url = "https://files.pythonhosted.org/packages/3e/06/d4bf0a8403ebc7d6b0fb2b45e41d6da6996b20f1dde1debffdac1b5ccb63/openai_agents-0.6.3-py3-none-any.whl", hash = "sha256:ada8b598f4db787939a62c8a291d07cbe68dae2d635955c44a0a0300746ee84f", size = 239015, upload-time = "2025-12-11T18:07:46.275Z" }, ] [[package]] diff --git a/examples/api/telegram/uv.lock b/examples/api/telegram/uv.lock index 90875f00..cf8845b3 100644 --- a/examples/api/telegram/uv.lock +++ b/examples/api/telegram/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14'", @@ -1856,7 +1856,7 @@ wheels = [ [[package]] name = "openai-agents" -version = "0.6.4" +version = "0.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "griffe" }, @@ -1867,9 +1867,9 @@ dependencies = [ { name = "types-requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/36/826ce8ad497904a1becdedef80326c2fe932c754646b77405a8dc4cd49f7/openai_agents-0.6.4.tar.gz", hash = "sha256:07836865ed9c37946523d44b2d87ad375673b6558e783fa086db004a892331ec", size = 2022961, upload-time = "2025-12-19T06:42:55.356Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/0b/1bfc1f47708ce5500ad6b05ba8a0a789232ee6f5b9dd68938131c4674533/openai_agents-0.6.3.tar.gz", hash = "sha256:436479f201910cfc466893854b47d0f3acbf7b3bdafa95eedb590ed0d40393ef", size = 2016166, upload-time = "2025-12-11T18:07:47.823Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/ed/f2282debf62c52241959b111d2e35105b43b2ea6ff060833734405bb4d0f/openai_agents-0.6.4-py3-none-any.whl", hash = "sha256:d14635c1fa0ee39e79b81e5cab2f22dd5024772d3dbc0770d8307fd2548b3951", size = 241982, upload-time = "2025-12-19T06:42:53.92Z" }, + { url = "https://files.pythonhosted.org/packages/3e/06/d4bf0a8403ebc7d6b0fb2b45e41d6da6996b20f1dde1debffdac1b5ccb63/openai_agents-0.6.3-py3-none-any.whl", hash = "sha256:ada8b598f4db787939a62c8a291d07cbe68dae2d635955c44a0a0300746ee84f", size = 239015, upload-time = "2025-12-11T18:07:46.275Z" }, ] [[package]] diff --git a/examples/api/whatsapp/uv.lock b/examples/api/whatsapp/uv.lock index e7c92bb9..f0622e6c 100644 --- a/examples/api/whatsapp/uv.lock +++ b/examples/api/whatsapp/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14'", @@ -10,7 +10,7 @@ resolution-markers = [ [[package]] name = "agentkernel" version = "0.2.9" -source = { registry = "../../../ak-py/dist" } +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "deprecated" }, { name = "pydantic" }, @@ -18,9 +18,9 @@ dependencies = [ { name = "pyyaml" }, { name = "singleton-type" }, ] -sdist = { path = "agentkernel-0.2.9.tar.gz" } +sdist = { url = "https://files.pythonhosted.org/packages/75/a4/c3b7b1663131bd275b99bb91d77d66983f598c9e0b686f57558ab9b6e1e2/agentkernel-0.2.9.tar.gz", hash = "sha256:976ccc343039f35ab4661e97f42f935ec88627ba1763980c68f983523ef09fdc", size = 76953, upload-time = "2025-12-18T12:28:27.293Z" } wheels = [ - { path = "agentkernel-0.2.9-py3-none-any.whl" }, + { url = "https://files.pythonhosted.org/packages/e6/6b/ea9652f7d83c29fdff8322bcb4a0ac8fa6c55214f9e0334a947a86ac2d95/agentkernel-0.2.9-py3-none-any.whl", hash = "sha256:307d6299baf7f9f579ab100492ddc2a66facbb8fc3b2960937b007f40bb66efa", size = 121841, upload-time = "2025-12-18T12:28:26.002Z" }, ] [package.optional-dependencies] diff --git a/examples/aws-containerized/openai-dynamodb/uv.lock b/examples/aws-containerized/openai-dynamodb/uv.lock index d2ef137c..b785bbd1 100644 --- a/examples/aws-containerized/openai-dynamodb/uv.lock +++ b/examples/aws-containerized/openai-dynamodb/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" [[package]] @@ -527,7 +527,7 @@ wheels = [ [[package]] name = "openai-agents" -version = "0.6.4" +version = "0.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "griffe" }, @@ -538,9 +538,9 @@ dependencies = [ { name = "types-requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/36/826ce8ad497904a1becdedef80326c2fe932c754646b77405a8dc4cd49f7/openai_agents-0.6.4.tar.gz", hash = "sha256:07836865ed9c37946523d44b2d87ad375673b6558e783fa086db004a892331ec", size = 2022961, upload-time = "2025-12-19T06:42:55.356Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/0b/1bfc1f47708ce5500ad6b05ba8a0a789232ee6f5b9dd68938131c4674533/openai_agents-0.6.3.tar.gz", hash = "sha256:436479f201910cfc466893854b47d0f3acbf7b3bdafa95eedb590ed0d40393ef", size = 2016166, upload-time = "2025-12-11T18:07:47.823Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/ed/f2282debf62c52241959b111d2e35105b43b2ea6ff060833734405bb4d0f/openai_agents-0.6.4-py3-none-any.whl", hash = "sha256:d14635c1fa0ee39e79b81e5cab2f22dd5024772d3dbc0770d8307fd2548b3951", size = 241982, upload-time = "2025-12-19T06:42:53.92Z" }, + { url = "https://files.pythonhosted.org/packages/3e/06/d4bf0a8403ebc7d6b0fb2b45e41d6da6996b20f1dde1debffdac1b5ccb63/openai_agents-0.6.3-py3-none-any.whl", hash = "sha256:ada8b598f4db787939a62c8a291d07cbe68dae2d635955c44a0a0300746ee84f", size = 239015, upload-time = "2025-12-11T18:07:46.275Z" }, ] [[package]] diff --git a/examples/aws-serverless/openai/uv.lock b/examples/aws-serverless/openai/uv.lock index 403794c0..51ff881d 100644 --- a/examples/aws-serverless/openai/uv.lock +++ b/examples/aws-serverless/openai/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" [[package]] @@ -449,7 +449,7 @@ wheels = [ [[package]] name = "openai-agents" -version = "0.6.4" +version = "0.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "griffe" }, @@ -460,9 +460,9 @@ dependencies = [ { name = "types-requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/36/826ce8ad497904a1becdedef80326c2fe932c754646b77405a8dc4cd49f7/openai_agents-0.6.4.tar.gz", hash = "sha256:07836865ed9c37946523d44b2d87ad375673b6558e783fa086db004a892331ec", size = 2022961, upload-time = "2025-12-19T06:42:55.356Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/0b/1bfc1f47708ce5500ad6b05ba8a0a789232ee6f5b9dd68938131c4674533/openai_agents-0.6.3.tar.gz", hash = "sha256:436479f201910cfc466893854b47d0f3acbf7b3bdafa95eedb590ed0d40393ef", size = 2016166, upload-time = "2025-12-11T18:07:47.823Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/ed/f2282debf62c52241959b111d2e35105b43b2ea6ff060833734405bb4d0f/openai_agents-0.6.4-py3-none-any.whl", hash = "sha256:d14635c1fa0ee39e79b81e5cab2f22dd5024772d3dbc0770d8307fd2548b3951", size = 241982, upload-time = "2025-12-19T06:42:53.92Z" }, + { url = "https://files.pythonhosted.org/packages/3e/06/d4bf0a8403ebc7d6b0fb2b45e41d6da6996b20f1dde1debffdac1b5ccb63/openai_agents-0.6.3-py3-none-any.whl", hash = "sha256:ada8b598f4db787939a62c8a291d07cbe68dae2d635955c44a0a0300746ee84f", size = 239015, upload-time = "2025-12-11T18:07:46.275Z" }, ] [[package]] diff --git a/examples/cli/multi/uv.lock b/examples/cli/multi/uv.lock index 0c0cb9ae..188e94e4 100644 --- a/examples/cli/multi/uv.lock +++ b/examples/cli/multi/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.13'", @@ -2031,7 +2031,7 @@ wheels = [ [[package]] name = "openai-agents" -version = "0.6.4" +version = "0.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "griffe" }, @@ -2042,9 +2042,9 @@ dependencies = [ { name = "types-requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/36/826ce8ad497904a1becdedef80326c2fe932c754646b77405a8dc4cd49f7/openai_agents-0.6.4.tar.gz", hash = "sha256:07836865ed9c37946523d44b2d87ad375673b6558e783fa086db004a892331ec", size = 2022961, upload-time = "2025-12-19T06:42:55.356Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/0b/1bfc1f47708ce5500ad6b05ba8a0a789232ee6f5b9dd68938131c4674533/openai_agents-0.6.3.tar.gz", hash = "sha256:436479f201910cfc466893854b47d0f3acbf7b3bdafa95eedb590ed0d40393ef", size = 2016166, upload-time = "2025-12-11T18:07:47.823Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/ed/f2282debf62c52241959b111d2e35105b43b2ea6ff060833734405bb4d0f/openai_agents-0.6.4-py3-none-any.whl", hash = "sha256:d14635c1fa0ee39e79b81e5cab2f22dd5024772d3dbc0770d8307fd2548b3951", size = 241982, upload-time = "2025-12-19T06:42:53.92Z" }, + { url = "https://files.pythonhosted.org/packages/3e/06/d4bf0a8403ebc7d6b0fb2b45e41d6da6996b20f1dde1debffdac1b5ccb63/openai_agents-0.6.3-py3-none-any.whl", hash = "sha256:ada8b598f4db787939a62c8a291d07cbe68dae2d635955c44a0a0300746ee84f", size = 239015, upload-time = "2025-12-11T18:07:46.275Z" }, ] [[package]] diff --git a/examples/cli/openai-dynamic/uv.lock b/examples/cli/openai-dynamic/uv.lock index 40771948..8ed195b8 100644 --- a/examples/cli/openai-dynamic/uv.lock +++ b/examples/cli/openai-dynamic/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14'", @@ -1647,7 +1647,7 @@ wheels = [ [[package]] name = "openai-agents" -version = "0.6.4" +version = "0.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "griffe" }, @@ -1658,9 +1658,9 @@ dependencies = [ { name = "types-requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/36/826ce8ad497904a1becdedef80326c2fe932c754646b77405a8dc4cd49f7/openai_agents-0.6.4.tar.gz", hash = "sha256:07836865ed9c37946523d44b2d87ad375673b6558e783fa086db004a892331ec", size = 2022961, upload-time = "2025-12-19T06:42:55.356Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/0b/1bfc1f47708ce5500ad6b05ba8a0a789232ee6f5b9dd68938131c4674533/openai_agents-0.6.3.tar.gz", hash = "sha256:436479f201910cfc466893854b47d0f3acbf7b3bdafa95eedb590ed0d40393ef", size = 2016166, upload-time = "2025-12-11T18:07:47.823Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/ed/f2282debf62c52241959b111d2e35105b43b2ea6ff060833734405bb4d0f/openai_agents-0.6.4-py3-none-any.whl", hash = "sha256:d14635c1fa0ee39e79b81e5cab2f22dd5024772d3dbc0770d8307fd2548b3951", size = 241982, upload-time = "2025-12-19T06:42:53.92Z" }, + { url = "https://files.pythonhosted.org/packages/3e/06/d4bf0a8403ebc7d6b0fb2b45e41d6da6996b20f1dde1debffdac1b5ccb63/openai_agents-0.6.3-py3-none-any.whl", hash = "sha256:ada8b598f4db787939a62c8a291d07cbe68dae2d635955c44a0a0300746ee84f", size = 239015, upload-time = "2025-12-11T18:07:46.275Z" }, ] [[package]] diff --git a/examples/cli/openai/uv.lock b/examples/cli/openai/uv.lock index f3f9af08..21e44a1d 100644 --- a/examples/cli/openai/uv.lock +++ b/examples/cli/openai/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14'", diff --git a/examples/containerized/openai/uv.lock b/examples/containerized/openai/uv.lock index 9f1bd857..3781b886 100644 --- a/examples/containerized/openai/uv.lock +++ b/examples/containerized/openai/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14'", @@ -1663,7 +1663,7 @@ wheels = [ [[package]] name = "openai-agents" -version = "0.6.4" +version = "0.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "griffe" }, @@ -1674,9 +1674,9 @@ dependencies = [ { name = "types-requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/36/826ce8ad497904a1becdedef80326c2fe932c754646b77405a8dc4cd49f7/openai_agents-0.6.4.tar.gz", hash = "sha256:07836865ed9c37946523d44b2d87ad375673b6558e783fa086db004a892331ec", size = 2022961, upload-time = "2025-12-19T06:42:55.356Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/0b/1bfc1f47708ce5500ad6b05ba8a0a789232ee6f5b9dd68938131c4674533/openai_agents-0.6.3.tar.gz", hash = "sha256:436479f201910cfc466893854b47d0f3acbf7b3bdafa95eedb590ed0d40393ef", size = 2016166, upload-time = "2025-12-11T18:07:47.823Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/ed/f2282debf62c52241959b111d2e35105b43b2ea6ff060833734405bb4d0f/openai_agents-0.6.4-py3-none-any.whl", hash = "sha256:d14635c1fa0ee39e79b81e5cab2f22dd5024772d3dbc0770d8307fd2548b3951", size = 241982, upload-time = "2025-12-19T06:42:53.92Z" }, + { url = "https://files.pythonhosted.org/packages/3e/06/d4bf0a8403ebc7d6b0fb2b45e41d6da6996b20f1dde1debffdac1b5ccb63/openai_agents-0.6.3-py3-none-any.whl", hash = "sha256:ada8b598f4db787939a62c8a291d07cbe68dae2d635955c44a0a0300746ee84f", size = 239015, upload-time = "2025-12-11T18:07:46.275Z" }, ] [[package]] From 5222577bdcf114dc2fb185fc8ea2cdecad819db2 Mon Sep 17 00:00:00 2001 From: pulinduvidmal Date: Tue, 23 Dec 2025 20:34:17 +0530 Subject: [PATCH 2/9] feat: add multimodal file/image support for telegram --- .../agentkernel/framework/openai/openai.py | 8 + .../integration/telegram/README.md | 62 +++++- .../integration/telegram/telegram_chat.py | 198 ++++++++++++++++-- 3 files changed, 250 insertions(+), 18 deletions(-) diff --git a/ak-py/src/agentkernel/framework/openai/openai.py b/ak-py/src/agentkernel/framework/openai/openai.py index d0ec1e61..91625e43 100644 --- a/ak-py/src/agentkernel/framework/openai/openai.py +++ b/ak-py/src/agentkernel/framework/openai/openai.py @@ -169,6 +169,14 @@ async def run(self, agent: Any, session: Session, requests: list[AgentRequest]) # Multimodal case with images/files. When using multimodal inputs, OpenAI cannot handle session. So these inputs are not saved in the context reply = (await Runner.run(agent.agent, message_content, session=None)).final_output + # Manually save the multimodal conversation to session for future reference + if session: + openai_session = session.get("openai") or session.set("openai", OpenAISession()) + # Add user message + await openai_session.add_items([{"role": "user", "content": prompt}]) + # Add assistant response + await openai_session.add_items([{"role": "assistant", "content": reply}]) + return AgentReplyText(text=str(reply), prompt=prompt) except Exception as e: return AgentReplyText(text=f"Error during agent execution: {str(e)}") diff --git a/ak-py/src/agentkernel/integration/telegram/README.md b/ak-py/src/agentkernel/integration/telegram/README.md index ef8ca92c..ca7bd561 100644 --- a/ak-py/src/agentkernel/integration/telegram/README.md +++ b/ak-py/src/agentkernel/integration/telegram/README.md @@ -21,16 +21,17 @@ The `AgentTelegramRequestHandler` class handles conversations with agents via Te ### Configuration Steps 1. **Create a Telegram Bot** + - Open Telegram and search for [@BotFather](https://t.me/botfather) - Send `/newbot` command - Follow the prompts to name your bot - Save the bot token provided - 2. **Get Your Credentials** + - **Bot Token**: Provided by BotFather when you create the bot - **Bot Username**: The username you chose (ending in `bot`) - 3. **Configure Webhook** + - The webhook is automatically set when your server starts - Or manually set via: `https://api.telegram.org/bot/setWebhook?url=https://your-domain.com/telegram/webhook` @@ -85,15 +86,46 @@ It is strongly recommended not to keep secrets and keys in the config file. Set ### Supported Message Types - **Text Messages**: Standard text messages +- **Images**: Photos sent directly to the bot +- **Documents**: Files including PDFs, TXT, CSV, DOCX, and other document formats - **Commands**: Bot commands starting with `/` - **Replies**: Reply to bot messages +### Multi-Modal Support + +The Telegram integration **fully supports** sending images and documents to agents. When a user sends a message with attachments: + +1. **Images**: Photos are extracted, base64-encoded, and sent to the agent as `AgentRequestImage` +2. **Documents**: Files are downloaded, base64-encoded, and sent to the agent as `AgentRequestFile` +3. **Combined**: A message can include both text and files/images which are all processed together + +**Status:** ✅ **FULLY IMPLEMENTED AND WORKING** + +- ✅ File/image detection and download infrastructure +- ✅ Base64 encoding of files/images +- ✅ Multi-modal requests forwarding (`service.run_multi()`) +- ✅ Session memory for follow-up questions with context +- ✅ Works with **OpenAI SDK** and **Google ADK** agents + +**Supported file types:** + +- **Images**: JPEG, PNG, GIF, WebP, and other image formats +- **Documents**: PDF, TXT, CSV, DOC, DOCX, and other document formats + +**Limitations:** + +- Maximum file size: ~20MB (Telegram API limit, may vary by file type) +- Processing time: Large files may take 30-60 seconds to download and analyze +- Session context: Cleared on server restart (can be persisted with external storage) +- Model requirements: Agent must support multimodal (OpenAI GPT-4o, Google ADK, etc.) + ### Message Handling - **Automatic Message Splitting**: Messages longer than 4096 characters are automatically split - **Session Management**: Uses chat ID as session ID to maintain conversation context - **Typing Indicator**: Shows typing status while processing - **Markdown Support**: Responses support Telegram's Markdown formatting +- **File Handling**: Supports images (JPEG, PNG, GIF, WebP) and documents (PDF, TXT, CSV, DOC, DOCX, etc.) ### Security @@ -107,11 +139,13 @@ It is strongly recommended not to keep secrets and keys in the config file. Set For local testing, use a tunneling service to expose your local server: **Using ngrok:** + ```bash ngrok http 8000 ``` **Using pinggy:** + ```bash ssh -p443 -R0:localhost:8000 a.pinggy.io ``` @@ -137,21 +171,31 @@ curl "https://api.telegram.org/bot/setWebhook?url=https://your-n ### Common Issues **Webhook not receiving messages:** + - Ensure webhook URL is HTTPS - Verify the URL is publicly accessible - Check bot token is correct - Use `getWebhookInfo` to debug: `https://api.telegram.org/bot/getWebhookInfo` **Messages not sending:** + - Verify bot token is correct - Check bot hasn't been blocked by user - Ensure chat_id is valid **Bot not responding:** + - Check server logs for errors - Verify agent is properly configured - Ensure OpenAI API key is set +**Files/Images not being processed:** + +- Verify your agent supports images/files (OpenAI SDK or Google ADK) +- Check the file size isn't too large +- Ensure `max_file_size` configuration allows the file +- Check logs for download errors from Telegram servers + ### Debug Logging Enable debug logging to troubleshoot: @@ -161,14 +205,24 @@ import logging logging.basicConfig(level=logging.DEBUG) ``` - ## API Rate Limits Telegram Bot API has rate limits: + - **Messages to same chat**: 1 message per second - **Bulk messages**: 30 messages per second to different chats - **Group messages**: 20 messages per minute per group +- **File downloads**: Files are downloaded on-demand when received + +## File Size Considerations + +- **Downloaded files**: Automatically downloaded and base64-encoded +- **Encoding overhead**: Base64 encoding increases size by ~33% +- **Agent framework limits**: OpenAI and Google ADK have their own file size limits +- **Performance**: Large files may take longer to download and process + ## References - [Telegram Bot API Documentation](https://core.telegram.org/bots/api) -- [BotFather](https://t.me/botfather) +- [Telegram Bot API Media Types](https://core.telegram.org/bots/api#mediagroupmessage) +- [BotFather](https://t.me/botfather) \ No newline at end of file diff --git a/ak-py/src/agentkernel/integration/telegram/telegram_chat.py b/ak-py/src/agentkernel/integration/telegram/telegram_chat.py index cb707287..35153c8f 100644 --- a/ak-py/src/agentkernel/integration/telegram/telegram_chat.py +++ b/ak-py/src/agentkernel/integration/telegram/telegram_chat.py @@ -1,4 +1,6 @@ +import base64 import logging +import mimetypes import traceback import httpx @@ -6,6 +8,7 @@ from ...api import RESTRequestHandler from ...core import AgentService, Config +from ...core.model import AgentRequestFile, AgentRequestImage, AgentRequestText class AgentTelegramRequestHandler(RESTRequestHandler): @@ -95,25 +98,29 @@ async def _handle_message(self, message: dict): chat_id = message.get("chat", {}).get("id") message_id = message.get("message_id") - text = message.get("text") - - # Handle different message types - if not text: - - self._log.warning("Message has no text content") - return + # Get text from either 'text' field (regular messages) or 'caption' field (media with captions) + text = (message.get("text") or message.get("caption") or "").strip() if not chat_id or not message_id: self._log.warning("Message missing required fields (chat_id/message_id)") return - self._log.debug(f"Processing message {message_id} from chat {chat_id}: {text}") + # Check if message has text, files, or images + has_text = bool(text) + has_files = "document" in message + has_images = "photo" in message + + if not has_text and not has_files and not has_images: + self._log.warning("Message has no text, files, or images") + return + + self._log.debug(f"Processing message {message_id} from chat {chat_id}") - # Check if it's a bot command - if text.startswith("/"): + # Check if it's a bot command (only if it has text) + if has_text and text.startswith("/"): await self._handle_command(chat_id, text) else: - await self._process_agent_message(chat_id, text) + await self._process_agent_message(chat_id, text if has_text else "", message) async def _handle_edited_message(self, message: dict): """ @@ -167,12 +174,13 @@ async def _handle_command(self, chat_id: int, command: str): # Unknown command - process as regular message await self._process_agent_message(chat_id, command) - async def _process_agent_message(self, chat_id: int, message_text: str): + async def _process_agent_message(self, chat_id: int, message_text: str, message: dict | None = None): """ Process message through agent. :param chat_id: Chat ID :param message_text: Message text + :param message: Full message dict from Telegram (for accessing files/images) """ service = AgentService() session_id = str(chat_id) # Use chat_id as session_id @@ -188,8 +196,27 @@ async def _process_agent_message(self, chat_id: int, message_text: str): await self._send_message(chat_id, "Sorry, no agent is available to handle your request.") return - # Run the agent - result = await service.run(message_text) + # Build requests list with text and files/images + requests = [] + + # Add text if present + if message_text: + requests.append(AgentRequestText(text=message_text)) + + # Process files and images if message object is provided + if message: + failed_files = await self._process_files(message, requests) + if failed_files: + self._log.warning(f"Failed to process files: {failed_files}") + + # If no content at all, nothing to process + if not requests: + self._log.warning("No valid content found in message") + await self._send_message(chat_id, "Sorry, your message appears to be empty.") + return + + # Run the agent with all requests (text + files/images) + result = await service.run_multi(requests=requests) if hasattr(result, "raw"): response_text = str(result.raw) @@ -297,3 +324,146 @@ async def _answer_callback_query(self, callback_query_id: str, text: str = None, self._log.debug(f"Callback query answered: {callback_query_id}") except Exception as e: self._log.warning(f"Failed to answer callback query: {e}") + + async def _get_file_info(self, file_id: str): + """ + Get file information from Telegram API. + + :param file_id: File ID from Telegram + :return: File info dict with file_path + """ + url = f"{self._base_url}/getFile" + payload = {"file_id": file_id} + + try: + async with httpx.AsyncClient() as client: + response = await client.post(url, json=payload) + response.raise_for_status() + result = response.json() + if result.get("ok"): + return result.get("result") + else: + self._log.error(f"Failed to get file info: {result}") + return None + except Exception as e: + self._log.error(f"Error getting file info: {e}") + return None + + async def _download_telegram_file(self, file_path: str) -> bytes | None: + """ + Download file content from Telegram server. + + :param file_path: File path from getFile API + :return: File content as bytes + """ + url = f"https://api.telegram.org/file/bot{self._bot_token}/{file_path}" + + try: + async with httpx.AsyncClient() as client: + response = await client.get(url) + response.raise_for_status() + return response.content + except Exception as e: + self._log.error(f"Error downloading file from Telegram: {e}") + return None + + async def _process_files(self, message: dict, requests: list) -> list[str]: + """ + Process files and images in a Telegram message. + + :param message: Message object from Telegram + :param requests: List to append AgentRequestFile/AgentRequestImage objects + :return: List of failed file names + """ + failed_files = [] + + # Process photos (images) + if "photo" in message: + photos = message.get("photo", []) + if photos: + # Get the largest photo + largest_photo = photos[-1] + file_id = largest_photo.get("file_id") + + try: + self._log.debug(f"Processing photo: {file_id}") + + # Get file info + file_info = await self._get_file_info(file_id) + if not file_info: + self._log.warning("Failed to get photo file info") + failed_files.append("photo") + return failed_files + + file_path = file_info.get("file_path") + file_size = file_info.get("file_size", 0) + + # Download file + file_content = await self._download_telegram_file(file_path) + if file_content is None: + self._log.warning(f"Failed to download photo") + failed_files.append("photo") + return failed_files + + # Base64 encode + image_data_base64 = base64.b64encode(file_content).decode("utf-8") + + # Add as image request + requests.append( + AgentRequestImage( + image_data=image_data_base64, + name="photo.jpg", + mime_type="image/jpeg", + ) + ) + self._log.debug(f"Added photo to request (size: {file_size} bytes)") + + except Exception as e: + self._log.error(f"Error processing photo: {e}\n{traceback.format_exc()}") + failed_files.append("photo") + + # Process documents (files) + if "document" in message: + document = message.get("document", {}) + file_id = document.get("file_id") + file_name = document.get("file_name", "document") + mime_type = document.get("mime_type", "application/octet-stream") + + try: + self._log.debug(f"Processing document: {file_id} ({file_name})") + + # Get file info + file_info = await self._get_file_info(file_id) + if not file_info: + self._log.warning(f"Failed to get file info for {file_name}") + failed_files.append(file_name) + return failed_files + + file_path = file_info.get("file_path") + file_size = file_info.get("file_size", 0) + + # Download file + file_content = await self._download_telegram_file(file_path) + if file_content is None: + self._log.warning(f"Failed to download file: {file_name}") + failed_files.append(file_name) + return failed_files + + # Base64 encode + file_data_base64 = base64.b64encode(file_content).decode("utf-8") + + # Add as file request + requests.append( + AgentRequestFile( + file_data=file_data_base64, + name=file_name, + mime_type=mime_type, + ) + ) + self._log.debug(f"Added file to request: {file_name} (size: {file_size} bytes)") + + except Exception as e: + self._log.error(f"Error processing document: {e}\n{traceback.format_exc()}") + failed_files.append(file_name) + + return failed_files From 54def413cb6e714099e2a48b00b4521ce18b960b Mon Sep 17 00:00:00 2001 From: pulinduvidmal Date: Tue, 23 Dec 2025 20:35:28 +0530 Subject: [PATCH 3/9] feat: add multimodal file/image support for telegram --- docs/docs/integrations/telegram.md | 156 ++++++++++++++-------------- examples/api/telegram/README.md | 111 ++++++++++++++++++-- examples/api/telegram/build.sh | 2 +- examples/api/telegram/server_adk.py | 25 +++++ 4 files changed, 211 insertions(+), 83 deletions(-) create mode 100644 examples/api/telegram/server_adk.py diff --git a/docs/docs/integrations/telegram.md b/docs/docs/integrations/telegram.md index c8eecf6b..8a394ea8 100644 --- a/docs/docs/integrations/telegram.md +++ b/docs/docs/integrations/telegram.md @@ -8,8 +8,8 @@ The `AgentTelegramRequestHandler` provides a seamless bridge between your Agent ### How it works: -1. User sends a message to your Telegram bot -2. Telegram delivers the message to your webhook endpoint +1. User sends a message to your Telegram bot +2. Telegram delivers the message to your webhook endpoint 3. Agent Kernel verifies and processes the message 4. Visual feedback is sent (typing indicator, etc.) 5. Agent generates a response and sends it back to the user @@ -51,11 +51,13 @@ export OPENAI_API_KEY="your_openai_api_key" Use a tunneling service for webhook delivery: **ngrok:** + ```bash ngrok http 8000 ``` **pinggy:** + ```bash ssh -p443 -R0:localhost:8000 a.pinggy.io ``` @@ -102,30 +104,69 @@ curl "https://api.telegram.org/bot/getWebhookInfo" ## Features -- Text message and command handling -- AI-powered responses via Agent Kernel -- Typing indicator -- Inline keyboards -- Markdown formatting -- Session management (per chat) -- Automatic message splitting for long responses +- **Text messages and commands** - Standard messaging with `/start`, `/help`, etc. +- **Multi-modal support** - Send and analyze images, PDFs, and document files alongside text +- **Typing indicator** - Visual feedback while processing +- **Inline keyboards** - Interactive button-based interactions +- **Markdown formatting** - Rich text formatting in responses +- **Session management** - Per-chat conversation context with memory +- **Automatic message splitting** - Long responses split to respect Telegram's 4096 character limit -## Advanced Usage +## Multi-Modal Support (Images & Documents) -### Custom Command Handler +The Telegram integration supports sending images and documents alongside text messages for intelligent analysis: + +### Supported File Types + +| Type | Formats | Use Cases | +| ------------------- | ------------------------ | ----------------------------------------------- | +| **Images** | JPEG, PNG, WebP, GIF | Photo analysis, vision tasks, object detection | +| **Documents** | PDF, TXT, CSV, DOCX, DOC | Document summarization, Q&A, content extraction | +| **Text** | Plain messages | Standard conversations | + +### How It Works + +1. User sends an image or document with optional text caption +2. Telegram webhook delivers the message to your agent +3. Agent Kernel downloads and processes the file +4. File is base64-encoded and sent to the agent for analysis +5. Agent generates response based on file content + text +6. Response is sent back through Telegram + +### Example Scenarios + +**Image Analysis:** + +- User sends photo of a receipt: "What's the total?" +- Bot analyzes image and responds with extracted total + +**Document Q&A:** + +- User sends PDF and asks: "Summarize key points" +- Bot reads and summarizes the document + +**Follow-up Questions:** + +- Message 1: User sends image, asks "What is this?" +- Bot analyzes and responds +- Message 2: User asks "What colors are dominant?" +- Bot remembers the image from previous context and answers + +### Session Memory with Context + +Each chat maintains conversation history: + +- Previous messages are remembered +- Images/files from earlier messages can be referenced +- Multi-turn conversations with rich context +- Works seamlessly with OpenAI GPT-4o and compatible models + if command == "/status": + await self._send_message(chat_id, "✅ Bot is running!") + elif command == "/about": + await self._send_message(chat_id, "I'm powered by Agent Kernel and OpenAI") + else: + await super()._handle_command(chat_id, command) -```python -from agentkernel.telegram import AgentTelegramRequestHandler - -class CustomTelegramHandler(AgentTelegramRequestHandler): - async def _handle_command(self, chat_id: int, command: str): - if command == "/status": - await self._send_message(chat_id, "✅ Bot is running!") - elif command == "/about": - await self._send_message(chat_id, "I'm powered by Agent Kernel and OpenAI") - else: - await super()._handle_command(chat_id, command) -``` ### Multi-Agent Setup @@ -171,52 +212,12 @@ await self._send_message( ) ``` - ## Supported Message Types - Text messages - Commands (e.g., /start, /help) - Inline keyboards and callback queries -## Troubleshooting - -### Webhook Verification Issues - -**Problem:** Webhook setup fails or Telegram can't reach your endpoint - -**Solutions:** -- Ensure your bot token is correct -- Webhook URL must be HTTPS and publicly accessible -- Check webhook info: `curl "https://api.telegram.org/bot/getWebhookInfo"` -- Review server logs for errors -- Verify agent configuration and OpenAI API key - -### No Messages Received - -**Problem:** Webhook is set but messages aren't reaching your agent - -**Solutions:** -- Verify your server is running and accessible -- Check that your webhook URL path is `/telegram/webhook` -- Review server logs for incoming webhook requests - -### Message Sending Failures - -**Problem:** Agent processes messages but responses don't appear in Telegram - -**Solutions:** -- Confirm you're using a valid bot token -- Check for API error codes in logs -- Ensure you are not exceeding rate limits - -### Authentication Errors - -**Problem:** "Invalid secret token" or authentication-related errors - -**Solutions:** -- Verify `AK_TELEGRAM__WEBHOOK_SECRET` matches what you set in Telegram -- Ensure the secret hasn't been changed -- Check server logs for validation details ### Enable Debug Logging @@ -234,6 +235,7 @@ logging.basicConfig( ## API Rate Limits Telegram Bot API enforces rate limits: + - **Messages to same chat:** 1 message per second - **Bulk messages:** 30 messages per second to different chats - **Group messages:** 20 messages per minute per group @@ -244,11 +246,11 @@ Best practice: Implement queuing for high-volume scenarios. ### Pre-Launch Checklist -✅ Use environment variables for all secrets \ +✅ Use environment variables for all secrets ✅ Deploy behind a reverse proxy (nginx, Apache) \ ✅ Set up health checks and monitoring \ ✅ Implement error handling and retry logic \ -✅ Review Telegram Bot API documentation for compliance +✅ Review Telegram Bot API documentation for compliance ### Deployment Architecture @@ -258,23 +260,25 @@ Best practice: Implement queuing for high-volume scenarios. ## Telegram vs Messenger Comparison -| Feature | Telegram | Facebook Messenger | -|------------------------|-------------------------|---------------------------| -| Message Limit | 4096 characters | 2000 characters | -| User Identifier | Chat ID | Page-Scoped ID (PSID) | -| Visual Feedback | Typing indicators | Typing, seen receipts | -| Interactive Elements | Inline keyboards | Buttons, quick replies | -| Authentication | Bot token + secret | Page access token + secret| -| App Review | Not required | Required for public access| -| Rich Media | Media, basic attachments| Extensive template support| +| Feature | Telegram | Facebook Messenger | +| -------------------- | ------------------------ | -------------------------- | +| Message Limit | 4096 characters | 2000 characters | +| User Identifier | Chat ID | Page-Scoped ID (PSID) | +| Visual Feedback | Typing indicators | Typing, seen receipts | +| Interactive Elements | Inline keyboards | Buttons, quick replies | +| Authentication | Bot token + secret | Page access token + secret | +| App Review | Not required | Required for public access | +| Rich Media | Media, basic attachments | Extensive template support | ## Example Projects -- Basic Example: `examples/api/telegram/server.py` +- Basic Example: \ +`examples/api/telegram/server.py`\ +`examples/api/telegram/server_adk.py` ## References - [Telegram Bot API Documentation](https://core.telegram.org/bots/api) - [BotFather](https://t.me/botfather) -*** +--- \ No newline at end of file diff --git a/examples/api/telegram/README.md b/examples/api/telegram/README.md index 5940d242..3aba9d48 100644 --- a/examples/api/telegram/README.md +++ b/examples/api/telegram/README.md @@ -59,24 +59,50 @@ For local development with Agent Kernel source: ## Run -Start the server: +### Option 1: Using OpenAI SDK (Recommended for beginners) ```bash uv run server.py ``` -The server will start on `http://0.0.0.0:8000` by default. +**Features:** + +- Uses OpenAI GPT-4o-mini model +- Built-in image and document analysis +- Session memory for multi-turn conversations +- Simple, straightforward setup + +### Option 2: Using Google ADK + +```bash +uv run server_adk.py +``` + +**Features:** + +- Uses Google ADK framework with OpenAI model via LiteLLM +- Same image/file analysis capabilities +- Different agent framework (for framework comparison) +- Requires `google-adk` package + +**Switching between frameworks:** + +- Both use the same Telegram bot configuration +- Same `config.yaml` file works for both +- Just run different server files to test ## Expose Local Server For local testing, expose your server using a tunnel: ### Using ngrok: + ```bash ngrok http 8000 ``` ### Using pinggy: + ```bash ssh -p443 -R0:localhost:8000 a.pinggy.io ``` @@ -86,6 +112,7 @@ Copy the HTTPS URL (e.g., `https://abc123.ngrok-free.app`) ## Configure Telegram Webhook ### Option 1: Using curl + ```bash curl -X POST "https://api.telegram.org/bot/setWebhook" \ -H "Content-Type: application/json" \ @@ -96,6 +123,7 @@ curl -X POST "https://api.telegram.org/bot/setWebhook" \ ``` ### Option 2: Using Python script + ```python import requests @@ -119,6 +147,59 @@ print(response.json()) curl "https://api.telegram.org/bot/getWebhookInfo" ``` +## Multimodal Features + +This integration supports images and documents: + +### Supported File Types + +| Type | Format | Use Case | +| ------------------- | --------------- | ---------------------- | +| **Images** | JPEG, PNG, WebP | Analysis, vision tasks | +| **Documents** | PDF, DOCX, TXT | Content analysis, Q&A | +| **Text** | Plain messages | Standard conversations | + +### Using Multimodal + +1. **Text Only** - Send a regular message + + ```bash + What is the capital of France? + ``` +2. **Image Analysis** - Send an image with caption + + ```bash + [Send image] What is in this image? + ``` +3. **Document Q&A** - Send a PDF with question + + ```bash + [Send PDF] Summarize this document + ``` +4. **Follow-up Questions** - Ask about previous context + + ```bash + [Send image first] + User: What is this? → Bot analyzes + User: Can you identify the parts? → Bot remembers image and answers + ``` + +### Session Memory + +The bot maintains conversation history: + +- Each Telegram chat has its own session +- Previous messages and context are remembered +- Images/PDFs from earlier in conversation can be referenced +- Session persists until bot restart + +### Multimodal Limitations + +- File size: Max 2MB +- Timeout: Requests take longer (30-60 seconds for large files) +- Context: Session cleared on server restart +- Supported models: OpenAI GPT-4o or compatible + ## Testing 1. Open Telegram and find your bot (by username you set in BotFather) @@ -129,9 +210,28 @@ curl "https://api.telegram.org/bot/getWebhookInfo" - Respond with AI-generated message 5. Check server logs to see the request/response flow -### Test Message Examples +### Test Multimodal Examples +```bash +# Test 1: Image Analysis +1. Send an image of a dog +2. User: "What animal is this?" +3. Expected: Bot identifies it as a dog + +# Test 2: Document Analysis +1. Send a PDF document +2. User: "Summarize the main points" +3. Expected: Bot reads and summarizes the document + +# Test 3: Follow-up Context +1. Send an image +2. User: "What is this?" → Bot responds +3. User: "What colors are dominant?" → Bot remembers the image ``` + +### Test Message Examples + +```bash /start Hello What can you help me with? @@ -142,6 +242,7 @@ Can you answer technical questions? ## Bot Commands Built-in commands: + - `/start` - Start conversation with the bot - `/help` - Show help message @@ -238,11 +339,9 @@ await self._send_message( - Ensure `AK_TELEGRAM__WEBHOOK_SECRET` matches what you set in webhook - Remove secret token temporarily for testing: `unset AK_TELEGRAM__WEBHOOK_SECRET` - ## Resources - [Telegram Bot API Documentation](https://core.telegram.org/bots/api) - [BotFather Commands](https://core.telegram.org/bots#botfather-commands) - [Telegram Bot Examples](https://core.telegram.org/bots/samples) -- [Agent Kernel Documentation](../../../docs/) - +- [Agent Kernel Documentation](../../../docs/) \ No newline at end of file diff --git a/examples/api/telegram/build.sh b/examples/api/telegram/build.sh index 23624f2a..ec8230e5 100755 --- a/examples/api/telegram/build.sh +++ b/examples/api/telegram/build.sh @@ -8,5 +8,5 @@ if [[ ${1-} != "local" ]]; then else # For local development of agentkernel, you can force reinstall from local dist uv sync --find-links ../../../ak-py/dist --all-extras - uv pip install --force-reinstall --find-links ../../../ak-py/dist agentkernel[api,openai,telegram] || true + uv pip install --force-reinstall --find-links ../../../ak-py/dist agentkernel[api,openai,adk,telegram] || true fi diff --git a/examples/api/telegram/server_adk.py b/examples/api/telegram/server_adk.py new file mode 100644 index 00000000..a779ed9f --- /dev/null +++ b/examples/api/telegram/server_adk.py @@ -0,0 +1,25 @@ +from agentkernel.adk import GoogleADKModule +from agentkernel.api import RESTAPI +from agentkernel.telegram import AgentTelegramRequestHandler +from google.adk.agents import Agent +from google.adk.models.lite_llm import LiteLlm + +# Create your Google ADK agent with correct API +general_agent = Agent( + name="general", + model=LiteLlm(model="openai/gpt-4o-mini"), + description="Agent for general questions", + instruction=""" + You provide assistance with general queries. + Give short and clear answers suitable for Telegram messaging. + If you receive images or files, analyze them and provide relevant insights. + """, +) + +# Initialize module with agent +GoogleADKModule([general_agent]) + + +if __name__ == "__main__": + handler = AgentTelegramRequestHandler() + RESTAPI.run([handler]) From d0199b6a0b001ed783ed298a62c68698cbd3621c Mon Sep 17 00:00:00 2001 From: pulinduvidmal Date: Tue, 23 Dec 2025 21:25:26 +0530 Subject: [PATCH 4/9] fix: resolve messenger integration conflicts from rebase --- .../integration/messenger/messenger_chat.py | 423 +----------------- 1 file changed, 10 insertions(+), 413 deletions(-) diff --git a/ak-py/src/agentkernel/integration/messenger/messenger_chat.py b/ak-py/src/agentkernel/integration/messenger/messenger_chat.py index a9cf6dc0..a5e4673d 100644 --- a/ak-py/src/agentkernel/integration/messenger/messenger_chat.py +++ b/ak-py/src/agentkernel/integration/messenger/messenger_chat.py @@ -1,4 +1,3 @@ -import base64 import hashlib import hmac import logging @@ -8,13 +7,7 @@ from fastapi import APIRouter, HTTPException, Request from ...api import RESTRequestHandler -from ...core import ( - AgentRequestFile, - AgentRequestImage, - AgentRequestText, - AgentService, - Config, -) +from ...core import AgentService, Config class AgentMessengerRequestHandler(RESTRequestHandler): @@ -151,22 +144,18 @@ async def _handle_message(self, messaging_event: dict): message = messaging_event.get("message", {}) message_id = message.get("mid") message_text = message.get("text") - attachments = message.get("attachments", []) if not sender_id or not message_id: self._log.warning("Message missing required fields (sender/mid)") return - # Allow message if it has text OR attachments - if not message_text and not attachments: - self._log.warning("Message has no text or attachments") + # Skip messages with attachments that don't have text + if not message_text: + self._log.warning("Message has no text content") return - if message_text: - self._log.debug(f"Processing message {message_id} from {sender_id}: {message_text}") - if attachments: - self._log.debug(f"Processing message {message_id} from {sender_id} with {len(attachments)} attachment(s)") - await self._process_agent_message(sender_id, message_text or "", attachments) + self._log.debug(f"Processing message {message_id} from {sender_id}: {message_text}") + await self._process_agent_message(sender_id, message_text) async def _handle_postback(self, messaging_event: dict): """ @@ -195,10 +184,7 @@ async def _handle_postback(self, messaging_event: dict): self._log.debug(f"Processing postback from {sender_id}: {message_text}") await self._process_agent_message(sender_id, message_text) - async def _process_agent_message(self, sender_id: str, message_text: str, attachments: list = None): - if attachments is None: - attachments = [] - + async def _process_agent_message(self, sender_id: str, message_text: str): service = AgentService() session_id = sender_id # Use sender_id as session_id to maintain conversation context try: @@ -216,66 +202,8 @@ async def _process_agent_message(self, sender_id: str, message_text: str, attach await self._send_typing_indicator(sender_id, False) return - # Extract and download attachments - processed_attachments = await self._extract_attachments(attachments) - - # Get session to store/retrieve attachment history - session = service.session - - # Store current attachment metadata in session - if processed_attachments: - self._store_attachment_metadata(session, processed_attachments) - - # Build request list: start with text only if it's not empty - requests = [] - if message_text.strip(): # Only add text if it's not empty - requests.append(AgentRequestText(text=message_text)) - requests.extend(processed_attachments) - - # Add conversation context if available - conversation_context = self._get_conversation_context(session) - if conversation_context: - requests.insert(0, AgentRequestText(text=conversation_context)) - self._log.info(f"[AGENT_CONTEXT] Added conversation history to context") - - # Add previous attachments to the request if available - previous_attachments = self._get_previous_attachments(session, processed_attachments) - self._log.info(f"[DEBUG] previous_attachments returned: {len(previous_attachments)} items") - if previous_attachments: - for att in previous_attachments: - self._log.info( - f"[DEBUG] Previous attachment: name={getattr(att, 'name', '?')}, has_data={bool(getattr(att, 'image_data' if 'Image' in type(att).__name__ else 'file_data', None))}" - ) - requests.extend(previous_attachments) - self._log.info(f"[AGENT_CONTEXT] Added {len(previous_attachments)} previous attachment(s) to context") - else: - self._log.info(f"[DEBUG] No previous attachments to add") - - # Log request summary - self._log.info( - f"[AGENT_INPUT] Total requests: {len(requests)} (text + {len(processed_attachments)} attachment(s) + context)" - ) - - # Store user message in conversation history for context (before agent runs) - if message_text.strip(): - self._store_user_message(session, message_text) - elif processed_attachments: - # Even if no text, store that user sent attachment(s) for context - attachment_names = [getattr(att, "name", "file") for att in processed_attachments] - self._store_user_message(session, f"[sent image/file: {', '.join(attachment_names)}]") - - # Run the agent - always use run_multi if there are requests - if len(requests) > 0: - self._log.info( - f"[AGENT_CALL] Running agent with {len(requests)} request(s) (text + {len(processed_attachments)} attachment(s))" - ) - result = await service.run_multi(requests) - else: - # No requests at all - nothing to process - self._log.warning("No text or attachments to process") - await self._send_message(sender_id, "Please send a message or attachment.") - await self._send_typing_indicator(sender_id, False) - return + # Run the agent + result = await service.run(message_text) if hasattr(result, "raw"): response_text = str(result.raw) @@ -284,9 +212,6 @@ async def _process_agent_message(self, sender_id: str, message_text: str, attach self._log.debug(f"Agent response: {response_text}") - # Store agent response in session for context - self._store_agent_response(session, response_text) - # Turn off typing indicator and send the response await self._send_typing_indicator(sender_id, False) await self._send_message(sender_id, response_text) @@ -376,332 +301,4 @@ async def _mark_seen(self, recipient_id: str): response.raise_for_status() self._log.debug(f"Message marked as seen: {recipient_id}") except Exception as e: - self._log.warning(f"Failed to mark message as seen: {e}") - - async def _extract_attachments(self, attachments: list) -> list: - """ - Extract and download attachments from Messenger message. - Messenger format: [{"type": "image", "payload": {"url": "..."}}] - - :param attachments: List of attachment objects from Messenger webhook - :return: List of AgentRequestImage or AgentRequestFile objects - """ - processed_attachments = [] - - if not attachments: - return processed_attachments - - try: - async with httpx.AsyncClient() as client: - for attachment in attachments: - attachment_type = attachment.get("type", "") - payload = attachment.get("payload", {}) - url = payload.get("url") - - if not url: - self._log.warning("Attachment missing URL") - continue - - try: - # Download attachment - response = await client.get(url) - response.raise_for_status() - - # Get MIME type from Content-Type header - mime_type = response.headers.get("content-type", "application/octet-stream").split(";")[0] - - # Convert to base64 - file_data = base64.b64encode(response.content).decode("utf-8") - - # Extract filename from URL or use default - filename = url.split("/")[-1].split("?")[0] or f"attachment_{len(processed_attachments)}" - - # Create appropriate request object - if attachment_type == "image" or mime_type.startswith("image/"): - request = AgentRequestImage(image_data=file_data, name=filename, mime_type=mime_type) - processed_attachments.append(request) - self._log.debug(f"Extracted image: {filename} ({mime_type})") - - elif attachment_type == "file" or mime_type in [ - "application/pdf", - "application/msword", - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - "application/vnd.ms-excel", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - ]: - request = AgentRequestFile(file_data=file_data, name=filename, mime_type=mime_type) - processed_attachments.append(request) - self._log.debug(f"Extracted file: {filename} ({mime_type})") - - else: - self._log.debug(f"Skipping unsupported attachment type: {mime_type}") - - except Exception as e: - self._log.warning(f"Error downloading attachment from {url}: {e}") - continue - - except Exception as e: - self._log.warning(f"Error processing attachments: {e}\n{traceback.format_exc()}") - - if processed_attachments: - self._log.info(f"Extracted {len(processed_attachments)} attachment(s)") - - return processed_attachments - - def _store_attachment_metadata(self, session, processed_attachments: list): - """ - Store attachment data and metadata in session for future reference. - Stores the actual base64-encoded data so images/files can be re-used. - - :param session: Agent session object - :param processed_attachments: List of AgentRequestImage or AgentRequestFile objects - """ - try: - import json - from datetime import datetime - - # Get existing attachment history - history = session.get_data(key="attachment_history") - if not history: - history = [] - else: - # Parse if it's a JSON string - if isinstance(history, str): - history = json.loads(history) - - # Add new attachments with full data - for attachment in processed_attachments: - # Extract base64 data based on type - if hasattr(attachment, "image_data"): - file_data = attachment.image_data - elif hasattr(attachment, "file_data"): - file_data = attachment.file_data - else: - file_data = None - - attachment_info = { - "name": getattr(attachment, "name", "unknown"), - "mime_type": getattr(attachment, "mime_type", "unknown"), - "type": type(attachment).__name__, # AgentRequestImage or AgentRequestFile - "data": file_data, # Store the actual base64 data - "timestamp": datetime.now().isoformat(), - } - history.append(attachment_info) - self._log.debug( - f"Stored attachment in session: {attachment_info['name']} ({attachment_info['mime_type']})" - ) - - # Keep only last 10 attachments to avoid session bloat (full data takes more space) - if len(history) > 10: - history = history[-10:] - - # Store back to session - session.set_data(key="attachment_history", data=json.dumps(history)) - self._log.info( - f"[DEBUG] Stored {len(processed_attachments)} attachment(s) in session. Total history: {len(history)}" - ) - self._log.info(f"[DEBUG] Attachment history keys: {[h['name'] for h in history]}") - - except Exception as e: - self._log.warning(f"Error storing attachment data: {e}") - - def _get_previous_attachments(self, session, current_attachments: list) -> list: - """ - Retrieve previous attachments from session and recreate request objects. - Only returns attachments NOT in the current message (to avoid duplicates). - - :param session: Agent session object - :param current_attachments: List of current attachments (to skip duplicates) - :return: List of AgentRequestImage or AgentRequestFile objects from history - """ - try: - import json - - history = session.get_data(key="attachment_history") - self._log.info( - f"[DEBUG] Retrieved history from session: {type(history)}, length: {len(history) if history else 0}" - ) - - if not history: - self._log.info(f"[DEBUG] History is empty or None") - return [] - - # Parse if it's a JSON string - if isinstance(history, str): - history = json.loads(history) - self._log.info(f"[DEBUG] Parsed JSON history, items: {len(history)}") - - if not history: - return [] - - # Get names of current attachments to skip duplicates - current_names = {getattr(att, "name", "") for att in current_attachments} - self._log.info(f"[DEBUG] Current attachment names: {current_names}") - - # Recreate request objects from history (excluding current) - previous_attachments = [] - for attachment_info in history: - att_name = attachment_info.get("name", "unknown") - self._log.info( - f"[DEBUG] Checking history item: {att_name}, in current_names? {att_name in current_names}" - ) - - # Skip if it's in the current message - if att_name in current_names: - self._log.info(f"[DEBUG] Skipping {att_name} (duplicate)") - continue - - attachment_type = attachment_info.get("type", "AgentRequestFile") - mime_type = attachment_info.get("mime_type", "unknown") - file_data = attachment_info.get("data") - name = attachment_info.get("name", "unknown") - - if not file_data: - self._log.warning(f"[DEBUG] No file_data for {name}") - continue - - # Recreate the appropriate request object - try: - if attachment_type == "AgentRequestImage": - request = AgentRequestImage(image_data=file_data, name=name, mime_type=mime_type) - previous_attachments.append(request) - self._log.info(f"[DEBUG] Recreated image: {name} ({len(file_data)} bytes)") - elif attachment_type == "AgentRequestFile": - request = AgentRequestFile(file_data=file_data, name=name, mime_type=mime_type) - previous_attachments.append(request) - self._log.info(f"[DEBUG] Recreated file: {name} ({len(file_data)} bytes)") - else: - self._log.warning(f"[DEBUG] Unknown attachment type: {attachment_type}") - except Exception as e: - self._log.warning(f"Error recreating attachment {name}: {e}") - continue - - self._log.info(f"[DEBUG] Total previous attachments recreated: {len(previous_attachments)}") - return previous_attachments - - except Exception as e: - self._log.warning(f"Error retrieving previous attachments: {e}\n{traceback.format_exc()}") - return [] - - def _store_agent_response(self, session, response_text: str): - """ - Store agent response in session for conversation context. - Helps agent understand what it already said in previous turns. - - :param session: Agent session object - :param response_text: Agent's response text - """ - try: - import json - from datetime import datetime - - # Get existing conversation history - history = session.get_data(key="conversation_history") - if not history: - history = [] - else: - # Parse if it's a JSON string - if isinstance(history, str): - history = json.loads(history) - - # Add agent response - history.append( - { - "role": "agent", - "content": response_text, - "timestamp": datetime.now().isoformat(), - } - ) - - # Keep only last 20 exchanges to avoid session bloat - if len(history) > 20: - history = history[-20:] - - # Store back to session - session.set_data(key="conversation_history", data=json.dumps(history)) - self._log.info(f"[DEBUG] Stored agent response in conversation history. Total: {len(history)}") - - except Exception as e: - self._log.warning(f"Error storing agent response: {e}") - - def _store_user_message(self, session, message_text: str): - """ - Store user message in session for conversation context. - Helps agent understand the full conversation flow and context. - - :param session: Agent session object - :param message_text: User's message text - """ - try: - import json - from datetime import datetime - - # Get existing conversation history - history = session.get_data(key="conversation_history") - if not history: - history = [] - else: - # Parse if it's a JSON string - if isinstance(history, str): - history = json.loads(history) - - # Add user message - history.append( - { - "role": "user", - "content": message_text, - "timestamp": datetime.now().isoformat(), - } - ) - - # Keep only last 20 exchanges to avoid session bloat - if len(history) > 20: - history = history[-20:] - - # Store back to session - session.set_data(key="conversation_history", data=json.dumps(history)) - self._log.info(f"[DEBUG] Stored user message in conversation history. Total: {len(history)}") - - except Exception as e: - self._log.warning(f"Error storing user message: {e}") - - def _get_conversation_context(self, session) -> str: - """ - Retrieve conversation history and format as context for agent. - Helps agent understand what was already discussed. - - :param session: Agent session object - :return: Formatted conversation context string - """ - try: - import json - - history = session.get_data(key="conversation_history") - if not history: - return "" - - # Parse if it's a JSON string - if isinstance(history, str): - history = json.loads(history) - - if not history: - return "" - - # Format as readable context - context_lines = ["[Previous conversation context:"] - for item in history[-10:]: # Last 10 messages to cover multi-turn conversations - role = item.get("role", "unknown") - content = item.get("content", "") - # Truncate long responses to 200 chars - if len(content) > 200: - content = content[:200] + "..." - context_lines.append(f" {role}: {content}") - context_lines.append("]") - - result = "\n".join(context_lines) - self._log.info(f"[DEBUG] Retrieved conversation context ({len(history)} items)") - return result - - except Exception as e: - self._log.warning(f"Error retrieving conversation context: {e}") - return "" + self._log.warning(f"Failed to mark message as seen: {e}") \ No newline at end of file From 698747dc3c2ccf08b8821b9049bcc116f13d3a66 Mon Sep 17 00:00:00 2001 From: pulinduvidmal Date: Tue, 23 Dec 2025 21:31:17 +0530 Subject: [PATCH 5/9] docs: update doc --- .../src/agentkernel/integration/telegram/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ak-py/src/agentkernel/integration/telegram/README.md b/ak-py/src/agentkernel/integration/telegram/README.md index ca7bd561..7f99878a 100644 --- a/ak-py/src/agentkernel/integration/telegram/README.md +++ b/ak-py/src/agentkernel/integration/telegram/README.md @@ -99,13 +99,13 @@ The Telegram integration **fully supports** sending images and documents to agen 2. **Documents**: Files are downloaded, base64-encoded, and sent to the agent as `AgentRequestFile` 3. **Combined**: A message can include both text and files/images which are all processed together -**Status:** ✅ **FULLY IMPLEMENTED AND WORKING** +**Status:** -- ✅ File/image detection and download infrastructure -- ✅ Base64 encoding of files/images -- ✅ Multi-modal requests forwarding (`service.run_multi()`) -- ✅ Session memory for follow-up questions with context -- ✅ Works with **OpenAI SDK** and **Google ADK** agents +- File/image detection and download infrastructure +- Base64 encoding of files/images +- Multi-modal requests forwarding (`service.run_multi()`) +- Session memory for follow-up questions with context +- Works with **OpenAI SDK** and **Google ADK** agents **Supported file types:** @@ -114,7 +114,7 @@ The Telegram integration **fully supports** sending images and documents to agen **Limitations:** -- Maximum file size: ~20MB (Telegram API limit, may vary by file type) +- Maximum file size: ~2MB - Processing time: Large files may take 30-60 seconds to download and analyze - Session context: Cleared on server restart (can be persisted with external storage) - Model requirements: Agent must support multimodal (OpenAI GPT-4o, Google ADK, etc.) From bb4c1b35e325b29372da2a883e7938b4b7ff8b3e Mon Sep 17 00:00:00 2001 From: pulinduvidmal Date: Tue, 23 Dec 2025 21:35:28 +0530 Subject: [PATCH 6/9] fix: resolve messenger integration conflicts from rebase --- ak-py/src/agentkernel/integration/messenger/messenger_chat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ak-py/src/agentkernel/integration/messenger/messenger_chat.py b/ak-py/src/agentkernel/integration/messenger/messenger_chat.py index a5e4673d..1cc6d883 100644 --- a/ak-py/src/agentkernel/integration/messenger/messenger_chat.py +++ b/ak-py/src/agentkernel/integration/messenger/messenger_chat.py @@ -301,4 +301,4 @@ async def _mark_seen(self, recipient_id: str): response.raise_for_status() self._log.debug(f"Message marked as seen: {recipient_id}") except Exception as e: - self._log.warning(f"Failed to mark message as seen: {e}") \ No newline at end of file + self._log.warning(f"Failed to mark message as seen: {e}") From ac4e5cd324755cb686305753eb860f323e893b91 Mon Sep 17 00:00:00 2001 From: pulinduvidmal Date: Wed, 24 Dec 2025 02:05:30 +0530 Subject: [PATCH 7/9] feat: add multimodal file/image support for messenger with documentation --- .../integration/messenger/README.md | 86 +++++++++---- .../integration/messenger/messenger_chat.py | 114 ++++++++++++++++-- docs/docs/integrations/messenger.md | 109 +++++++++++++---- examples/api/messenger/README.md | 52 +++++++- examples/api/messenger/build.sh | 2 +- examples/api/messenger/server_adk.py | 25 ++++ 6 files changed, 320 insertions(+), 68 deletions(-) create mode 100644 examples/api/messenger/server_adk.py diff --git a/ak-py/src/agentkernel/integration/messenger/README.md b/ak-py/src/agentkernel/integration/messenger/README.md index 9d3edd91..e727cc11 100644 --- a/ak-py/src/agentkernel/integration/messenger/README.md +++ b/ak-py/src/agentkernel/integration/messenger/README.md @@ -15,28 +15,28 @@ The `AgentMessengerRequestHandler` class handles conversations with agents via F ## Facebook Messenger Setup ### Prerequisites + Please follow the steps in the [Messenger Platform Getting Started Guide](https://developers.facebook.com/docs/messenger-platform/getting-started) ### Configuration Steps 1. **Create a Facebook App** + - Go to https://developers.facebook.com/apps - Create a new app and select "Business" type - Add Messenger product to your app - 2. **Create or Link a Facebook Page** + - You need a Facebook Page to use Messenger Platform - Link your page to the app in Messenger Settings - 3. **Get Your Credentials** - - **Page Access Token**: App > Left side panel > Select 'Use cases' > Select 'Engage with customers on messenger from Meta' > Select 'Customise' - - Click on 'Messenger API settings' > Select "Generate Access Tokens" and do the needful + - **Page Access Token**: App > Left side panel > Select 'Use cases' > Select 'Engage with customers on messenger from Meta' > Select 'Customise' + - Click on 'Messenger API settings' > Select "Generate Access Tokens" and do the needful - **App Secret**: [Optional but recommended for security]. App > App Setting > Basic - - **Verify Token**: Create your own secure random string for webhook verification - 4. **Configure Webhook** + - Go to 'Messenger API settings' (described above) > Select "Config Webhooks" - click "Add Callback URL". Set callback URL: `https://your-domain.com/messenger/webhook` - Set verify token: Your chosen verify token. Click "Verify and Save" @@ -46,18 +46,18 @@ Please follow the steps in the [Messenger Platform Getting Started Guide](https: - `messaging_postbacks` - To handle button clicks - `message_deliveries` - For delivery confirmations (optional) - `message_reads` - For read receipts (optional) - 5. **Subscribe Your Page** - * After webhook verification, subscribe your page to receive events. To do that go back to "Generate Access Token" > "Webhook Subcription" - * Subscribe to these webhook fields: - * `messages` - To receive messages - * `messaging_optins` - To receive messages - * `messaging_postbacks` - To handle button clicks + * After webhook verification, subscribe your page to receive events. To do that go back to "Generate Access Token" > "Webhook Subcription" + * Subscribe to these webhook fields: + * `messages` - To receive messages + * `messaging_optins` - To receive messages + * `messaging_postbacks` - To handle button clicks 6. **Testing** - * See the Testing steps at the bottom - * You need to get the approval for the FB page to enable message flow from the page to the Agent. - * From 'Messenger API settings' > 'Complete App Review' section > Select 'Request Permission' + +* See the Testing steps at the bottom +* You need to get the approval for the FB page to enable message flow from the page to the Agent. + * From 'Messenger API settings' > 'Complete App Review' section > Select 'Request Permission' ### Required Environment Variables @@ -69,6 +69,7 @@ export AK_MESSENGER__API_VERSION="v21.0" # Optional, defaults to v24.0. Only ch ``` ### Webhook Verification + The handler automatically responds to Facebook's webhook verification challenge via `/messenger/webhook` (e.g., `http://localhost:8000/messenger/webhook`). When you configure the webhook URL in Facebook's developer portal, Facebook will send a GET request to verify the endpoint. The handler processes this automatically. ## Simple Facebook Messenger Integration Code @@ -103,6 +104,7 @@ messenger: agent: "general" # Name of the agent to handle Messenger messages api_version: "v21.0" # Optional, defaults to v24.0 ``` + It is strongly recommended not to keep secrets and keys in the config file. Set them as environment variables. ## Features @@ -111,16 +113,42 @@ It is strongly recommended not to keep secrets and keys in the config file. Set - **Text Messages**: Standard text messages - **Postbacks**: Button clicks and quick reply selections -- **Attachments**: Images, videos, audio, files (basic detection) +- **Attachments**: Images, videos, audio, files with full multimodal analysis ✅ - **Delivery Receipts**: Message delivery confirmations - **Read Receipts**: Message read notifications +### Multi-Modal Support (Images & Files) ✅ + +The Messenger integration fully supports sending and analyzing images and files: + +**Supported File Types:** + +- **Images**: JPEG, PNG, GIF, WebP +- **Documents**: PDF, Word (.docx), Excel (.xlsx), PowerPoint (.pptx), Text files +- **Media**: Audio and video files + +**How It Works:** + +1. User sends message with attachment (image or file) +2. Handler downloads attachment from Messenger +3. File is validated against size limit (default 2 MB, configurable) +4. File is base64-encoded for transmission to AI agent +5. Agent analyzes and responds with insights + +**File Size Limits:** + +- Default maximum: **2 MB per file** +- Base64 encoding increases size by ~33%, so effective usable size is ~1.5 MB +- Configurable via `api.max_file_size` in config.yaml + ### Message Handling - **Automatic Message Splitting**: Messages longer than 2000 characters are automatically split - **Session Management**: Uses sender ID as session ID to maintain conversation context - **Typing Indicators**: Shows typing indicator while processing - **Message Threading**: Maintains conversation flow +- **Multimodal Processing**: Automatically detects and processes images and files alongside text +- **File Size Validation**: Prevents oversized files from being processed ### Security @@ -135,11 +163,13 @@ It is strongly recommended not to keep secrets and keys in the config file. Set For local testing, use a tunneling service to expose your local server: **Using ngrok:** + ```bash ngrok http 8000 ``` **Using pinggy:** + ```bash ssh -p443 -R0:localhost:8000 a.pinggy.io ``` @@ -147,10 +177,11 @@ ssh -p443 -R0:localhost:8000 a.pinggy.io Update your Facebook webhook URL with the tunnel URL. ### Testing Steps + This assumes that you have successfully verified the Webhook URL and correct subscriptions are enabled. 1. App > Left side panel > Select 'Use cases' > Select 'Engage with customers on messenger from Meta' > Select 'Customise' - - Click on 'Messenger API settings' > Select "API integration helper" + - Click on 'Messenger API settings' > Select "API integration helper" 2. In the API integration helper, insert your page access token and the page will automatically get selected. 3. From there select a recipient, insert the test message and click on "Send message" @@ -168,7 +199,7 @@ class CustomMessengerHandler(AgentMessengerRequestHandler): # Add custom preprocessing message = messaging_event.get("message", {}) message_text = message.get("text", "") - + # Custom logic here if message_text.startswith("/help"): sender_id = messaging_event["sender"]["id"] @@ -177,7 +208,7 @@ class CustomMessengerHandler(AgentMessengerRequestHandler): "Available commands: ..." ) return - + # Call parent handler await super()._handle_message(messaging_event) ``` @@ -194,7 +225,7 @@ async def _send_quick_replies(self, recipient_id: str, text: str, quick_replies: "Authorization": f"Bearer {self._access_token}", "Content-Type": "application/json" } - + payload = { "recipient": {"id": recipient_id}, "message": { @@ -203,7 +234,7 @@ async def _send_quick_replies(self, recipient_id: str, text: str, quick_replies: }, "messaging_type": "RESPONSE" } - + async with httpx.AsyncClient() as client: response = await client.post(url, json=payload, headers=headers) response.raise_for_status() @@ -214,12 +245,14 @@ async def _send_quick_replies(self, recipient_id: str, text: str, quick_replies: ### Common Issues **Webhook verification fails:** + - Ensure verify token in config matches what you set in Facebook developer portal - Check that your endpoint is accessible via HTTPS - Verify the callback URL is correct - Make sure the handler returns the challenge as an integer **No Messages Received:** + - Check webhook subscription is active - Verify page is subscribed to webhook events - Check app secret and access token are correct @@ -227,12 +260,14 @@ async def _send_quick_replies(self, recipient_id: str, text: str, quick_replies: - Ensure the app is not in development mode restrictions **Agent Not Responding:** + - Verify access token has correct permissions - Check agent configuration in config.yaml - Review server logs for agent errors - Ensure the page access token hasn't expired **Messages Not Sending:** + - Verify access token is a page access token, not user token - Check token permissions include `pages_messaging` - Ensure recipient has messaged your page first (24-hour window rule) @@ -250,6 +285,7 @@ logging.basicConfig(level=logging.DEBUG) ## API Rate Limits Facebook Messenger API has rate limits: + - **Standard Messaging**: 10,000 messages per day per page (varies by tier) - **Broadcast Messages**: Requires approval and different messaging type - **Response Time**: Must respond to user messages within 24 hours @@ -261,6 +297,7 @@ For higher limits, apply for standard messaging access. ### Message Types The platform supports three messaging types: + - **RESPONSE**: Reply to user messages (default, used in this integration) - **UPDATE**: Send proactive notifications (requires approval) - **MESSAGE_TAG**: Send messages outside 24-hour window with approved tags @@ -272,6 +309,7 @@ Quick replies provide predefined response options to users. Extend the handler t ### Buttons and Templates Messenger supports rich templates like: + - Button template - Generic template (cards) - List template @@ -282,12 +320,14 @@ These can be added by extending the `_send_message` method. ## Privacy and Compliance ### User Data + - Store user data securely - Implement data retention policies - Provide data deletion capabilities - Follow GDPR/CCPA requirements ### Facebook Policies + - Review Facebook Platform Policy - Ensure compliance with Messenger Platform Policy - Get required permissions before going live @@ -306,7 +346,9 @@ For production deployment: 7. **Get App Reviewed**: Submit your app for review to remove development mode restrictions ### App Review + Before going live: + 1. Complete app review process 2. Request `pages_messaging` permission 3. Provide test credentials and instructions @@ -330,4 +372,4 @@ Key differences between Messenger and WhatsApp integrations: - [Send API Reference](https://developers.facebook.com/docs/messenger-platform/reference/send-api) - [Webhook Reference](https://developers.facebook.com/docs/messenger-platform/webhooks) - [Platform Policy](https://developers.facebook.com/docs/messenger-platform/policy-overview) -- [Agent Kernel Documentation](../../../docs/) +- [Agent Kernel Documentation](../../../docs/) \ No newline at end of file diff --git a/ak-py/src/agentkernel/integration/messenger/messenger_chat.py b/ak-py/src/agentkernel/integration/messenger/messenger_chat.py index 1cc6d883..81754cf0 100644 --- a/ak-py/src/agentkernel/integration/messenger/messenger_chat.py +++ b/ak-py/src/agentkernel/integration/messenger/messenger_chat.py @@ -1,6 +1,8 @@ +import base64 import hashlib import hmac import logging +import mimetypes import traceback import httpx @@ -8,6 +10,7 @@ from ...api import RESTRequestHandler from ...core import AgentService, Config +from ...core.model import AgentRequestFile, AgentRequestImage, AgentRequestText class AgentMessengerRequestHandler(RESTRequestHandler): @@ -30,6 +33,7 @@ def __init__(self): self._app_secret = Config.get().messenger.app_secret self._api_version = Config.get().messenger.api_version or "v24.0" self._base_url = f"https://graph.facebook.com/{self._api_version}" + self._max_file_size = Config.get().api.max_file_size if not all([self._access_token, self._verify_token]): self._log.error("Facebook Messenger configuration is incomplete. Please set access_token and verify_token.") raise ValueError("Incomplete Facebook Messenger configuration.") @@ -143,19 +147,20 @@ async def _handle_message(self, messaging_event: dict): sender_id = messaging_event.get("sender", {}).get("id") message = messaging_event.get("message", {}) message_id = message.get("mid") - message_text = message.get("text") + message_text = message.get("text", "").strip() + attachments = message.get("attachments", []) if not sender_id or not message_id: self._log.warning("Message missing required fields (sender/mid)") return - # Skip messages with attachments that don't have text - if not message_text: - self._log.warning("Message has no text content") + # Skip if no text and no attachments + if not message_text and not attachments: + self._log.warning("Message has no text content or attachments") return - self._log.debug(f"Processing message {message_id} from {sender_id}: {message_text}") - await self._process_agent_message(sender_id, message_text) + self._log.debug(f"Processing message {message_id} from {sender_id}: text='{message_text}', attachments={len(attachments)}") + await self._process_agent_message(sender_id, message_text, attachments) async def _handle_postback(self, messaging_event: dict): """ @@ -184,7 +189,7 @@ async def _handle_postback(self, messaging_event: dict): self._log.debug(f"Processing postback from {sender_id}: {message_text}") await self._process_agent_message(sender_id, message_text) - async def _process_agent_message(self, sender_id: str, message_text: str): + async def _process_agent_message(self, sender_id: str, message_text: str, attachments: list = None): service = AgentService() session_id = sender_id # Use sender_id as session_id to maintain conversation context try: @@ -202,13 +207,35 @@ async def _process_agent_message(self, sender_id: str, message_text: str): await self._send_typing_indicator(sender_id, False) return + # Build requests list with text and attachments + requests = [] + + # Add text if present + if message_text: + requests.append(AgentRequestText(text=message_text)) + + # Process attachments (images and files) + if attachments: + for attachment in attachments: + await self._process_attachment(attachment, requests) + # Run the agent - result = await service.run(message_text) - - if hasattr(result, "raw"): - response_text = str(result.raw) + if requests: + # Use run_multi for multimodal requests + if len(requests) > 1 or any(isinstance(r, (AgentRequestFile, AgentRequestImage)) for r in requests): + result = await service.run_multi(requests=requests) + else: + result = await service.run(message_text) if message_text else None else: - response_text = str(result) + result = None + + if result: + if hasattr(result, "raw"): + response_text = str(result.raw) + else: + response_text = str(result) + else: + response_text = "Sorry, I could not process your message." self._log.debug(f"Agent response: {response_text}") @@ -221,6 +248,67 @@ async def _process_agent_message(self, sender_id: str, message_text: str): await self._send_typing_indicator(sender_id, False) await self._send_message(sender_id, "Sorry, there was an error processing your request.") + async def _process_attachment(self, attachment: dict, requests: list): + """ + Process a Messenger attachment (image or file). + + :param attachment: Attachment object from message + :param requests: List to append the processed request to + """ + attachment_type = attachment.get("type") + payload = attachment.get("payload", {}) + url = payload.get("url") + + if not url: + self._log.warning(f"Attachment has no URL: {attachment}") + return + + try: + # Download the attachment + async with httpx.AsyncClient() as client: + response = await client.get(url, timeout=10.0) + response.raise_for_status() + file_data = response.content + + # Check file size + if len(file_data) > self._max_file_size: + self._log.warning(f"Attachment size ({len(file_data) / (1024 * 1024):.2f} MB) exceeds maximum allowed size of {self._max_file_size / (1024 * 1024):.2f} MB") + return + + # Encode to base64 + file_data_base64 = base64.b64encode(file_data).decode("utf-8") + + # Get MIME type + mime_type = response.headers.get("content-type", "application/octet-stream") + + # Extract filename from URL if available + filename = url.split("/")[-1].split("?")[0] or f"attachment_{len(requests)}" + + self._log.debug(f"Downloaded {attachment_type} attachment: {filename} (size: {len(file_data)} bytes, type: {mime_type})") + + # Classify based on attachment type and MIME type + if attachment_type == "image" or (mime_type and mime_type.startswith("image/")): + self._log.debug(f"Adding image: {filename}") + requests.append( + AgentRequestImage( + image_data=file_data_base64, + name=filename, + mime_type=mime_type, + ) + ) + else: + self._log.debug(f"Adding file: {filename}") + requests.append( + AgentRequestFile( + file_data=file_data_base64, + name=filename, + mime_type=mime_type, + ) + ) + + except Exception as e: + self._log.error(f"Error processing attachment: {e}\n{traceback.format_exc()}") + async def _send_message(self, recipient_id: str, text: str): """ Send a Messenger message using the Send API. @@ -301,4 +389,4 @@ async def _mark_seen(self, recipient_id: str): response.raise_for_status() self._log.debug(f"Message marked as seen: {recipient_id}") except Exception as e: - self._log.warning(f"Failed to mark message as seen: {e}") + self._log.warning(f"Failed to mark message as seen: {e}") \ No newline at end of file diff --git a/docs/docs/integrations/messenger.md b/docs/docs/integrations/messenger.md index 04b8a04d..2d6a9ac9 100644 --- a/docs/docs/integrations/messenger.md +++ b/docs/docs/integrations/messenger.md @@ -69,9 +69,11 @@ Before you begin, you'll need: **Create a Verify Token:** This is a random string you create yourself for webhook verification. Choose something secure like: + ```bash openssl rand -hex 32 ``` + Save this as `AK_MESSENGER__VERIFY_TOKEN` ### 3. Configure Environment Variables @@ -107,6 +109,7 @@ ssh -p443 -R0:localhost:8000 a.pinggy.io **Subscribe to webhook events:** Still in **Messenger API settings**, under "Webhooks", subscribe to: + - `messages` - To receive user messages - `messaging_postbacks` - To handle button clicks - `messaging_optins` - To receive opt-in events @@ -201,26 +204,26 @@ class CustomMessengerHandler(AgentMessengerRequestHandler): message = messaging_event.get("message", {}) message_text = message.get("text", "").strip() sender_id = messaging_event.get("sender", {}).get("id") - + # Handle special commands if message_text.startswith("/"): await self._handle_command(message_text, sender_id) return - + # Preprocess messages before sending to agent processed_text = self._preprocess_message(message_text) - + # Continue with normal processing await super()._handle_message(messaging_event) - + async def _handle_command(self, command: str, sender_id: str): """Handle custom commands""" await self._mark_seen(sender_id) await self._send_typing_indicator(sender_id, True) - + if command == "/help": help_text = """🤖 Available Commands: - + /help - Show this help message /start - Start a new conversation @@ -236,9 +239,9 @@ Just send any message to chat with me!""" sender_id, f"Unknown command. Try /help for available commands." ) - + await self._send_typing_indicator(sender_id, False) - + def _preprocess_message(self, text: str) -> str: """Clean up or enhance user messages""" # Expand common abbreviations @@ -290,15 +293,54 @@ OpenAIModule([sales_agent, support_agent, general_agent]) ## Supported Message Types ### Text Messages + Standard text messages are fully supported with automatic context management. ### Postbacks + Handle button clicks and quick reply selections. Postbacks are processed as text using the button title or payload. -### Attachments -The integration detects images, videos, audio, and files. Basic attachment information is logged, with extensibility for custom handling. +### Multi-Modal: Images and Files + +The integration provides full support for images and file attachments with AI analysis: + +**Supported Formats:** + +- **Images**: JPEG, PNG, GIF, WebP (detected automatically) +- **Documents**: PDF, Word (.docx), Excel (.xlsx), PowerPoint (.pptx), Text files +- **Media**: Audio and video files + +**How It Works:** + +1. **Detection**: When user sends message with attachment, handler identifies type +2. **Download**: File is downloaded from Messenger's servers +3. **Validation**: File size checked against limit (default 2 MB) +4. **Encoding**: File converted to base64 for transmission +5. **Processing**: Agent receives file and can analyze it +6. **Response**: Agent provides insights or answers about the content + +**File Size Limits:** + +- **Default**: 2 MB per file (2,097,152 bytes) +- **Base64 Overhead**: ~33% increase in size +- **Effective Size**: ~1.5 MB usable after base64 encoding +- **Configurable**: Set `api.max_file_size` in config.yaml + +**Example User Interactions:** + +```bash +User: "What's in this photo?" [sends image] +Agent: [analyzes image] "This appears to be..." + +User: "Extract the text from this PDF" [sends PDF] +Agent: [processes PDF] "The document contains..." + +User: "Can you read this note?" [sends image] +Agent: [uses OCR] "The note says..." +``` ### Delivery and Read Receipts + Automatically logged for monitoring and debugging purposes. ## Troubleshooting @@ -308,6 +350,7 @@ Automatically logged for monitoring and debugging purposes. **Problem:** "Webhook verification failed" error when configuring callback URL **Solutions:** + - Ensure your verify token in the environment variable exactly matches what you enter in Facebook - Verify your server is running and accessible via HTTPS - Check that your webhook URL path is `/messenger/webhook` @@ -319,6 +362,7 @@ Automatically logged for monitoring and debugging purposes. **Problem:** Webhook is verified but messages aren't reaching your agent **Solutions:** + - Check that webhook subscriptions include `messages` and are active - Verify your page is subscribed to the webhook (in Webhook Subscriptions) - Ensure your access token hasn't expired @@ -330,6 +374,7 @@ Automatically logged for monitoring and debugging purposes. **Problem:** Agent processes messages but responses don't appear in Messenger **Solutions:** + - Confirm you're using a **page access token**, not a user access token - Verify the token has `pages_messaging` permission - Check that you're responding within the 24-hour messaging window @@ -341,6 +386,7 @@ Automatically logged for monitoring and debugging purposes. **Problem:** "Invalid signature" or authentication-related errors **Solutions:** + - Verify `AK_MESSENGER__APP_SECRET` matches your app's actual secret - Ensure the app secret hasn't been regenerated in Facebook - Check that webhook payloads haven't been modified in transit @@ -361,6 +407,7 @@ logging.basicConfig( ``` This will show: + - Incoming webhook requests - Signature verification steps - Agent processing details @@ -372,11 +419,13 @@ This will show: Facebook Messenger enforces rate limits to ensure platform stability: ### Message Limits + - **Standard tier**: ~10,000 messages per day per page - **Rate**: Varies by tier and page quality score - **Best practice**: Implement queuing for high-volume scenarios ### Response Window + - You must respond to user messages within **24 hours** - After 24 hours, you need special permissions (Message Tags) - To send messages outside this window, apply for advanced messaging features @@ -391,7 +440,7 @@ class RateLimitedMessengerHandler(AgentMessengerRequestHandler): def __init__(self): super().__init__() self.message_queue = Queue() - + async def _send_message(self, recipient_id: str, text: str): # Add delay to respect rate limits await asyncio.sleep(0.1) # 10 messages per second @@ -405,36 +454,37 @@ class RateLimitedMessengerHandler(AgentMessengerRequestHandler): Before deploying to production: 1. ✅ **Complete Facebook App Review** + - Request `pages_messaging` permission - Submit app for review with clear use case documentation - Provide test credentials and instructions - Typical approval time: 3-5 business days - 2. ✅ **Security Measures** + - Use environment variables for all secrets - Enable app secret verification (`AK_MESSENGER__APP_SECRET`) - Implement HTTPS with valid SSL certificate - Use secure secret management (AWS Secrets Manager, HashiCorp Vault) - 3. ✅ **Infrastructure** + - Deploy behind a reverse proxy (nginx, Apache) - Set up load balancing for high traffic - Implement health checks and monitoring - Configure auto-scaling if using cloud services - 4. ✅ **Monitoring & Logging** + - Set up centralized logging (CloudWatch, Datadog, ELK) - Configure alerts for errors and anomalies - Track conversation metrics and performance - Monitor API rate limits - 5. ✅ **Error Handling** + - Implement retry logic with exponential backoff - Handle network failures gracefully - Provide fallback responses for errors - Log errors for debugging - 6. ✅ **Compliance** + - Review Facebook Platform Policies - Ensure GDPR/CCPA compliance for user data - Implement data retention policies @@ -445,16 +495,19 @@ Before deploying to production: For production deployments, consider: **Serverless (AWS Lambda):** + - Cost-effective for low-to-medium traffic - Auto-scaling built-in - See `examples/aws-serverless` for reference **Containerized (Docker/Kubernetes):** + - Better for high traffic and complex workflows - Full control over environment - See `examples/aws-containerized` for reference **Traditional Server:** + - Simple deployment for small-scale applications - Use systemd or supervisor for process management - Configure nginx as reverse proxy @@ -466,6 +519,7 @@ For production deployments, consider: Extend the integration to send structured content: **Quick Replies:** + ```python async def send_with_quick_replies(self, recipient_id: str, text: str): payload = { @@ -482,6 +536,7 @@ async def send_with_quick_replies(self, recipient_id: str, text: str): ``` **Button Templates:** + ```python async def send_button_template(self, recipient_id: str): payload = { @@ -526,21 +581,23 @@ async def setup_persistent_menu(self): ## Messenger vs WhatsApp Comparison -| Feature | Facebook Messenger | WhatsApp | -|---------|-------------------|----------| -| **Message Limit** | 2,000 characters | 4,096 characters | -| **User Identifier** | Page-Scoped ID (PSID) | Phone number | -| **Visual Feedback** | Typing indicators, seen receipts | Read receipts | +| Feature | Facebook Messenger | WhatsApp | +| ------------------------------ | --------------------------------- | ----------------------------- | +| **Message Limit** | 2,000 characters | 4,096 characters | +| **User Identifier** | Page-Scoped ID (PSID) | Phone number | +| **Visual Feedback** | Typing indicators, seen receipts | Read receipts | | **Interactive Elements** | Buttons, quick replies, templates | Interactive messages, buttons | -| **Authentication** | Page access token | Phone number ID + token | -| **App Review** | Required for public access | Required for production | -| **Rich Media** | Extensive template support | Limited to media messages | +| **Authentication** | Page access token | Phone number ID + token | +| **App Review** | Required for public access | Required for production | +| **Rich Media** | Extensive template support | Limited to media messages | ## Example Projects Complete working examples with different configurations: -- **Basic Example**: `examples/api/messenger/server.py` +- **Basic Example**: \ +`examples/api/messenger/server.py` +`examples/api/messenger/server_adk.py` - **Custom Handler**: `examples/api/messenger/example_custom_handler.py` ## Additional Resources @@ -560,4 +617,4 @@ If you encounter issues: 2. Enable debug logging to see detailed request/response information 3. Review the Facebook Messenger Platform documentation 4. Check the [Agent Kernel GitHub Issues](https://github.com/yaalalabs/agent-kernel/issues) -5. Visit the [Facebook Developer Community](https://developers.facebook.com/community/) +5. Visit the [Facebook Developer Community](https://developers.facebook.com/community/) \ No newline at end of file diff --git a/examples/api/messenger/README.md b/examples/api/messenger/README.md index a0058330..d9908b75 100644 --- a/examples/api/messenger/README.md +++ b/examples/api/messenger/README.md @@ -17,6 +17,7 @@ This example demonstrates how to create a Facebook Messenger Platform integratio Follow the setup guide in [AgentMessengerRequestHandler](../../../ak-py/src/agentkernel/integrations/messenger/README.md) You'll need: + - Page Access Token - App Secret (optional but recommended) - Verify Token (you create this) @@ -74,11 +75,13 @@ The server will start on `http://localhost:8000` by default. For local testing, expose your server using a tunnel: ### Using ngrok: + ```bash ngrok http 8000 ``` ### Using pinggy: + ```bash ssh -p443 -R0:localhost:8000 a.pinggy.io ``` @@ -113,13 +116,23 @@ Copy the HTTPS URL and configure it in your Facebook Messenger webhook settings. ### Test Message Examples -``` +**Text Messages:** + +```bash Hello What can you help me with? Tell me about your services Can you answer technical questions? ``` +**Multimodal Messages (with attachments):** + +```bash +"What's in this photo?" [attach image] +"Can you analyze this document?" [attach PDF] +"Extract text from this image" [attach screenshot] +``` + ## Advanced Examples ### Custom Message Handler @@ -135,12 +148,12 @@ class CustomMessengerHandler(AgentMessengerRequestHandler): message = messaging_event.get("message", {}) text = message.get("text", "") sender_id = messaging_event["sender"]["id"] - + # Handle commands if text.startswith("/start"): await self._send_message(sender_id, "Welcome! How can I help you?") return - + # Default behavior await super()._handle_message(messaging_event) ``` @@ -206,6 +219,7 @@ OpenAIModule([support_agent, sales_agent]) ### Rate Limiting If you hit rate limits: + - Implement message queuing - Add retry logic with exponential backoff - Monitor rate limit headers in API responses @@ -231,17 +245,18 @@ For production deployment: Before going live with your app: 1. **Complete Platform Policy Review** + - Review Facebook Platform Policies - Review Messenger Platform Policies - Ensure compliance with data handling requirements - 2. **Submit for Review** + - Request `pages_messaging` permission - Provide test credentials and detailed instructions - Submit screencast demonstrating functionality - Wait for approval (typically 3-5 business days) - 3. **Switch to Live Mode** + - After approval, switch app from development to live mode - Update webhook URLs if needed - Monitor for issues @@ -249,11 +264,35 @@ Before going live with your app: ### Deployment Architectures See deployment documentation for: + - AWS Lambda + API Gateway (serverless) - AWS ECS/EKS (containerized) - Google Cloud Run - Azure Container Apps +## Multimodal Features + +The integration now supports file and image analysis: + +**Supported Attachments:** + +- Images (JPEG, PNG, GIF, WebP) +- Documents (PDF, Word, Excel, PowerPoint) +- Audio and video files + +**File Size Limits:** + +- Default: 2 MB per file (configurable) +- Base64 encoding adds ~33% overhead +- Check logs for size validation messages + +**Configuration:** + +```yaml +api: + max_file_size: 2097152 # 2 MB in bytes +``` + ## Features to Implement Extend this example with: @@ -278,7 +317,8 @@ Extend this example with: ## Support For issues and questions: + - Check the [troubleshooting section](#troubleshooting) - Review server logs for detailed error messages - Consult Facebook Developer Community -- Review Agent Kernel documentation +- Review Agent Kernel documentation \ No newline at end of file diff --git a/examples/api/messenger/build.sh b/examples/api/messenger/build.sh index 83e4238d..e7732cda 100755 --- a/examples/api/messenger/build.sh +++ b/examples/api/messenger/build.sh @@ -8,5 +8,5 @@ if [[ ${1-} != "local" ]]; then else # For local development of agentkernel, you can force reinstall from local dist uv sync --find-links ../../../ak-py/dist --all-extras - uv pip install --force-reinstall --find-links ../../../ak-py/dist agentkernel[api,openai,messenger] || true + uv pip install --force-reinstall --find-links ../../../ak-py/dist agentkernel[api,openai,adk,messenger] || true fi diff --git a/examples/api/messenger/server_adk.py b/examples/api/messenger/server_adk.py new file mode 100644 index 00000000..208c9867 --- /dev/null +++ b/examples/api/messenger/server_adk.py @@ -0,0 +1,25 @@ +from agentkernel.adk import GoogleADKModule +from agentkernel.api import RESTAPI +from agentkernel.messenger import AgentMessengerRequestHandler +from google.adk.agents import Agent +from google.adk.models.lite_llm import LiteLlm + +# Create your Google ADK agent with correct API +general_agent = Agent( + name="general", + model=LiteLlm(model="openai/gpt-4o-mini"), + description="Agent for general questions", + instruction=""" + You provide assistance with general queries. + Give short and clear answers suitable for Messenger messaging. + If you receive images or files, analyze them and provide relevant insights. + """, +) + +# Initialize module with agent +GoogleADKModule([general_agent]) + + +if __name__ == "__main__": + handler = AgentMessengerRequestHandler() + RESTAPI.run([handler]) \ No newline at end of file From 7227653f9e668eda9e596be6aa0ccda31228771f Mon Sep 17 00:00:00 2001 From: pulinduvidmal Date: Wed, 24 Dec 2025 02:48:12 +0530 Subject: [PATCH 8/9] feat: add multimodal file/image support for instagram with documentation --- .../integration/instagram/README.md | 36 +++++- .../integration/instagram/instagram_chat.py | 115 +++++++++++++++-- docs/docs/integrations/instagram.md | 116 +++++++++++++----- examples/api/instagram/README.md | 40 +++++- examples/api/instagram/build.sh | 2 +- examples/api/instagram/server_adk.py | 25 ++++ 6 files changed, 285 insertions(+), 49 deletions(-) create mode 100644 examples/api/instagram/server_adk.py diff --git a/ak-py/src/agentkernel/integration/instagram/README.md b/ak-py/src/agentkernel/integration/instagram/README.md index 5d12aebb..fe1783f9 100644 --- a/ak-py/src/agentkernel/integration/instagram/README.md +++ b/ak-py/src/agentkernel/integration/instagram/README.md @@ -33,29 +33,30 @@ This requires an **Instagram User Access Token** (starts with `IGAA...`). ### Configuration Steps 1. **Create a Meta App** + - Go to https://developers.facebook.com/apps - Create a new app and select "Business" type - Add **Instagram API** product (with Business Login) - 2. **Set Up Business Login for Instagram** + - In **Use Cases**, select "Instagram Business" - Click "API setup with Instagram login" - Add required permissions: - `instagram_business_basic` - `instagram_business_manage_messages` - 3. **Generate Access Token** + - Go to "Generate access tokens" section - Add your Instagram Professional account - Generate a token with the required permissions - The token will start with `IGAA...` - 4. **Get Your Credentials** + - **Access Token**: Generated from the step above (starts with `IGAA...`) - **App Secret**: App > App Settings > Basic (for webhook signature verification) - **Verify Token**: Create your own secure random string for webhook verification - 5. **Configure Webhook** + - In "Configure webhooks" section, enter: - **Callback URL**: Your public HTTPS endpoint + `/instagram/webhook` - **Verify Token**: Your chosen verify token @@ -114,6 +115,30 @@ RESTAPI.run([handler]) - `messaging_reads`: Read receipts (logged only) - `messaging_reactions`: Message reactions (logged only) +## Multi-Modal Support (Images & Files) + +The Instagram integration fully supports sending and analyzing images and files: + +**Supported File Types:** + +- **Images**: JPEG, PNG, GIF, WebP +- **Documents**: PDF, Word (.docx), Excel (.xlsx), PowerPoint (.pptx), Text files +- **Media**: Audio and video files + +**How It Works:** + +1. User sends message with attachment (image or file) +2. Handler downloads attachment from Instagram +3. File is validated against size limit (default 2 MB, configurable) +4. File is base64-encoded for transmission to AI agent +5. Agent analyzes and responds with insights + +**File Size Limits:** + +- Default maximum: **2 MB per file** +- Base64 encoding increases size by ~33%, so effective usable size is ~1.5 MB +- Configurable via `api.max_file_size` in config.yaml + ## Character Limits Instagram DM messages have a 1000 character limit. Long responses are automatically split into multiple messages. @@ -123,6 +148,7 @@ Instagram DM messages have a 1000 character limit. Long responses are automatica ### 401 Unauthorized / Cannot Parse Access Token This is the most common issue. It occurs when: + - Using a Facebook Page Access Token instead of Instagram User Access Token - Token has expired (tokens are valid for 60 days) - Token doesn't have required permissions @@ -150,4 +176,4 @@ This is the most common issue. It occurs when: ## Resources - [Instagram API with Instagram Login](https://developers.facebook.com/docs/instagram-platform/instagram-api-with-instagram-login) -- [Webhook Setup Guide](https://developers.facebook.com/docs/instagram-platform/webhooks) +- [Webhook Setup Guide](https://developers.facebook.com/docs/instagram-platform/webhooks) \ No newline at end of file diff --git a/ak-py/src/agentkernel/integration/instagram/instagram_chat.py b/ak-py/src/agentkernel/integration/instagram/instagram_chat.py index f7f9ccd0..2974a08c 100644 --- a/ak-py/src/agentkernel/integration/instagram/instagram_chat.py +++ b/ak-py/src/agentkernel/integration/instagram/instagram_chat.py @@ -1,6 +1,8 @@ +import base64 import hashlib import hmac import logging +import mimetypes import traceback import httpx @@ -8,6 +10,7 @@ from ...api import RESTRequestHandler from ...core import AgentService, Config +from ...core.model import AgentRequestFile, AgentRequestImage, AgentRequestText class AgentInstagramRequestHandler(RESTRequestHandler): @@ -35,6 +38,7 @@ def __init__(self): self._app_secret = Config.get().instagram.app_secret self._instagram_account_id = Config.get().instagram.instagram_account_id self._api_version = Config.get().instagram.api_version or "v21.0" + self._max_file_size = Config.get().api.max_file_size # Use graph.instagram.com for Business Login for Instagram (without Facebook) self._base_url = f"https://graph.instagram.com/{self._api_version}" if not all([self._access_token, self._verify_token]): @@ -151,7 +155,8 @@ async def _handle_message(self, messaging_event: dict): sender_id = messaging_event.get("sender", {}).get("id") message = messaging_event.get("message", {}) message_id = message.get("mid") - message_text = message.get("text") + message_text = message.get("text", "").strip() + attachments = message.get("attachments", []) if not sender_id or not message_id: self._log.warning("Message missing required fields (sender/mid)") @@ -162,13 +167,13 @@ async def _handle_message(self, messaging_event: dict): self._log.debug(f"Skipping echo message {message_id}") return - # Skip messages with attachments that don't have text - if not message_text: - self._log.warning("Message has no text content") + # Skip if no text and no attachments + if not message_text and not attachments: + self._log.warning("Message has no text content or attachments") return - self._log.debug(f"Processing message {message_id} from {sender_id}: {message_text}") - await self._process_agent_message(sender_id, message_text) + self._log.debug(f"Processing message {message_id} from {sender_id}: text='{message_text}', attachments={len(attachments)}") + await self._process_agent_message(sender_id, message_text, attachments) async def _handle_postback(self, messaging_event: dict): """ @@ -195,12 +200,13 @@ async def _handle_postback(self, messaging_event: dict): self._log.debug(f"Processing postback from {sender_id}: {message_text}") await self._process_agent_message(sender_id, message_text) - async def _process_agent_message(self, sender_id: str, message_text: str): + async def _process_agent_message(self, sender_id: str, message_text: str, attachments: list = None): """ Process a message through the agent and send the response. :param sender_id: Instagram-scoped user ID :param message_text: The message text to process + :param attachments: Optional list of attachments """ service = AgentService() session_id = sender_id # Use sender_id as session_id to maintain conversation context @@ -220,13 +226,35 @@ async def _process_agent_message(self, sender_id: str, message_text: str): await self._send_typing_indicator(sender_id, False) return + # Build requests list with text and attachments + requests = [] + + # Add text if present + if message_text: + requests.append(AgentRequestText(text=message_text)) + + # Process attachments (images and files) + if attachments: + for attachment in attachments: + await self._process_attachment(attachment, requests) + # Run the agent - result = await service.run(message_text) - - if hasattr(result, "raw"): - response_text = str(result.raw) + if requests: + # Use run_multi for multimodal requests + if len(requests) > 1 or any(isinstance(r, (AgentRequestFile, AgentRequestImage)) for r in requests): + result = await service.run_multi(requests=requests) + else: + result = await service.run(message_text) if message_text else None else: - response_text = str(result) + result = None + + if result: + if hasattr(result, "raw"): + response_text = str(result.raw) + else: + response_text = str(result) + else: + response_text = "Sorry, I could not process your message." self._log.debug(f"Agent response: {response_text}") @@ -239,6 +267,67 @@ async def _process_agent_message(self, sender_id: str, message_text: str): await self._send_typing_indicator(sender_id, False) await self._send_message(sender_id, "Sorry, there was an error processing your request.") + async def _process_attachment(self, attachment: dict, requests: list): + """ + Process an Instagram attachment (image or file). + + :param attachment: Attachment object from message + :param requests: List to append the processed request to + """ + attachment_type = attachment.get("type") + payload = attachment.get("payload", {}) + url = payload.get("url") + + if not url: + self._log.warning(f"Attachment has no URL: {attachment}") + return + + try: + # Download the attachment + async with httpx.AsyncClient() as client: + response = await client.get(url, timeout=10.0) + response.raise_for_status() + file_data = response.content + + # Check file size + if len(file_data) > self._max_file_size: + self._log.warning(f"Attachment size ({len(file_data) / (1024 * 1024):.2f} MB) exceeds maximum allowed size of {self._max_file_size / (1024 * 1024):.2f} MB") + return + + # Encode to base64 + file_data_base64 = base64.b64encode(file_data).decode("utf-8") + + # Get MIME type + mime_type = response.headers.get("content-type", "application/octet-stream") + + # Extract filename from URL if available + filename = url.split("/")[-1].split("?")[0] or f"attachment_{len(requests)}" + + self._log.debug(f"Downloaded {attachment_type} attachment: {filename} (size: {len(file_data)} bytes, type: {mime_type})") + + # Classify based on attachment type and MIME type + if attachment_type == "image" or (mime_type and mime_type.startswith("image/")): + self._log.debug(f"Adding image: {filename}") + requests.append( + AgentRequestImage( + image_data=file_data_base64, + name=filename, + mime_type=mime_type, + ) + ) + else: + self._log.debug(f"Adding file: {filename}") + requests.append( + AgentRequestFile( + file_data=file_data_base64, + name=filename, + mime_type=mime_type, + ) + ) + + except Exception as e: + self._log.error(f"Error processing attachment: {e}\n{traceback.format_exc()}") + async def _send_message(self, recipient_id: str, text: str): """ Send an Instagram message using the Instagram Messaging API. @@ -321,4 +410,4 @@ async def _mark_seen(self, recipient_id: str): response.raise_for_status() self._log.debug(f"Message marked as seen: {recipient_id}") except Exception as e: - self._log.warning(f"Failed to mark message as seen: {e}") + self._log.warning(f"Failed to mark message as seen: {e}") \ No newline at end of file diff --git a/docs/docs/integrations/instagram.md b/docs/docs/integrations/instagram.md index e559c5ca..e86869ae 100644 --- a/docs/docs/integrations/instagram.md +++ b/docs/docs/integrations/instagram.md @@ -74,9 +74,11 @@ Before you begin, you'll need: **Create a Verify Token:** This is a random string you create yourself for webhook verification. Choose something secure like: + ```bash openssl rand -hex 32 ``` + Save this as `AK_INSTAGRAM__VERIFY_TOKEN` ### 3. Configure Environment Variables @@ -112,6 +114,7 @@ ssh -p443 -R0:localhost:8000 a.pinggy.io **Subscribe to webhook events:** In the webhook configuration, subscribe to: + - `messages` - To receive user messages **Enable the webhook:** @@ -200,25 +203,25 @@ class CustomInstagramHandler(AgentInstagramRequestHandler): message = messaging_event.get("message", {}) message_text = message.get("text", "").strip() sender_id = messaging_event.get("sender", {}).get("id") - + # Handle special commands if message_text.startswith("/"): await self._handle_command(message_text, sender_id) return - + # Preprocess messages before sending to agent processed_text = self._preprocess_message(message_text) - + # Continue with normal processing await super()._handle_message(messaging_event) - + async def _handle_command(self, command: str, sender_id: str): """Handle custom commands""" await self._send_typing_indicator(sender_id, True) - + if command == "/help": help_text = """🤖 Available Commands: - + /help - Show this help message /start - Start a new conversation @@ -234,9 +237,9 @@ Just send any message to chat with me!""" sender_id, f"Unknown command. Try /help for available commands." ) - + await self._send_typing_indicator(sender_id, False) - + def _preprocess_message(self, text: str) -> str: """Clean up or enhance user messages""" # Expand common abbreviations @@ -288,15 +291,58 @@ OpenAIModule([sales_agent, support_agent, general_agent]) ## Supported Message Types ### Text Messages + Standard text messages are fully supported with automatic context management. ### Postbacks + Handle button clicks and quick reply selections. Postbacks are processed as text using the button title or payload. +### Multi-Modal: Images and Files + +The integration provides full support for images and file attachments with AI analysis: + +**Supported Formats:** + +- **Images**: JPEG, PNG, GIF, WebP (detected automatically) +- **Documents**: PDF, Word (.docx), Excel (.xlsx), PowerPoint (.pptx), Text files +- **Media**: Audio and video files + +**How It Works:** + +1. **Detection**: When user sends message with attachment, handler identifies type +2. **Download**: File is downloaded from Instagram's servers +3. **Validation**: File size checked against limit (default 2 MB) +4. **Encoding**: File converted to base64 for transmission +5. **Processing**: Agent receives file and can analyze it +6. **Response**: Agent provides insights or answers about the content + +**File Size Limits:** + +- **Default**: 2 MB per file (2,097,152 bytes) +- **Base64 Overhead**: ~33% increase in size +- **Effective Size**: ~1.5 MB usable after base64 encoding +- **Configurable**: Set `api.max_file_size` in config.yaml + +**Example User Interactions:** + +```bash +User: "What's in this photo?" [sends image] +Agent: [analyzes image] "This appears to be..." + +User: "Extract the text from this PDF" [sends PDF] +Agent: [processes PDF] "The document contains..." + +User: "Can you read this note?" [sends image] +Agent: [uses OCR] "The note says..." +``` + ### Reactions + Message reactions are logged with extensibility for custom handling. ### Read Receipts + Read receipts are automatically logged for debugging purposes. ## Character Limits @@ -320,11 +366,13 @@ This requires an **Instagram User Access Token** (starts with `IGAA...`), not a **Problem:** API calls fail with authentication errors **This is the most common issue.** It occurs when: + - Using a Facebook Page Access Token instead of Instagram User Access Token - Token has expired (tokens are valid for 60 days) - Token doesn't have required permissions **Solutions:** + - Generate a new token from Business Login for Instagram - The token should start with `IGAA...` - Do NOT use tokens starting with `EAAG...` (Facebook Page tokens) @@ -335,6 +383,7 @@ This requires an **Instagram User Access Token** (starts with `IGAA...`), not a **Problem:** "Webhook verification failed" error when configuring callback URL **Solutions:** + - Ensure your verify token in the environment variable exactly matches what you enter in Meta portal - Verify your server is running and accessible via HTTPS - Check that your webhook URL path is `/instagram/webhook` @@ -346,6 +395,7 @@ This requires an **Instagram User Access Token** (starts with `IGAA...`), not a **Problem:** Webhook is verified but messages aren't reaching your agent **Solutions:** + - Check that webhook subscriptions include `messages` and are active - Verify Instagram account is a Professional account (Business or Creator) - Ensure app has `instagram_business_manage_messages` permission @@ -357,6 +407,7 @@ This requires an **Instagram User Access Token** (starts with `IGAA...`), not a **Problem:** API returns permission denied errors **Solutions:** + - Ensure all required permissions are granted: - `instagram_business_basic` - `instagram_business_manage_messages` @@ -368,6 +419,7 @@ This requires an **Instagram User Access Token** (starts with `IGAA...`), not a **Problem:** Agent processes messages but responses don't appear in Instagram **Solutions:** + - Confirm you're using an **Instagram User Access Token** (starts with `IGAA...`) - Verify the token has required permissions - Check that you're responding within a reasonable time @@ -389,6 +441,7 @@ logging.basicConfig( ``` This will show: + - Incoming webhook requests - Signature verification steps - Agent processing details @@ -400,6 +453,7 @@ This will show: Instagram enforces rate limits to ensure platform stability: ### Message Limits + - Rate limits vary by account type and quality score - **Best practice**: Implement queuing for high-volume scenarios @@ -413,7 +467,7 @@ class RateLimitedInstagramHandler(AgentInstagramRequestHandler): def __init__(self): super().__init__() self.message_queue = Queue() - + async def _send_message(self, recipient_id: str, text: str): # Add delay to respect rate limits await asyncio.sleep(0.1) # 10 messages per second @@ -427,41 +481,42 @@ class RateLimitedInstagramHandler(AgentInstagramRequestHandler): Before deploying to production: 1. ✅ **Complete Meta App Review** + - Request `instagram_business_manage_messages` permission - Submit app for review with clear use case documentation - Provide test credentials and instructions - Typical approval time: 3-5 business days - 2. ✅ **Security Measures** + - Use environment variables for all secrets - Enable app secret verification (`AK_INSTAGRAM__APP_SECRET`) - Implement HTTPS with valid SSL certificate - Use secure secret management (AWS Secrets Manager, HashiCorp Vault) - 3. ✅ **Infrastructure** + - Deploy behind a reverse proxy (nginx, Apache) - Set up load balancing for high traffic - Implement health checks and monitoring - Configure auto-scaling if using cloud services - 4. ✅ **Monitoring & Logging** + - Set up centralized logging (CloudWatch, Datadog, ELK) - Configure alerts for errors and anomalies - Track conversation metrics and performance - Monitor API rate limits - 5. ✅ **Error Handling** + - Implement retry logic with exponential backoff - Handle network failures gracefully - Provide fallback responses for errors - Log errors for debugging - 6. ✅ **Token Management** + - Instagram User Access Tokens expire in 60 days - Implement token refresh before expiration - Set up alerts for token expiration - 7. ✅ **Compliance** + - Review Meta Platform Policies - Ensure GDPR/CCPA compliance for user data - Implement data retention policies @@ -472,38 +527,44 @@ Before deploying to production: For production deployments, consider: **Serverless (AWS Lambda):** + - Cost-effective for low-to-medium traffic - Auto-scaling built-in - See `examples/aws-serverless` for reference **Containerized (Docker/Kubernetes):** + - Better for high traffic and complex workflows - Full control over environment - See `examples/aws-containerized` for reference **Traditional Server:** + - Simple deployment for small-scale applications - Use systemd or supervisor for process management - Configure nginx as reverse proxy ## Instagram vs Messenger vs WhatsApp Comparison -| Feature | Instagram | Facebook Messenger | WhatsApp | -|---------|-----------|-------------------|----------| -| **Message Limit** | 1,000 characters | 2,000 characters | 4,096 characters | -| **User Identifier** | Instagram-Scoped ID | Page-Scoped ID (PSID) | Phone number | -| **Visual Feedback** | Typing indicators | Typing indicators, seen receipts | Read receipts | -| **Authentication** | Instagram User Token (IGAA) | Page access token | Phone number ID + token | -| **Facebook Page Required** | No (with Business Login) | Yes | No | -| **Account Type** | Professional (Business/Creator) | Facebook Page | WhatsApp Business | -| **App Review** | Required for production | Required for public access | Required for production | +| Feature | Instagram | Facebook Messenger | WhatsApp | +| -------------------------------- | ------------------------------- | -------------------------------- | ----------------------- | +| **Message Limit** | 1,000 characters | 2,000 characters | 4,096 characters | +| **User Identifier** | Instagram-Scoped ID | Page-Scoped ID (PSID) | Phone number | +| **Visual Feedback** | Typing indicators | Typing indicators, seen receipts | Read receipts | +| **Authentication** | Instagram User Token (IGAA) | Page access token | Phone number ID + token | +| **Facebook Page Required** | No (with Business Login) | Yes | No | +| **Account Type** | Professional (Business/Creator) | Facebook Page | WhatsApp Business | +| **App Review** | Required for production | Required for public access | Required for production | ## Example Projects Complete working examples with different configurations: -- **Basic Example**: `examples/api/instagram/server.py` -- **Integration Guide**: `ak-py/src/agentkernel/integration/instagram/README.md` +- **Basic Example**: \ +`examples/api/instagram/server.py`\ +`examples/api/instagram/server_adk.py` +- **Integration Guide**: \ +`ak-py/src/agentkernel/integration/instagram/README.md` ## Additional Resources @@ -522,5 +583,4 @@ If you encounter issues: 2. Enable debug logging to see detailed request/response information 3. Review the Instagram Platform documentation 4. Check the [Agent Kernel GitHub Issues](https://github.com/yaalalabs/agent-kernel/issues) -5. Visit the [Meta Developer Community](https://developers.facebook.com/community/) - +5. Visit the [Meta Developer Community](https://developers.facebook.com/community/) \ No newline at end of file diff --git a/examples/api/instagram/README.md b/examples/api/instagram/README.md index 1a49ec24..cc335868 100644 --- a/examples/api/instagram/README.md +++ b/examples/api/instagram/README.md @@ -22,6 +22,7 @@ This integration uses the Instagram Graph API (`graph.instagram.com`) directly, Follow the setup guide in [AgentInstagramRequestHandler](../../../ak-py/src/agentkernel/integration/instagram/README.md) You'll need: + - Access Token (from Business Login - starts with `IGAA...`) - App Secret (optional but recommended for webhook signature verification) - Verify Token (you create this - any secure random string) @@ -82,11 +83,13 @@ The server will start on `http://localhost:8000` by default. For local testing, expose your server using a tunnel: ### Using ngrok: + ```bash ngrok http 8000 ``` ### Using pinggy: + ```bash ssh -p443 -R0:localhost:8000 a.pinggy.io ``` @@ -114,13 +117,46 @@ Copy the HTTPS URL and configure it in your Instagram webhook settings. ### Test Message Examples -``` +**Text Messages:** + +```bash Hello What services do you offer? Can you help me with a question? Tell me more about yourself ``` +**Multimodal Messages (with attachments):** + +```bash +"What's in this photo?" [attach image] +"Can you analyze this document?" [attach PDF] +"Extract text from this image" [attach screenshot] +``` + +## Multimodal Features + +The integration now supports file and image analysis: + +**Supported Attachments:** + +- Images (JPEG, PNG, GIF, WebP) +- Documents (PDF, Word, Excel, PowerPoint) +- Audio and video files + +**File Size Limits:** + +- Default: 2 MB per file (configurable) +- Base64 encoding adds ~33% overhead +- Check logs for size validation messages + +**Configuration:** + +```yaml +api: + max_file_size: 2097152 # 2 MB in bytes +``` + ## API Endpoint This integration uses the Instagram Graph API directly: @@ -183,4 +219,4 @@ See deployment documentation for AWS and other platforms. - [Instagram API with Instagram Login](https://developers.facebook.com/docs/instagram-platform/instagram-api-with-instagram-login) - [Webhook Setup Guide](https://developers.facebook.com/docs/instagram-platform/webhooks) - [Agent Kernel Documentation](../../../docs/) -- [Instagram Integration Guide](../../../ak-py/src/agentkernel/integration/instagram/README.md) +- [Instagram Integration Guide](../../../ak-py/src/agentkernel/integration/instagram/README.md) \ No newline at end of file diff --git a/examples/api/instagram/build.sh b/examples/api/instagram/build.sh index bf2ab667..0bf8a00f 100755 --- a/examples/api/instagram/build.sh +++ b/examples/api/instagram/build.sh @@ -8,6 +8,6 @@ if [[ ${1-} != "local" ]]; then else # For local development of agentkernel, you can force reinstall from local dist uv sync --find-links ../../../ak-py/dist --all-extras - uv pip install --force-reinstall --find-links ../../../ak-py/dist agentkernel[api,openai,instagram] || true + uv pip install --force-reinstall --find-links ../../../ak-py/dist agentkernel[api,openai,adk,instagram] || true fi diff --git a/examples/api/instagram/server_adk.py b/examples/api/instagram/server_adk.py new file mode 100644 index 00000000..6c42013b --- /dev/null +++ b/examples/api/instagram/server_adk.py @@ -0,0 +1,25 @@ +from agentkernel.adk import GoogleADKModule +from agentkernel.api import RESTAPI +from agentkernel.instagram import AgentInstagramRequestHandler +from google.adk.agents import Agent +from google.adk.models.lite_llm import LiteLlm + +# Create your Google ADK agent with correct API +general_agent = Agent( + name="general", + model=LiteLlm(model="openai/gpt-4o-mini"), + description="Agent for general questions", + instruction=""" + You provide assistance with general queries. + Give short and clear answers suitable for Instagram messaging. + If you receive images or files, analyze them and provide relevant insights. + """, +) + +# Initialize module with agent +GoogleADKModule([general_agent]) + + +if __name__ == "__main__": + handler = AgentInstagramRequestHandler() + RESTAPI.run([handler]) \ No newline at end of file From c0b334b491f279dd048949a55dd3e4022b325d50 Mon Sep 17 00:00:00 2001 From: pulinduvidmal Date: Wed, 24 Dec 2025 09:38:58 +0530 Subject: [PATCH 9/9] feat: add file support for Instagram integration --- .../integration/instagram/instagram_chat.py | 22 ++++++++++++------- examples/api/instagram/server_adk.py | 2 +- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/ak-py/src/agentkernel/integration/instagram/instagram_chat.py b/ak-py/src/agentkernel/integration/instagram/instagram_chat.py index 2974a08c..8133bfb3 100644 --- a/ak-py/src/agentkernel/integration/instagram/instagram_chat.py +++ b/ak-py/src/agentkernel/integration/instagram/instagram_chat.py @@ -172,7 +172,9 @@ async def _handle_message(self, messaging_event: dict): self._log.warning("Message has no text content or attachments") return - self._log.debug(f"Processing message {message_id} from {sender_id}: text='{message_text}', attachments={len(attachments)}") + self._log.debug( + f"Processing message {message_id} from {sender_id}: text='{message_text}', attachments={len(attachments)}" + ) await self._process_agent_message(sender_id, message_text, attachments) async def _handle_postback(self, messaging_event: dict): @@ -228,16 +230,16 @@ async def _process_agent_message(self, sender_id: str, message_text: str, attach # Build requests list with text and attachments requests = [] - + # Add text if present if message_text: requests.append(AgentRequestText(text=message_text)) - + # Process attachments (images and files) if attachments: for attachment in attachments: await self._process_attachment(attachment, requests) - + # Run the agent if requests: # Use run_multi for multimodal requests @@ -247,7 +249,7 @@ async def _process_agent_message(self, sender_id: str, message_text: str, attach result = await service.run(message_text) if message_text else None else: result = None - + if result: if hasattr(result, "raw"): response_text = str(result.raw) @@ -291,7 +293,9 @@ async def _process_attachment(self, attachment: dict, requests: list): # Check file size if len(file_data) > self._max_file_size: - self._log.warning(f"Attachment size ({len(file_data) / (1024 * 1024):.2f} MB) exceeds maximum allowed size of {self._max_file_size / (1024 * 1024):.2f} MB") + self._log.warning( + f"Attachment size ({len(file_data) / (1024 * 1024):.2f} MB) exceeds maximum allowed size of {self._max_file_size / (1024 * 1024):.2f} MB" + ) return # Encode to base64 @@ -303,7 +307,9 @@ async def _process_attachment(self, attachment: dict, requests: list): # Extract filename from URL if available filename = url.split("/")[-1].split("?")[0] or f"attachment_{len(requests)}" - self._log.debug(f"Downloaded {attachment_type} attachment: {filename} (size: {len(file_data)} bytes, type: {mime_type})") + self._log.debug( + f"Downloaded {attachment_type} attachment: {filename} (size: {len(file_data)} bytes, type: {mime_type})" + ) # Classify based on attachment type and MIME type if attachment_type == "image" or (mime_type and mime_type.startswith("image/")): @@ -410,4 +416,4 @@ async def _mark_seen(self, recipient_id: str): response.raise_for_status() self._log.debug(f"Message marked as seen: {recipient_id}") except Exception as e: - self._log.warning(f"Failed to mark message as seen: {e}") \ No newline at end of file + self._log.warning(f"Failed to mark message as seen: {e}") diff --git a/examples/api/instagram/server_adk.py b/examples/api/instagram/server_adk.py index 6c42013b..d171e429 100644 --- a/examples/api/instagram/server_adk.py +++ b/examples/api/instagram/server_adk.py @@ -22,4 +22,4 @@ if __name__ == "__main__": handler = AgentInstagramRequestHandler() - RESTAPI.run([handler]) \ No newline at end of file + RESTAPI.run([handler])