diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/_models/_completions.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/_models/_completions.py new file mode 100644 index 000000000000..0fa5bca388fc --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/_models/_completions.py @@ -0,0 +1,73 @@ +from typing import Optional, List +from pydantic import BaseModel + +from azure.ai.evaluation._common._models._messages import ChatMessage + +class InputTokenDetails(BaseModel): + """ + Represents details about input tokens. + + Attributes: + cached_tokens (Optional[int]): The number of cached tokens. + """ + cached_tokens: Optional[int] = None + +class OutputTokenDetails(BaseModel): + """ + Represents details about output tokens. + + Attributes: + reasoning_tokens (Optional[int]): The number of reasoning tokens. + """ + reasoning_tokens: Optional[int] = None + +class CompletionUsage(BaseModel): + """ + Represents the usage details of a completion. + + Attributes: + output_tokens (int): The number of output tokens. + input_tokens (int): The number of input tokens. + total_tokens (int): The total number of tokens. + input_token_details (Optional[InputTokenDetails]): Details about input tokens. + output_token_details (Optional[OutputTokenDetails]): Details about output tokens. + """ + output_tokens: int + input_tokens: int + total_tokens: int + input_token_details: Optional[InputTokenDetails] = None + output_token_details: Optional[OutputTokenDetails] = None + +class IncompleteDetails(BaseModel): + """ + Represents details about incomplete completions. + + Attributes: + reason (str): The reason for the incomplete status. + """ + reason: str + +class Completion(BaseModel): + """ + Represents a completion. + + Attributes: + agent_id (str): The ID of the agent. + completion_id (str): The ID of the completion. + created_at (int): The timestamp when the completion was created. + completed_at (int): The timestamp when the completion was completed. + status (str): The status of the completion. + output (List[ChatMessage]): The output messages of the completion. + thread_id (str): The ID of the thread the completion belongs to. + usage (CompletionUsage): The usage details of the completion. + incomplete_details (Optional[IncompleteDetails]): Details about incomplete completions. + """ + agent_id: str + completion_id: str + created_at: int + completed_at: int + status: str + output: List[ChatMessage] = [] + thread_id: str + usage: CompletionUsage + incomplete_details: Optional[IncompleteDetails] = None diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/_models/_messages.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/_models/_messages.py new file mode 100644 index 000000000000..904c3bca2af8 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/_models/_messages.py @@ -0,0 +1,199 @@ +import datetime +import json +from enum import Enum + +from pydantic import BaseModel + +from typing import List, Optional, Union, Dict, Any + + +class AuthorRole(Enum): + USER = "user" + AGENT = "agent" + SYSTEM = "system" + TOOL = "tool" + DEVELOPER = "developer" + +class AIContent(BaseModel): + """ + Base class for content of a message. + + Attributes: + type (str): The type of the content. + """ + type: str + +class Annotation(BaseModel): + """ + Represents an annotation that links a portion of message content to a specific tool result or an external reference. + + Attributes: + tool_call_id (Optional[str]): The ID of the tool call that produced the result being referenced by this annotation. + Optional if the annotation references an external URL instead. + json_path (Optional[str]): A JSONPath query into the result object of the tool call. + This path should locate the specific value within the tool result relevant to this annotation. + Optional if referencing an external URL. + url (Optional[str]): A URL to an external resource that provides additional context or references the annotated content. + Optional if referencing a tool call result. + start (Optional[int]): The start index of the text span in the message content that this annotation applies to. + end (Optional[int]): The end index of the text span in the message content that this annotation applies to. + """ + tool_call_id: Optional[str] = None + json_path: Optional[str] = None + url: Optional[str] = None + start: Optional[int] = None + end: Optional[int] = None + +class ContentFilterContent(AIContent): + """ + Represents content filtered by a content filter. + + Attributes: + type (str): The type of the content, which is always 'content_filter'. + content_filter (str): The content filter applied. + detected (bool): Indicates whether the content was detected by the filter. + """ + type: str = "content_filter" + content_filter: str + detected: bool + +class RefusalContent(AIContent): + """ + Represents refusal content. + + Attributes: + type (str): The type of the content, which is always 'refusal'. + refusal (str): The refusal message. + """ + type: str = "refusal" + refusal: str + +class TextContent(AIContent): + """ + Represents text content. + + Attributes: + type (str): The type of the content, which is always 'text'. + text (str): The text content. + annotations (List[Annotation]): A list of annotations associated with the text content. + """ + type: str = "text" + text: str + annotations: List[Annotation] = [] + +class ToolCallContent(AIContent): + """ + Represents tool call content. + + Attributes: + type (str): The type of the content, which is always 'tool_call'. + name (str): The name of the tool. + tool_call_id (str): The ID of the tool call. + arguments (Optional[Dict[str, Any]]): The arguments for the tool call. + """ + type: str = "tool_call" + name: str + tool_call_id: str + arguments: Optional[Dict[str, Any]] = None + +class ToolResultContent(AIContent): + """ + Represents tool result content. + + Attributes: + type (str): The type of the content, which is always 'tool_result'. + tool_call_id (str): The ID of the tool call. + results (Optional[Any]): The results of the tool call. + """ + type: str = "tool_result" + tool_call_id: str + results: Optional[Any] = None + +class FileContent(AIContent): + """ + Represents file content. + + Attributes: + type (str): The type of the content, which is always 'file'. + file_name (Optional[str]): The name of the file. + mime_type (Optional[str]): The MIME type of the file. + uri (Optional[str]): The URI of the file. + data_uri (Optional[str]): The data URI of the file. + data (Optional[bytes]): The binary data of the file. + """ + type: str = "file" + file_name: Optional[str] = None + mime_type: Optional[str] = None + uri: Optional[str] = None + data_uri: Optional[str] = None + data: Optional[bytes] = None + +class AudioContent(FileContent): + """ + Represents audio content. + + Attributes: + type (str): The type of the content, which is always 'audio'. + duration (Optional[short]): The duration of the audio content. + """ + type: str = "audio" + duration: Optional[int] = None + +class ImageContent(FileContent): + """ + Represents image content. + + Attributes: + type (str): The type of the content, which is always 'image'. + width (Optional[int]): The width of the image. + height (Optional[int]): The height of the image. + """ + type: str = "image" + width: Optional[int] = None + height: Optional[int] = None + +class VideoContent(FileContent): + """ + Represents video content. + + Attributes: + type (str): The type of the content, which is always 'video'. + duration (Optional[int]): The duration of the video. + width (Optional[int]): The width of the video. + height (Optional[int]): The height of the video. + """ + type: str = "video" + duration: Optional[int] = None + width: Optional[int] = None + height: Optional[int] = None + +class ChatMessage(BaseModel): + """ + Represents a chat message. + + Attributes: + user_id (Optional[str]): The ID of the user who sent the message. + agent_id (Optional[str]): The ID of the agent who sent the message. + message_id (str): The ID of the message. + completion_id (Optional[str]): The ID of the completion associated with the message. + thread_id (str): The ID of the thread the message belongs to. + role (AuthorRole): The role of the message sender. + content (List[AIContent]): The content of the message. + author_name (Optional[str]): The name of the author of the message. + created_at (Optional[int]): The timestamp when the message was created. + completed_at (Optional[int]): The timestamp when the message was completed. + """ + user_id: Optional[str] = None + agent_id: Optional[str] = None + message_id: str + completion_id: Optional[str] = None + thread_id: str + role: AuthorRole + content: List[AIContent] + author_name: Optional[str] = None + created_at: Optional[int] = None + completed_at: Optional[int] = None + + + + diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/_models/_threads.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/_models/_threads.py new file mode 100644 index 000000000000..ea1cb26409fc --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/_models/_threads.py @@ -0,0 +1,16 @@ +from typing import List +from pydantic import BaseModel + +from azure.ai.evaluation._common._models._messages import ChatMessage + + +class ActivityThread(BaseModel): + """ + Represents an activity thread. + + Attributes: + thread_id (str): The ID of the thread. + messages (List[ChatMessage]): The list of chat messages in the thread. + """ + thread_id: str + messages: List[ChatMessage] = [] \ No newline at end of file diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/_models/_tools.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/_models/_tools.py new file mode 100644 index 000000000000..a0aa988ca9b1 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/_models/_tools.py @@ -0,0 +1,109 @@ +from typing import Optional, List +from pydantic import BaseModel +import json + +class ToolOverrides(BaseModel): + """ + Optional overrides for tool metadata. + + Attributes: + name (Optional[str]): The name of the tool. + description (Optional[str]): The description of the tool. + parameters (Optional[dict]): The parameters for the tool. + """ + name: Optional[str] = None + description: Optional[str] = None + parameters: Optional[dict] = None + +class AgentToolDefinition(BaseModel): + """ + Represents an agent tool definition. + + Attributes: + type (str): The type of the tool. + override (Optional[ToolOverrides]): Optional overrides for tool metadata. + """ + type: str + override: Optional[ToolOverrides] = None + +class FunctionToolDefinition(AgentToolDefinition): + """ + Represents a function tool definition. + + Attributes: + type (str): The type of the tool, which is always 'function'. + name (str): The name of the function. + description (Optional[str]): The description of the function. + parameters (Optional[dict]): JSON Schema describing the parameters, stored as raw JSON. + strict (Optional[bool]): Indicates if the function is strict. + """ + type: str = "function" + name: str + description: Optional[str] = None + parameters: Optional[dict] = None + strict: Optional[bool] = None + +class BingGroundingToolDefinition(AgentToolDefinition): + """ + Represents a Bing Grounding tool definition. + + Attributes: + type (str): The type of the tool, which is always 'Microsoft.BingGrounding'. + connection_name (str): The name of the connection. + """ + type: str = "Microsoft.BingGrounding" + connection_name: str + +class CodeInterpreterToolDefinition(AgentToolDefinition): + """ + Represents a Code Interpreter tool definition. + + Attributes: + type (str): The type of the tool, which is always 'OpenAI.CodeInterpreter'. + file_ids (Optional[List[str]]): A list of file IDs. + """ + type: str = "OpenAI.CodeInterpreter" + file_ids: Optional[List[str]] = None + +class RankingOptions(BaseModel): + """ + Represents ranking options. + + Attributes: + score_threshold (float): The score threshold for ranking. + ranker (Optional[str]): The ranker used for ranking. + """ + score_threshold: float + ranker: Optional[str] = None + +class FileSearchToolDefinition(AgentToolDefinition): + """ + Represents a File Search tool definition. + + Attributes: + type (str): The type of the tool, which is always 'OpenAI.FileSearch'. + max_num_results (Optional[int]): The maximum number of results. + ranking_options (Optional[RankingOptions]): The ranking options. + vector_store_id (Optional[str]): The ID of the vector store. + """ + type: str = "OpenAI.FileSearch" + max_num_results: Optional[int] = None + ranking_options: Optional[RankingOptions] = None + vector_store_id: Optional[str] = None + +# class OpenApiToolDefinition(AgentToolDefinition): +# """ +# Represents an OpenAPI tool definition. +# +# Attributes: +# type (str): The type of the tool, which is always 'OpenApi'. +# name (str): The name of the tool. +# description (Optional[str]): The description of the tool. +# spec (BinaryData): The OpenAPI specification. +# auth (OpenApiAuthDetails): The authentication details. +# """ +# type: str = "OpenApi" +# name: str +# description: Optional[str] = None +# spec: BinaryData +# auth: OpenApiAuthDetails \ No newline at end of file