[Feat] Add Tetrate Agent Router Service (TARS) provider

litellm/__init__.py

@@ -528,6 +528,7 @@ def identify(event_details):
 ovhcloud_models: Set = set()
 ovhcloud_embedding_models: Set = set()
 lemonade_models: Set = set()
+tars_models: Set = set()
 
 
 def is_bedrock_pricing_only_model(key: str) -> bool:
@@ -752,6 +753,8 @@ def add_known_models():
             ovhcloud_embedding_models.add(key)
         elif value.get("litellm_provider") == "lemonade":
             lemonade_models.add(key)
+        elif value.get("litellm_provider") == "tars":
+            tars_models.add(key)
 
 
 add_known_models()
@@ -854,6 +857,7 @@ def add_known_models():
     | ovhcloud_models
     | lemonade_models
     | set(clarifai_models)
+    | tars_models
 )
 
 model_list_set = set(model_list)
@@ -940,6 +944,7 @@ def add_known_models():
     "ovhcloud": ovhcloud_models | ovhcloud_embedding_models,
     "lemonade": lemonade_models,
     "clarifai": clarifai_models,
+    "tars": tars_models,
 }
 
 # mapping for those models which have larger equivalents
@@ -1284,6 +1289,7 @@ def add_known_models():
 from .llms.watsonx.embed.transformation import IBMWatsonXEmbeddingConfig
 from .llms.github_copilot.chat.transformation import GithubCopilotConfig
 from .llms.nebius.chat.transformation import NebiusConfig
+from .llms.tars.chat.transformation import TarsConfig
 from .llms.wandb.chat.transformation import WandbConfig
 from .llms.dashscope.chat.transformation import DashScopeChatConfig
 from .llms.moonshot.chat.transformation import MoonshotChatConfig

litellm/constants.py

@@ -361,7 +361,8 @@
     "vercel_ai_gateway",
     "wandb",
     "ovhcloud",
-    "lemonade"
+    "lemonade",
+    "tars"
 ]
 
 LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS = [
@@ -547,6 +548,7 @@
     "wandb",
     "cometapi",
     "clarifai",
+    "tars",
 ]
 openai_text_completion_compatible_providers: List = (
     [  # providers that support `/v1/completions`

litellm/cost_calculator.py

@@ -375,6 +375,11 @@ def cost_per_token(  # noqa: PLR0915
             cost_per_token as dashscope_cost_per_token,
         )
         return dashscope_cost_per_token(model=model, usage=usage_block)
+    elif custom_llm_provider == "tars":
+        from litellm.llms.tars.cost_calculator import (
+            cost_per_token as tars_cost_per_token,
+        )
+        return tars_cost_per_token(model=model, usage=usage_block)
     else:
         model_info = _cached_get_model_info_helper(
             model=model, custom_llm_provider=custom_llm_provider

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -803,6 +803,14 @@ def _get_openai_compatible_provider_info(  # noqa: PLR0915
         ) = litellm.ClarifaiConfig()._get_openai_compatible_provider_info(
             api_base, api_key
         )
+    elif custom_llm_provider == "tars":
+        # TARS (Tetrate Agent Router Service) is OpenAI compatible
+        api_base = (
+            api_base
+            or get_secret_str("TARS_API_BASE")
+            or "https://api.router.tetrate.ai/v1"
+        )  # type: ignore
+        dynamic_api_key = api_key or get_secret_str("TARS_API_KEY")
 
     if api_base is not None and not isinstance(api_base, str):
         raise Exception("api base needs to be a string. api_base={}".format(api_base))

litellm/llms/tars/chat/transformation.py

@@ -0,0 +1,79 @@
+"""
+Support for OpenAI's `/v1/chat/completions` endpoint.
+
+TARS (Tetrate Agent Router Service) is OpenAI-compatible.
+
+Docs: https://router.tetrate.ai
+API: https://api.router.tetrate.ai/v1
+"""
+
+from typing import Optional, Union
+
+import httpx
+
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
+
+from ..common_utils import TarsException, TarsModelInfo
+
+
+class TarsConfig(OpenAIGPTConfig, TarsModelInfo):
+    """
+    Configuration for TARS (Tetrate Agent Router Service).
+
+    TARS is OpenAI-compatible and routes to multiple LLM providers.
+    Supports dynamic model fetching from the TARS API.
+    """
+
+    def get_error_class(
+        self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
+    ) -> BaseLLMException:
+        return TarsException(
+            message=error_message,
+            status_code=status_code,
+            headers=headers,
+        )
+
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        model: str,
+        optional_params: dict,
+        litellm_params: dict,
+        stream: Optional[bool] = None,
+    ) -> str:
+        if not api_base:
+            api_base = "https://api.router.tetrate.ai/v1"
+
+        endpoint = "chat/completions"
+        api_base = api_base.rstrip("/")
+
+        if endpoint in api_base:
+            result = api_base
+        else:
+            result = f"{api_base}/{endpoint}"
+
+        return result
+
+    def get_models(self, api_key: Optional[str] = None, api_base: Optional[str] = None):
+        """
+        Override OpenAIGPTConfig.get_models() to use TARS API instead of OpenAI API.
+        """
+        # Use TarsModelInfo.get_models() method instead of OpenAIGPTConfig.get_models()
+        return TarsModelInfo.get_models(self, api_key=api_key, api_base=api_base)
+
+    @staticmethod
+    def get_api_base(api_base: Optional[str] = None) -> str:
+        """
+        Override OpenAIGPTConfig.get_api_base() to use TARS API base instead of OpenAI API base.
+        """
+        return TarsModelInfo.get_api_base(api_base)
+
+    @staticmethod
+    def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
+        """
+        Override OpenAIGPTConfig.get_api_key() to use TARS API key instead of OpenAI API key.
+        """
+        return TarsModelInfo.get_api_key(api_key)
+

litellm/llms/tars/common_utils.py

@@ -0,0 +1,111 @@
+"""
+TARS (Tetrate Agent Router Service) common utilities and model info.
+"""
+
+from typing import List, Optional
+
+import httpx
+
+from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.secret_managers.main import get_secret_str
+
+
+class TarsException(BaseLLMException):
+    """Exception class for TARS provider errors."""
+    pass
+
+
+class TarsModelInfo(BaseLLMModelInfo):
+    """
+    Model info for TARS (Tetrate Agent Router Service) provider.
+
+    Supports dynamic model fetching from the TARS API.
+    """
+
+    @staticmethod
+    def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
+        """Get TARS API key from parameter or environment variable."""
+        return api_key or get_secret_str("TARS_API_KEY")
+
+    @staticmethod
+    def get_api_base(api_base: Optional[str] = None) -> str:
+        """Get TARS API base URL from parameter or environment variable."""
+        return api_base or get_secret_str("TARS_API_BASE") or "https://api.router.tetrate.ai/v1"
+
+    @staticmethod
+    def get_base_model(model: str) -> Optional[str]:
+        """Remove tars/ prefix from model name."""
+        return model.replace("tars/", "")
+
+    def get_models(
+        self, api_key: Optional[str] = None, api_base: Optional[str] = None
+    ) -> List[str]:
+        """
+        Fetch available models from TARS API.
+
+        Args:
+            api_key: TARS API key (optional, will use TARS_API_KEY env var if not provided)
+            api_base: TARS API base URL (optional, defaults to https://api.router.tetrate.ai/v1)
+
+        Returns:
+            List of model names prefixed with "tars/"
+        """
+        api_base = self.get_api_base(api_base)
+        api_key = self.get_api_key(api_key)
+
+        if api_key is None:
+            raise ValueError(
+                "TARS_API_KEY is not set. Please set the environment variable to query TARS's /models endpoint."
+            )
+
+        try:
+            # Use a fresh httpx client to avoid any global configuration issues
+            url = f"{api_base}/models"
+            with httpx.Client() as client:
+                response = client.get(
+                    url=url,
+                    headers={"Authorization": f"Bearer {api_key}"},
+                    timeout=10.0
+                )
+                response.raise_for_status()
+        except httpx.HTTPStatusError as e:
+            raise ValueError(
+                f"Failed to fetch models from TARS. Status code: {e.response.status_code}, Response: {e.response.text}"
+            )
+        except Exception as e:
+            raise ValueError(f"Failed to fetch models from TARS. Error: {e}")
+
+        models_data = response.json().get("data", [])
+
+        # Extract model IDs and prefix with "tars/"
+        litellm_model_names = []
+        for model in models_data:
+            if isinstance(model, dict) and "id" in model:
+                model_id = model["id"]
+                litellm_model_name = f"tars/{model_id}"
+                litellm_model_names.append(litellm_model_name)
+
+        return sorted(litellm_model_names)
+
+    def validate_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: list,
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> dict:
+        """Validate TARS environment and add authentication headers."""
+        api_key = self.get_api_key(api_key)
+        api_base = self.get_api_base(api_base)
+
+        if api_key is None:
+            raise ValueError(
+                "TARS_API_KEY is not set. Please set the environment variable."
+            )
+
+        headers["Authorization"] = f"Bearer {api_key}"
+        return headers

litellm/llms/tars/cost_calculator.py

@@ -0,0 +1,55 @@
+"""
+Helper util for handling TARS-specific cost calculation.
+- Uses the generic cost calculator which already handles tiered pricing correctly.
+- Adds a 5% margin to the base model costs.
+- Returns (0.0, 0.0) when no pricing is available.
+"""
+
+from typing import Tuple
+
+from litellm.types.utils import Usage
+from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
+from litellm.utils import get_model_info
+
+
+def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]:
+    """
+    Calculates the cost per token for a given TARS model with a 5% margin.
+    Uses the generic cost calculator for all pricing logic.
+
+    Input:
+        - model: str, the model name without provider prefix.
+        - usage: LiteLLM Usage block, containing usage information.
+
+    Returns:
+        Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd.
+        Returns (0.0, 0.0) if no pricing is available.
+    """
+    try:
+        # Check if pricing is available for this model.
+        model_info = get_model_info(model=model, custom_llm_provider="tars")
+
+        # If no pricing is available, return (0.0, 0.0).
+        if not model_info or (
+            model_info.get("input_cost_per_token", 0) == 0 and 
+            model_info.get("output_cost_per_token", 0) == 0
+        ):
+            return (0.0, 0.0)
+
+        # Calculate base cost using generic calculator.
+        prompt_cost, completion_cost = generic_cost_per_token(
+            model=model,
+            usage=usage,
+            custom_llm_provider="tars"
+        )
+
+        # Add 5% margin to both costs.
+        margin_multiplier = 1.05
+        prompt_cost_with_margin = prompt_cost * margin_multiplier
+        completion_cost_with_margin = completion_cost * margin_multiplier
+
+        return prompt_cost_with_margin, completion_cost_with_margin
+    except Exception:
+        # If any error occurs (e.g., model not found), return (0.0, 0.0).
+        return (0.0, 0.0)
+

litellm/llms/tars/embedding/transformation.py

@@ -0,0 +1,52 @@
+"""
+Support for OpenAI's `/v1/embeddings` endpoint.
+
+TARS (Tetrate Agent Router Service) is OpenAI-compatible for embeddings.
+
+Docs: https://router.tetrate.ai
+API: https://api.router.tetrate.ai/v1
+"""
+
+from typing import Optional, Union
+
+import httpx
+
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.openai.embedding.transformation import OpenAIEmbeddingConfig
+from litellm.secret_managers.main import get_secret_str
+
+from ..common_utils import TarsException
+
+
+class TarsEmbeddingConfig(OpenAIEmbeddingConfig):
+    """
+    Configuration for TARS embeddings.
+
+    TARS supports embeddings through OpenAI-compatible API.
+    """
+
+    def get_error_class(
+        self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
+    ) -> BaseLLMException:
+        return TarsException(
+            message=error_message,
+            status_code=status_code,
+            headers=headers,
+        )
+
+    @staticmethod
+    def get_api_base(api_base: Optional[str] = None) -> str:
+        """
+        Get TARS API base URL from parameter or environment variable.
+        Override to use TARS-specific defaults instead of OpenAI defaults.
+        """
+        return api_base or get_secret_str("TARS_API_BASE") or "https://api.router.tetrate.ai/v1"
+
+    @staticmethod
+    def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
+        """
+        Get TARS API key from parameter or environment variable.
+        Override to use TARS-specific API key instead of OpenAI key.
+        """
+        return api_key or get_secret_str("TARS_API_KEY")
+