diff --git a/config/codex_chatgpt_handler.py b/config/codex_chatgpt_handler.py index c9c666e6..f3906567 100644 --- a/config/codex_chatgpt_handler.py +++ b/config/codex_chatgpt_handler.py @@ -487,6 +487,21 @@ def _model_response(model: str, payload: dict[str, Any]) -> ModelResponse: usage = payload.get("usage") or {} input_tokens = usage.get("input_tokens", 0) or 0 output_tokens = usage.get("output_tokens", 0) or 0 + # The OpenAI Responses API reports prompt-cache hits under + # ``input_tokens_details.cached_tokens``. Surface it as the OpenAI-style + # ``prompt_tokens_details.cached_tokens`` that LiteLLM reads for cost + # calculation, so the ChatGPT/Codex OAuth path is billed at the + # cache-read rate instead of full input rate (mirrors + # claude_code_handler surfacing ``cache_read_input_tokens``). When the + # field is absent the value is 0 and nothing is stamped — harmless no-op. + cached_tokens = (usage.get("input_tokens_details") or {}).get("cached_tokens", 0) or 0 + usage_out: dict[str, Any] = { + "prompt_tokens": input_tokens, + "completion_tokens": output_tokens, + "total_tokens": usage.get("total_tokens", input_tokens + output_tokens), + } + if cached_tokens: + usage_out["prompt_tokens_details"] = {"cached_tokens": cached_tokens} return ModelResponse( id=payload.get("id", f"chatcmpl-{model}"), model=model, @@ -497,11 +512,7 @@ def _model_response(model: str, payload: dict[str, Any]) -> ModelResponse: "finish_reason": "tool_calls" if tool_calls else "stop", } ], - usage={ - "prompt_tokens": input_tokens, - "completion_tokens": output_tokens, - "total_tokens": usage.get("total_tokens", input_tokens + output_tokens), - }, + usage=usage_out, ) @@ -576,6 +587,14 @@ def _response_to_chunks(self, response: ModelResponse) -> list[dict[str, Any]]: "completion_tokens": response.usage.completion_tokens if response.usage else 0, "total_tokens": response.usage.total_tokens if response.usage else 0, } + # Carry the prompt-cache hit count into the streaming final-usage chunk + # so streamed calls are cost-calculated at the cache-read rate too. + _ptd = getattr(response.usage, "prompt_tokens_details", None) if response.usage else None + _cached = getattr(_ptd, "cached_tokens", None) if _ptd is not None else None + if not _cached and isinstance(_ptd, dict): + _cached = _ptd.get("cached_tokens") + if _cached: + usage["prompt_tokens_details"] = {"cached_tokens": _cached} if raw_tool_calls: chunks = [] if content: diff --git a/config/litellm_dynamic_config.py b/config/litellm_dynamic_config.py index 1c7fa4c7..62d34891 100644 --- a/config/litellm_dynamic_config.py +++ b/config/litellm_dynamic_config.py @@ -688,15 +688,22 @@ def build_model_entry(model_name: str) -> dict[str, Any]: # Perplexity Sonar numbers are best-effort against the published rate # card (Sonar $1/$1, Sonar Pro $3/$15) — search-call surcharge not # modeled. -_SUBSCRIPTION_SHADOW_PRICING: dict[str, tuple[float, float]] = { - "auth/gpt-5.5": (0.000005, 0.000030), - "auth/gpt-5.4": (0.0000025, 0.000015), - "auth/gpt-5.4-mini": (0.00000075, 0.0000045), - "auth/gpt-5.3-codex": (0.00000175, 0.000014), +# Values are (input, output) or (input, output, cache_read) USD per token. +# The optional third element is the prompt-cache-read rate: OpenAI's GPT-5 +# generation prices cached input at ~1/10th of input (90% off), so the +# ChatGPT/Codex OAuth routes carry cache_read = input * 0.1. LiteLLM only +# discounts cached tokens when this cost is present in ``model_info`` AND the +# codex handler surfaces ``prompt_tokens_details.cached_tokens`` (it now does). +# Track OpenAI's published cached-input rate if it changes. +_SUBSCRIPTION_SHADOW_PRICING: dict[str, tuple[float, ...]] = { + "auth/gpt-5.5": (0.000005, 0.000030, 0.0000005), + "auth/gpt-5.4": (0.0000025, 0.000015, 0.00000025), + "auth/gpt-5.4-mini": (0.00000075, 0.0000045, 0.000000075), + "auth/gpt-5.3-codex": (0.00000175, 0.000014, 0.000000175), # gpt-5.3-codex-spark is the agentic-coding model the Codex subscription # actually serves (verified 2026-06-25 via /backend-api/codex/models); # gpt-5.3-codex is the API slug. Same shadow pricing. - "auth/gpt-5.3-codex-spark": (0.00000175, 0.000014), + "auth/gpt-5.3-codex-spark": (0.00000175, 0.000014, 0.000000175), "gemini-sub/gemini-2.5-pro": (0.00000125, 0.00001), "gemini-sub/gemini-2.5-flash": (0.0000003, 0.0000025), "copilot/gpt-5.5": (0.000005, 0.000030), @@ -723,10 +730,16 @@ def _with_shadow_pricing(route: dict[str, Any]) -> dict[str, Any]: if pricing is None: return route enriched = dict(route) - enriched["model_info"] = { + model_info: dict[str, float] = { "input_cost_per_token": pricing[0], "output_cost_per_token": pricing[1], } + # Optional third element = prompt-cache-read cost per token. Stamped only + # when present so LiteLLM discounts cached input on routes whose handler + # surfaces cached_tokens (codex/ChatGPT); routes without it are unchanged. + if len(pricing) > 2: + model_info["cache_read_input_token_cost"] = pricing[2] + enriched["model_info"] = model_info return enriched diff --git a/packages/decepticon/tests/unit/llm/test_codex_chatgpt_handler_cache.py b/packages/decepticon/tests/unit/llm/test_codex_chatgpt_handler_cache.py new file mode 100644 index 00000000..fd819ae2 --- /dev/null +++ b/packages/decepticon/tests/unit/llm/test_codex_chatgpt_handler_cache.py @@ -0,0 +1,119 @@ +"""codex_chatgpt_handler surfaces Responses-API prompt-cache hits. + +The ChatGPT/Codex OAuth backend reports cache hits under +``usage.input_tokens_details.cached_tokens``. The handler must re-emit them as +``prompt_tokens_details.cached_tokens`` so LiteLLM bills the cache-read rate. +""" + +from __future__ import annotations + +import importlib.util +import sys +import types +from pathlib import Path +from typing import Any + +_MODULE_PATH = Path(__file__).resolve().parents[5] / "config" / "codex_chatgpt_handler.py" + + +class _CapturingModelResponse: + """Stand-in for litellm.ModelResponse that records the usage kwarg.""" + + def __init__(self, **kwargs: Any) -> None: + self.kwargs = kwargs + self.usage = kwargs.get("usage") + + +def _load_handler() -> Any: + fake_litellm = types.ModuleType("litellm") + fake_litellm.CustomLLM = object + fake_litellm.ModelResponse = _CapturingModelResponse + fake_litellm.AuthenticationError = type("AuthenticationError", (Exception,), {}) + fake_litellm.APIError = type("APIError", (Exception,), {}) + + fake_oauth = types.ModuleType("oauth_token_store") + for _name in ( + "DEFAULT_JWT_SKEW_SECONDS", + "FileBackedCache", + "decode_jwt_payload", + "is_jwt_expired", + "oauth_refresh_request", + "read_json_file", + "with_retry_on_401", + "write_json_atomic", + ): + setattr(fake_oauth, _name, (lambda *_a, **_kw: None)) + fake_oauth.DEFAULT_JWT_SKEW_SECONDS = 300 + + fake_http = types.ModuleType("http_client") + fake_http.post = lambda *_a, **_kw: None + fake_http.async_post = lambda *_a, **_kw: None + + # Force complete fakes while loading the handler, then restore sys.modules + # so this test neither depends on nor leaks stubs another test installed + # (the modules are only referenced at import time; we call _model_response + # only, which touches none of them). + overrides = { + "litellm": fake_litellm, + "oauth_token_store": fake_oauth, + "http_client": fake_http, + "httpx": types.ModuleType("httpx"), + } + saved = {name: sys.modules.get(name) for name in overrides} + sys.modules.update(overrides) + try: + spec = importlib.util.spec_from_file_location("_codex_handler_src", _MODULE_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + finally: + for name, prev in saved.items(): + if prev is None: + sys.modules.pop(name, None) + else: + sys.modules[name] = prev + + +_module = _load_handler() +_model_response = _module._model_response + +_MSG_OUTPUT = [{"type": "message", "content": [{"type": "output_text", "text": "OK"}]}] + + +def test_cached_tokens_surfaced_as_prompt_tokens_details() -> None: + payload = { + "output": _MSG_OUTPUT, + "usage": { + "input_tokens": 5000, + "output_tokens": 10, + "total_tokens": 5010, + "input_tokens_details": {"cached_tokens": 4800}, + }, + } + resp = _model_response("gpt-5.5", payload) + assert resp.usage["prompt_tokens"] == 5000 + assert resp.usage["prompt_tokens_details"] == {"cached_tokens": 4800} + + +def test_no_cache_field_omits_prompt_tokens_details() -> None: + payload = { + "output": _MSG_OUTPUT, + "usage": {"input_tokens": 5000, "output_tokens": 10, "total_tokens": 5010}, + } + resp = _model_response("gpt-5.5", payload) + assert "prompt_tokens_details" not in resp.usage + + +def test_zero_cached_tokens_omits_prompt_tokens_details() -> None: + payload = { + "output": _MSG_OUTPUT, + "usage": { + "input_tokens": 5000, + "output_tokens": 10, + "total_tokens": 5010, + "input_tokens_details": {"cached_tokens": 0}, + }, + } + resp = _model_response("gpt-5.5", payload) + assert "prompt_tokens_details" not in resp.usage