From cff9d1f22270940dacd0f9c4f3c9de2ccdd5d694 Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Mon, 21 Sep 2026 15:07:53 +0200
Subject: [PATCH 01/30] feat(aio): add Jev boolean evaluations with TypeSafe
 keys

---
 .../internal/ai-observability-judge-inputs.md |  20 +++
 .../costs/providers/manual-providers.test.ts  |   1 +
 .../ai/costs/providers/manual-providers.ts    |   4 +
 posthog/settings/web.py                       |   2 +
 .../eval_reports/report_agent/prompts.py      |   9 +-
 .../ai_observability/evaluation_llm_judge.py  |  59 ++++++++-
 .../ai_observability/evaluation_types.py      |   1 +
 .../evaluation_workflow_activities.py         |   2 +
 .../ai_observability/test_run_evaluation.py   |  84 +++++++++++-
 .../backend/api/evaluation_config.py          |   8 +-
 .../backend/api/evaluations.py                |   6 +
 .../backend/api/provider_keys.py              |   4 +
 .../ai_observability/backend/api/proxy.py     |   5 +-
 .../ai_observability/backend/api/taggers.py   |   6 +-
 .../api/test/test_evaluation_config.py        |  14 ++
 .../backend/api/test/test_evaluations.py      |  27 +++-
 .../backend/api/test/test_provider_keys.py    |  48 ++++++-
 .../ai_observability/backend/llm/client.py    |   7 +
 .../backend/llm/test/test_typesafe.py         | 122 ++++++++++++++++++
 .../ai_observability/backend/llm/typesafe.py  | 120 +++++++++++++++++
 .../backend/models/provider_keys.py           |   5 +
 .../ByokModelPickerNotice.stories.tsx         |  47 +++++--
 .../frontend/ByokModelPickerNotice.tsx        |   5 +-
 .../ConversationDisplay/EvaluationDisplay.tsx |  14 +-
 .../EvaluationExplanation.stories.tsx         |  21 +++
 .../components/EvaluationExplanation.tsx      |  24 ++++
 .../components/GenerationEvalRunsTable.tsx    |  13 +-
 .../evaluations/AIObservabilityEvaluation.tsx |  21 ++-
 .../components/EvaluationRunsTable.tsx        |  13 +-
 .../frontend/evaluations/types.ts             |   1 +
 .../frontend/generated/api.schemas.ts         |  33 ++++-
 .../frontend/generated/api.zod.ts             |  21 ++-
 .../frontend/modelPickerLogic.test.ts         |  34 +++++
 .../frontend/modelPickerLogic.ts              |  92 +++++++++++--
 .../playground/llmPlaygroundModelLogic.ts     |   5 +-
 .../settings/LLMProviderKeysSettings.tsx      |   6 +-
 .../frontend/settings/llmProviderKeysLogic.ts |  14 +-
 .../ai_observability/frontend/utils.test.ts   |   9 ++
 products/ai_observability/frontend/utils.ts   |   7 +-
 services/mcp/src/api/generated.ts             |  33 ++++-
 .../mcp/src/generated/ai_observability/api.ts |   7 +-
 41 files changed, 863 insertions(+), 111 deletions(-)
 create mode 100644 products/ai_observability/backend/llm/test/test_typesafe.py
 create mode 100644 products/ai_observability/backend/llm/typesafe.py
 create mode 100644 products/ai_observability/frontend/components/EvaluationExplanation.stories.tsx
 create mode 100644 products/ai_observability/frontend/components/EvaluationExplanation.tsx

diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index f2adaa6ca809..a8682d2eac3e 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -33,3 +33,23 @@ Generation evaluations extract message text without the per-message character cu
 They sample the combined input, tool definitions, and output only when that text exceeds 150,000 characters, with a final character slice enforcing the limit.
 
 Implementation: [trace judge](../../posthog/temporal/ai_observability/run_trace_evaluation.py), [session judge](../../posthog/temporal/ai_observability/run_session_evaluation.py), and [generation judge](../../posthog/temporal/ai_observability/evaluation_llm_judge.py).
+
+## Jev boolean judge
+
+Jev is available under the existing LLM judge option with a customer-provided TypeSafe API key.
+Select the key and `jev-1.13.0` on each evaluation; TypeSafe keys cannot become the shared active provider key used by other AI features.
+Provider keys keep the provider they were created with; switching providers requires a new key.
+Jev supports boolean evaluations only and uses the same formatted text for generation, trace, and session targets.
+
+The evaluation prompt becomes a [Noul question](https://docs.typesafe.ai/primitives/noul).
+A probability of at least 0.5 produces `true`; the evaluation's existing pass/fail polarity still applies.
+For evaluations that allow N/A, a separate question checks whether the criteria apply, using the same threshold.
+Uncertainty alone does not produce N/A.
+The raw probability is stored in `$ai_evaluation_probability`, with token usage and the resolved model version.
+Jev provides no written reasoning, so reports inspect the original source when explaining outcomes.
+
+TypeSafe rate limits and overload responses are retried through Temporal, honoring `Retry-After` up to five minutes.
+If retries fail, the run fails and the evaluation stays enabled.
+Invalid probabilities or missing answers fail the evaluation rather than producing a false result.
+Inputs rejected for exceeding the model's context window are skipped.
+See TypeSafe's [API reference](https://docs.typesafe.ai/api) and [model limits and pricing](https://docs.typesafe.ai/models).
diff --git a/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.test.ts b/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.test.ts
index f2824fe114c2..bdbdea4850c6 100644
--- a/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.test.ts
+++ b/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.test.ts
@@ -5,6 +5,7 @@ describe('manualCosts', () => {
         model: string
         expected: { prompt_token: number; completion_token: number; cache_read_token?: number }
     }> = [
+        { model: 'jev-1.13.0', expected: { prompt_token: 0.000000042, completion_token: 0 } },
         {
             model: 'gpt-4.5',
             expected: {
diff --git a/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.ts b/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.ts
index 0781a502ea6c..4085be25813d 100644
--- a/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.ts
+++ b/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.ts
@@ -1,6 +1,10 @@
 import type { ModelCostRow } from './types'
 
 const manualProviderCosts: ModelCostRow[] = [
+    {
+        model: 'jev-1.13.0',
+        cost: { default: { prompt_token: 0.000000042, completion_token: 0 } },
+    },
     {
         model: 'gpt-4.5',
         cost: {
diff --git a/posthog/settings/web.py b/posthog/settings/web.py
index b57420c28dac..3d797dc6c897 100644
--- a/posthog/settings/web.py
+++ b/posthog/settings/web.py
@@ -614,6 +614,8 @@ def static_varies_origin(headers, path, url):
             "TaskArtifactStatusEnum": ["active", "failed"],
             "RunSourceEnum": ["manual", "signal_report", "agent"],
             "TaskBootstrapRunSourceEnum": ["manual", "signal_report"],
+            # Completion providers are a subset of LLMProvider that excludes evaluation-only models.
+            "LLMCompletionProviderEnum": "products.ai_observability.backend.models.provider_keys.llm_completion_provider_choices",
             #
             # The same choice set is declared in more than one product. A shared Choices
             # class would cross a product boundary, so the entry names the set centrally.
diff --git a/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.py b/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.py
index 1a8430701fd4..94450ac4e60b 100644
--- a/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.py
+++ b/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.py
@@ -154,9 +154,14 @@ def build_eval_report_system_prompt(
             "- **`get_top_outcome_reasons(outcome, limit)`**: grouped reasoning strings for one outcome. "
             f"If omitted, outcome defaults to `{analysis_outcome}`.\n"
         )
-        result_overview_detail = "Includes truncated reasoning."
+        result_overview_detail = "Includes truncated reasoning when the judge provides it."
         sample_ordering_signature = ""
-        sample_ordering_instruction = 'Rows include full reasoning. Use the default `order_by="recent"`.'
+        sample_ordering_instruction = (
+            'Rows include full reasoning when available. Use the default `order_by="recent"`. '
+            "Some judges, including Jev, return no written reasoning. For those results, inspect the original "
+            "generation, trace, or session with the detail tools and ground your analysis in that source. "
+            "Do not invent a judge explanation or treat absent reasoning as an evaluation failure."
+        )
         analysis_sample_arguments = f'outcome="{analysis_outcome}"'
         outcome_analysis_step = (
             f"Inspect grouped reasons and sample relevant outcomes, using `{analysis_outcome}` and `{primary_outcome}` "
diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.py
index 6b34453ef95f..76386be60251 100644
--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py
+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py
@@ -41,10 +41,13 @@
     ModelNotFoundError,
     ModelPermissionError,
     ProviderConnectionError,
+    ProviderMismatchError,
     QuotaExceededError,
     RateLimitError,
     StructuredOutputParseError,
 )
+from products.ai_observability.backend.llm.types import CompletionResponse, Usage
+from products.ai_observability.backend.llm.typesafe import TypeSafeClient, TypeSafeRateLimitError
 from products.ai_observability.backend.text_repr.formatters import add_line_numbers, reduce_by_uniform_sampling
 
 logger = structlog.get_logger(__name__)
@@ -335,16 +338,55 @@ def call_llm_judge(
         capture_analytics=False,
     )
 
+    probability: float | None = None
     try:
-        response = client.complete(
-            CompletionRequest(
+        if provider == "typesafe":
+            if provider_key is not None and provider_key.provider != provider:
+                raise ProviderMismatchError(provider_key.provider, provider)
+            typesafe_result = TypeSafeClient.evaluate_boolean(
+                api_key=provider_key.encrypted_config.get("api_key", "") if provider_key else "",
                 model=model,
-                system=system_prompt,
-                messages=[{"role": "user", "content": user_prompt}],
-                provider=provider,
-                response_format=response_format,
+                prompt=evaluation["evaluation_config"]["prompt"],
+                source=user_prompt,
+                allows_na=allows_na,
             )
-        )
+            probability = typesafe_result.answers["verdict"].noul
+            applicable = not allows_na or typesafe_result.answers["applicable"].noul >= 0.5
+            parsed: BooleanEvalResult | BooleanWithNAEvalResult = (
+                BooleanWithNAEvalResult(
+                    reasoning="", applicable=applicable, verdict=probability >= 0.5 if applicable else None
+                )
+                if allows_na
+                else BooleanEvalResult(reasoning="", verdict=probability >= 0.5)
+            )
+            model = typesafe_result.model
+            response = CompletionResponse(
+                content="",
+                model=model,
+                parsed=parsed,
+                usage=Usage(
+                    input_tokens=typesafe_result.usage.input_tokens,
+                    output_tokens=typesafe_result.usage.output_tokens,
+                    total_tokens=typesafe_result.usage.input_tokens + typesafe_result.usage.output_tokens,
+                ),
+            )
+        else:
+            response = client.complete(
+                CompletionRequest(
+                    model=model,
+                    system=system_prompt,
+                    messages=[{"role": "user", "content": user_prompt}],
+                    provider=provider,
+                    response_format=response_format,
+                )
+            )
+    except TypeSafeRateLimitError as e:
+        increment_errors("rate_limit", provider=provider)
+        raise ApplicationError(
+            str(e),
+            {"error_type": "provider_unavailable", "provider": provider},
+            next_retry_delay=timedelta(seconds=e.retry_after) if e.retry_after is not None else None,
+        ) from e
     except AuthenticationError:
         if is_byok:
             increment_user_errors("auth_error", provider=provider)
@@ -496,4 +538,7 @@ def call_llm_judge(
     else:
         raise ValueError(f"Unexpected result type: {type(parsed_result)}")
 
+    if probability is not None:
+        result_dict["probability"] = probability
+
     return result_dict
diff --git a/posthog/temporal/ai_observability/evaluation_types.py b/posthog/temporal/ai_observability/evaluation_types.py
index 5a35886967cb..d3b72f942f41 100644
--- a/posthog/temporal/ai_observability/evaluation_types.py
+++ b/posthog/temporal/ai_observability/evaluation_types.py
@@ -33,6 +33,7 @@ class EvaluationActivityResult(TypedDict, total=False):
     result_type: Required[Literal["boolean", "sentiment"]]
     reasoning: Required[str]
     verdict: NotRequired[bool | None]
+    probability: NotRequired[float]
     allows_na: NotRequired[bool]
     input_tokens: NotRequired[int]
     output_tokens: NotRequired[int]
diff --git a/posthog/temporal/ai_observability/evaluation_workflow_activities.py b/posthog/temporal/ai_observability/evaluation_workflow_activities.py
index 377271e1d712..c0ec2ceffc30 100644
--- a/posthog/temporal/ai_observability/evaluation_workflow_activities.py
+++ b/posthog/temporal/ai_observability/evaluation_workflow_activities.py
@@ -286,6 +286,8 @@ def build_evaluation_event_properties(
         properties["$ai_evaluation_provider"] = result.get("provider", "openai")
         properties["$ai_evaluation_key_type"] = "byok" if result.get("is_byok") else "posthog"
         properties["$ai_evaluation_key_id"] = result.get("key_id")
+        if "probability" in result:
+            properties["$ai_evaluation_probability"] = result["probability"]
 
     if result["result_type"] == "sentiment":
         if not result.get("skipped"):
diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index 7b10a5f48fbb..601c77e6ae70 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -1,6 +1,6 @@
 import json
 import uuid
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
 from typing import Any
 
 import pytest
@@ -42,8 +42,17 @@
     status_reason_detail_for_terminal_user_error,
     terminal_user_error_result_from_application_error,
 )
-from .evaluation_llm_judge import JUDGE_EVENT_MAX_CHARS, TransientJudgeError, _execute_llm_judge_activity
-from .evaluation_workflow_activities import LocalEvaluationOutcome, backfill_verdict_timestamp
+from .evaluation_llm_judge import (
+    JUDGE_EVENT_MAX_CHARS,
+    TransientJudgeError,
+    _execute_llm_judge_activity,
+    call_llm_judge,
+)
+from .evaluation_workflow_activities import (
+    LocalEvaluationOutcome,
+    backfill_verdict_timestamp,
+    build_evaluation_event_properties,
+)
 from .run_evaluation import (
     BooleanEvalResult,
     BooleanWithNAEvalResult,
@@ -75,6 +84,75 @@ def _mock_config_with_active_key(provider: str = "openai") -> MagicMock:
     return MagicMock(active_provider_key=key)
 
 
+@pytest.mark.parametrize(
+    "probability,applicability,allows_na,verdict",
+    [(0.49, 1.0, False, False), (0.5, 1.0, False, True), (0.9, 0.1, True, None), (0.0, 0.9, True, False)],
+)
+def test_typesafe_judge_emits_boolean_probability_without_reasoning(
+    probability: float, applicability: float, allows_na: bool, verdict: bool | None
+) -> None:
+    key = MagicMock(provider="typesafe", encrypted_config={"api_key": "test-typesafe-key"})
+    resolved = MagicMock(provider="typesafe", model="jev-1.13.0", provider_key=key, is_byok=True)
+    response = MagicMock(status_code=200)
+    response.json.return_value = {
+        "model": "jev-1.13.0",
+        "answers": {
+            "verdict": {"type": "noul", "noul": probability},
+            "applicable": {"type": "noul", "noul": applicability},
+        },
+        "usage": {"input_tokens": 120, "output_tokens": 10},
+    }
+    evaluation = {
+        "id": "test-evaluation",
+        "name": "Politeness",
+        "team_id": 1,
+        "evaluation_config": {"prompt": "Is the response polite?"},
+    }
+    with (
+        patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
+        patch("requests.request", return_value=response),
+    ):
+        spec.return_value.resolve.return_value = resolved
+        result = call_llm_judge(
+            evaluation=evaluation,
+            system_prompt="Unused generation instructions",
+            user_prompt="Hello!",
+            allows_na=allows_na,
+        )
+
+    assert result["verdict"] is verdict
+    assert result["reasoning"] == ""
+    assert result["probability"] == probability
+    assert result["total_tokens"] == 130
+    if allows_na:
+        assert result["applicable"] is (applicability >= 0.5)
+    properties = build_evaluation_event_properties(evaluation, result, datetime.now(UTC))
+    assert properties["$ai_evaluation_probability"] == probability
+    assert properties["$ai_model"] == "jev-1.13.0"
+    assert properties["$ai_evaluation_key_type"] == "byok"
+
+
+def test_typesafe_rate_limit_retries_without_disabling_the_evaluation() -> None:
+    key = MagicMock(provider="typesafe", encrypted_config={"api_key": "test-typesafe-key"})
+    with (
+        patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
+        patch("requests.request", return_value=MagicMock(status_code=429, headers={"Retry-After": "15"})),
+        pytest.raises(ApplicationError) as error,
+    ):
+        spec.return_value.resolve.return_value = MagicMock(
+            provider="typesafe", model="jev-1.13.0", provider_key=key, is_byok=True
+        )
+        call_llm_judge(
+            evaluation={"team_id": 1, "evaluation_config": {"prompt": "Polite?"}},
+            system_prompt="",
+            user_prompt="Hello!",
+            allows_na=False,
+        )
+    assert not error.value.non_retryable
+    assert error.value.next_retry_delay == timedelta(seconds=15)
+    assert terminal_user_error_result_from_application_error(error.value, allows_na=False) is None
+
+
 def test_status_reason_detail_for_terminal_user_error_only_keeps_truncated_hog_errors():
     hog_spec = require_user_error_spec("hog_error")
     permission_spec = require_user_error_spec("permission_error")
diff --git a/products/ai_observability/backend/api/evaluation_config.py b/products/ai_observability/backend/api/evaluation_config.py
index 4f658caf8da0..20f56270b836 100644
--- a/products/ai_observability/backend/api/evaluation_config.py
+++ b/products/ai_observability/backend/api/evaluation_config.py
@@ -14,7 +14,7 @@
 from posthog.scopes import APIScopeObjectOrNotSupported
 
 from ..models.evaluation_config import EvaluationConfig
-from ..models.provider_keys import LLMProviderKey
+from ..models.provider_keys import LLMProvider, LLMProviderKey
 from .metrics import llma_track_latency
 from .provider_keys import LLMProviderKeySerializer
 
@@ -109,6 +109,12 @@ def set_active_key(self, request: ValidatedRequest, **kwargs) -> Response:
                 status=status.HTTP_404_NOT_FOUND,
             )
 
+        if key.provider == LLMProvider.TYPESAFE:
+            return Response(
+                {"detail": "Select the TypeSafe key on an evaluation instead."},
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+
         if key.state != LLMProviderKey.State.OK:
             return Response(
                 {"detail": f"Cannot activate key with state '{key.state}'. Please validate the key first."},
diff --git a/products/ai_observability/backend/api/evaluations.py b/products/ai_observability/backend/api/evaluations.py
index dcbf33887400..ccc5c5b84856 100644
--- a/products/ai_observability/backend/api/evaluations.py
+++ b/products/ai_observability/backend/api/evaluations.py
@@ -43,6 +43,7 @@
 from ..evaluation_conditions import build_condition_filter
 from ..hog import compile_ai_observability_hog
 from ..llm import DEFAULT_MODEL_BY_PROVIDER
+from ..llm.typesafe import TypeSafeClient
 from ..models.evaluation_config import EvaluationConfig
 from ..models.evaluation_configs import (
     EVALUATION_TEST_LOOKBACK_DAYS,
@@ -251,6 +252,11 @@ def validate(self, data: dict[str, Any]) -> dict[str, Any]:
         errors = {field: "This field is required." for field in ("provider", "model") if field not in data}
         if errors:
             raise serializers.ValidationError(errors, code="required")
+        if data["provider"] == LLMProvider.TYPESAFE:
+            if data["model"] != TypeSafeClient.MODEL:
+                raise serializers.ValidationError({"model": "Select a supported Jev model."})
+            if not data.get("provider_key_id"):
+                raise serializers.ValidationError({"provider_key_id": "Select a TypeSafe API key for this evaluation."})
         return data
 
     def get_provider_key_name(self, obj: LLMModelConfiguration) -> str | None:
diff --git a/products/ai_observability/backend/api/provider_keys.py b/products/ai_observability/backend/api/provider_keys.py
index f5dae204f375..c8637d3608af 100644
--- a/products/ai_observability/backend/api/provider_keys.py
+++ b/products/ai_observability/backend/api/provider_keys.py
@@ -159,6 +159,10 @@ def validate(self, data):
             raise serializers.ValidationError({"api_key": "API key is required when creating a new provider key."})
 
         provider = data.get("provider", getattr(self.instance, "provider", None))
+        if self.instance is not None and provider != self.instance.provider:
+            raise serializers.ValidationError({"provider": "A key's provider cannot change. Create a new key instead."})
+        if provider == LLMProvider.TYPESAFE and data.get("set_as_active"):
+            raise serializers.ValidationError({"set_as_active": "Select the TypeSafe key on an evaluation instead."})
         if provider == LLMProvider.AZURE_OPENAI:
             has_endpoint = bool(data.get("azure_endpoint"))
             has_existing_endpoint = self.instance and self.instance.encrypted_config.get("azure_endpoint")
diff --git a/products/ai_observability/backend/api/proxy.py b/products/ai_observability/backend/api/proxy.py
index 37d8cc696a0f..f1f0d27dc1c6 100644
--- a/products/ai_observability/backend/api/proxy.py
+++ b/products/ai_observability/backend/api/proxy.py
@@ -49,7 +49,7 @@
     get_playground_models,
 )
 from products.ai_observability.backend.llm.errors import UnsupportedProviderError
-from products.ai_observability.backend.models.provider_keys import LLMProvider, LLMProviderKey
+from products.ai_observability.backend.models.provider_keys import LLMProviderKey, llm_completion_provider_choices
 
 from ee.hogai.utils.asgi import SyncIterableToAsync
 
@@ -63,6 +63,7 @@ def models_cache_key(provider_key_id: str | uuid.UUID) -> str:
 
 
 PROVIDER_DISPLAY_NAMES: dict[str, str] = {
+    "typesafe": "TypeSafe",
     "openai": "OpenAI",
     "anthropic": "Anthropic",
     "gemini": "Gemini",
@@ -78,7 +79,7 @@ class LLMProxyCompletionSerializer(serializers.Serializer):
     system = serializers.CharField(allow_blank=True)
     messages = serializers.ListField(child=serializers.DictField())
     model = serializers.CharField()
-    provider = serializers.ChoiceField(choices=LLMProvider.choices)
+    provider = serializers.ChoiceField(choices=llm_completion_provider_choices())
     thinking = serializers.BooleanField(default=False, required=False)
     temperature = serializers.FloatField(required=False)
     top_p = serializers.FloatField(required=False)
diff --git a/products/ai_observability/backend/api/taggers.py b/products/ai_observability/backend/api/taggers.py
index 27e599e15958..1af3bb8cf5eb 100644
--- a/products/ai_observability/backend/api/taggers.py
+++ b/products/ai_observability/backend/api/taggers.py
@@ -39,7 +39,7 @@
 
 from ..hog import compile_ai_observability_hog
 from ..models.model_configuration import LLMModelConfiguration
-from ..models.provider_keys import LLMProvider, LLMProviderKey
+from ..models.provider_keys import LLMProviderKey, llm_completion_provider_choices
 from ..models.taggers import Tagger, TaggerType, validate_tagger_config
 from .metrics import llma_track_latency
 
@@ -121,7 +121,9 @@ class TaggerConfigField(serializers.JSONField):
 
 
 class TaggerModelConfigurationWriteSerializer(serializers.Serializer):
-    provider = serializers.ChoiceField(choices=LLMProvider.choices, help_text="LLM provider to use for this tagger.")
+    provider = serializers.ChoiceField(
+        choices=llm_completion_provider_choices(), help_text="LLM provider to use for this tagger."
+    )
     model = serializers.CharField(max_length=100, help_text="Provider model identifier to use for this tagger.")
     provider_key_id = serializers.UUIDField(
         required=False,
diff --git a/products/ai_observability/backend/api/test/test_evaluation_config.py b/products/ai_observability/backend/api/test/test_evaluation_config.py
index ec18af3aa0a9..ba3a43f60b8f 100644
--- a/products/ai_observability/backend/api/test/test_evaluation_config.py
+++ b/products/ai_observability/backend/api/test/test_evaluation_config.py
@@ -36,6 +36,20 @@ def _setup_team():
 
 
 class TestEvaluationConfigViewSet(APIBaseTest):
+    def test_typesafe_cannot_become_the_shared_active_key(self) -> None:
+        key = LLMProviderKey.objects.create(
+            team=self.team,
+            provider="typesafe",
+            name="TypeSafe",
+            state="ok",
+            encrypted_config={"api_key": "test-typesafe-key"},
+        )
+        response = self.client.post(
+            f"/api/environments/{self.team.id}/llm_analytics/evaluation_config/set_active_key/", {"key_id": str(key.id)}
+        )
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+        self.assertFalse(EvaluationConfig.objects.filter(team=self.team, active_provider_key=key).exists())
+
     def test_unauthenticated_user_cannot_access_config(self):
         self.client.logout()
         response = self.client.get(f"/api/environments/{self.team.id}/llm_analytics/evaluation_config/")
diff --git a/products/ai_observability/backend/api/test/test_evaluations.py b/products/ai_observability/backend/api/test/test_evaluations.py
index cca163537cf8..004d55bc3ea4 100644
--- a/products/ai_observability/backend/api/test/test_evaluations.py
+++ b/products/ai_observability/backend/api/test/test_evaluations.py
@@ -54,6 +54,21 @@ def _setup_team():
 
 
 class TestModelConfigurationSerializer(SimpleTestCase):
+    @parameterized.expand(
+        [
+            ("missing_key", "jev-1.13.0", None, False),
+            ("unsupported_model", "other-model", str(uuid4()), False),
+            ("configured", "jev-1.13.0", str(uuid4()), True),
+        ]
+    )
+    def test_typesafe_requires_supported_model_and_explicit_key(
+        self, _name: str, model: str, key_id: str | None, valid: bool
+    ) -> None:
+        serializer = ModelConfigurationSerializer(
+            data={"provider": "typesafe", "model": model, "provider_key_id": key_id}
+        )
+        self.assertEqual(serializer.is_valid(), valid, serializer.errors)
+
     @parameterized.expand(
         [
             ("missing_provider", {"model": "gpt-5-mini"}, "provider"),
@@ -114,16 +129,18 @@ def test_unauthenticated_user_cannot_access_evaluation_configs(self):
         response = self.client.get(f"/api/environments/{self.team.id}/evaluations/")
         self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)
 
-    def test_can_create_evaluation_config(self):
+    @parameterized.expand([("openai", "gpt-5-mini"), ("typesafe", "jev-1.13.0")])
+    def test_can_create_evaluation_config(self, provider: str, model: str) -> None:
         key = LLMProviderKey.objects.create(
             team=self.team,
-            provider="openai",
+            provider=provider,
             name="Active Key",
             state=LLMProviderKey.State.OK,
             encrypted_config={"api_key": "sk-test"},
             created_by=self.user,
         )
-        EvaluationConfig.objects.create(team=self.team, active_provider_key=key)
+        if provider == "openai":
+            EvaluationConfig.objects.create(team=self.team, active_provider_key=key)
         response = self.client.post(
             f"/api/environments/{self.team.id}/evaluations/",
             {
@@ -131,7 +148,9 @@ def test_can_create_evaluation_config(self):
                 "description": "Test Description",
                 "enabled": True,
                 "evaluation_type": "llm_judge",
-                "model_configuration": _DEFAULT_MODEL_CONFIGURATION,
+                "model_configuration": {"provider": provider, "model": model, "provider_key_id": str(key.id)}
+                if provider == "typesafe"
+                else _DEFAULT_MODEL_CONFIGURATION,
                 "evaluation_config": {"prompt": "Test prompt"},
                 "output_type": "boolean",
                 "output_config": {},
diff --git a/products/ai_observability/backend/api/test/test_provider_keys.py b/products/ai_observability/backend/api/test/test_provider_keys.py
index 2a26070f7d54..f840c2af3209 100644
--- a/products/ai_observability/backend/api/test/test_provider_keys.py
+++ b/products/ai_observability/backend/api/test/test_provider_keys.py
@@ -1,19 +1,22 @@
 from uuid import uuid4
 
 from posthog.test.base import APIBaseTest
-from unittest.mock import patch
+from unittest.mock import Mock, patch
 
 from django.core.cache import cache
+from django.test import SimpleTestCase
 from django.utils import timezone
 
 from parameterized import parameterized
-from rest_framework import status
+from rest_framework import serializers, status
 
 from posthog.constants import AvailableFeature
 from posthog.models import Organization, OrganizationMembership, Project, Team, User
 
 from products.access_control.backend.models.access_control import AccessControl
-from products.ai_observability.backend.api.proxy import models_cache_key
+from products.ai_observability.backend.api.provider_keys import LLMProviderKeySerializer
+from products.ai_observability.backend.api.proxy import LLMProxyCompletionSerializer, models_cache_key
+from products.ai_observability.backend.api.taggers import TaggerModelConfigurationWriteSerializer
 from products.ai_observability.backend.llm.providers.azure_openai import DEFAULT_API_VERSION
 from products.ai_observability.backend.models.evaluation_config import EvaluationConfig
 from products.ai_observability.backend.models.evaluations import Evaluation
@@ -22,6 +25,36 @@
 from products.ai_observability.backend.models.taggers import Tagger
 
 
+class TestProviderKeySerializer(SimpleTestCase):
+    @parameterized.expand([(LLMProxyCompletionSerializer,), (TaggerModelConfigurationWriteSerializer,)])
+    def test_typesafe_is_not_a_completion_provider(self, serializer_class: type[serializers.Serializer]) -> None:
+        serializer = serializer_class(
+            data={
+                "provider": "typesafe",
+                "model": "jev-1.13.0",
+                "provider_key_id": str(uuid4()),
+                "system": "Reply politely.",
+                "messages": [{"role": "user", "content": "Hello!"}],
+            }
+        )
+        self.assertFalse(serializer.is_valid())
+        self.assertIn("provider", serializer.errors)
+
+    @parameterized.expand([("openai", "typesafe"), ("typesafe", "openai")])
+    def test_cannot_change_provider_of_an_existing_key(self, current: str, requested: str) -> None:
+        key = LLMProviderKey(provider=current, state="ok", encrypted_config={"api_key": "test-key"})
+        serializer = LLMProviderKeySerializer(key, data={"provider": requested}, partial=True)
+        self.assertFalse(serializer.is_valid())
+        self.assertIn("provider", serializer.errors)
+
+    def test_typesafe_cannot_be_created_as_the_shared_key(self) -> None:
+        serializer = LLMProviderKeySerializer(
+            data={"provider": "typesafe", "name": "TypeSafe", "api_key": "test-key", "set_as_active": True}
+        )
+        self.assertFalse(serializer.is_valid())
+        self.assertIn("set_as_active", serializer.errors)
+
+
 def _setup_team():
     org = Organization.objects.create(name="test")
     project = Project.objects.create(id=Team.objects.increment_id_sequence(), organization=org)
@@ -85,13 +118,14 @@ def test_unauthenticated_user_cannot_access_provider_keys(self):
         response = self.client.get(f"/api/environments/{self.team.id}/llm_analytics/provider_keys/")
         self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)
 
+    @parameterized.expand([("openai",), ("typesafe",)])
     @patch("products.ai_observability.backend.api.provider_keys.validate_provider_key")
-    def test_can_create_provider_key(self, mock_validate):
+    def test_can_create_provider_key(self, provider: str, mock_validate: Mock) -> None:
         mock_validate.return_value = (LLMProviderKey.State.OK, None)
 
         response = self.client.post(
             f"/api/environments/{self.team.id}/llm_analytics/provider_keys/",
-            {"provider": "openai", "name": "My Key", "api_key": "sk-test-key-12345"},
+            {"provider": provider, "name": "My Key", "api_key": "sk-test-key-12345"},
         )
         self.assertEqual(response.status_code, status.HTTP_201_CREATED)
         self.assertEqual(LLMProviderKey.objects.count(), 1)
@@ -99,14 +133,14 @@ def test_can_create_provider_key(self, mock_validate):
         key = LLMProviderKey.objects.first()
         assert key is not None
         self.assertEqual(key.name, "My Key")
-        self.assertEqual(key.provider, "openai")
+        self.assertEqual(key.provider, provider)
         self.assertEqual(key.state, LLMProviderKey.State.OK)
         self.assertEqual(key.team, self.team)
         self.assertEqual(key.created_by, self.user)
 
         self.assertEqual(response.data["api_key_masked"], "sk-t...2345")
         self.assertNotIn("api_key", response.data)
-        mock_validate.assert_called_once_with("openai", "sk-test-key-12345")
+        mock_validate.assert_called_once_with(provider, "sk-test-key-12345")
 
     @patch("products.ai_observability.backend.api.provider_keys.validate_provider_key")
     def test_can_create_provider_key_with_set_as_active(self, mock_validate):
diff --git a/products/ai_observability/backend/llm/client.py b/products/ai_observability/backend/llm/client.py
index 8e496868a2cf..ad85f43a247a 100644
--- a/products/ai_observability/backend/llm/client.py
+++ b/products/ai_observability/backend/llm/client.py
@@ -15,6 +15,7 @@
     CompletionResponse,
     StreamChunk,
 )
+from products.ai_observability.backend.llm.typesafe import TypeSafeClient
 
 if TYPE_CHECKING:
     from products.ai_observability.backend.llm.config import ProviderConfig
@@ -81,16 +82,22 @@ def _resolve_credentials(self) -> tuple[str | None, str | None]:
     @classmethod
     def validate_key(cls, provider: str, api_key: str, **kwargs: Any) -> tuple[str, str | None]:
         """Validate an API key for a provider. Returns (state, error_message)."""
+        if provider == "typesafe":
+            return TypeSafeClient.validate_key(api_key)
         return _get_provider(provider).validate_key(api_key, **kwargs)
 
     @classmethod
     def list_models(cls, provider: str, api_key: str | None = None, **kwargs: Any) -> list[str]:
         """List available models for a provider."""
+        if provider == "typesafe":
+            return [TypeSafeClient.MODEL]
         return _get_provider(provider).list_models(api_key, **kwargs)
 
     @classmethod
     def recommended_models(cls, provider: str) -> set[str]:
         """Return the set of curated/recommended model IDs for a provider."""
+        if provider == "typesafe":
+            return {TypeSafeClient.MODEL}
         return _get_provider(provider).recommended_models()
 
 
diff --git a/products/ai_observability/backend/llm/test/test_typesafe.py b/products/ai_observability/backend/llm/test/test_typesafe.py
new file mode 100644
index 000000000000..7815eccd0188
--- /dev/null
+++ b/products/ai_observability/backend/llm/test/test_typesafe.py
@@ -0,0 +1,122 @@
+import pytest
+from unittest.mock import Mock, patch
+
+from products.ai_observability.backend.llm.client import Client
+from products.ai_observability.backend.llm.errors import (
+    AuthenticationError,
+    ContextWindowExceededError,
+    ModelPermissionError,
+    ProviderConnectionError,
+    StructuredOutputParseError,
+)
+from products.ai_observability.backend.llm.typesafe import TypeSafeClient, TypeSafeRateLimitError
+
+
+@pytest.mark.parametrize("status, expected_state", [(200, "ok"), (401, "invalid"), (403, "invalid"), (500, "error")])
+def test_typesafe_key_validation(status: int, expected_state: str) -> None:
+    response = Mock(status_code=status)
+    response.json.return_value = {"models": [{"name": "jev-latest"}]}
+    with patch("requests.request", return_value=response) as request:
+        state, message = Client.validate_key("typesafe", "test-typesafe-key")
+
+    assert state == expected_state
+    assert (message is None) == (expected_state == "ok")
+    assert request.call_args.args == ("GET", "https://api.typesafe.ai/v1/models")
+    assert request.call_args.kwargs["headers"]["Authorization"] == "Bearer test-typesafe-key"
+
+
+@pytest.mark.parametrize("allows_na", [False, True])
+def test_typesafe_boolean_request(allows_na: bool) -> None:
+    response = Mock(status_code=200)
+    response.json.return_value = {
+        "model": "jev-1.13.0",
+        "answers": {"verdict": {"type": "noul", "noul": 0.8}, "applicable": {"type": "noul", "noul": 0.2}},
+        "usage": {"input_tokens": 120, "output_tokens": 10},
+    }
+    with patch("requests.request", return_value=response) as request:
+        result = TypeSafeClient.evaluate_boolean(
+            api_key="test-typesafe-key",
+            model="jev-1.13.0",
+            prompt="Is the response polite?",
+            source="Hello!",
+            allows_na=allows_na,
+        )
+
+    assert result.answers["verdict"].noul == 0.8
+    body = request.call_args.kwargs["json"]
+    assert body["state"] == "Hello!"
+    assert body["questions"]["verdict"] == {"type": "noul", "instructions": "Is the response polite?"}
+    assert ("applicable" in body["questions"]) == allows_na
+    assert request.call_args.kwargs["allow_redirects"] is False
+
+
+@pytest.mark.parametrize("probability", [-0.1, 1.1, float("nan"), float("inf"), "0.8", True, None])
+def test_typesafe_rejects_invalid_probabilities(probability: object) -> None:
+    response = Mock(status_code=200)
+    response.json.return_value = {
+        "model": "jev-1.13.0",
+        "answers": {"verdict": {"type": "noul", "noul": probability}},
+        "usage": {"input_tokens": 120, "output_tokens": 10},
+    }
+    with patch("requests.request", return_value=response), pytest.raises(StructuredOutputParseError):
+        TypeSafeClient.evaluate_boolean(
+            api_key="test-typesafe-key",
+            model="jev-1.13.0",
+            prompt="Is the response polite?",
+            source="Hello!",
+            allows_na=False,
+        )
+
+
+@pytest.mark.parametrize("status", [429, 529])
+def test_typesafe_rate_limits_are_retryable(status: int) -> None:
+    response = Mock(status_code=status, headers={"Retry-After": "15"})
+    with patch("requests.request", return_value=response), pytest.raises(TypeSafeRateLimitError) as error:
+        TypeSafeClient.evaluate_boolean(
+            api_key="test-typesafe-key",
+            model="jev-1.13.0",
+            prompt="Is the response polite?",
+            source="Hello!",
+            allows_na=False,
+        )
+    assert error.value.retry_after == 15
+
+
+def test_typesafe_requires_a_key() -> None:
+    with patch("requests.request") as request, pytest.raises(AuthenticationError):
+        TypeSafeClient.evaluate_boolean(
+            api_key="", model="jev-1.13.0", prompt="Polite?", source="Hello!", allows_na=False
+        )
+    request.assert_not_called()
+
+
+@pytest.mark.parametrize("answers", [{}, {"verdict": {"type": "noul", "noul": 0.9}}])
+def test_typesafe_requires_every_requested_answer(answers: dict[str, object]) -> None:
+    response = Mock(status_code=200)
+    response.json.return_value = {
+        "model": "jev-1.13.0",
+        "answers": answers,
+        "usage": {"input_tokens": 12, "output_tokens": 2},
+    }
+    with patch("requests.request", return_value=response), pytest.raises(StructuredOutputParseError):
+        TypeSafeClient.evaluate_boolean(
+            api_key="test-typesafe-key", model="jev-1.13.0", prompt="Polite?", source="Hello!", allows_na=True
+        )
+
+
+@pytest.mark.parametrize(
+    "status,message,error_type",
+    [
+        (401, "Invalid key", AuthenticationError),
+        (403, "Access denied", ModelPermissionError),
+        (500, "Unavailable", ProviderConnectionError),
+        (422, "Input exceeds the context window", ContextWindowExceededError),
+        (422, "Invalid question", StructuredOutputParseError),
+    ],
+)
+def test_typesafe_preserves_error_categories(status: int, message: str, error_type: type[Exception]) -> None:
+    response = Mock(status_code=status, text=message)
+    with patch("requests.request", return_value=response), pytest.raises(error_type):
+        TypeSafeClient.evaluate_boolean(
+            api_key="test-typesafe-key", model="jev-1.13.0", prompt="Polite?", source="Hello!", allows_na=False
+        )
diff --git a/products/ai_observability/backend/llm/typesafe.py b/products/ai_observability/backend/llm/typesafe.py
new file mode 100644
index 000000000000..a005e3389fab
--- /dev/null
+++ b/products/ai_observability/backend/llm/typesafe.py
@@ -0,0 +1,120 @@
+import math
+from datetime import UTC, datetime
+from email.utils import parsedate_to_datetime
+from typing import Literal
+
+import requests
+from pydantic import BaseModel, Field, ValidationError
+
+from products.ai_observability.backend.llm.errors import (
+    AuthenticationError,
+    ContextWindowExceededError,
+    ModelNotFoundError,
+    ModelPermissionError,
+    ProviderConnectionError,
+    RateLimitError,
+    StructuredOutputParseError,
+    is_context_window_error_message,
+)
+
+
+class NoulAnswer(BaseModel):
+    type: Literal["noul"]
+    noul: float = Field(strict=True, ge=0, le=1, allow_inf_nan=False)
+
+
+class TypeSafeUsage(BaseModel):
+    input_tokens: int = Field(strict=True, ge=0)
+    output_tokens: int = Field(strict=True, ge=0)
+
+
+class TypeSafeResponse(BaseModel):
+    model: str = Field(min_length=1)
+    answers: dict[str, NoulAnswer]
+    usage: TypeSafeUsage
+
+
+class TypeSafeRateLimitError(RateLimitError):
+    def __init__(self, retry_after: str | None) -> None:
+        super().__init__("TypeSafe is temporarily rate limiting requests.")
+        self.retry_after: float | None = None
+        if retry_after:
+            try:
+                delay = float(retry_after)
+            except ValueError:
+                try:
+                    delay = (parsedate_to_datetime(retry_after) - datetime.now(UTC)).total_seconds()
+                except (ValueError, TypeError, OverflowError):
+                    return
+            if math.isfinite(delay):
+                self.retry_after = max(1, min(delay, 300))
+
+
+class TypeSafeClient:
+    MODEL = "jev-1.13.0"
+
+    @staticmethod
+    def evaluate_boolean(*, api_key: str, model: str, prompt: str, source: str, allows_na: bool) -> TypeSafeResponse:
+        if not api_key:
+            raise AuthenticationError("A TypeSafe API key is required.")
+        questions = {"verdict": {"type": "noul", "instructions": prompt}}
+        if allows_na:
+            questions["applicable"] = {
+                "type": "noul",
+                "instructions": (
+                    "Do these evaluation criteria apply to this input? Answer true when the criteria can be "
+                    "evaluated, even if they are not met. Answer false only when they are not relevant.\n\n" + prompt
+                ),
+            }
+        try:
+            response = requests.request(
+                "POST",
+                "https://api.typesafe.ai/v1/systemone",
+                headers={"Authorization": f"Bearer {api_key}"},
+                json={"model": model, "state": source, "questions": questions},
+                timeout=60,
+                allow_redirects=False,
+            )
+        except requests.RequestException as error:
+            raise ProviderConnectionError("Could not reach TypeSafe.") from error
+
+        if response.status_code == 401:
+            raise AuthenticationError("TypeSafe rejected this API key.")
+        if response.status_code == 403:
+            raise ModelPermissionError(model)
+        if response.status_code == 404:
+            raise ModelNotFoundError(model)
+        if response.status_code in (429, 529):
+            raise TypeSafeRateLimitError(response.headers.get("Retry-After"))
+        if response.status_code >= 500:
+            raise ProviderConnectionError("TypeSafe is temporarily unavailable.")
+        if response.status_code == 422 and is_context_window_error_message(response.text):
+            raise ContextWindowExceededError("This input exceeds Jev's context window.")
+        if response.status_code != 200:
+            raise StructuredOutputParseError("TypeSafe rejected the evaluation request. Check the model and criteria.")
+        try:
+            result = TypeSafeResponse.model_validate(response.json())
+        except (ValidationError, ValueError) as error:
+            raise StructuredOutputParseError("TypeSafe returned an invalid evaluation response.") from error
+        # boffin: Missing answers cannot become a false verdict.
+        if not questions.keys() <= result.answers.keys():
+            raise StructuredOutputParseError("TypeSafe did not answer every evaluation question.")
+        return result
+
+    @staticmethod
+    def validate_key(api_key: str) -> tuple[str, str | None]:
+        try:
+            response = requests.request(
+                "GET",
+                "https://api.typesafe.ai/v1/models",
+                headers={"Authorization": f"Bearer {api_key}"},
+                timeout=30,
+                allow_redirects=False,
+            )
+        except requests.RequestException:
+            return "error", "Could not reach TypeSafe. Try again."
+        if response.status_code == 200:
+            return "ok", None
+        if response.status_code in (401, 403):
+            return "invalid", "TypeSafe rejected this API key. Check the key and try again."
+        return "error", "Could not validate the TypeSafe key. Try again."
diff --git a/products/ai_observability/backend/models/provider_keys.py b/products/ai_observability/backend/models/provider_keys.py
index b98abb31da22..b51f27473086 100644
--- a/products/ai_observability/backend/models/provider_keys.py
+++ b/products/ai_observability/backend/models/provider_keys.py
@@ -22,6 +22,7 @@ class LLMProvider(models.TextChoices):
     TOGETHER_AI = "together_ai", "Together AI"
     MINIMAX = "minimax", "MiniMax"
     ZEABUR = "zeabur", "Zeabur AI Hub"
+    TYPESAFE = "typesafe", "TypeSafe"
 
 
 def llm_provider_choices() -> list[tuple[str, str | Promise]]:
@@ -29,6 +30,10 @@ def llm_provider_choices() -> list[tuple[str, str | Promise]]:
     return list(LLMProvider.choices)
 
 
+def llm_completion_provider_choices() -> list[tuple[str, str | Promise]]:
+    return [(provider, label) for provider, label in LLMProvider.choices if provider != LLMProvider.TYPESAFE]
+
+
 class LLMProviderKey(UUIDTModel):
     class State(models.TextChoices):
         UNKNOWN = "unknown"
diff --git a/products/ai_observability/frontend/ByokModelPickerNotice.stories.tsx b/products/ai_observability/frontend/ByokModelPickerNotice.stories.tsx
index ae66fddb9b58..a16df5161d64 100644
--- a/products/ai_observability/frontend/ByokModelPickerNotice.stories.tsx
+++ b/products/ai_observability/frontend/ByokModelPickerNotice.stories.tsx
@@ -11,49 +11,72 @@ import { LLMProviderKey, LLMProviderKeyState } from './settings/llmProviderKeysL
 interface StoryArgs {
     keyState: LLMProviderKeyState | null
     modelsFail: boolean
+    typesafe?: boolean
 }
 
 function providerKey(state: LLMProviderKeyState): Partial<LLMProviderKey> {
     return { id: 'key-1', provider: 'openai', name: 'Production', state }
 }
 
-function PickerWithNotice(): JSX.Element {
-    const { providerModelGroups, hasByokKeys, byokModelsLoading, providerKeysLoading } = useValues(modelPickerLogic)
+function PickerWithNotice({ typesafe = false }: { typesafe?: boolean }): JSX.Element {
+    const { providerModelGroups, evaluationProviderModelGroups, hasByokKeys, byokModelsLoading, providerKeysLoading } =
+        useValues(modelPickerLogic)
 
     return (
         // The story root has no width of its own, so the picker column is sized here to match the form it sits in.
-        <div className="w-[560px]">
+        <div className="w-full max-w-[560px]">
             <ModelPicker
                 model=""
                 selectedProviderKeyId={null}
                 onSelect={() => {}}
-                groups={providerModelGroups}
+                groups={typesafe ? evaluationProviderModelGroups : providerModelGroups}
                 loading={byokModelsLoading || providerKeysLoading}
-                footerLink={getModelPickerFooterLink(hasByokKeys)}
+                footerLink={getModelPickerFooterLink(
+                    typesafe ? evaluationProviderModelGroups.some((group) => !group.disabledReason) : hasByokKeys
+                )}
             />
-            <ByokModelPickerNotice />
+            <ByokModelPickerNotice forEvaluation={typesafe} />
         </div>
     )
 }
 
 const meta: Meta<StoryArgs> = {
     title: 'Scenes-App/AI observability/BYOK model picker notice',
-    render: ({ keyState, modelsFail }) => {
+    render: ({ keyState, modelsFail, typesafe }) => {
         useStorybookMocks({
             get: {
                 '/api/environments/:team_id/llm_analytics/provider_keys/': {
-                    results: keyState ? [providerKey(keyState)] : [],
+                    results: keyState
+                        ? [
+                              {
+                                  ...providerKey(keyState),
+                                  ...(typesafe ? { provider: 'typesafe', name: 'TypeSafe' } : {}),
+                              },
+                          ]
+                        : [],
                 },
                 '/api/environments/:team_id/llm_analytics/evaluation_config/': { active_provider_key: null },
                 // Only the per-key request fails. The playground list uses the same path without a key id.
                 '/api/llm_proxy/models/': ({ request }) =>
                     modelsFail && new URL(request.url).searchParams.get('provider_key_id')
                         ? [500, { error: 'Internal error' }]
-                        : [200, []],
+                        : [
+                              200,
+                              typesafe
+                                  ? [
+                                        {
+                                            id: 'jev-1.13.0',
+                                            name: 'jev-1.13.0',
+                                            provider: 'TypeSafe',
+                                            is_recommended: true,
+                                        },
+                                    ]
+                                  : [],
+                          ],
             },
         })
 
-        return <PickerWithNotice />
+        return <PickerWithNotice typesafe={typesafe} />
     },
 }
 export default meta
@@ -69,3 +92,7 @@ export const NoUsableProviderKeys: StoryObj<StoryArgs> = {
 export const ModelsFailedToLoad: StoryObj<StoryArgs> = {
     args: { keyState: 'ok', modelsFail: true },
 }
+
+export const Jev: StoryObj<StoryArgs> = {
+    args: { keyState: 'ok', modelsFail: false, typesafe: true },
+}
diff --git a/products/ai_observability/frontend/ByokModelPickerNotice.tsx b/products/ai_observability/frontend/ByokModelPickerNotice.tsx
index 0be94b7ed9be..2db1153ae964 100644
--- a/products/ai_observability/frontend/ByokModelPickerNotice.tsx
+++ b/products/ai_observability/frontend/ByokModelPickerNotice.tsx
@@ -7,8 +7,9 @@ import { modelPickerLogic } from './modelPickerLogic'
 import { providerLabel } from './settings/providerKeyStateUtils'
 
 /** Explains an empty or short model picker on the surfaces that only run on the team's own provider keys. */
-export function ByokModelPickerNotice(): JSX.Element | null {
-    const { byokModelNotice } = useValues(modelPickerLogic)
+export function ByokModelPickerNotice({ forEvaluation = false }: { forEvaluation?: boolean }): JSX.Element | null {
+    const { byokModelNotice: generativeModelNotice, evaluationModelNotice } = useValues(modelPickerLogic)
+    const byokModelNotice = forEvaluation ? evaluationModelNotice : generativeModelNotice
     const { loadByokModels } = useActions(modelPickerLogic)
 
     switch (byokModelNotice?.kind) {
diff --git a/products/ai_observability/frontend/ConversationDisplay/EvaluationDisplay.tsx b/products/ai_observability/frontend/ConversationDisplay/EvaluationDisplay.tsx
index f614e648e94c..fdb90285bfff 100644
--- a/products/ai_observability/frontend/ConversationDisplay/EvaluationDisplay.tsx
+++ b/products/ai_observability/frontend/ConversationDisplay/EvaluationDisplay.tsx
@@ -7,14 +7,16 @@ import { urls } from 'scenes/urls'
 
 import { EventType } from '~/types'
 
+import { EvaluationExplanation } from '../components/EvaluationExplanation'
 import { EvaluationResultTag } from '../components/EvaluationResultTag'
 import { MetadataTag } from '../components/MetadataTag'
 import { llmEvaluationsLogic } from '../evaluations/llmEvaluationsLogic'
-import { normalizeEvaluationResultProperties } from '../utils'
+import { normalizeEvaluationResultProperties, normalizeOptionalNumber } from '../utils'
 
 export function EvaluationDisplay({ eventProperties }: { eventProperties: EventType['properties'] }): JSX.Element {
     const { detectorEvaluationIds } = useValues(llmEvaluationsLogic)
     const reasoning = eventProperties.$ai_evaluation_reasoning
+    const probability = normalizeOptionalNumber(eventProperties.$ai_evaluation_probability)
     const evaluationName = eventProperties.$ai_evaluation_name
     const model = eventProperties.$ai_model ?? eventProperties.$ai_evaluation_model
     const traceId = eventProperties.$ai_trace_id
@@ -55,10 +57,14 @@ export function EvaluationDisplay({ eventProperties }: { eventProperties: EventT
                 )}
             </div>
 
-            {reasoning && (
+            {(reasoning || probability !== null) && (
                 <div className="p-3 border rounded bg-surface-primary">
-                    <div className="font-medium text-xs text-muted mb-1.5">REASONING</div>
-                    <div className="text-sm whitespace-pre-wrap">{reasoning}</div>
+                    <div className="font-medium text-xs text-muted mb-1.5">Evaluation details</div>
+                    {probability !== null ? (
+                        <EvaluationExplanation probability={probability} />
+                    ) : (
+                        <div className="text-sm whitespace-pre-wrap">{reasoning}</div>
+                    )}
                 </div>
             )}
         </div>
diff --git a/products/ai_observability/frontend/components/EvaluationExplanation.stories.tsx b/products/ai_observability/frontend/components/EvaluationExplanation.stories.tsx
new file mode 100644
index 000000000000..308817da1549
--- /dev/null
+++ b/products/ai_observability/frontend/components/EvaluationExplanation.stories.tsx
@@ -0,0 +1,21 @@
+import type { Meta, StoryObj } from '@storybook/react'
+
+import { EvaluationExplanation } from './EvaluationExplanation'
+
+const meta: Meta<typeof EvaluationExplanation> = {
+    title: 'Scenes-App/AI observability/Evaluation explanation',
+    component: EvaluationExplanation,
+}
+export default meta
+
+export const Reasoning: StoryObj<typeof EvaluationExplanation> = {
+    args: { reasoning: 'The response answers the question and includes a polite greeting.' },
+}
+
+export const Jev: StoryObj<typeof EvaluationExplanation> = {
+    args: { probability: 0.84 },
+}
+
+export const JevZeroProbability: StoryObj<typeof EvaluationExplanation> = {
+    args: { probability: 0 },
+}
diff --git a/products/ai_observability/frontend/components/EvaluationExplanation.tsx b/products/ai_observability/frontend/components/EvaluationExplanation.tsx
new file mode 100644
index 000000000000..1a6262e677d4
--- /dev/null
+++ b/products/ai_observability/frontend/components/EvaluationExplanation.tsx
@@ -0,0 +1,24 @@
+import { Tooltip } from '@posthog/lemon-ui'
+
+export function EvaluationExplanation({
+    reasoning,
+    probability,
+}: {
+    reasoning?: string
+    probability?: number | null
+}): JSX.Element {
+    if (probability != null) {
+        return (
+            <Tooltip title="Jev's estimated probability that the evaluation criteria are true. Jev does not provide written reasoning.">
+                <span>{`${(probability * 100).toFixed(1)}% probability of true`}</span>
+            </Tooltip>
+        )
+    }
+    return (
+        <Tooltip title={reasoning}>
+            <div className="max-w-md cursor-default">
+                <div className="text-sm text-default line-clamp-2">{reasoning || 'No reasoning provided'}</div>
+            </div>
+        </Tooltip>
+    )
+}
diff --git a/products/ai_observability/frontend/components/GenerationEvalRunsTable.tsx b/products/ai_observability/frontend/components/GenerationEvalRunsTable.tsx
index 2c97a54e5291..6afc9894c5da 100644
--- a/products/ai_observability/frontend/components/GenerationEvalRunsTable.tsx
+++ b/products/ai_observability/frontend/components/GenerationEvalRunsTable.tsx
@@ -1,6 +1,6 @@
 import { BuiltLogic, useValues } from 'kea'
 
-import { LemonTable, Link, Tooltip } from '@posthog/lemon-ui'
+import { LemonTable, Link } from '@posthog/lemon-ui'
 
 import { TZLabel } from 'lib/components/TZLabel'
 import { LemonTableColumns } from 'lib/lemon-ui/LemonTable'
@@ -9,6 +9,7 @@ import { urls } from 'scenes/urls'
 import { llmEvaluationsLogic } from '../evaluations/llmEvaluationsLogic'
 import { EvaluationRun } from '../evaluations/types'
 import type { generationEvaluationRunsLogicType } from '../generationEvaluationRunsLogic'
+import { EvaluationExplanation } from './EvaluationExplanation'
 import { EvaluationResultTag, getEvaluationResultSortValue } from './EvaluationResultTag'
 import { EvaluationRunTargetCell } from './EvaluationRunTargetCell'
 
@@ -59,15 +60,9 @@ export function GenerationEvalRunsTable({
             },
         },
         {
-            title: 'Reasoning',
+            title: 'Details',
             key: 'reasoning',
-            render: (_, run) => (
-                <Tooltip title={run.reasoning}>
-                    <div className="max-w-md cursor-default">
-                        <div className="text-sm text-default line-clamp-2">{run.reasoning}</div>
-                    </div>
-                </Tooltip>
-            ),
+            render: (_, run) => <EvaluationExplanation reasoning={run.reasoning} probability={run.probability} />,
         },
     ]
 
diff --git a/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx b/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx
index 9137d65d1e60..2d3a05d85ad1 100644
--- a/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx
+++ b/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx
@@ -126,7 +126,9 @@ export function AIObservabilityEvaluation(): JSX.Element {
         return <NotFound object="evaluation" />
     }
     const openInPlaygroundUrl =
-        evaluationTypeUsesModelConfiguration(evaluation.evaluation_type) && evaluation.id
+        evaluationTypeUsesModelConfiguration(evaluation.evaluation_type) &&
+        evaluation.id &&
+        evaluation.model_configuration?.provider !== 'typesafe'
             ? combineUrl(urls.aiObservabilityPlayground(), { source_evaluation_id: evaluation.id }).url
             : null
 
@@ -903,17 +905,18 @@ export function AIObservabilityEvaluation(): JSX.Element {
 }
 
 function EvaluationModelPicker(): JSX.Element {
-    const { hasByokKeys, byokModels, providerModelGroups, byokModelsLoading, providerKeysLoading } =
+    const { byokModels, evaluationProviderModelGroups, byokModelsLoading, providerKeysLoading } =
         useValues(modelPickerLogic)
-    const { selectedModel, selectedPickerProviderKeyId, modelSelectionRequired } = useValues(llmEvaluationLogic)
+    const { selectedModel, selectedPickerProviderKeyId, modelSelectionRequired, evaluation } =
+        useValues(llmEvaluationLogic)
     const { selectModelFromPicker } = useActions(llmEvaluationLogic)
 
     // Evals always run on the team's own provider key, so only BYOK models are offered.
     const selectedModelName = byokModels.find((m) => m.id === selectedModel)?.name
-    const groups = providerModelGroups
+    const groups = evaluationProviderModelGroups
     const loading = byokModelsLoading || providerKeysLoading
 
-    const footerLink = getModelPickerFooterLink(hasByokKeys)
+    const footerLink = getModelPickerFooterLink(groups.some((group) => !group.disabledReason))
 
     return (
         <div className="bg-bg-light border rounded p-6">
@@ -935,7 +938,13 @@ function EvaluationModelPicker(): JSX.Element {
                             selectedModelName={selectedModelName}
                             data-attr="evaluation-model-selector"
                         />
-                        <ByokModelPickerNotice />
+                        <ByokModelPickerNotice forEvaluation />
+                        {evaluation?.model_configuration?.provider === 'typesafe' && (
+                            <p className="text-sm text-muted mt-2">
+                                Jev returns a probability without written reasoning. A probability of 50% or higher
+                                produces a true result.
+                            </p>
+                        )}
                         {modelSelectionRequired && !selectedModel && (
                             <p className="text-sm text-danger mt-1">Select a judge model.</p>
                         )}
diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationRunsTable.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationRunsTable.tsx
index 4c0ba8361983..aa1b938395d6 100644
--- a/products/ai_observability/frontend/evaluations/components/EvaluationRunsTable.tsx
+++ b/products/ai_observability/frontend/evaluations/components/EvaluationRunsTable.tsx
@@ -1,11 +1,12 @@
 import { useActions, useValues } from 'kea'
 
 import { IconRefresh } from '@posthog/icons'
-import { LemonBanner, LemonButton, LemonSegmentedButton, LemonTable, LemonTag, Tooltip } from '@posthog/lemon-ui'
+import { LemonBanner, LemonButton, LemonSegmentedButton, LemonTable, LemonTag } from '@posthog/lemon-ui'
 
 import { TZLabel } from 'lib/components/TZLabel'
 import { LemonTableColumns } from 'lib/lemon-ui/LemonTable'
 
+import { EvaluationExplanation } from '../../components/EvaluationExplanation'
 import { EvaluationResultTag, getEvaluationResultSortValue } from '../../components/EvaluationResultTag'
 import { EvaluationRunTargetCell } from '../../components/EvaluationRunTargetCell'
 import { evaluationIsDetector } from '../constants'
@@ -107,15 +108,9 @@ export function EvaluationRunsTable(): JSX.Element {
             },
         },
         {
-            title: 'Reasoning',
+            title: 'Details',
             key: 'reasoning',
-            render: (_, run) => (
-                <Tooltip title={run.reasoning}>
-                    <div className="max-w-md cursor-default">
-                        <div className="text-sm text-default line-clamp-2">{run.reasoning}</div>
-                    </div>
-                </Tooltip>
-            ),
+            render: (_, run) => <EvaluationExplanation reasoning={run.reasoning} probability={run.probability} />,
         },
         {
             title: 'Status',
diff --git a/products/ai_observability/frontend/evaluations/types.ts b/products/ai_observability/frontend/evaluations/types.ts
index 35aa972762b8..d92281b64eea 100644
--- a/products/ai_observability/frontend/evaluations/types.ts
+++ b/products/ai_observability/frontend/evaluations/types.ts
@@ -143,6 +143,7 @@ export interface EvaluationRun {
     // evaluation disallows N/A, so it has to be read alongside this rather than on its own.
     skipped?: boolean
     reasoning: string
+    probability?: number | null
     status: 'completed' | 'failed' | 'running'
 }
 
diff --git a/products/ai_observability/frontend/generated/api.schemas.ts b/products/ai_observability/frontend/generated/api.schemas.ts
index ca36ebd4e839..7d5dd381ca0d 100644
--- a/products/ai_observability/frontend/generated/api.schemas.ts
+++ b/products/ai_observability/frontend/generated/api.schemas.ts
@@ -850,6 +850,7 @@ export const EvaluationTargetEnumApi = {
  * * `together_ai` - Together AI
  * * `minimax` - MiniMax
  * * `zeabur` - Zeabur AI Hub
+ * * `typesafe` - TypeSafe
  */
 export type LLMProviderEnumApi = (typeof LLMProviderEnumApi)[keyof typeof LLMProviderEnumApi]
 
@@ -863,6 +864,7 @@ export const LLMProviderEnumApi = {
     TogetherAi: 'together_ai',
     Minimax: 'minimax',
     Zeabur: 'zeabur',
+    Typesafe: 'typesafe',
 } as const
 
 /**
@@ -3041,6 +3043,32 @@ export interface TaggerConditionApi {
     properties?: TaggerConditionApiPropertiesItem[]
 }
 
+/**
+ * * `openai` - Openai
+ * * `anthropic` - Anthropic
+ * * `gemini` - Gemini
+ * * `openrouter` - Openrouter
+ * * `fireworks` - Fireworks
+ * * `azure_openai` - Azure OpenAI
+ * * `together_ai` - Together AI
+ * * `minimax` - MiniMax
+ * * `zeabur` - Zeabur AI Hub
+ */
+export type LLMCompletionProviderEnumApi =
+    (typeof LLMCompletionProviderEnumApi)[keyof typeof LLMCompletionProviderEnumApi]
+
+export const LLMCompletionProviderEnumApi = {
+    Openai: 'openai',
+    Anthropic: 'anthropic',
+    Gemini: 'gemini',
+    Openrouter: 'openrouter',
+    Fireworks: 'fireworks',
+    AzureOpenai: 'azure_openai',
+    TogetherAi: 'together_ai',
+    Minimax: 'minimax',
+    Zeabur: 'zeabur',
+} as const
+
 /**
  * Nested serializer for model configuration.
  */
@@ -3056,7 +3084,7 @@ export interface TaggerModelConfigurationApi {
      * * `together_ai` - Together AI
      * * `minimax` - MiniMax
      * * `zeabur` - Zeabur AI Hub */
-    provider: LLMProviderEnumApi
+    provider: LLMCompletionProviderEnumApi
     /**
      * Provider model identifier to use for this tagger.
      * @maxLength 100
@@ -3110,7 +3138,7 @@ export interface TaggerModelConfigurationWriteApi {
      * * `together_ai` - Together AI
      * * `minimax` - MiniMax
      * * `zeabur` - Zeabur AI Hub */
-    provider: LLMProviderEnumApi
+    provider: LLMCompletionProviderEnumApi
     /**
      * Provider model identifier to use for this tagger.
      * @maxLength 100
@@ -3537,6 +3565,7 @@ export const LlmAnalyticsModelsRetrieveProvider = {
     Openai: 'openai',
     Openrouter: 'openrouter',
     TogetherAi: 'together_ai',
+    Typesafe: 'typesafe',
     Zeabur: 'zeabur',
 } as const
 
diff --git a/products/ai_observability/frontend/generated/api.zod.ts b/products/ai_observability/frontend/generated/api.zod.ts
index 4e21ad828c8c..5fab62024e2b 100644
--- a/products/ai_observability/frontend/generated/api.zod.ts
+++ b/products/ai_observability/frontend/generated/api.zod.ts
@@ -438,9 +438,10 @@ export const EvaluationsCreateBody = /* @__PURE__ */ zod
                                 'together_ai',
                                 'minimax',
                                 'zeabur',
+                                'typesafe',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - TypeSafe'
                             ),
                         model: zod.string().max(evaluationsCreateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -742,9 +743,10 @@ export const EvaluationsUpdateBody = /* @__PURE__ */ zod
                                 'together_ai',
                                 'minimax',
                                 'zeabur',
+                                'typesafe',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - TypeSafe'
                             ),
                         model: zod.string().max(evaluationsUpdateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -950,9 +952,10 @@ export const EvaluationsPartialUpdateBody = /* @__PURE__ */ zod
                                 'together_ai',
                                 'minimax',
                                 'zeabur',
+                                'typesafe',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - TypeSafe'
                             ),
                         model: zod.string().max(evaluationsPartialUpdateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -1530,9 +1533,10 @@ export const LlmAnalyticsProviderKeysCreateBody = /* @__PURE__ */ zod.object({
             'together_ai',
             'minimax',
             'zeabur',
+            'typesafe',
         ])
         .describe(
-            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub'
+            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - TypeSafe'
         ),
     name: zod.string().max(llmAnalyticsProviderKeysCreateBodyNameMax),
     api_key: zod.string().optional(),
@@ -1563,9 +1567,10 @@ export const LlmAnalyticsProviderKeysUpdateBody = /* @__PURE__ */ zod.object({
             'together_ai',
             'minimax',
             'zeabur',
+            'typesafe',
         ])
         .describe(
-            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub'
+            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - TypeSafe'
         ),
     name: zod.string().max(llmAnalyticsProviderKeysUpdateBodyNameMax),
     api_key: zod.string().optional(),
@@ -1596,10 +1601,11 @@ export const LlmAnalyticsProviderKeysPartialUpdateBody = /* @__PURE__ */ zod.obj
             'together_ai',
             'minimax',
             'zeabur',
+            'typesafe',
         ])
         .optional()
         .describe(
-            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub'
+            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - TypeSafe'
         ),
     name: zod.string().max(llmAnalyticsProviderKeysPartialUpdateBodyNameMax).optional(),
     api_key: zod.string().optional(),
@@ -1630,9 +1636,10 @@ export const LlmAnalyticsProviderKeysValidateCreateBody = /* @__PURE__ */ zod.ob
             'together_ai',
             'minimax',
             'zeabur',
+            'typesafe',
         ])
         .describe(
-            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub'
+            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - TypeSafe'
         ),
     name: zod.string().max(llmAnalyticsProviderKeysValidateCreateBodyNameMax),
     api_key: zod.string().optional(),
diff --git a/products/ai_observability/frontend/modelPickerLogic.test.ts b/products/ai_observability/frontend/modelPickerLogic.test.ts
index 2fbf369d3d21..ede2501f8805 100644
--- a/products/ai_observability/frontend/modelPickerLogic.test.ts
+++ b/products/ai_observability/frontend/modelPickerLogic.test.ts
@@ -45,6 +45,40 @@ describe('modelPickerLogic', () => {
     })
 
     describe('loadByokModels', () => {
+        it('offers Jev only to evaluation model pickers', async () => {
+            useMocks({
+                get: {
+                    '/api/environments/:team_id/llm_analytics/provider_keys/': {
+                        results: [{ id: 'key-typesafe', provider: 'typesafe', name: 'TypeSafe', state: 'ok' }],
+                    },
+                    '/api/environments/:team_id/llm_analytics/evaluation_config/': { active_provider_key: null },
+                    '/api/llm_proxy/models/': ({ request }) =>
+                        new URL(request.url).searchParams.get('provider_key_id')
+                            ? [
+                                  200,
+                                  [
+                                      {
+                                          id: 'jev-1.13.0',
+                                          name: 'jev-1.13.0',
+                                          provider: 'TypeSafe',
+                                          is_recommended: true,
+                                      },
+                                  ],
+                              ]
+                            : [200, []],
+                },
+            })
+            logic = modelPickerLogic()
+            logic.mount()
+            await expectLogic(logic).toFinishAllListeners()
+
+            expect(logic.values.evaluationProviderModelGroups[0].models[0].id).toBe('jev-1.13.0')
+            expect(logic.values.providerModelGroups).toEqual([])
+            expect(logic.values.generativeByokModels).toEqual([])
+            expect(logic.values.hasByokKeys).toBe(false)
+            expect(logic.values.evaluationModelNotice).toBeNull()
+        })
+
         it('should load and attach providerKeyId to models from valid keys', async () => {
             useMocks({
                 get: {
diff --git a/products/ai_observability/frontend/modelPickerLogic.ts b/products/ai_observability/frontend/modelPickerLogic.ts
index 841a7e9ef124..9f1da5bf9c08 100644
--- a/products/ai_observability/frontend/modelPickerLogic.ts
+++ b/products/ai_observability/frontend/modelPickerLogic.ts
@@ -42,6 +42,26 @@ const UNHEALTHY_KEY_REASON = 'This provider key has an issue. Check your provide
 const UNAVAILABLE_KEY_REASON = "Couldn't load models for this key. Try again in a moment."
 const UNAVAILABLE_KEY_SUFFIX = ' (Unavailable)'
 
+function getByokModelNotice(
+    providerKeys: LLMProviderKey[],
+    providerKeysLoading: boolean,
+    byokModelsLoading: boolean,
+    failedByokProviderKeyIds: string[],
+    providerModelGroups: ProviderModelGroup[]
+): ByokModelNotice | null {
+    if (providerKeysLoading || byokModelsLoading) {
+        return null
+    }
+    const failedKeys = providerKeys.filter((key) => failedByokProviderKeyIds.includes(key.id))
+    if (failedKeys.length > 0) {
+        return { kind: 'models-failed', keys: failedKeys }
+    }
+    if (providerModelGroups.some((group) => !group.disabledReason)) {
+        return null
+    }
+    return providerKeys.length === 0 ? { kind: 'no-keys' } : { kind: 'no-usable-keys' }
+}
+
 function providerKeyGroupLabel(key: LLMProviderKey, keysPerProvider: Record<string, number>, suffix = ''): string {
     const label = providerLabel(key.provider)
     return (keysPerProvider[key.provider] ?? 0) > 1 ? `${label} (${key.name})${suffix}` : `${label}${suffix}`
@@ -94,7 +114,10 @@ export interface modelPickerLogicValues {
     byokModelNotice: ByokModelNotice | null
     byokModels: ModelOption[]
     byokModelsLoading: boolean
+    evaluationModelNotice: ByokModelNotice | null
+    evaluationProviderModelGroups: ProviderModelGroup[]
     failedByokProviderKeyIds: string[]
+    generativeByokModels: ModelOption[]
     hasByokKeys: boolean
     playgroundModels: ModelOption[]
     playgroundModelsLoading: boolean
@@ -178,12 +201,21 @@ export interface modelPickerLogicActions {
 export interface modelPickerLogicMeta {
     __keaTypeGenInternalSelectorTypes: {
         hasByokKeys: (providerKeys: LLMProviderKey[]) => boolean
+        generativeByokModels: (byokModels: ModelOption[], providerKeys: LLMProviderKey[]) => ModelOption[]
         playgroundProviderModelGroups: (playgroundModels: ModelOption[]) => ProviderModelGroup[]
-        providerModelGroups: (
+        evaluationProviderModelGroups: (
             byokModels: ModelOption[],
             providerKeys: LLMProviderKey[],
             failedByokProviderKeyIds: string[]
         ) => ProviderModelGroup[]
+        providerModelGroups: (evaluationProviderModelGroups: ProviderModelGroup[]) => ProviderModelGroup[]
+        evaluationModelNotice: (
+            providerKeys: LLMProviderKey[],
+            providerKeysLoading: boolean,
+            byokModelsLoading: boolean,
+            failedByokProviderKeyIds: string[],
+            evaluationProviderModelGroups: ProviderModelGroup[]
+        ) => ByokModelNotice | null
         byokModelNotice: (
             providerKeys: LLMProviderKey[],
             providerKeysLoading: boolean,
@@ -314,14 +346,22 @@ export const modelPickerLogic = kea<modelPickerLogicType>([
     selectors({
         hasByokKeys: [
             (s) => [s.providerKeys],
-            (providerKeys: LLMProviderKey[]): boolean => providerKeys.some((k) => k.state === 'ok'),
+            (providerKeys: LLMProviderKey[]): boolean =>
+                providerKeys.some((k) => k.state === 'ok' && k.provider !== 'typesafe'),
+        ],
+        generativeByokModels: [
+            (s) => [s.byokModels, s.providerKeys],
+            (models: ModelOption[], keys: LLMProviderKey[]): ModelOption[] =>
+                models.filter(
+                    (model) => !keys.some((key) => key.id === model.providerKeyId && key.provider === 'typesafe')
+                ),
         ],
         playgroundProviderModelGroups: [
             (s) => [s.playgroundModels],
             (playgroundModels: ModelOption[]): ProviderModelGroup[] =>
                 buildPlaygroundProviderModelGroups(Array.isArray(playgroundModels) ? playgroundModels : []),
         ],
-        providerModelGroups: [
+        evaluationProviderModelGroups: [
             (s) => [s.byokModels, s.providerKeys, s.failedByokProviderKeyIds],
             (
                 byokModels: ModelOption[],
@@ -378,6 +418,34 @@ export const modelPickerLogic = kea<modelPickerLogicType>([
                 })
             },
         ],
+        providerModelGroups: [
+            (s) => [s.evaluationProviderModelGroups],
+            (groups: ProviderModelGroup[]): ProviderModelGroup[] =>
+                groups.filter((group) => group.provider !== 'typesafe'),
+        ],
+        evaluationModelNotice: [
+            (s) => [
+                s.providerKeys,
+                s.providerKeysLoading,
+                s.byokModelsLoading,
+                s.failedByokProviderKeyIds,
+                s.evaluationProviderModelGroups,
+            ],
+            (
+                providerKeys: LLMProviderKey[],
+                providerKeysLoading: boolean,
+                byokModelsLoading: boolean,
+                failedByokProviderKeyIds: string[],
+                groups: ProviderModelGroup[]
+            ): ByokModelNotice | null =>
+                getByokModelNotice(
+                    providerKeys,
+                    providerKeysLoading,
+                    byokModelsLoading,
+                    failedByokProviderKeyIds,
+                    groups
+                ),
+        ],
         byokModelNotice: [
             (s) => [
                 s.providerKeys,
@@ -393,17 +461,13 @@ export const modelPickerLogic = kea<modelPickerLogicType>([
                 failedByokProviderKeyIds: string[],
                 providerModelGroups: ProviderModelGroup[]
             ): ByokModelNotice | null => {
-                if (providerKeysLoading || byokModelsLoading) {
-                    return null
-                }
-                const failedKeys = providerKeys.filter((key) => failedByokProviderKeyIds.includes(key.id))
-                if (failedKeys.length > 0) {
-                    return { kind: 'models-failed', keys: failedKeys }
-                }
-                if (providerModelGroups.some((group) => !group.disabledReason)) {
-                    return null
-                }
-                return providerKeys.length === 0 ? { kind: 'no-keys' } : { kind: 'no-usable-keys' }
+                return getByokModelNotice(
+                    providerKeys.filter((key) => key.provider !== 'typesafe'),
+                    providerKeysLoading,
+                    byokModelsLoading,
+                    failedByokProviderKeyIds,
+                    providerModelGroups
+                )
             },
         ],
     }),
diff --git a/products/ai_observability/frontend/playground/llmPlaygroundModelLogic.ts b/products/ai_observability/frontend/playground/llmPlaygroundModelLogic.ts
index 3453adc24a56..efca80bd18e4 100644
--- a/products/ai_observability/frontend/playground/llmPlaygroundModelLogic.ts
+++ b/products/ai_observability/frontend/playground/llmPlaygroundModelLogic.ts
@@ -204,7 +204,7 @@ export const llmPlaygroundModelLogic = kea<llmPlaygroundModelLogicType>([
             ],
             modelPickerLogic,
             [
-                'byokModels',
+                'generativeByokModels as byokModels',
                 'byokModelsLoading',
                 'playgroundModels',
                 'playgroundModelsLoading',
@@ -422,7 +422,8 @@ export const llmPlaygroundModelLogic = kea<llmPlaygroundModelLogicType>([
             loadProviderKeysFailure: () => resolvePendingTarget(),
             loadPlaygroundModelsFailure: () => resolvePendingTarget(),
             loadByokModelsFailure: () => resolvePendingTarget(),
-            loadByokModelsSuccess: ({ byokModels }: { byokModels: ModelOption[] }) => {
+            loadByokModelsSuccess: () => {
+                const byokModels = values.byokModels
                 if (byokModels.length === 0) {
                     return
                 }
diff --git a/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx b/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
index 972d6a5a7967..2d581f873f1e 100644
--- a/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
+++ b/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
@@ -102,6 +102,8 @@ function getKeyPlaceholder(provider: LLMProvider): string {
             return 'Enter your MiniMax API key'
         case 'zeabur':
             return 'sk-...'
+        case 'typesafe':
+            return 'Enter your TypeSafe API key'
     }
 }
 
@@ -195,7 +197,7 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
                     provider,
                     name,
                     api_key: apiKey,
-                    set_as_active: !evaluationConfig?.active_provider_key,
+                    set_as_active: provider !== 'typesafe' && !evaluationConfig?.active_provider_key,
                 }
                 if (isAzure) {
                     payload.azure_endpoint = azureEndpoint
@@ -229,7 +231,7 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
                 provider,
                 name,
                 api_key: apiKey,
-                set_as_active: !evaluationConfig?.active_provider_key,
+                set_as_active: provider !== 'typesafe' && !evaluationConfig?.active_provider_key,
             }
             if (isAzure) {
                 payload.azure_endpoint = azureEndpoint
diff --git a/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts b/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts
index 299adb970721..369dfdf1cb00 100644
--- a/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts
+++ b/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts
@@ -5,17 +5,10 @@ import api, { ApiError } from 'lib/api'
 import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast'
 import { teamLogic } from 'scenes/teamLogic'
 
+import type { LLMProviderEnumApi } from '../generated/api.schemas'
+
 export type LLMProviderKeyState = 'unknown' | 'ok' | 'invalid' | 'error'
-export type LLMProvider =
-    | 'openai'
-    | 'anthropic'
-    | 'gemini'
-    | 'openrouter'
-    | 'fireworks'
-    | 'azure_openai'
-    | 'together_ai'
-    | 'minimax'
-    | 'zeabur'
+export type LLMProvider = LLMProviderEnumApi
 
 /** Default Azure OpenAI API version — keep in sync with backend DEFAULT_API_VERSION. */
 export const DEFAULT_AZURE_API_VERSION = '2024-10-21'
@@ -30,6 +23,7 @@ export const LLM_PROVIDER_LABELS: Record<LLMProvider, string> = {
     together_ai: 'Together AI',
     minimax: 'MiniMax',
     zeabur: 'Zeabur AI Hub',
+    typesafe: 'TypeSafe',
 }
 
 const LLM_PROVIDERS = new Set<string>(Object.keys(LLM_PROVIDER_LABELS))
diff --git a/products/ai_observability/frontend/utils.test.ts b/products/ai_observability/frontend/utils.test.ts
index 8b5a655c07e8..5a119d6472fc 100644
--- a/products/ai_observability/frontend/utils.test.ts
+++ b/products/ai_observability/frontend/utils.test.ts
@@ -75,6 +75,15 @@ function makeEvaluationRunRow({
 }
 
 describe('mapEvaluationRunRow', () => {
+    it.each([0, 0.49, 1])('preserves a Jev probability of %s without inventing reasoning', (probability) => {
+        const row = makeEvaluationRunRow()
+        row[7] = ''
+        row[15] = probability
+        const run = mapEvaluationRunRow(row)
+        expect(run.probability).toBe(probability)
+        expect(run.reasoning).toBe('')
+    })
+
     it('maps sentiment rows without coercing missing boolean results to false', () => {
         const run = mapEvaluationRunRow(
             makeEvaluationRunRow({
diff --git a/products/ai_observability/frontend/utils.ts b/products/ai_observability/frontend/utils.ts
index d0f3d50267e4..896c69cd9321 100644
--- a/products/ai_observability/frontend/utils.ts
+++ b/products/ai_observability/frontend/utils.ts
@@ -1127,6 +1127,7 @@ type RawEvaluationRunRow = [
     sentiment_score: number | string | null,
     session_id: string | null,
     skipped: boolean | string | null,
+    probability?: number | string | null,
 ]
 
 export function normalizeEvaluationType(value: unknown): EvaluationType | undefined {
@@ -1231,7 +1232,8 @@ export function mapEvaluationRunRow(row: RawEvaluationRunRow): EvaluationRun {
         session_id: row[13] || null,
         ...normalizedResult,
         skipped: isExplicitEvaluationPass(row[14]),
-        reasoning: row[7] || 'No reasoning provided',
+        reasoning: row[7] ?? '',
+        probability: normalizeOptionalNumber(row[15]),
         status: 'completed' as const,
     }
 }
@@ -1275,7 +1277,8 @@ export async function queryEvaluationRuns(params: {
             properties.$ai_sentiment_label as sentiment_label,
             properties.$ai_sentiment_score as sentiment_score,
             properties.$ai_session_id as session_id,
-            properties.$ai_evaluation_skipped as skipped
+            properties.$ai_evaluation_skipped as skipped,
+            properties.$ai_evaluation_probability as probability
         FROM events
         WHERE
             event = '$ai_evaluation'
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index a2dc91309343..ec94dfa350c7 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -36051,6 +36051,7 @@ export namespace Schemas {
      * * `together_ai` - Together AI
      * * `minimax` - MiniMax
      * * `zeabur` - Zeabur AI Hub
+     * * `typesafe` - TypeSafe
      */
     export type LLMProviderEnum = typeof LLMProviderEnum[keyof typeof LLMProviderEnum];
 
@@ -36065,6 +36066,7 @@ export namespace Schemas {
       TogetherAi: 'together_ai',
       Minimax: 'minimax',
       Zeabur: 'zeabur',
+      Typesafe: 'typesafe',
     } as const;
 
     /**
@@ -51320,6 +51322,32 @@ export namespace Schemas {
       createdAt: string | null;
     }
 
+    /**
+     * * `openai` - Openai
+     * * `anthropic` - Anthropic
+     * * `gemini` - Gemini
+     * * `openrouter` - Openrouter
+     * * `fireworks` - Fireworks
+     * * `azure_openai` - Azure OpenAI
+     * * `together_ai` - Together AI
+     * * `minimax` - MiniMax
+     * * `zeabur` - Zeabur AI Hub
+     */
+    export type LLMCompletionProviderEnum = typeof LLMCompletionProviderEnum[keyof typeof LLMCompletionProviderEnum];
+
+
+    export const LLMCompletionProviderEnum = {
+      Openai: 'openai',
+      Anthropic: 'anthropic',
+      Gemini: 'gemini',
+      Openrouter: 'openrouter',
+      Fireworks: 'fireworks',
+      AzureOpenai: 'azure_openai',
+      TogetherAi: 'together_ai',
+      Minimax: 'minimax',
+      Zeabur: 'zeabur',
+    } as const;
+
     export interface LLMModelInfo {
       /** Provider-specific model identifier (e.g. 'gpt-4o-mini', 'claude-3-5-sonnet-20241022'). */
       id: string;
@@ -63502,7 +63530,7 @@ export namespace Schemas {
        * * `together_ai` - Together AI
        * * `minimax` - MiniMax
        * * `zeabur` - Zeabur AI Hub */
-      provider: LLMProviderEnum;
+      provider: LLMCompletionProviderEnum;
       /**
          * Provider model identifier to use for this tagger.
          * @maxLength 100
@@ -73486,7 +73514,7 @@ export namespace Schemas {
        * * `together_ai` - Together AI
        * * `minimax` - MiniMax
        * * `zeabur` - Zeabur AI Hub */
-      provider: LLMProviderEnum;
+      provider: LLMCompletionProviderEnum;
       /**
          * Provider model identifier to use for this tagger.
          * @maxLength 100
@@ -106108,6 +106136,7 @@ export namespace Schemas {
       Openai: 'openai',
       Openrouter: 'openrouter',
       TogetherAi: 'together_ai',
+      Typesafe: 'typesafe',
       Zeabur: 'zeabur',
     } as const;
 
diff --git a/services/mcp/src/generated/ai_observability/api.ts b/services/mcp/src/generated/ai_observability/api.ts
index b9908e33273d..d3bae257e86b 100644
--- a/services/mcp/src/generated/ai_observability/api.ts
+++ b/services/mcp/src/generated/ai_observability/api.ts
@@ -740,9 +740,10 @@ export const EvaluationsCreateBody = () => zod
                                 'together_ai',
                                 'minimax',
                                 'zeabur',
+                                'typesafe',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - TypeSafe'
                             ),
                         model: zod.string().max(evaluationsCreateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -966,9 +967,10 @@ export const EvaluationsPartialUpdateBody = () => zod
                                 'together_ai',
                                 'minimax',
                                 'zeabur',
+                                'typesafe',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - TypeSafe'
                             ),
                         model: zod.string().max(evaluationsPartialUpdateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -1477,6 +1479,7 @@ export const LlmAnalyticsModelsRetrieveQueryParams = () => zod.object({
             'openai',
             'openrouter',
             'together_ai',
+            'typesafe',
             'zeabur',
         ])
         .optional()

From 98cf112a6cb78c80a352eadcd33ab05a97f84473 Mon Sep 17 00:00:00 2001
From: "tests-posthog[bot]"
 <250237707+tests-posthog[bot]@users.noreply.github.com>
Date: Mon, 21 Sep 2026 13:18:36 +0000
Subject: [PATCH 02/30] test(mcp): update unit test snapshots

---
 .../__snapshots__/tool-schemas/llma-evaluation-create.json   | 5 +++--
 .../tool-schemas/llma-evaluation-judge-models.json           | 1 +
 .../__snapshots__/tool-schemas/llma-evaluation-update.json   | 5 +++--
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json
index bbcc23cd980e..de7e987e8fce 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json
@@ -112,7 +112,7 @@
                             "type": "string"
                         },
                         "provider": {
-                            "description": "* `openai` - Openai\n* `anthropic` - Anthropic\n* `gemini` - Gemini\n* `openrouter` - Openrouter\n* `fireworks` - Fireworks\n* `azure_openai` - Azure OpenAI\n* `together_ai` - Together AI\n* `minimax` - MiniMax\n* `zeabur` - Zeabur AI Hub",
+                            "description": "* `openai` - Openai\n* `anthropic` - Anthropic\n* `gemini` - Gemini\n* `openrouter` - Openrouter\n* `fireworks` - Fireworks\n* `azure_openai` - Azure OpenAI\n* `together_ai` - Together AI\n* `minimax` - MiniMax\n* `zeabur` - Zeabur AI Hub\n* `typesafe` - TypeSafe",
                             "enum": [
                                 "openai",
                                 "anthropic",
@@ -122,7 +122,8 @@
                                 "azure_openai",
                                 "together_ai",
                                 "minimax",
-                                "zeabur"
+                                "zeabur",
+                                "typesafe"
                             ],
                             "type": "string"
                         },
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-judge-models.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-judge-models.json
index 51d148e109ab..cef5749759f5 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-judge-models.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-judge-models.json
@@ -16,6 +16,7 @@
                 "openai",
                 "openrouter",
                 "together_ai",
+                "typesafe",
                 "zeabur"
             ],
             "type": "string"
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json
index e87591b624f1..e2f6c7616058 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json
@@ -115,7 +115,7 @@
                             "type": "string"
                         },
                         "provider": {
-                            "description": "* `openai` - Openai\n* `anthropic` - Anthropic\n* `gemini` - Gemini\n* `openrouter` - Openrouter\n* `fireworks` - Fireworks\n* `azure_openai` - Azure OpenAI\n* `together_ai` - Together AI\n* `minimax` - MiniMax\n* `zeabur` - Zeabur AI Hub",
+                            "description": "* `openai` - Openai\n* `anthropic` - Anthropic\n* `gemini` - Gemini\n* `openrouter` - Openrouter\n* `fireworks` - Fireworks\n* `azure_openai` - Azure OpenAI\n* `together_ai` - Together AI\n* `minimax` - MiniMax\n* `zeabur` - Zeabur AI Hub\n* `typesafe` - TypeSafe",
                             "enum": [
                                 "openai",
                                 "anthropic",
@@ -125,7 +125,8 @@
                                 "azure_openai",
                                 "together_ai",
                                 "minimax",
-                                "zeabur"
+                                "zeabur",
+                                "typesafe"
                             ],
                             "type": "string"
                         },

From 6703c4dee688da883acfbfb96607f697b409676c Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Fri, 25 Sep 2026 12:46:50 +0200
Subject: [PATCH 03/30] feat(aio): support compatible System One judge
 endpoints

---
 .../internal/ai-observability-judge-inputs.md |  20 +-
 .../ai_observability/evaluation_llm_judge.py  |  21 +-
 .../ai_observability/test_run_evaluation.py   |  32 ++-
 .../backend/api/evaluations.py                |   7 +-
 .../backend/api/provider_keys.py              |  82 +++++++-
 .../ai_observability/backend/api/proxy.py     |  10 +-
 .../backend/api/test/test_evaluations.py      |   6 +-
 .../backend/api/test/test_provider_keys.py    |  65 +++++-
 .../ai_observability/backend/llm/client.py    |   8 +-
 .../backend/llm/system_one.py                 | 158 ++++++++++++++
 .../backend/llm/test/test_system_one.py       | 193 ++++++++++++++++++
 .../backend/llm/test/test_typesafe.py         | 122 -----------
 .../ai_observability/backend/llm/typesafe.py  | 120 -----------
 .../backend/models/model_configuration.py     |   2 +-
 .../backend/models/provider_keys.py           |   6 +-
 .../frontend/generated/api.schemas.ts         |  36 +++-
 .../frontend/generated/api.zod.ts             |  46 ++++-
 .../frontend/modelPickerLogic.test.ts         |  67 +++---
 .../settings/LLMProviderKeysSettings.tsx      |  81 +++++++-
 .../SystemOneConnectionFields.stories.tsx     |  28 +++
 .../settings/SystemOneConnectionFields.tsx    |  48 +++++
 .../settings/llmProviderKeysLogic.test.ts     |   2 +
 .../frontend/settings/llmProviderKeysLogic.ts |  43 +++-
 services/mcp/src/api/generated.ts             |  36 +++-
 .../mcp/src/generated/ai_observability/api.ts |   4 +-
 25 files changed, 904 insertions(+), 339 deletions(-)
 create mode 100644 products/ai_observability/backend/llm/system_one.py
 create mode 100644 products/ai_observability/backend/llm/test/test_system_one.py
 delete mode 100644 products/ai_observability/backend/llm/test/test_typesafe.py
 delete mode 100644 products/ai_observability/backend/llm/typesafe.py
 create mode 100644 products/ai_observability/frontend/settings/SystemOneConnectionFields.stories.tsx
 create mode 100644 products/ai_observability/frontend/settings/SystemOneConnectionFields.tsx

diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index a8682d2eac3e..ef8c8338e7f4 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -34,12 +34,22 @@ They sample the combined input, tool definitions, and output only when that text
 
 Implementation: [trace judge](../../posthog/temporal/ai_observability/run_trace_evaluation.py), [session judge](../../posthog/temporal/ai_observability/run_session_evaluation.py), and [generation judge](../../posthog/temporal/ai_observability/evaluation_llm_judge.py).
 
-## Jev boolean judge
+## System One boolean judges
 
-Jev is available under the existing LLM judge option with a customer-provided TypeSafe API key.
-Select the key and `jev-1.13.0` on each evaluation; TypeSafe keys cannot become the shared active provider key used by other AI features.
+System One-compatible models are available under the existing LLM judge option.
+Add a connection under **System One (Jev)** in provider key settings.
+The default endpoint is TypeSafe's `https://api.typesafe.ai/v1`, with model `jev-1.13.0` and a TypeSafe API key.
+Advanced configuration accepts a different public HTTPS base URL and model ID for compatible services.
+The client appends `/systemone` to the base URL and sends the API key as a bearer token.
+An empty key selects no authentication for a custom endpoint; TypeSafe requires a key.
+Changing the endpoint requires entering its credential again, or explicitly choosing no authentication, so an existing key is not forwarded to a new host.
+Private network destinations and redirects are blocked by the shared DNS-pinned HTTP transport.
+Saving a connection validates it with a short synthetic input and two Noul questions, including applicability, without sending evaluation data.
+Select the connection and configured model on each evaluation; these connections cannot become the shared active provider key used by other AI features.
 Provider keys keep the provider they were created with; switching providers requires a new key.
-Jev supports boolean evaluations only and uses the same formatted text for generation, trace, and session targets.
+This integration supports boolean evaluations only and uses the same formatted text for generation, trace, and session targets.
+API compatibility does not guarantee equivalent judgments or calibration across models.
+Compare results on representative inputs when changing models.
 
 The evaluation prompt becomes a [Noul question](https://docs.typesafe.ai/primitives/noul).
 A probability of at least 0.5 produces `true`; the evaluation's existing pass/fail polarity still applies.
@@ -48,7 +58,7 @@ Uncertainty alone does not produce N/A.
 The raw probability is stored in `$ai_evaluation_probability`, with token usage and the resolved model version.
 Jev provides no written reasoning, so reports inspect the original source when explaining outcomes.
 
-TypeSafe rate limits and overload responses are retried through Temporal, honoring `Retry-After` up to five minutes.
+Rate limits and overload responses are retried through Temporal, honoring `Retry-After` up to five minutes.
 If retries fail, the run fails and the evaluation stays enabled.
 Invalid probabilities or missing answers fail the evaluation rather than producing a false result.
 Inputs rejected for exceeding the model's context window are skipped.
diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.py
index 76386be60251..e95a5da3b76d 100644
--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py
+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py
@@ -46,8 +46,8 @@
     RateLimitError,
     StructuredOutputParseError,
 )
+from products.ai_observability.backend.llm.system_one import SystemOneClient, SystemOneRateLimitError
 from products.ai_observability.backend.llm.types import CompletionResponse, Usage
-from products.ai_observability.backend.llm.typesafe import TypeSafeClient, TypeSafeRateLimitError
 from products.ai_observability.backend.text_repr.formatters import add_line_numbers, reduce_by_uniform_sampling
 
 logger = structlog.get_logger(__name__)
@@ -343,15 +343,18 @@ def call_llm_judge(
         if provider == "typesafe":
             if provider_key is not None and provider_key.provider != provider:
                 raise ProviderMismatchError(provider_key.provider, provider)
-            typesafe_result = TypeSafeClient.evaluate_boolean(
+            system_one_result = SystemOneClient.evaluate_boolean(
                 api_key=provider_key.encrypted_config.get("api_key", "") if provider_key else "",
+                base_url=provider_key.encrypted_config.get("base_url", SystemOneClient.BASE_URL)
+                if provider_key
+                else SystemOneClient.BASE_URL,
                 model=model,
                 prompt=evaluation["evaluation_config"]["prompt"],
                 source=user_prompt,
                 allows_na=allows_na,
             )
-            probability = typesafe_result.answers["verdict"].noul
-            applicable = not allows_na or typesafe_result.answers["applicable"].noul >= 0.5
+            probability = system_one_result.answers["verdict"].noul
+            applicable = not allows_na or system_one_result.answers["applicable"].noul >= 0.5
             parsed: BooleanEvalResult | BooleanWithNAEvalResult = (
                 BooleanWithNAEvalResult(
                     reasoning="", applicable=applicable, verdict=probability >= 0.5 if applicable else None
@@ -359,15 +362,15 @@ def call_llm_judge(
                 if allows_na
                 else BooleanEvalResult(reasoning="", verdict=probability >= 0.5)
             )
-            model = typesafe_result.model
+            model = system_one_result.model
             response = CompletionResponse(
                 content="",
                 model=model,
                 parsed=parsed,
                 usage=Usage(
-                    input_tokens=typesafe_result.usage.input_tokens,
-                    output_tokens=typesafe_result.usage.output_tokens,
-                    total_tokens=typesafe_result.usage.input_tokens + typesafe_result.usage.output_tokens,
+                    input_tokens=system_one_result.usage.input_tokens,
+                    output_tokens=system_one_result.usage.output_tokens,
+                    total_tokens=system_one_result.usage.input_tokens + system_one_result.usage.output_tokens,
                 ),
             )
         else:
@@ -380,7 +383,7 @@ def call_llm_judge(
                     response_format=response_format,
                 )
             )
-    except TypeSafeRateLimitError as e:
+    except SystemOneRateLimitError as e:
         increment_errors("rate_limit", provider=provider)
         raise ApplicationError(
             str(e),
diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index 601c77e6ae70..d1dc346c3d5d 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -84,15 +84,32 @@ def _mock_config_with_active_key(provider: str = "openai") -> MagicMock:
     return MagicMock(active_provider_key=key)
 
 
+@pytest.mark.parametrize(
+    "connection_config,base_url,model",
+    [
+        ({"api_key": "test-typesafe-key"}, "https://api.typesafe.ai/v1", "jev-1.13.0"),
+        (
+            {"api_key": "", "base_url": "https://decisions.example.com/v1"},
+            "https://decisions.example.com/v1",
+            "custom-model",
+        ),
+    ],
+)
 @pytest.mark.parametrize(
     "probability,applicability,allows_na,verdict",
     [(0.49, 1.0, False, False), (0.5, 1.0, False, True), (0.9, 0.1, True, None), (0.0, 0.9, True, False)],
 )
 def test_typesafe_judge_emits_boolean_probability_without_reasoning(
-    probability: float, applicability: float, allows_na: bool, verdict: bool | None
+    probability: float,
+    applicability: float,
+    allows_na: bool,
+    verdict: bool | None,
+    connection_config: dict[str, str],
+    base_url: str,
+    model: str,
 ) -> None:
-    key = MagicMock(provider="typesafe", encrypted_config={"api_key": "test-typesafe-key"})
-    resolved = MagicMock(provider="typesafe", model="jev-1.13.0", provider_key=key, is_byok=True)
+    key = MagicMock(provider="typesafe", encrypted_config=connection_config)
+    resolved = MagicMock(provider="typesafe", model=model, provider_key=key, is_byok=True)
     response = MagicMock(status_code=200)
     response.json.return_value = {
         "model": "jev-1.13.0",
@@ -110,7 +127,7 @@ def test_typesafe_judge_emits_boolean_probability_without_reasoning(
     }
     with (
         patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
-        patch("requests.request", return_value=response),
+        patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response) as request,
     ):
         spec.return_value.resolve.return_value = resolved
         result = call_llm_judge(
@@ -120,6 +137,8 @@ def test_typesafe_judge_emits_boolean_probability_without_reasoning(
             allows_na=allows_na,
         )
 
+    assert request.call_args.args[1] == f"{base_url}/systemone"
+    assert request.call_args.kwargs["json"]["model"] == model
     assert result["verdict"] is verdict
     assert result["reasoning"] == ""
     assert result["probability"] == probability
@@ -136,7 +155,10 @@ def test_typesafe_rate_limit_retries_without_disabling_the_evaluation() -> None:
     key = MagicMock(provider="typesafe", encrypted_config={"api_key": "test-typesafe-key"})
     with (
         patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
-        patch("requests.request", return_value=MagicMock(status_code=429, headers={"Retry-After": "15"})),
+        patch(
+            "products.ai_observability.backend.llm.system_one.pinned_request",
+            return_value=MagicMock(status_code=429, headers={"Retry-After": "15"}),
+        ),
         pytest.raises(ApplicationError) as error,
     ):
         spec.return_value.resolve.return_value = MagicMock(
diff --git a/products/ai_observability/backend/api/evaluations.py b/products/ai_observability/backend/api/evaluations.py
index ccc5c5b84856..5471a8548bc7 100644
--- a/products/ai_observability/backend/api/evaluations.py
+++ b/products/ai_observability/backend/api/evaluations.py
@@ -43,7 +43,6 @@
 from ..evaluation_conditions import build_condition_filter
 from ..hog import compile_ai_observability_hog
 from ..llm import DEFAULT_MODEL_BY_PROVIDER
-from ..llm.typesafe import TypeSafeClient
 from ..models.evaluation_config import EvaluationConfig
 from ..models.evaluation_configs import (
     EVALUATION_TEST_LOOKBACK_DAYS,
@@ -253,10 +252,10 @@ def validate(self, data: dict[str, Any]) -> dict[str, Any]:
         if errors:
             raise serializers.ValidationError(errors, code="required")
         if data["provider"] == LLMProvider.TYPESAFE:
-            if data["model"] != TypeSafeClient.MODEL:
-                raise serializers.ValidationError({"model": "Select a supported Jev model."})
             if not data.get("provider_key_id"):
-                raise serializers.ValidationError({"provider_key_id": "Select a TypeSafe API key for this evaluation."})
+                raise serializers.ValidationError(
+                    {"provider_key_id": "Select a System One connection for this evaluation."}
+                )
         return data
 
     def get_provider_key_name(self, obj: LLMModelConfiguration) -> str | None:
diff --git a/products/ai_observability/backend/api/provider_keys.py b/products/ai_observability/backend/api/provider_keys.py
index c8637d3608af..2289420f4ea4 100644
--- a/products/ai_observability/backend/api/provider_keys.py
+++ b/products/ai_observability/backend/api/provider_keys.py
@@ -30,6 +30,7 @@
     error_field_for_validation_message,
     is_allowed_azure_endpoint,
 )
+from ..llm.system_one import SystemOneClient
 from ..models.evaluation_config import EvaluationConfig
 from ..models.evaluations import Evaluation
 from ..models.model_configuration import LLMModelConfiguration
@@ -90,7 +91,18 @@ def _validation_error_field(provider: str, error_message: str | None) -> str:
 
 
 class LLMProviderKeySerializer(serializers.ModelSerializer):
-    api_key = serializers.CharField(write_only=True, required=False)
+    api_key = serializers.CharField(write_only=True, required=False, allow_blank=True)
+    base_url = serializers.URLField(
+        write_only=True, required=False, help_text="System One API base URL, including /v1. Defaults to TypeSafe."
+    )
+    system_one_model = serializers.CharField(
+        write_only=True,
+        required=False,
+        max_length=100,
+        help_text="Model ID served by the System One endpoint. Defaults to jev-1.13.0.",
+    )
+    base_url_display = serializers.SerializerMethodField(help_text="Configured System One base URL.")
+    system_one_model_display = serializers.SerializerMethodField(help_text="Configured System One model ID.")
     api_key_masked = serializers.SerializerMethodField()
     azure_endpoint = serializers.URLField(write_only=True, required=False, help_text="Azure OpenAI endpoint URL")
     api_version = serializers.CharField(
@@ -111,6 +123,10 @@ class Meta:
             "error_message",
             "api_key",
             "api_key_masked",
+            "base_url",
+            "system_one_model",
+            "base_url_display",
+            "system_one_model_display",
             "azure_endpoint",
             "api_version",
             "azure_endpoint_display",
@@ -125,6 +141,24 @@ class Meta:
     def get_api_key_masked(self, obj: LLMProviderKey) -> str:
         return mask_key_value(obj.encrypted_config.get("api_key", ""))
 
+    def get_base_url_display(self, obj: LLMProviderKey) -> str | None:
+        return (
+            obj.encrypted_config.get("base_url", SystemOneClient.BASE_URL)
+            if obj.provider == LLMProvider.TYPESAFE
+            else None
+        )
+
+    def get_system_one_model_display(self, obj: LLMProviderKey) -> str | None:
+        return (
+            obj.encrypted_config.get("model", SystemOneClient.MODEL) if obj.provider == LLMProvider.TYPESAFE else None
+        )
+
+    def validate_base_url(self, value: str) -> str:
+        try:
+            return SystemOneClient.normalize_base_url(value)
+        except ValueError as error:
+            raise serializers.ValidationError(str(error)) from error
+
     def get_azure_endpoint_display(self, obj: LLMProviderKey) -> str | None:
         if obj.provider != LLMProvider.AZURE_OPENAI:
             return None
@@ -159,6 +193,19 @@ def validate(self, data):
             raise serializers.ValidationError({"api_key": "API key is required when creating a new provider key."})
 
         provider = data.get("provider", getattr(self.instance, "provider", None))
+        if provider != LLMProvider.TYPESAFE:
+            if "base_url" in data or "system_one_model" in data:
+                raise serializers.ValidationError(
+                    {"base_url": "These settings are only available for System One connections."}
+                )
+            if data.get("api_key") == "":
+                raise serializers.ValidationError({"api_key": "An API key is required."})
+        elif self.instance is not None and "base_url" in data:
+            current_url = self.instance.encrypted_config.get("base_url", SystemOneClient.BASE_URL)
+            if data["base_url"] != current_url and "api_key" not in data:
+                raise serializers.ValidationError(
+                    {"api_key": "Enter the credential for the new endpoint, or an empty value for no authentication."}
+                )
         if self.instance is not None and provider != self.instance.provider:
             raise serializers.ValidationError({"provider": "A key's provider cannot change. Create a new key instead."})
         if provider == LLMProvider.TYPESAFE and data.get("set_as_active"):
@@ -176,6 +223,12 @@ def validate(self, data):
 
         return data
 
+    def _system_one_config(self, validated_data: dict, current: dict | None = None) -> dict:
+        config = dict(current or {})
+        config["base_url"] = validated_data.pop("base_url", config.get("base_url", SystemOneClient.BASE_URL))
+        config["model"] = validated_data.pop("system_one_model", config.get("model", SystemOneClient.MODEL))
+        return config
+
     def _pop_azure_kwargs(self, validated_data: dict) -> dict:
         """Pop Azure-specific write-only fields out of ``validated_data`` and return them as kwargs.
 
@@ -217,6 +270,16 @@ def create(self, validated_data):
 
         azure_kwargs = self._normalize_azure_config(provider, azure_kwargs)
 
+        if provider == LLMProvider.TYPESAFE:
+            connection_config = self._system_one_config(validated_data)
+            state, error_message = validate_provider_key(provider, api_key or "", **connection_config)
+            if state != LLMProviderKey.State.OK:
+                raise serializers.ValidationError({"api_key": error_message})
+            validated_data["encrypted_config"] = {"api_key": api_key or "", **connection_config}
+            validated_data["state"] = state
+            validated_data["error_message"] = None
+            return super().create(validated_data)
+
         if api_key:
             state, error_message = validate_provider_key(provider, api_key, **azure_kwargs)
             if state != LLMProviderKey.State.OK:
@@ -236,6 +299,17 @@ def create(self, validated_data):
         return instance
 
     def update(self, instance, validated_data):
+        if instance.provider == LLMProvider.TYPESAFE:
+            if any(field in validated_data for field in ("api_key", "base_url", "system_one_model")):
+                config = self._system_one_config(validated_data, instance.encrypted_config)
+                config["api_key"] = validated_data.pop("api_key", config.get("api_key", ""))
+                state, error_message = validate_provider_key(instance.provider, **config)
+                if state != LLMProviderKey.State.OK:
+                    raise serializers.ValidationError({"api_key": error_message})
+                instance.encrypted_config = config
+                instance.state = state
+                instance.error_message = None
+            return super().update(instance, validated_data)
         api_key = validated_data.pop("api_key", None)
         azure_kwargs = self._normalize_azure_config(instance.provider, self._pop_azure_kwargs(validated_data))
 
@@ -358,13 +432,15 @@ def validate(self, request: Request, **_kwargs) -> Response:
         instance = self.get_object()
         api_key = instance.encrypted_config.get("api_key")
 
-        if not api_key:
+        if not api_key and instance.provider != LLMProvider.TYPESAFE:
             return Response(
                 {"detail": "No API key configured for this provider key."},
                 status=status.HTTP_400_BAD_REQUEST,
             )
 
-        state, error_message = validate_provider_key(instance.provider, api_key, **instance.provider_extra_kwargs())
+        state, error_message = validate_provider_key(
+            instance.provider, api_key or "", **instance.provider_extra_kwargs()
+        )
         instance.state = state
         instance.error_message = error_message
         instance.save(update_fields=["state", "error_message"])
diff --git a/products/ai_observability/backend/api/proxy.py b/products/ai_observability/backend/api/proxy.py
index f1f0d27dc1c6..77bee39e080c 100644
--- a/products/ai_observability/backend/api/proxy.py
+++ b/products/ai_observability/backend/api/proxy.py
@@ -49,7 +49,11 @@
     get_playground_models,
 )
 from products.ai_observability.backend.llm.errors import UnsupportedProviderError
-from products.ai_observability.backend.models.provider_keys import LLMProviderKey, llm_completion_provider_choices
+from products.ai_observability.backend.models.provider_keys import (
+    LLMProvider,
+    LLMProviderKey,
+    llm_completion_provider_choices,
+)
 
 from ee.hogai.utils.asgi import SyncIterableToAsync
 
@@ -63,7 +67,7 @@ def models_cache_key(provider_key_id: str | uuid.UUID) -> str:
 
 
 PROVIDER_DISPLAY_NAMES: dict[str, str] = {
-    "typesafe": "TypeSafe",
+    "typesafe": "System One (Jev)",
     "openai": "OpenAI",
     "anthropic": "Anthropic",
     "gemini": "Gemini",
@@ -190,7 +194,7 @@ def _get_provider_key(
             raise ValueError("Provider key not found")
 
         api_key = key.encrypted_config.get("api_key")
-        if not api_key:
+        if not api_key and key.provider != LLMProvider.TYPESAFE:
             raise ValueError("No API key configured for this provider key")
 
         if touch_last_used:
diff --git a/products/ai_observability/backend/api/test/test_evaluations.py b/products/ai_observability/backend/api/test/test_evaluations.py
index 004d55bc3ea4..64a6eaf10c64 100644
--- a/products/ai_observability/backend/api/test/test_evaluations.py
+++ b/products/ai_observability/backend/api/test/test_evaluations.py
@@ -57,13 +57,11 @@ class TestModelConfigurationSerializer(SimpleTestCase):
     @parameterized.expand(
         [
             ("missing_key", "jev-1.13.0", None, False),
-            ("unsupported_model", "other-model", str(uuid4()), False),
+            ("custom_model", "other-model", str(uuid4()), True),
             ("configured", "jev-1.13.0", str(uuid4()), True),
         ]
     )
-    def test_typesafe_requires_supported_model_and_explicit_key(
-        self, _name: str, model: str, key_id: str | None, valid: bool
-    ) -> None:
+    def test_system_one_requires_explicit_key(self, _name: str, model: str, key_id: str | None, valid: bool) -> None:
         serializer = ModelConfigurationSerializer(
             data={"provider": "typesafe", "model": model, "provider_key_id": key_id}
         )
diff --git a/products/ai_observability/backend/api/test/test_provider_keys.py b/products/ai_observability/backend/api/test/test_provider_keys.py
index f840c2af3209..3ef5deaa9d5f 100644
--- a/products/ai_observability/backend/api/test/test_provider_keys.py
+++ b/products/ai_observability/backend/api/test/test_provider_keys.py
@@ -17,7 +17,9 @@
 from products.ai_observability.backend.api.provider_keys import LLMProviderKeySerializer
 from products.ai_observability.backend.api.proxy import LLMProxyCompletionSerializer, models_cache_key
 from products.ai_observability.backend.api.taggers import TaggerModelConfigurationWriteSerializer
+from products.ai_observability.backend.llm.client import Client
 from products.ai_observability.backend.llm.providers.azure_openai import DEFAULT_API_VERSION
+from products.ai_observability.backend.llm.system_one import SystemOneClient
 from products.ai_observability.backend.models.evaluation_config import EvaluationConfig
 from products.ai_observability.backend.models.evaluations import Evaluation
 from products.ai_observability.backend.models.model_configuration import LLMModelConfiguration
@@ -54,6 +56,12 @@ def test_typesafe_cannot_be_created_as_the_shared_key(self) -> None:
         self.assertFalse(serializer.is_valid())
         self.assertIn("set_as_active", serializer.errors)
 
+    @parameterized.expand([("https://decisions.example.com/v1", False), (SystemOneClient.BASE_URL, True)])
+    def test_endpoint_change_requires_explicit_credentials(self, base_url: str, valid: bool) -> None:
+        key = LLMProviderKey(provider="typesafe", encrypted_config={"api_key": "example-token"})
+        serializer = LLMProviderKeySerializer(key, data={"base_url": base_url}, partial=True)
+        self.assertEqual(serializer.is_valid(), valid, serializer.errors)
+
 
 def _setup_team():
     org = Organization.objects.create(name="test")
@@ -140,7 +148,62 @@ def test_can_create_provider_key(self, provider: str, mock_validate: Mock) -> No
 
         self.assertEqual(response.data["api_key_masked"], "sk-t...2345")
         self.assertNotIn("api_key", response.data)
-        mock_validate.assert_called_once_with(provider, "sk-test-key-12345")
+        expected_config = (
+            {"base_url": SystemOneClient.BASE_URL, "model": SystemOneClient.MODEL} if provider == "typesafe" else {}
+        )
+        mock_validate.assert_called_once_with(provider, "sk-test-key-12345", **expected_config)
+
+    @patch("products.ai_observability.backend.llm.system_one.pinned_request")
+    def test_custom_system_one_connection_round_trip(self, request: Mock) -> None:
+        request.return_value = Mock(status_code=200)
+        request.return_value.json.return_value = {
+            "model": "custom-model",
+            "answers": {"verdict": {"type": "noul", "noul": 1.0}, "applicable": {"type": "noul", "noul": 1.0}},
+            "usage": {"input_tokens": 12, "output_tokens": 0},
+        }
+        url = f"/api/environments/{self.team.id}/llm_analytics/provider_keys/"
+        response = self.client.post(
+            url,
+            {
+                "provider": "typesafe",
+                "name": "Custom",
+                "api_key": "",
+                "base_url": "https://decisions.example.com/v1/",
+                "system_one_model": "custom-model",
+            },
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertEqual(response.data["base_url_display"], "https://decisions.example.com/v1")
+        key = LLMProviderKey.objects.get(id=response.data["id"])
+        self.assertEqual(Client.list_models("typesafe", **key.provider_extra_kwargs()), ["custom-model"])
+        model_config = LLMModelConfiguration(provider="typesafe", model="custom-model", provider_key=key)
+        self.assertEqual(model_config.get_available_models(), ["custom-model"])
+        self.assertEqual(request.call_args.kwargs["headers"], {})
+        request.reset_mock()
+        response = self.client.patch(f"{url}{key.id}/", {"base_url": "https://other.example.com/v1"})
+        self.assertEqual(response.status_code, 400)
+        request.assert_not_called()
+        key.refresh_from_db()
+        self.assertEqual(key.encrypted_config["base_url"], "https://decisions.example.com/v1")
+        request.return_value.status_code = 401
+        response = self.client.patch(
+            f"{url}{key.id}/", {"base_url": "https://other.example.com/v1", "api_key": "fake-token"}
+        )
+        self.assertEqual(response.status_code, 400)
+        key.refresh_from_db()
+        self.assertEqual(key.encrypted_config["base_url"], "https://decisions.example.com/v1")
+        self.assertEqual(key.encrypted_config["api_key"], "")
+        request.return_value.status_code = 200
+        response = self.client.patch(
+            f"{url}{key.id}/", {"base_url": "https://other.example.com/v1", "api_key": "fake-token"}
+        )
+        self.assertEqual(response.status_code, 200, response.data)
+        key.refresh_from_db()
+        self.assertEqual(key.encrypted_config["base_url"], "https://other.example.com/v1")
+        self.assertEqual(request.call_args.kwargs["headers"], {"Authorization": "Bearer fake-token"})
+        response = self.client.post(f"{url}{key.id}/validate/")
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(request.call_args.args[1], "https://other.example.com/v1/systemone")
 
     @patch("products.ai_observability.backend.api.provider_keys.validate_provider_key")
     def test_can_create_provider_key_with_set_as_active(self, mock_validate):
diff --git a/products/ai_observability/backend/llm/client.py b/products/ai_observability/backend/llm/client.py
index ad85f43a247a..6227f80a2b01 100644
--- a/products/ai_observability/backend/llm/client.py
+++ b/products/ai_observability/backend/llm/client.py
@@ -9,13 +9,13 @@
 from typing import TYPE_CHECKING, Any
 
 from products.ai_observability.backend.llm.errors import ProviderMismatchError, UnsupportedProviderError
+from products.ai_observability.backend.llm.system_one import SystemOneClient
 from products.ai_observability.backend.llm.types import (
     AnalyticsContext,
     CompletionRequest,
     CompletionResponse,
     StreamChunk,
 )
-from products.ai_observability.backend.llm.typesafe import TypeSafeClient
 
 if TYPE_CHECKING:
     from products.ai_observability.backend.llm.config import ProviderConfig
@@ -83,21 +83,21 @@ def _resolve_credentials(self) -> tuple[str | None, str | None]:
     def validate_key(cls, provider: str, api_key: str, **kwargs: Any) -> tuple[str, str | None]:
         """Validate an API key for a provider. Returns (state, error_message)."""
         if provider == "typesafe":
-            return TypeSafeClient.validate_key(api_key)
+            return SystemOneClient.validate_key(api_key, **kwargs)
         return _get_provider(provider).validate_key(api_key, **kwargs)
 
     @classmethod
     def list_models(cls, provider: str, api_key: str | None = None, **kwargs: Any) -> list[str]:
         """List available models for a provider."""
         if provider == "typesafe":
-            return [TypeSafeClient.MODEL]
+            return [kwargs.get("model", SystemOneClient.MODEL)]
         return _get_provider(provider).list_models(api_key, **kwargs)
 
     @classmethod
     def recommended_models(cls, provider: str) -> set[str]:
         """Return the set of curated/recommended model IDs for a provider."""
         if provider == "typesafe":
-            return {TypeSafeClient.MODEL}
+            return {SystemOneClient.MODEL}
         return _get_provider(provider).recommended_models()
 
 
diff --git a/products/ai_observability/backend/llm/system_one.py b/products/ai_observability/backend/llm/system_one.py
new file mode 100644
index 000000000000..76b74da2dca8
--- /dev/null
+++ b/products/ai_observability/backend/llm/system_one.py
@@ -0,0 +1,158 @@
+import math
+from datetime import UTC, datetime
+from email.utils import parsedate_to_datetime
+from typing import Literal
+from urllib.parse import urlsplit
+
+import requests
+from pydantic import BaseModel, Field, ValidationError
+
+from posthog.security.pinned_requests import SSRFBlockedError, pinned_request
+from posthog.security.url_validation import has_authority_bypass_chars
+
+from products.ai_observability.backend.llm.errors import (
+    AuthenticationError,
+    ContextWindowExceededError,
+    ModelNotFoundError,
+    ModelPermissionError,
+    ProviderConnectionError,
+    RateLimitError,
+    StructuredOutputParseError,
+    is_context_window_error_message,
+)
+
+
+class NoulAnswer(BaseModel):
+    type: Literal["noul"]
+    noul: float = Field(strict=True, ge=0, le=1, allow_inf_nan=False)
+
+
+class SystemOneUsage(BaseModel):
+    input_tokens: int = Field(strict=True, ge=0)
+    output_tokens: int = Field(strict=True, ge=0)
+
+
+class SystemOneResponse(BaseModel):
+    model: str = Field(min_length=1)
+    answers: dict[str, NoulAnswer]
+    usage: SystemOneUsage
+
+
+class SystemOneRateLimitError(RateLimitError):
+    def __init__(self, retry_after: str | None) -> None:
+        super().__init__("The System One endpoint is busy. Try again later.")
+        self.retry_after: float | None = None
+        if retry_after:
+            try:
+                delay = float(retry_after)
+            except ValueError:
+                try:
+                    delay = (parsedate_to_datetime(retry_after) - datetime.now(UTC)).total_seconds()
+                except (ValueError, TypeError, OverflowError):
+                    return
+            if math.isfinite(delay):
+                self.retry_after = max(1, min(delay, 300))
+
+
+class SystemOneClient:
+    MODEL = "jev-1.13.0"
+    BASE_URL = "https://api.typesafe.ai/v1"
+
+    @staticmethod
+    def normalize_base_url(base_url: str) -> str:
+        parsed = urlsplit(base_url)
+        if (
+            has_authority_bypass_chars(base_url)
+            or parsed.scheme != "https"
+            or not parsed.hostname
+            or parsed.username is not None
+            or parsed.password is not None
+            or parsed.query
+            or parsed.fragment
+        ):
+            raise ValueError("Use an HTTPS base URL without credentials, a query, or a fragment.")
+        return base_url.rstrip("/")
+
+    @staticmethod
+    def evaluate_boolean(
+        *, api_key: str, model: str, prompt: str, source: str, allows_na: bool, base_url: str = BASE_URL
+    ) -> SystemOneResponse:
+        base_url = SystemOneClient.normalize_base_url(base_url)
+        if not api_key and base_url == SystemOneClient.BASE_URL:
+            raise AuthenticationError("A TypeSafe API key is required.")
+        questions = {"verdict": {"type": "noul", "instructions": prompt}}
+        if allows_na:
+            questions["applicable"] = {
+                "type": "noul",
+                "instructions": (
+                    "Do these evaluation criteria apply to this input? Answer true when the criteria can be "
+                    "evaluated, even if they are not met. Answer false only when they are not relevant.\n\n" + prompt
+                ),
+            }
+        try:
+            response = pinned_request(
+                "POST",
+                f"{base_url}/systemone",
+                headers={"Authorization": f"Bearer {api_key}"} if api_key else {},
+                json={"model": model, "state": source, "questions": questions},
+                timeout=60,
+            )
+        except SSRFBlockedError as error:
+            raise StructuredOutputParseError("This endpoint is not allowed. Use a public HTTPS endpoint.") from error
+        except requests.RequestException as error:
+            raise ProviderConnectionError("Could not reach the System One endpoint. Try again.") from error
+
+        if response.status_code == 401:
+            raise AuthenticationError("The endpoint rejected this credential. Check the bearer token.")
+        if response.status_code == 403:
+            raise ModelPermissionError(model)
+        if response.status_code == 404:
+            raise ModelNotFoundError(model)
+        if response.status_code in (429, 503, 529):
+            raise SystemOneRateLimitError(response.headers.get("Retry-After"))
+        if response.status_code >= 500:
+            raise ProviderConnectionError("The System One endpoint is temporarily unavailable. Try again.")
+        if response.status_code == 413 or (
+            response.status_code == 422 and is_context_window_error_message(response.text)
+        ):
+            raise ContextWindowExceededError("This input exceeds the endpoint's size limit. Reduce the input.")
+        if response.status_code != 200:
+            raise StructuredOutputParseError(
+                "The endpoint rejected the evaluation request. Check the model and criteria."
+            )
+        try:
+            result = SystemOneResponse.model_validate(response.json())
+        except (ValidationError, ValueError) as error:
+            raise StructuredOutputParseError(
+                "The endpoint returned an invalid System One response. Check compatibility."
+            ) from error
+        # boffin: Missing answers cannot become a false verdict.
+        if not questions.keys() <= result.answers.keys():
+            raise StructuredOutputParseError(
+                "The endpoint did not answer every evaluation question. Check compatibility."
+            )
+        return result
+
+    @staticmethod
+    def validate_key(api_key: str, *, base_url: str = BASE_URL, model: str = MODEL) -> tuple[str, str | None]:
+        try:
+            SystemOneClient.evaluate_boolean(
+                api_key=api_key,
+                base_url=base_url,
+                model=model,
+                prompt="Does the text contain a greeting?",
+                source="Hello!",
+                allows_na=True,
+            )
+        except (AuthenticationError, ModelPermissionError) as error:
+            return "invalid", str(error)
+        except (
+            ValueError,
+            ModelNotFoundError,
+            ProviderConnectionError,
+            RateLimitError,
+            StructuredOutputParseError,
+            ContextWindowExceededError,
+        ) as error:
+            return "error", str(error)
+        return "ok", None
diff --git a/products/ai_observability/backend/llm/test/test_system_one.py b/products/ai_observability/backend/llm/test/test_system_one.py
new file mode 100644
index 000000000000..1f56f2cad602
--- /dev/null
+++ b/products/ai_observability/backend/llm/test/test_system_one.py
@@ -0,0 +1,193 @@
+import pytest
+from unittest.mock import Mock, patch
+
+from django.test import override_settings
+
+from products.ai_observability.backend.llm.client import Client
+from products.ai_observability.backend.llm.errors import (
+    AuthenticationError,
+    ContextWindowExceededError,
+    ModelPermissionError,
+    ProviderConnectionError,
+    StructuredOutputParseError,
+)
+from products.ai_observability.backend.llm.system_one import SystemOneClient, SystemOneRateLimitError
+
+
+@pytest.mark.parametrize("status, expected_state", [(200, "ok"), (401, "invalid"), (403, "invalid"), (500, "error")])
+def test_typesafe_key_validation(status: int, expected_state: str) -> None:
+    response = Mock(status_code=status)
+    response.json.return_value = {
+        "model": "jev-1.13.0",
+        "answers": {"verdict": {"type": "noul", "noul": 0.9}, "applicable": {"type": "noul", "noul": 0.9}},
+        "usage": {"input_tokens": 12, "output_tokens": 0},
+    }
+    with patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response) as request:
+        state, message = Client.validate_key("typesafe", "test-typesafe-key")
+
+    assert state == expected_state
+    assert (message is None) == (expected_state == "ok")
+    assert request.call_args.args == ("POST", "https://api.typesafe.ai/v1/systemone")
+    assert request.call_args.kwargs["headers"]["Authorization"] == "Bearer test-typesafe-key"
+
+
+@pytest.mark.parametrize("allows_na", [False, True])
+def test_typesafe_boolean_request(allows_na: bool) -> None:
+    response = Mock(status_code=200)
+    response.json.return_value = {
+        "model": "jev-1.13.0",
+        "answers": {"verdict": {"type": "noul", "noul": 0.8}, "applicable": {"type": "noul", "noul": 0.2}},
+        "usage": {"input_tokens": 120, "output_tokens": 10},
+    }
+    with patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response) as request:
+        result = SystemOneClient.evaluate_boolean(
+            api_key="test-typesafe-key",
+            model="jev-1.13.0",
+            prompt="Is the response polite?",
+            source="Hello!",
+            allows_na=allows_na,
+        )
+
+    assert result.answers["verdict"].noul == 0.8
+    body = request.call_args.kwargs["json"]
+    assert body["state"] == "Hello!"
+    assert body["questions"]["verdict"] == {"type": "noul", "instructions": "Is the response polite?"}
+    assert ("applicable" in body["questions"]) == allows_na
+    assert request.call_args.args[1] == "https://api.typesafe.ai/v1/systemone"
+
+
+@pytest.mark.parametrize("probability", [-0.1, 1.1, float("nan"), float("inf"), "0.8", True, None])
+def test_typesafe_rejects_invalid_probabilities(probability: object) -> None:
+    response = Mock(status_code=200)
+    response.json.return_value = {
+        "model": "jev-1.13.0",
+        "answers": {"verdict": {"type": "noul", "noul": probability}},
+        "usage": {"input_tokens": 120, "output_tokens": 10},
+    }
+    with (
+        patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response),
+        pytest.raises(StructuredOutputParseError),
+    ):
+        SystemOneClient.evaluate_boolean(
+            api_key="test-typesafe-key",
+            model="jev-1.13.0",
+            prompt="Is the response polite?",
+            source="Hello!",
+            allows_na=False,
+        )
+
+
+@pytest.mark.parametrize("status", [429, 503, 529])
+def test_typesafe_rate_limits_are_retryable(status: int) -> None:
+    response = Mock(status_code=status, headers={"Retry-After": "15"})
+    with (
+        patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response),
+        pytest.raises(SystemOneRateLimitError) as error,
+    ):
+        SystemOneClient.evaluate_boolean(
+            api_key="test-typesafe-key",
+            model="jev-1.13.0",
+            prompt="Is the response polite?",
+            source="Hello!",
+            allows_na=False,
+        )
+    assert error.value.retry_after == 15
+
+
+def test_typesafe_requires_a_key() -> None:
+    with (
+        patch("products.ai_observability.backend.llm.system_one.pinned_request") as request,
+        pytest.raises(AuthenticationError),
+    ):
+        SystemOneClient.evaluate_boolean(
+            api_key="", model="jev-1.13.0", prompt="Polite?", source="Hello!", allows_na=False
+        )
+    request.assert_not_called()
+
+
+@pytest.mark.parametrize("answers", [{}, {"verdict": {"type": "noul", "noul": 0.9}}])
+def test_typesafe_requires_every_requested_answer(answers: dict[str, object]) -> None:
+    response = Mock(status_code=200)
+    response.json.return_value = {
+        "model": "jev-1.13.0",
+        "answers": answers,
+        "usage": {"input_tokens": 12, "output_tokens": 2},
+    }
+    with (
+        patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response),
+        pytest.raises(StructuredOutputParseError),
+    ):
+        SystemOneClient.evaluate_boolean(
+            api_key="test-typesafe-key", model="jev-1.13.0", prompt="Polite?", source="Hello!", allows_na=True
+        )
+
+
+@pytest.mark.parametrize(
+    "status,message,error_type",
+    [
+        (401, "Invalid key", AuthenticationError),
+        (403, "Access denied", ModelPermissionError),
+        (500, "Unavailable", ProviderConnectionError),
+        (413, "Request too large", ContextWindowExceededError),
+        (422, "Input exceeds the context window", ContextWindowExceededError),
+        (422, "Invalid question", StructuredOutputParseError),
+    ],
+)
+def test_typesafe_preserves_error_categories(status: int, message: str, error_type: type[Exception]) -> None:
+    response = Mock(status_code=status, text=message)
+    with (
+        patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response),
+        pytest.raises(error_type),
+    ):
+        SystemOneClient.evaluate_boolean(
+            api_key="test-typesafe-key", model="jev-1.13.0", prompt="Polite?", source="Hello!", allows_na=False
+        )
+
+
+@pytest.mark.parametrize("api_key", ["example-token", ""])
+def test_custom_endpoint_and_model(api_key: str) -> None:
+    response = Mock(status_code=200)
+    response.json.return_value = {
+        "model": "custom-model-revision",
+        "answers": {"verdict": {"type": "noul", "noul": 0.7}, "applicable": {"type": "noul", "noul": 1.0}},
+        "usage": {"input_tokens": 15, "output_tokens": 0},
+        "latency_ms": 42,
+    }
+    with patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response) as request:
+        result = SystemOneClient.evaluate_boolean(
+            api_key=api_key,
+            base_url="https://decisions.example.com/v1/",
+            model="custom-model",
+            prompt="Polite?",
+            source="Hello!",
+            allows_na=True,
+        )
+    assert request.call_args.args == ("POST", "https://decisions.example.com/v1/systemone")
+    assert request.call_args.kwargs["headers"] == ({"Authorization": f"Bearer {api_key}"} if api_key else {})
+    assert request.call_args.kwargs["json"]["model"] == "custom-model"
+    assert result.model == "custom-model-revision"
+    assert result.usage.output_tokens == 0
+    assert result.answers["verdict"].noul == 0.7
+
+
+@pytest.mark.parametrize(
+    "base_url",
+    [
+        "http://example.com/v1",
+        "https://user:secret@example.com/v1",
+        "https://example.com/v1?token=secret",
+        "https://example.com/v1#fragment",
+    ],
+)
+def test_invalid_endpoint_is_rejected_before_sending_credentials(base_url: str) -> None:
+    with patch("products.ai_observability.backend.llm.system_one.pinned_request") as request:
+        state, _ = SystemOneClient.validate_key("example-token", base_url=base_url)
+    assert state == "error"
+    request.assert_not_called()
+
+
+def test_private_endpoint_is_blocked() -> None:
+    with override_settings(DEBUG=False, TEST=False), patch("requests.Session.request") as request:
+        state, _ = SystemOneClient.validate_key("example-token", base_url="https://127.0.0.1/v1")
+    assert state == "error"
+    request.assert_not_called()
diff --git a/products/ai_observability/backend/llm/test/test_typesafe.py b/products/ai_observability/backend/llm/test/test_typesafe.py
deleted file mode 100644
index 7815eccd0188..000000000000
--- a/products/ai_observability/backend/llm/test/test_typesafe.py
+++ /dev/null
@@ -1,122 +0,0 @@
-import pytest
-from unittest.mock import Mock, patch
-
-from products.ai_observability.backend.llm.client import Client
-from products.ai_observability.backend.llm.errors import (
-    AuthenticationError,
-    ContextWindowExceededError,
-    ModelPermissionError,
-    ProviderConnectionError,
-    StructuredOutputParseError,
-)
-from products.ai_observability.backend.llm.typesafe import TypeSafeClient, TypeSafeRateLimitError
-
-
-@pytest.mark.parametrize("status, expected_state", [(200, "ok"), (401, "invalid"), (403, "invalid"), (500, "error")])
-def test_typesafe_key_validation(status: int, expected_state: str) -> None:
-    response = Mock(status_code=status)
-    response.json.return_value = {"models": [{"name": "jev-latest"}]}
-    with patch("requests.request", return_value=response) as request:
-        state, message = Client.validate_key("typesafe", "test-typesafe-key")
-
-    assert state == expected_state
-    assert (message is None) == (expected_state == "ok")
-    assert request.call_args.args == ("GET", "https://api.typesafe.ai/v1/models")
-    assert request.call_args.kwargs["headers"]["Authorization"] == "Bearer test-typesafe-key"
-
-
-@pytest.mark.parametrize("allows_na", [False, True])
-def test_typesafe_boolean_request(allows_na: bool) -> None:
-    response = Mock(status_code=200)
-    response.json.return_value = {
-        "model": "jev-1.13.0",
-        "answers": {"verdict": {"type": "noul", "noul": 0.8}, "applicable": {"type": "noul", "noul": 0.2}},
-        "usage": {"input_tokens": 120, "output_tokens": 10},
-    }
-    with patch("requests.request", return_value=response) as request:
-        result = TypeSafeClient.evaluate_boolean(
-            api_key="test-typesafe-key",
-            model="jev-1.13.0",
-            prompt="Is the response polite?",
-            source="Hello!",
-            allows_na=allows_na,
-        )
-
-    assert result.answers["verdict"].noul == 0.8
-    body = request.call_args.kwargs["json"]
-    assert body["state"] == "Hello!"
-    assert body["questions"]["verdict"] == {"type": "noul", "instructions": "Is the response polite?"}
-    assert ("applicable" in body["questions"]) == allows_na
-    assert request.call_args.kwargs["allow_redirects"] is False
-
-
-@pytest.mark.parametrize("probability", [-0.1, 1.1, float("nan"), float("inf"), "0.8", True, None])
-def test_typesafe_rejects_invalid_probabilities(probability: object) -> None:
-    response = Mock(status_code=200)
-    response.json.return_value = {
-        "model": "jev-1.13.0",
-        "answers": {"verdict": {"type": "noul", "noul": probability}},
-        "usage": {"input_tokens": 120, "output_tokens": 10},
-    }
-    with patch("requests.request", return_value=response), pytest.raises(StructuredOutputParseError):
-        TypeSafeClient.evaluate_boolean(
-            api_key="test-typesafe-key",
-            model="jev-1.13.0",
-            prompt="Is the response polite?",
-            source="Hello!",
-            allows_na=False,
-        )
-
-
-@pytest.mark.parametrize("status", [429, 529])
-def test_typesafe_rate_limits_are_retryable(status: int) -> None:
-    response = Mock(status_code=status, headers={"Retry-After": "15"})
-    with patch("requests.request", return_value=response), pytest.raises(TypeSafeRateLimitError) as error:
-        TypeSafeClient.evaluate_boolean(
-            api_key="test-typesafe-key",
-            model="jev-1.13.0",
-            prompt="Is the response polite?",
-            source="Hello!",
-            allows_na=False,
-        )
-    assert error.value.retry_after == 15
-
-
-def test_typesafe_requires_a_key() -> None:
-    with patch("requests.request") as request, pytest.raises(AuthenticationError):
-        TypeSafeClient.evaluate_boolean(
-            api_key="", model="jev-1.13.0", prompt="Polite?", source="Hello!", allows_na=False
-        )
-    request.assert_not_called()
-
-
-@pytest.mark.parametrize("answers", [{}, {"verdict": {"type": "noul", "noul": 0.9}}])
-def test_typesafe_requires_every_requested_answer(answers: dict[str, object]) -> None:
-    response = Mock(status_code=200)
-    response.json.return_value = {
-        "model": "jev-1.13.0",
-        "answers": answers,
-        "usage": {"input_tokens": 12, "output_tokens": 2},
-    }
-    with patch("requests.request", return_value=response), pytest.raises(StructuredOutputParseError):
-        TypeSafeClient.evaluate_boolean(
-            api_key="test-typesafe-key", model="jev-1.13.0", prompt="Polite?", source="Hello!", allows_na=True
-        )
-
-
-@pytest.mark.parametrize(
-    "status,message,error_type",
-    [
-        (401, "Invalid key", AuthenticationError),
-        (403, "Access denied", ModelPermissionError),
-        (500, "Unavailable", ProviderConnectionError),
-        (422, "Input exceeds the context window", ContextWindowExceededError),
-        (422, "Invalid question", StructuredOutputParseError),
-    ],
-)
-def test_typesafe_preserves_error_categories(status: int, message: str, error_type: type[Exception]) -> None:
-    response = Mock(status_code=status, text=message)
-    with patch("requests.request", return_value=response), pytest.raises(error_type):
-        TypeSafeClient.evaluate_boolean(
-            api_key="test-typesafe-key", model="jev-1.13.0", prompt="Polite?", source="Hello!", allows_na=False
-        )
diff --git a/products/ai_observability/backend/llm/typesafe.py b/products/ai_observability/backend/llm/typesafe.py
deleted file mode 100644
index a005e3389fab..000000000000
--- a/products/ai_observability/backend/llm/typesafe.py
+++ /dev/null
@@ -1,120 +0,0 @@
-import math
-from datetime import UTC, datetime
-from email.utils import parsedate_to_datetime
-from typing import Literal
-
-import requests
-from pydantic import BaseModel, Field, ValidationError
-
-from products.ai_observability.backend.llm.errors import (
-    AuthenticationError,
-    ContextWindowExceededError,
-    ModelNotFoundError,
-    ModelPermissionError,
-    ProviderConnectionError,
-    RateLimitError,
-    StructuredOutputParseError,
-    is_context_window_error_message,
-)
-
-
-class NoulAnswer(BaseModel):
-    type: Literal["noul"]
-    noul: float = Field(strict=True, ge=0, le=1, allow_inf_nan=False)
-
-
-class TypeSafeUsage(BaseModel):
-    input_tokens: int = Field(strict=True, ge=0)
-    output_tokens: int = Field(strict=True, ge=0)
-
-
-class TypeSafeResponse(BaseModel):
-    model: str = Field(min_length=1)
-    answers: dict[str, NoulAnswer]
-    usage: TypeSafeUsage
-
-
-class TypeSafeRateLimitError(RateLimitError):
-    def __init__(self, retry_after: str | None) -> None:
-        super().__init__("TypeSafe is temporarily rate limiting requests.")
-        self.retry_after: float | None = None
-        if retry_after:
-            try:
-                delay = float(retry_after)
-            except ValueError:
-                try:
-                    delay = (parsedate_to_datetime(retry_after) - datetime.now(UTC)).total_seconds()
-                except (ValueError, TypeError, OverflowError):
-                    return
-            if math.isfinite(delay):
-                self.retry_after = max(1, min(delay, 300))
-
-
-class TypeSafeClient:
-    MODEL = "jev-1.13.0"
-
-    @staticmethod
-    def evaluate_boolean(*, api_key: str, model: str, prompt: str, source: str, allows_na: bool) -> TypeSafeResponse:
-        if not api_key:
-            raise AuthenticationError("A TypeSafe API key is required.")
-        questions = {"verdict": {"type": "noul", "instructions": prompt}}
-        if allows_na:
-            questions["applicable"] = {
-                "type": "noul",
-                "instructions": (
-                    "Do these evaluation criteria apply to this input? Answer true when the criteria can be "
-                    "evaluated, even if they are not met. Answer false only when they are not relevant.\n\n" + prompt
-                ),
-            }
-        try:
-            response = requests.request(
-                "POST",
-                "https://api.typesafe.ai/v1/systemone",
-                headers={"Authorization": f"Bearer {api_key}"},
-                json={"model": model, "state": source, "questions": questions},
-                timeout=60,
-                allow_redirects=False,
-            )
-        except requests.RequestException as error:
-            raise ProviderConnectionError("Could not reach TypeSafe.") from error
-
-        if response.status_code == 401:
-            raise AuthenticationError("TypeSafe rejected this API key.")
-        if response.status_code == 403:
-            raise ModelPermissionError(model)
-        if response.status_code == 404:
-            raise ModelNotFoundError(model)
-        if response.status_code in (429, 529):
-            raise TypeSafeRateLimitError(response.headers.get("Retry-After"))
-        if response.status_code >= 500:
-            raise ProviderConnectionError("TypeSafe is temporarily unavailable.")
-        if response.status_code == 422 and is_context_window_error_message(response.text):
-            raise ContextWindowExceededError("This input exceeds Jev's context window.")
-        if response.status_code != 200:
-            raise StructuredOutputParseError("TypeSafe rejected the evaluation request. Check the model and criteria.")
-        try:
-            result = TypeSafeResponse.model_validate(response.json())
-        except (ValidationError, ValueError) as error:
-            raise StructuredOutputParseError("TypeSafe returned an invalid evaluation response.") from error
-        # boffin: Missing answers cannot become a false verdict.
-        if not questions.keys() <= result.answers.keys():
-            raise StructuredOutputParseError("TypeSafe did not answer every evaluation question.")
-        return result
-
-    @staticmethod
-    def validate_key(api_key: str) -> tuple[str, str | None]:
-        try:
-            response = requests.request(
-                "GET",
-                "https://api.typesafe.ai/v1/models",
-                headers={"Authorization": f"Bearer {api_key}"},
-                timeout=30,
-                allow_redirects=False,
-            )
-        except requests.RequestException:
-            return "error", "Could not reach TypeSafe. Try again."
-        if response.status_code == 200:
-            return "ok", None
-        if response.status_code in (401, 403):
-            return "invalid", "TypeSafe rejected this API key. Check the key and try again."
-        return "error", "Could not validate the TypeSafe key. Try again."
diff --git a/products/ai_observability/backend/models/model_configuration.py b/products/ai_observability/backend/models/model_configuration.py
index 1c7d199bdc27..7babb8fba51b 100644
--- a/products/ai_observability/backend/models/model_configuration.py
+++ b/products/ai_observability/backend/models/model_configuration.py
@@ -60,7 +60,7 @@ def get_available_models(self) -> list[str]:
             from products.ai_observability.backend.llm.client import Client
 
             api_key = self.provider_key.encrypted_config.get("api_key")
-            return Client.list_models(self.provider, api_key)
+            return Client.list_models(self.provider, api_key, **self.provider_key.provider_extra_kwargs())
 
         from products.ai_observability.backend.llm import PLAYGROUND_MODELS_BY_PROVIDER
 
diff --git a/products/ai_observability/backend/models/provider_keys.py b/products/ai_observability/backend/models/provider_keys.py
index b51f27473086..f2077b51f7b1 100644
--- a/products/ai_observability/backend/models/provider_keys.py
+++ b/products/ai_observability/backend/models/provider_keys.py
@@ -22,7 +22,7 @@ class LLMProvider(models.TextChoices):
     TOGETHER_AI = "together_ai", "Together AI"
     MINIMAX = "minimax", "MiniMax"
     ZEABUR = "zeabur", "Zeabur AI Hub"
-    TYPESAFE = "typesafe", "TypeSafe"
+    TYPESAFE = "typesafe", "System One (Jev)"
 
 
 def llm_provider_choices() -> list[tuple[str, str | Promise]]:
@@ -67,6 +67,10 @@ def provider_extra_kwargs(self) -> dict[str, Any]:
         extra config (e.g. Azure's ``azure_endpoint`` and ``api_version``). Most
         providers return an empty dict.
         """
+        if self.provider == LLMProvider.TYPESAFE:
+            return {
+                field: self.encrypted_config[field] for field in ("base_url", "model") if field in self.encrypted_config
+            }
         if self.provider == LLMProvider.AZURE_OPENAI:
             return {
                 "azure_endpoint": self.encrypted_config.get("azure_endpoint", ""),
diff --git a/products/ai_observability/frontend/generated/api.schemas.ts b/products/ai_observability/frontend/generated/api.schemas.ts
index 7d5dd381ca0d..26145676d499 100644
--- a/products/ai_observability/frontend/generated/api.schemas.ts
+++ b/products/ai_observability/frontend/generated/api.schemas.ts
@@ -850,7 +850,7 @@ export const EvaluationTargetEnumApi = {
  * * `together_ai` - Together AI
  * * `minimax` - MiniMax
  * * `zeabur` - Zeabur AI Hub
- * * `typesafe` - TypeSafe
+ * * `typesafe` - System One (Jev)
  */
 export type LLMProviderEnumApi = (typeof LLMProviderEnumApi)[keyof typeof LLMProviderEnumApi]
 
@@ -1567,6 +1567,23 @@ export interface LLMProviderKeyApi {
     readonly error_message: string | null
     api_key?: string
     readonly api_key_masked: string
+    /** System One API base URL, including /v1. Defaults to TypeSafe. */
+    base_url?: string
+    /**
+     * Model ID served by the System One endpoint. Defaults to jev-1.13.0.
+     * @maxLength 100
+     */
+    system_one_model?: string
+    /**
+     * Configured System One base URL.
+     * @nullable
+     */
+    readonly base_url_display: string | null
+    /**
+     * Configured System One model ID.
+     * @nullable
+     */
+    readonly system_one_model_display: string | null
     /** Azure OpenAI endpoint URL */
     azure_endpoint?: string
     /**
@@ -2103,6 +2120,23 @@ export interface PatchedLLMProviderKeyApi {
     readonly error_message?: string | null
     api_key?: string
     readonly api_key_masked?: string
+    /** System One API base URL, including /v1. Defaults to TypeSafe. */
+    base_url?: string
+    /**
+     * Model ID served by the System One endpoint. Defaults to jev-1.13.0.
+     * @maxLength 100
+     */
+    system_one_model?: string
+    /**
+     * Configured System One base URL.
+     * @nullable
+     */
+    readonly base_url_display?: string | null
+    /**
+     * Configured System One model ID.
+     * @nullable
+     */
+    readonly system_one_model_display?: string | null
     /** Azure OpenAI endpoint URL */
     azure_endpoint?: string
     /**
diff --git a/products/ai_observability/frontend/generated/api.zod.ts b/products/ai_observability/frontend/generated/api.zod.ts
index 5fab62024e2b..4caf45710557 100644
--- a/products/ai_observability/frontend/generated/api.zod.ts
+++ b/products/ai_observability/frontend/generated/api.zod.ts
@@ -441,7 +441,7 @@ export const EvaluationsCreateBody = /* @__PURE__ */ zod
                                 'typesafe',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - TypeSafe'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
                             ),
                         model: zod.string().max(evaluationsCreateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -746,7 +746,7 @@ export const EvaluationsUpdateBody = /* @__PURE__ */ zod
                                 'typesafe',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - TypeSafe'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
                             ),
                         model: zod.string().max(evaluationsUpdateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -955,7 +955,7 @@ export const EvaluationsPartialUpdateBody = /* @__PURE__ */ zod
                                 'typesafe',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - TypeSafe'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
                             ),
                         model: zod.string().max(evaluationsPartialUpdateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -1517,6 +1517,8 @@ export const LlmAnalyticsParserRecipesPartialUpdateBody = /* @__PURE__ */ zod.ob
 
 export const llmAnalyticsProviderKeysCreateBodyNameMax = 255
 
+export const llmAnalyticsProviderKeysCreateBodySystemOneModelMax = 100
+
 export const llmAnalyticsProviderKeysCreateBodyApiVersionMax = 20
 
 export const llmAnalyticsProviderKeysCreateBodySetAsActiveDefault = false
@@ -1536,10 +1538,16 @@ export const LlmAnalyticsProviderKeysCreateBody = /* @__PURE__ */ zod.object({
             'typesafe',
         ])
         .describe(
-            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - TypeSafe'
+            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
         ),
     name: zod.string().max(llmAnalyticsProviderKeysCreateBodyNameMax),
     api_key: zod.string().optional(),
+    base_url: zod.url().optional().describe('System One API base URL, including \/v1. Defaults to TypeSafe.'),
+    system_one_model: zod
+        .string()
+        .max(llmAnalyticsProviderKeysCreateBodySystemOneModelMax)
+        .optional()
+        .describe('Model ID served by the System One endpoint. Defaults to jev-1.13.0.'),
     azure_endpoint: zod.url().optional().describe('Azure OpenAI endpoint URL'),
     api_version: zod
         .string()
@@ -1551,6 +1559,8 @@ export const LlmAnalyticsProviderKeysCreateBody = /* @__PURE__ */ zod.object({
 
 export const llmAnalyticsProviderKeysUpdateBodyNameMax = 255
 
+export const llmAnalyticsProviderKeysUpdateBodySystemOneModelMax = 100
+
 export const llmAnalyticsProviderKeysUpdateBodyApiVersionMax = 20
 
 export const llmAnalyticsProviderKeysUpdateBodySetAsActiveDefault = false
@@ -1570,10 +1580,16 @@ export const LlmAnalyticsProviderKeysUpdateBody = /* @__PURE__ */ zod.object({
             'typesafe',
         ])
         .describe(
-            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - TypeSafe'
+            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
         ),
     name: zod.string().max(llmAnalyticsProviderKeysUpdateBodyNameMax),
     api_key: zod.string().optional(),
+    base_url: zod.url().optional().describe('System One API base URL, including \/v1. Defaults to TypeSafe.'),
+    system_one_model: zod
+        .string()
+        .max(llmAnalyticsProviderKeysUpdateBodySystemOneModelMax)
+        .optional()
+        .describe('Model ID served by the System One endpoint. Defaults to jev-1.13.0.'),
     azure_endpoint: zod.url().optional().describe('Azure OpenAI endpoint URL'),
     api_version: zod
         .string()
@@ -1585,6 +1601,8 @@ export const LlmAnalyticsProviderKeysUpdateBody = /* @__PURE__ */ zod.object({
 
 export const llmAnalyticsProviderKeysPartialUpdateBodyNameMax = 255
 
+export const llmAnalyticsProviderKeysPartialUpdateBodySystemOneModelMax = 100
+
 export const llmAnalyticsProviderKeysPartialUpdateBodyApiVersionMax = 20
 
 export const llmAnalyticsProviderKeysPartialUpdateBodySetAsActiveDefault = false
@@ -1605,10 +1623,16 @@ export const LlmAnalyticsProviderKeysPartialUpdateBody = /* @__PURE__ */ zod.obj
         ])
         .optional()
         .describe(
-            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - TypeSafe'
+            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
         ),
     name: zod.string().max(llmAnalyticsProviderKeysPartialUpdateBodyNameMax).optional(),
     api_key: zod.string().optional(),
+    base_url: zod.url().optional().describe('System One API base URL, including \/v1. Defaults to TypeSafe.'),
+    system_one_model: zod
+        .string()
+        .max(llmAnalyticsProviderKeysPartialUpdateBodySystemOneModelMax)
+        .optional()
+        .describe('Model ID served by the System One endpoint. Defaults to jev-1.13.0.'),
     azure_endpoint: zod.url().optional().describe('Azure OpenAI endpoint URL'),
     api_version: zod
         .string()
@@ -1620,6 +1644,8 @@ export const LlmAnalyticsProviderKeysPartialUpdateBody = /* @__PURE__ */ zod.obj
 
 export const llmAnalyticsProviderKeysValidateCreateBodyNameMax = 255
 
+export const llmAnalyticsProviderKeysValidateCreateBodySystemOneModelMax = 100
+
 export const llmAnalyticsProviderKeysValidateCreateBodyApiVersionMax = 20
 
 export const llmAnalyticsProviderKeysValidateCreateBodySetAsActiveDefault = false
@@ -1639,10 +1665,16 @@ export const LlmAnalyticsProviderKeysValidateCreateBody = /* @__PURE__ */ zod.ob
             'typesafe',
         ])
         .describe(
-            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - TypeSafe'
+            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
         ),
     name: zod.string().max(llmAnalyticsProviderKeysValidateCreateBodyNameMax),
     api_key: zod.string().optional(),
+    base_url: zod.url().optional().describe('System One API base URL, including \/v1. Defaults to TypeSafe.'),
+    system_one_model: zod
+        .string()
+        .max(llmAnalyticsProviderKeysValidateCreateBodySystemOneModelMax)
+        .optional()
+        .describe('Model ID served by the System One endpoint. Defaults to jev-1.13.0.'),
     azure_endpoint: zod.url().optional().describe('Azure OpenAI endpoint URL'),
     api_version: zod
         .string()
diff --git a/products/ai_observability/frontend/modelPickerLogic.test.ts b/products/ai_observability/frontend/modelPickerLogic.test.ts
index ede2501f8805..d997dd864bc8 100644
--- a/products/ai_observability/frontend/modelPickerLogic.test.ts
+++ b/products/ai_observability/frontend/modelPickerLogic.test.ts
@@ -45,39 +45,42 @@ describe('modelPickerLogic', () => {
     })
 
     describe('loadByokModels', () => {
-        it('offers Jev only to evaluation model pickers', async () => {
-            useMocks({
-                get: {
-                    '/api/environments/:team_id/llm_analytics/provider_keys/': {
-                        results: [{ id: 'key-typesafe', provider: 'typesafe', name: 'TypeSafe', state: 'ok' }],
+        it.each(['jev-1.13.0', 'custom-model'])(
+            'offers System One model %s only to evaluation model pickers',
+            async (model) => {
+                useMocks({
+                    get: {
+                        '/api/environments/:team_id/llm_analytics/provider_keys/': {
+                            results: [{ id: 'key-typesafe', provider: 'typesafe', name: 'TypeSafe', state: 'ok' }],
+                        },
+                        '/api/environments/:team_id/llm_analytics/evaluation_config/': { active_provider_key: null },
+                        '/api/llm_proxy/models/': ({ request }) =>
+                            new URL(request.url).searchParams.get('provider_key_id')
+                                ? [
+                                      200,
+                                      [
+                                          {
+                                              id: model,
+                                              name: model,
+                                              provider: 'System One (Jev)',
+                                              is_recommended: true,
+                                          },
+                                      ],
+                                  ]
+                                : [200, []],
                     },
-                    '/api/environments/:team_id/llm_analytics/evaluation_config/': { active_provider_key: null },
-                    '/api/llm_proxy/models/': ({ request }) =>
-                        new URL(request.url).searchParams.get('provider_key_id')
-                            ? [
-                                  200,
-                                  [
-                                      {
-                                          id: 'jev-1.13.0',
-                                          name: 'jev-1.13.0',
-                                          provider: 'TypeSafe',
-                                          is_recommended: true,
-                                      },
-                                  ],
-                              ]
-                            : [200, []],
-                },
-            })
-            logic = modelPickerLogic()
-            logic.mount()
-            await expectLogic(logic).toFinishAllListeners()
-
-            expect(logic.values.evaluationProviderModelGroups[0].models[0].id).toBe('jev-1.13.0')
-            expect(logic.values.providerModelGroups).toEqual([])
-            expect(logic.values.generativeByokModels).toEqual([])
-            expect(logic.values.hasByokKeys).toBe(false)
-            expect(logic.values.evaluationModelNotice).toBeNull()
-        })
+                })
+                logic = modelPickerLogic()
+                logic.mount()
+                await expectLogic(logic).toFinishAllListeners()
+
+                expect(logic.values.evaluationProviderModelGroups[0].models[0].id).toBe(model)
+                expect(logic.values.providerModelGroups).toEqual([])
+                expect(logic.values.generativeByokModels).toEqual([])
+                expect(logic.values.hasByokKeys).toBe(false)
+                expect(logic.values.evaluationModelNotice).toBeNull()
+            }
+        )
 
         it('should load and attach providerKeyId to models from valid keys', async () => {
             useMocks({
diff --git a/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx b/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
index 2d581f873f1e..8e04f248b65d 100644
--- a/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
+++ b/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
@@ -24,6 +24,8 @@ import {
     AlternativeKey,
     CreateLLMProviderKeyPayload,
     DEFAULT_AZURE_API_VERSION,
+    DEFAULT_SYSTEM_ONE_BASE_URL,
+    DEFAULT_SYSTEM_ONE_MODEL,
     DependentConfigsResponse,
     KeyValidationResult,
     LLMProvider,
@@ -34,6 +36,7 @@ import {
     llmProviderKeysLogic,
     sortProviderKeys,
 } from './llmProviderKeysLogic'
+import { SystemOneConnectionFields } from './SystemOneConnectionFields'
 
 function StateTag({ state, errorMessage }: { state: LLMProviderKeyState; errorMessage: string | null }): JSX.Element {
     const tagProps: { type: 'success' | 'danger' | 'warning' | 'default'; children: string } = {
@@ -103,7 +106,7 @@ function getKeyPlaceholder(provider: LLMProvider): string {
         case 'zeabur':
             return 'sk-...'
         case 'typesafe':
-            return 'Enter your TypeSafe API key'
+            return 'Enter your endpoint’s bearer token'
     }
 }
 
@@ -134,7 +137,11 @@ function KeyValidationStatus({
     const bullets = (
         <ul className="text-xs text-muted mt-1 list-disc pl-4 space-y-0.5">
             <li>Your key will be encrypted and stored securely</li>
-            <li>You pay {LLM_PROVIDER_LABELS[provider]} directly for model usage</li>
+            <li>
+                {provider === 'typesafe'
+                    ? 'Model usage is billed by your endpoint provider'
+                    : `You pay ${LLM_PROVIDER_LABELS[provider]} directly for model usage`}
+            </li>
             <li>Each evaluation counts as an AI observability event</li>
         </ul>
     )
@@ -158,6 +165,7 @@ function KeyValidationStatus({
 }
 
 function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }): JSX.Element {
+    const { systemOneBaseUrl, systemOneModel } = useValues(llmProviderKeysLogic)
     const { newKeyModalOpen, providerKeysLoading, preValidationResult, preValidationResultLoading, evaluationConfig } =
         useValues(llmProviderKeysLogic)
     const { setNewKeyModalOpen, createProviderKey, preValidateKey, clearPreValidation } =
@@ -172,7 +180,15 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
 
     const isAzure = provider === 'azure_openai'
     const keyValidated = preValidationResult?.state === 'ok'
-    const isValid = name.length > 0 && apiKey.length > 0 && (!isAzure || azureEndpoint.length > 0)
+    const isSystemOne = provider === 'typesafe'
+    const isValid =
+        name.length > 0 &&
+        (isSystemOne
+            ? systemOneBaseUrl.length > 0 &&
+              systemOneModel.length > 0 &&
+              (apiKey.length > 0 || systemOneBaseUrl !== DEFAULT_SYSTEM_ONE_BASE_URL)
+            : apiKey.length > 0) &&
+        (!isAzure || azureEndpoint.length > 0)
     const validationFailed = !!preValidationResult && preValidationResult.state !== 'ok'
     const azureErrorField = isAzure && validationFailed ? azureErrorFieldFromResult(preValidationResult) : null
 
@@ -226,12 +242,24 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
     }
 
     const handleSubmit = (): void => {
+        if (isSystemOne) {
+            createProviderKey({
+                payload: {
+                    provider,
+                    name,
+                    api_key: apiKey,
+                    base_url: systemOneBaseUrl,
+                    system_one_model: systemOneModel,
+                },
+            })
+            return
+        }
         if (keyValidated) {
             const payload: CreateLLMProviderKeyPayload = {
                 provider,
                 name,
                 api_key: apiKey,
-                set_as_active: provider !== 'typesafe' && !evaluationConfig?.active_provider_key,
+                set_as_active: !evaluationConfig?.active_provider_key,
             }
             if (isAzure) {
                 payload.azure_endpoint = azureEndpoint
@@ -249,6 +277,9 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
     }
 
     const handleApiKeyBlur = (): void => {
+        if (isSystemOne) {
+            return
+        }
         if (apiKey.length > 0 && !preValidationResult) {
             preValidateKey({
                 apiKey,
@@ -287,7 +318,7 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
                     <LemonButton
                         type="primary"
                         onClick={handleSubmit}
-                        loading={providerKeysLoading}
+                        loading={providerKeysLoading || preValidationResultLoading}
                         disabled={!isValid}
                         disabledReason={restrictionReason}
                     >
@@ -344,6 +375,7 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
                         </div>
                     </>
                 )}
+                {isSystemOne && <SystemOneConnectionFields />}
                 <div>
                     <label className="text-sm font-medium">Name</label>
                     <LemonInput
@@ -374,6 +406,12 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
                         provider={provider}
                         suppressError={azureErrorField === 'endpoint'}
                     />
+                    {isSystemOne && (
+                        <p className="text-xs text-muted">
+                            Sent as a bearer token. Leave empty only if your custom endpoint does not require
+                            authentication.
+                        </p>
+                    )}
                 </div>
             </div>
         </LemonModal>
@@ -388,8 +426,11 @@ function EditKeyModal({
     restrictionReason: string | null
 }): JSX.Element {
     const { providerKeysLoading, preValidationResult, preValidationResultLoading } = useValues(llmProviderKeysLogic)
+    const { systemOneBaseUrl, systemOneModel } = useValues(llmProviderKeysLogic)
     const { setEditingKey, updateProviderKey, preValidateKey, clearPreValidation } = useActions(llmProviderKeysLogic)
     const isAzureEdit = keyToEdit.provider === 'azure_openai'
+    const isSystemOne = keyToEdit.provider === 'typesafe'
+    const endpointChanged = systemOneBaseUrl !== (keyToEdit.base_url_display ?? DEFAULT_SYSTEM_ONE_BASE_URL)
 
     const [name, setName] = useState(keyToEdit.name)
     const [apiKey, setApiKey] = useState('')
@@ -409,6 +450,15 @@ function EditKeyModal({
         if (apiKey.length > 0) {
             payload.api_key = apiKey
         }
+        if (isSystemOne) {
+            if (endpointChanged) {
+                payload.base_url = systemOneBaseUrl
+                payload.api_key = apiKey
+            }
+            if (systemOneModel !== (keyToEdit.system_one_model_display ?? DEFAULT_SYSTEM_ONE_MODEL)) {
+                payload.system_one_model = systemOneModel
+            }
+        }
         if (isAzureEdit) {
             if (azureEndpoint !== (keyToEdit.azure_endpoint_display ?? '')) {
                 payload.azure_endpoint = azureEndpoint
@@ -421,6 +471,9 @@ function EditKeyModal({
     }
 
     const handleApiKeyBlur = (): void => {
+        if (isSystemOne) {
+            return
+        }
         if (apiKey.length > 0) {
             preValidateKey({
                 apiKey,
@@ -437,8 +490,9 @@ function EditKeyModal({
         }
     }
 
-    const keyValidated = apiKey.length === 0 || preValidationResult?.state === 'ok'
-    const isValid = name.length > 0 && keyValidated
+    const keyValidated = isSystemOne || apiKey.length === 0 || preValidationResult?.state === 'ok'
+    const isValid =
+        name.length > 0 && keyValidated && (!isSystemOne || (systemOneBaseUrl.length > 0 && systemOneModel.length > 0))
     const validationFailed = !!preValidationResult && preValidationResult.state !== 'ok'
     const azureErrorField = isAzureEdit && validationFailed ? azureErrorFieldFromResult(preValidationResult) : null
 
@@ -502,6 +556,7 @@ function EditKeyModal({
                         </div>
                     </>
                 )}
+                {isSystemOne && <SystemOneConnectionFields />}
                 <div>
                     <label className="text-sm font-medium">Name</label>
                     <LemonInput value={name} onChange={setName} className="mt-1" fullWidth />
@@ -512,7 +567,11 @@ function EditKeyModal({
                         value={apiKey}
                         onChange={handleApiKeyChange}
                         onBlur={handleApiKeyBlur}
-                        placeholder={`Leave empty to keep current (${keyToEdit.api_key_masked})`}
+                        placeholder={
+                            isSystemOne && endpointChanged
+                                ? 'Enter the new endpoint’s bearer token'
+                                : `Leave empty to keep current (${keyToEdit.api_key_masked})`
+                        }
                         type="password"
                         autoComplete="off"
                         className="mt-1"
@@ -527,7 +586,11 @@ function EditKeyModal({
                             suppressError={azureErrorField === 'endpoint'}
                         />
                     ) : (
-                        <p className="text-xs text-muted mt-1">Leave empty to keep the current key</p>
+                        <p className="text-xs text-muted mt-1">
+                            {isSystemOne && endpointChanged
+                                ? 'The saved key will not be sent to the new endpoint. Enter its bearer token, or leave empty for no authentication.'
+                                : 'Leave empty to keep the current key'}
+                        </p>
                     )}
                 </div>
             </div>
diff --git a/products/ai_observability/frontend/settings/SystemOneConnectionFields.stories.tsx b/products/ai_observability/frontend/settings/SystemOneConnectionFields.stories.tsx
new file mode 100644
index 000000000000..ba7483ff4310
--- /dev/null
+++ b/products/ai_observability/frontend/settings/SystemOneConnectionFields.stories.tsx
@@ -0,0 +1,28 @@
+import type { Meta, StoryObj } from '@storybook/react'
+
+import { useStorybookMocks } from '~/mocks/browser'
+
+import { LLMProviderKeysSettings } from './LLMProviderKeysSettings'
+import { SystemOneConnectionFields } from './SystemOneConnectionFields'
+
+const meta: Meta<typeof SystemOneConnectionFields> = {
+    title: 'Scenes-App/AI observability/System One connection',
+    component: SystemOneConnectionFields,
+    decorators: [
+        (Story) => {
+            useStorybookMocks({
+                get: {
+                    '/api/environments/:team_id/llm_analytics/provider_keys/': { results: [] },
+                    '/api/environments/:team_id/llm_analytics/evaluation_config/': { active_provider_key: null },
+                },
+            })
+            return <Story />
+        },
+    ],
+}
+
+export default meta
+type Story = StoryObj<typeof meta>
+
+export const Default: Story = {}
+export const ProviderSettings: Story = { render: () => <LLMProviderKeysSettings /> }
diff --git a/products/ai_observability/frontend/settings/SystemOneConnectionFields.tsx b/products/ai_observability/frontend/settings/SystemOneConnectionFields.tsx
new file mode 100644
index 000000000000..a80dc7022be7
--- /dev/null
+++ b/products/ai_observability/frontend/settings/SystemOneConnectionFields.tsx
@@ -0,0 +1,48 @@
+import { useActions, useValues } from 'kea'
+
+import { LemonInput } from '@posthog/lemon-ui'
+
+import { LemonLabel } from 'lib/lemon-ui/LemonLabel'
+
+import { llmProviderKeysLogic } from './llmProviderKeysLogic'
+
+export function SystemOneConnectionFields(): JSX.Element {
+    const { systemOneBaseUrl, systemOneModel, providerKeysLoading } = useValues(llmProviderKeysLogic)
+    const { setSystemOneBaseUrl, setSystemOneModel } = useActions(llmProviderKeysLogic)
+
+    return (
+        <details>
+            <summary className="cursor-pointer text-sm font-medium">Advanced configuration</summary>
+            <div className="space-y-3 mt-3">
+                <div>
+                    <LemonLabel htmlFor="system-one-base-url">Base URL</LemonLabel>
+                    <LemonInput
+                        id="system-one-base-url"
+                        value={systemOneBaseUrl}
+                        onChange={setSystemOneBaseUrl}
+                        disabled={providerKeysLoading}
+                        placeholder="https://api.typesafe.ai/v1"
+                        data-attr="system-one-base-url"
+                        fullWidth
+                    />
+                </div>
+                <div>
+                    <LemonLabel htmlFor="system-one-model">Model ID</LemonLabel>
+                    <LemonInput
+                        id="system-one-model"
+                        value={systemOneModel}
+                        onChange={setSystemOneModel}
+                        disabled={providerKeysLoading}
+                        placeholder="jev-1.13.0"
+                        data-attr="system-one-model"
+                        fullWidth
+                    />
+                </div>
+                <p className="text-xs text-muted">
+                    Use a public HTTPS endpoint that supports the System One API. Include /v1 in the base URL if
+                    required by your service. Validation sends a short synthetic example to the selected model.
+                </p>
+            </div>
+        </details>
+    )
+}
diff --git a/products/ai_observability/frontend/settings/llmProviderKeysLogic.test.ts b/products/ai_observability/frontend/settings/llmProviderKeysLogic.test.ts
index ba250b022182..4bbcd45411ac 100644
--- a/products/ai_observability/frontend/settings/llmProviderKeysLogic.test.ts
+++ b/products/ai_observability/frontend/settings/llmProviderKeysLogic.test.ts
@@ -17,6 +17,8 @@ describe('normalizeLLMProvider', () => {
         ['zeabur-ai-hub', 'zeabur'],
         ['zeabur', 'zeabur'],
         ['openai', 'openai'],
+        ['System One (Jev)', 'typesafe'],
+        ['typesafe', 'typesafe'],
     ])('maps %s to %s', (input, expected) => {
         expect(normalizeLLMProvider(input)).toBe(expected)
     })
diff --git a/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts b/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts
index 369dfdf1cb00..d35e8ea9d2ef 100644
--- a/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts
+++ b/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts
@@ -5,13 +5,15 @@ import api, { ApiError } from 'lib/api'
 import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast'
 import { teamLogic } from 'scenes/teamLogic'
 
-import type { LLMProviderEnumApi } from '../generated/api.schemas'
+import type { LLMProviderEnumApi, LLMProviderKeyApi } from '../generated/api.schemas'
 
 export type LLMProviderKeyState = 'unknown' | 'ok' | 'invalid' | 'error'
 export type LLMProvider = LLMProviderEnumApi
 
 /** Default Azure OpenAI API version — keep in sync with backend DEFAULT_API_VERSION. */
 export const DEFAULT_AZURE_API_VERSION = '2024-10-21'
+export const DEFAULT_SYSTEM_ONE_BASE_URL: string = 'https://api.typesafe.ai/v1'
+export const DEFAULT_SYSTEM_ONE_MODEL: string = 'jev-1.13.0'
 
 export const LLM_PROVIDER_LABELS: Record<LLMProvider, string> = {
     openai: 'OpenAI',
@@ -23,7 +25,7 @@ export const LLM_PROVIDER_LABELS: Record<LLMProvider, string> = {
     together_ai: 'Together AI',
     minimax: 'MiniMax',
     zeabur: 'Zeabur AI Hub',
-    typesafe: 'TypeSafe',
+    typesafe: 'System One (Jev)',
 }
 
 const LLM_PROVIDERS = new Set<string>(Object.keys(LLM_PROVIDER_LABELS))
@@ -57,6 +59,9 @@ export function normalizeLLMProvider(provider: string | undefined): LLMProvider
     }
 
     const normalized = provider.trim().toLowerCase()
+    if (normalized === 'system one (jev)') {
+        return 'typesafe'
+    }
     if (normalized === 'google' || normalized === 'google-ai-studio') {
         return 'gemini'
     }
@@ -76,7 +81,9 @@ export function normalizeLLMProvider(provider: string | undefined): LLMProvider
     return normalized in LLM_PROVIDER_LABELS ? (normalized as LLMProvider) : null
 }
 
-export interface LLMProviderKey {
+export interface LLMProviderKey extends Partial<
+    Pick<LLMProviderKeyApi, 'base_url_display' | 'system_one_model_display'>
+> {
     id: string
     provider: LLMProvider
     name: string
@@ -138,7 +145,7 @@ export interface EvaluationConfig {
     updated_at: string
 }
 
-export interface CreateLLMProviderKeyPayload {
+export interface CreateLLMProviderKeyPayload extends Pick<LLMProviderKeyApi, 'base_url' | 'system_one_model'> {
     provider: LLMProvider
     name: string
     api_key: string
@@ -147,7 +154,7 @@ export interface CreateLLMProviderKeyPayload {
     api_version?: string
 }
 
-export interface UpdateLLMProviderKeyPayload {
+export interface UpdateLLMProviderKeyPayload extends Pick<LLMProviderKeyApi, 'base_url' | 'system_one_model'> {
     name?: string
     api_key?: string
     azure_endpoint?: string
@@ -194,6 +201,8 @@ export interface llmProviderKeysLogicValues {
     providerKeys: LLMProviderKey[]
     providerKeysLoading: boolean
     requiresProviderKey: boolean
+    systemOneBaseUrl: string
+    systemOneModel: string
     validatingKeyId: string | null
 }
 
@@ -350,6 +359,12 @@ export interface llmProviderKeysLogicActions {
     setNewKeyModalOpen: (open: boolean) => {
         open: boolean
     }
+    setSystemOneBaseUrl: (baseUrl: string) => {
+        baseUrl: string
+    }
+    setSystemOneModel: (model: string) => {
+        model: string
+    }
     updateProviderKey: ({ id, payload }: { id: string; payload: UpdateLLMProviderKeyPayload }) => {
         id: string
         payload: UpdateLLMProviderKeyPayload
@@ -416,6 +431,8 @@ export const llmProviderKeysLogic = kea<llmProviderKeysLogicType>([
     path(['products', 'ai_observability', 'settings', 'llmProviderKeysLogic']),
 
     actions({
+        setSystemOneBaseUrl: (baseUrl: string) => ({ baseUrl }),
+        setSystemOneModel: (model: string) => ({ model }),
         clearPreValidation: true,
         setNewKeyModalOpen: (open: boolean) => ({ open }),
         setEditingKey: (key: LLMProviderKey | null) => ({ key }),
@@ -424,6 +441,22 @@ export const llmProviderKeysLogic = kea<llmProviderKeysLogicType>([
     }),
 
     reducers({
+        systemOneBaseUrl: [
+            DEFAULT_SYSTEM_ONE_BASE_URL,
+            {
+                setSystemOneBaseUrl: (_, { baseUrl }) => baseUrl,
+                setNewKeyModalOpen: () => DEFAULT_SYSTEM_ONE_BASE_URL,
+                setEditingKey: (_, { key }) => key?.base_url_display ?? DEFAULT_SYSTEM_ONE_BASE_URL,
+            },
+        ],
+        systemOneModel: [
+            DEFAULT_SYSTEM_ONE_MODEL,
+            {
+                setSystemOneModel: (_, { model }) => model,
+                setNewKeyModalOpen: () => DEFAULT_SYSTEM_ONE_MODEL,
+                setEditingKey: (_, { key }) => key?.system_one_model_display ?? DEFAULT_SYSTEM_ONE_MODEL,
+            },
+        ],
         newKeyModalOpen: [
             false,
             {
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 8bf9eccedcdc..35421d7ff314 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -36259,7 +36259,7 @@ export namespace Schemas {
      * * `together_ai` - Together AI
      * * `minimax` - MiniMax
      * * `zeabur` - Zeabur AI Hub
-     * * `typesafe` - TypeSafe
+     * * `typesafe` - System One (Jev)
      */
     export type LLMProviderEnum = typeof LLMProviderEnum[keyof typeof LLMProviderEnum];
 
@@ -36477,6 +36477,23 @@ export namespace Schemas {
       readonly error_message: string | null;
       api_key?: string;
       readonly api_key_masked: string;
+      /** System One API base URL, including /v1. Defaults to TypeSafe. */
+      base_url?: string;
+      /**
+         * Model ID served by the System One endpoint. Defaults to jev-1.13.0.
+         * @maxLength 100
+         */
+      system_one_model?: string;
+      /**
+         * Configured System One base URL.
+         * @nullable
+         */
+      readonly base_url_display: string | null;
+      /**
+         * Configured System One model ID.
+         * @nullable
+         */
+      readonly system_one_model_display: string | null;
       /** Azure OpenAI endpoint URL */
       azure_endpoint?: string;
       /**
@@ -70055,6 +70072,23 @@ export namespace Schemas {
       readonly error_message?: string | null;
       api_key?: string;
       readonly api_key_masked?: string;
+      /** System One API base URL, including /v1. Defaults to TypeSafe. */
+      base_url?: string;
+      /**
+         * Model ID served by the System One endpoint. Defaults to jev-1.13.0.
+         * @maxLength 100
+         */
+      system_one_model?: string;
+      /**
+         * Configured System One base URL.
+         * @nullable
+         */
+      readonly base_url_display?: string | null;
+      /**
+         * Configured System One model ID.
+         * @nullable
+         */
+      readonly system_one_model_display?: string | null;
       /** Azure OpenAI endpoint URL */
       azure_endpoint?: string;
       /**
diff --git a/services/mcp/src/generated/ai_observability/api.ts b/services/mcp/src/generated/ai_observability/api.ts
index d3bae257e86b..2c2e077da6cb 100644
--- a/services/mcp/src/generated/ai_observability/api.ts
+++ b/services/mcp/src/generated/ai_observability/api.ts
@@ -743,7 +743,7 @@ export const EvaluationsCreateBody = () => zod
                                 'typesafe',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - TypeSafe'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
                             ),
                         model: zod.string().max(evaluationsCreateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -970,7 +970,7 @@ export const EvaluationsPartialUpdateBody = () => zod
                                 'typesafe',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - TypeSafe'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
                             ),
                         model: zod.string().max(evaluationsPartialUpdateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod

From 6f46380ffcb2bf71b29f10e78c76bdb0d8d947e9 Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Fri, 25 Sep 2026 12:58:32 +0200
Subject: [PATCH 04/30] fix(aio): distinguish rejected System One requests

---
 .../internal/ai-observability-judge-inputs.md |  1 +
 .../ai_observability/evaluation_errors.py     |  8 ++++++
 .../ai_observability/evaluation_llm_judge.py  | 16 ++++++++++-
 .../ai_observability/test_run_evaluation.py   | 27 +++++++++++++++++++
 .../backend/llm/system_one.py                 | 10 +++++--
 .../backend/llm/test/test_system_one.py       |  8 ++++--
 6 files changed, 65 insertions(+), 5 deletions(-)

diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index ea1dae6944b1..ad90c30e5006 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -65,6 +65,7 @@ Jev provides no written reasoning, so reports inspect the original source when e
 
 Rate limits and overload responses are retried through Temporal, honoring `Retry-After` up to five minutes.
 If retries fail, the run fails and the evaluation stays enabled.
+Blocked endpoints and rejected requests disable the evaluation and mark the connection for revalidation, without recording model usage.
 Invalid probabilities or missing answers fail the evaluation rather than producing a false result.
 Inputs rejected for exceeding the model's context window are skipped.
 See TypeSafe's [API reference](https://docs.typesafe.ai/api) and [model limits and pricing](https://docs.typesafe.ai/models).
diff --git a/posthog/temporal/ai_observability/evaluation_errors.py b/posthog/temporal/ai_observability/evaluation_errors.py
index 6b70bf5182c7..884cfa425bd1 100644
--- a/posthog/temporal/ai_observability/evaluation_errors.py
+++ b/posthog/temporal/ai_observability/evaluation_errors.py
@@ -31,6 +31,14 @@ class EvaluationErrorSpec:
 
 
 USER_ERROR_SPECS: dict[str, EvaluationErrorSpec] = {
+    "request_rejected": EvaluationErrorSpec(
+        error_type="request_rejected",
+        owner="user",
+        safe_message="The judge endpoint rejected the request. Check the connection settings and evaluation criteria.",
+        status_reason=EvaluationStatusReason.PROVIDER_KEY_INVALID,
+        disables_evaluation=True,
+        provider_key_state=LLMProviderKey.State.ERROR,
+    ),
     "provider_key_required": EvaluationErrorSpec(
         error_type="provider_key_required",
         owner="user",
diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.py
index 7b0102c5d7bd..684ae7c67c60 100644
--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py
+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py
@@ -48,7 +48,11 @@
     RateLimitError,
     StructuredOutputParseError,
 )
-from products.ai_observability.backend.llm.system_one import SystemOneClient, SystemOneRateLimitError
+from products.ai_observability.backend.llm.system_one import (
+    SystemOneClient,
+    SystemOneRateLimitError,
+    SystemOneRequestRejectedError,
+)
 from products.ai_observability.backend.llm.types import CompletionResponse
 from products.ai_observability.backend.models.evaluation_configs import NumericOutputConfig, NumericScoreOutOfBounds
 from products.ai_observability.backend.text_repr.formatters import add_line_numbers, reduce_by_uniform_sampling
@@ -504,6 +508,16 @@ def call_llm_judge(
                     response_format=response_format,
                 )
             )
+    except SystemOneRequestRejectedError as e:
+        increment_user_errors("request_rejected", provider=provider)
+        return terminal_user_error_result(
+            spec=require_user_error_spec("request_rejected", is_byok=is_byok),
+            message=str(e),
+            allows_na=allows_na,
+            output_type=output_type,
+            key_id=key_id,
+            is_byok=is_byok,
+        )
     except SystemOneRateLimitError as e:
         increment_errors("rate_limit", provider=provider)
         raise ApplicationError(
diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index 963d07f1eb93..b6d7d3630cc3 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -158,6 +158,33 @@ def test_typesafe_judge_emits_boolean_probability_without_reasoning(
     assert properties["$ai_evaluation_key_type"] == "byok"
 
 
+@pytest.mark.parametrize("status", [301, 400, 422])
+def test_system_one_rejected_requests_disable_without_model_cost_attribution(status: int) -> None:
+    key = MagicMock(provider="typesafe", encrypted_config={"api_key": "example-token"})
+    with (
+        patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
+        patch(
+            "products.ai_observability.backend.llm.system_one.pinned_request",
+            return_value=MagicMock(status_code=status, text="Invalid request"),
+        ),
+    ):
+        spec.return_value.resolve.return_value = MagicMock(
+            provider="typesafe", model="jev-1.13.0", provider_key=key, is_byok=True
+        )
+        result = call_llm_judge(
+            evaluation={"id": "test-evaluation", "team_id": 1, "evaluation_config": {"prompt": "Polite?"}},
+            system_prompt="",
+            user_prompt="Hello!",
+            allows_na=False,
+        )
+    assert result["terminal_user_error"] is True
+    assert result["skip_reason"] == "request_rejected"
+    assert result["provider_key_state"] == "error"
+    assert result["status_reason"] == "provider_key_invalid"
+    assert "model" not in result
+    assert "provider" not in result
+
+
 def test_typesafe_rate_limit_retries_without_disabling_the_evaluation() -> None:
     key = MagicMock(provider="typesafe", encrypted_config={"api_key": "test-typesafe-key"})
     with (
diff --git a/products/ai_observability/backend/llm/system_one.py b/products/ai_observability/backend/llm/system_one.py
index 76b74da2dca8..d797c6f819f9 100644
--- a/products/ai_observability/backend/llm/system_one.py
+++ b/products/ai_observability/backend/llm/system_one.py
@@ -13,6 +13,7 @@
 from products.ai_observability.backend.llm.errors import (
     AuthenticationError,
     ContextWindowExceededError,
+    LLMError,
     ModelNotFoundError,
     ModelPermissionError,
     ProviderConnectionError,
@@ -38,6 +39,10 @@ class SystemOneResponse(BaseModel):
     usage: SystemOneUsage
 
 
+class SystemOneRequestRejectedError(LLMError):
+    pass
+
+
 class SystemOneRateLimitError(RateLimitError):
     def __init__(self, retry_after: str | None) -> None:
         super().__init__("The System One endpoint is busy. Try again later.")
@@ -98,7 +103,7 @@ def evaluate_boolean(
                 timeout=60,
             )
         except SSRFBlockedError as error:
-            raise StructuredOutputParseError("This endpoint is not allowed. Use a public HTTPS endpoint.") from error
+            raise SystemOneRequestRejectedError("This endpoint is not allowed. Use a public HTTPS endpoint.") from error
         except requests.RequestException as error:
             raise ProviderConnectionError("Could not reach the System One endpoint. Try again.") from error
 
@@ -117,7 +122,7 @@ def evaluate_boolean(
         ):
             raise ContextWindowExceededError("This input exceeds the endpoint's size limit. Reduce the input.")
         if response.status_code != 200:
-            raise StructuredOutputParseError(
+            raise SystemOneRequestRejectedError(
                 "The endpoint rejected the evaluation request. Check the model and criteria."
             )
         try:
@@ -152,6 +157,7 @@ def validate_key(api_key: str, *, base_url: str = BASE_URL, model: str = MODEL)
             ProviderConnectionError,
             RateLimitError,
             StructuredOutputParseError,
+            SystemOneRequestRejectedError,
             ContextWindowExceededError,
         ) as error:
             return "error", str(error)
diff --git a/products/ai_observability/backend/llm/test/test_system_one.py b/products/ai_observability/backend/llm/test/test_system_one.py
index 1f56f2cad602..f9d0b998d3c7 100644
--- a/products/ai_observability/backend/llm/test/test_system_one.py
+++ b/products/ai_observability/backend/llm/test/test_system_one.py
@@ -11,7 +11,11 @@
     ProviderConnectionError,
     StructuredOutputParseError,
 )
-from products.ai_observability.backend.llm.system_one import SystemOneClient, SystemOneRateLimitError
+from products.ai_observability.backend.llm.system_one import (
+    SystemOneClient,
+    SystemOneRateLimitError,
+    SystemOneRequestRejectedError,
+)
 
 
 @pytest.mark.parametrize("status, expected_state", [(200, "ok"), (401, "invalid"), (403, "invalid"), (500, "error")])
@@ -130,7 +134,7 @@ def test_typesafe_requires_every_requested_answer(answers: dict[str, object]) ->
         (500, "Unavailable", ProviderConnectionError),
         (413, "Request too large", ContextWindowExceededError),
         (422, "Input exceeds the context window", ContextWindowExceededError),
-        (422, "Invalid question", StructuredOutputParseError),
+        (422, "Invalid question", SystemOneRequestRejectedError),
     ],
 )
 def test_typesafe_preserves_error_categories(status: int, message: str, error_type: type[Exception]) -> None:

From 25623e9ba2b7ee74e196256153f82f71a4e5b88d Mon Sep 17 00:00:00 2001
From: "tests-posthog[bot]"
 <250237707+tests-posthog[bot]@users.noreply.github.com>
Date: Fri, 25 Sep 2026 11:04:12 +0000
Subject: [PATCH 05/30] test(mcp): update unit test snapshots

---
 .../unit/__snapshots__/tool-schemas/llma-evaluation-create.json | 2 +-
 .../unit/__snapshots__/tool-schemas/llma-evaluation-update.json | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json
index 4b8de8d2ab63..c7acc821350a 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json
@@ -112,7 +112,7 @@
                             "type": "string"
                         },
                         "provider": {
-                            "description": "* `openai` - Openai\n* `anthropic` - Anthropic\n* `gemini` - Gemini\n* `openrouter` - Openrouter\n* `fireworks` - Fireworks\n* `azure_openai` - Azure OpenAI\n* `together_ai` - Together AI\n* `minimax` - MiniMax\n* `zeabur` - Zeabur AI Hub\n* `typesafe` - TypeSafe",
+                            "description": "* `openai` - Openai\n* `anthropic` - Anthropic\n* `gemini` - Gemini\n* `openrouter` - Openrouter\n* `fireworks` - Fireworks\n* `azure_openai` - Azure OpenAI\n* `together_ai` - Together AI\n* `minimax` - MiniMax\n* `zeabur` - Zeabur AI Hub\n* `typesafe` - System One (Jev)",
                             "enum": [
                                 "openai",
                                 "anthropic",
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json
index 004588151d45..69dc4c521d7a 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json
@@ -115,7 +115,7 @@
                             "type": "string"
                         },
                         "provider": {
-                            "description": "* `openai` - Openai\n* `anthropic` - Anthropic\n* `gemini` - Gemini\n* `openrouter` - Openrouter\n* `fireworks` - Fireworks\n* `azure_openai` - Azure OpenAI\n* `together_ai` - Together AI\n* `minimax` - MiniMax\n* `zeabur` - Zeabur AI Hub\n* `typesafe` - TypeSafe",
+                            "description": "* `openai` - Openai\n* `anthropic` - Anthropic\n* `gemini` - Gemini\n* `openrouter` - Openrouter\n* `fireworks` - Fireworks\n* `azure_openai` - Azure OpenAI\n* `together_ai` - Together AI\n* `minimax` - MiniMax\n* `zeabur` - Zeabur AI Hub\n* `typesafe` - System One (Jev)",
                             "enum": [
                                 "openai",
                                 "anthropic",

From 46d304558501ed7429e464d4268ff1780b8ea198 Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Fri, 25 Sep 2026 13:43:32 +0200
Subject: [PATCH 06/30] feat(aio): support typed system one questions and
 numeric judges

---
 .../internal/ai-observability-judge-inputs.md |  29 ++-
 .../eval_reports/report_agent/prompts.py      |   2 +-
 .../ai_observability/evaluation_llm_judge.py  |  73 ++++++--
 .../ai_observability/test_run_evaluation.py   |  67 +++++++
 .../backend/api/evaluations.py                |  21 ++-
 .../ai_observability/backend/api/proxy.py     |   2 +-
 .../backend/api/test/test_evaluations.py      |  13 +-
 .../backend/llm/system_one.py                 | 109 ++++++++---
 .../backend/llm/test/test_system_one.py       | 175 +++++++++++++++---
 .../backend/models/evaluation_configs.py      |  18 ++
 .../backend/models/provider_keys.py           |   2 +-
 .../models/test/test_evaluation_configs.py    |  12 ++
 .../components/EvaluationExplanation.tsx      |   2 +-
 .../evaluations/AIObservabilityEvaluation.tsx |  26 ++-
 .../NumericEvaluationConfig.stories.tsx       |   7 +-
 .../NumericEvaluationConfig.test.tsx          |  17 ++
 .../components/NumericEvaluationConfig.tsx    |  35 +++-
 .../frontend/evaluations/constants.ts         |  18 +-
 .../evaluations/llmEvaluationLogic.test.ts    |  32 ++--
 .../evaluations/llmEvaluationLogic.ts         |   8 +-
 .../frontend/generated/api.schemas.ts         |  42 ++++-
 .../frontend/generated/api.zod.ts             |  64 ++++++-
 .../frontend/modelPickerLogic.test.ts         |   2 +-
 .../settings/SystemOneConnectionFields.tsx    |   5 +-
 .../settings/llmProviderKeysLogic.test.ts     |   1 +
 .../frontend/settings/llmProviderKeysLogic.ts |   4 +-
 services/mcp/src/api/generated.ts             |  42 ++++-
 .../mcp/src/generated/ai_observability/api.ts |  41 +++-
 28 files changed, 730 insertions(+), 139 deletions(-)

diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index ad90c30e5006..876436fedac5 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -39,34 +39,45 @@ They sample the combined input, tool definitions, and output only when that text
 
 Implementation: [trace judge](../../posthog/temporal/ai_observability/run_trace_evaluation.py), [session judge](../../posthog/temporal/ai_observability/run_session_evaluation.py), and [generation judge](../../posthog/temporal/ai_observability/evaluation_llm_judge.py).
 
-## System One boolean judges
+## System One judges
 
 System One-compatible models are available under the existing LLM judge option.
-Add a connection under **System One (Jev)** in provider key settings.
+Add a connection under **System One** in provider key settings.
 The default endpoint is TypeSafe's `https://api.typesafe.ai/v1`, with model `jev-1.13.0` and a TypeSafe API key.
 Advanced configuration accepts a different public HTTPS base URL and model ID for compatible services.
 The client appends `/systemone` to the base URL and sends the API key as a bearer token.
 An empty key selects no authentication for a custom endpoint; TypeSafe requires a key.
 Changing the endpoint requires entering its credential again, or explicitly choosing no authentication, so an existing key is not forwarded to a new host.
 Private network destinations and redirects are blocked by the shared DNS-pinned HTTP transport.
-Saving a connection validates it with a short synthetic input and two Noul questions, including applicability, without sending evaluation data.
+Saving a connection validates it with a short synthetic input and a Noul question, without sending evaluation data.
 Select the connection and configured model on each evaluation; these connections cannot become the shared active provider key used by other AI features.
 Provider keys keep the provider they were created with; switching providers requires a new key.
-This integration supports boolean evaluations only and uses the same formatted text for generation, trace, and session targets.
+This integration supports boolean and numeric evaluations and uses the same formatted text for generation, trace, and session targets.
+The client accepts typed Noul, Score, and Choice questions independently of PostHog's evaluation output types.
+Choice support in the client does not enable categorical evaluations in the product.
 API compatibility does not guarantee equivalent judgments or calibration across models.
 Compare results on representative inputs when changing models.
 
-The evaluation prompt becomes a [Noul question](https://docs.typesafe.ai/primitives/noul).
+For boolean evaluations, the prompt becomes a [Noul question](https://docs.typesafe.ai/primitives/noul).
 A probability of at least 0.5 produces `true`; the evaluation's existing pass/fail polarity still applies.
-For evaluations that allow N/A, a separate question checks whether the criteria apply, using the same threshold.
-Uncertainty alone does not produce N/A.
 The raw probability is stored in `$ai_evaluation_probability`, with token usage and the resolved model version.
-Jev provides no written reasoning, so reports inspect the original source when explaining outcomes.
+
+Numeric evaluations use a [Score question](https://docs.typesafe.ai/primitives/score).
+Configure 2 to 10 ordered score levels and a minimum and maximum.
+The levels are evenly spaced across those bounds.
+The response is a probability-weighted level index, which maps to the configured scale without rounding.
+For example, three levels on a 0–10 scale map an answer of 1.25 to a score of 6.25.
+Scores are stored in `$ai_evaluation_numeric_result`; they are not boolean probabilities.
+The rubric remains part of the prompt when switching to a completion-based judge.
+
+For either output type, evaluations that allow N/A send a separate Noul question about whether the criteria apply, using the 0.5 threshold.
+Uncertainty alone does not produce N/A.
+System One answers contain no written reasoning, so reports inspect the original source when explaining outcomes.
 
 Rate limits and overload responses are retried through Temporal, honoring `Retry-After` up to five minutes.
 If retries fail, the run fails and the evaluation stays enabled.
 Blocked endpoints and rejected requests disable the evaluation and mark the connection for revalidation, without recording model usage.
-Invalid probabilities or missing answers fail the evaluation rather than producing a false result.
+Invalid probabilities, missing answers, mismatched answer types, and invalid score scales skip the item as an unparsable response.
 Inputs rejected for exceeding the model's context window are skipped.
 See TypeSafe's [API reference](https://docs.typesafe.ai/api) and [model limits and pricing](https://docs.typesafe.ai/models).
 
diff --git a/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.py b/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.py
index c67d70be2797..4163b72f598c 100644
--- a/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.py
+++ b/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.py
@@ -170,7 +170,7 @@ def build_eval_report_system_prompt(
         sample_ordering_signature = ""
         sample_ordering_instruction = (
             'Rows include full reasoning when available. Use the default `order_by="recent"`. '
-            "Some judges, including Jev, return no written reasoning. For those results, inspect the original "
+            "Some judges return no written reasoning. For those results, inspect the original "
             "generation, trace, or session with the detail tools and ground your analysis in that source. "
             "Do not invent a judge explanation or treat absent reasoning as an evaluation failure."
         )
diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.py
index 684ae7c67c60..b331a9148c7b 100644
--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py
+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py
@@ -49,7 +49,12 @@
     StructuredOutputParseError,
 )
 from products.ai_observability.backend.llm.system_one import (
+    NoulAnswer,
+    NoulQuestion,
+    ScoreAnswer,
+    ScoreQuestion,
     SystemOneClient,
+    SystemOneQuestion,
     SystemOneRateLimitError,
     SystemOneRequestRejectedError,
 )
@@ -175,6 +180,11 @@ def get_output_type_config(
             instructions += f" The score must be at most {numeric_config.max}."
         if numeric_config.step is not None:
             instructions += f" Suggested score increment: {numeric_config.step}; do not round an otherwise valid score."
+        if numeric_config.score_levels is not None:
+            instructions += " Use these rubric levels, interpolating between them when needed:\n" + "\n".join(
+                f"{numeric_config.score_from_level(index)}: {description}"
+                for index, description in enumerate(numeric_config.score_levels)
+            )
         if allows_na:
             instructions += " Return score=null when the criteria does not apply."
         return OutputTypeConfig(
@@ -444,8 +454,8 @@ def call_llm_judge(
         raise
 
     provider = resolved.provider
-    if provider == "typesafe" and output_type != "boolean":
-        raise ApplicationError("System One connections support boolean evaluations only.", non_retryable=True)
+    if provider == "typesafe" and output_type not in ("boolean", "numeric"):
+        raise ApplicationError("This System One evaluation output type is not supported.", non_retryable=True)
     model = resolved.model
     provider_key = resolved.provider_key
     is_byok = resolved.is_byok
@@ -467,26 +477,59 @@ def call_llm_judge(
         if provider == "typesafe":
             if provider_key is not None and provider_key.provider != provider:
                 raise ProviderMismatchError(provider_key.provider, provider)
-            system_one_result = SystemOneClient.evaluate_boolean(
+            prompt = evaluation["evaluation_config"]["prompt"]
+            numeric_config = NumericOutputConfig.model_validate(output_config) if output_type == "numeric" else None
+            questions: dict[str, SystemOneQuestion]
+            if numeric_config is not None:
+                if numeric_config.score_levels is None:
+                    raise ApplicationError("Add score levels to this numeric evaluation.", non_retryable=True)
+                questions = {"score": ScoreQuestion(instructions=prompt, criteria=list(numeric_config.score_levels))}
+            else:
+                questions = {"verdict": NoulQuestion(instructions=prompt)}
+            if allows_na:
+                questions["applicable"] = NoulQuestion(
+                    instructions=(
+                        "Do these evaluation criteria apply to this input? Answer true when the criteria can be "
+                        "evaluated, even if they are not met. Answer false only when they are not relevant.\n\n"
+                        + prompt
+                    )
+                )
+            system_one_result = SystemOneClient.evaluate(
                 api_key=provider_key.encrypted_config.get("api_key", "") if provider_key else "",
                 base_url=provider_key.encrypted_config.get("base_url", SystemOneClient.BASE_URL)
                 if provider_key
                 else SystemOneClient.BASE_URL,
                 model=model,
-                prompt=evaluation["evaluation_config"]["prompt"],
-                source=user_prompt,
-                allows_na=allows_na,
+                state=user_prompt,
+                questions=questions,
             )
-            probability = system_one_result.answers["verdict"].noul
-            applicable = not allows_na or system_one_result.answers["applicable"].noul >= 0.5
-            parsed: BooleanEvalResult | BooleanWithNAEvalResult = (
-                BooleanWithNAEvalResult(
-                    reasoning="",
-                    outcome="not_applicable" if not applicable else "pass" if probability >= 0.5 else "fail",
+            applicable = True
+            if allows_na:
+                applicability_answer = system_one_result.answers["applicable"]
+                assert isinstance(applicability_answer, NoulAnswer)
+                applicable = applicability_answer.noul >= 0.5
+            parsed: BooleanEvalResult | BooleanWithNAEvalResult | NumericEvalResult | NumericWithNAEvalResult
+            if numeric_config is not None:
+                score_answer = system_one_result.answers["score"]
+                assert isinstance(score_answer, ScoreAnswer)
+                score = numeric_config.score_from_level(score_answer.score)
+                parsed = (
+                    NumericWithNAEvalResult(reasoning="", score=score if applicable else None)
+                    if allows_na
+                    else NumericEvalResult(reasoning="", score=score)
+                )
+            else:
+                verdict_answer = system_one_result.answers["verdict"]
+                assert isinstance(verdict_answer, NoulAnswer)
+                probability = verdict_answer.noul
+                parsed = (
+                    BooleanWithNAEvalResult(
+                        reasoning="",
+                        outcome="not_applicable" if not applicable else "pass" if probability >= 0.5 else "fail",
+                    )
+                    if allows_na
+                    else BooleanEvalResult(reasoning="", verdict=probability >= 0.5)
                 )
-                if allows_na
-                else BooleanEvalResult(reasoning="", verdict=probability >= 0.5)
-            )
             model = system_one_result.model
             response = CompletionResponse(
                 content="",
diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index b6d7d3630cc3..cf84b6dad99c 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -158,6 +158,73 @@ def test_typesafe_judge_emits_boolean_probability_without_reasoning(
     assert properties["$ai_evaluation_key_type"] == "byok"
 
 
+@pytest.mark.parametrize(
+    "raw_score,probabilities,allows_na,applicability,expected",
+    [
+        (0, [1, 0, 0], False, 1, -10),
+        (1, [0, 1, 0], False, 1, 0),
+        (1.25, [0.1, 0.55, 0.35], True, 1, 2.5),
+        (2, [0, 0, 1], True, 0.1, None),
+    ],
+)
+def test_system_one_numeric_scores_keep_the_rubric_scale(
+    raw_score: float, probabilities: list[float], allows_na: bool, applicability: float, expected: float | None
+) -> None:
+    levels = ["Poor", "Fair", "Good"]
+    evaluation = {
+        "id": "test-evaluation",
+        "name": "Quality",
+        "team_id": 1,
+        "evaluation_config": {"prompt": "Assess response quality"},
+        "output_type": "numeric",
+        "output_config": {"min": -10, "max": 10, "score_levels": levels, "allows_na": allows_na},
+    }
+    key = MagicMock(provider="typesafe", encrypted_config={"api_key": "example-token"})
+    response = MagicMock(status_code=200)
+    response.json.return_value = {
+        "model": "jev-1.13.0",
+        "answers": {
+            "score": {
+                "type": "score",
+                "score": raw_score,
+                "confidence": 0.5,
+                "legend": {str(index): value for index, value in enumerate(levels)},
+                "probabilities": {str(index): value for index, value in enumerate(probabilities)},
+            },
+            "applicable": {"type": "noul", "noul": applicability},
+        },
+        "usage": {"input_tokens": 120, "output_tokens": 10},
+    }
+    with (
+        patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
+        patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response) as request,
+    ):
+        spec.return_value.resolve.return_value = MagicMock(
+            provider="typesafe", model="jev-1.13.0", provider_key=key, is_byok=True
+        )
+        result = call_llm_judge(evaluation=evaluation, system_prompt="", user_prompt="Hello!", allows_na=allows_na)
+    assert request.call_args.kwargs["json"]["questions"]["score"] == {
+        "type": "score",
+        "instructions": "Assess response quality",
+        "criteria": levels,
+    }
+    assert ("applicable" in request.call_args.kwargs["json"]["questions"]) is allows_na
+    assert result.get("score") == expected
+    if expected is None:
+        assert "score" not in result
+        assert result["applicable"] is False
+    assert result["reasoning"] == ""
+    assert result["total_tokens"] == 130
+    assert "verdict" not in result
+    assert "probability" not in result
+    properties = build_evaluation_event_properties(evaluation, result, datetime.now(UTC))
+    assert properties["$ai_evaluation_result_type"] == "numeric"
+    assert properties.get("$ai_evaluation_numeric_result") == expected
+    assert "$ai_evaluation_result" not in properties
+    assert "$ai_evaluation_probability" not in properties
+    assert properties["$ai_model"] == "jev-1.13.0"
+
+
 @pytest.mark.parametrize("status", [301, 400, 422])
 def test_system_one_rejected_requests_disable_without_model_cost_attribution(status: int) -> None:
     key = MagicMock(provider="typesafe", encrypted_config={"api_key": "example-token"})
diff --git a/products/ai_observability/backend/api/evaluations.py b/products/ai_observability/backend/api/evaluations.py
index f4e51c928b3a..e146aab155d6 100644
--- a/products/ai_observability/backend/api/evaluations.py
+++ b/products/ai_observability/backend/api/evaluations.py
@@ -195,6 +195,14 @@ class _EvaluationConfigField(serializers.JSONField):
                 "minimum": 0,
                 "description": "Optional positive input increment. Does not round evaluation results.",
             },
+            "score_levels": {
+                "type": "array",
+                "nullable": True,
+                "items": {"type": "string", "minLength": 1},
+                "minItems": 2,
+                "maxItems": 10,
+                "description": "Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.",
+            },
             "passing_rule": {
                 "type": "object",
                 "nullable": True,
@@ -400,7 +408,7 @@ class EvaluationSerializer(UserAccessControlSerializerMixin, serializers.ModelSe
         help_text=(
             "Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and "
             "{true_is_failure} to declare that a true result means the evaluation found a problem. "
-            "For 'numeric': only min/max/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. "
+            "For 'numeric': min/max/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. "
             "Do not send true_is_failure for numeric output. For 'sentiment': {}."
         ),
     )
@@ -516,9 +524,9 @@ def validate(self, data):
             if isinstance(model_configuration, dict)
             else getattr(model_configuration, "provider", None)
         )
-        if model_provider == LLMProvider.TYPESAFE and output_type != "boolean":
+        if model_provider == LLMProvider.TYPESAFE and output_type not in ("boolean", "numeric"):
             raise serializers.ValidationError(
-                {"model_configuration": "System One connections support boolean evaluations only."}
+                {"model_configuration": "Select a model that supports this evaluation output type."}
             )
 
         if not evaluation_uses_model_configuration(evaluation_type) and model_configuration is not None:
@@ -564,6 +572,13 @@ def validate(self, data):
             except ValueError as e:
                 raise serializers.ValidationError({"config": str(e)})
 
+        if model_provider == LLMProvider.TYPESAFE and output_type == "numeric":
+            numeric_output = data.get("output_config", getattr(self.instance, "output_config", {}))
+            if not numeric_output.get("score_levels"):
+                raise serializers.ValidationError(
+                    {"output_config": "Add 2 to 10 score levels and set both bounds for this System One judge."}
+                )
+
         # Sentiment is addressed per-message within one generation event ($ai_target_event_id +
         # message index). An aggregate target emits a single evaluation event for the whole unit,
         # where the message index is ambiguous and that per-generation linkage is absent.
diff --git a/products/ai_observability/backend/api/proxy.py b/products/ai_observability/backend/api/proxy.py
index 77bee39e080c..73f09dad9f1f 100644
--- a/products/ai_observability/backend/api/proxy.py
+++ b/products/ai_observability/backend/api/proxy.py
@@ -67,7 +67,7 @@ def models_cache_key(provider_key_id: str | uuid.UUID) -> str:
 
 
 PROVIDER_DISPLAY_NAMES: dict[str, str] = {
-    "typesafe": "System One (Jev)",
+    "typesafe": "System One",
     "openai": "OpenAI",
     "anthropic": "Anthropic",
     "gemini": "Gemini",
diff --git a/products/ai_observability/backend/api/test/test_evaluations.py b/products/ai_observability/backend/api/test/test_evaluations.py
index f3ee8f9e232e..537d3739f044 100644
--- a/products/ai_observability/backend/api/test/test_evaluations.py
+++ b/products/ai_observability/backend/api/test/test_evaluations.py
@@ -108,16 +108,21 @@ def test_existing_boolean_cannot_change_to_numeric(self):
 
 
 class TestModelConfigurationSerializer(SimpleTestCase):
-    def test_numeric_evaluation_rejects_system_one_connection(self) -> None:
+    @parameterized.expand([({}, False), ({"min": 0, "max": 10, "score_levels": ["Poor", "Good"]}, True)])
+    def test_numeric_system_one_requires_score_levels(self, config: dict, valid: bool) -> None:
         evaluation = Evaluation(
             evaluation_type="llm_judge",
             evaluation_config={"prompt": "Score quality"},
             output_type="numeric",
-            output_config={},
+            output_config=config,
         )
         serializer = EvaluationSerializer(instance=evaluation, partial=True)
-        with self.assertRaisesMessage(ValidationError, "System One connections support boolean evaluations only"):
-            serializer.validate({"model_configuration": {"provider": "typesafe", "model": "custom-model"}})
+        data = {"model_configuration": {"provider": "typesafe", "model": "custom-model"}}
+        if valid:
+            self.assertEqual(serializer.validate(data)["model_configuration"], data["model_configuration"])
+        else:
+            with self.assertRaisesMessage(ValidationError, "Add 2 to 10 score levels"):
+                serializer.validate(data)
 
     @parameterized.expand(
         [
diff --git a/products/ai_observability/backend/llm/system_one.py b/products/ai_observability/backend/llm/system_one.py
index d797c6f819f9..51579efa5c61 100644
--- a/products/ai_observability/backend/llm/system_one.py
+++ b/products/ai_observability/backend/llm/system_one.py
@@ -1,11 +1,12 @@
 import math
+from collections.abc import Mapping
 from datetime import UTC, datetime
 from email.utils import parsedate_to_datetime
-from typing import Literal
+from typing import Annotated, Literal
 from urllib.parse import urlsplit
 
 import requests
-from pydantic import BaseModel, Field, ValidationError
+from pydantic import BaseModel, Field, JsonValue, ValidationError
 
 from posthog.security.pinned_requests import SSRFBlockedError, pinned_request
 from posthog.security.url_validation import has_authority_bypass_chars
@@ -22,10 +23,49 @@
     is_context_window_error_message,
 )
 
+type SystemOneContent = str | dict[str, JsonValue] | list[JsonValue]
+type Probability = Annotated[float, Field(strict=True, ge=0, le=1, allow_inf_nan=False)]
+
+
+class NoulQuestion(BaseModel):
+    type: Literal["noul"] = "noul"
+    instructions: SystemOneContent | None = None
+    criteria: dict[Literal["true", "false"], SystemOneContent | None] | None = None
+
+
+class ScoreQuestion(BaseModel):
+    type: Literal["score"] = "score"
+    instructions: SystemOneContent | None = None
+    criteria: list[SystemOneContent] = Field(min_length=1, max_length=10)
+
+
+class ChoiceQuestion(BaseModel):
+    type: Literal["choice"] = "choice"
+    instructions: SystemOneContent | None = None
+    criteria: dict[str, SystemOneContent | None] = Field(min_length=1, max_length=255)
+
+
+type SystemOneQuestion = NoulQuestion | ScoreQuestion | ChoiceQuestion
+
 
 class NoulAnswer(BaseModel):
     type: Literal["noul"]
-    noul: float = Field(strict=True, ge=0, le=1, allow_inf_nan=False)
+    noul: Probability
+
+
+class ScoreAnswer(BaseModel):
+    type: Literal["score"]
+    score: float = Field(strict=True, ge=0, allow_inf_nan=False)
+    legend: dict[str, SystemOneContent]
+    probabilities: dict[str, Probability]
+    confidence: Probability
+
+
+class ChoiceAnswer(BaseModel):
+    type: Literal["choice"]
+    choice: str
+    probabilities: dict[str, Probability]
+    confidence: Probability
 
 
 class SystemOneUsage(BaseModel):
@@ -35,7 +75,7 @@ class SystemOneUsage(BaseModel):
 
 class SystemOneResponse(BaseModel):
     model: str = Field(min_length=1)
-    answers: dict[str, NoulAnswer]
+    answers: dict[str, Annotated[NoulAnswer | ScoreAnswer | ChoiceAnswer, Field(discriminator="type")]]
     usage: SystemOneUsage
 
 
@@ -79,27 +119,31 @@ def normalize_base_url(base_url: str) -> str:
         return base_url.rstrip("/")
 
     @staticmethod
-    def evaluate_boolean(
-        *, api_key: str, model: str, prompt: str, source: str, allows_na: bool, base_url: str = BASE_URL
+    def evaluate(
+        *,
+        api_key: str,
+        model: str,
+        state: SystemOneContent,
+        questions: Mapping[str, SystemOneQuestion],
+        base_url: str = BASE_URL,
     ) -> SystemOneResponse:
         base_url = SystemOneClient.normalize_base_url(base_url)
         if not api_key and base_url == SystemOneClient.BASE_URL:
             raise AuthenticationError("A TypeSafe API key is required.")
-        questions = {"verdict": {"type": "noul", "instructions": prompt}}
-        if allows_na:
-            questions["applicable"] = {
-                "type": "noul",
-                "instructions": (
-                    "Do these evaluation criteria apply to this input? Answer true when the criteria can be "
-                    "evaluated, even if they are not met. Answer false only when they are not relevant.\n\n" + prompt
-                ),
-            }
+        if not questions:
+            raise ValueError("Provide at least one System One question.")
         try:
             response = pinned_request(
                 "POST",
                 f"{base_url}/systemone",
                 headers={"Authorization": f"Bearer {api_key}"} if api_key else {},
-                json={"model": model, "state": source, "questions": questions},
+                json={
+                    "model": model,
+                    "state": state,
+                    "questions": {
+                        key: question.model_dump(mode="json", exclude_none=True) for key, question in questions.items()
+                    },
+                },
                 timeout=60,
             )
         except SSRFBlockedError as error:
@@ -131,23 +175,46 @@ def evaluate_boolean(
             raise StructuredOutputParseError(
                 "The endpoint returned an invalid System One response. Check compatibility."
             ) from error
-        # boffin: Missing answers cannot become a false verdict.
+        # Missing or mismatched answers cannot become valid evaluation results.
         if not questions.keys() <= result.answers.keys():
             raise StructuredOutputParseError(
                 "The endpoint did not answer every evaluation question. Check compatibility."
             )
+        for key, question in questions.items():
+            answer = result.answers[key]
+            if answer.type != question.type:
+                raise StructuredOutputParseError("The endpoint returned the wrong answer type. Check compatibility.")
+            if isinstance(question, ChoiceQuestion) and isinstance(answer, ChoiceAnswer):
+                if set(answer.probabilities) != set(question.criteria) or answer.choice not in question.criteria:
+                    raise StructuredOutputParseError("The endpoint returned different choices. Check compatibility.")
+            if isinstance(question, ScoreQuestion) and isinstance(answer, ScoreAnswer):
+                levels = {str(index) for index in range(len(question.criteria))}
+                if (
+                    set(answer.probabilities) != levels
+                    or set(answer.legend) != levels
+                    or answer.score > len(levels) - 1
+                ):
+                    raise StructuredOutputParseError(
+                        "The endpoint returned an invalid score scale. Check compatibility."
+                    )
+            # Compatible servers round each probability to four decimal places.
+            if isinstance(answer, ScoreAnswer | ChoiceAnswer) and not math.isclose(
+                sum(answer.probabilities.values()), 1.0, abs_tol=max(0.001, len(answer.probabilities) * 0.00005)
+            ):
+                raise StructuredOutputParseError(
+                    "The endpoint returned an invalid probability distribution. Check compatibility."
+                )
         return result
 
     @staticmethod
     def validate_key(api_key: str, *, base_url: str = BASE_URL, model: str = MODEL) -> tuple[str, str | None]:
         try:
-            SystemOneClient.evaluate_boolean(
+            SystemOneClient.evaluate(
                 api_key=api_key,
                 base_url=base_url,
                 model=model,
-                prompt="Does the text contain a greeting?",
-                source="Hello!",
-                allows_na=True,
+                state="Hello!",
+                questions={"verdict": NoulQuestion(instructions="Does the text contain a greeting?")},
             )
         except (AuthenticationError, ModelPermissionError) as error:
             return "invalid", str(error)
diff --git a/products/ai_observability/backend/llm/test/test_system_one.py b/products/ai_observability/backend/llm/test/test_system_one.py
index f9d0b998d3c7..4cb03bf4b674 100644
--- a/products/ai_observability/backend/llm/test/test_system_one.py
+++ b/products/ai_observability/backend/llm/test/test_system_one.py
@@ -12,7 +12,14 @@
     StructuredOutputParseError,
 )
 from products.ai_observability.backend.llm.system_one import (
+    ChoiceAnswer,
+    ChoiceQuestion,
+    NoulAnswer,
+    NoulQuestion,
+    ScoreAnswer,
+    ScoreQuestion,
     SystemOneClient,
+    SystemOneQuestion,
     SystemOneRateLimitError,
     SystemOneRequestRejectedError,
 )
@@ -35,31 +42,135 @@ def test_typesafe_key_validation(status: int, expected_state: str) -> None:
     assert request.call_args.kwargs["headers"]["Authorization"] == "Bearer test-typesafe-key"
 
 
-@pytest.mark.parametrize("allows_na", [False, True])
-def test_typesafe_boolean_request(allows_na: bool) -> None:
+def test_typesafe_mixed_question_request() -> None:
     response = Mock(status_code=200)
     response.json.return_value = {
         "model": "jev-1.13.0",
-        "answers": {"verdict": {"type": "noul", "noul": 0.8}, "applicable": {"type": "noul", "noul": 0.2}},
+        "answers": {
+            "verdict": {"type": "noul", "noul": 0.8},
+            "quality": {
+                "type": "score",
+                "score": 1.25,
+                "legend": {"0": "Poor", "1": "Fair", "2": "Good"},
+                "probabilities": {"0": 0.1, "1": 0.55, "2": 0.35},
+                "confidence": 0.6,
+            },
+            "language": {"type": "choice", "choice": "en", "probabilities": {"en": 0.7, "es": 0.3}, "confidence": 0.4},
+        },
         "usage": {"input_tokens": 120, "output_tokens": 10},
     }
     with patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response) as request:
-        result = SystemOneClient.evaluate_boolean(
+        result = SystemOneClient.evaluate(
             api_key="test-typesafe-key",
             model="jev-1.13.0",
-            prompt="Is the response polite?",
-            source="Hello!",
-            allows_na=allows_na,
+            state={"text": "Hello!"},
+            questions={
+                "verdict": NoulQuestion(instructions="Is the response polite?"),
+                "quality": ScoreQuestion(instructions="Assess quality", criteria=["Poor", "Fair", "Good"]),
+                "language": ChoiceQuestion(
+                    instructions={"question": "Which language?"}, criteria={"en": None, "es": None}
+                ),
+            },
         )
 
+    assert isinstance(result.answers["verdict"], NoulAnswer)
     assert result.answers["verdict"].noul == 0.8
+    assert isinstance(result.answers["quality"], ScoreAnswer)
+    assert result.answers["quality"].score == 1.25
+    assert isinstance(result.answers["language"], ChoiceAnswer)
+    assert result.answers["language"].choice == "en"
     body = request.call_args.kwargs["json"]
-    assert body["state"] == "Hello!"
+    assert body["state"] == {"text": "Hello!"}
     assert body["questions"]["verdict"] == {"type": "noul", "instructions": "Is the response polite?"}
-    assert ("applicable" in body["questions"]) == allows_na
+    assert body["questions"]["quality"] == {
+        "type": "score",
+        "instructions": "Assess quality",
+        "criteria": ["Poor", "Fair", "Good"],
+    }
+    assert body["questions"]["language"] == {
+        "type": "choice",
+        "instructions": {"question": "Which language?"},
+        "criteria": {"en": None, "es": None},
+    }
     assert request.call_args.args[1] == "https://api.typesafe.ai/v1/systemone"
 
 
+@pytest.mark.parametrize(
+    "question,answer",
+    [
+        (ScoreQuestion(criteria=["Poor", "Good"]), {"type": "noul", "noul": 0.8}),
+        *[
+            (
+                ScoreQuestion(criteria=["Poor", "Good"]),
+                {
+                    "type": "score",
+                    "score": 0.75,
+                    "confidence": 0.5,
+                    "legend": {"0": "Poor", "1": "Good"},
+                    "probabilities": {"0": 0.25, "1": 0.75},
+                    field: value,
+                },
+            )
+            for field, value in [
+                ("score", 1.1),
+                ("score", True),
+                ("score", float("nan")),
+                ("legend", {"0": "Poor"}),
+                ("probabilities", {"0": 0.25, "2": 0.75}),
+                ("probabilities", {"0": 0.25, "1": 0.25}),
+            ]
+        ],
+        (
+            ChoiceQuestion(criteria={"en": None, "es": None}),
+            {"type": "choice", "choice": "fr", "confidence": 0.5, "probabilities": {"en": 0.25, "es": 0.75}},
+        ),
+    ],
+)
+def test_rejects_answers_that_do_not_match_the_requested_scale(
+    question: SystemOneQuestion, answer: dict[str, object]
+) -> None:
+    response = Mock(status_code=200)
+    response.json.return_value = {
+        "model": "custom-model",
+        "answers": {"result": answer},
+        "usage": {"input_tokens": 12, "output_tokens": 2},
+    }
+    with (
+        patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response),
+        pytest.raises(StructuredOutputParseError),
+    ):
+        SystemOneClient.evaluate(
+            api_key="example-token", model="custom-model", state="Hello!", questions={"result": question}
+        )
+
+
+def test_choice_accepts_a_rounded_distribution_with_many_options() -> None:
+    criteria = {str(index): None for index in range(255)}
+    response = Mock(status_code=200)
+    response.json.return_value = {
+        "model": "custom-model",
+        "answers": {
+            "result": {
+                "type": "choice",
+                "choice": "0",
+                "confidence": 0,
+                "probabilities": dict.fromkeys(criteria, 0.0039),
+            }
+        },
+        "usage": {"input_tokens": 12, "output_tokens": 0},
+    }
+    with patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response):
+        result = SystemOneClient.evaluate(
+            api_key="",
+            base_url="https://decisions.example.com/v1",
+            model="custom-model",
+            state="Hello!",
+            questions={"result": ChoiceQuestion(criteria=criteria)},
+        )
+    assert isinstance(result.answers["result"], ChoiceAnswer)
+    assert result.answers["result"].choice == "0"
+
+
 @pytest.mark.parametrize("probability", [-0.1, 1.1, float("nan"), float("inf"), "0.8", True, None])
 def test_typesafe_rejects_invalid_probabilities(probability: object) -> None:
     response = Mock(status_code=200)
@@ -72,12 +183,11 @@ def test_typesafe_rejects_invalid_probabilities(probability: object) -> None:
         patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response),
         pytest.raises(StructuredOutputParseError),
     ):
-        SystemOneClient.evaluate_boolean(
+        SystemOneClient.evaluate(
             api_key="test-typesafe-key",
             model="jev-1.13.0",
-            prompt="Is the response polite?",
-            source="Hello!",
-            allows_na=False,
+            state="Hello!",
+            questions={"verdict": NoulQuestion(instructions="Is the response polite?")},
         )
 
 
@@ -88,12 +198,11 @@ def test_typesafe_rate_limits_are_retryable(status: int) -> None:
         patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response),
         pytest.raises(SystemOneRateLimitError) as error,
     ):
-        SystemOneClient.evaluate_boolean(
+        SystemOneClient.evaluate(
             api_key="test-typesafe-key",
             model="jev-1.13.0",
-            prompt="Is the response polite?",
-            source="Hello!",
-            allows_na=False,
+            state="Hello!",
+            questions={"verdict": NoulQuestion(instructions="Is the response polite?")},
         )
     assert error.value.retry_after == 15
 
@@ -103,8 +212,8 @@ def test_typesafe_requires_a_key() -> None:
         patch("products.ai_observability.backend.llm.system_one.pinned_request") as request,
         pytest.raises(AuthenticationError),
     ):
-        SystemOneClient.evaluate_boolean(
-            api_key="", model="jev-1.13.0", prompt="Polite?", source="Hello!", allows_na=False
+        SystemOneClient.evaluate(
+            api_key="", model="jev-1.13.0", state="Hello!", questions={"verdict": NoulQuestion(instructions="Polite?")}
         )
     request.assert_not_called()
 
@@ -121,8 +230,14 @@ def test_typesafe_requires_every_requested_answer(answers: dict[str, object]) ->
         patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response),
         pytest.raises(StructuredOutputParseError),
     ):
-        SystemOneClient.evaluate_boolean(
-            api_key="test-typesafe-key", model="jev-1.13.0", prompt="Polite?", source="Hello!", allows_na=True
+        SystemOneClient.evaluate(
+            api_key="test-typesafe-key",
+            model="jev-1.13.0",
+            state="Hello!",
+            questions={
+                "verdict": NoulQuestion(instructions="Polite?"),
+                "applicable": NoulQuestion(instructions="Relevant?"),
+            },
         )
 
 
@@ -143,8 +258,11 @@ def test_typesafe_preserves_error_categories(status: int, message: str, error_ty
         patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response),
         pytest.raises(error_type),
     ):
-        SystemOneClient.evaluate_boolean(
-            api_key="test-typesafe-key", model="jev-1.13.0", prompt="Polite?", source="Hello!", allows_na=False
+        SystemOneClient.evaluate(
+            api_key="test-typesafe-key",
+            model="jev-1.13.0",
+            state="Hello!",
+            questions={"verdict": NoulQuestion(instructions="Polite?")},
         )
 
 
@@ -158,19 +276,22 @@ def test_custom_endpoint_and_model(api_key: str) -> None:
         "latency_ms": 42,
     }
     with patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response) as request:
-        result = SystemOneClient.evaluate_boolean(
+        result = SystemOneClient.evaluate(
             api_key=api_key,
             base_url="https://decisions.example.com/v1/",
             model="custom-model",
-            prompt="Polite?",
-            source="Hello!",
-            allows_na=True,
+            state="Hello!",
+            questions={
+                "verdict": NoulQuestion(instructions="Polite?"),
+                "applicable": NoulQuestion(instructions="Relevant?"),
+            },
         )
     assert request.call_args.args == ("POST", "https://decisions.example.com/v1/systemone")
     assert request.call_args.kwargs["headers"] == ({"Authorization": f"Bearer {api_key}"} if api_key else {})
     assert request.call_args.kwargs["json"]["model"] == "custom-model"
     assert result.model == "custom-model-revision"
     assert result.usage.output_tokens == 0
+    assert isinstance(result.answers["verdict"], NoulAnswer)
     assert result.answers["verdict"].noul == 0.7
 
 
diff --git a/products/ai_observability/backend/models/evaluation_configs.py b/products/ai_observability/backend/models/evaluation_configs.py
index a45fd06ec3f4..2c928a3eaf02 100644
--- a/products/ai_observability/backend/models/evaluation_configs.py
+++ b/products/ai_observability/backend/models/evaluation_configs.py
@@ -81,17 +81,35 @@ class NumericOutputConfig(BaseModel):
     min: float | None = None
     max: float | None = None
     step: float | None = Field(default=None, gt=0)
+    score_levels: list[str] | None = Field(default=None, min_length=2, max_length=10)
     allows_na: bool = False
     passing_rule: NumericPassingRule | None = None
 
+    @field_validator("score_levels")
+    @classmethod
+    def validate_score_levels(cls, value: list[str] | None) -> list[str] | None:
+        if value is not None:
+            value = [level.strip() for level in value]
+            if any(not level for level in value):
+                raise ValueError("Describe every score level")
+        return value
+
     @model_validator(mode="after")
     def validate_bounds(self) -> Self:
         if self.min is not None and self.max is not None and self.min > self.max:
             raise ValueError("Minimum score cannot exceed maximum score")
+        if self.score_levels is not None and (self.min is None or self.max is None or self.min >= self.max):
+            raise ValueError("Score levels require a minimum and a greater maximum")
         if self.passing_rule is not None:
             self.passing_rule.threshold = self.validate_score(self.passing_rule.threshold)
         return self
 
+    def score_from_level(self, level: float) -> float:
+        if self.score_levels is None or self.min is None or self.max is None:
+            raise ValueError("Configure score levels and bounds before using this judge")
+        fraction = level / (len(self.score_levels) - 1)
+        return self.validate_score((1 - fraction) * self.min + fraction * self.max)
+
     def validate_score(self, value: object) -> float:
         if isinstance(value, bool) or not isinstance(value, int | float):
             raise ValueError("Numeric evaluations must return a finite number")
diff --git a/products/ai_observability/backend/models/provider_keys.py b/products/ai_observability/backend/models/provider_keys.py
index f2077b51f7b1..1d3acbdc409e 100644
--- a/products/ai_observability/backend/models/provider_keys.py
+++ b/products/ai_observability/backend/models/provider_keys.py
@@ -22,7 +22,7 @@ class LLMProvider(models.TextChoices):
     TOGETHER_AI = "together_ai", "Together AI"
     MINIMAX = "minimax", "MiniMax"
     ZEABUR = "zeabur", "Zeabur AI Hub"
-    TYPESAFE = "typesafe", "System One (Jev)"
+    TYPESAFE = "typesafe", "System One"
 
 
 def llm_provider_choices() -> list[tuple[str, str | Promise]]:
diff --git a/products/ai_observability/backend/models/test/test_evaluation_configs.py b/products/ai_observability/backend/models/test/test_evaluation_configs.py
index f780826f31d5..f075adb85ec9 100644
--- a/products/ai_observability/backend/models/test/test_evaluation_configs.py
+++ b/products/ai_observability/backend/models/test/test_evaluation_configs.py
@@ -57,6 +57,12 @@ def test_numeric_configuration(self, runtime, config):
             {"passing_rule": {"operator": "gte", "threshold": True}},
             {"passing_rule": {"operator": "gte", "threshold": float("nan")}},
             {"min": 0, "passing_rule": {"operator": "gte", "threshold": -1}},
+            {"score_levels": ["Poor", "Good"]},
+            {"min": 0, "score_levels": ["Poor", "Good"]},
+            {"min": 0, "max": 0, "score_levels": ["Poor", "Good"]},
+            {"min": 0, "max": 10, "score_levels": ["Only one"]},
+            {"min": 0, "max": 10, "score_levels": ["Level"] * 11},
+            {"min": 0, "max": 10, "score_levels": ["Poor", "  "]},
         ],
     )
     def test_invalid_numeric_configuration(self, output):
@@ -72,6 +78,12 @@ def test_unbounded_and_nullable_configuration(self):
         )
         assert output == {"allows_na": True}
 
+    @pytest.mark.parametrize("level,expected", [(0, -10), (1.25, 2.5), (2, 10)])
+    def test_score_levels_keep_fractional_values(self, level: float, expected: float) -> None:
+        config = NumericOutputConfig(min=-10, max=10, score_levels=[" Poor ", "Fair", "Good"])
+        assert config.score_levels == ["Poor", "Fair", "Good"]
+        assert config.score_from_level(level) == expected
+
 
 class TestValidateTargetConfig:
     @pytest.mark.parametrize(
diff --git a/products/ai_observability/frontend/components/EvaluationExplanation.tsx b/products/ai_observability/frontend/components/EvaluationExplanation.tsx
index 1a6262e677d4..9cebc004cd3c 100644
--- a/products/ai_observability/frontend/components/EvaluationExplanation.tsx
+++ b/products/ai_observability/frontend/components/EvaluationExplanation.tsx
@@ -9,7 +9,7 @@ export function EvaluationExplanation({
 }): JSX.Element {
     if (probability != null) {
         return (
-            <Tooltip title="Jev's estimated probability that the evaluation criteria are true. Jev does not provide written reasoning.">
+            <Tooltip title="The judge's estimated probability that the evaluation criteria are true. This result has no written reasoning.">
                 <span>{`${(probability * 100).toFixed(1)}% probability of true`}</span>
             </Tooltip>
         )
diff --git a/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx b/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx
index 0b74aeeb660d..d32f6b3c1345 100644
--- a/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx
+++ b/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx
@@ -180,7 +180,10 @@ export function AIObservabilityEvaluation(): JSX.Element {
           : !hasSelectedJudgeModel
             ? 'Select a judge model before saving'
             : evaluation.output_type === 'numeric'
-              ? (numericOutputConfigError(evaluation.output_config) ?? undefined)
+              ? (numericOutputConfigError(
+                    evaluation.output_config,
+                    evaluation.model_configuration?.provider === 'typesafe'
+                ) ?? undefined)
               : undefined
 
     const focusTriggers = (): void => {
@@ -746,6 +749,9 @@ export function AIObservabilityEvaluation(): JSX.Element {
                                                 <NumericEvaluationConfig
                                                     config={evaluation.output_config}
                                                     onChange={patchOutputConfig}
+                                                    requiresScoreLevels={
+                                                        evaluation.model_configuration?.provider === 'typesafe'
+                                                    }
                                                 />
                                             )}
                                             <LemonField.Pure label="Description (optional)">
@@ -930,7 +936,10 @@ function EvaluationModelPicker(): JSX.Element {
     // Evals always run on the team's own provider key, so only BYOK models are offered.
     const selectedModelName = byokModels.find((m) => m.id === selectedModel)?.name
     const groups = evaluationProviderModelGroups.filter(
-        (group) => evaluation?.output_type === 'boolean' || group.provider !== 'typesafe'
+        (group) =>
+            evaluation?.output_type === 'boolean' ||
+            evaluation?.output_type === 'numeric' ||
+            group.provider !== 'typesafe'
     )
     const loading = byokModelsLoading || providerKeysLoading
 
@@ -957,12 +966,13 @@ function EvaluationModelPicker(): JSX.Element {
                             data-attr="evaluation-model-selector"
                         />
                         <ByokModelPickerNotice forEvaluation />
-                        {evaluation?.model_configuration?.provider === 'typesafe' && (
-                            <p className="text-sm text-muted mt-2">
-                                System One returns a probability without written reasoning. A probability of 50% or higher
-                                produces a true result.
-                            </p>
-                        )}
+                        {evaluation?.model_configuration?.provider === 'typesafe' &&
+                            evaluation.output_type === 'boolean' && (
+                                <p className="text-sm text-muted mt-2">
+                                    This judge returns a probability without written reasoning. A probability of 50% or
+                                    higher produces a true result.
+                                </p>
+                            )}
                         {modelSelectionRequired && !selectedModel && (
                             <p className="text-sm text-danger mt-1">Select a judge model.</p>
                         )}
diff --git a/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.stories.tsx b/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.stories.tsx
index ccde3ae802c9..a34411314f2e 100644
--- a/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.stories.tsx
+++ b/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.stories.tsx
@@ -13,18 +13,22 @@ export default meta
 
 type Story = StoryObj<typeof NumericEvaluationConfig>
 
-function Template({ narrow = false }: { narrow?: boolean }): JSX.Element {
+function Template({ narrow = false, systemOne = false }: { narrow?: boolean; systemOne?: boolean }): JSX.Element {
     const [config, setConfig] = useState<EvaluationOutputConfig>({
         min: 0,
         max: 10,
         step: 0.5,
         allows_na: true,
         passing_rule: { operator: 'gte', threshold: 7 },
+        ...(systemOne
+            ? { score_levels: ['Does not meet the criteria', 'Partly meets the criteria', 'Fully meets the criteria'] }
+            : {}),
     })
     return (
         <div className={narrow ? 'max-w-lg' : 'max-w-4xl'}>
             <NumericEvaluationConfig
                 config={config}
+                requiresScoreLevels={systemOne}
                 onChange={(patch) => setConfig((current) => ({ ...current, ...patch }))}
             />
         </div>
@@ -33,3 +37,4 @@ function Template({ narrow = false }: { narrow?: boolean }): JSX.Element {
 
 export const Default: Story = { render: () => <Template /> }
 export const Narrow: Story = { render: () => <Template narrow /> }
+export const SystemOne: Story = { render: () => <Template systemOne narrow /> }
diff --git a/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.test.tsx b/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.test.tsx
index e263c954fef7..7d4ccd35941c 100644
--- a/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.test.tsx
+++ b/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.test.tsx
@@ -7,6 +7,23 @@ import { NumericEvaluationConfig } from './NumericEvaluationConfig'
 describe('NumericEvaluationConfig', () => {
     afterEach(cleanup)
 
+    it('requires a bounded rubric for System One and keeps level edits', () => {
+        const onChange = jest.fn()
+        const config: EvaluationOutputConfig = { min: 0, max: 10 }
+        expect(numericOutputConfigError(config)).toBeNull()
+        expect(numericOutputConfigError(config, true)).not.toBeNull()
+        const { getByLabelText, rerender } = render(
+            <NumericEvaluationConfig config={config} onChange={onChange} requiresScoreLevels />
+        )
+        fireEvent.change(getByLabelText('Score levels'), { target: { value: 'Poor\nFair\nGood' } })
+        expect(onChange).toHaveBeenLastCalledWith({ score_levels: ['Poor', 'Fair', 'Good'] })
+        const edited = { ...config, ...onChange.mock.calls[0][0] }
+        expect(numericOutputConfigError(edited, true)).toBeNull()
+        expect(numericOutputConfigError({ ...edited, min: null }, true)).not.toBeNull()
+        rerender(<NumericEvaluationConfig config={edited} onChange={onChange} />)
+        expect((getByLabelText('Score levels') as HTMLTextAreaElement).value).toBe('Poor\nFair\nGood')
+    })
+
     it.each([
         ['min', 'Minimum (optional)'],
         ['max', 'Maximum (optional)'],
diff --git a/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.tsx b/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.tsx
index 848d6addc8c6..e6d4d0cf12e2 100644
--- a/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.tsx
+++ b/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.tsx
@@ -1,6 +1,6 @@
 import { useId } from 'react'
 
-import { LemonBanner, LemonInput, LemonSelect, LemonSwitch } from '@posthog/lemon-ui'
+import { LemonBanner, LemonInput, LemonSelect, LemonSwitch, LemonTextArea } from '@posthog/lemon-ui'
 
 import { LemonField } from 'lib/lemon-ui/LemonField'
 
@@ -10,13 +10,16 @@ import type { EvaluationOutputConfig } from '../types'
 export function NumericEvaluationConfig({
     config,
     onChange,
+    requiresScoreLevels = false,
 }: {
     config: EvaluationOutputConfig
     onChange: (patch: EvaluationOutputConfig) => void
+    requiresScoreLevels?: boolean
 }): JSX.Element {
     const id = useId()
     const rule = config.passing_rule
-    const error = numericOutputConfigError(config)
+    const hasScoreLevels = requiresScoreLevels || config.score_levels != null
+    const error = numericOutputConfigError(config, requiresScoreLevels)
     return (
         <div className="space-y-4">
             {error && <LemonBanner type="error">{error}</LemonBanner>}
@@ -25,7 +28,13 @@ export function NumericEvaluationConfig({
                     <LemonField.Pure
                         key={field}
                         htmlFor={`${id}-${field}`}
-                        label={{ min: 'Minimum (optional)', max: 'Maximum (optional)', step: 'Step (optional)' }[field]}
+                        label={
+                            {
+                                min: hasScoreLevels ? 'Minimum' : 'Minimum (optional)',
+                                max: hasScoreLevels ? 'Maximum' : 'Maximum (optional)',
+                                step: 'Step (optional)',
+                            }[field]
+                        }
                     >
                         <LemonInput
                             id={`${id}-${field}`}
@@ -38,7 +47,25 @@ export function NumericEvaluationConfig({
                     </LemonField.Pure>
                 ))}
             </div>
-            <p className="text-muted text-sm">Bounds are inclusive. Step guides scoring without rounding results.</p>
+            <p className="text-muted text-sm">
+                Bounds are inclusive. Step controls input increments without rounding results.
+            </p>
+            {hasScoreLevels && (
+                <LemonField.Pure label="Score levels" htmlFor={`${id}-score-levels`}>
+                    <LemonTextArea
+                        id={`${id}-score-levels`}
+                        value={config.score_levels?.join('\n') ?? ''}
+                        onChange={(value) => onChange({ score_levels: value.split('\n') })}
+                        placeholder={'Does not meet the criteria\nPartly meets the criteria\nFully meets the criteria'}
+                        minRows={3}
+                        data-attr="llma-evaluation-score-levels"
+                    />
+                    <p className="text-muted text-sm">
+                        Describe 2 to 10 levels, one per line, from the minimum to the maximum. Levels are spaced
+                        evenly. Scores can fall between levels. They measure the rubric, not the probability of passing.
+                    </p>
+                </LemonField.Pure>
+            )}
             <LemonSwitch
                 label="Allow N/A responses"
                 checked={config.allows_na ?? false}
diff --git a/products/ai_observability/frontend/evaluations/constants.ts b/products/ai_observability/frontend/evaluations/constants.ts
index 0498837fbed3..25c8687abd51 100644
--- a/products/ai_observability/frontend/evaluations/constants.ts
+++ b/products/ai_observability/frontend/evaluations/constants.ts
@@ -21,7 +21,10 @@ const EVALUATION_RESULT_FALSE_HOGQL = "properties.$ai_evaluation_result = 'false
 export const EVALUATION_NOT_SKIPPED_HOGQL =
     "(isNull(properties.$ai_evaluation_skipped) OR properties.$ai_evaluation_skipped != 'true')"
 
-export function numericOutputConfigError(config: EvaluationOutputConfig): string | null {
+export function numericOutputConfigError(
+    config: EvaluationOutputConfig,
+    requiresScoreLevels: boolean = false
+): string | null {
     const { min, max, step, passing_rule } = config
     if ([min, max, step, passing_rule?.threshold].some((value) => value != null && !Number.isFinite(value))) {
         return 'Enter finite numbers for the score bounds, step, and threshold.'
@@ -29,6 +32,19 @@ export function numericOutputConfigError(config: EvaluationOutputConfig): string
     if (min != null && max != null && min > max) {
         return 'Minimum must be less than or equal to maximum.'
     }
+    if (requiresScoreLevels || config.score_levels != null) {
+        if (min == null || max == null || min >= max) {
+            return 'Set a minimum and a greater maximum for the score levels.'
+        }
+        if (
+            !config.score_levels ||
+            config.score_levels.length < 2 ||
+            config.score_levels.length > 10 ||
+            config.score_levels.some((level) => !level.trim())
+        ) {
+            return 'Describe 2 to 10 score levels, one per line.'
+        }
+    }
     if (step != null && step <= 0) {
         return 'Step must be greater than zero.'
     }
diff --git a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts
index b76fa3d1ad95..e3a91bc5231d 100644
--- a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts
+++ b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts
@@ -658,19 +658,27 @@ return result`,
                 await expectLogic(logic).toMatchValues({ formValid: false })
             })
 
-            it('returns true when all fields valid', async () => {
-                await expectLogic(logic).toDispatchActions(['loadEvaluationSuccess'])
-                logic.actions.setEvaluationName('Valid Name')
-                logic.actions.setEvaluationPrompt('Valid prompt')
-                logic.actions.setTriggerConditions([{ id: 'c1', rollout_percentage: 50, properties: [] }])
-                logic.actions.setModelConfiguration({
-                    provider: 'openai',
-                    model: 'gpt-5-mini',
-                    provider_key_id: 'key-1',
-                })
+            it.each(['openai', 'typesafe'] as const)(
+                'validates boolean and numeric fields for %s',
+                async (provider) => {
+                    await expectLogic(logic).toDispatchActions(['loadEvaluationSuccess'])
+                    logic.actions.setEvaluationName('Valid Name')
+                    logic.actions.setEvaluationPrompt('Valid prompt')
+                    logic.actions.setTriggerConditions([{ id: 'c1', rollout_percentage: 50, properties: [] }])
+                    logic.actions.setModelConfiguration({
+                        provider,
+                        model: provider === 'typesafe' ? 'jev-1.13.0' : 'gpt-5-mini',
+                        provider_key_id: 'key-1',
+                    })
 
-                await expectLogic(logic).toMatchValues({ formValid: true })
-            })
+                    await expectLogic(logic).toMatchValues({ formValid: true })
+                    logic.actions.setOutputType('numeric')
+                    logic.actions.patchOutputConfig({ min: 0, max: 10 })
+                    await expectLogic(logic).toMatchValues({ formValid: provider !== 'typesafe' })
+                    logic.actions.patchOutputConfig({ score_levels: ['Poor', 'Good'] })
+                    await expectLogic(logic).toMatchValues({ formValid: true })
+                }
+            )
 
             // A loaded evaluation whose stored shape doesn't match its type (e.g. an llm_judge
             // record with no prompt) used to crash formValid with a TypeError on render.
diff --git a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts
index bc11da83705e..35d2d86a02fb 100644
--- a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts
+++ b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts
@@ -1437,7 +1437,13 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
                 if (!evaluation) {
                     return false
                 }
-                if (evaluation.output_type === 'numeric' && numericOutputConfigError(evaluation.output_config)) {
+                if (
+                    evaluation.output_type === 'numeric' &&
+                    numericOutputConfigError(
+                        evaluation.output_config,
+                        evaluation.model_configuration?.provider === 'typesafe'
+                    )
+                ) {
                     return false
                 }
                 const hasValidName = (evaluation.name?.length ?? 0) > 0
diff --git a/products/ai_observability/frontend/generated/api.schemas.ts b/products/ai_observability/frontend/generated/api.schemas.ts
index c7a074714f84..aff4c56651ce 100644
--- a/products/ai_observability/frontend/generated/api.schemas.ts
+++ b/products/ai_observability/frontend/generated/api.schemas.ts
@@ -870,7 +870,7 @@ export const EvaluationTargetEnumApi = {
  * * `together_ai` - Together AI
  * * `minimax` - MiniMax
  * * `zeabur` - Zeabur AI Hub
- * * `typesafe` - System One (Jev)
+ * * `typesafe` - System One
  */
 export type LLMProviderEnumApi = (typeof LLMProviderEnumApi)[keyof typeof LLMProviderEnumApi]
 
@@ -938,7 +938,7 @@ export type EvaluationApiOutputConfigPassingRule = {
 } | null
 
 /**
- * Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min/max/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.
+ * Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min/max/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.
  */
 export type EvaluationApiOutputConfig = {
     /** Whether the evaluation can return N/A for non-applicable generations. */
@@ -962,6 +962,14 @@ export type EvaluationApiOutputConfig = {
      * @nullable
      */
     step?: number | null
+    /**
+     * Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.
+     * @minItems 2
+     * @maxItems 10
+     * @nullable
+     * @items.minLength 1
+     */
+    score_levels?: string[] | null
     /**
      * Optional numeric passing rule. Null removes the rule; historical scores use the current rule.
      * @nullable
@@ -1040,7 +1048,7 @@ export interface EvaluationApi {
      * * `numeric` - Numeric
      * * `sentiment` - Sentiment */
     output_type: OutputTypeEnumApi
-    /** Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min/max/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}. */
+    /** Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min/max/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}. */
     output_config?: EvaluationApiOutputConfig
     /** Trigger conditions that filter which events are evaluated. OR between condition sets, AND within each. Each set is {id, rollout_percentage, properties[]} — `rollout_percentage` (0-100, defaults to 100) is the sampling field the dispatcher reads. */
     conditions?: EvaluationConditionApi[]
@@ -1212,7 +1220,7 @@ export type PatchedEvaluationApiOutputConfigPassingRule = {
 } | null
 
 /**
- * Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min/max/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.
+ * Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min/max/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.
  */
 export type PatchedEvaluationApiOutputConfig = {
     /** Whether the evaluation can return N/A for non-applicable generations. */
@@ -1236,6 +1244,14 @@ export type PatchedEvaluationApiOutputConfig = {
      * @nullable
      */
     step?: number | null
+    /**
+     * Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.
+     * @minItems 2
+     * @maxItems 10
+     * @nullable
+     * @items.minLength 1
+     */
+    score_levels?: string[] | null
     /**
      * Optional numeric passing rule. Null removes the rule; historical scores use the current rule.
      * @nullable
@@ -1314,7 +1330,7 @@ export interface PatchedEvaluationApi {
      * * `numeric` - Numeric
      * * `sentiment` - Sentiment */
     output_type?: OutputTypeEnumApi
-    /** Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min/max/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}. */
+    /** Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min/max/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}. */
     output_config?: PatchedEvaluationApiOutputConfig
     /** Trigger conditions that filter which events are evaluated. OR between condition sets, AND within each. Each set is {id, rollout_percentage, properties[]} — `rollout_percentage` (0-100, defaults to 100) is the sampling field the dispatcher reads. */
     conditions?: EvaluationConditionApi[]
@@ -1377,6 +1393,14 @@ export type TestHogRequestApiOutputConfig = {
      * @nullable
      */
     step?: number | null
+    /**
+     * Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.
+     * @minItems 2
+     * @maxItems 10
+     * @nullable
+     * @items.minLength 1
+     */
+    score_levels?: string[] | null
     /**
      * Optional numeric passing rule. Null removes the rule; historical scores use the current rule.
      * @nullable
@@ -2064,6 +2088,14 @@ export type EvaluationReportMetricsApiOutputConfig = {
      * @nullable
      */
     step?: number | null
+    /**
+     * Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.
+     * @minItems 2
+     * @maxItems 10
+     * @nullable
+     * @items.minLength 1
+     */
+    score_levels?: string[] | null
     /**
      * Optional numeric passing rule. Null removes the rule; historical scores use the current rule.
      * @nullable
diff --git a/products/ai_observability/frontend/generated/api.zod.ts b/products/ai_observability/frontend/generated/api.zod.ts
index 28729b394c69..477dfaeac6d6 100644
--- a/products/ai_observability/frontend/generated/api.zod.ts
+++ b/products/ai_observability/frontend/generated/api.zod.ts
@@ -261,6 +261,9 @@ export const evaluationsCreateBodyNameMax = 400
 export const evaluationsCreateBodyEvaluationConfigThreeSourceDefault = `user_messages`
 export const evaluationsCreateBodyOutputConfigStepExclusiveMin = 0
 
+export const evaluationsCreateBodyOutputConfigScoreLevelsMin = 2
+export const evaluationsCreateBodyOutputConfigScoreLevelsMax = 10
+
 export const evaluationsCreateBodyConditionsItemIdMax = 100
 
 export const evaluationsCreateBodyConditionsItemRolloutPercentageDefault = 100
@@ -350,6 +353,14 @@ export const EvaluationsCreateBody = /* @__PURE__ */ zod
                     .gt(evaluationsCreateBodyOutputConfigStepExclusiveMin)
                     .nullish()
                     .describe('Optional positive input increment. Does not round evaluation results.'),
+                score_levels: zod
+                    .array(zod.string().min(1))
+                    .min(evaluationsCreateBodyOutputConfigScoreLevelsMin)
+                    .max(evaluationsCreateBodyOutputConfigScoreLevelsMax)
+                    .nullish()
+                    .describe(
+                        'Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.'
+                    ),
                 passing_rule: zod
                     .object({
                         operator: zod
@@ -366,7 +377,7 @@ export const EvaluationsCreateBody = /* @__PURE__ */ zod
             })
             .optional()
             .describe(
-                "Output config. For 'boolean' output_type: {allows_na} to permit N\/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min\/max\/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}."
+                "Output config. For 'boolean' output_type: {allows_na} to permit N\/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min\/max\/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}."
             ),
         conditions: zod
             .array(
@@ -463,7 +474,7 @@ export const EvaluationsCreateBody = /* @__PURE__ */ zod
                                 'typesafe',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One'
                             ),
                         model: zod.string().max(evaluationsCreateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -588,6 +599,9 @@ export const evaluationsUpdateBodyNameMax = 400
 export const evaluationsUpdateBodyEvaluationConfigThreeSourceDefault = `user_messages`
 export const evaluationsUpdateBodyOutputConfigStepExclusiveMin = 0
 
+export const evaluationsUpdateBodyOutputConfigScoreLevelsMin = 2
+export const evaluationsUpdateBodyOutputConfigScoreLevelsMax = 10
+
 export const evaluationsUpdateBodyConditionsItemIdMax = 100
 
 export const evaluationsUpdateBodyConditionsItemRolloutPercentageDefault = 100
@@ -677,6 +691,14 @@ export const EvaluationsUpdateBody = /* @__PURE__ */ zod
                     .gt(evaluationsUpdateBodyOutputConfigStepExclusiveMin)
                     .nullish()
                     .describe('Optional positive input increment. Does not round evaluation results.'),
+                score_levels: zod
+                    .array(zod.string().min(1))
+                    .min(evaluationsUpdateBodyOutputConfigScoreLevelsMin)
+                    .max(evaluationsUpdateBodyOutputConfigScoreLevelsMax)
+                    .nullish()
+                    .describe(
+                        'Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.'
+                    ),
                 passing_rule: zod
                     .object({
                         operator: zod
@@ -693,7 +715,7 @@ export const EvaluationsUpdateBody = /* @__PURE__ */ zod
             })
             .optional()
             .describe(
-                "Output config. For 'boolean' output_type: {allows_na} to permit N\/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min\/max\/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}."
+                "Output config. For 'boolean' output_type: {allows_na} to permit N\/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min\/max\/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}."
             ),
         conditions: zod
             .array(
@@ -790,7 +812,7 @@ export const EvaluationsUpdateBody = /* @__PURE__ */ zod
                                 'typesafe',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One'
                             ),
                         model: zod.string().max(evaluationsUpdateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -817,6 +839,9 @@ export const evaluationsPartialUpdateBodyNameMax = 400
 export const evaluationsPartialUpdateBodyEvaluationConfigThreeSourceDefault = `user_messages`
 export const evaluationsPartialUpdateBodyOutputConfigStepExclusiveMin = 0
 
+export const evaluationsPartialUpdateBodyOutputConfigScoreLevelsMin = 2
+export const evaluationsPartialUpdateBodyOutputConfigScoreLevelsMax = 10
+
 export const evaluationsPartialUpdateBodyConditionsItemIdMax = 100
 
 export const evaluationsPartialUpdateBodyConditionsItemRolloutPercentageDefault = 100
@@ -908,6 +933,14 @@ export const EvaluationsPartialUpdateBody = /* @__PURE__ */ zod
                     .gt(evaluationsPartialUpdateBodyOutputConfigStepExclusiveMin)
                     .nullish()
                     .describe('Optional positive input increment. Does not round evaluation results.'),
+                score_levels: zod
+                    .array(zod.string().min(1))
+                    .min(evaluationsPartialUpdateBodyOutputConfigScoreLevelsMin)
+                    .max(evaluationsPartialUpdateBodyOutputConfigScoreLevelsMax)
+                    .nullish()
+                    .describe(
+                        'Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.'
+                    ),
                 passing_rule: zod
                     .object({
                         operator: zod
@@ -924,7 +957,7 @@ export const EvaluationsPartialUpdateBody = /* @__PURE__ */ zod
             })
             .optional()
             .describe(
-                "Output config. For 'boolean' output_type: {allows_na} to permit N\/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min\/max\/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}."
+                "Output config. For 'boolean' output_type: {allows_na} to permit N\/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min\/max\/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}."
             ),
         conditions: zod
             .array(
@@ -1021,7 +1054,7 @@ export const EvaluationsPartialUpdateBody = /* @__PURE__ */ zod
                                 'typesafe',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One'
                             ),
                         model: zod.string().max(evaluationsPartialUpdateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -1049,6 +1082,9 @@ export const EvaluationsPartialUpdateBody = /* @__PURE__ */ zod
 export const evaluationsTestHogCreateBodyOutputTypeDefault = `boolean`
 export const evaluationsTestHogCreateBodyOutputConfigStepExclusiveMin = 0
 
+export const evaluationsTestHogCreateBodyOutputConfigScoreLevelsMin = 2
+export const evaluationsTestHogCreateBodyOutputConfigScoreLevelsMax = 10
+
 export const evaluationsTestHogCreateBodySampleCountDefault = 5
 export const evaluationsTestHogCreateBodySampleCountMax = 10
 
@@ -1089,6 +1125,14 @@ export const EvaluationsTestHogCreateBody = /* @__PURE__ */ zod.object({
                 .gt(evaluationsTestHogCreateBodyOutputConfigStepExclusiveMin)
                 .nullish()
                 .describe('Optional positive input increment. Does not round evaluation results.'),
+            score_levels: zod
+                .array(zod.string().min(1))
+                .min(evaluationsTestHogCreateBodyOutputConfigScoreLevelsMin)
+                .max(evaluationsTestHogCreateBodyOutputConfigScoreLevelsMax)
+                .nullish()
+                .describe(
+                    'Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.'
+                ),
             passing_rule: zod
                 .object({
                     operator: zod
@@ -1646,7 +1690,7 @@ export const LlmAnalyticsProviderKeysCreateBody = /* @__PURE__ */ zod.object({
             'typesafe',
         ])
         .describe(
-            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
+            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One'
         ),
     name: zod.string().max(llmAnalyticsProviderKeysCreateBodyNameMax),
     api_key: zod.string().optional(),
@@ -1688,7 +1732,7 @@ export const LlmAnalyticsProviderKeysUpdateBody = /* @__PURE__ */ zod.object({
             'typesafe',
         ])
         .describe(
-            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
+            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One'
         ),
     name: zod.string().max(llmAnalyticsProviderKeysUpdateBodyNameMax),
     api_key: zod.string().optional(),
@@ -1731,7 +1775,7 @@ export const LlmAnalyticsProviderKeysPartialUpdateBody = /* @__PURE__ */ zod.obj
         ])
         .optional()
         .describe(
-            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
+            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One'
         ),
     name: zod.string().max(llmAnalyticsProviderKeysPartialUpdateBodyNameMax).optional(),
     api_key: zod.string().optional(),
@@ -1773,7 +1817,7 @@ export const LlmAnalyticsProviderKeysValidateCreateBody = /* @__PURE__ */ zod.ob
             'typesafe',
         ])
         .describe(
-            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
+            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One'
         ),
     name: zod.string().max(llmAnalyticsProviderKeysValidateCreateBodyNameMax),
     api_key: zod.string().optional(),
diff --git a/products/ai_observability/frontend/modelPickerLogic.test.ts b/products/ai_observability/frontend/modelPickerLogic.test.ts
index d997dd864bc8..4c2900bfa81d 100644
--- a/products/ai_observability/frontend/modelPickerLogic.test.ts
+++ b/products/ai_observability/frontend/modelPickerLogic.test.ts
@@ -62,7 +62,7 @@ describe('modelPickerLogic', () => {
                                           {
                                               id: model,
                                               name: model,
-                                              provider: 'System One (Jev)',
+                                              provider: 'System One',
                                               is_recommended: true,
                                           },
                                       ],
diff --git a/products/ai_observability/frontend/settings/SystemOneConnectionFields.tsx b/products/ai_observability/frontend/settings/SystemOneConnectionFields.tsx
index a80dc7022be7..6c42142c414f 100644
--- a/products/ai_observability/frontend/settings/SystemOneConnectionFields.tsx
+++ b/products/ai_observability/frontend/settings/SystemOneConnectionFields.tsx
@@ -39,8 +39,9 @@ export function SystemOneConnectionFields(): JSX.Element {
                     />
                 </div>
                 <p className="text-xs text-muted">
-                    Use a public HTTPS endpoint that supports the System One API. Include /v1 in the base URL if
-                    required by your service. Validation sends a short synthetic example to the selected model.
+                    Defaults to TypeSafe's hosted service and the Jev model. Use a public HTTPS endpoint that supports
+                    the System One API. Include /v1 in the base URL if required by your service. Validation sends a
+                    short synthetic example to the selected model.
                 </p>
             </div>
         </details>
diff --git a/products/ai_observability/frontend/settings/llmProviderKeysLogic.test.ts b/products/ai_observability/frontend/settings/llmProviderKeysLogic.test.ts
index 4bbcd45411ac..574ccf3f2432 100644
--- a/products/ai_observability/frontend/settings/llmProviderKeysLogic.test.ts
+++ b/products/ai_observability/frontend/settings/llmProviderKeysLogic.test.ts
@@ -18,6 +18,7 @@ describe('normalizeLLMProvider', () => {
         ['zeabur', 'zeabur'],
         ['openai', 'openai'],
         ['System One (Jev)', 'typesafe'],
+        ['System One', 'typesafe'],
         ['typesafe', 'typesafe'],
     ])('maps %s to %s', (input, expected) => {
         expect(normalizeLLMProvider(input)).toBe(expected)
diff --git a/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts b/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts
index 95fa115b4e6a..26143e39e72c 100644
--- a/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts
+++ b/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts
@@ -25,7 +25,7 @@ export const LLM_PROVIDER_LABELS: Record<LLMProvider, string> = {
     together_ai: 'Together AI',
     minimax: 'MiniMax',
     zeabur: 'Zeabur AI Hub',
-    typesafe: 'System One (Jev)',
+    typesafe: 'System One',
 }
 
 const LLM_PROVIDERS = new Set<string>(Object.keys(LLM_PROVIDER_LABELS))
@@ -59,7 +59,7 @@ export function normalizeLLMProvider(provider: string | undefined): LLMProvider
     }
 
     const normalized = provider.trim().toLowerCase()
-    if (normalized === 'system one (jev)') {
+    if (normalized === 'system one' || normalized === 'system one (jev)') {
         return 'typesafe'
     }
     if (normalized === 'google' || normalized === 'google-ai-studio') {
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 2a7ff7c390b8..3f652652d89b 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -37668,7 +37668,7 @@ export namespace Schemas {
     } | null;
 
     /**
-     * Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min/max/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.
+     * Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min/max/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.
      */
     export type EvaluationOutputConfig = {
       /** Whether the evaluation can return N/A for non-applicable generations. */
@@ -37692,6 +37692,14 @@ export namespace Schemas {
          * @nullable
          */
       step?: number | null;
+      /**
+         * Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.
+         * @minItems 2
+         * @maxItems 10
+         * @nullable
+         * @items.minLength 1
+         */
+      score_levels?: string[] | null;
       /**
          * Optional numeric passing rule. Null removes the rule; historical scores use the current rule.
          * @nullable
@@ -37841,7 +37849,7 @@ export namespace Schemas {
      * * `together_ai` - Together AI
      * * `minimax` - MiniMax
      * * `zeabur` - Zeabur AI Hub
-     * * `typesafe` - System One (Jev)
+     * * `typesafe` - System One
      */
     export type LLMProviderEnum = typeof LLMProviderEnum[keyof typeof LLMProviderEnum];
 
@@ -37915,7 +37923,7 @@ export namespace Schemas {
        * * `numeric` - Numeric
        * * `sentiment` - Sentiment */
       output_type: OutputTypeEnum;
-      /** Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min/max/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}. */
+      /** Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min/max/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}. */
       output_config?: EvaluationOutputConfig;
       /** Trigger conditions that filter which events are evaluated. OR between condition sets, AND within each. Each set is {id, rollout_percentage, properties[]} — `rollout_percentage` (0-100, defaults to 100) is the sampling field the dispatcher reads. */
       conditions?: EvaluationCondition[];
@@ -38274,6 +38282,14 @@ export namespace Schemas {
          * @nullable
          */
       step?: number | null;
+      /**
+         * Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.
+         * @minItems 2
+         * @maxItems 10
+         * @nullable
+         * @items.minLength 1
+         */
+      score_levels?: string[] | null;
       /**
          * Optional numeric passing rule. Null removes the rule; historical scores use the current rule.
          * @nullable
@@ -71661,7 +71677,7 @@ export namespace Schemas {
     } | null;
 
     /**
-     * Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min/max/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.
+     * Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min/max/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.
      */
     export type PatchedEvaluationOutputConfig = {
       /** Whether the evaluation can return N/A for non-applicable generations. */
@@ -71685,6 +71701,14 @@ export namespace Schemas {
          * @nullable
          */
       step?: number | null;
+      /**
+         * Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.
+         * @minItems 2
+         * @maxItems 10
+         * @nullable
+         * @items.minLength 1
+         */
+      score_levels?: string[] | null;
       /**
          * Optional numeric passing rule. Null removes the rule; historical scores use the current rule.
          * @nullable
@@ -71761,7 +71785,7 @@ export namespace Schemas {
        * * `numeric` - Numeric
        * * `sentiment` - Sentiment */
       output_type?: OutputTypeEnum;
-      /** Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min/max/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}. */
+      /** Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min/max/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}. */
       output_config?: PatchedEvaluationOutputConfig;
       /** Trigger conditions that filter which events are evaluated. OR between condition sets, AND within each. Each set is {id, rollout_percentage, properties[]} — `rollout_percentage` (0-100, defaults to 100) is the sampling field the dispatcher reads. */
       conditions?: EvaluationCondition[];
@@ -98474,6 +98498,14 @@ export namespace Schemas {
          * @nullable
          */
       step?: number | null;
+      /**
+         * Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.
+         * @minItems 2
+         * @maxItems 10
+         * @nullable
+         * @items.minLength 1
+         */
+      score_levels?: string[] | null;
       /**
          * Optional numeric passing rule. Null removes the rule; historical scores use the current rule.
          * @nullable
diff --git a/services/mcp/src/generated/ai_observability/api.ts b/services/mcp/src/generated/ai_observability/api.ts
index ff6bb3457533..f664445fbe30 100644
--- a/services/mcp/src/generated/ai_observability/api.ts
+++ b/services/mcp/src/generated/ai_observability/api.ts
@@ -563,6 +563,9 @@ export const evaluationsCreateBodyNameMax = 400
 export const evaluationsCreateBodyEvaluationConfigThreeSourceDefault = `user_messages`
 export const evaluationsCreateBodyOutputConfigStepExclusiveMin = 0
 
+export const evaluationsCreateBodyOutputConfigScoreLevelsMin = 2
+export const evaluationsCreateBodyOutputConfigScoreLevelsMax = 10
+
 export const evaluationsCreateBodyConditionsItemIdMax = 100
 
 export const evaluationsCreateBodyConditionsItemRolloutPercentageDefault = 100
@@ -652,6 +655,14 @@ export const EvaluationsCreateBody = () => zod
                     .gt(evaluationsCreateBodyOutputConfigStepExclusiveMin)
                     .nullish()
                     .describe('Optional positive input increment. Does not round evaluation results.'),
+                score_levels: zod
+                    .array(zod.string().min(1))
+                    .min(evaluationsCreateBodyOutputConfigScoreLevelsMin)
+                    .max(evaluationsCreateBodyOutputConfigScoreLevelsMax)
+                    .nullish()
+                    .describe(
+                        'Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.'
+                    ),
                 passing_rule: zod
                     .object({
                         operator: zod
@@ -668,7 +679,7 @@ export const EvaluationsCreateBody = () => zod
             })
             .optional()
             .describe(
-                "Output config. For 'boolean' output_type: {allows_na} to permit N\/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min\/max\/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}."
+                "Output config. For 'boolean' output_type: {allows_na} to permit N\/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min\/max\/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}."
             ),
         conditions: zod
             .array(
@@ -765,7 +776,7 @@ export const EvaluationsCreateBody = () => zod
                                 'typesafe',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One'
                             ),
                         model: zod.string().max(evaluationsCreateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -810,6 +821,9 @@ export const evaluationsPartialUpdateBodyNameMax = 400
 export const evaluationsPartialUpdateBodyEvaluationConfigThreeSourceDefault = `user_messages`
 export const evaluationsPartialUpdateBodyOutputConfigStepExclusiveMin = 0
 
+export const evaluationsPartialUpdateBodyOutputConfigScoreLevelsMin = 2
+export const evaluationsPartialUpdateBodyOutputConfigScoreLevelsMax = 10
+
 export const evaluationsPartialUpdateBodyConditionsItemIdMax = 100
 
 export const evaluationsPartialUpdateBodyConditionsItemRolloutPercentageDefault = 100
@@ -901,6 +915,14 @@ export const EvaluationsPartialUpdateBody = () => zod
                     .gt(evaluationsPartialUpdateBodyOutputConfigStepExclusiveMin)
                     .nullish()
                     .describe('Optional positive input increment. Does not round evaluation results.'),
+                score_levels: zod
+                    .array(zod.string().min(1))
+                    .min(evaluationsPartialUpdateBodyOutputConfigScoreLevelsMin)
+                    .max(evaluationsPartialUpdateBodyOutputConfigScoreLevelsMax)
+                    .nullish()
+                    .describe(
+                        'Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.'
+                    ),
                 passing_rule: zod
                     .object({
                         operator: zod
@@ -917,7 +939,7 @@ export const EvaluationsPartialUpdateBody = () => zod
             })
             .optional()
             .describe(
-                "Output config. For 'boolean' output_type: {allows_na} to permit N\/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min\/max\/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}."
+                "Output config. For 'boolean' output_type: {allows_na} to permit N\/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min\/max\/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}."
             ),
         conditions: zod
             .array(
@@ -1014,7 +1036,7 @@ export const EvaluationsPartialUpdateBody = () => zod
                                 'typesafe',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One'
                             ),
                         model: zod.string().max(evaluationsPartialUpdateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -1062,6 +1084,9 @@ export const EvaluationsTestHogCreateParams = () => zod.object({
 export const evaluationsTestHogCreateBodyOutputTypeDefault = `boolean`
 export const evaluationsTestHogCreateBodyOutputConfigStepExclusiveMin = 0
 
+export const evaluationsTestHogCreateBodyOutputConfigScoreLevelsMin = 2
+export const evaluationsTestHogCreateBodyOutputConfigScoreLevelsMax = 10
+
 export const evaluationsTestHogCreateBodySampleCountDefault = 5
 export const evaluationsTestHogCreateBodySampleCountMax = 10
 
@@ -1102,6 +1127,14 @@ export const EvaluationsTestHogCreateBody = () => zod.object({
                 .gt(evaluationsTestHogCreateBodyOutputConfigStepExclusiveMin)
                 .nullish()
                 .describe('Optional positive input increment. Does not round evaluation results.'),
+            score_levels: zod
+                .array(zod.string().min(1))
+                .min(evaluationsTestHogCreateBodyOutputConfigScoreLevelsMin)
+                .max(evaluationsTestHogCreateBodyOutputConfigScoreLevelsMax)
+                .nullish()
+                .describe(
+                    'Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.'
+                ),
             passing_rule: zod
                 .object({
                     operator: zod

From 102cdcdce3f8eaabe05f9fe99171bce8b904b4e9 Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Fri, 25 Sep 2026 13:47:13 +0200
Subject: [PATCH 07/30] chore(aio): use fixed timestamps in system one judge
 tests

---
 posthog/temporal/ai_observability/test_run_evaluation.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index cf84b6dad99c..b6f3ed286bb4 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -152,7 +152,7 @@ def test_typesafe_judge_emits_boolean_probability_without_reasoning(
     assert result["total_tokens"] == 130
     if allows_na:
         assert result["applicable"] is (applicability >= 0.5)
-    properties = build_evaluation_event_properties(evaluation, result, datetime.now(UTC))
+    properties = build_evaluation_event_properties(evaluation, result, datetime(2026, 1, 1, tzinfo=UTC))
     assert properties["$ai_evaluation_probability"] == probability
     assert properties["$ai_model"] == "jev-1.13.0"
     assert properties["$ai_evaluation_key_type"] == "byok"
@@ -217,7 +217,7 @@ def test_system_one_numeric_scores_keep_the_rubric_scale(
     assert result["total_tokens"] == 130
     assert "verdict" not in result
     assert "probability" not in result
-    properties = build_evaluation_event_properties(evaluation, result, datetime.now(UTC))
+    properties = build_evaluation_event_properties(evaluation, result, datetime(2026, 1, 1, tzinfo=UTC))
     assert properties["$ai_evaluation_result_type"] == "numeric"
     assert properties.get("$ai_evaluation_numeric_result") == expected
     assert "$ai_evaluation_result" not in properties

From b773d3bff0acdb9577ca206e36a8894c940d0741 Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Fri, 25 Sep 2026 13:50:13 +0200
Subject: [PATCH 08/30] chore(aio): regenerate numeric rubric mcp snapshots

---
 .../tool-schemas/llma-evaluation-create.json  | 21 +++++++++++++++++--
 .../llma-evaluation-test-hog.json             | 17 +++++++++++++++
 .../tool-schemas/llma-evaluation-update.json  | 21 +++++++++++++++++--
 3 files changed, 55 insertions(+), 4 deletions(-)

diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json
index c7acc821350a..3f0ed7e8b5da 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json
@@ -112,7 +112,7 @@
                             "type": "string"
                         },
                         "provider": {
-                            "description": "* `openai` - Openai\n* `anthropic` - Anthropic\n* `gemini` - Gemini\n* `openrouter` - Openrouter\n* `fireworks` - Fireworks\n* `azure_openai` - Azure OpenAI\n* `together_ai` - Together AI\n* `minimax` - MiniMax\n* `zeabur` - Zeabur AI Hub\n* `typesafe` - System One (Jev)",
+                            "description": "* `openai` - Openai\n* `anthropic` - Anthropic\n* `gemini` - Gemini\n* `openrouter` - Openrouter\n* `fireworks` - Fireworks\n* `azure_openai` - Azure OpenAI\n* `together_ai` - Together AI\n* `minimax` - MiniMax\n* `zeabur` - Zeabur AI Hub\n* `typesafe` - System One",
                             "enum": [
                                 "openai",
                                 "anthropic",
@@ -164,7 +164,7 @@
             "type": "string"
         },
         "output_config": {
-            "description": "Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min/max/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.",
+            "description": "Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min/max/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.",
             "properties": {
                 "allows_na": {
                     "description": "Whether the evaluation can return N/A for non-applicable generations.",
@@ -215,6 +215,23 @@
                     ],
                     "description": "Optional numeric passing rule. Null removes the rule; historical scores use the current rule."
                 },
+                "score_levels": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "minLength": 1,
+                                "type": "string"
+                            },
+                            "maxItems": 10,
+                            "minItems": 2,
+                            "type": "array"
+                        },
+                        {
+                            "type": "null"
+                        }
+                    ],
+                    "description": "Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set."
+                },
                 "step": {
                     "anyOf": [
                         {
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-test-hog.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-test-hog.json
index 777e7b66f4df..3eafd9567df8 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-test-hog.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-test-hog.json
@@ -69,6 +69,23 @@
                     ],
                     "description": "Optional numeric passing rule. Null removes the rule; historical scores use the current rule."
                 },
+                "score_levels": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "minLength": 1,
+                                "type": "string"
+                            },
+                            "maxItems": 10,
+                            "minItems": 2,
+                            "type": "array"
+                        },
+                        {
+                            "type": "null"
+                        }
+                    ],
+                    "description": "Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set."
+                },
                 "step": {
                     "anyOf": [
                         {
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json
index 69dc4c521d7a..e1df61130c2a 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json
@@ -115,7 +115,7 @@
                             "type": "string"
                         },
                         "provider": {
-                            "description": "* `openai` - Openai\n* `anthropic` - Anthropic\n* `gemini` - Gemini\n* `openrouter` - Openrouter\n* `fireworks` - Fireworks\n* `azure_openai` - Azure OpenAI\n* `together_ai` - Together AI\n* `minimax` - MiniMax\n* `zeabur` - Zeabur AI Hub\n* `typesafe` - System One (Jev)",
+                            "description": "* `openai` - Openai\n* `anthropic` - Anthropic\n* `gemini` - Gemini\n* `openrouter` - Openrouter\n* `fireworks` - Fireworks\n* `azure_openai` - Azure OpenAI\n* `together_ai` - Together AI\n* `minimax` - MiniMax\n* `zeabur` - Zeabur AI Hub\n* `typesafe` - System One",
                             "enum": [
                                 "openai",
                                 "anthropic",
@@ -167,7 +167,7 @@
             "type": "string"
         },
         "output_config": {
-            "description": "Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min/max/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.",
+            "description": "Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min/max/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.",
             "properties": {
                 "allows_na": {
                     "description": "Whether the evaluation can return N/A for non-applicable generations.",
@@ -218,6 +218,23 @@
                     ],
                     "description": "Optional numeric passing rule. Null removes the rule; historical scores use the current rule."
                 },
+                "score_levels": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "minLength": 1,
+                                "type": "string"
+                            },
+                            "maxItems": 10,
+                            "minItems": 2,
+                            "type": "array"
+                        },
+                        {
+                            "type": "null"
+                        }
+                    ],
+                    "description": "Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set."
+                },
                 "step": {
                     "anyOf": [
                         {

From 5457ae7b44164c2b3e00cbbe30c7197fcbb8545a Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Fri, 25 Sep 2026 14:23:33 +0200
Subject: [PATCH 09/30] fix(aio): keep numeric mapping separate from system one
 client

---
 .../internal/ai-observability-judge-inputs.md | 18 ++---
 .../ai_observability/evaluation_llm_judge.py  | 49 ++++---------
 .../ai_observability/test_run_evaluation.py   | 72 +++----------------
 .../backend/api/evaluations.py                | 19 +----
 .../ai_observability/backend/api/proxy.py     |  2 +-
 .../backend/api/test/test_evaluations.py      | 13 ++--
 .../backend/models/evaluation_configs.py      | 18 -----
 .../backend/models/provider_keys.py           |  2 +-
 .../models/test/test_evaluation_configs.py    | 12 ----
 .../evaluations/AIObservabilityEvaluation.tsx | 26 +++----
 .../NumericEvaluationConfig.stories.tsx       |  7 +-
 .../NumericEvaluationConfig.test.tsx          | 17 -----
 .../components/NumericEvaluationConfig.tsx    | 35 ++-------
 .../frontend/evaluations/constants.ts         | 18 +----
 .../evaluations/llmEvaluationLogic.test.ts    | 32 ++++-----
 .../evaluations/llmEvaluationLogic.ts         |  8 +--
 .../frontend/generated/api.schemas.ts         | 42 ++---------
 .../frontend/generated/api.zod.ts             | 64 +++--------------
 .../frontend/modelPickerLogic.test.ts         |  2 +-
 .../frontend/settings/llmProviderKeysLogic.ts |  2 +-
 services/mcp/src/api/generated.ts             | 42 ++---------
 .../mcp/src/generated/ai_observability/api.ts | 41 ++---------
 .../tool-schemas/llma-evaluation-create.json  | 21 +-----
 .../llma-evaluation-test-hog.json             | 17 -----
 .../tool-schemas/llma-evaluation-update.json  | 21 +-----
 25 files changed, 94 insertions(+), 506 deletions(-)

diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index 876436fedac5..fcb8d21faed8 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -42,7 +42,7 @@ Implementation: [trace judge](../../posthog/temporal/ai_observability/run_trace_
 ## System One judges
 
 System One-compatible models are available under the existing LLM judge option.
-Add a connection under **System One** in provider key settings.
+Add a connection under **System One (Jev)** in provider key settings.
 The default endpoint is TypeSafe's `https://api.typesafe.ai/v1`, with model `jev-1.13.0` and a TypeSafe API key.
 Advanced configuration accepts a different public HTTPS base URL and model ID for compatible services.
 The client appends `/systemone` to the base URL and sends the API key as a bearer token.
@@ -52,9 +52,11 @@ Private network destinations and redirects are blocked by the shared DNS-pinned
 Saving a connection validates it with a short synthetic input and a Noul question, without sending evaluation data.
 Select the connection and configured model on each evaluation; these connections cannot become the shared active provider key used by other AI features.
 Provider keys keep the provider they were created with; switching providers requires a new key.
-This integration supports boolean and numeric evaluations and uses the same formatted text for generation, trace, and session targets.
+The evaluation integration uses Noul for boolean outputs, with the same formatted text for generation, trace, and session targets.
 The client accepts typed Noul, Score, and Choice questions independently of PostHog's evaluation output types.
-Choice support in the client does not enable categorical evaluations in the product.
+The client preserves raw Score values and Choice labels without applying evaluation policy.
+Mapping Score and Choice answers onto numeric and categorical evaluations is separate from this integration.
+Numeric evaluations retain their existing arbitrary ranges and completion-based judges.
 API compatibility does not guarantee equivalent judgments or calibration across models.
 Compare results on representative inputs when changing models.
 
@@ -62,15 +64,7 @@ For boolean evaluations, the prompt becomes a [Noul question](https://docs.types
 A probability of at least 0.5 produces `true`; the evaluation's existing pass/fail polarity still applies.
 The raw probability is stored in `$ai_evaluation_probability`, with token usage and the resolved model version.
 
-Numeric evaluations use a [Score question](https://docs.typesafe.ai/primitives/score).
-Configure 2 to 10 ordered score levels and a minimum and maximum.
-The levels are evenly spaced across those bounds.
-The response is a probability-weighted level index, which maps to the configured scale without rounding.
-For example, three levels on a 0–10 scale map an answer of 1.25 to a score of 6.25.
-Scores are stored in `$ai_evaluation_numeric_result`; they are not boolean probabilities.
-The rubric remains part of the prompt when switching to a completion-based judge.
-
-For either output type, evaluations that allow N/A send a separate Noul question about whether the criteria apply, using the 0.5 threshold.
+Evaluations that allow N/A send a separate Noul question about whether the criteria apply, using the 0.5 threshold.
 Uncertainty alone does not produce N/A.
 System One answers contain no written reasoning, so reports inspect the original source when explaining outcomes.
 
diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.py
index b331a9148c7b..eca7d8153c67 100644
--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py
+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py
@@ -51,8 +51,6 @@
 from products.ai_observability.backend.llm.system_one import (
     NoulAnswer,
     NoulQuestion,
-    ScoreAnswer,
-    ScoreQuestion,
     SystemOneClient,
     SystemOneQuestion,
     SystemOneRateLimitError,
@@ -180,11 +178,6 @@ def get_output_type_config(
             instructions += f" The score must be at most {numeric_config.max}."
         if numeric_config.step is not None:
             instructions += f" Suggested score increment: {numeric_config.step}; do not round an otherwise valid score."
-        if numeric_config.score_levels is not None:
-            instructions += " Use these rubric levels, interpolating between them when needed:\n" + "\n".join(
-                f"{numeric_config.score_from_level(index)}: {description}"
-                for index, description in enumerate(numeric_config.score_levels)
-            )
         if allows_na:
             instructions += " Return score=null when the criteria does not apply."
         return OutputTypeConfig(
@@ -454,7 +447,7 @@ def call_llm_judge(
         raise
 
     provider = resolved.provider
-    if provider == "typesafe" and output_type not in ("boolean", "numeric"):
+    if provider == "typesafe" and output_type != "boolean":
         raise ApplicationError("This System One evaluation output type is not supported.", non_retryable=True)
     model = resolved.model
     provider_key = resolved.provider_key
@@ -478,14 +471,7 @@ def call_llm_judge(
             if provider_key is not None and provider_key.provider != provider:
                 raise ProviderMismatchError(provider_key.provider, provider)
             prompt = evaluation["evaluation_config"]["prompt"]
-            numeric_config = NumericOutputConfig.model_validate(output_config) if output_type == "numeric" else None
-            questions: dict[str, SystemOneQuestion]
-            if numeric_config is not None:
-                if numeric_config.score_levels is None:
-                    raise ApplicationError("Add score levels to this numeric evaluation.", non_retryable=True)
-                questions = {"score": ScoreQuestion(instructions=prompt, criteria=list(numeric_config.score_levels))}
-            else:
-                questions = {"verdict": NoulQuestion(instructions=prompt)}
+            questions: dict[str, SystemOneQuestion] = {"verdict": NoulQuestion(instructions=prompt)}
             if allows_na:
                 questions["applicable"] = NoulQuestion(
                     instructions=(
@@ -508,28 +494,17 @@ def call_llm_judge(
                 applicability_answer = system_one_result.answers["applicable"]
                 assert isinstance(applicability_answer, NoulAnswer)
                 applicable = applicability_answer.noul >= 0.5
-            parsed: BooleanEvalResult | BooleanWithNAEvalResult | NumericEvalResult | NumericWithNAEvalResult
-            if numeric_config is not None:
-                score_answer = system_one_result.answers["score"]
-                assert isinstance(score_answer, ScoreAnswer)
-                score = numeric_config.score_from_level(score_answer.score)
-                parsed = (
-                    NumericWithNAEvalResult(reasoning="", score=score if applicable else None)
-                    if allows_na
-                    else NumericEvalResult(reasoning="", score=score)
-                )
-            else:
-                verdict_answer = system_one_result.answers["verdict"]
-                assert isinstance(verdict_answer, NoulAnswer)
-                probability = verdict_answer.noul
-                parsed = (
-                    BooleanWithNAEvalResult(
-                        reasoning="",
-                        outcome="not_applicable" if not applicable else "pass" if probability >= 0.5 else "fail",
-                    )
-                    if allows_na
-                    else BooleanEvalResult(reasoning="", verdict=probability >= 0.5)
+            verdict_answer = system_one_result.answers["verdict"]
+            assert isinstance(verdict_answer, NoulAnswer)
+            probability = verdict_answer.noul
+            parsed = (
+                BooleanWithNAEvalResult(
+                    reasoning="",
+                    outcome="not_applicable" if not applicable else "pass" if probability >= 0.5 else "fail",
                 )
+                if allows_na
+                else BooleanEvalResult(reasoning="", verdict=probability >= 0.5)
+            )
             model = system_one_result.model
             response = CompletionResponse(
                 content="",
diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index b6f3ed286bb4..2ea7a07eb8ee 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -158,71 +158,21 @@ def test_typesafe_judge_emits_boolean_probability_without_reasoning(
     assert properties["$ai_evaluation_key_type"] == "byok"
 
 
-@pytest.mark.parametrize(
-    "raw_score,probabilities,allows_na,applicability,expected",
-    [
-        (0, [1, 0, 0], False, 1, -10),
-        (1, [0, 1, 0], False, 1, 0),
-        (1.25, [0.1, 0.55, 0.35], True, 1, 2.5),
-        (2, [0, 0, 1], True, 0.1, None),
-    ],
-)
-def test_system_one_numeric_scores_keep_the_rubric_scale(
-    raw_score: float, probabilities: list[float], allows_na: bool, applicability: float, expected: float | None
-) -> None:
-    levels = ["Poor", "Fair", "Good"]
-    evaluation = {
-        "id": "test-evaluation",
-        "name": "Quality",
-        "team_id": 1,
-        "evaluation_config": {"prompt": "Assess response quality"},
-        "output_type": "numeric",
-        "output_config": {"min": -10, "max": 10, "score_levels": levels, "allows_na": allows_na},
-    }
-    key = MagicMock(provider="typesafe", encrypted_config={"api_key": "example-token"})
-    response = MagicMock(status_code=200)
-    response.json.return_value = {
-        "model": "jev-1.13.0",
-        "answers": {
-            "score": {
-                "type": "score",
-                "score": raw_score,
-                "confidence": 0.5,
-                "legend": {str(index): value for index, value in enumerate(levels)},
-                "probabilities": {str(index): value for index, value in enumerate(probabilities)},
-            },
-            "applicable": {"type": "noul", "noul": applicability},
-        },
-        "usage": {"input_tokens": 120, "output_tokens": 10},
-    }
+def test_system_one_numeric_mapping_is_not_enabled() -> None:
     with (
         patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
-        patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response) as request,
+        patch("products.ai_observability.backend.llm.system_one.pinned_request") as request,
+        pytest.raises(ApplicationError) as error,
     ):
-        spec.return_value.resolve.return_value = MagicMock(
-            provider="typesafe", model="jev-1.13.0", provider_key=key, is_byok=True
+        spec.return_value.resolve.return_value = MagicMock(provider="typesafe")
+        call_llm_judge(
+            evaluation={"team_id": 1, "output_type": "numeric"},
+            system_prompt="",
+            user_prompt="Hello!",
+            allows_na=False,
         )
-        result = call_llm_judge(evaluation=evaluation, system_prompt="", user_prompt="Hello!", allows_na=allows_na)
-    assert request.call_args.kwargs["json"]["questions"]["score"] == {
-        "type": "score",
-        "instructions": "Assess response quality",
-        "criteria": levels,
-    }
-    assert ("applicable" in request.call_args.kwargs["json"]["questions"]) is allows_na
-    assert result.get("score") == expected
-    if expected is None:
-        assert "score" not in result
-        assert result["applicable"] is False
-    assert result["reasoning"] == ""
-    assert result["total_tokens"] == 130
-    assert "verdict" not in result
-    assert "probability" not in result
-    properties = build_evaluation_event_properties(evaluation, result, datetime(2026, 1, 1, tzinfo=UTC))
-    assert properties["$ai_evaluation_result_type"] == "numeric"
-    assert properties.get("$ai_evaluation_numeric_result") == expected
-    assert "$ai_evaluation_result" not in properties
-    assert "$ai_evaluation_probability" not in properties
-    assert properties["$ai_model"] == "jev-1.13.0"
+    assert error.value.non_retryable
+    request.assert_not_called()
 
 
 @pytest.mark.parametrize("status", [301, 400, 422])
diff --git a/products/ai_observability/backend/api/evaluations.py b/products/ai_observability/backend/api/evaluations.py
index e146aab155d6..9c29ddd2a991 100644
--- a/products/ai_observability/backend/api/evaluations.py
+++ b/products/ai_observability/backend/api/evaluations.py
@@ -195,14 +195,6 @@ class _EvaluationConfigField(serializers.JSONField):
                 "minimum": 0,
                 "description": "Optional positive input increment. Does not round evaluation results.",
             },
-            "score_levels": {
-                "type": "array",
-                "nullable": True,
-                "items": {"type": "string", "minLength": 1},
-                "minItems": 2,
-                "maxItems": 10,
-                "description": "Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.",
-            },
             "passing_rule": {
                 "type": "object",
                 "nullable": True,
@@ -408,7 +400,7 @@ class EvaluationSerializer(UserAccessControlSerializerMixin, serializers.ModelSe
         help_text=(
             "Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and "
             "{true_is_failure} to declare that a true result means the evaluation found a problem. "
-            "For 'numeric': min/max/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. "
+            "For 'numeric': only min/max/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. "
             "Do not send true_is_failure for numeric output. For 'sentiment': {}."
         ),
     )
@@ -524,7 +516,7 @@ def validate(self, data):
             if isinstance(model_configuration, dict)
             else getattr(model_configuration, "provider", None)
         )
-        if model_provider == LLMProvider.TYPESAFE and output_type not in ("boolean", "numeric"):
+        if model_provider == LLMProvider.TYPESAFE and output_type != "boolean":
             raise serializers.ValidationError(
                 {"model_configuration": "Select a model that supports this evaluation output type."}
             )
@@ -572,13 +564,6 @@ def validate(self, data):
             except ValueError as e:
                 raise serializers.ValidationError({"config": str(e)})
 
-        if model_provider == LLMProvider.TYPESAFE and output_type == "numeric":
-            numeric_output = data.get("output_config", getattr(self.instance, "output_config", {}))
-            if not numeric_output.get("score_levels"):
-                raise serializers.ValidationError(
-                    {"output_config": "Add 2 to 10 score levels and set both bounds for this System One judge."}
-                )
-
         # Sentiment is addressed per-message within one generation event ($ai_target_event_id +
         # message index). An aggregate target emits a single evaluation event for the whole unit,
         # where the message index is ambiguous and that per-generation linkage is absent.
diff --git a/products/ai_observability/backend/api/proxy.py b/products/ai_observability/backend/api/proxy.py
index 73f09dad9f1f..77bee39e080c 100644
--- a/products/ai_observability/backend/api/proxy.py
+++ b/products/ai_observability/backend/api/proxy.py
@@ -67,7 +67,7 @@ def models_cache_key(provider_key_id: str | uuid.UUID) -> str:
 
 
 PROVIDER_DISPLAY_NAMES: dict[str, str] = {
-    "typesafe": "System One",
+    "typesafe": "System One (Jev)",
     "openai": "OpenAI",
     "anthropic": "Anthropic",
     "gemini": "Gemini",
diff --git a/products/ai_observability/backend/api/test/test_evaluations.py b/products/ai_observability/backend/api/test/test_evaluations.py
index 537d3739f044..e1a75919621a 100644
--- a/products/ai_observability/backend/api/test/test_evaluations.py
+++ b/products/ai_observability/backend/api/test/test_evaluations.py
@@ -108,21 +108,16 @@ def test_existing_boolean_cannot_change_to_numeric(self):
 
 
 class TestModelConfigurationSerializer(SimpleTestCase):
-    @parameterized.expand([({}, False), ({"min": 0, "max": 10, "score_levels": ["Poor", "Good"]}, True)])
-    def test_numeric_system_one_requires_score_levels(self, config: dict, valid: bool) -> None:
+    def test_numeric_evaluation_rejects_system_one_connection(self) -> None:
         evaluation = Evaluation(
             evaluation_type="llm_judge",
             evaluation_config={"prompt": "Score quality"},
             output_type="numeric",
-            output_config=config,
+            output_config={},
         )
         serializer = EvaluationSerializer(instance=evaluation, partial=True)
-        data = {"model_configuration": {"provider": "typesafe", "model": "custom-model"}}
-        if valid:
-            self.assertEqual(serializer.validate(data)["model_configuration"], data["model_configuration"])
-        else:
-            with self.assertRaisesMessage(ValidationError, "Add 2 to 10 score levels"):
-                serializer.validate(data)
+        with self.assertRaisesMessage(ValidationError, "Select a model that supports this evaluation output type"):
+            serializer.validate({"model_configuration": {"provider": "typesafe", "model": "custom-model"}})
 
     @parameterized.expand(
         [
diff --git a/products/ai_observability/backend/models/evaluation_configs.py b/products/ai_observability/backend/models/evaluation_configs.py
index 2c928a3eaf02..a45fd06ec3f4 100644
--- a/products/ai_observability/backend/models/evaluation_configs.py
+++ b/products/ai_observability/backend/models/evaluation_configs.py
@@ -81,35 +81,17 @@ class NumericOutputConfig(BaseModel):
     min: float | None = None
     max: float | None = None
     step: float | None = Field(default=None, gt=0)
-    score_levels: list[str] | None = Field(default=None, min_length=2, max_length=10)
     allows_na: bool = False
     passing_rule: NumericPassingRule | None = None
 
-    @field_validator("score_levels")
-    @classmethod
-    def validate_score_levels(cls, value: list[str] | None) -> list[str] | None:
-        if value is not None:
-            value = [level.strip() for level in value]
-            if any(not level for level in value):
-                raise ValueError("Describe every score level")
-        return value
-
     @model_validator(mode="after")
     def validate_bounds(self) -> Self:
         if self.min is not None and self.max is not None and self.min > self.max:
             raise ValueError("Minimum score cannot exceed maximum score")
-        if self.score_levels is not None and (self.min is None or self.max is None or self.min >= self.max):
-            raise ValueError("Score levels require a minimum and a greater maximum")
         if self.passing_rule is not None:
             self.passing_rule.threshold = self.validate_score(self.passing_rule.threshold)
         return self
 
-    def score_from_level(self, level: float) -> float:
-        if self.score_levels is None or self.min is None or self.max is None:
-            raise ValueError("Configure score levels and bounds before using this judge")
-        fraction = level / (len(self.score_levels) - 1)
-        return self.validate_score((1 - fraction) * self.min + fraction * self.max)
-
     def validate_score(self, value: object) -> float:
         if isinstance(value, bool) or not isinstance(value, int | float):
             raise ValueError("Numeric evaluations must return a finite number")
diff --git a/products/ai_observability/backend/models/provider_keys.py b/products/ai_observability/backend/models/provider_keys.py
index 1d3acbdc409e..f2077b51f7b1 100644
--- a/products/ai_observability/backend/models/provider_keys.py
+++ b/products/ai_observability/backend/models/provider_keys.py
@@ -22,7 +22,7 @@ class LLMProvider(models.TextChoices):
     TOGETHER_AI = "together_ai", "Together AI"
     MINIMAX = "minimax", "MiniMax"
     ZEABUR = "zeabur", "Zeabur AI Hub"
-    TYPESAFE = "typesafe", "System One"
+    TYPESAFE = "typesafe", "System One (Jev)"
 
 
 def llm_provider_choices() -> list[tuple[str, str | Promise]]:
diff --git a/products/ai_observability/backend/models/test/test_evaluation_configs.py b/products/ai_observability/backend/models/test/test_evaluation_configs.py
index f075adb85ec9..f780826f31d5 100644
--- a/products/ai_observability/backend/models/test/test_evaluation_configs.py
+++ b/products/ai_observability/backend/models/test/test_evaluation_configs.py
@@ -57,12 +57,6 @@ def test_numeric_configuration(self, runtime, config):
             {"passing_rule": {"operator": "gte", "threshold": True}},
             {"passing_rule": {"operator": "gte", "threshold": float("nan")}},
             {"min": 0, "passing_rule": {"operator": "gte", "threshold": -1}},
-            {"score_levels": ["Poor", "Good"]},
-            {"min": 0, "score_levels": ["Poor", "Good"]},
-            {"min": 0, "max": 0, "score_levels": ["Poor", "Good"]},
-            {"min": 0, "max": 10, "score_levels": ["Only one"]},
-            {"min": 0, "max": 10, "score_levels": ["Level"] * 11},
-            {"min": 0, "max": 10, "score_levels": ["Poor", "  "]},
         ],
     )
     def test_invalid_numeric_configuration(self, output):
@@ -78,12 +72,6 @@ def test_unbounded_and_nullable_configuration(self):
         )
         assert output == {"allows_na": True}
 
-    @pytest.mark.parametrize("level,expected", [(0, -10), (1.25, 2.5), (2, 10)])
-    def test_score_levels_keep_fractional_values(self, level: float, expected: float) -> None:
-        config = NumericOutputConfig(min=-10, max=10, score_levels=[" Poor ", "Fair", "Good"])
-        assert config.score_levels == ["Poor", "Fair", "Good"]
-        assert config.score_from_level(level) == expected
-
 
 class TestValidateTargetConfig:
     @pytest.mark.parametrize(
diff --git a/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx b/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx
index d32f6b3c1345..dbd2f01f9b74 100644
--- a/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx
+++ b/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx
@@ -180,10 +180,7 @@ export function AIObservabilityEvaluation(): JSX.Element {
           : !hasSelectedJudgeModel
             ? 'Select a judge model before saving'
             : evaluation.output_type === 'numeric'
-              ? (numericOutputConfigError(
-                    evaluation.output_config,
-                    evaluation.model_configuration?.provider === 'typesafe'
-                ) ?? undefined)
+              ? (numericOutputConfigError(evaluation.output_config) ?? undefined)
               : undefined
 
     const focusTriggers = (): void => {
@@ -749,9 +746,6 @@ export function AIObservabilityEvaluation(): JSX.Element {
                                                 <NumericEvaluationConfig
                                                     config={evaluation.output_config}
                                                     onChange={patchOutputConfig}
-                                                    requiresScoreLevels={
-                                                        evaluation.model_configuration?.provider === 'typesafe'
-                                                    }
                                                 />
                                             )}
                                             <LemonField.Pure label="Description (optional)">
@@ -936,10 +930,7 @@ function EvaluationModelPicker(): JSX.Element {
     // Evals always run on the team's own provider key, so only BYOK models are offered.
     const selectedModelName = byokModels.find((m) => m.id === selectedModel)?.name
     const groups = evaluationProviderModelGroups.filter(
-        (group) =>
-            evaluation?.output_type === 'boolean' ||
-            evaluation?.output_type === 'numeric' ||
-            group.provider !== 'typesafe'
+        (group) => evaluation?.output_type === 'boolean' || group.provider !== 'typesafe'
     )
     const loading = byokModelsLoading || providerKeysLoading
 
@@ -966,13 +957,12 @@ function EvaluationModelPicker(): JSX.Element {
                             data-attr="evaluation-model-selector"
                         />
                         <ByokModelPickerNotice forEvaluation />
-                        {evaluation?.model_configuration?.provider === 'typesafe' &&
-                            evaluation.output_type === 'boolean' && (
-                                <p className="text-sm text-muted mt-2">
-                                    This judge returns a probability without written reasoning. A probability of 50% or
-                                    higher produces a true result.
-                                </p>
-                            )}
+                        {evaluation?.model_configuration?.provider === 'typesafe' && (
+                            <p className="text-sm text-muted mt-2">
+                                This judge returns a probability without written reasoning. A probability of 50% or
+                                higher produces a true result.
+                            </p>
+                        )}
                         {modelSelectionRequired && !selectedModel && (
                             <p className="text-sm text-danger mt-1">Select a judge model.</p>
                         )}
diff --git a/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.stories.tsx b/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.stories.tsx
index a34411314f2e..ccde3ae802c9 100644
--- a/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.stories.tsx
+++ b/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.stories.tsx
@@ -13,22 +13,18 @@ export default meta
 
 type Story = StoryObj<typeof NumericEvaluationConfig>
 
-function Template({ narrow = false, systemOne = false }: { narrow?: boolean; systemOne?: boolean }): JSX.Element {
+function Template({ narrow = false }: { narrow?: boolean }): JSX.Element {
     const [config, setConfig] = useState<EvaluationOutputConfig>({
         min: 0,
         max: 10,
         step: 0.5,
         allows_na: true,
         passing_rule: { operator: 'gte', threshold: 7 },
-        ...(systemOne
-            ? { score_levels: ['Does not meet the criteria', 'Partly meets the criteria', 'Fully meets the criteria'] }
-            : {}),
     })
     return (
         <div className={narrow ? 'max-w-lg' : 'max-w-4xl'}>
             <NumericEvaluationConfig
                 config={config}
-                requiresScoreLevels={systemOne}
                 onChange={(patch) => setConfig((current) => ({ ...current, ...patch }))}
             />
         </div>
@@ -37,4 +33,3 @@ function Template({ narrow = false, systemOne = false }: { narrow?: boolean; sys
 
 export const Default: Story = { render: () => <Template /> }
 export const Narrow: Story = { render: () => <Template narrow /> }
-export const SystemOne: Story = { render: () => <Template systemOne narrow /> }
diff --git a/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.test.tsx b/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.test.tsx
index 7d4ccd35941c..e263c954fef7 100644
--- a/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.test.tsx
+++ b/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.test.tsx
@@ -7,23 +7,6 @@ import { NumericEvaluationConfig } from './NumericEvaluationConfig'
 describe('NumericEvaluationConfig', () => {
     afterEach(cleanup)
 
-    it('requires a bounded rubric for System One and keeps level edits', () => {
-        const onChange = jest.fn()
-        const config: EvaluationOutputConfig = { min: 0, max: 10 }
-        expect(numericOutputConfigError(config)).toBeNull()
-        expect(numericOutputConfigError(config, true)).not.toBeNull()
-        const { getByLabelText, rerender } = render(
-            <NumericEvaluationConfig config={config} onChange={onChange} requiresScoreLevels />
-        )
-        fireEvent.change(getByLabelText('Score levels'), { target: { value: 'Poor\nFair\nGood' } })
-        expect(onChange).toHaveBeenLastCalledWith({ score_levels: ['Poor', 'Fair', 'Good'] })
-        const edited = { ...config, ...onChange.mock.calls[0][0] }
-        expect(numericOutputConfigError(edited, true)).toBeNull()
-        expect(numericOutputConfigError({ ...edited, min: null }, true)).not.toBeNull()
-        rerender(<NumericEvaluationConfig config={edited} onChange={onChange} />)
-        expect((getByLabelText('Score levels') as HTMLTextAreaElement).value).toBe('Poor\nFair\nGood')
-    })
-
     it.each([
         ['min', 'Minimum (optional)'],
         ['max', 'Maximum (optional)'],
diff --git a/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.tsx b/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.tsx
index e6d4d0cf12e2..848d6addc8c6 100644
--- a/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.tsx
+++ b/products/ai_observability/frontend/evaluations/components/NumericEvaluationConfig.tsx
@@ -1,6 +1,6 @@
 import { useId } from 'react'
 
-import { LemonBanner, LemonInput, LemonSelect, LemonSwitch, LemonTextArea } from '@posthog/lemon-ui'
+import { LemonBanner, LemonInput, LemonSelect, LemonSwitch } from '@posthog/lemon-ui'
 
 import { LemonField } from 'lib/lemon-ui/LemonField'
 
@@ -10,16 +10,13 @@ import type { EvaluationOutputConfig } from '../types'
 export function NumericEvaluationConfig({
     config,
     onChange,
-    requiresScoreLevels = false,
 }: {
     config: EvaluationOutputConfig
     onChange: (patch: EvaluationOutputConfig) => void
-    requiresScoreLevels?: boolean
 }): JSX.Element {
     const id = useId()
     const rule = config.passing_rule
-    const hasScoreLevels = requiresScoreLevels || config.score_levels != null
-    const error = numericOutputConfigError(config, requiresScoreLevels)
+    const error = numericOutputConfigError(config)
     return (
         <div className="space-y-4">
             {error && <LemonBanner type="error">{error}</LemonBanner>}
@@ -28,13 +25,7 @@ export function NumericEvaluationConfig({
                     <LemonField.Pure
                         key={field}
                         htmlFor={`${id}-${field}`}
-                        label={
-                            {
-                                min: hasScoreLevels ? 'Minimum' : 'Minimum (optional)',
-                                max: hasScoreLevels ? 'Maximum' : 'Maximum (optional)',
-                                step: 'Step (optional)',
-                            }[field]
-                        }
+                        label={{ min: 'Minimum (optional)', max: 'Maximum (optional)', step: 'Step (optional)' }[field]}
                     >
                         <LemonInput
                             id={`${id}-${field}`}
@@ -47,25 +38,7 @@ export function NumericEvaluationConfig({
                     </LemonField.Pure>
                 ))}
             </div>
-            <p className="text-muted text-sm">
-                Bounds are inclusive. Step controls input increments without rounding results.
-            </p>
-            {hasScoreLevels && (
-                <LemonField.Pure label="Score levels" htmlFor={`${id}-score-levels`}>
-                    <LemonTextArea
-                        id={`${id}-score-levels`}
-                        value={config.score_levels?.join('\n') ?? ''}
-                        onChange={(value) => onChange({ score_levels: value.split('\n') })}
-                        placeholder={'Does not meet the criteria\nPartly meets the criteria\nFully meets the criteria'}
-                        minRows={3}
-                        data-attr="llma-evaluation-score-levels"
-                    />
-                    <p className="text-muted text-sm">
-                        Describe 2 to 10 levels, one per line, from the minimum to the maximum. Levels are spaced
-                        evenly. Scores can fall between levels. They measure the rubric, not the probability of passing.
-                    </p>
-                </LemonField.Pure>
-            )}
+            <p className="text-muted text-sm">Bounds are inclusive. Step guides scoring without rounding results.</p>
             <LemonSwitch
                 label="Allow N/A responses"
                 checked={config.allows_na ?? false}
diff --git a/products/ai_observability/frontend/evaluations/constants.ts b/products/ai_observability/frontend/evaluations/constants.ts
index 25c8687abd51..0498837fbed3 100644
--- a/products/ai_observability/frontend/evaluations/constants.ts
+++ b/products/ai_observability/frontend/evaluations/constants.ts
@@ -21,10 +21,7 @@ const EVALUATION_RESULT_FALSE_HOGQL = "properties.$ai_evaluation_result = 'false
 export const EVALUATION_NOT_SKIPPED_HOGQL =
     "(isNull(properties.$ai_evaluation_skipped) OR properties.$ai_evaluation_skipped != 'true')"
 
-export function numericOutputConfigError(
-    config: EvaluationOutputConfig,
-    requiresScoreLevels: boolean = false
-): string | null {
+export function numericOutputConfigError(config: EvaluationOutputConfig): string | null {
     const { min, max, step, passing_rule } = config
     if ([min, max, step, passing_rule?.threshold].some((value) => value != null && !Number.isFinite(value))) {
         return 'Enter finite numbers for the score bounds, step, and threshold.'
@@ -32,19 +29,6 @@ export function numericOutputConfigError(
     if (min != null && max != null && min > max) {
         return 'Minimum must be less than or equal to maximum.'
     }
-    if (requiresScoreLevels || config.score_levels != null) {
-        if (min == null || max == null || min >= max) {
-            return 'Set a minimum and a greater maximum for the score levels.'
-        }
-        if (
-            !config.score_levels ||
-            config.score_levels.length < 2 ||
-            config.score_levels.length > 10 ||
-            config.score_levels.some((level) => !level.trim())
-        ) {
-            return 'Describe 2 to 10 score levels, one per line.'
-        }
-    }
     if (step != null && step <= 0) {
         return 'Step must be greater than zero.'
     }
diff --git a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts
index e3a91bc5231d..b76fa3d1ad95 100644
--- a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts
+++ b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts
@@ -658,27 +658,19 @@ return result`,
                 await expectLogic(logic).toMatchValues({ formValid: false })
             })
 
-            it.each(['openai', 'typesafe'] as const)(
-                'validates boolean and numeric fields for %s',
-                async (provider) => {
-                    await expectLogic(logic).toDispatchActions(['loadEvaluationSuccess'])
-                    logic.actions.setEvaluationName('Valid Name')
-                    logic.actions.setEvaluationPrompt('Valid prompt')
-                    logic.actions.setTriggerConditions([{ id: 'c1', rollout_percentage: 50, properties: [] }])
-                    logic.actions.setModelConfiguration({
-                        provider,
-                        model: provider === 'typesafe' ? 'jev-1.13.0' : 'gpt-5-mini',
-                        provider_key_id: 'key-1',
-                    })
+            it('returns true when all fields valid', async () => {
+                await expectLogic(logic).toDispatchActions(['loadEvaluationSuccess'])
+                logic.actions.setEvaluationName('Valid Name')
+                logic.actions.setEvaluationPrompt('Valid prompt')
+                logic.actions.setTriggerConditions([{ id: 'c1', rollout_percentage: 50, properties: [] }])
+                logic.actions.setModelConfiguration({
+                    provider: 'openai',
+                    model: 'gpt-5-mini',
+                    provider_key_id: 'key-1',
+                })
 
-                    await expectLogic(logic).toMatchValues({ formValid: true })
-                    logic.actions.setOutputType('numeric')
-                    logic.actions.patchOutputConfig({ min: 0, max: 10 })
-                    await expectLogic(logic).toMatchValues({ formValid: provider !== 'typesafe' })
-                    logic.actions.patchOutputConfig({ score_levels: ['Poor', 'Good'] })
-                    await expectLogic(logic).toMatchValues({ formValid: true })
-                }
-            )
+                await expectLogic(logic).toMatchValues({ formValid: true })
+            })
 
             // A loaded evaluation whose stored shape doesn't match its type (e.g. an llm_judge
             // record with no prompt) used to crash formValid with a TypeError on render.
diff --git a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts
index 35d2d86a02fb..bc11da83705e 100644
--- a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts
+++ b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts
@@ -1437,13 +1437,7 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
                 if (!evaluation) {
                     return false
                 }
-                if (
-                    evaluation.output_type === 'numeric' &&
-                    numericOutputConfigError(
-                        evaluation.output_config,
-                        evaluation.model_configuration?.provider === 'typesafe'
-                    )
-                ) {
+                if (evaluation.output_type === 'numeric' && numericOutputConfigError(evaluation.output_config)) {
                     return false
                 }
                 const hasValidName = (evaluation.name?.length ?? 0) > 0
diff --git a/products/ai_observability/frontend/generated/api.schemas.ts b/products/ai_observability/frontend/generated/api.schemas.ts
index 8e99c6790f5f..3425e3858a9d 100644
--- a/products/ai_observability/frontend/generated/api.schemas.ts
+++ b/products/ai_observability/frontend/generated/api.schemas.ts
@@ -870,7 +870,7 @@ export const EvaluationTargetEnumApi = {
  * * `together_ai` - Together AI
  * * `minimax` - MiniMax
  * * `zeabur` - Zeabur AI Hub
- * * `typesafe` - System One
+ * * `typesafe` - System One (Jev)
  */
 export type LLMProviderEnumApi = (typeof LLMProviderEnumApi)[keyof typeof LLMProviderEnumApi]
 
@@ -938,7 +938,7 @@ export type EvaluationApiOutputConfigPassingRule = {
 } | null
 
 /**
- * Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min/max/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.
+ * Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min/max/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.
  */
 export type EvaluationApiOutputConfig = {
     /** Whether the evaluation can return N/A for non-applicable generations. */
@@ -962,14 +962,6 @@ export type EvaluationApiOutputConfig = {
      * @nullable
      */
     step?: number | null
-    /**
-     * Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.
-     * @minItems 2
-     * @maxItems 10
-     * @nullable
-     * @items.minLength 1
-     */
-    score_levels?: string[] | null
     /**
      * Optional numeric passing rule. Null removes the rule; historical scores use the current rule.
      * @nullable
@@ -1048,7 +1040,7 @@ export interface EvaluationApi {
      * * `numeric` - Numeric
      * * `sentiment` - Sentiment */
     output_type: OutputTypeEnumApi
-    /** Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min/max/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}. */
+    /** Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min/max/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}. */
     output_config?: EvaluationApiOutputConfig
     /** Trigger conditions that filter which events are evaluated. OR between condition sets, AND within each. Each set is {id, rollout_percentage, properties[]} — `rollout_percentage` (0-100, defaults to 100) is the sampling field the dispatcher reads. */
     conditions?: EvaluationConditionApi[]
@@ -1227,7 +1219,7 @@ export type PatchedEvaluationApiOutputConfigPassingRule = {
 } | null
 
 /**
- * Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min/max/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.
+ * Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min/max/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.
  */
 export type PatchedEvaluationApiOutputConfig = {
     /** Whether the evaluation can return N/A for non-applicable generations. */
@@ -1251,14 +1243,6 @@ export type PatchedEvaluationApiOutputConfig = {
      * @nullable
      */
     step?: number | null
-    /**
-     * Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.
-     * @minItems 2
-     * @maxItems 10
-     * @nullable
-     * @items.minLength 1
-     */
-    score_levels?: string[] | null
     /**
      * Optional numeric passing rule. Null removes the rule; historical scores use the current rule.
      * @nullable
@@ -1337,7 +1321,7 @@ export interface PatchedEvaluationApi {
      * * `numeric` - Numeric
      * * `sentiment` - Sentiment */
     output_type?: OutputTypeEnumApi
-    /** Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min/max/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}. */
+    /** Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min/max/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}. */
     output_config?: PatchedEvaluationApiOutputConfig
     /** Trigger conditions that filter which events are evaluated. OR between condition sets, AND within each. Each set is {id, rollout_percentage, properties[]} — `rollout_percentage` (0-100, defaults to 100) is the sampling field the dispatcher reads. */
     conditions?: EvaluationConditionApi[]
@@ -1400,14 +1384,6 @@ export type TestHogRequestApiOutputConfig = {
      * @nullable
      */
     step?: number | null
-    /**
-     * Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.
-     * @minItems 2
-     * @maxItems 10
-     * @nullable
-     * @items.minLength 1
-     */
-    score_levels?: string[] | null
     /**
      * Optional numeric passing rule. Null removes the rule; historical scores use the current rule.
      * @nullable
@@ -2095,14 +2071,6 @@ export type EvaluationReportMetricsApiOutputConfig = {
      * @nullable
      */
     step?: number | null
-    /**
-     * Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.
-     * @minItems 2
-     * @maxItems 10
-     * @nullable
-     * @items.minLength 1
-     */
-    score_levels?: string[] | null
     /**
      * Optional numeric passing rule. Null removes the rule; historical scores use the current rule.
      * @nullable
diff --git a/products/ai_observability/frontend/generated/api.zod.ts b/products/ai_observability/frontend/generated/api.zod.ts
index 477dfaeac6d6..28729b394c69 100644
--- a/products/ai_observability/frontend/generated/api.zod.ts
+++ b/products/ai_observability/frontend/generated/api.zod.ts
@@ -261,9 +261,6 @@ export const evaluationsCreateBodyNameMax = 400
 export const evaluationsCreateBodyEvaluationConfigThreeSourceDefault = `user_messages`
 export const evaluationsCreateBodyOutputConfigStepExclusiveMin = 0
 
-export const evaluationsCreateBodyOutputConfigScoreLevelsMin = 2
-export const evaluationsCreateBodyOutputConfigScoreLevelsMax = 10
-
 export const evaluationsCreateBodyConditionsItemIdMax = 100
 
 export const evaluationsCreateBodyConditionsItemRolloutPercentageDefault = 100
@@ -353,14 +350,6 @@ export const EvaluationsCreateBody = /* @__PURE__ */ zod
                     .gt(evaluationsCreateBodyOutputConfigStepExclusiveMin)
                     .nullish()
                     .describe('Optional positive input increment. Does not round evaluation results.'),
-                score_levels: zod
-                    .array(zod.string().min(1))
-                    .min(evaluationsCreateBodyOutputConfigScoreLevelsMin)
-                    .max(evaluationsCreateBodyOutputConfigScoreLevelsMax)
-                    .nullish()
-                    .describe(
-                        'Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.'
-                    ),
                 passing_rule: zod
                     .object({
                         operator: zod
@@ -377,7 +366,7 @@ export const EvaluationsCreateBody = /* @__PURE__ */ zod
             })
             .optional()
             .describe(
-                "Output config. For 'boolean' output_type: {allows_na} to permit N\/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min\/max\/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}."
+                "Output config. For 'boolean' output_type: {allows_na} to permit N\/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min\/max\/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}."
             ),
         conditions: zod
             .array(
@@ -474,7 +463,7 @@ export const EvaluationsCreateBody = /* @__PURE__ */ zod
                                 'typesafe',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
                             ),
                         model: zod.string().max(evaluationsCreateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -599,9 +588,6 @@ export const evaluationsUpdateBodyNameMax = 400
 export const evaluationsUpdateBodyEvaluationConfigThreeSourceDefault = `user_messages`
 export const evaluationsUpdateBodyOutputConfigStepExclusiveMin = 0
 
-export const evaluationsUpdateBodyOutputConfigScoreLevelsMin = 2
-export const evaluationsUpdateBodyOutputConfigScoreLevelsMax = 10
-
 export const evaluationsUpdateBodyConditionsItemIdMax = 100
 
 export const evaluationsUpdateBodyConditionsItemRolloutPercentageDefault = 100
@@ -691,14 +677,6 @@ export const EvaluationsUpdateBody = /* @__PURE__ */ zod
                     .gt(evaluationsUpdateBodyOutputConfigStepExclusiveMin)
                     .nullish()
                     .describe('Optional positive input increment. Does not round evaluation results.'),
-                score_levels: zod
-                    .array(zod.string().min(1))
-                    .min(evaluationsUpdateBodyOutputConfigScoreLevelsMin)
-                    .max(evaluationsUpdateBodyOutputConfigScoreLevelsMax)
-                    .nullish()
-                    .describe(
-                        'Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.'
-                    ),
                 passing_rule: zod
                     .object({
                         operator: zod
@@ -715,7 +693,7 @@ export const EvaluationsUpdateBody = /* @__PURE__ */ zod
             })
             .optional()
             .describe(
-                "Output config. For 'boolean' output_type: {allows_na} to permit N\/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min\/max\/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}."
+                "Output config. For 'boolean' output_type: {allows_na} to permit N\/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min\/max\/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}."
             ),
         conditions: zod
             .array(
@@ -812,7 +790,7 @@ export const EvaluationsUpdateBody = /* @__PURE__ */ zod
                                 'typesafe',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
                             ),
                         model: zod.string().max(evaluationsUpdateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -839,9 +817,6 @@ export const evaluationsPartialUpdateBodyNameMax = 400
 export const evaluationsPartialUpdateBodyEvaluationConfigThreeSourceDefault = `user_messages`
 export const evaluationsPartialUpdateBodyOutputConfigStepExclusiveMin = 0
 
-export const evaluationsPartialUpdateBodyOutputConfigScoreLevelsMin = 2
-export const evaluationsPartialUpdateBodyOutputConfigScoreLevelsMax = 10
-
 export const evaluationsPartialUpdateBodyConditionsItemIdMax = 100
 
 export const evaluationsPartialUpdateBodyConditionsItemRolloutPercentageDefault = 100
@@ -933,14 +908,6 @@ export const EvaluationsPartialUpdateBody = /* @__PURE__ */ zod
                     .gt(evaluationsPartialUpdateBodyOutputConfigStepExclusiveMin)
                     .nullish()
                     .describe('Optional positive input increment. Does not round evaluation results.'),
-                score_levels: zod
-                    .array(zod.string().min(1))
-                    .min(evaluationsPartialUpdateBodyOutputConfigScoreLevelsMin)
-                    .max(evaluationsPartialUpdateBodyOutputConfigScoreLevelsMax)
-                    .nullish()
-                    .describe(
-                        'Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.'
-                    ),
                 passing_rule: zod
                     .object({
                         operator: zod
@@ -957,7 +924,7 @@ export const EvaluationsPartialUpdateBody = /* @__PURE__ */ zod
             })
             .optional()
             .describe(
-                "Output config. For 'boolean' output_type: {allows_na} to permit N\/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min\/max\/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}."
+                "Output config. For 'boolean' output_type: {allows_na} to permit N\/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min\/max\/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}."
             ),
         conditions: zod
             .array(
@@ -1054,7 +1021,7 @@ export const EvaluationsPartialUpdateBody = /* @__PURE__ */ zod
                                 'typesafe',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
                             ),
                         model: zod.string().max(evaluationsPartialUpdateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -1082,9 +1049,6 @@ export const EvaluationsPartialUpdateBody = /* @__PURE__ */ zod
 export const evaluationsTestHogCreateBodyOutputTypeDefault = `boolean`
 export const evaluationsTestHogCreateBodyOutputConfigStepExclusiveMin = 0
 
-export const evaluationsTestHogCreateBodyOutputConfigScoreLevelsMin = 2
-export const evaluationsTestHogCreateBodyOutputConfigScoreLevelsMax = 10
-
 export const evaluationsTestHogCreateBodySampleCountDefault = 5
 export const evaluationsTestHogCreateBodySampleCountMax = 10
 
@@ -1125,14 +1089,6 @@ export const EvaluationsTestHogCreateBody = /* @__PURE__ */ zod.object({
                 .gt(evaluationsTestHogCreateBodyOutputConfigStepExclusiveMin)
                 .nullish()
                 .describe('Optional positive input increment. Does not round evaluation results.'),
-            score_levels: zod
-                .array(zod.string().min(1))
-                .min(evaluationsTestHogCreateBodyOutputConfigScoreLevelsMin)
-                .max(evaluationsTestHogCreateBodyOutputConfigScoreLevelsMax)
-                .nullish()
-                .describe(
-                    'Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.'
-                ),
             passing_rule: zod
                 .object({
                     operator: zod
@@ -1690,7 +1646,7 @@ export const LlmAnalyticsProviderKeysCreateBody = /* @__PURE__ */ zod.object({
             'typesafe',
         ])
         .describe(
-            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One'
+            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
         ),
     name: zod.string().max(llmAnalyticsProviderKeysCreateBodyNameMax),
     api_key: zod.string().optional(),
@@ -1732,7 +1688,7 @@ export const LlmAnalyticsProviderKeysUpdateBody = /* @__PURE__ */ zod.object({
             'typesafe',
         ])
         .describe(
-            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One'
+            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
         ),
     name: zod.string().max(llmAnalyticsProviderKeysUpdateBodyNameMax),
     api_key: zod.string().optional(),
@@ -1775,7 +1731,7 @@ export const LlmAnalyticsProviderKeysPartialUpdateBody = /* @__PURE__ */ zod.obj
         ])
         .optional()
         .describe(
-            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One'
+            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
         ),
     name: zod.string().max(llmAnalyticsProviderKeysPartialUpdateBodyNameMax).optional(),
     api_key: zod.string().optional(),
@@ -1817,7 +1773,7 @@ export const LlmAnalyticsProviderKeysValidateCreateBody = /* @__PURE__ */ zod.ob
             'typesafe',
         ])
         .describe(
-            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One'
+            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
         ),
     name: zod.string().max(llmAnalyticsProviderKeysValidateCreateBodyNameMax),
     api_key: zod.string().optional(),
diff --git a/products/ai_observability/frontend/modelPickerLogic.test.ts b/products/ai_observability/frontend/modelPickerLogic.test.ts
index 4c2900bfa81d..d997dd864bc8 100644
--- a/products/ai_observability/frontend/modelPickerLogic.test.ts
+++ b/products/ai_observability/frontend/modelPickerLogic.test.ts
@@ -62,7 +62,7 @@ describe('modelPickerLogic', () => {
                                           {
                                               id: model,
                                               name: model,
-                                              provider: 'System One',
+                                              provider: 'System One (Jev)',
                                               is_recommended: true,
                                           },
                                       ],
diff --git a/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts b/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts
index 26143e39e72c..1406f2cae877 100644
--- a/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts
+++ b/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts
@@ -25,7 +25,7 @@ export const LLM_PROVIDER_LABELS: Record<LLMProvider, string> = {
     together_ai: 'Together AI',
     minimax: 'MiniMax',
     zeabur: 'Zeabur AI Hub',
-    typesafe: 'System One',
+    typesafe: 'System One (Jev)',
 }
 
 const LLM_PROVIDERS = new Set<string>(Object.keys(LLM_PROVIDER_LABELS))
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index feb29c0d3d40..fd58bad34c34 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -37680,7 +37680,7 @@ export namespace Schemas {
     } | null;
 
     /**
-     * Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min/max/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.
+     * Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min/max/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.
      */
     export type EvaluationOutputConfig = {
       /** Whether the evaluation can return N/A for non-applicable generations. */
@@ -37704,14 +37704,6 @@ export namespace Schemas {
          * @nullable
          */
       step?: number | null;
-      /**
-         * Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.
-         * @minItems 2
-         * @maxItems 10
-         * @nullable
-         * @items.minLength 1
-         */
-      score_levels?: string[] | null;
       /**
          * Optional numeric passing rule. Null removes the rule; historical scores use the current rule.
          * @nullable
@@ -37861,7 +37853,7 @@ export namespace Schemas {
      * * `together_ai` - Together AI
      * * `minimax` - MiniMax
      * * `zeabur` - Zeabur AI Hub
-     * * `typesafe` - System One
+     * * `typesafe` - System One (Jev)
      */
     export type LLMProviderEnum = typeof LLMProviderEnum[keyof typeof LLMProviderEnum];
 
@@ -37935,7 +37927,7 @@ export namespace Schemas {
        * * `numeric` - Numeric
        * * `sentiment` - Sentiment */
       output_type: OutputTypeEnum;
-      /** Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min/max/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}. */
+      /** Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min/max/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}. */
       output_config?: EvaluationOutputConfig;
       /** Trigger conditions that filter which events are evaluated. OR between condition sets, AND within each. Each set is {id, rollout_percentage, properties[]} — `rollout_percentage` (0-100, defaults to 100) is the sampling field the dispatcher reads. */
       conditions?: EvaluationCondition[];
@@ -38301,14 +38293,6 @@ export namespace Schemas {
          * @nullable
          */
       step?: number | null;
-      /**
-         * Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.
-         * @minItems 2
-         * @maxItems 10
-         * @nullable
-         * @items.minLength 1
-         */
-      score_levels?: string[] | null;
       /**
          * Optional numeric passing rule. Null removes the rule; historical scores use the current rule.
          * @nullable
@@ -71697,7 +71681,7 @@ export namespace Schemas {
     } | null;
 
     /**
-     * Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min/max/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.
+     * Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min/max/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.
      */
     export type PatchedEvaluationOutputConfig = {
       /** Whether the evaluation can return N/A for non-applicable generations. */
@@ -71721,14 +71705,6 @@ export namespace Schemas {
          * @nullable
          */
       step?: number | null;
-      /**
-         * Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.
-         * @minItems 2
-         * @maxItems 10
-         * @nullable
-         * @items.minLength 1
-         */
-      score_levels?: string[] | null;
       /**
          * Optional numeric passing rule. Null removes the rule; historical scores use the current rule.
          * @nullable
@@ -71805,7 +71781,7 @@ export namespace Schemas {
        * * `numeric` - Numeric
        * * `sentiment` - Sentiment */
       output_type?: OutputTypeEnum;
-      /** Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min/max/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}. */
+      /** Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min/max/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}. */
       output_config?: PatchedEvaluationOutputConfig;
       /** Trigger conditions that filter which events are evaluated. OR between condition sets, AND within each. Each set is {id, rollout_percentage, properties[]} — `rollout_percentage` (0-100, defaults to 100) is the sampling field the dispatcher reads. */
       conditions?: EvaluationCondition[];
@@ -98519,14 +98495,6 @@ export namespace Schemas {
          * @nullable
          */
       step?: number | null;
-      /**
-         * Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.
-         * @minItems 2
-         * @maxItems 10
-         * @nullable
-         * @items.minLength 1
-         */
-      score_levels?: string[] | null;
       /**
          * Optional numeric passing rule. Null removes the rule; historical scores use the current rule.
          * @nullable
diff --git a/services/mcp/src/generated/ai_observability/api.ts b/services/mcp/src/generated/ai_observability/api.ts
index f664445fbe30..ff6bb3457533 100644
--- a/services/mcp/src/generated/ai_observability/api.ts
+++ b/services/mcp/src/generated/ai_observability/api.ts
@@ -563,9 +563,6 @@ export const evaluationsCreateBodyNameMax = 400
 export const evaluationsCreateBodyEvaluationConfigThreeSourceDefault = `user_messages`
 export const evaluationsCreateBodyOutputConfigStepExclusiveMin = 0
 
-export const evaluationsCreateBodyOutputConfigScoreLevelsMin = 2
-export const evaluationsCreateBodyOutputConfigScoreLevelsMax = 10
-
 export const evaluationsCreateBodyConditionsItemIdMax = 100
 
 export const evaluationsCreateBodyConditionsItemRolloutPercentageDefault = 100
@@ -655,14 +652,6 @@ export const EvaluationsCreateBody = () => zod
                     .gt(evaluationsCreateBodyOutputConfigStepExclusiveMin)
                     .nullish()
                     .describe('Optional positive input increment. Does not round evaluation results.'),
-                score_levels: zod
-                    .array(zod.string().min(1))
-                    .min(evaluationsCreateBodyOutputConfigScoreLevelsMin)
-                    .max(evaluationsCreateBodyOutputConfigScoreLevelsMax)
-                    .nullish()
-                    .describe(
-                        'Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.'
-                    ),
                 passing_rule: zod
                     .object({
                         operator: zod
@@ -679,7 +668,7 @@ export const EvaluationsCreateBody = () => zod
             })
             .optional()
             .describe(
-                "Output config. For 'boolean' output_type: {allows_na} to permit N\/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min\/max\/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}."
+                "Output config. For 'boolean' output_type: {allows_na} to permit N\/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min\/max\/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}."
             ),
         conditions: zod
             .array(
@@ -776,7 +765,7 @@ export const EvaluationsCreateBody = () => zod
                                 'typesafe',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
                             ),
                         model: zod.string().max(evaluationsCreateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -821,9 +810,6 @@ export const evaluationsPartialUpdateBodyNameMax = 400
 export const evaluationsPartialUpdateBodyEvaluationConfigThreeSourceDefault = `user_messages`
 export const evaluationsPartialUpdateBodyOutputConfigStepExclusiveMin = 0
 
-export const evaluationsPartialUpdateBodyOutputConfigScoreLevelsMin = 2
-export const evaluationsPartialUpdateBodyOutputConfigScoreLevelsMax = 10
-
 export const evaluationsPartialUpdateBodyConditionsItemIdMax = 100
 
 export const evaluationsPartialUpdateBodyConditionsItemRolloutPercentageDefault = 100
@@ -915,14 +901,6 @@ export const EvaluationsPartialUpdateBody = () => zod
                     .gt(evaluationsPartialUpdateBodyOutputConfigStepExclusiveMin)
                     .nullish()
                     .describe('Optional positive input increment. Does not round evaluation results.'),
-                score_levels: zod
-                    .array(zod.string().min(1))
-                    .min(evaluationsPartialUpdateBodyOutputConfigScoreLevelsMin)
-                    .max(evaluationsPartialUpdateBodyOutputConfigScoreLevelsMax)
-                    .nullish()
-                    .describe(
-                        'Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.'
-                    ),
                 passing_rule: zod
                     .object({
                         operator: zod
@@ -939,7 +917,7 @@ export const EvaluationsPartialUpdateBody = () => zod
             })
             .optional()
             .describe(
-                "Output config. For 'boolean' output_type: {allows_na} to permit N\/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min\/max\/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}."
+                "Output config. For 'boolean' output_type: {allows_na} to permit N\/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min\/max\/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}."
             ),
         conditions: zod
             .array(
@@ -1036,7 +1014,7 @@ export const EvaluationsPartialUpdateBody = () => zod
                                 'typesafe',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
                             ),
                         model: zod.string().max(evaluationsPartialUpdateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -1084,9 +1062,6 @@ export const EvaluationsTestHogCreateParams = () => zod.object({
 export const evaluationsTestHogCreateBodyOutputTypeDefault = `boolean`
 export const evaluationsTestHogCreateBodyOutputConfigStepExclusiveMin = 0
 
-export const evaluationsTestHogCreateBodyOutputConfigScoreLevelsMin = 2
-export const evaluationsTestHogCreateBodyOutputConfigScoreLevelsMax = 10
-
 export const evaluationsTestHogCreateBodySampleCountDefault = 5
 export const evaluationsTestHogCreateBodySampleCountMax = 10
 
@@ -1127,14 +1102,6 @@ export const EvaluationsTestHogCreateBody = () => zod.object({
                 .gt(evaluationsTestHogCreateBodyOutputConfigStepExclusiveMin)
                 .nullish()
                 .describe('Optional positive input increment. Does not round evaluation results.'),
-            score_levels: zod
-                .array(zod.string().min(1))
-                .min(evaluationsTestHogCreateBodyOutputConfigScoreLevelsMin)
-                .max(evaluationsTestHogCreateBodyOutputConfigScoreLevelsMax)
-                .nullish()
-                .describe(
-                    'Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set.'
-                ),
             passing_rule: zod
                 .object({
                     operator: zod
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json
index 3f0ed7e8b5da..c7acc821350a 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json
@@ -112,7 +112,7 @@
                             "type": "string"
                         },
                         "provider": {
-                            "description": "* `openai` - Openai\n* `anthropic` - Anthropic\n* `gemini` - Gemini\n* `openrouter` - Openrouter\n* `fireworks` - Fireworks\n* `azure_openai` - Azure OpenAI\n* `together_ai` - Together AI\n* `minimax` - MiniMax\n* `zeabur` - Zeabur AI Hub\n* `typesafe` - System One",
+                            "description": "* `openai` - Openai\n* `anthropic` - Anthropic\n* `gemini` - Gemini\n* `openrouter` - Openrouter\n* `fireworks` - Fireworks\n* `azure_openai` - Azure OpenAI\n* `together_ai` - Together AI\n* `minimax` - MiniMax\n* `zeabur` - Zeabur AI Hub\n* `typesafe` - System One (Jev)",
                             "enum": [
                                 "openai",
                                 "anthropic",
@@ -164,7 +164,7 @@
             "type": "string"
         },
         "output_config": {
-            "description": "Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min/max/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.",
+            "description": "Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min/max/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.",
             "properties": {
                 "allows_na": {
                     "description": "Whether the evaluation can return N/A for non-applicable generations.",
@@ -215,23 +215,6 @@
                     ],
                     "description": "Optional numeric passing rule. Null removes the rule; historical scores use the current rule."
                 },
-                "score_levels": {
-                    "anyOf": [
-                        {
-                            "items": {
-                                "minLength": 1,
-                                "type": "string"
-                            },
-                            "maxItems": 10,
-                            "minItems": 2,
-                            "type": "array"
-                        },
-                        {
-                            "type": "null"
-                        }
-                    ],
-                    "description": "Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set."
-                },
                 "step": {
                     "anyOf": [
                         {
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-test-hog.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-test-hog.json
index 3eafd9567df8..777e7b66f4df 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-test-hog.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-test-hog.json
@@ -69,23 +69,6 @@
                     ],
                     "description": "Optional numeric passing rule. Null removes the rule; historical scores use the current rule."
                 },
-                "score_levels": {
-                    "anyOf": [
-                        {
-                            "items": {
-                                "minLength": 1,
-                                "type": "string"
-                            },
-                            "maxItems": 10,
-                            "minItems": 2,
-                            "type": "array"
-                        },
-                        {
-                            "type": "null"
-                        }
-                    ],
-                    "description": "Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set."
-                },
                 "step": {
                     "anyOf": [
                         {
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json
index e1df61130c2a..69dc4c521d7a 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json
@@ -115,7 +115,7 @@
                             "type": "string"
                         },
                         "provider": {
-                            "description": "* `openai` - Openai\n* `anthropic` - Anthropic\n* `gemini` - Gemini\n* `openrouter` - Openrouter\n* `fireworks` - Fireworks\n* `azure_openai` - Azure OpenAI\n* `together_ai` - Together AI\n* `minimax` - MiniMax\n* `zeabur` - Zeabur AI Hub\n* `typesafe` - System One",
+                            "description": "* `openai` - Openai\n* `anthropic` - Anthropic\n* `gemini` - Gemini\n* `openrouter` - Openrouter\n* `fireworks` - Fireworks\n* `azure_openai` - Azure OpenAI\n* `together_ai` - Together AI\n* `minimax` - MiniMax\n* `zeabur` - Zeabur AI Hub\n* `typesafe` - System One (Jev)",
                             "enum": [
                                 "openai",
                                 "anthropic",
@@ -167,7 +167,7 @@
             "type": "string"
         },
         "output_config": {
-            "description": "Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': min/max/step, allows_na, score_levels, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.",
+            "description": "Output config. For 'boolean' output_type: {allows_na} to permit N/A results, and {true_is_failure} to declare that a true result means the evaluation found a problem. For 'numeric': only min/max/step, allows_na, and passing_rule {operator: 'gte'|'lte', threshold}. Do not send true_is_failure for numeric output. For 'sentiment': {}.",
             "properties": {
                 "allows_na": {
                     "description": "Whether the evaluation can return N/A for non-applicable generations.",
@@ -218,23 +218,6 @@
                     ],
                     "description": "Optional numeric passing rule. Null removes the rule; historical scores use the current rule."
                 },
-                "score_levels": {
-                    "anyOf": [
-                        {
-                            "items": {
-                                "minLength": 1,
-                                "type": "string"
-                            },
-                            "maxItems": 10,
-                            "minItems": 2,
-                            "type": "array"
-                        },
-                        {
-                            "type": "null"
-                        }
-                    ],
-                    "description": "Ordered rubric descriptions, spaced evenly from min to max. Required for System One numeric judges. Both bounds must be set."
-                },
                 "step": {
                     "anyOf": [
                         {

From d52c0495b153836d2128cda85fbb167eeb1ede8b Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Fri, 25 Sep 2026 14:51:50 +0200
Subject: [PATCH 10/30] fix(aio): reuse typesafe egress for boolean evaluations

---
 .../internal/ai-observability-judge-inputs.md |  11 +-
 posthog/egress/test/test_typesafe.py          |  78 +++++--
 posthog/egress/typesafe/README.md             |  25 ++-
 posthog/egress/typesafe/client.py             |  46 +++-
 posthog/egress/typesafe/limiter.py            |  18 +-
 posthog/egress/typesafe/transport.py          |   9 +-
 .../ai_observability/evaluation_llm_judge.py  |  16 +-
 .../ai_observability/test_run_evaluation.py   |  26 ++-
 .../backend/api/test/test_provider_keys.py    |  11 +-
 .../backend/llm/system_one.py                 | 194 ++++++-----------
 .../backend/llm/test/test_system_one.py       | 198 +++++-------------
 11 files changed, 286 insertions(+), 346 deletions(-)

diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index fcb8d21faed8..7f50ca2c3ab2 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -53,9 +53,8 @@ Saving a connection validates it with a short synthetic input and a Noul questio
 Select the connection and configured model on each evaluation; these connections cannot become the shared active provider key used by other AI features.
 Provider keys keep the provider they were created with; switching providers requires a new key.
 The evaluation integration uses Noul for boolean outputs, with the same formatted text for generation, trace, and session targets.
-The client accepts typed Noul, Score, and Choice questions independently of PostHog's evaluation output types.
-The client preserves raw Score values and Choice labels without applying evaluation policy.
-Mapping Score and Choice answers onto numeric and categorical evaluations is separate from this integration.
+The integration reuses the typed Noul client, rate limiter, and request telemetry in `posthog/egress/typesafe`.
+Numeric and categorical support is separate from this integration.
 Numeric evaluations retain their existing arbitrary ranges and completion-based judges.
 API compatibility does not guarantee equivalent judgments or calibration across models.
 Compare results on representative inputs when changing models.
@@ -68,10 +67,12 @@ Evaluations that allow N/A send a separate Noul question about whether the crite
 Uncertainty alone does not produce N/A.
 System One answers contain no written reasoning, so reports inspect the original source when explaining outcomes.
 
-Rate limits and overload responses are retried through Temporal, honoring `Retry-After` up to five minutes.
+Each endpoint and credential pair has a separate, hashed rate-limit scope shared across workers.
+Evaluations use the batch lane; connection validation uses the normal lane.
+Local budget exhaustion, rate limits, and overload responses are retried through Temporal, honoring `Retry-After` up to five minutes.
 If retries fail, the run fails and the evaluation stays enabled.
 Blocked endpoints and rejected requests disable the evaluation and mark the connection for revalidation, without recording model usage.
-Invalid probabilities, missing answers, mismatched answer types, and invalid score scales skip the item as an unparsable response.
+Invalid probabilities, missing answers, and mismatched answer types skip the item as an unparsable response.
 Inputs rejected for exceeding the model's context window are skipped.
 See TypeSafe's [API reference](https://docs.typesafe.ai/api) and [model limits and pricing](https://docs.typesafe.ai/models).
 
diff --git a/posthog/egress/test/test_typesafe.py b/posthog/egress/test/test_typesafe.py
index 8d8156fd1738..961d9a1f9db7 100644
--- a/posthog/egress/test/test_typesafe.py
+++ b/posthog/egress/test/test_typesafe.py
@@ -1,7 +1,7 @@
 import json
 from typing import Any
 
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
 
 from django.test import SimpleTestCase, override_settings
 
@@ -10,6 +10,7 @@
 from prometheus_client import REGISTRY
 
 from posthog.egress.limiter.policies import Priority, resolve_policy
+from posthog.egress.observability.observability import scope_fingerprint
 from posthog.egress.typesafe.client import (
     ChoiceAnswer,
     ChoiceQuestion,
@@ -65,18 +66,42 @@ def _with_answer(question_id: str, answer: dict[str, Any]) -> str:
 
 @override_settings(TYPESAFE_API_KEY=_FAKE_API_KEY)
 class TestTypeSafeEgress(SimpleTestCase):
-    def test_sends_the_documented_request_and_records_it_without_the_key(self) -> None:
-        consume = MagicMock(return_value=True)
-        before = REGISTRY.get_sample_value("typesafe_api_requests_total", _COUNTER_LABELS) or 0.0
+    @parameterized.expand(
+        [
+            ("instance_key", None, "https://api.typesafe.ai/v1"),
+            ("explicit_instance_key", _FAKE_API_KEY, "https://api.typesafe.ai/v1"),
+            ("customer_key", "fake-customer-key", "https://api.typesafe.ai/v1"),
+            ("custom_endpoint", "fake-customer-key", "https://decisions.example.com/v1"),
+            ("custom_no_auth", "", "https://decisions.example.com/v1"),
+        ]
+    )
+    def test_sends_the_documented_request_and_records_it_without_the_key(
+        self, _name: str, api_key: str | None, base_url: str
+    ) -> None:
+        resolved_key = _FAKE_API_KEY if api_key is None else api_key
+        scope = (
+            "default"
+            if base_url == "https://api.typesafe.ai/v1" and resolved_key == _FAKE_API_KEY
+            else scope_fingerprint(base_url, resolved_key)
+        )
+        labels = {**_COUNTER_LABELS, "account": scope}
+        before = REGISTRY.get_sample_value("typesafe_api_requests_total", labels) or 0.0
         with (
-            patch("posthog.egress.typesafe.transport.consume_typesafe_sync", consume),
+            patch("posthog.egress.limiter.backends.LimitsBackend.consume_sync", return_value=True) as consume,
             patch("requests.request", return_value=_response(200, json.dumps(_ANSWERS))) as request,
         ):
-            result = system_one(state={"ticket": "Payouts fail"}, questions=_QUESTIONS, source="test")
-
-        assert request.call_args.args == ("POST", "https://api.typesafe.ai/v1/systemone")
+            result = system_one(
+                state={"ticket": "Payouts fail"},
+                questions=_QUESTIONS,
+                source="test",
+                api_key=api_key,
+                base_url=base_url,
+            )
+
+        assert request.call_args.args == ("POST", f"{base_url}/systemone")
         kwargs = request.call_args.kwargs
-        assert kwargs["headers"]["Authorization"] == f"Bearer {_FAKE_API_KEY}"
+        assert kwargs["headers"].get("Authorization") == (f"Bearer {resolved_key}" if resolved_key else None)
+        assert kwargs["allow_redirects"] is False
         assert kwargs["json"] == {
             "model": "jev-latest",
             "state": {"ticket": "Payouts fail"},
@@ -95,14 +120,20 @@ def test_sends_the_documented_request_and_records_it_without_the_key(self) -> No
             "team": ChoiceAnswer(choice="billing", confidence=0.81, probabilities={"billing": 0.88, "support": 0.12}),
         }
         assert result.input_tokens == 296
-        assert consume.call_args.kwargs["priority"] is Priority.NORMAL
-        assert REGISTRY.get_sample_value("typesafe_api_requests_total", _COUNTER_LABELS) == before + 1
-        assert _FAKE_API_KEY not in str(list(REGISTRY.collect()))
+        assert result.output_tokens == 20
+        assert consume.call_args.args[0] == typesafe_account_key(scope)
+        assert consume.call_args.args[3] is Priority.NORMAL
+        assert REGISTRY.get_sample_value("typesafe_api_requests_total", labels) == before + 1
+        metrics = str(list(REGISTRY.collect()))
+        assert _FAKE_API_KEY not in metrics
+        assert "fake-customer-key" not in metrics
+        assert "decisions.example.com" not in metrics
 
     @parameterized.expand(
         [
             ("rate_limited", 429, json.dumps({"error": "rate limited"}), 429),
             ("overloaded", 529, json.dumps({"error": "overloaded"}), 529),
+            ("redirect", 302, "", 302),
             ("non_json_body", 200, "<html>bad gateway</html>", None),
             (
                 "missing_answer",
@@ -149,17 +180,32 @@ def test_raises_rather_than_returning_a_partial_answer(
 
     @parameterized.expand(
         [
-            ("no_configured_key", "", Priority.NORMAL, TypeSafeNotConfigured),
-            ("critical_lane_skips_the_spend_ceiling", _FAKE_API_KEY, Priority.CRITICAL, ValueError),
+            ("no_configured_key", "", "https://api.typesafe.ai/v1", Priority.NORMAL, TypeSafeNotConfigured),
+            (
+                "critical_lane_skips_the_spend_ceiling",
+                _FAKE_API_KEY,
+                "https://api.typesafe.ai/v1",
+                Priority.CRITICAL,
+                ValueError,
+            ),
+            (
+                "custom_endpoint_cannot_inherit_instance_key",
+                _FAKE_API_KEY,
+                "https://decisions.example.com/v1",
+                Priority.NORMAL,
+                ValueError,
+            ),
         ]
     )
-    def test_never_calls_out(self, _name: str, api_key: str, priority: Priority, error: type[Exception]) -> None:
+    def test_never_calls_out(
+        self, _name: str, api_key: str, base_url: str, priority: Priority, error: type[Exception]
+    ) -> None:
         with (
             override_settings(TYPESAFE_API_KEY=api_key),
             patch("requests.request") as request,
             self.assertRaises(error),
         ):
-            system_one(state="hi", questions=_QUESTIONS, source="test", priority=priority)
+            system_one(state="hi", questions=_QUESTIONS, source="test", priority=priority, base_url=base_url)
         request.assert_not_called()
 
     @override_settings(TYPESAFE_EGRESS_PER_MINUTE_BUDGET=7, TYPESAFE_EGRESS_HOURLY_BUDGET=11)
diff --git a/posthog/egress/typesafe/README.md b/posthog/egress/typesafe/README.md
index b4d7490d54fd..c264c35559c7 100644
--- a/posthog/egress/typesafe/README.md
+++ b/posthog/egress/typesafe/README.md
@@ -13,14 +13,18 @@ Every caller meets these rules before it merges, and a reviewer blocks a caller
 - **A launch that sends customer data needs an explicit opt-in.** Each customer turns it on before any of their data goes to TypeSafe. Use the approved opt-in copy. Until that copy exists, no caller sends customer data.
 - **A launch that sends customer data needs sign-off from leadership** before it ships, in addition to the opt-in.
 
-A self-hosted instance has no `TYPESAFE_API_KEY`, so it sends nothing.
+A self-hosted instance without `TYPESAFE_API_KEY` makes no default-service requests unless a caller passes an explicit credential.
 
 ## Identity
 
-An instance holds a single TypeSafe API key.
-The whole instance draws from one budget under the key `typesafe:account:default`.
+Calls with the configured instance TypeSafe API key draw from one budget under `typesafe:account:default`.
+An explicit credential for the official endpoint uses that same budget when it matches the instance key.
+Other credentials and compatible endpoints use a fingerprint of the base URL and credential as their scope.
+The same connection shares a budget across workers and projects; different credentials or endpoints have separate budgets.
+No credential or custom URL reaches a metric label.
 TypeSafe does not say whether it meters its limits per API key or per account, so treat that as unverified.
-With one key per instance, both readings give the same single budget.
+For customer-owned keys, a credential scope is a conservative local budget, not a claim that separate keys on one vendor account have separate quotas.
+Provider rate-limit responses remain the backstop.
 
 ## Budget
 
@@ -38,13 +42,15 @@ At typical request sizes of a few hundred tokens, the request budget keeps the t
 At the 64k-token maximum, 600 requests a minute would exceed it, and TypeSafe answers with a 429.
 A caller that sends large states lowers its own request rate.
 Raise both settings when real traffic outgrows them.
+Compatible endpoints use these same settings as operator ceilings; their vendor limits are unverified.
 
 ## Lanes and callers
 
 The default reserve ladder applies, and `typesafe_request` defaults to `NORMAL`.
 `typesafe_request` rejects `CRITICAL`, because a `CRITICAL` call is never shed and would skip the hourly spend ceiling.
 Give every caller an explicit lane: `NORMAL` when a person waits for the answer, `BATCH` for background work.
-No caller exists on master yet. Each new caller adds itself here with its lane and its feature flag.
+AI observability uses `llma_evaluations` as its source, `BATCH` for evaluation runs, and `NORMAL` for synthetic connection validation.
+The evaluation caller must meet the usage policy above before launch.
 
 ## Rate-limit headers
 
@@ -56,14 +62,17 @@ The counter is `typesafe_api_requests_total`, labeled `account, method, endpoint
 
 ## Auth
 
-`TYPESAFE_API_KEY` authenticates every call as a bearer token.
-An instance without one makes no request at all: `system_one` raises `TypeSafeNotConfigured`.
+`system_one` uses `TYPESAFE_API_KEY` when no credential is passed.
+An explicit `api_key` selects a caller-owned bearer token; an empty string selects no authentication for a compatible endpoint.
+The official endpoint always requires a key and raises `TypeSafeNotConfigured` without one.
+Custom URLs supplied by users require a DNS-pinned session; AI observability validates them as public HTTPS URLs and supplies that session.
+Redirects are disabled for every request.
 
 ## Typed client
 
 Callers use `client.py` rather than `typesafe_request`.
 `system_one(state=..., questions=..., source=..., model=...)` sends one state with a map of `NoulQuestion` and `ChoiceQuestion` entries.
-It returns a `SystemOneResult` with one `NoulAnswer` or `ChoiceAnswer` per question id, the versioned model that answered, and the input token count.
+It returns a `SystemOneResult` with one `NoulAnswer` or `ChoiceAnswer` per question id, the versioned model that answered, and available input and output token counts.
 It raises `TypeSafeRequestFailed` on an HTTP error, and on a body that lacks the answering model or a complete answer for any question.
 A choice outside the options the caller sent, or a choice without a probability for every option, counts as incomplete.
 `model` defaults to the `jev-latest` alias. A caller that tunes thresholds against one version pins that version's id, such as `jev-1.13.0`.
diff --git a/posthog/egress/typesafe/client.py b/posthog/egress/typesafe/client.py
index 5cb358213b7a..2bf4847785bf 100644
--- a/posthog/egress/typesafe/client.py
+++ b/posthog/egress/typesafe/client.py
@@ -21,8 +21,12 @@
 
 from django.conf import settings
 
+import requests
+
 from posthog.dataclasses import frozen
 from posthog.egress.limiter.policies import Priority
+from posthog.egress.observability.observability import scope_fingerprint
+from posthog.egress.typesafe.limiter import ACCOUNT_SCOPE_ID
 from posthog.egress.typesafe.transport import DEFAULT_TIMEOUT, typesafe_request
 
 TYPESAFE_API_BASE = "https://api.typesafe.ai"
@@ -48,9 +52,12 @@ class TypeSafeRequestFailed(Exception):
     match the documented shape). ``status_code`` is set for an HTTP error, so a caller can defer a
     429 or 529 and drop the rest."""
 
-    def __init__(self, message: str, *, status_code: int | None = None) -> None:
+    def __init__(
+        self, message: str, *, status_code: int | None = None, response: requests.Response | None = None
+    ) -> None:
         super().__init__(message)
         self.status_code = status_code
+        self.response = response
 
 
 @frozen
@@ -115,6 +122,7 @@ class SystemOneResult:
     model: str
     answers: Mapping[str, Answer]
     input_tokens: int | None
+    output_tokens: int | None = None
 
 
 def _as_mapping(value: object) -> Mapping[str, object] | None:
@@ -169,6 +177,9 @@ def system_one(
     model: str = JEV_LATEST,
     priority: Priority = Priority.NORMAL,
     timeout: float | tuple[float, float] = DEFAULT_TIMEOUT,
+    api_key: str | None = None,
+    base_url: str = f"{TYPESAFE_API_BASE}/v1",
+    session: requests.Session | None = None,
 ) -> SystemOneResult:
     """Evaluate ``state`` against every question in one call. TypeSafe answers the questions in
     parallel, so a caller asks everything it needs in one request.
@@ -180,19 +191,30 @@ def system_one(
     """
     if not questions:
         raise ValueError("system_one needs at least one question")
-    api_key = settings.TYPESAFE_API_KEY
-    if not api_key:
+    base_url = base_url.rstrip("/")
+    if api_key is None and base_url != f"{TYPESAFE_API_BASE}/v1":
+        raise ValueError("Pass an explicit credential or an empty string for a custom endpoint")
+    resolved_api_key = settings.TYPESAFE_API_KEY if api_key is None else api_key
+    if not resolved_api_key and base_url == f"{TYPESAFE_API_BASE}/v1":
         raise TypeSafeNotConfigured("No TYPESAFE_API_KEY configured")
+    # Customer credentials and custom endpoints must not share the instance account's budget.
+    scope = (
+        ACCOUNT_SCOPE_ID
+        if base_url == f"{TYPESAFE_API_BASE}/v1" and resolved_api_key == settings.TYPESAFE_API_KEY
+        else scope_fingerprint(base_url, resolved_api_key)
+    )
 
     response = typesafe_request(
         "POST",
-        f"{TYPESAFE_API_BASE}{SYSTEM_ONE_ENDPOINT}",
-        api_key=api_key,
+        f"{base_url}/systemone",
+        api_key=resolved_api_key,
+        scope=scope,
         source=source,
         endpoint=SYSTEM_ONE_ENDPOINT,
         priority=priority,
         timeout=timeout,
         allow_redirects=False,
+        session=session,
         json={
             "model": model,
             "state": state,
@@ -200,10 +222,12 @@ def system_one(
         },
     )
 
-    if not response.ok:
+    if response.status_code != 200:
         # A 422 body echoes the offending field, which can carry the state, so keep the body out of
         # the exception that gets logged.
-        raise TypeSafeRequestFailed(f"TypeSafe returned HTTP {response.status_code}", status_code=response.status_code)
+        raise TypeSafeRequestFailed(
+            f"TypeSafe returned HTTP {response.status_code}", status_code=response.status_code, response=response
+        )
     try:
         payload: object = response.json()
     except ValueError as exc:
@@ -219,11 +243,17 @@ def system_one(
 
     usage = _as_mapping(body.get("usage")) or {}
     input_tokens = usage.get("input_tokens")
+    output_tokens = usage.get("output_tokens")
     return SystemOneResult(
         model=answered_model,
         answers={
             question_id: _parse_answer(question_id, question, raw_answers.get(question_id))
             for question_id, question in questions.items()
         },
-        input_tokens=input_tokens if isinstance(input_tokens, int) and not isinstance(input_tokens, bool) else None,
+        input_tokens=input_tokens
+        if isinstance(input_tokens, int) and not isinstance(input_tokens, bool) and input_tokens >= 0
+        else None,
+        output_tokens=output_tokens
+        if isinstance(output_tokens, int) and not isinstance(output_tokens, bool) and output_tokens >= 0
+        else None,
     )
diff --git a/posthog/egress/typesafe/limiter.py b/posthog/egress/typesafe/limiter.py
index 512e8c41168b..862b0e28c57d 100644
--- a/posthog/egress/typesafe/limiter.py
+++ b/posthog/egress/typesafe/limiter.py
@@ -1,7 +1,8 @@
 """TypeSafe egress budget.
 
-A PostHog instance holds a single TypeSafe API key, so the whole instance draws from one shared
-budget under a constant scope. The per-minute default stays at half of TypeSafe's published request
+A PostHog instance's TypeSafe API key draws from one shared budget under a constant scope.
+Caller-owned credentials and compatible endpoints use separate fingerprinted scopes.
+The per-minute default stays at half of TypeSafe's published request
 limit, because TypeSafe changes its limits without notice. The hourly default is an operator ceiling
 on spend, because TypeSafe bills every input token.
 
@@ -27,12 +28,13 @@
 )
 
 
-def typesafe_account_key() -> str:
-    """Limiter key for the instance's single TypeSafe API key."""
-    return f"{TYPESAFE_DOMAIN}:account:{ACCOUNT_SCOPE_ID}"
+def typesafe_account_key(scope: str = ACCOUNT_SCOPE_ID) -> str:
+    return f"{TYPESAFE_DOMAIN}:account:{scope}"
 
 
-def consume_typesafe_sync(n: int = 1, *, priority: Priority = Priority.NORMAL, source: str = "unknown") -> bool:
-    """Reserve ``n`` requests against the instance's TypeSafe budget. Returns False when the budget
+def consume_typesafe_sync(
+    n: int = 1, *, scope: str = ACCOUNT_SCOPE_ID, priority: Priority = Priority.NORMAL, source: str = "unknown"
+) -> bool:
+    """Reserve ``n`` requests against the scope's TypeSafe budget. Returns False when the budget
     (or this ``priority``'s reserved floor) is exhausted, so degrade gracefully rather than calling out."""
-    return get_outbound_rate_limiter().consume_sync(typesafe_account_key(), n, priority=priority, source=source)
+    return get_outbound_rate_limiter().consume_sync(typesafe_account_key(scope), n, priority=priority, source=source)
diff --git a/posthog/egress/typesafe/transport.py b/posthog/egress/typesafe/transport.py
index 5a7ae6e4660e..196d99327b40 100644
--- a/posthog/egress/typesafe/transport.py
+++ b/posthog/egress/typesafe/transport.py
@@ -1,7 +1,7 @@
 """TypeSafe incarnation of the egress transport.
 
 ``typesafe_request`` is the one way to call TypeSafe from anywhere in the codebase: it gates on the
-instance's shared account budget and records telemetry by construction. It stays token-agnostic
+selected account budget and records telemetry by construction. It stays token-agnostic
 like the other incarnations, so the caller owns where the API key comes from:
 :mod:`posthog.egress.typesafe.client` reads it from settings.
 """
@@ -31,7 +31,7 @@ def _standard_headers(self) -> dict[str, str]:
         return {"Accept": "application/json", "Content-Type": "application/json"}
 
     def _consume(self, scope: str, priority: Priority, source: str, url: str) -> bool:
-        return consume_typesafe_sync(priority=priority, source=source)
+        return consume_typesafe_sync(scope=scope, priority=priority, source=source)
 
     def _budget_exhausted_error(self, scope: str) -> TypeSafeEgressBudgetExhausted:
         return TypeSafeEgressBudgetExhausted("TypeSafe egress budget exhausted; degrading", scope=scope)
@@ -51,6 +51,7 @@ def typesafe_request(
     api_key: str,
     source: str,
     endpoint: str,
+    scope: str = ACCOUNT_SCOPE_ID,
     priority: Priority = Priority.NORMAL,
     timeout: float | tuple[float, float] = DEFAULT_TIMEOUT,
     **kwargs: Any,
@@ -67,8 +68,8 @@ def typesafe_request(
         method,
         url,
         source=source,
-        headers={"Authorization": f"Bearer {api_key}"},
-        scope=ACCOUNT_SCOPE_ID,
+        headers={"Authorization": f"Bearer {api_key}"} if api_key else {},
+        scope=scope,
         priority=priority,
         endpoint=endpoint,
         timeout=timeout,
diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.py
index eca7d8153c67..d92e5899d11a 100644
--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py
+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py
@@ -12,6 +12,7 @@
 from temporalio.exceptions import ApplicationError
 
 from posthog.dataclasses import frozen
+from posthog.egress.typesafe.client import NoulAnswer, NoulQuestion, Question
 from posthog.temporal.ai_observability.evaluation_errors import (
     require_user_error_spec,
     terminal_user_error_result,
@@ -49,10 +50,7 @@
     StructuredOutputParseError,
 )
 from products.ai_observability.backend.llm.system_one import (
-    NoulAnswer,
-    NoulQuestion,
     SystemOneClient,
-    SystemOneQuestion,
     SystemOneRateLimitError,
     SystemOneRequestRejectedError,
 )
@@ -471,7 +469,7 @@ def call_llm_judge(
             if provider_key is not None and provider_key.provider != provider:
                 raise ProviderMismatchError(provider_key.provider, provider)
             prompt = evaluation["evaluation_config"]["prompt"]
-            questions: dict[str, SystemOneQuestion] = {"verdict": NoulQuestion(instructions=prompt)}
+            questions: dict[str, Question] = {"verdict": NoulQuestion(instructions=prompt)}
             if allows_na:
                 questions["applicable"] = NoulQuestion(
                     instructions=(
@@ -493,10 +491,10 @@ def call_llm_judge(
             if allows_na:
                 applicability_answer = system_one_result.answers["applicable"]
                 assert isinstance(applicability_answer, NoulAnswer)
-                applicable = applicability_answer.noul >= 0.5
+                applicable = applicability_answer.probability >= 0.5
             verdict_answer = system_one_result.answers["verdict"]
             assert isinstance(verdict_answer, NoulAnswer)
-            probability = verdict_answer.noul
+            probability = verdict_answer.probability
             parsed = (
                 BooleanWithNAEvalResult(
                     reasoning="",
@@ -511,9 +509,9 @@ def call_llm_judge(
                 model=model,
                 parsed=parsed,
                 usage=Usage(
-                    input_tokens=system_one_result.usage.input_tokens,
-                    output_tokens=system_one_result.usage.output_tokens,
-                    total_tokens=system_one_result.usage.input_tokens + system_one_result.usage.output_tokens,
+                    input_tokens=(system_one_result.input_tokens or 0),
+                    output_tokens=(system_one_result.output_tokens or 0),
+                    total_tokens=(system_one_result.input_tokens or 0) + (system_one_result.output_tokens or 0),
                 ),
             )
         else:
diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index 2ea7a07eb8ee..b1f0efce999d 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -1,6 +1,7 @@
 import json
 import uuid
 from datetime import UTC, datetime, timedelta
+from ipaddress import ip_address
 from typing import Any, cast
 
 import pytest
@@ -133,8 +134,10 @@ def test_typesafe_judge_emits_boolean_probability_without_reasoning(
         "evaluation_config": {"prompt": "Is the response polite?"},
     }
     with (
+        patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),
+        patch("posthog.egress.limiter.backends.LimitsBackend.consume_sync", return_value=True),
         patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
-        patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response) as request,
+        patch("requests.Session.request", return_value=response) as request,
     ):
         spec.return_value.resolve.return_value = resolved
         result = call_llm_judge(
@@ -160,8 +163,10 @@ def test_typesafe_judge_emits_boolean_probability_without_reasoning(
 
 def test_system_one_numeric_mapping_is_not_enabled() -> None:
     with (
+        patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),
+        patch("posthog.egress.limiter.backends.LimitsBackend.consume_sync", return_value=True),
         patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
-        patch("products.ai_observability.backend.llm.system_one.pinned_request") as request,
+        patch("requests.Session.request") as request,
         pytest.raises(ApplicationError) as error,
     ):
         spec.return_value.resolve.return_value = MagicMock(provider="typesafe")
@@ -179,9 +184,11 @@ def test_system_one_numeric_mapping_is_not_enabled() -> None:
 def test_system_one_rejected_requests_disable_without_model_cost_attribution(status: int) -> None:
     key = MagicMock(provider="typesafe", encrypted_config={"api_key": "example-token"})
     with (
+        patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),
+        patch("posthog.egress.limiter.backends.LimitsBackend.consume_sync", return_value=True),
         patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
         patch(
-            "products.ai_observability.backend.llm.system_one.pinned_request",
+            "requests.Session.request",
             return_value=MagicMock(status_code=status, text="Invalid request"),
         ),
     ):
@@ -202,14 +209,17 @@ def test_system_one_rejected_requests_disable_without_model_cost_attribution(sta
     assert "provider" not in result
 
 
-def test_typesafe_rate_limit_retries_without_disabling_the_evaluation() -> None:
+@pytest.mark.parametrize("budget_granted", [True, False])
+def test_typesafe_rate_limit_retries_without_disabling_the_evaluation(budget_granted: bool) -> None:
     key = MagicMock(provider="typesafe", encrypted_config={"api_key": "test-typesafe-key"})
     with (
+        patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),
+        patch("posthog.egress.limiter.backends.LimitsBackend.consume_sync", return_value=budget_granted),
         patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
         patch(
-            "products.ai_observability.backend.llm.system_one.pinned_request",
+            "requests.Session.request",
             return_value=MagicMock(status_code=429, headers={"Retry-After": "15"}),
-        ),
+        ) as request,
         pytest.raises(ApplicationError) as error,
     ):
         spec.return_value.resolve.return_value = MagicMock(
@@ -222,7 +232,9 @@ def test_typesafe_rate_limit_retries_without_disabling_the_evaluation() -> None:
             allows_na=False,
         )
     assert not error.value.non_retryable
-    assert error.value.next_retry_delay == timedelta(seconds=15)
+    assert error.value.next_retry_delay == (timedelta(seconds=15) if budget_granted else None)
+    if not budget_granted:
+        request.assert_not_called()
     assert terminal_user_error_result_from_application_error(error.value, allows_na=False) is None
 
 
diff --git a/products/ai_observability/backend/api/test/test_provider_keys.py b/products/ai_observability/backend/api/test/test_provider_keys.py
index 7dca755bf66a..08caff087ee2 100644
--- a/products/ai_observability/backend/api/test/test_provider_keys.py
+++ b/products/ai_observability/backend/api/test/test_provider_keys.py
@@ -1,3 +1,4 @@
+from ipaddress import ip_address
 from uuid import uuid4
 
 from posthog.test.base import APIBaseTest
@@ -153,8 +154,10 @@ def test_can_create_provider_key(self, provider: str, mock_validate: Mock) -> No
         )
         mock_validate.assert_called_once_with(provider, "sk-test-key-12345", **expected_config)
 
-    @patch("products.ai_observability.backend.llm.system_one.pinned_request")
-    def test_custom_system_one_connection_round_trip(self, request: Mock) -> None:
+    @patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")})
+    @patch("posthog.egress.limiter.backends.LimitsBackend.consume_sync", return_value=True)
+    @patch("requests.Session.request")
+    def test_custom_system_one_connection_round_trip(self, request: Mock, _budget: Mock, _dns: Mock) -> None:
         request.return_value = Mock(status_code=200)
         request.return_value.json.return_value = {
             "model": "custom-model",
@@ -178,7 +181,7 @@ def test_custom_system_one_connection_round_trip(self, request: Mock) -> None:
         self.assertEqual(Client.list_models("typesafe", **key.provider_extra_kwargs()), ["custom-model"])
         model_config = LLMModelConfiguration(provider="typesafe", model="custom-model", provider_key=key)
         self.assertEqual(model_config.get_available_models(), ["custom-model"])
-        self.assertEqual(request.call_args.kwargs["headers"], {})
+        self.assertNotIn("Authorization", request.call_args.kwargs["headers"])
         request.reset_mock()
         response = self.client.patch(f"{url}{key.id}/", {"base_url": "https://other.example.com/v1"})
         self.assertEqual(response.status_code, 400)
@@ -200,7 +203,7 @@ def test_custom_system_one_connection_round_trip(self, request: Mock) -> None:
         self.assertEqual(response.status_code, 200, response.data)
         key.refresh_from_db()
         self.assertEqual(key.encrypted_config["base_url"], "https://other.example.com/v1")
-        self.assertEqual(request.call_args.kwargs["headers"], {"Authorization": "Bearer fake-token"})
+        self.assertEqual(request.call_args.kwargs["headers"]["Authorization"], "Bearer fake-token")
         response = self.client.post(f"{url}{key.id}/validate/")
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(request.call_args.args[1], "https://other.example.com/v1/systemone")
diff --git a/products/ai_observability/backend/llm/system_one.py b/products/ai_observability/backend/llm/system_one.py
index 51579efa5c61..bfb22260fe5f 100644
--- a/products/ai_observability/backend/llm/system_one.py
+++ b/products/ai_observability/backend/llm/system_one.py
@@ -2,13 +2,23 @@
 from collections.abc import Mapping
 from datetime import UTC, datetime
 from email.utils import parsedate_to_datetime
-from typing import Annotated, Literal
 from urllib.parse import urlsplit
 
 import requests
-from pydantic import BaseModel, Field, JsonValue, ValidationError
 
-from posthog.security.pinned_requests import SSRFBlockedError, pinned_request
+from posthog.egress.limiter.policies import Priority
+from posthog.egress.typesafe.client import (
+    TYPESAFE_API_BASE,
+    JsonValue,
+    NoulQuestion,
+    Question,
+    SystemOneResult,
+    TypeSafeNotConfigured,
+    TypeSafeRequestFailed,
+    system_one,
+)
+from posthog.egress.typesafe.transport import TypeSafeEgressBudgetExhausted
+from posthog.security.pinned_requests import SSRFBlockedError, pinned_session
 from posthog.security.url_validation import has_authority_bypass_chars
 
 from products.ai_observability.backend.llm.errors import (
@@ -23,61 +33,6 @@
     is_context_window_error_message,
 )
 
-type SystemOneContent = str | dict[str, JsonValue] | list[JsonValue]
-type Probability = Annotated[float, Field(strict=True, ge=0, le=1, allow_inf_nan=False)]
-
-
-class NoulQuestion(BaseModel):
-    type: Literal["noul"] = "noul"
-    instructions: SystemOneContent | None = None
-    criteria: dict[Literal["true", "false"], SystemOneContent | None] | None = None
-
-
-class ScoreQuestion(BaseModel):
-    type: Literal["score"] = "score"
-    instructions: SystemOneContent | None = None
-    criteria: list[SystemOneContent] = Field(min_length=1, max_length=10)
-
-
-class ChoiceQuestion(BaseModel):
-    type: Literal["choice"] = "choice"
-    instructions: SystemOneContent | None = None
-    criteria: dict[str, SystemOneContent | None] = Field(min_length=1, max_length=255)
-
-
-type SystemOneQuestion = NoulQuestion | ScoreQuestion | ChoiceQuestion
-
-
-class NoulAnswer(BaseModel):
-    type: Literal["noul"]
-    noul: Probability
-
-
-class ScoreAnswer(BaseModel):
-    type: Literal["score"]
-    score: float = Field(strict=True, ge=0, allow_inf_nan=False)
-    legend: dict[str, SystemOneContent]
-    probabilities: dict[str, Probability]
-    confidence: Probability
-
-
-class ChoiceAnswer(BaseModel):
-    type: Literal["choice"]
-    choice: str
-    probabilities: dict[str, Probability]
-    confidence: Probability
-
-
-class SystemOneUsage(BaseModel):
-    input_tokens: int = Field(strict=True, ge=0)
-    output_tokens: int = Field(strict=True, ge=0)
-
-
-class SystemOneResponse(BaseModel):
-    model: str = Field(min_length=1)
-    answers: dict[str, Annotated[NoulAnswer | ScoreAnswer | ChoiceAnswer, Field(discriminator="type")]]
-    usage: SystemOneUsage
-
 
 class SystemOneRequestRejectedError(LLMError):
     pass
@@ -101,7 +56,7 @@ def __init__(self, retry_after: str | None) -> None:
 
 class SystemOneClient:
     MODEL = "jev-1.13.0"
-    BASE_URL = "https://api.typesafe.ai/v1"
+    BASE_URL = f"{TYPESAFE_API_BASE}/v1"
 
     @staticmethod
     def normalize_base_url(base_url: str) -> str:
@@ -123,88 +78,68 @@ def evaluate(
         *,
         api_key: str,
         model: str,
-        state: SystemOneContent,
-        questions: Mapping[str, SystemOneQuestion],
+        state: JsonValue,
+        questions: Mapping[str, Question],
         base_url: str = BASE_URL,
-    ) -> SystemOneResponse:
+        priority: Priority = Priority.BATCH,
+    ) -> SystemOneResult:
         base_url = SystemOneClient.normalize_base_url(base_url)
         if not api_key and base_url == SystemOneClient.BASE_URL:
             raise AuthenticationError("A TypeSafe API key is required.")
-        if not questions:
-            raise ValueError("Provide at least one System One question.")
         try:
-            response = pinned_request(
-                "POST",
-                f"{base_url}/systemone",
-                headers={"Authorization": f"Bearer {api_key}"} if api_key else {},
-                json={
-                    "model": model,
-                    "state": state,
-                    "questions": {
-                        key: question.model_dump(mode="json", exclude_none=True) for key, question in questions.items()
-                    },
-                },
-                timeout=60,
-            )
+            with pinned_session(f"{base_url}/systemone") as session:
+                result = system_one(
+                    api_key=api_key,
+                    base_url=base_url,
+                    model=model,
+                    state=state,
+                    questions=questions,
+                    source="llma_evaluations",
+                    priority=priority,
+                    timeout=60,
+                    session=session,
+                )
+            if result.input_tokens is None or result.output_tokens is None:
+                raise StructuredOutputParseError("The endpoint returned invalid token usage. Check compatibility.")
+            return result
+        except TypeSafeNotConfigured as error:
+            raise AuthenticationError("A TypeSafe API key is required.") from error
+        except TypeSafeEgressBudgetExhausted as error:
+            raise SystemOneRateLimitError(None) from error
         except SSRFBlockedError as error:
             raise SystemOneRequestRejectedError("This endpoint is not allowed. Use a public HTTPS endpoint.") from error
         except requests.RequestException as error:
             raise ProviderConnectionError("Could not reach the System One endpoint. Try again.") from error
-
-        if response.status_code == 401:
-            raise AuthenticationError("The endpoint rejected this credential. Check the bearer token.")
-        if response.status_code == 403:
-            raise ModelPermissionError(model)
-        if response.status_code == 404:
-            raise ModelNotFoundError(model)
-        if response.status_code in (429, 503, 529):
-            raise SystemOneRateLimitError(response.headers.get("Retry-After"))
-        if response.status_code >= 500:
-            raise ProviderConnectionError("The System One endpoint is temporarily unavailable. Try again.")
-        if response.status_code == 413 or (
-            response.status_code == 422 and is_context_window_error_message(response.text)
-        ):
-            raise ContextWindowExceededError("This input exceeds the endpoint's size limit. Reduce the input.")
-        if response.status_code != 200:
+        except TypeSafeRequestFailed as error:
+            status = error.status_code
+            response = error.response
+            if status is None:
+                raise StructuredOutputParseError(
+                    "The endpoint returned an invalid System One response. Check compatibility."
+                ) from error
+            if status == 401:
+                raise AuthenticationError("The endpoint rejected this credential. Check the bearer token.") from error
+            if status == 403:
+                raise ModelPermissionError(model) from error
+            if status == 404:
+                raise ModelNotFoundError(model) from error
+            if status in (429, 503, 529):
+                raise SystemOneRateLimitError(
+                    response.headers.get("Retry-After") if response is not None else None
+                ) from error
+            if status >= 500:
+                raise ProviderConnectionError(
+                    "The System One endpoint is temporarily unavailable. Try again."
+                ) from error
+            if status == 413 or (
+                status == 422 and response is not None and is_context_window_error_message(response.text)
+            ):
+                raise ContextWindowExceededError(
+                    "This input exceeds the endpoint's size limit. Reduce the input."
+                ) from error
             raise SystemOneRequestRejectedError(
                 "The endpoint rejected the evaluation request. Check the model and criteria."
-            )
-        try:
-            result = SystemOneResponse.model_validate(response.json())
-        except (ValidationError, ValueError) as error:
-            raise StructuredOutputParseError(
-                "The endpoint returned an invalid System One response. Check compatibility."
             ) from error
-        # Missing or mismatched answers cannot become valid evaluation results.
-        if not questions.keys() <= result.answers.keys():
-            raise StructuredOutputParseError(
-                "The endpoint did not answer every evaluation question. Check compatibility."
-            )
-        for key, question in questions.items():
-            answer = result.answers[key]
-            if answer.type != question.type:
-                raise StructuredOutputParseError("The endpoint returned the wrong answer type. Check compatibility.")
-            if isinstance(question, ChoiceQuestion) and isinstance(answer, ChoiceAnswer):
-                if set(answer.probabilities) != set(question.criteria) or answer.choice not in question.criteria:
-                    raise StructuredOutputParseError("The endpoint returned different choices. Check compatibility.")
-            if isinstance(question, ScoreQuestion) and isinstance(answer, ScoreAnswer):
-                levels = {str(index) for index in range(len(question.criteria))}
-                if (
-                    set(answer.probabilities) != levels
-                    or set(answer.legend) != levels
-                    or answer.score > len(levels) - 1
-                ):
-                    raise StructuredOutputParseError(
-                        "The endpoint returned an invalid score scale. Check compatibility."
-                    )
-            # Compatible servers round each probability to four decimal places.
-            if isinstance(answer, ScoreAnswer | ChoiceAnswer) and not math.isclose(
-                sum(answer.probabilities.values()), 1.0, abs_tol=max(0.001, len(answer.probabilities) * 0.00005)
-            ):
-                raise StructuredOutputParseError(
-                    "The endpoint returned an invalid probability distribution. Check compatibility."
-                )
-        return result
 
     @staticmethod
     def validate_key(api_key: str, *, base_url: str = BASE_URL, model: str = MODEL) -> tuple[str, str | None]:
@@ -214,6 +149,7 @@ def validate_key(api_key: str, *, base_url: str = BASE_URL, model: str = MODEL)
                 base_url=base_url,
                 model=model,
                 state="Hello!",
+                priority=Priority.NORMAL,
                 questions={"verdict": NoulQuestion(instructions="Does the text contain a greeting?")},
             )
         except (AuthenticationError, ModelPermissionError) as error:
diff --git a/products/ai_observability/backend/llm/test/test_system_one.py b/products/ai_observability/backend/llm/test/test_system_one.py
index 4cb03bf4b674..0eb0449456e4 100644
--- a/products/ai_observability/backend/llm/test/test_system_one.py
+++ b/products/ai_observability/backend/llm/test/test_system_one.py
@@ -1,8 +1,13 @@
+from collections.abc import Iterator
+from ipaddress import ip_address
+
 import pytest
 from unittest.mock import Mock, patch
 
 from django.test import override_settings
 
+from posthog.egress.typesafe.client import NoulAnswer, NoulQuestion
+
 from products.ai_observability.backend.llm.client import Client
 from products.ai_observability.backend.llm.errors import (
     AuthenticationError,
@@ -12,19 +17,21 @@
     StructuredOutputParseError,
 )
 from products.ai_observability.backend.llm.system_one import (
-    ChoiceAnswer,
-    ChoiceQuestion,
-    NoulAnswer,
-    NoulQuestion,
-    ScoreAnswer,
-    ScoreQuestion,
     SystemOneClient,
-    SystemOneQuestion,
     SystemOneRateLimitError,
     SystemOneRequestRejectedError,
 )
 
 
+@pytest.fixture(autouse=True)
+def isolated_egress_budget() -> Iterator[None]:
+    with (
+        patch("posthog.egress.limiter.backends.LimitsBackend.consume_sync", return_value=True),
+        patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),
+    ):
+        yield
+
+
 @pytest.mark.parametrize("status, expected_state", [(200, "ok"), (401, "invalid"), (403, "invalid"), (500, "error")])
 def test_typesafe_key_validation(status: int, expected_state: str) -> None:
     response = Mock(status_code=status)
@@ -33,7 +40,7 @@ def test_typesafe_key_validation(status: int, expected_state: str) -> None:
         "answers": {"verdict": {"type": "noul", "noul": 0.9}, "applicable": {"type": "noul", "noul": 0.9}},
         "usage": {"input_tokens": 12, "output_tokens": 0},
     }
-    with patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response) as request:
+    with patch("requests.Session.request", return_value=response) as request:
         state, message = Client.validate_key("typesafe", "test-typesafe-key")
 
     assert state == expected_state
@@ -42,135 +49,6 @@ def test_typesafe_key_validation(status: int, expected_state: str) -> None:
     assert request.call_args.kwargs["headers"]["Authorization"] == "Bearer test-typesafe-key"
 
 
-def test_typesafe_mixed_question_request() -> None:
-    response = Mock(status_code=200)
-    response.json.return_value = {
-        "model": "jev-1.13.0",
-        "answers": {
-            "verdict": {"type": "noul", "noul": 0.8},
-            "quality": {
-                "type": "score",
-                "score": 1.25,
-                "legend": {"0": "Poor", "1": "Fair", "2": "Good"},
-                "probabilities": {"0": 0.1, "1": 0.55, "2": 0.35},
-                "confidence": 0.6,
-            },
-            "language": {"type": "choice", "choice": "en", "probabilities": {"en": 0.7, "es": 0.3}, "confidence": 0.4},
-        },
-        "usage": {"input_tokens": 120, "output_tokens": 10},
-    }
-    with patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response) as request:
-        result = SystemOneClient.evaluate(
-            api_key="test-typesafe-key",
-            model="jev-1.13.0",
-            state={"text": "Hello!"},
-            questions={
-                "verdict": NoulQuestion(instructions="Is the response polite?"),
-                "quality": ScoreQuestion(instructions="Assess quality", criteria=["Poor", "Fair", "Good"]),
-                "language": ChoiceQuestion(
-                    instructions={"question": "Which language?"}, criteria={"en": None, "es": None}
-                ),
-            },
-        )
-
-    assert isinstance(result.answers["verdict"], NoulAnswer)
-    assert result.answers["verdict"].noul == 0.8
-    assert isinstance(result.answers["quality"], ScoreAnswer)
-    assert result.answers["quality"].score == 1.25
-    assert isinstance(result.answers["language"], ChoiceAnswer)
-    assert result.answers["language"].choice == "en"
-    body = request.call_args.kwargs["json"]
-    assert body["state"] == {"text": "Hello!"}
-    assert body["questions"]["verdict"] == {"type": "noul", "instructions": "Is the response polite?"}
-    assert body["questions"]["quality"] == {
-        "type": "score",
-        "instructions": "Assess quality",
-        "criteria": ["Poor", "Fair", "Good"],
-    }
-    assert body["questions"]["language"] == {
-        "type": "choice",
-        "instructions": {"question": "Which language?"},
-        "criteria": {"en": None, "es": None},
-    }
-    assert request.call_args.args[1] == "https://api.typesafe.ai/v1/systemone"
-
-
-@pytest.mark.parametrize(
-    "question,answer",
-    [
-        (ScoreQuestion(criteria=["Poor", "Good"]), {"type": "noul", "noul": 0.8}),
-        *[
-            (
-                ScoreQuestion(criteria=["Poor", "Good"]),
-                {
-                    "type": "score",
-                    "score": 0.75,
-                    "confidence": 0.5,
-                    "legend": {"0": "Poor", "1": "Good"},
-                    "probabilities": {"0": 0.25, "1": 0.75},
-                    field: value,
-                },
-            )
-            for field, value in [
-                ("score", 1.1),
-                ("score", True),
-                ("score", float("nan")),
-                ("legend", {"0": "Poor"}),
-                ("probabilities", {"0": 0.25, "2": 0.75}),
-                ("probabilities", {"0": 0.25, "1": 0.25}),
-            ]
-        ],
-        (
-            ChoiceQuestion(criteria={"en": None, "es": None}),
-            {"type": "choice", "choice": "fr", "confidence": 0.5, "probabilities": {"en": 0.25, "es": 0.75}},
-        ),
-    ],
-)
-def test_rejects_answers_that_do_not_match_the_requested_scale(
-    question: SystemOneQuestion, answer: dict[str, object]
-) -> None:
-    response = Mock(status_code=200)
-    response.json.return_value = {
-        "model": "custom-model",
-        "answers": {"result": answer},
-        "usage": {"input_tokens": 12, "output_tokens": 2},
-    }
-    with (
-        patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response),
-        pytest.raises(StructuredOutputParseError),
-    ):
-        SystemOneClient.evaluate(
-            api_key="example-token", model="custom-model", state="Hello!", questions={"result": question}
-        )
-
-
-def test_choice_accepts_a_rounded_distribution_with_many_options() -> None:
-    criteria = {str(index): None for index in range(255)}
-    response = Mock(status_code=200)
-    response.json.return_value = {
-        "model": "custom-model",
-        "answers": {
-            "result": {
-                "type": "choice",
-                "choice": "0",
-                "confidence": 0,
-                "probabilities": dict.fromkeys(criteria, 0.0039),
-            }
-        },
-        "usage": {"input_tokens": 12, "output_tokens": 0},
-    }
-    with patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response):
-        result = SystemOneClient.evaluate(
-            api_key="",
-            base_url="https://decisions.example.com/v1",
-            model="custom-model",
-            state="Hello!",
-            questions={"result": ChoiceQuestion(criteria=criteria)},
-        )
-    assert isinstance(result.answers["result"], ChoiceAnswer)
-    assert result.answers["result"].choice == "0"
-
-
 @pytest.mark.parametrize("probability", [-0.1, 1.1, float("nan"), float("inf"), "0.8", True, None])
 def test_typesafe_rejects_invalid_probabilities(probability: object) -> None:
     response = Mock(status_code=200)
@@ -180,7 +58,7 @@ def test_typesafe_rejects_invalid_probabilities(probability: object) -> None:
         "usage": {"input_tokens": 120, "output_tokens": 10},
     }
     with (
-        patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response),
+        patch("requests.Session.request", return_value=response),
         pytest.raises(StructuredOutputParseError),
     ):
         SystemOneClient.evaluate(
@@ -195,7 +73,7 @@ def test_typesafe_rejects_invalid_probabilities(probability: object) -> None:
 def test_typesafe_rate_limits_are_retryable(status: int) -> None:
     response = Mock(status_code=status, headers={"Retry-After": "15"})
     with (
-        patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response),
+        patch("requests.Session.request", return_value=response),
         pytest.raises(SystemOneRateLimitError) as error,
     ):
         SystemOneClient.evaluate(
@@ -207,9 +85,28 @@ def test_typesafe_rate_limits_are_retryable(status: int) -> None:
     assert error.value.retry_after == 15
 
 
+@pytest.mark.parametrize(
+    "usage", [{}, {"input_tokens": -1, "output_tokens": 0}, {"input_tokens": 1, "output_tokens": True}]
+)
+def test_invalid_usage_does_not_become_a_billable_evaluation(usage: dict[str, object]) -> None:
+    response = Mock(status_code=200)
+    response.json.return_value = {
+        "model": "jev-1.13.0",
+        "answers": {"verdict": {"type": "noul", "noul": 0.9}},
+        "usage": usage,
+    }
+    with patch("requests.Session.request", return_value=response), pytest.raises(StructuredOutputParseError):
+        SystemOneClient.evaluate(
+            api_key="example-token",
+            model="jev-1.13.0",
+            state="Hello!",
+            questions={"verdict": NoulQuestion(instructions="Polite?")},
+        )
+
+
 def test_typesafe_requires_a_key() -> None:
     with (
-        patch("products.ai_observability.backend.llm.system_one.pinned_request") as request,
+        patch("requests.Session.request") as request,
         pytest.raises(AuthenticationError),
     ):
         SystemOneClient.evaluate(
@@ -227,7 +124,7 @@ def test_typesafe_requires_every_requested_answer(answers: dict[str, object]) ->
         "usage": {"input_tokens": 12, "output_tokens": 2},
     }
     with (
-        patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response),
+        patch("requests.Session.request", return_value=response),
         pytest.raises(StructuredOutputParseError),
     ):
         SystemOneClient.evaluate(
@@ -255,7 +152,7 @@ def test_typesafe_requires_every_requested_answer(answers: dict[str, object]) ->
 def test_typesafe_preserves_error_categories(status: int, message: str, error_type: type[Exception]) -> None:
     response = Mock(status_code=status, text=message)
     with (
-        patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response),
+        patch("requests.Session.request", return_value=response),
         pytest.raises(error_type),
     ):
         SystemOneClient.evaluate(
@@ -275,7 +172,7 @@ def test_custom_endpoint_and_model(api_key: str) -> None:
         "usage": {"input_tokens": 15, "output_tokens": 0},
         "latency_ms": 42,
     }
-    with patch("products.ai_observability.backend.llm.system_one.pinned_request", return_value=response) as request:
+    with patch("requests.Session.request", return_value=response) as request:
         result = SystemOneClient.evaluate(
             api_key=api_key,
             base_url="https://decisions.example.com/v1/",
@@ -287,12 +184,13 @@ def test_custom_endpoint_and_model(api_key: str) -> None:
             },
         )
     assert request.call_args.args == ("POST", "https://decisions.example.com/v1/systemone")
-    assert request.call_args.kwargs["headers"] == ({"Authorization": f"Bearer {api_key}"} if api_key else {})
+    assert request.call_args.kwargs["headers"].get("Authorization") == (f"Bearer {api_key}" if api_key else None)
+    assert request.call_args.kwargs["allow_redirects"] is False
     assert request.call_args.kwargs["json"]["model"] == "custom-model"
     assert result.model == "custom-model-revision"
-    assert result.usage.output_tokens == 0
+    assert result.output_tokens == 0
     assert isinstance(result.answers["verdict"], NoulAnswer)
-    assert result.answers["verdict"].noul == 0.7
+    assert result.answers["verdict"].probability == 0.7
 
 
 @pytest.mark.parametrize(
@@ -305,14 +203,18 @@ def test_custom_endpoint_and_model(api_key: str) -> None:
     ],
 )
 def test_invalid_endpoint_is_rejected_before_sending_credentials(base_url: str) -> None:
-    with patch("products.ai_observability.backend.llm.system_one.pinned_request") as request:
+    with patch("requests.Session.request") as request:
         state, _ = SystemOneClient.validate_key("example-token", base_url=base_url)
     assert state == "error"
     request.assert_not_called()
 
 
 def test_private_endpoint_is_blocked() -> None:
-    with override_settings(DEBUG=False, TEST=False), patch("requests.Session.request") as request:
+    with (
+        override_settings(DEBUG=False, TEST=False),
+        patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("127.0.0.1")}),
+        patch("requests.Session.request") as request,
+    ):
         state, _ = SystemOneClient.validate_key("example-token", base_url="https://127.0.0.1/v1")
     assert state == "error"
     request.assert_not_called()

From 2bcc0f83f9b480a2b6711bd51c28fe2c55a55a3a Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Fri, 25 Sep 2026 15:24:50 +0200
Subject: [PATCH 11/30] fix(aio): gate system one evaluation experiments

---
 .../internal/ai-observability-judge-inputs.md |  5 ++
 frontend/src/lib/constants.tsx                |  1 +
 posthog/egress/typesafe/README.md             |  4 +-
 .../ai_observability/evaluation_llm_judge.py  |  3 +
 .../ai_observability/test_run_evaluation.py   | 32 +++++++++
 .../backend/api/provider_keys.py              | 20 +++---
 .../backend/api/test/test_provider_keys.py    | 69 +++++++++++++++----
 .../backend/llm/system_one.py                 | 20 ++++++
 .../backend/llm/test/test_system_one.py       | 19 +++++
 .../settings/LLMProviderKeysSettings.tsx      | 10 ++-
 .../SystemOneConnectionFields.stories.tsx     |  8 ++-
 11 files changed, 164 insertions(+), 27 deletions(-)

diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index 7f50ca2c3ab2..e3344f7a9375 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -42,6 +42,11 @@ Implementation: [trace judge](../../posthog/temporal/ai_observability/run_trace_
 ## System One judges
 
 System One-compatible models are available under the existing LLM judge option.
+The experiment requires the `llm-analytics-system-one-evaluations` feature flag and an organization in `POSTHOG_INTERNAL_ORG_IDS`.
+Configure the flag as a project-group flag targeting only PostHog staff projects so the browser and background workers evaluate the same rollout.
+Connection validation and every evaluation check this gate; an absent flag, failed flag lookup, or customer project blocks the call.
+Turning the flag off stops subsequent runs, including queued work, without disabling the saved evaluation.
+Use only synthetic or PostHog-owned data, following the [TypeSafe usage policy](../../posthog/egress/typesafe/README.md#usage-policy).
 Add a connection under **System One (Jev)** in provider key settings.
 The default endpoint is TypeSafe's `https://api.typesafe.ai/v1`, with model `jev-1.13.0` and a TypeSafe API key.
 Advanced configuration accepts a different public HTTPS base URL and model ID for compatible services.
diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx
index cf2016b151d2..2ddb9a8ce164 100644
--- a/frontend/src/lib/constants.tsx
+++ b/frontend/src/lib/constants.tsx
@@ -363,6 +363,7 @@ export const FEATURE_FLAGS = {
     LLM_ANALYTICS_EVAL_SETTLING_STRATEGY: 'llm-analytics-eval-settling-strategy', // owner: #team-ai-observability
     LLM_ANALYTICS_NUMERIC_EVALS: 'llm-analytics-numeric-evaluations', // owner: #team-ai-observability
     LLM_ANALYTICS_OFFLINE_EVALS: 'llm-analytics-offline-evals', // owner: #team-ai-observability
+    LLM_ANALYTICS_SYSTEM_ONE_EVALUATIONS: 'llm-analytics-system-one-evaluations', // owner: #team-ai-observability
     LLM_ANALYTICS_TAGS: 'llm-analytics-tags', // owner: #team-ai-observability
     LLM_ANALYTICS_TRACE_NAVIGATION: 'llm-analytics-trace-navigation', // owner: #team-ai-observability
     LLM_OBSERVABILITY_TRACE_SEARCH: 'llm-observability-trace-search', // owner: #team-ai-observability
diff --git a/posthog/egress/typesafe/README.md b/posthog/egress/typesafe/README.md
index c264c35559c7..9194e78493ee 100644
--- a/posthog/egress/typesafe/README.md
+++ b/posthog/egress/typesafe/README.md
@@ -50,7 +50,9 @@ The default reserve ladder applies, and `typesafe_request` defaults to `NORMAL`.
 `typesafe_request` rejects `CRITICAL`, because a `CRITICAL` call is never shed and would skip the hourly spend ceiling.
 Give every caller an explicit lane: `NORMAL` when a person waits for the answer, `BATCH` for background work.
 AI observability uses `llma_evaluations` as its source, `BATCH` for evaluation runs, and `NORMAL` for synthetic connection validation.
-The evaluation caller must meet the usage policy above before launch.
+The evaluation caller checks `llm-analytics-system-one-evaluations` and `POSTHOG_INTERNAL_ORG_IDS` before connection validation and every evaluation.
+The project-group flag stays off until it targets PostHog staff projects; customer projects are blocked regardless of the flag.
+Customer-data launch still requires the opt-in and sign-off above.
 
 ## Rate-limit headers
 
diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.py
index d92e5899d11a..488343b4055e 100644
--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py
+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py
@@ -53,6 +53,7 @@
     SystemOneClient,
     SystemOneRateLimitError,
     SystemOneRequestRejectedError,
+    system_one_evaluations_enabled,
 )
 from products.ai_observability.backend.llm.types import CompletionResponse
 from products.ai_observability.backend.models.evaluation_configs import NumericOutputConfig, NumericScoreOutOfBounds
@@ -447,6 +448,8 @@ def call_llm_judge(
     provider = resolved.provider
     if provider == "typesafe" and output_type != "boolean":
         raise ApplicationError("This System One evaluation output type is not supported.", non_retryable=True)
+    if provider == "typesafe" and not system_one_evaluations_enabled(team_id):
+        raise ApplicationError("System One evaluations are not available for this project.", non_retryable=True)
     model = resolved.model
     provider_key = resolved.provider_key
     is_byok = resolved.is_byok
diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index b1f0efce999d..e820888773e7 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -137,6 +137,9 @@ def test_typesafe_judge_emits_boolean_probability_without_reasoning(
         patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),
         patch("posthog.egress.limiter.backends.LimitsBackend.consume_sync", return_value=True),
         patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
+        patch(
+            "posthog.temporal.ai_observability.evaluation_llm_judge.system_one_evaluations_enabled", return_value=True
+        ),
         patch("requests.Session.request", return_value=response) as request,
     ):
         spec.return_value.resolve.return_value = resolved
@@ -166,6 +169,9 @@ def test_system_one_numeric_mapping_is_not_enabled() -> None:
         patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),
         patch("posthog.egress.limiter.backends.LimitsBackend.consume_sync", return_value=True),
         patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
+        patch(
+            "posthog.temporal.ai_observability.evaluation_llm_judge.system_one_evaluations_enabled", return_value=True
+        ),
         patch("requests.Session.request") as request,
         pytest.raises(ApplicationError) as error,
     ):
@@ -180,6 +186,26 @@ def test_system_one_numeric_mapping_is_not_enabled() -> None:
     request.assert_not_called()
 
 
+def test_system_one_disabled_experiment_does_not_send_evaluation_data() -> None:
+    with (
+        patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
+        patch(
+            "posthog.temporal.ai_observability.evaluation_llm_judge.system_one_evaluations_enabled", return_value=False
+        ),
+        patch("requests.Session.request") as request,
+        pytest.raises(ApplicationError) as error,
+    ):
+        spec.return_value.resolve.return_value = MagicMock(provider="typesafe")
+        call_llm_judge(
+            evaluation={"team_id": 1, "evaluation_config": {"prompt": "Is this a greeting?"}},
+            system_prompt="",
+            user_prompt="Hello!",
+            allows_na=False,
+        )
+    assert error.value.non_retryable
+    request.assert_not_called()
+
+
 @pytest.mark.parametrize("status", [301, 400, 422])
 def test_system_one_rejected_requests_disable_without_model_cost_attribution(status: int) -> None:
     key = MagicMock(provider="typesafe", encrypted_config={"api_key": "example-token"})
@@ -187,6 +213,9 @@ def test_system_one_rejected_requests_disable_without_model_cost_attribution(sta
         patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),
         patch("posthog.egress.limiter.backends.LimitsBackend.consume_sync", return_value=True),
         patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
+        patch(
+            "posthog.temporal.ai_observability.evaluation_llm_judge.system_one_evaluations_enabled", return_value=True
+        ),
         patch(
             "requests.Session.request",
             return_value=MagicMock(status_code=status, text="Invalid request"),
@@ -216,6 +245,9 @@ def test_typesafe_rate_limit_retries_without_disabling_the_evaluation(budget_gra
         patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),
         patch("posthog.egress.limiter.backends.LimitsBackend.consume_sync", return_value=budget_granted),
         patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
+        patch(
+            "posthog.temporal.ai_observability.evaluation_llm_judge.system_one_evaluations_enabled", return_value=True
+        ),
         patch(
             "requests.Session.request",
             return_value=MagicMock(status_code=429, headers={"Retry-After": "15"}),
diff --git a/products/ai_observability/backend/api/provider_keys.py b/products/ai_observability/backend/api/provider_keys.py
index 85b5fcb3c722..ed39010827b8 100644
--- a/products/ai_observability/backend/api/provider_keys.py
+++ b/products/ai_observability/backend/api/provider_keys.py
@@ -30,7 +30,7 @@
     error_field_for_validation_message,
     is_allowed_azure_endpoint,
 )
-from ..llm.system_one import SystemOneClient
+from ..llm.system_one import SystemOneClient, system_one_evaluations_enabled
 from ..models.evaluation_config import EvaluationConfig
 from ..models.evaluations import Evaluation
 from ..models.model_configuration import LLMModelConfiguration
@@ -69,8 +69,10 @@ def _reload_model_config_dependents_on_commit(team_id: int, model_config_ids: li
         transaction.on_commit(lambda: reload_taggers_on_workers(team_id=team_id, tagger_ids=tagger_ids))
 
 
-def validate_provider_key(provider: str, api_key: str, **kwargs) -> tuple[str, str | None]:
+def validate_provider_key(provider: str, api_key: str, *, team_id: int, **kwargs: str) -> tuple[str, str | None]:
     """Validate an API key for any supported provider using the unified client."""
+    if provider == LLMProvider.TYPESAFE and not system_one_evaluations_enabled(team_id):
+        raise exceptions.PermissionDenied("System One evaluations are not available for this project.")
     try:
         return Client.validate_key(provider, api_key, **kwargs)
     except Exception:
@@ -274,7 +276,7 @@ def create(self, validated_data):
 
         if provider == LLMProvider.TYPESAFE:
             connection_config = self._system_one_config(validated_data)
-            state, error_message = validate_provider_key(provider, api_key or "", **connection_config)
+            state, error_message = validate_provider_key(provider, api_key or "", team_id=team.id, **connection_config)
             if state != LLMProviderKey.State.OK:
                 raise serializers.ValidationError({"api_key": error_message})
             validated_data["encrypted_config"] = {"api_key": api_key or "", **connection_config}
@@ -283,7 +285,7 @@ def create(self, validated_data):
             return super().create(validated_data)
 
         if api_key:
-            state, error_message = validate_provider_key(provider, api_key, **azure_kwargs)
+            state, error_message = validate_provider_key(provider, api_key, team_id=team.id, **azure_kwargs)
             if state != LLMProviderKey.State.OK:
                 error_field = _validation_error_field(provider, error_message)
                 raise serializers.ValidationError({error_field: error_message or "Key validation failed"})
@@ -305,7 +307,7 @@ def update(self, instance, validated_data):
             if any(field in validated_data for field in ("api_key", "base_url", "system_one_model")):
                 config = self._system_one_config(validated_data, instance.encrypted_config)
                 config["api_key"] = validated_data.pop("api_key", config.get("api_key", ""))
-                state, error_message = validate_provider_key(instance.provider, **config)
+                state, error_message = validate_provider_key(instance.provider, team_id=instance.team_id, **config)
                 if state != LLMProviderKey.State.OK:
                     raise serializers.ValidationError({"api_key": error_message})
                 instance.encrypted_config = config
@@ -319,7 +321,9 @@ def update(self, instance, validated_data):
             # Fall back to existing config for Azure fields not provided in the update.
             extra_kwargs = {**instance.provider_extra_kwargs(), **azure_kwargs}
 
-            state, error_message = validate_provider_key(instance.provider, api_key, **extra_kwargs)
+            state, error_message = validate_provider_key(
+                instance.provider, api_key, team_id=instance.team_id, **extra_kwargs
+            )
             if state != LLMProviderKey.State.OK:
                 error_field = _validation_error_field(instance.provider, error_message)
                 raise serializers.ValidationError({error_field: error_message or "Key validation failed"})
@@ -443,7 +447,7 @@ def validate(self, request: Request, **_kwargs) -> Response:
             )
 
         state, error_message = validate_provider_key(
-            instance.provider, api_key or "", **instance.provider_extra_kwargs()
+            instance.provider, api_key or "", team_id=self.team_id, **instance.provider_extra_kwargs()
         )
         instance.state = state
         instance.error_message = error_message
@@ -637,7 +641,7 @@ def create(self, request: Request, **_kwargs) -> Response:
             if api_version:
                 extra_kwargs["api_version"] = api_version
 
-        state, error_message = validate_provider_key(provider, api_key, **extra_kwargs)
+        state, error_message = validate_provider_key(provider, api_key, team_id=self.team_id, **extra_kwargs)
         error_field = (
             error_field_for_validation_message(error_message) if provider == LLMProvider.AZURE_OPENAI else None
         )
diff --git a/products/ai_observability/backend/api/test/test_provider_keys.py b/products/ai_observability/backend/api/test/test_provider_keys.py
index 08caff087ee2..e6baf258e782 100644
--- a/products/ai_observability/backend/api/test/test_provider_keys.py
+++ b/products/ai_observability/backend/api/test/test_provider_keys.py
@@ -122,6 +122,39 @@ def setUp(self):
         self.organization_membership.level = OrganizationMembership.Level.ADMIN
         self.organization_membership.save()
 
+    @parameterized.expand([("create",), ("update",), ("validate",), ("prevalidate",)])
+    def test_system_one_customer_project_cannot_make_requests(self, operation: str) -> None:
+        key = LLMProviderKey.objects.create(
+            team=self.team,
+            provider="typesafe",
+            name="Example connection",
+            encrypted_config={"api_key": "example-token"},
+            created_by=self.user,
+        )
+        base_url = f"/api/environments/{self.team.id}/llm_analytics/"
+        with (
+            self.settings(POSTHOG_INTERNAL_ORG_IDS=[]),
+            patch("products.ai_observability.backend.llm.system_one.get_feature_flag_or_none", return_value=True),
+            patch("requests.Session.request") as request,
+        ):
+            if operation == "create":
+                response = self.client.post(
+                    f"{base_url}provider_keys/",
+                    {"provider": "typesafe", "name": "Example connection", "api_key": "example-token"},
+                )
+            elif operation == "update":
+                response = self.client.patch(f"{base_url}provider_keys/{key.id}/", {"api_key": "new-example-token"})
+            elif operation == "validate":
+                response = self.client.post(f"{base_url}provider_keys/{key.id}/validate/")
+            else:
+                response = self.client.post(
+                    f"{base_url}provider_key_validations/", {"provider": "typesafe", "api_key": "example-token"}
+                )
+        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN, response.data)
+        request.assert_not_called()
+        key.refresh_from_db()
+        self.assertEqual(key.encrypted_config["api_key"], "example-token")
+
     def test_unauthenticated_user_cannot_access_provider_keys(self):
         self.client.logout()
         response = self.client.get(f"/api/environments/{self.team.id}/llm_analytics/provider_keys/")
@@ -152,12 +185,15 @@ def test_can_create_provider_key(self, provider: str, mock_validate: Mock) -> No
         expected_config = (
             {"base_url": SystemOneClient.BASE_URL, "model": SystemOneClient.MODEL} if provider == "typesafe" else {}
         )
-        mock_validate.assert_called_once_with(provider, "sk-test-key-12345", **expected_config)
+        mock_validate.assert_called_once_with(provider, "sk-test-key-12345", team_id=self.team.id, **expected_config)
 
     @patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")})
     @patch("posthog.egress.limiter.backends.LimitsBackend.consume_sync", return_value=True)
     @patch("requests.Session.request")
-    def test_custom_system_one_connection_round_trip(self, request: Mock, _budget: Mock, _dns: Mock) -> None:
+    @patch("products.ai_observability.backend.api.provider_keys.system_one_evaluations_enabled", return_value=True)
+    def test_custom_system_one_connection_round_trip(
+        self, _enabled: Mock, request: Mock, _budget: Mock, _dns: Mock
+    ) -> None:
         request.return_value = Mock(status_code=200)
         request.return_value.json.return_value = {
             "model": "custom-model",
@@ -357,7 +393,7 @@ def test_can_update_provider_key_api_key(self, mock_validate):
 
         key.refresh_from_db()
         self.assertEqual(key.encrypted_config["api_key"], "sk-new-key-12345")
-        mock_validate.assert_called_once_with("openai", "sk-new-key-12345")
+        mock_validate.assert_called_once_with("openai", "sk-new-key-12345", team_id=self.team.id)
 
     @patch("products.ai_observability.backend.api.provider_keys.validate_provider_key")
     def test_can_update_fireworks_provider_key_api_key(self, mock_validate):
@@ -380,7 +416,7 @@ def test_can_update_fireworks_provider_key_api_key(self, mock_validate):
 
         key.refresh_from_db()
         self.assertEqual(key.encrypted_config["api_key"], "fw-new-key-12345")
-        mock_validate.assert_called_once_with("fireworks", "fw-new-key-12345")
+        mock_validate.assert_called_once_with("fireworks", "fw-new-key-12345", team_id=self.team.id)
 
     def test_can_delete_provider_key(self):
         key = LLMProviderKey.objects.create(
@@ -537,7 +573,7 @@ def test_can_create_openrouter_provider_key(self, mock_validate):
         assert key is not None
         self.assertEqual(key.provider, "openrouter")
         self.assertEqual(key.state, LLMProviderKey.State.OK)
-        mock_validate.assert_called_once_with("openrouter", "sk-or-v1-test-key-12345")
+        mock_validate.assert_called_once_with("openrouter", "sk-or-v1-test-key-12345", team_id=self.team.id)
 
     @patch("products.ai_observability.backend.api.provider_keys.validate_provider_key")
     def test_openrouter_key_accepts_any_format(self, mock_validate):
@@ -563,7 +599,7 @@ def test_can_create_fireworks_provider_key(self, mock_validate):
         assert key is not None
         self.assertEqual(key.provider, "fireworks")
         self.assertEqual(key.state, LLMProviderKey.State.OK)
-        mock_validate.assert_called_once_with("fireworks", "fw-test-key-12345")
+        mock_validate.assert_called_once_with("fireworks", "fw-test-key-12345", team_id=self.team.id)
 
     @patch("products.ai_observability.backend.api.provider_keys.validate_provider_key")
     def test_fireworks_key_accepts_any_format(self, mock_validate):
@@ -589,7 +625,7 @@ def test_can_create_together_provider_key(self, mock_validate):
         assert key is not None
         self.assertEqual(key.provider, "together_ai")
         self.assertEqual(key.state, LLMProviderKey.State.OK)
-        mock_validate.assert_called_once_with("together_ai", "together-test-key-12345")
+        mock_validate.assert_called_once_with("together_ai", "together-test-key-12345", team_id=self.team.id)
 
     @patch("products.ai_observability.backend.api.provider_keys.validate_provider_key")
     def test_together_key_accepts_any_format(self, mock_validate):
@@ -615,7 +651,7 @@ def test_can_create_minimax_provider_key(self, mock_validate):
         assert key is not None
         self.assertEqual(key.provider, "minimax")
         self.assertEqual(key.state, LLMProviderKey.State.OK)
-        mock_validate.assert_called_once_with("minimax", "minimax-test-key-12345")
+        mock_validate.assert_called_once_with("minimax", "minimax-test-key-12345", team_id=self.team.id)
 
     @patch("products.ai_observability.backend.api.provider_keys.validate_provider_key")
     def test_minimax_key_accepts_any_format(self, mock_validate):
@@ -641,7 +677,7 @@ def test_can_create_zeabur_provider_key(self, mock_validate):
         assert key is not None
         self.assertEqual(key.provider, "zeabur")
         self.assertEqual(key.state, LLMProviderKey.State.OK)
-        mock_validate.assert_called_once_with("zeabur", "sk-zeabur-test-key-12345")
+        mock_validate.assert_called_once_with("zeabur", "sk-zeabur-test-key-12345", team_id=self.team.id)
 
     @patch("products.ai_observability.backend.api.provider_keys.validate_provider_key")
     def test_zeabur_key_rejects_invalid_format(self, mock_validate):
@@ -680,6 +716,7 @@ def test_can_create_azure_openai_provider_key(self, mock_validate):
         mock_validate.assert_called_once_with(
             "azure_openai",
             "azure-hex-123",
+            team_id=self.team.id,
             azure_endpoint="https://contoso.openai.azure.com/",
             api_version="2024-10-21",
         )
@@ -757,6 +794,7 @@ def test_validate_azure_key_reuses_encrypted_config(self, mock_validate):
         mock_validate.assert_called_once_with(
             "azure_openai",
             "azure-hex-123",
+            team_id=self.team.id,
             azure_endpoint="https://contoso.openai.azure.com/",
             api_version="2024-10-21",
         )
@@ -814,6 +852,7 @@ def test_create_azure_openai_persists_default_api_version_when_omitted(self, moc
         mock_validate.assert_called_once_with(
             "azure_openai",
             "azure-hex-123",
+            team_id=self.team.id,
             azure_endpoint="https://contoso.openai.azure.com/",
             api_version=DEFAULT_API_VERSION,
         )
@@ -955,7 +994,7 @@ def test_can_pre_validate_api_key(self, mock_validate):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(response.data["state"], "ok")
         self.assertIsNone(response.data["error_message"])
-        mock_validate.assert_called_once_with("openai", "sk-test-key")
+        mock_validate.assert_called_once_with("openai", "sk-test-key", team_id=self.team.id)
 
     @patch("products.ai_observability.backend.api.provider_keys.validate_provider_key")
     def test_pre_validate_returns_error_state(self, mock_validate):
@@ -979,7 +1018,7 @@ def test_can_pre_validate_openrouter_key(self, mock_validate):
         )
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(response.data["state"], "ok")
-        mock_validate.assert_called_once_with("openrouter", "sk-or-v1-test-key")
+        mock_validate.assert_called_once_with("openrouter", "sk-or-v1-test-key", team_id=self.team.id)
 
     @patch("products.ai_observability.backend.api.provider_keys.validate_provider_key")
     def test_can_pre_validate_fireworks_key(self, mock_validate):
@@ -991,7 +1030,7 @@ def test_can_pre_validate_fireworks_key(self, mock_validate):
         )
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(response.data["state"], "ok")
-        mock_validate.assert_called_once_with("fireworks", "fw-test-key")
+        mock_validate.assert_called_once_with("fireworks", "fw-test-key", team_id=self.team.id)
 
     @patch("products.ai_observability.backend.api.provider_keys.validate_provider_key")
     def test_can_pre_validate_together_key(self, mock_validate):
@@ -1003,7 +1042,7 @@ def test_can_pre_validate_together_key(self, mock_validate):
         )
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(response.data["state"], "ok")
-        mock_validate.assert_called_once_with("together_ai", "together-test-key")
+        mock_validate.assert_called_once_with("together_ai", "together-test-key", team_id=self.team.id)
 
     @patch("products.ai_observability.backend.api.provider_keys.validate_provider_key")
     def test_can_pre_validate_minimax_key(self, mock_validate):
@@ -1015,7 +1054,7 @@ def test_can_pre_validate_minimax_key(self, mock_validate):
         )
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(response.data["state"], "ok")
-        mock_validate.assert_called_once_with("minimax", "minimax-test-key")
+        mock_validate.assert_called_once_with("minimax", "minimax-test-key", team_id=self.team.id)
 
     @patch("products.ai_observability.backend.api.provider_keys.validate_provider_key")
     def test_can_pre_validate_zeabur_key(self, mock_validate):
@@ -1027,7 +1066,7 @@ def test_can_pre_validate_zeabur_key(self, mock_validate):
         )
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(response.data["state"], "ok")
-        mock_validate.assert_called_once_with("zeabur", "sk-zeabur-test-key")
+        mock_validate.assert_called_once_with("zeabur", "sk-zeabur-test-key", team_id=self.team.id)
 
     def test_pre_validate_requires_api_key(self):
         response = self.client.post(
diff --git a/products/ai_observability/backend/llm/system_one.py b/products/ai_observability/backend/llm/system_one.py
index bfb22260fe5f..ac08fc3eae20 100644
--- a/products/ai_observability/backend/llm/system_one.py
+++ b/products/ai_observability/backend/llm/system_one.py
@@ -4,6 +4,8 @@
 from email.utils import parsedate_to_datetime
 from urllib.parse import urlsplit
 
+from django.conf import settings
+
 import requests
 
 from posthog.egress.limiter.policies import Priority
@@ -18,6 +20,8 @@
     system_one,
 )
 from posthog.egress.typesafe.transport import TypeSafeEgressBudgetExhausted
+from posthog.models import Team
+from posthog.ph_client import get_feature_flag_or_none
 from posthog.security.pinned_requests import SSRFBlockedError, pinned_session
 from posthog.security.url_validation import has_authority_bypass_chars
 
@@ -34,6 +38,22 @@
 )
 
 
+def system_one_evaluations_enabled(team_id: int) -> bool:
+    team = Team.objects.only("uuid", "organization_id").get(id=team_id)
+    # Staff access to a customer project does not make its data eligible for this experiment.
+    if str(team.organization_id) not in settings.POSTHOG_INTERNAL_ORG_IDS:
+        return False
+    return (
+        get_feature_flag_or_none(
+            "llm-analytics-system-one-evaluations",
+            str(team.uuid),
+            groups={"organization": str(team.organization_id), "project": str(team.id)},
+            send_feature_flag_events=False,
+        )
+        is True
+    )
+
+
 class SystemOneRequestRejectedError(LLMError):
     pass
 
diff --git a/products/ai_observability/backend/llm/test/test_system_one.py b/products/ai_observability/backend/llm/test/test_system_one.py
index 0eb0449456e4..52f87b414c48 100644
--- a/products/ai_observability/backend/llm/test/test_system_one.py
+++ b/products/ai_observability/backend/llm/test/test_system_one.py
@@ -1,5 +1,6 @@
 from collections.abc import Iterator
 from ipaddress import ip_address
+from uuid import uuid4
 
 import pytest
 from unittest.mock import Mock, patch
@@ -7,6 +8,7 @@
 from django.test import override_settings
 
 from posthog.egress.typesafe.client import NoulAnswer, NoulQuestion
+from posthog.models import Team
 
 from products.ai_observability.backend.llm.client import Client
 from products.ai_observability.backend.llm.errors import (
@@ -20,6 +22,7 @@
     SystemOneClient,
     SystemOneRateLimitError,
     SystemOneRequestRejectedError,
+    system_one_evaluations_enabled,
 )
 
 
@@ -32,6 +35,22 @@ def isolated_egress_budget() -> Iterator[None]:
         yield
 
 
+@pytest.mark.parametrize(
+    "internal,flag,enabled", [(False, True, False), (True, False, False), (True, None, False), (True, True, True)]
+)
+def test_system_one_experiment_requires_internal_project_and_flag(
+    internal: bool, flag: bool | None, enabled: bool
+) -> None:
+    team = Team(id=1, organization_id=uuid4(), uuid=uuid4())
+    with (
+        override_settings(POSTHOG_INTERNAL_ORG_IDS=[str(team.organization_id)] if internal else []),
+        patch("products.ai_observability.backend.llm.system_one.Team.objects.only") as teams,
+        patch("products.ai_observability.backend.llm.system_one.get_feature_flag_or_none", return_value=flag),
+    ):
+        teams.return_value.get.return_value = team
+        assert system_one_evaluations_enabled(team.id) is enabled
+
+
 @pytest.mark.parametrize("status, expected_state", [(200, "ok"), (401, "invalid"), (403, "invalid"), (500, "error")])
 def test_typesafe_key_validation(status: int, expected_state: str) -> None:
     response = Mock(status_code=status)
diff --git a/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx b/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
index 8e04f248b65d..53e15f1d01df 100644
--- a/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
+++ b/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
@@ -15,9 +15,10 @@ import {
 } from '@posthog/lemon-ui'
 
 import { RestrictionScope, useRestrictedArea } from 'lib/components/RestrictedArea'
-import { TeamMembershipLevel } from 'lib/constants'
+import { FEATURE_FLAGS, TeamMembershipLevel } from 'lib/constants'
 import { IconKey } from 'lib/lemon-ui/icons'
 import { LemonTableColumns } from 'lib/lemon-ui/LemonTable'
+import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
 
 import { LLMProviderIcon, LLM_PROVIDER_SELECT_OPTIONS } from '../LLMProviderIcon'
 import {
@@ -165,6 +166,7 @@ function KeyValidationStatus({
 }
 
 function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }): JSX.Element {
+    const { featureFlags } = useValues(featureFlagLogic)
     const { systemOneBaseUrl, systemOneModel } = useValues(llmProviderKeysLogic)
     const { newKeyModalOpen, providerKeysLoading, preValidationResult, preValidationResultLoading, evaluationConfig } =
         useValues(llmProviderKeysLogic)
@@ -333,7 +335,11 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
                     <LemonSelect
                         value={provider}
                         onChange={handleProviderChange}
-                        options={LLM_PROVIDER_SELECT_OPTIONS}
+                        options={LLM_PROVIDER_SELECT_OPTIONS.filter(
+                            ({ value }) =>
+                                value !== 'typesafe' ||
+                                featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_SYSTEM_ONE_EVALUATIONS] === true
+                        )}
                         className="mt-1"
                         fullWidth
                     />
diff --git a/products/ai_observability/frontend/settings/SystemOneConnectionFields.stories.tsx b/products/ai_observability/frontend/settings/SystemOneConnectionFields.stories.tsx
index ba7483ff4310..04831a61e3cf 100644
--- a/products/ai_observability/frontend/settings/SystemOneConnectionFields.stories.tsx
+++ b/products/ai_observability/frontend/settings/SystemOneConnectionFields.stories.tsx
@@ -1,5 +1,7 @@
 import type { Meta, StoryObj } from '@storybook/react'
 
+import { FEATURE_FLAGS } from 'lib/constants'
+
 import { useStorybookMocks } from '~/mocks/browser'
 
 import { LLMProviderKeysSettings } from './LLMProviderKeysSettings'
@@ -25,4 +27,8 @@ export default meta
 type Story = StoryObj<typeof meta>
 
 export const Default: Story = {}
-export const ProviderSettings: Story = { render: () => <LLMProviderKeysSettings /> }
+export const ProviderSettings: Story = {
+    parameters: { featureFlags: [FEATURE_FLAGS.LLM_ANALYTICS_SYSTEM_ONE_EVALUATIONS] },
+    render: () => <LLMProviderKeysSettings />,
+}
+export const ProviderSettingsFlagOff: Story = { render: () => <LLMProviderKeysSettings /> }

From 322035a279f132fcd6e9ef84215513021a2361fb Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Fri, 25 Sep 2026 15:46:57 +0200
Subject: [PATCH 12/30] fix(aio): separate customer system one connections

---
 .../internal/ai-observability-judge-inputs.md | 10 ++++----
 posthog/egress/typesafe/README.md             |  6 +++--
 .../ai_observability/evaluation_llm_judge.py  | 19 +++++++++------
 .../ai_observability/test_run_evaluation.py   | 24 +++++++++++++++----
 .../backend/api/provider_keys.py              |  4 +++-
 .../backend/api/test/test_provider_keys.py    | 21 +++++++++++-----
 .../backend/llm/system_one.py                 | 13 +++++++---
 .../backend/llm/test/test_system_one.py       | 21 ++++++++++++----
 8 files changed, 86 insertions(+), 32 deletions(-)

diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index e3344f7a9375..cfd9976c422d 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -42,11 +42,13 @@ Implementation: [trace judge](../../posthog/temporal/ai_observability/run_trace_
 ## System One judges
 
 System One-compatible models are available under the existing LLM judge option.
-The experiment requires the `llm-analytics-system-one-evaluations` feature flag and an organization in `POSTHOG_INTERNAL_ORG_IDS`.
-Configure the flag as a project-group flag targeting only PostHog staff projects so the browser and background workers evaluate the same rollout.
-Connection validation and every evaluation check this gate; an absent flag, failed flag lookup, or customer project blocks the call.
+The `llm-analytics-system-one-evaluations` project-group feature flag controls access in the browser and background workers.
+Customers configure their own TypeSafe key or a compatible deployment with its own authentication.
+Evaluation connections never fall back to an instance TypeSafe key or gateway credential.
+PostHog's regional AI gateway endpoints additionally require an organization in `POSTHOG_INTERNAL_ORG_IDS`; customer projects cannot use those endpoints.
+Connection validation and every evaluation check these gates; an absent flag or failed flag lookup blocks the call.
 Turning the flag off stops subsequent runs, including queued work, without disabling the saved evaluation.
-Use only synthetic or PostHog-owned data, following the [TypeSafe usage policy](../../posthog/egress/typesafe/README.md#usage-policy).
+Keep the experimental flag limited to staff projects until customer rollout meets the [TypeSafe usage policy](../../posthog/egress/typesafe/README.md#usage-policy).
 Add a connection under **System One (Jev)** in provider key settings.
 The default endpoint is TypeSafe's `https://api.typesafe.ai/v1`, with model `jev-1.13.0` and a TypeSafe API key.
 Advanced configuration accepts a different public HTTPS base URL and model ID for compatible services.
diff --git a/posthog/egress/typesafe/README.md b/posthog/egress/typesafe/README.md
index 9194e78493ee..fae05f6956e8 100644
--- a/posthog/egress/typesafe/README.md
+++ b/posthog/egress/typesafe/README.md
@@ -50,8 +50,10 @@ The default reserve ladder applies, and `typesafe_request` defaults to `NORMAL`.
 `typesafe_request` rejects `CRITICAL`, because a `CRITICAL` call is never shed and would skip the hourly spend ceiling.
 Give every caller an explicit lane: `NORMAL` when a person waits for the answer, `BATCH` for background work.
 AI observability uses `llma_evaluations` as its source, `BATCH` for evaluation runs, and `NORMAL` for synthetic connection validation.
-The evaluation caller checks `llm-analytics-system-one-evaluations` and `POSTHOG_INTERNAL_ORG_IDS` before connection validation and every evaluation.
-The project-group flag stays off until it targets PostHog staff projects; customer projects are blocked regardless of the flag.
+The evaluation caller checks the project-group flag `llm-analytics-system-one-evaluations` before connection validation and every evaluation.
+Customer connections require their own TypeSafe credential or compatible deployment; the evaluation adapter never falls back to instance credentials.
+PostHog's regional AI gateway endpoints additionally require an organization in `POSTHOG_INTERNAL_ORG_IDS`.
+Keep the experimental flag limited to PostHog staff projects until customer rollout is approved.
 Customer-data launch still requires the opt-in and sign-off above.
 
 ## Rate-limit headers
diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.py
index 488343b4055e..416a931c051f 100644
--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py
+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py
@@ -446,15 +446,22 @@ def call_llm_judge(
         raise
 
     provider = resolved.provider
-    if provider == "typesafe" and output_type != "boolean":
-        raise ApplicationError("This System One evaluation output type is not supported.", non_retryable=True)
-    if provider == "typesafe" and not system_one_evaluations_enabled(team_id):
-        raise ApplicationError("System One evaluations are not available for this project.", non_retryable=True)
     model = resolved.model
     provider_key = resolved.provider_key
     is_byok = resolved.is_byok
     key_id = str(provider_key.id) if provider_key else None
 
+    if provider == "typesafe":
+        if output_type != "boolean":
+            raise ApplicationError("This System One evaluation output type is not supported.", non_retryable=True)
+        base_url = (
+            provider_key.encrypted_config.get("base_url", SystemOneClient.BASE_URL)
+            if provider_key
+            else SystemOneClient.BASE_URL
+        )
+        if not system_one_evaluations_enabled(team_id, base_url=base_url):
+            raise ApplicationError("System One evaluations are not available for this project.", non_retryable=True)
+
     type_config = get_output_type_config(allows_na, output_type=output_type, output_config=output_config)
     response_format = type_config.response_format
 
@@ -483,9 +490,7 @@ def call_llm_judge(
                 )
             system_one_result = SystemOneClient.evaluate(
                 api_key=provider_key.encrypted_config.get("api_key", "") if provider_key else "",
-                base_url=provider_key.encrypted_config.get("base_url", SystemOneClient.BASE_URL)
-                if provider_key
-                else SystemOneClient.BASE_URL,
+                base_url=base_url,
                 model=model,
                 state=user_prompt,
                 questions=questions,
diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index e820888773e7..d7ef0d15b4b6 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -7,6 +7,8 @@
 import pytest
 from unittest.mock import AsyncMock, MagicMock, patch
 
+from django.test import override_settings
+
 import posthoganalytics
 from asgiref.sync import async_to_sync, sync_to_async
 from parameterized import parameterized
@@ -186,16 +188,28 @@ def test_system_one_numeric_mapping_is_not_enabled() -> None:
     request.assert_not_called()
 
 
-def test_system_one_disabled_experiment_does_not_send_evaluation_data() -> None:
+@pytest.mark.parametrize(
+    "base_url,flag",
+    [("https://decisions.example.com/v1", False), ("https://ai-gateway.us.posthog.com/v1", True)],
+)
+def test_system_one_restricted_connection_does_not_send_evaluation_data(base_url: str, flag: bool) -> None:
     with (
+        override_settings(POSTHOG_INTERNAL_ORG_IDS=[]),
         patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
-        patch(
-            "posthog.temporal.ai_observability.evaluation_llm_judge.system_one_evaluations_enabled", return_value=False
-        ),
+        patch("products.ai_observability.backend.llm.system_one.Team.objects.only") as teams,
+        patch("products.ai_observability.backend.llm.system_one.get_feature_flag_or_none", return_value=flag),
         patch("requests.Session.request") as request,
         pytest.raises(ApplicationError) as error,
     ):
-        spec.return_value.resolve.return_value = MagicMock(provider="typesafe")
+        teams.return_value.get.return_value = Team(id=1, organization_id=uuid.uuid4(), uuid=uuid.uuid4())
+        spec.return_value.resolve.return_value = MagicMock(
+            provider="typesafe",
+            model="custom-model",
+            provider_key=MagicMock(
+                provider="typesafe", encrypted_config={"base_url": base_url, "api_key": "example-token"}
+            ),
+            is_byok=True,
+        )
         call_llm_judge(
             evaluation={"team_id": 1, "evaluation_config": {"prompt": "Is this a greeting?"}},
             system_prompt="",
diff --git a/products/ai_observability/backend/api/provider_keys.py b/products/ai_observability/backend/api/provider_keys.py
index ed39010827b8..98dd2abb2830 100644
--- a/products/ai_observability/backend/api/provider_keys.py
+++ b/products/ai_observability/backend/api/provider_keys.py
@@ -71,7 +71,9 @@ def _reload_model_config_dependents_on_commit(team_id: int, model_config_ids: li
 
 def validate_provider_key(provider: str, api_key: str, *, team_id: int, **kwargs: str) -> tuple[str, str | None]:
     """Validate an API key for any supported provider using the unified client."""
-    if provider == LLMProvider.TYPESAFE and not system_one_evaluations_enabled(team_id):
+    if provider == LLMProvider.TYPESAFE and not system_one_evaluations_enabled(
+        team_id, base_url=kwargs.get("base_url", SystemOneClient.BASE_URL)
+    ):
         raise exceptions.PermissionDenied("System One evaluations are not available for this project.")
     try:
         return Client.validate_key(provider, api_key, **kwargs)
diff --git a/products/ai_observability/backend/api/test/test_provider_keys.py b/products/ai_observability/backend/api/test/test_provider_keys.py
index e6baf258e782..7278e5364705 100644
--- a/products/ai_observability/backend/api/test/test_provider_keys.py
+++ b/products/ai_observability/backend/api/test/test_provider_keys.py
@@ -123,24 +123,33 @@ def setUp(self):
         self.organization_membership.save()
 
     @parameterized.expand([("create",), ("update",), ("validate",), ("prevalidate",)])
-    def test_system_one_customer_project_cannot_make_requests(self, operation: str) -> None:
+    def test_system_one_customer_project_cannot_use_posthog_gateway_or_disabled_feature(self, operation: str) -> None:
+        gateway_url = "https://ai-gateway.us.posthog.com/v1"
         key = LLMProviderKey.objects.create(
             team=self.team,
             provider="typesafe",
             name="Example connection",
-            encrypted_config={"api_key": "example-token"},
+            encrypted_config={"api_key": "example-token", "base_url": gateway_url},
             created_by=self.user,
         )
         base_url = f"/api/environments/{self.team.id}/llm_analytics/"
         with (
             self.settings(POSTHOG_INTERNAL_ORG_IDS=[]),
-            patch("products.ai_observability.backend.llm.system_one.get_feature_flag_or_none", return_value=True),
+            patch(
+                "products.ai_observability.backend.llm.system_one.get_feature_flag_or_none",
+                return_value=operation != "prevalidate",
+            ),
             patch("requests.Session.request") as request,
         ):
             if operation == "create":
                 response = self.client.post(
                     f"{base_url}provider_keys/",
-                    {"provider": "typesafe", "name": "Example connection", "api_key": "example-token"},
+                    {
+                        "provider": "typesafe",
+                        "name": "Example connection",
+                        "api_key": "example-token",
+                        "base_url": gateway_url,
+                    },
                 )
             elif operation == "update":
                 response = self.client.patch(f"{base_url}provider_keys/{key.id}/", {"api_key": "new-example-token"})
@@ -190,9 +199,9 @@ def test_can_create_provider_key(self, provider: str, mock_validate: Mock) -> No
     @patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")})
     @patch("posthog.egress.limiter.backends.LimitsBackend.consume_sync", return_value=True)
     @patch("requests.Session.request")
-    @patch("products.ai_observability.backend.api.provider_keys.system_one_evaluations_enabled", return_value=True)
+    @patch("products.ai_observability.backend.llm.system_one.get_feature_flag_or_none", return_value=True)
     def test_custom_system_one_connection_round_trip(
-        self, _enabled: Mock, request: Mock, _budget: Mock, _dns: Mock
+        self, _flag: Mock, request: Mock, _budget: Mock, _dns: Mock
     ) -> None:
         request.return_value = Mock(status_code=200)
         request.return_value.json.return_value = {
diff --git a/products/ai_observability/backend/llm/system_one.py b/products/ai_observability/backend/llm/system_one.py
index ac08fc3eae20..71856bf7451b 100644
--- a/products/ai_observability/backend/llm/system_one.py
+++ b/products/ai_observability/backend/llm/system_one.py
@@ -38,10 +38,17 @@
 )
 
 
-def system_one_evaluations_enabled(team_id: int) -> bool:
+def system_one_evaluations_enabled(team_id: int, *, base_url: str) -> bool:
+    try:
+        host = (urlsplit(base_url).hostname or "").encode("idna").decode("ascii").lower().rstrip(".")
+    except ValueError:
+        return False
     team = Team.objects.only("uuid", "organization_id").get(id=team_id)
-    # Staff access to a customer project does not make its data eligible for this experiment.
-    if str(team.organization_id) not in settings.POSTHOG_INTERNAL_ORG_IDS:
+    # Customer connections use their own provider account or deployment, never PostHog's gateway.
+    if (
+        host in {"ai-gateway.us.posthog.com", "ai-gateway.eu.posthog.com"}
+        and str(team.organization_id) not in settings.POSTHOG_INTERNAL_ORG_IDS
+    ):
         return False
     return (
         get_feature_flag_or_none(
diff --git a/products/ai_observability/backend/llm/test/test_system_one.py b/products/ai_observability/backend/llm/test/test_system_one.py
index 52f87b414c48..03d852269cc4 100644
--- a/products/ai_observability/backend/llm/test/test_system_one.py
+++ b/products/ai_observability/backend/llm/test/test_system_one.py
@@ -36,10 +36,22 @@ def isolated_egress_budget() -> Iterator[None]:
 
 
 @pytest.mark.parametrize(
-    "internal,flag,enabled", [(False, True, False), (True, False, False), (True, None, False), (True, True, True)]
+    "base_url,internal,flag,enabled",
+    [
+        (SystemOneClient.BASE_URL, False, True, True),
+        ("https://decisions.example.com/v1", False, True, True),
+        ("https://decisions.example.com/v1", False, False, False),
+        (SystemOneClient.BASE_URL, False, None, False),
+        ("https://ai-gateway.us.posthog.com/v1", False, True, False),
+        ("https://ai-gateway.eu.posthog.com/v1", False, True, False),
+        ("https://AI-GATEWAY.US.POSTHOG.COM.:443/v1", False, True, False),
+        ("https://ａｉ-gateway.us.posthog.com/v1", False, True, False),
+        ("https://ai-gateway.us.posthog.com/v1", True, False, False),
+        ("https://ai-gateway.us.posthog.com/v1", True, True, True),
+    ],
 )
-def test_system_one_experiment_requires_internal_project_and_flag(
-    internal: bool, flag: bool | None, enabled: bool
+def test_system_one_connections_require_flag_and_reserve_posthog_gateway_for_internal_projects(
+    base_url: str, internal: bool, flag: bool | None, enabled: bool
 ) -> None:
     team = Team(id=1, organization_id=uuid4(), uuid=uuid4())
     with (
@@ -48,7 +60,7 @@ def test_system_one_experiment_requires_internal_project_and_flag(
         patch("products.ai_observability.backend.llm.system_one.get_feature_flag_or_none", return_value=flag),
     ):
         teams.return_value.get.return_value = team
-        assert system_one_evaluations_enabled(team.id) is enabled
+        assert system_one_evaluations_enabled(team.id, base_url=base_url) is enabled
 
 
 @pytest.mark.parametrize("status, expected_state", [(200, "ok"), (401, "invalid"), (403, "invalid"), (500, "error")])
@@ -123,6 +135,7 @@ def test_invalid_usage_does_not_become_a_billable_evaluation(usage: dict[str, ob
         )
 
 
+@override_settings(TYPESAFE_API_KEY="example-instance-key")
 def test_typesafe_requires_a_key() -> None:
     with (
         patch("requests.Session.request") as request,

From f452de6e3a74ce0997796731f33a571569295cd9 Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Fri, 25 Sep 2026 16:14:30 +0200
Subject: [PATCH 13/30] fix(aio): leave custom system one costs unknown

---
 .../internal/ai-observability-judge-inputs.md |  6 ++++
 .../core-filter-definitions-by-group.json     | 14 ++++++++
 .../src/ingestion/pipelines/ai/costs/index.ts |  5 +++
 .../pipelines/ai/process-ai-event.test.ts     | 30 +++++++++++++++-
 posthog/taxonomy/taxonomy.py                  |  5 +++
 .../ai_observability/evaluation_llm_judge.py  |  3 ++
 .../ai_observability/evaluation_types.py      |  1 +
 .../evaluation_workflow_activities.py         |  3 ++
 .../ai_observability/test_run_evaluation.py   | 35 +++++++++++++++++--
 .../lib/trace-property-allowlist.generated.ts |  1 +
 10 files changed, 100 insertions(+), 3 deletions(-)

diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index cfd9976c422d..5846cf7ca8c7 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -43,6 +43,7 @@ Implementation: [trace judge](../../posthog/temporal/ai_observability/run_trace_
 
 System One-compatible models are available under the existing LLM judge option.
 The `llm-analytics-system-one-evaluations` project-group feature flag controls access in the browser and background workers.
+Deploy the ingestion and evaluation worker changes before enabling the flag.
 Customers configure their own TypeSafe key or a compatible deployment with its own authentication.
 Evaluation connections never fall back to an instance TypeSafe key or gateway credential.
 PostHog's regional AI gateway endpoints additionally require an organization in `POSTHOG_INTERNAL_ORG_IDS`; customer projects cannot use those endpoints.
@@ -69,6 +70,11 @@ Compare results on representative inputs when changing models.
 For boolean evaluations, the prompt becomes a [Noul question](https://docs.typesafe.ai/primitives/noul).
 A probability of at least 0.5 produces `true`; the evaluation's existing pass/fail polarity still applies.
 The raw probability is stored in `$ai_evaluation_probability`, with token usage and the resolved model version.
+Evaluations sent to the default TypeSafe endpoint use known model pricing to estimate cost.
+Custom deployments, including PostHog's gateway, retain their model name and token usage with cost left unknown.
+They set `$ai_cost_estimation_enabled` to `false`, so a matching model name cannot apply another service's price.
+System One results without endpoint pricing information also leave cost unknown.
+Ingestion still accepts explicitly reported costs and custom token prices; System One connection settings do not expose pricing overrides.
 
 Evaluations that allow N/A send a separate Noul question about whether the criteria apply, using the 0.5 threshold.
 Uncertainty alone does not produce N/A.
diff --git a/frontend/src/taxonomy/core-filter-definitions-by-group.json b/frontend/src/taxonomy/core-filter-definitions-by-group.json
index 297e1991d1be..0e436b2ad560 100644
--- a/frontend/src/taxonomy/core-filter-definitions-by-group.json
+++ b/frontend/src/taxonomy/core-filter-definitions-by-group.json
@@ -240,6 +240,13 @@
             "label": "AI cache write token price (LLM)",
             "type": "Numeric"
         },
+        "$ai_cost_estimation_enabled": {
+            "description": "Set to false to skip automatic model pricing for custom deployments. Reported costs and custom token prices still apply. Without either, cost stays unknown.",
+            "examples": [
+                false
+            ],
+            "label": "AI cost estimation enabled (LLM)"
+        },
         "$ai_cost_model_provider": {
             "description": "The provider used to look up the cost for this model.",
             "examples": [
@@ -5103,6 +5110,13 @@
             "label": "AI cache write token price (LLM)",
             "type": "Numeric"
         },
+        "$ai_cost_estimation_enabled": {
+            "description": "Set to false to skip automatic model pricing for custom deployments. Reported costs and custom token prices still apply. Without either, cost stays unknown.",
+            "examples": [
+                false
+            ],
+            "label": "AI cost estimation enabled (LLM)"
+        },
         "$ai_cost_model_provider": {
             "description": "The provider used to look up the cost for this model.",
             "examples": [
diff --git a/nodejs/src/ingestion/pipelines/ai/costs/index.ts b/nodejs/src/ingestion/pipelines/ai/costs/index.ts
index 583d7811409d..191cacc8df8b 100644
--- a/nodejs/src/ingestion/pipelines/ai/costs/index.ts
+++ b/nodejs/src/ingestion/pipelines/ai/costs/index.ts
@@ -261,6 +261,11 @@ export const processCost = (event: EventWithProperties): EventWithProperties =>
         return event
     }
 
+    const costEstimationEnabled = event.properties['$ai_cost_estimation_enabled']
+    if (costEstimationEnabled === false || costEstimationEnabled === 'false') {
+        return event
+    }
+
     if (!event.properties['$ai_model']) {
         return event
     }
diff --git a/nodejs/src/ingestion/pipelines/ai/process-ai-event.test.ts b/nodejs/src/ingestion/pipelines/ai/process-ai-event.test.ts
index 45f1e31ebd24..6f7b9fb56811 100644
--- a/nodejs/src/ingestion/pipelines/ai/process-ai-event.test.ts
+++ b/nodejs/src/ingestion/pipelines/ai/process-ai-event.test.ts
@@ -823,6 +823,32 @@ describe('processAiEvent()', () => {
     })
 
     describe('cost optimization bypass', () => {
+        it.each([
+            ['claude-2', false, undefined],
+            ['claude-2', 'false', undefined],
+            ['claude-2', true, 90],
+            ['claude-2', undefined, 90],
+            ['testing_model', false, undefined],
+        ])('prices %s with catalog estimation set to %p', (model, enabled, total) => {
+            event.event = '$ai_evaluation'
+            event.properties!.$ai_model = model
+            event.properties!.$ai_provider = 'typesafe'
+            event.properties!.$ai_cost_estimation_enabled = enabled
+
+            const result = processAiEvent(event)
+
+            expect(result.properties!.$ai_total_cost_usd).toBe(total)
+            expect(result.properties!.$ai_model).toBe(model)
+            expect(result.properties!.$ai_input_tokens).toBe(100)
+            expect(result.properties!.$ai_output_tokens).toBe(50)
+            if (total === undefined) {
+                expect(result.properties!.$ai_input_cost_usd).toBeUndefined()
+                expect(result.properties!.$ai_output_cost_usd).toBeUndefined()
+                expect(result.properties!.$ai_model_cost_used).toBeUndefined()
+                expect(result.properties!.$ai_cost_model_source).toBeUndefined()
+            }
+        })
+
         it('skips calculation when costs are pre-calculated', () => {
             event.properties!.$ai_input_cost_usd = 10.5
             event.properties!.$ai_output_cost_usd = 5.5
@@ -939,6 +965,7 @@ describe('processAiEvent()', () => {
 
         it.each([true, 'true'])('passes the total through when $ai_cost_passthrough is %p', (flag) => {
             event.properties!.$ai_cost_passthrough = flag
+            event.properties!.$ai_cost_estimation_enabled = false
             event.properties!.$ai_total_cost_usd = 0.000372
 
             const result = processAiEvent(event)
@@ -983,7 +1010,8 @@ describe('processAiEvent()', () => {
     })
 
     describe('custom token pricing', () => {
-        it('uses custom pricing when provided', () => {
+        it.each([undefined, false])('uses custom pricing when catalog estimation is %p', (enabled) => {
+            event.properties!.$ai_cost_estimation_enabled = enabled
             event.properties!.$ai_input_token_price = 0.001
             event.properties!.$ai_output_token_price = 0.002
             event.properties!.$ai_input_tokens = 100
diff --git a/posthog/taxonomy/taxonomy.py b/posthog/taxonomy/taxonomy.py
index 5146d4b72921..d6c814c155fe 100644
--- a/posthog/taxonomy/taxonomy.py
+++ b/posthog/taxonomy/taxonomy.py
@@ -2451,6 +2451,11 @@ def visible_definitions(group: str) -> Iterator[tuple[str, CoreFilterDefinition]
             "description": "Set this to keep the reported total cost and skip PostHog's token-based estimate. Use it when the provider reports an authoritative cost, such as an LLM gateway. The input and output cost split is left unset.",
             "examples": [True],
         },
+        "$ai_cost_estimation_enabled": {
+            "label": "AI cost estimation enabled (LLM)",
+            "description": "Set to false to skip automatic model pricing for custom deployments. Reported costs and custom token prices still apply. Without either, cost stays unknown.",
+            "examples": [False],
+        },
         "$ai_cost_model_provider": {
             "label": "AI cost model provider (LLM)",
             "description": "The provider used to look up the cost for this model.",
diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.py
index 416a931c051f..5a46945395f7 100644
--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py
+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py
@@ -777,4 +777,7 @@ def call_llm_judge(
     if probability is not None:
         result_dict["probability"] = probability
 
+    if provider == "typesafe":
+        result_dict["cost_estimation_enabled"] = base_url.rstrip("/") == SystemOneClient.BASE_URL
+
     return result_dict
diff --git a/posthog/temporal/ai_observability/evaluation_types.py b/posthog/temporal/ai_observability/evaluation_types.py
index 6ca039669f19..354d653fb262 100644
--- a/posthog/temporal/ai_observability/evaluation_types.py
+++ b/posthog/temporal/ai_observability/evaluation_types.py
@@ -46,6 +46,7 @@ class EvaluationActivityResult(TypedDict, total=False):
     key_id: NotRequired[str | None]
     model: NotRequired[str]
     provider: NotRequired[str]
+    cost_estimation_enabled: NotRequired[bool]
     applicable: NotRequired[bool]
     skipped: NotRequired[bool]
     skip_reason: NotRequired[str]
diff --git a/posthog/temporal/ai_observability/evaluation_workflow_activities.py b/posthog/temporal/ai_observability/evaluation_workflow_activities.py
index dce05036342c..f31c4874c3ef 100644
--- a/posthog/temporal/ai_observability/evaluation_workflow_activities.py
+++ b/posthog/temporal/ai_observability/evaluation_workflow_activities.py
@@ -302,6 +302,9 @@ def build_evaluation_event_properties(
         properties["$ai_evaluation_provider"] = result.get("provider", "openai")
         properties["$ai_evaluation_key_type"] = "byok" if result.get("is_byok") else "posthog"
         properties["$ai_evaluation_key_id"] = result.get("key_id")
+        if result.get("provider") == "typesafe":
+            # A model name alone does not establish a System One deployment's price.
+            properties["$ai_cost_estimation_enabled"] = result.get("cost_estimation_enabled", False)
         if "probability" in result:
             properties["$ai_evaluation_probability"] = result["probability"]
 
diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index d7ef0d15b4b6..d02ad7e8ef80 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -95,13 +95,20 @@ def _mock_config_with_active_key(provider: str = "openai") -> MagicMock:
 
 
 @pytest.mark.parametrize(
-    "connection_config,base_url,model",
+    "connection_config,base_url,model,cost_estimation_enabled",
     [
-        ({"api_key": "test-typesafe-key"}, "https://api.typesafe.ai/v1", "jev-1.13.0"),
+        ({"api_key": "test-typesafe-key"}, "https://api.typesafe.ai/v1", "jev-1.13.0", True),
         (
             {"api_key": "", "base_url": "https://decisions.example.com/v1"},
             "https://decisions.example.com/v1",
             "custom-model",
+            False,
+        ),
+        (
+            {"api_key": "example-token", "base_url": "https://ai-gateway.us.posthog.com/v1"},
+            "https://ai-gateway.us.posthog.com/v1",
+            "custom-model",
+            False,
         ),
     ],
 )
@@ -117,6 +124,7 @@ def test_typesafe_judge_emits_boolean_probability_without_reasoning(
     connection_config: dict[str, str],
     base_url: str,
     model: str,
+    cost_estimation_enabled: bool,
 ) -> None:
     key = MagicMock(provider="typesafe", encrypted_config=connection_config)
     resolved = MagicMock(provider="typesafe", model=model, provider_key=key, is_byok=True)
@@ -164,6 +172,29 @@ def test_typesafe_judge_emits_boolean_probability_without_reasoning(
     assert properties["$ai_evaluation_probability"] == probability
     assert properties["$ai_model"] == "jev-1.13.0"
     assert properties["$ai_evaluation_key_type"] == "byok"
+    assert properties["$ai_input_tokens"] == 120
+    assert properties["$ai_output_tokens"] == 10
+    assert properties["$ai_cost_estimation_enabled"] is cost_estimation_enabled
+
+
+@pytest.mark.parametrize("provider,enabled", [("typesafe", False), ("openai", None)])
+def test_judge_cost_estimation_without_endpoint_information(provider: str, enabled: bool | None) -> None:
+    properties = build_evaluation_event_properties(
+        {"id": "example-evaluation", "name": "Greeting"},
+        {
+            "result_type": "boolean",
+            "reasoning": "",
+            "verdict": True,
+            "provider": provider,
+            "model": "example-model",
+            "input_tokens": 120,
+            "output_tokens": 10,
+        },
+        datetime(2026, 1, 1, tzinfo=UTC),
+    )
+    assert properties.get("$ai_cost_estimation_enabled") is enabled
+    assert properties["$ai_input_tokens"] == 120
+    assert properties["$ai_output_tokens"] == 10
 
 
 def test_system_one_numeric_mapping_is_not_enabled() -> None:
diff --git a/services/mcp/src/lib/trace-property-allowlist.generated.ts b/services/mcp/src/lib/trace-property-allowlist.generated.ts
index c53d58cfdbcd..5659410d2a40 100644
--- a/services/mcp/src/lib/trace-property-allowlist.generated.ts
+++ b/services/mcp/src/lib/trace-property-allowlist.generated.ts
@@ -22,6 +22,7 @@ export const AI_TAXONOMY_EVENT_PROPERTIES = [
     '$ai_cache_reporting_exclusive',
     '$ai_cache_write_1h_token_price',
     '$ai_cache_write_token_price',
+    '$ai_cost_estimation_enabled',
     '$ai_cost_model_provider',
     '$ai_cost_model_source',
     '$ai_cost_passthrough',

From 0a6ea3e7c0480ff3990222fb4a04ff5c65912385 Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Fri, 25 Sep 2026 16:36:20 +0200
Subject: [PATCH 14/30] fix(aio): defer system one pricing without extra
 metadata

---
 .../internal/ai-observability-judge-inputs.md |  7 ++--
 .../core-filter-definitions-by-group.json     | 14 --------
 .../src/ingestion/pipelines/ai/costs/index.ts |  4 +--
 .../costs/providers/manual-providers.test.ts  |  1 -
 .../ai/costs/providers/manual-providers.ts    |  4 ---
 .../pipelines/ai/process-ai-event.test.ts     | 18 ++++------
 posthog/taxonomy/taxonomy.py                  |  5 ---
 .../ai_observability/evaluation_llm_judge.py  |  3 --
 .../ai_observability/evaluation_types.py      |  1 -
 .../evaluation_workflow_activities.py         |  3 --
 .../ai_observability/test_run_evaluation.py   | 35 ++-----------------
 .../lib/trace-property-allowlist.generated.ts |  1 -
 12 files changed, 12 insertions(+), 84 deletions(-)

diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index 5846cf7ca8c7..c40812122707 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -70,11 +70,8 @@ Compare results on representative inputs when changing models.
 For boolean evaluations, the prompt becomes a [Noul question](https://docs.typesafe.ai/primitives/noul).
 A probability of at least 0.5 produces `true`; the evaluation's existing pass/fail polarity still applies.
 The raw probability is stored in `$ai_evaluation_probability`, with token usage and the resolved model version.
-Evaluations sent to the default TypeSafe endpoint use known model pricing to estimate cost.
-Custom deployments, including PostHog's gateway, retain their model name and token usage with cost left unknown.
-They set `$ai_cost_estimation_enabled` to `false`, so a matching model name cannot apply another service's price.
-System One results without endpoint pricing information also leave cost unknown.
-Ingestion still accepts explicitly reported costs and custom token prices; System One connection settings do not expose pricing overrides.
+Automatic cost estimates are deferred for all System One evaluations, including official TypeSafe connections.
+Events retain their model name and token usage, with cost left unknown.
 
 Evaluations that allow N/A send a separate Noul question about whether the criteria apply, using the 0.5 threshold.
 Uncertainty alone does not produce N/A.
diff --git a/frontend/src/taxonomy/core-filter-definitions-by-group.json b/frontend/src/taxonomy/core-filter-definitions-by-group.json
index 0e436b2ad560..297e1991d1be 100644
--- a/frontend/src/taxonomy/core-filter-definitions-by-group.json
+++ b/frontend/src/taxonomy/core-filter-definitions-by-group.json
@@ -240,13 +240,6 @@
             "label": "AI cache write token price (LLM)",
             "type": "Numeric"
         },
-        "$ai_cost_estimation_enabled": {
-            "description": "Set to false to skip automatic model pricing for custom deployments. Reported costs and custom token prices still apply. Without either, cost stays unknown.",
-            "examples": [
-                false
-            ],
-            "label": "AI cost estimation enabled (LLM)"
-        },
         "$ai_cost_model_provider": {
             "description": "The provider used to look up the cost for this model.",
             "examples": [
@@ -5110,13 +5103,6 @@
             "label": "AI cache write token price (LLM)",
             "type": "Numeric"
         },
-        "$ai_cost_estimation_enabled": {
-            "description": "Set to false to skip automatic model pricing for custom deployments. Reported costs and custom token prices still apply. Without either, cost stays unknown.",
-            "examples": [
-                false
-            ],
-            "label": "AI cost estimation enabled (LLM)"
-        },
         "$ai_cost_model_provider": {
             "description": "The provider used to look up the cost for this model.",
             "examples": [
diff --git a/nodejs/src/ingestion/pipelines/ai/costs/index.ts b/nodejs/src/ingestion/pipelines/ai/costs/index.ts
index 191cacc8df8b..97fbea273275 100644
--- a/nodejs/src/ingestion/pipelines/ai/costs/index.ts
+++ b/nodejs/src/ingestion/pipelines/ai/costs/index.ts
@@ -261,8 +261,8 @@ export const processCost = (event: EventWithProperties): EventWithProperties =>
         return event
     }
 
-    const costEstimationEnabled = event.properties['$ai_cost_estimation_enabled']
-    if (costEstimationEnabled === false || costEstimationEnabled === 'false') {
+    // System One endpoints can be self-hosted, so model names do not establish prices.
+    if (event.event === '$ai_evaluation' && event.properties['$ai_provider'] === 'typesafe') {
         return event
     }
 
diff --git a/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.test.ts b/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.test.ts
index bdbdea4850c6..f2824fe114c2 100644
--- a/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.test.ts
+++ b/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.test.ts
@@ -5,7 +5,6 @@ describe('manualCosts', () => {
         model: string
         expected: { prompt_token: number; completion_token: number; cache_read_token?: number }
     }> = [
-        { model: 'jev-1.13.0', expected: { prompt_token: 0.000000042, completion_token: 0 } },
         {
             model: 'gpt-4.5',
             expected: {
diff --git a/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.ts b/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.ts
index 4085be25813d..0781a502ea6c 100644
--- a/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.ts
+++ b/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.ts
@@ -1,10 +1,6 @@
 import type { ModelCostRow } from './types'
 
 const manualProviderCosts: ModelCostRow[] = [
-    {
-        model: 'jev-1.13.0',
-        cost: { default: { prompt_token: 0.000000042, completion_token: 0 } },
-    },
     {
         model: 'gpt-4.5',
         cost: {
diff --git a/nodejs/src/ingestion/pipelines/ai/process-ai-event.test.ts b/nodejs/src/ingestion/pipelines/ai/process-ai-event.test.ts
index 6f7b9fb56811..fbb2b9be0c7e 100644
--- a/nodejs/src/ingestion/pipelines/ai/process-ai-event.test.ts
+++ b/nodejs/src/ingestion/pipelines/ai/process-ai-event.test.ts
@@ -824,16 +824,13 @@ describe('processAiEvent()', () => {
 
     describe('cost optimization bypass', () => {
         it.each([
-            ['claude-2', false, undefined],
-            ['claude-2', 'false', undefined],
-            ['claude-2', true, 90],
-            ['claude-2', undefined, 90],
-            ['testing_model', false, undefined],
-        ])('prices %s with catalog estimation set to %p', (model, enabled, total) => {
+            ['gpt-4', 'typesafe', undefined],
+            ['claude-2', 'typesafe', undefined],
+            ['gpt-4', 'openai', 30],
+        ])('prices evaluations using %s through %s', (model, provider, total) => {
             event.event = '$ai_evaluation'
             event.properties!.$ai_model = model
-            event.properties!.$ai_provider = 'typesafe'
-            event.properties!.$ai_cost_estimation_enabled = enabled
+            event.properties!.$ai_provider = provider
 
             const result = processAiEvent(event)
 
@@ -845,7 +842,6 @@ describe('processAiEvent()', () => {
                 expect(result.properties!.$ai_input_cost_usd).toBeUndefined()
                 expect(result.properties!.$ai_output_cost_usd).toBeUndefined()
                 expect(result.properties!.$ai_model_cost_used).toBeUndefined()
-                expect(result.properties!.$ai_cost_model_source).toBeUndefined()
             }
         })
 
@@ -965,7 +961,6 @@ describe('processAiEvent()', () => {
 
         it.each([true, 'true'])('passes the total through when $ai_cost_passthrough is %p', (flag) => {
             event.properties!.$ai_cost_passthrough = flag
-            event.properties!.$ai_cost_estimation_enabled = false
             event.properties!.$ai_total_cost_usd = 0.000372
 
             const result = processAiEvent(event)
@@ -1010,8 +1005,7 @@ describe('processAiEvent()', () => {
     })
 
     describe('custom token pricing', () => {
-        it.each([undefined, false])('uses custom pricing when catalog estimation is %p', (enabled) => {
-            event.properties!.$ai_cost_estimation_enabled = enabled
+        it('uses custom pricing when provided', () => {
             event.properties!.$ai_input_token_price = 0.001
             event.properties!.$ai_output_token_price = 0.002
             event.properties!.$ai_input_tokens = 100
diff --git a/posthog/taxonomy/taxonomy.py b/posthog/taxonomy/taxonomy.py
index d6c814c155fe..5146d4b72921 100644
--- a/posthog/taxonomy/taxonomy.py
+++ b/posthog/taxonomy/taxonomy.py
@@ -2451,11 +2451,6 @@ def visible_definitions(group: str) -> Iterator[tuple[str, CoreFilterDefinition]
             "description": "Set this to keep the reported total cost and skip PostHog's token-based estimate. Use it when the provider reports an authoritative cost, such as an LLM gateway. The input and output cost split is left unset.",
             "examples": [True],
         },
-        "$ai_cost_estimation_enabled": {
-            "label": "AI cost estimation enabled (LLM)",
-            "description": "Set to false to skip automatic model pricing for custom deployments. Reported costs and custom token prices still apply. Without either, cost stays unknown.",
-            "examples": [False],
-        },
         "$ai_cost_model_provider": {
             "label": "AI cost model provider (LLM)",
             "description": "The provider used to look up the cost for this model.",
diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.py
index 5a46945395f7..416a931c051f 100644
--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py
+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py
@@ -777,7 +777,4 @@ def call_llm_judge(
     if probability is not None:
         result_dict["probability"] = probability
 
-    if provider == "typesafe":
-        result_dict["cost_estimation_enabled"] = base_url.rstrip("/") == SystemOneClient.BASE_URL
-
     return result_dict
diff --git a/posthog/temporal/ai_observability/evaluation_types.py b/posthog/temporal/ai_observability/evaluation_types.py
index 354d653fb262..6ca039669f19 100644
--- a/posthog/temporal/ai_observability/evaluation_types.py
+++ b/posthog/temporal/ai_observability/evaluation_types.py
@@ -46,7 +46,6 @@ class EvaluationActivityResult(TypedDict, total=False):
     key_id: NotRequired[str | None]
     model: NotRequired[str]
     provider: NotRequired[str]
-    cost_estimation_enabled: NotRequired[bool]
     applicable: NotRequired[bool]
     skipped: NotRequired[bool]
     skip_reason: NotRequired[str]
diff --git a/posthog/temporal/ai_observability/evaluation_workflow_activities.py b/posthog/temporal/ai_observability/evaluation_workflow_activities.py
index f31c4874c3ef..dce05036342c 100644
--- a/posthog/temporal/ai_observability/evaluation_workflow_activities.py
+++ b/posthog/temporal/ai_observability/evaluation_workflow_activities.py
@@ -302,9 +302,6 @@ def build_evaluation_event_properties(
         properties["$ai_evaluation_provider"] = result.get("provider", "openai")
         properties["$ai_evaluation_key_type"] = "byok" if result.get("is_byok") else "posthog"
         properties["$ai_evaluation_key_id"] = result.get("key_id")
-        if result.get("provider") == "typesafe":
-            # A model name alone does not establish a System One deployment's price.
-            properties["$ai_cost_estimation_enabled"] = result.get("cost_estimation_enabled", False)
         if "probability" in result:
             properties["$ai_evaluation_probability"] = result["probability"]
 
diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index d02ad7e8ef80..d7ef0d15b4b6 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -95,20 +95,13 @@ def _mock_config_with_active_key(provider: str = "openai") -> MagicMock:
 
 
 @pytest.mark.parametrize(
-    "connection_config,base_url,model,cost_estimation_enabled",
+    "connection_config,base_url,model",
     [
-        ({"api_key": "test-typesafe-key"}, "https://api.typesafe.ai/v1", "jev-1.13.0", True),
+        ({"api_key": "test-typesafe-key"}, "https://api.typesafe.ai/v1", "jev-1.13.0"),
         (
             {"api_key": "", "base_url": "https://decisions.example.com/v1"},
             "https://decisions.example.com/v1",
             "custom-model",
-            False,
-        ),
-        (
-            {"api_key": "example-token", "base_url": "https://ai-gateway.us.posthog.com/v1"},
-            "https://ai-gateway.us.posthog.com/v1",
-            "custom-model",
-            False,
         ),
     ],
 )
@@ -124,7 +117,6 @@ def test_typesafe_judge_emits_boolean_probability_without_reasoning(
     connection_config: dict[str, str],
     base_url: str,
     model: str,
-    cost_estimation_enabled: bool,
 ) -> None:
     key = MagicMock(provider="typesafe", encrypted_config=connection_config)
     resolved = MagicMock(provider="typesafe", model=model, provider_key=key, is_byok=True)
@@ -172,29 +164,6 @@ def test_typesafe_judge_emits_boolean_probability_without_reasoning(
     assert properties["$ai_evaluation_probability"] == probability
     assert properties["$ai_model"] == "jev-1.13.0"
     assert properties["$ai_evaluation_key_type"] == "byok"
-    assert properties["$ai_input_tokens"] == 120
-    assert properties["$ai_output_tokens"] == 10
-    assert properties["$ai_cost_estimation_enabled"] is cost_estimation_enabled
-
-
-@pytest.mark.parametrize("provider,enabled", [("typesafe", False), ("openai", None)])
-def test_judge_cost_estimation_without_endpoint_information(provider: str, enabled: bool | None) -> None:
-    properties = build_evaluation_event_properties(
-        {"id": "example-evaluation", "name": "Greeting"},
-        {
-            "result_type": "boolean",
-            "reasoning": "",
-            "verdict": True,
-            "provider": provider,
-            "model": "example-model",
-            "input_tokens": 120,
-            "output_tokens": 10,
-        },
-        datetime(2026, 1, 1, tzinfo=UTC),
-    )
-    assert properties.get("$ai_cost_estimation_enabled") is enabled
-    assert properties["$ai_input_tokens"] == 120
-    assert properties["$ai_output_tokens"] == 10
 
 
 def test_system_one_numeric_mapping_is_not_enabled() -> None:
diff --git a/services/mcp/src/lib/trace-property-allowlist.generated.ts b/services/mcp/src/lib/trace-property-allowlist.generated.ts
index 5659410d2a40..c53d58cfdbcd 100644
--- a/services/mcp/src/lib/trace-property-allowlist.generated.ts
+++ b/services/mcp/src/lib/trace-property-allowlist.generated.ts
@@ -22,7 +22,6 @@ export const AI_TAXONOMY_EVENT_PROPERTIES = [
     '$ai_cache_reporting_exclusive',
     '$ai_cache_write_1h_token_price',
     '$ai_cache_write_token_price',
-    '$ai_cost_estimation_enabled',
     '$ai_cost_model_provider',
     '$ai_cost_model_source',
     '$ai_cost_passthrough',

From 693576e0df4c8f7ff72c2cd2162ed0a6675be6a5 Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Fri, 25 Sep 2026 16:44:31 +0200
Subject: [PATCH 15/30] fix(aio): use existing model pricing for system one

---
 .../internal/ai-observability-judge-inputs.md |  5 +++--
 .../src/ingestion/pipelines/ai/costs/index.ts |  5 -----
 .../costs/providers/manual-providers.test.ts  |  1 +
 .../ai/costs/providers/manual-providers.ts    |  4 ++++
 .../pipelines/ai/process-ai-event.test.ts     | 22 -------------------
 5 files changed, 8 insertions(+), 29 deletions(-)

diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index c40812122707..5a6b70f7104f 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -70,8 +70,9 @@ Compare results on representative inputs when changing models.
 For boolean evaluations, the prompt becomes a [Noul question](https://docs.typesafe.ai/primitives/noul).
 A probability of at least 0.5 produces `true`; the evaluation's existing pass/fail polarity still applies.
 The raw probability is stored in `$ai_evaluation_probability`, with token usage and the resolved model version.
-Automatic cost estimates are deferred for all System One evaluations, including official TypeSafe connections.
-Events retain their model name and token usage, with cost left unknown.
+Ingestion estimates cost from the reported model and token usage using the existing pricing catalog, including Jev 1.13.0.
+Models without a catalog match retain their usage with cost left unknown.
+A custom deployment reporting a recognized model name can inherit that model's catalog estimate; this does not measure its hosting cost.
 
 Evaluations that allow N/A send a separate Noul question about whether the criteria apply, using the 0.5 threshold.
 Uncertainty alone does not produce N/A.
diff --git a/nodejs/src/ingestion/pipelines/ai/costs/index.ts b/nodejs/src/ingestion/pipelines/ai/costs/index.ts
index 97fbea273275..583d7811409d 100644
--- a/nodejs/src/ingestion/pipelines/ai/costs/index.ts
+++ b/nodejs/src/ingestion/pipelines/ai/costs/index.ts
@@ -261,11 +261,6 @@ export const processCost = (event: EventWithProperties): EventWithProperties =>
         return event
     }
 
-    // System One endpoints can be self-hosted, so model names do not establish prices.
-    if (event.event === '$ai_evaluation' && event.properties['$ai_provider'] === 'typesafe') {
-        return event
-    }
-
     if (!event.properties['$ai_model']) {
         return event
     }
diff --git a/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.test.ts b/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.test.ts
index f2824fe114c2..bdbdea4850c6 100644
--- a/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.test.ts
+++ b/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.test.ts
@@ -5,6 +5,7 @@ describe('manualCosts', () => {
         model: string
         expected: { prompt_token: number; completion_token: number; cache_read_token?: number }
     }> = [
+        { model: 'jev-1.13.0', expected: { prompt_token: 0.000000042, completion_token: 0 } },
         {
             model: 'gpt-4.5',
             expected: {
diff --git a/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.ts b/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.ts
index 0781a502ea6c..4085be25813d 100644
--- a/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.ts
+++ b/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.ts
@@ -1,6 +1,10 @@
 import type { ModelCostRow } from './types'
 
 const manualProviderCosts: ModelCostRow[] = [
+    {
+        model: 'jev-1.13.0',
+        cost: { default: { prompt_token: 0.000000042, completion_token: 0 } },
+    },
     {
         model: 'gpt-4.5',
         cost: {
diff --git a/nodejs/src/ingestion/pipelines/ai/process-ai-event.test.ts b/nodejs/src/ingestion/pipelines/ai/process-ai-event.test.ts
index fbb2b9be0c7e..45f1e31ebd24 100644
--- a/nodejs/src/ingestion/pipelines/ai/process-ai-event.test.ts
+++ b/nodejs/src/ingestion/pipelines/ai/process-ai-event.test.ts
@@ -823,28 +823,6 @@ describe('processAiEvent()', () => {
     })
 
     describe('cost optimization bypass', () => {
-        it.each([
-            ['gpt-4', 'typesafe', undefined],
-            ['claude-2', 'typesafe', undefined],
-            ['gpt-4', 'openai', 30],
-        ])('prices evaluations using %s through %s', (model, provider, total) => {
-            event.event = '$ai_evaluation'
-            event.properties!.$ai_model = model
-            event.properties!.$ai_provider = provider
-
-            const result = processAiEvent(event)
-
-            expect(result.properties!.$ai_total_cost_usd).toBe(total)
-            expect(result.properties!.$ai_model).toBe(model)
-            expect(result.properties!.$ai_input_tokens).toBe(100)
-            expect(result.properties!.$ai_output_tokens).toBe(50)
-            if (total === undefined) {
-                expect(result.properties!.$ai_input_cost_usd).toBeUndefined()
-                expect(result.properties!.$ai_output_cost_usd).toBeUndefined()
-                expect(result.properties!.$ai_model_cost_used).toBeUndefined()
-            }
-        })
-
         it('skips calculation when costs are pre-calculated', () => {
             event.properties!.$ai_input_cost_usd = 10.5
             event.properties!.$ai_output_cost_usd = 5.5

From 8ccdf2a0109b46af36a4c8546e34cc7bf142727c Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Fri, 25 Sep 2026 19:05:07 +0200
Subject: [PATCH 16/30] feat(aio): support generic system one evaluation
 endpoints

---
 .../internal/ai-observability-judge-inputs.md | 18 ++--
 .../costs/providers/manual-providers.test.ts  |  1 -
 .../ai/costs/providers/manual-providers.ts    |  4 -
 .../ai_observability/evaluation_llm_judge.py  | 10 +--
 .../ai_observability/test_run_evaluation.py   | 45 ++++++----
 .../backend/api/evaluation_config.py          |  2 +-
 .../backend/api/evaluations.py                |  4 +-
 .../backend/api/provider_keys.py              | 38 ++++-----
 .../ai_observability/backend/api/proxy.py     |  4 +-
 .../api/test/test_evaluation_config.py        |  8 +-
 .../backend/api/test/test_evaluations.py      | 12 +--
 .../backend/api/test/test_provider_keys.py    | 84 +++++++++++++++----
 .../ai_observability/backend/llm/client.py    | 10 +--
 .../backend/llm/system_one.py                 | 21 +++--
 .../backend/llm/test/test_system_one.py       | 73 +++++++++-------
 .../backend/models/provider_keys.py           |  6 +-
 .../ByokModelPickerNotice.stories.tsx         | 30 +++----
 .../frontend/LLMProviderIcon.test.ts          | 10 ++-
 .../frontend/LLMProviderIcon.tsx              |  3 +
 .../EvaluationExplanation.stories.tsx         |  4 +-
 .../evaluations/AIObservabilityEvaluation.tsx |  6 +-
 .../frontend/generated/api.schemas.ts         | 14 ++--
 .../frontend/generated/api.zod.ts             | 44 +++++-----
 .../frontend/modelPickerLogic.test.ts         |  8 +-
 .../frontend/modelPickerLogic.ts              |  8 +-
 .../settings/LLMProviderKeysSettings.tsx      | 18 ++--
 .../settings/SystemOneConnectionFields.tsx    | 62 +++++++-------
 .../settings/llmProviderKeysLogic.test.ts     |  5 +-
 .../frontend/settings/llmProviderKeysLogic.ts | 10 +--
 .../ai_observability/frontend/utils.test.ts   |  2 +-
 services/mcp/src/api/generated.ts             | 14 ++--
 .../mcp/src/generated/ai_observability/api.ts | 10 +--
 32 files changed, 329 insertions(+), 259 deletions(-)

diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index aab0670882a4..c6539c72f7c8 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -44,17 +44,17 @@ Implementation: [trace judge](../../posthog/temporal/ai_observability/run_trace_
 System One-compatible models are available under the existing LLM judge option.
 The `llm-analytics-system-one-evaluations` project-group feature flag controls access in the browser and background workers.
 Deploy the ingestion and evaluation worker changes before enabling the flag.
-Customers configure their own TypeSafe key or a compatible deployment with its own authentication.
-Evaluation connections never fall back to an instance TypeSafe key or gateway credential.
+Projects configure a System One-compatible deployment and its authentication.
+Evaluation connections never fall back to an instance credential or gateway configuration.
 PostHog's regional AI gateway endpoints additionally require an organization in `POSTHOG_INTERNAL_ORG_IDS`; customer projects cannot use those endpoints.
 Connection validation and every evaluation check these gates; an absent flag or failed flag lookup blocks the call.
 Turning the flag off stops subsequent runs, including queued work, without disabling the saved evaluation.
-Keep the experimental flag limited to staff projects until customer rollout meets the [TypeSafe usage policy](../../posthog/egress/typesafe/README.md#usage-policy).
-Add a connection under **System One (Jev)** in provider key settings.
-The default endpoint is TypeSafe's `https://api.typesafe.ai/v1`, with model `jev-1.13.0` and a TypeSafe API key.
-Advanced configuration accepts a different public HTTPS base URL and model ID for compatible services.
+Keep the experimental flag limited to staff projects during rollout.
+Add a connection under **System One** in provider key settings.
+Enter the public HTTPS base URL and model ID of a compatible service; neither has a default.
+TypeSafe's hosted endpoint is not supported by this integration.
 The client appends `/systemone` to the base URL and sends the API key as a bearer token.
-An empty key selects no authentication for a custom endpoint; TypeSafe requires a key.
+An empty key selects no authentication.
 Changing the endpoint requires entering its credential again, or explicitly choosing no authentication, so an existing key is not forwarded to a new host.
 Private network destinations and redirects are blocked by the shared DNS-pinned HTTP transport.
 Saving a connection validates it with a short synthetic input and a Noul question, without sending evaluation data.
@@ -73,7 +73,7 @@ For boolean evaluations, the prompt becomes a [Noul question](https://docs.types
 A probability of at least 0.5 produces `true`; the evaluation's existing pass/fail polarity still applies.
 The raw probability is stored in `$ai_evaluation_probability`, with available token usage and the resolved model version.
 Missing or invalid token counts remain unknown and do not discard a valid answer.
-Ingestion estimates cost from the reported model and token usage using the existing pricing catalog, including Jev 1.13.0.
+Ingestion estimates cost from the reported model and token usage when the model appears in the existing pricing catalog.
 Models without a catalog match retain their usage with cost left unknown.
 A custom deployment reporting a recognized model name can inherit that model's catalog estimate; this does not measure its hosting cost.
 
@@ -88,7 +88,7 @@ If retries fail, the run fails and the evaluation stays enabled.
 Blocked endpoints and rejected requests disable the evaluation and mark the connection for revalidation, without recording model usage.
 Invalid probabilities, missing answers, and mismatched answer types skip the item as an unparsable response.
 Inputs rejected for exceeding the model's context window are skipped.
-See TypeSafe's [API reference](https://docs.typesafe.ai/api) and [model limits and pricing](https://docs.typesafe.ai/models).
+See TypeSafe's [API reference](https://docs.typesafe.ai/api) for the System One protocol.
 
 ## Model output limits
 
diff --git a/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.test.ts b/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.test.ts
index bdbdea4850c6..f2824fe114c2 100644
--- a/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.test.ts
+++ b/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.test.ts
@@ -5,7 +5,6 @@ describe('manualCosts', () => {
         model: string
         expected: { prompt_token: number; completion_token: number; cache_read_token?: number }
     }> = [
-        { model: 'jev-1.13.0', expected: { prompt_token: 0.000000042, completion_token: 0 } },
         {
             model: 'gpt-4.5',
             expected: {
diff --git a/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.ts b/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.ts
index 4085be25813d..0781a502ea6c 100644
--- a/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.ts
+++ b/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.ts
@@ -1,10 +1,6 @@
 import type { ModelCostRow } from './types'
 
 const manualProviderCosts: ModelCostRow[] = [
-    {
-        model: 'jev-1.13.0',
-        cost: { default: { prompt_token: 0.000000042, completion_token: 0 } },
-    },
     {
         model: 'gpt-4.5',
         cost: {
diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.py
index c76241b06a3a..e33ed205d346 100644
--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py
+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py
@@ -451,14 +451,10 @@ def call_llm_judge(
     is_byok = resolved.is_byok
     key_id = str(provider_key.id) if provider_key else None
 
-    if provider == "typesafe":
+    if provider == "system_one":
         if output_type != "boolean":
             raise ApplicationError("This System One evaluation output type is not supported.", non_retryable=True)
-        base_url = (
-            provider_key.encrypted_config.get("base_url", SystemOneClient.BASE_URL)
-            if provider_key
-            else SystemOneClient.BASE_URL
-        )
+        base_url = provider_key.encrypted_config.get("base_url", "") if provider_key else ""
         if not system_one_evaluations_enabled(team_id, base_url=base_url):
             raise ApplicationError("System One evaluations are not available for this project.", non_retryable=True)
 
@@ -476,7 +472,7 @@ def call_llm_judge(
     probability: float | None = None
     system_one_result = None
     try:
-        if provider == "typesafe":
+        if provider == "system_one":
             if provider_key is not None and provider_key.provider != provider:
                 raise ProviderMismatchError(provider_key.provider, provider)
             prompt = evaluation["evaluation_config"]["prompt"]
diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index ca983a70bccb..68a5701e7c49 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -98,9 +98,9 @@ def _mock_config_with_active_key(provider: str = "openai") -> MagicMock:
     "connection_config,base_url,model,usage",
     [
         (
-            {"api_key": "test-typesafe-key"},
-            "https://api.typesafe.ai/v1",
-            "jev-1.13.0",
+            {"api_key": "example-token", "base_url": "https://decisions.example.com/v1"},
+            "https://decisions.example.com/v1",
+            "example-judge-v1",
             {"input_tokens": 120, "output_tokens": 10},
         ),
         (
@@ -109,14 +109,19 @@ def _mock_config_with_active_key(provider: str = "openai") -> MagicMock:
             "custom-model",
             {"input_tokens": 120},
         ),
-        ({"api_key": "example-token"}, "https://api.typesafe.ai/v1", "jev-1.13.0", {}),
+        (
+            {"api_key": "example-token", "base_url": "https://decisions.example.com/v1"},
+            "https://decisions.example.com/v1",
+            "example-judge-v1",
+            {},
+        ),
     ],
 )
 @pytest.mark.parametrize(
     "probability,applicability,allows_na,verdict",
     [(0.49, 1.0, False, False), (0.5, 1.0, False, True), (0.9, 0.1, True, None), (0.0, 0.9, True, False)],
 )
-def test_typesafe_judge_emits_boolean_probability_without_reasoning(
+def test_system_one_judge_emits_boolean_probability_without_reasoning(
     probability: float,
     applicability: float,
     allows_na: bool,
@@ -126,11 +131,11 @@ def test_typesafe_judge_emits_boolean_probability_without_reasoning(
     model: str,
     usage: dict[str, int],
 ) -> None:
-    key = MagicMock(provider="typesafe", encrypted_config=connection_config)
-    resolved = MagicMock(provider="typesafe", model=model, provider_key=key, is_byok=True)
+    key = MagicMock(provider="system_one", encrypted_config=connection_config)
+    resolved = MagicMock(provider="system_one", model=model, provider_key=key, is_byok=True)
     response = MagicMock(status_code=200)
     response.json.return_value = {
-        "model": "jev-1.13.0",
+        "model": "example-judge-v1",
         "answers": {
             "verdict": {"type": "noul", "noul": probability},
             "applicable": {"type": "noul", "noul": applicability},
@@ -172,7 +177,7 @@ def test_typesafe_judge_emits_boolean_probability_without_reasoning(
     assert properties["$ai_input_tokens"] == usage.get("input_tokens")
     assert properties["$ai_output_tokens"] == usage.get("output_tokens")
     assert properties["$ai_evaluation_probability"] == probability
-    assert properties["$ai_model"] == "jev-1.13.0"
+    assert properties["$ai_model"] == "example-judge-v1"
     assert properties["$ai_evaluation_key_type"] == "byok"
 
 
@@ -187,7 +192,7 @@ def test_system_one_numeric_mapping_is_not_enabled() -> None:
         patch("requests.Session.request") as request,
         pytest.raises(ApplicationError) as error,
     ):
-        spec.return_value.resolve.return_value = MagicMock(provider="typesafe")
+        spec.return_value.resolve.return_value = MagicMock(provider="system_one")
         call_llm_judge(
             evaluation={"team_id": 1, "output_type": "numeric"},
             system_prompt="",
@@ -213,10 +218,10 @@ def test_system_one_restricted_connection_does_not_send_evaluation_data(base_url
     ):
         teams.return_value.get.return_value = Team(id=1, organization_id=uuid.uuid4(), uuid=uuid.uuid4())
         spec.return_value.resolve.return_value = MagicMock(
-            provider="typesafe",
+            provider="system_one",
             model="custom-model",
             provider_key=MagicMock(
-                provider="typesafe", encrypted_config={"base_url": base_url, "api_key": "example-token"}
+                provider="system_one", encrypted_config={"base_url": base_url, "api_key": "example-token"}
             ),
             is_byok=True,
         )
@@ -232,7 +237,10 @@ def test_system_one_restricted_connection_does_not_send_evaluation_data(base_url
 
 @pytest.mark.parametrize("status", [301, 400, 422])
 def test_system_one_rejected_requests_disable_without_model_cost_attribution(status: int) -> None:
-    key = MagicMock(provider="typesafe", encrypted_config={"api_key": "example-token"})
+    key = MagicMock(
+        provider="system_one",
+        encrypted_config={"api_key": "example-token", "base_url": "https://decisions.example.com/v1"},
+    )
     with (
         patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),
         patch("posthog.egress.limiter.backends.LimitsBackend.consume_sync", return_value=True),
@@ -246,7 +254,7 @@ def test_system_one_rejected_requests_disable_without_model_cost_attribution(sta
         ),
     ):
         spec.return_value.resolve.return_value = MagicMock(
-            provider="typesafe", model="jev-1.13.0", provider_key=key, is_byok=True
+            provider="system_one", model="example-judge-v1", provider_key=key, is_byok=True
         )
         result = call_llm_judge(
             evaluation={"id": "test-evaluation", "team_id": 1, "evaluation_config": {"prompt": "Polite?"}},
@@ -263,8 +271,11 @@ def test_system_one_rejected_requests_disable_without_model_cost_attribution(sta
 
 
 @pytest.mark.parametrize("budget_granted", [True, False])
-def test_typesafe_rate_limit_retries_without_disabling_the_evaluation(budget_granted: bool) -> None:
-    key = MagicMock(provider="typesafe", encrypted_config={"api_key": "test-typesafe-key"})
+def test_system_one_rate_limit_retries_without_disabling_the_evaluation(budget_granted: bool) -> None:
+    key = MagicMock(
+        provider="system_one",
+        encrypted_config={"api_key": "example-token", "base_url": "https://decisions.example.com/v1"},
+    )
     with (
         patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),
         patch("posthog.egress.limiter.backends.LimitsBackend.consume_sync", return_value=budget_granted),
@@ -279,7 +290,7 @@ def test_typesafe_rate_limit_retries_without_disabling_the_evaluation(budget_gra
         pytest.raises(ApplicationError) as error,
     ):
         spec.return_value.resolve.return_value = MagicMock(
-            provider="typesafe", model="jev-1.13.0", provider_key=key, is_byok=True
+            provider="system_one", model="example-judge-v1", provider_key=key, is_byok=True
         )
         call_llm_judge(
             evaluation={"team_id": 1, "evaluation_config": {"prompt": "Polite?"}},
diff --git a/products/ai_observability/backend/api/evaluation_config.py b/products/ai_observability/backend/api/evaluation_config.py
index d534df1b7e2a..dc60e83d0798 100644
--- a/products/ai_observability/backend/api/evaluation_config.py
+++ b/products/ai_observability/backend/api/evaluation_config.py
@@ -109,7 +109,7 @@ def set_active_key(self, request: ValidatedRequest, **kwargs) -> Response:
                 status=status.HTTP_404_NOT_FOUND,
             )
 
-        if key.provider == LLMProvider.TYPESAFE:
+        if key.provider == LLMProvider.SYSTEM_ONE:
             return Response(
                 {"detail": "Select the System One connection on an evaluation instead."},
                 status=status.HTTP_400_BAD_REQUEST,
diff --git a/products/ai_observability/backend/api/evaluations.py b/products/ai_observability/backend/api/evaluations.py
index 9c29ddd2a991..7b573b7ad7e9 100644
--- a/products/ai_observability/backend/api/evaluations.py
+++ b/products/ai_observability/backend/api/evaluations.py
@@ -316,7 +316,7 @@ def validate(self, data: dict[str, Any]) -> dict[str, Any]:
         errors = {field: "This field is required." for field in ("provider", "model") if field not in data}
         if errors:
             raise serializers.ValidationError(errors, code="required")
-        if data["provider"] == LLMProvider.TYPESAFE:
+        if data["provider"] == LLMProvider.SYSTEM_ONE:
             if not data.get("provider_key_id"):
                 raise serializers.ValidationError(
                     {"provider_key_id": "Select a System One connection for this evaluation."}
@@ -516,7 +516,7 @@ def validate(self, data):
             if isinstance(model_configuration, dict)
             else getattr(model_configuration, "provider", None)
         )
-        if model_provider == LLMProvider.TYPESAFE and output_type != "boolean":
+        if model_provider == LLMProvider.SYSTEM_ONE and output_type != "boolean":
             raise serializers.ValidationError(
                 {"model_configuration": "Select a model that supports this evaluation output type."}
             )
diff --git a/products/ai_observability/backend/api/provider_keys.py b/products/ai_observability/backend/api/provider_keys.py
index 98dd2abb2830..d5948266d608 100644
--- a/products/ai_observability/backend/api/provider_keys.py
+++ b/products/ai_observability/backend/api/provider_keys.py
@@ -71,8 +71,8 @@ def _reload_model_config_dependents_on_commit(team_id: int, model_config_ids: li
 
 def validate_provider_key(provider: str, api_key: str, *, team_id: int, **kwargs: str) -> tuple[str, str | None]:
     """Validate an API key for any supported provider using the unified client."""
-    if provider == LLMProvider.TYPESAFE and not system_one_evaluations_enabled(
-        team_id, base_url=kwargs.get("base_url", SystemOneClient.BASE_URL)
+    if provider == LLMProvider.SYSTEM_ONE and not system_one_evaluations_enabled(
+        team_id, base_url=kwargs.get("base_url", "")
     ):
         raise exceptions.PermissionDenied("System One evaluations are not available for this project.")
     try:
@@ -97,13 +97,13 @@ def _validation_error_field(provider: str, error_message: str | None) -> str:
 class LLMProviderKeySerializer(serializers.ModelSerializer):
     api_key = serializers.CharField(write_only=True, required=False, allow_blank=True)
     base_url = serializers.URLField(
-        write_only=True, required=False, help_text="System One API base URL, including /v1. Defaults to TypeSafe."
+        write_only=True, required=False, help_text="System One API base URL, including /v1."
     )
     system_one_model = serializers.CharField(
         write_only=True,
         required=False,
         max_length=100,
-        help_text="Model ID served by the System One endpoint. Defaults to jev-1.13.0.",
+        help_text="Model ID served by the System One endpoint.",
     )
     base_url_display = serializers.SerializerMethodField(help_text="Configured System One base URL.")
     system_one_model_display = serializers.SerializerMethodField(help_text="Configured System One model ID.")
@@ -146,16 +146,10 @@ def get_api_key_masked(self, obj: LLMProviderKey) -> str:
         return mask_key_value(obj.encrypted_config.get("api_key", ""))
 
     def get_base_url_display(self, obj: LLMProviderKey) -> str | None:
-        return (
-            obj.encrypted_config.get("base_url", SystemOneClient.BASE_URL)
-            if obj.provider == LLMProvider.TYPESAFE
-            else None
-        )
+        return obj.encrypted_config.get("base_url") if obj.provider == LLMProvider.SYSTEM_ONE else None
 
     def get_system_one_model_display(self, obj: LLMProviderKey) -> str | None:
-        return (
-            obj.encrypted_config.get("model", SystemOneClient.MODEL) if obj.provider == LLMProvider.TYPESAFE else None
-        )
+        return obj.encrypted_config.get("model") if obj.provider == LLMProvider.SYSTEM_ONE else None
 
     def validate_base_url(self, value: str) -> str:
         try:
@@ -197,7 +191,11 @@ def validate(self, data):
             raise serializers.ValidationError({"api_key": "API key is required when creating a new provider key."})
 
         provider = data.get("provider", getattr(self.instance, "provider", None))
-        if provider != LLMProvider.TYPESAFE:
+        if provider == LLMProvider.SYSTEM_ONE and self.instance is None:
+            for field in ("base_url", "system_one_model"):
+                if not data.get(field):
+                    raise serializers.ValidationError({field: "This field is required for System One connections."})
+        if provider != LLMProvider.SYSTEM_ONE:
             if "base_url" in data or "system_one_model" in data:
                 raise serializers.ValidationError(
                     {"base_url": "These settings are only available for System One connections."}
@@ -205,14 +203,14 @@ def validate(self, data):
             if data.get("api_key") == "":
                 raise serializers.ValidationError({"api_key": "An API key is required."})
         elif self.instance is not None and "base_url" in data:
-            current_url = self.instance.encrypted_config.get("base_url", SystemOneClient.BASE_URL)
+            current_url = self.instance.encrypted_config.get("base_url")
             if data["base_url"] != current_url and "api_key" not in data:
                 raise serializers.ValidationError(
                     {"api_key": "Enter the credential for the new endpoint, or an empty value for no authentication."}
                 )
         if self.instance is not None and provider != self.instance.provider:
             raise serializers.ValidationError({"provider": "A key's provider cannot change. Create a new key instead."})
-        if provider == LLMProvider.TYPESAFE and data.get("set_as_active"):
+        if provider == LLMProvider.SYSTEM_ONE and data.get("set_as_active"):
             raise serializers.ValidationError(
                 {"set_as_active": "Select the System One connection on an evaluation instead."}
             )
@@ -231,8 +229,8 @@ def validate(self, data):
 
     def _system_one_config(self, validated_data: dict, current: dict | None = None) -> dict:
         config = dict(current or {})
-        config["base_url"] = validated_data.pop("base_url", config.get("base_url", SystemOneClient.BASE_URL))
-        config["model"] = validated_data.pop("system_one_model", config.get("model", SystemOneClient.MODEL))
+        config["base_url"] = validated_data.pop("base_url", config.get("base_url"))
+        config["model"] = validated_data.pop("system_one_model", config.get("model"))
         return config
 
     def _pop_azure_kwargs(self, validated_data: dict) -> dict:
@@ -276,7 +274,7 @@ def create(self, validated_data):
 
         azure_kwargs = self._normalize_azure_config(provider, azure_kwargs)
 
-        if provider == LLMProvider.TYPESAFE:
+        if provider == LLMProvider.SYSTEM_ONE:
             connection_config = self._system_one_config(validated_data)
             state, error_message = validate_provider_key(provider, api_key or "", team_id=team.id, **connection_config)
             if state != LLMProviderKey.State.OK:
@@ -305,7 +303,7 @@ def create(self, validated_data):
         return instance
 
     def update(self, instance, validated_data):
-        if instance.provider == LLMProvider.TYPESAFE:
+        if instance.provider == LLMProvider.SYSTEM_ONE:
             if any(field in validated_data for field in ("api_key", "base_url", "system_one_model")):
                 config = self._system_one_config(validated_data, instance.encrypted_config)
                 config["api_key"] = validated_data.pop("api_key", config.get("api_key", ""))
@@ -442,7 +440,7 @@ def validate(self, request: Request, **_kwargs) -> Response:
         instance = self.get_object()
         api_key = instance.encrypted_config.get("api_key")
 
-        if not api_key and instance.provider != LLMProvider.TYPESAFE:
+        if not api_key and instance.provider != LLMProvider.SYSTEM_ONE:
             return Response(
                 {"detail": "No API key configured for this provider key."},
                 status=status.HTTP_400_BAD_REQUEST,
diff --git a/products/ai_observability/backend/api/proxy.py b/products/ai_observability/backend/api/proxy.py
index 77bee39e080c..b8658a254a77 100644
--- a/products/ai_observability/backend/api/proxy.py
+++ b/products/ai_observability/backend/api/proxy.py
@@ -67,7 +67,7 @@ def models_cache_key(provider_key_id: str | uuid.UUID) -> str:
 
 
 PROVIDER_DISPLAY_NAMES: dict[str, str] = {
-    "typesafe": "System One (Jev)",
+    "system_one": "System One",
     "openai": "OpenAI",
     "anthropic": "Anthropic",
     "gemini": "Gemini",
@@ -194,7 +194,7 @@ def _get_provider_key(
             raise ValueError("Provider key not found")
 
         api_key = key.encrypted_config.get("api_key")
-        if not api_key and key.provider != LLMProvider.TYPESAFE:
+        if not api_key and key.provider != LLMProvider.SYSTEM_ONE:
             raise ValueError("No API key configured for this provider key")
 
         if touch_last_used:
diff --git a/products/ai_observability/backend/api/test/test_evaluation_config.py b/products/ai_observability/backend/api/test/test_evaluation_config.py
index ba3a43f60b8f..3f2eb94a5a4d 100644
--- a/products/ai_observability/backend/api/test/test_evaluation_config.py
+++ b/products/ai_observability/backend/api/test/test_evaluation_config.py
@@ -36,13 +36,13 @@ def _setup_team():
 
 
 class TestEvaluationConfigViewSet(APIBaseTest):
-    def test_typesafe_cannot_become_the_shared_active_key(self) -> None:
+    def test_system_one_cannot_become_the_shared_active_key(self) -> None:
         key = LLMProviderKey.objects.create(
             team=self.team,
-            provider="typesafe",
-            name="TypeSafe",
+            provider="system_one",
+            name="System One",
             state="ok",
-            encrypted_config={"api_key": "test-typesafe-key"},
+            encrypted_config={"api_key": "example-token"},
         )
         response = self.client.post(
             f"/api/environments/{self.team.id}/llm_analytics/evaluation_config/set_active_key/", {"key_id": str(key.id)}
diff --git a/products/ai_observability/backend/api/test/test_evaluations.py b/products/ai_observability/backend/api/test/test_evaluations.py
index e1a75919621a..abf98dbb71ee 100644
--- a/products/ai_observability/backend/api/test/test_evaluations.py
+++ b/products/ai_observability/backend/api/test/test_evaluations.py
@@ -117,18 +117,18 @@ def test_numeric_evaluation_rejects_system_one_connection(self) -> None:
         )
         serializer = EvaluationSerializer(instance=evaluation, partial=True)
         with self.assertRaisesMessage(ValidationError, "Select a model that supports this evaluation output type"):
-            serializer.validate({"model_configuration": {"provider": "typesafe", "model": "custom-model"}})
+            serializer.validate({"model_configuration": {"provider": "system_one", "model": "custom-model"}})
 
     @parameterized.expand(
         [
-            ("missing_key", "jev-1.13.0", None, False),
+            ("missing_key", "example-judge-v1", None, False),
             ("custom_model", "other-model", str(uuid4()), True),
-            ("configured", "jev-1.13.0", str(uuid4()), True),
+            ("configured", "example-judge-v1", str(uuid4()), True),
         ]
     )
     def test_system_one_requires_explicit_key(self, _name: str, model: str, key_id: str | None, valid: bool) -> None:
         serializer = ModelConfigurationSerializer(
-            data={"provider": "typesafe", "model": model, "provider_key_id": key_id}
+            data={"provider": "system_one", "model": model, "provider_key_id": key_id}
         )
         self.assertEqual(serializer.is_valid(), valid, serializer.errors)
 
@@ -300,7 +300,7 @@ def test_unauthenticated_user_cannot_access_evaluation_configs(self):
         response = self.client.get(f"/api/environments/{self.team.id}/evaluations/")
         self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)
 
-    @parameterized.expand([("openai", "gpt-5-mini"), ("typesafe", "jev-1.13.0")])
+    @parameterized.expand([("openai", "gpt-5-mini"), ("system_one", "example-judge-v1")])
     def test_can_create_evaluation_config(self, provider: str, model: str) -> None:
         key = LLMProviderKey.objects.create(
             team=self.team,
@@ -320,7 +320,7 @@ def test_can_create_evaluation_config(self, provider: str, model: str) -> None:
                 "enabled": True,
                 "evaluation_type": "llm_judge",
                 "model_configuration": {"provider": provider, "model": model, "provider_key_id": str(key.id)}
-                if provider == "typesafe"
+                if provider == "system_one"
                 else _DEFAULT_MODEL_CONFIGURATION,
                 "evaluation_config": {"prompt": "Test prompt"},
                 "output_type": "boolean",
diff --git a/products/ai_observability/backend/api/test/test_provider_keys.py b/products/ai_observability/backend/api/test/test_provider_keys.py
index 7278e5364705..25733ee21133 100644
--- a/products/ai_observability/backend/api/test/test_provider_keys.py
+++ b/products/ai_observability/backend/api/test/test_provider_keys.py
@@ -20,7 +20,6 @@
 from products.ai_observability.backend.api.taggers import TaggerModelConfigurationWriteSerializer
 from products.ai_observability.backend.llm.client import Client
 from products.ai_observability.backend.llm.providers.azure_openai import DEFAULT_API_VERSION
-from products.ai_observability.backend.llm.system_one import SystemOneClient
 from products.ai_observability.backend.models.evaluation_config import EvaluationConfig
 from products.ai_observability.backend.models.evaluations import Evaluation
 from products.ai_observability.backend.models.model_configuration import LLMModelConfiguration
@@ -30,11 +29,11 @@
 
 class TestProviderKeySerializer(SimpleTestCase):
     @parameterized.expand([(LLMProxyCompletionSerializer,), (TaggerModelConfigurationWriteSerializer,)])
-    def test_typesafe_is_not_a_completion_provider(self, serializer_class: type[serializers.Serializer]) -> None:
+    def test_system_one_is_not_a_completion_provider(self, serializer_class: type[serializers.Serializer]) -> None:
         serializer = serializer_class(
             data={
-                "provider": "typesafe",
-                "model": "jev-1.13.0",
+                "provider": "system_one",
+                "model": "example-judge-v1",
                 "provider_key_id": str(uuid4()),
                 "system": "Reply politely.",
                 "messages": [{"role": "user", "content": "Hello!"}],
@@ -43,23 +42,60 @@ def test_typesafe_is_not_a_completion_provider(self, serializer_class: type[seri
         self.assertFalse(serializer.is_valid())
         self.assertIn("provider", serializer.errors)
 
-    @parameterized.expand([("openai", "typesafe"), ("typesafe", "openai")])
+    @parameterized.expand([("openai", "system_one"), ("system_one", "openai")])
     def test_cannot_change_provider_of_an_existing_key(self, current: str, requested: str) -> None:
         key = LLMProviderKey(provider=current, state="ok", encrypted_config={"api_key": "test-key"})
         serializer = LLMProviderKeySerializer(key, data={"provider": requested}, partial=True)
         self.assertFalse(serializer.is_valid())
         self.assertIn("provider", serializer.errors)
 
-    def test_typesafe_cannot_be_created_as_the_shared_key(self) -> None:
+    def test_system_one_cannot_be_created_as_the_shared_key(self) -> None:
         serializer = LLMProviderKeySerializer(
-            data={"provider": "typesafe", "name": "TypeSafe", "api_key": "test-key", "set_as_active": True}
+            data={
+                "provider": "system_one",
+                "name": "Example connection",
+                "api_key": "example-token",
+                "base_url": "https://decisions.example.com/v1",
+                "system_one_model": "custom-model",
+                "set_as_active": True,
+            }
         )
         self.assertFalse(serializer.is_valid())
         self.assertIn("set_as_active", serializer.errors)
 
-    @parameterized.expand([("https://decisions.example.com/v1", False), (SystemOneClient.BASE_URL, True)])
+    @parameterized.expand([("base_url",), ("system_one_model",)])
+    def test_system_one_connection_requires_endpoint_and_model(self, missing_field: str) -> None:
+        data = {
+            "provider": "system_one",
+            "name": "Example connection",
+            "api_key": "",
+            "base_url": "https://decisions.example.com/v1",
+            "system_one_model": "custom-model",
+        }
+        del data[missing_field]
+        serializer = LLMProviderKeySerializer(data=data)
+        self.assertFalse(serializer.is_valid())
+        self.assertIn(missing_field, serializer.errors)
+
+    def test_system_one_connection_rejects_hosted_typesafe_endpoint(self) -> None:
+        serializer = LLMProviderKeySerializer(
+            data={
+                "provider": "system_one",
+                "name": "Example connection",
+                "api_key": "example-token",
+                "base_url": "https://api.typesafe.ai/v1",
+                "system_one_model": "example-judge-v1",
+            }
+        )
+        self.assertFalse(serializer.is_valid())
+        self.assertIn("base_url", serializer.errors)
+
+    @parameterized.expand([("https://other.example.com/v1", False), ("https://decisions.example.com/v1", True)])
     def test_endpoint_change_requires_explicit_credentials(self, base_url: str, valid: bool) -> None:
-        key = LLMProviderKey(provider="typesafe", encrypted_config={"api_key": "example-token"})
+        key = LLMProviderKey(
+            provider="system_one",
+            encrypted_config={"api_key": "example-token", "base_url": "https://decisions.example.com/v1"},
+        )
         serializer = LLMProviderKeySerializer(key, data={"base_url": base_url}, partial=True)
         self.assertEqual(serializer.is_valid(), valid, serializer.errors)
 
@@ -127,7 +163,7 @@ def test_system_one_customer_project_cannot_use_posthog_gateway_or_disabled_feat
         gateway_url = "https://ai-gateway.us.posthog.com/v1"
         key = LLMProviderKey.objects.create(
             team=self.team,
-            provider="typesafe",
+            provider="system_one",
             name="Example connection",
             encrypted_config={"api_key": "example-token", "base_url": gateway_url},
             created_by=self.user,
@@ -145,10 +181,11 @@ def test_system_one_customer_project_cannot_use_posthog_gateway_or_disabled_feat
                 response = self.client.post(
                     f"{base_url}provider_keys/",
                     {
-                        "provider": "typesafe",
+                        "provider": "system_one",
                         "name": "Example connection",
                         "api_key": "example-token",
                         "base_url": gateway_url,
+                        "system_one_model": "custom-model",
                     },
                 )
             elif operation == "update":
@@ -157,7 +194,7 @@ def test_system_one_customer_project_cannot_use_posthog_gateway_or_disabled_feat
                 response = self.client.post(f"{base_url}provider_keys/{key.id}/validate/")
             else:
                 response = self.client.post(
-                    f"{base_url}provider_key_validations/", {"provider": "typesafe", "api_key": "example-token"}
+                    f"{base_url}provider_key_validations/", {"provider": "system_one", "api_key": "example-token"}
                 )
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN, response.data)
         request.assert_not_called()
@@ -169,14 +206,23 @@ def test_unauthenticated_user_cannot_access_provider_keys(self):
         response = self.client.get(f"/api/environments/{self.team.id}/llm_analytics/provider_keys/")
         self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)
 
-    @parameterized.expand([("openai",), ("typesafe",)])
+    @parameterized.expand([("openai",), ("system_one",)])
     @patch("products.ai_observability.backend.api.provider_keys.validate_provider_key")
     def test_can_create_provider_key(self, provider: str, mock_validate: Mock) -> None:
         mock_validate.return_value = (LLMProviderKey.State.OK, None)
 
         response = self.client.post(
             f"/api/environments/{self.team.id}/llm_analytics/provider_keys/",
-            {"provider": provider, "name": "My Key", "api_key": "sk-test-key-12345"},
+            {
+                "provider": provider,
+                "name": "My Key",
+                "api_key": "sk-test-key-12345",
+                **(
+                    {"base_url": "https://decisions.example.com/v1", "system_one_model": "custom-model"}
+                    if provider == "system_one"
+                    else {}
+                ),
+            },
         )
         self.assertEqual(response.status_code, status.HTTP_201_CREATED)
         self.assertEqual(LLMProviderKey.objects.count(), 1)
@@ -192,7 +238,9 @@ def test_can_create_provider_key(self, provider: str, mock_validate: Mock) -> No
         self.assertEqual(response.data["api_key_masked"], "sk-t...2345")
         self.assertNotIn("api_key", response.data)
         expected_config = (
-            {"base_url": SystemOneClient.BASE_URL, "model": SystemOneClient.MODEL} if provider == "typesafe" else {}
+            {"base_url": "https://decisions.example.com/v1", "model": "custom-model"}
+            if provider == "system_one"
+            else {}
         )
         mock_validate.assert_called_once_with(provider, "sk-test-key-12345", team_id=self.team.id, **expected_config)
 
@@ -213,7 +261,7 @@ def test_custom_system_one_connection_round_trip(
         response = self.client.post(
             url,
             {
-                "provider": "typesafe",
+                "provider": "system_one",
                 "name": "Custom",
                 "api_key": "",
                 "base_url": "https://decisions.example.com/v1/",
@@ -223,8 +271,8 @@ def test_custom_system_one_connection_round_trip(
         self.assertEqual(response.status_code, 201, response.data)
         self.assertEqual(response.data["base_url_display"], "https://decisions.example.com/v1")
         key = LLMProviderKey.objects.get(id=response.data["id"])
-        self.assertEqual(Client.list_models("typesafe", **key.provider_extra_kwargs()), ["custom-model"])
-        model_config = LLMModelConfiguration(provider="typesafe", model="custom-model", provider_key=key)
+        self.assertEqual(Client.list_models("system_one", **key.provider_extra_kwargs()), ["custom-model"])
+        model_config = LLMModelConfiguration(provider="system_one", model="custom-model", provider_key=key)
         self.assertEqual(model_config.get_available_models(), ["custom-model"])
         self.assertNotIn("Authorization", request.call_args.kwargs["headers"])
         request.reset_mock()
diff --git a/products/ai_observability/backend/llm/client.py b/products/ai_observability/backend/llm/client.py
index 6227f80a2b01..ba4308e8491a 100644
--- a/products/ai_observability/backend/llm/client.py
+++ b/products/ai_observability/backend/llm/client.py
@@ -82,22 +82,22 @@ def _resolve_credentials(self) -> tuple[str | None, str | None]:
     @classmethod
     def validate_key(cls, provider: str, api_key: str, **kwargs: Any) -> tuple[str, str | None]:
         """Validate an API key for a provider. Returns (state, error_message)."""
-        if provider == "typesafe":
+        if provider == "system_one":
             return SystemOneClient.validate_key(api_key, **kwargs)
         return _get_provider(provider).validate_key(api_key, **kwargs)
 
     @classmethod
     def list_models(cls, provider: str, api_key: str | None = None, **kwargs: Any) -> list[str]:
         """List available models for a provider."""
-        if provider == "typesafe":
-            return [kwargs.get("model", SystemOneClient.MODEL)]
+        if provider == "system_one":
+            return [kwargs["model"]] if kwargs.get("model") else []
         return _get_provider(provider).list_models(api_key, **kwargs)
 
     @classmethod
     def recommended_models(cls, provider: str) -> set[str]:
         """Return the set of curated/recommended model IDs for a provider."""
-        if provider == "typesafe":
-            return {SystemOneClient.MODEL}
+        if provider == "system_one":
+            return set()
         return _get_provider(provider).recommended_models()
 
 
diff --git a/products/ai_observability/backend/llm/system_one.py b/products/ai_observability/backend/llm/system_one.py
index 23d487b29d02..e5baf031bbe8 100644
--- a/products/ai_observability/backend/llm/system_one.py
+++ b/products/ai_observability/backend/llm/system_one.py
@@ -9,7 +9,7 @@
 import requests
 
 from posthog.egress.limiter.policies import Priority
-from posthog.egress.typesafe.client import TYPESAFE_API_BASE, TypeSafeNotConfigured, TypeSafeRequestFailed
+from posthog.egress.typesafe.client import TypeSafeRequestFailed
 from posthog.egress.typesafe.transport import TypeSafeEgressBudgetExhausted
 from posthog.llm.system_one import JsonValue, NoulQuestion, Question, SystemOneResult
 from posthog.llm.system_one_client import TypeSafeSystemOneClient
@@ -34,7 +34,9 @@
 def system_one_evaluations_enabled(team_id: int, *, base_url: str) -> bool:
     try:
         host = (urlsplit(base_url).hostname or "").encode("idna").decode("ascii").lower().rstrip(".")
-    except ValueError:
+    except (ValueError, UnicodeError):
+        return False
+    if not host or host == "typesafe.ai" or host.endswith(".typesafe.ai"):
         return False
     team = Team.objects.only("uuid", "organization_id").get(id=team_id)
     # Customer connections use their own provider account or deployment, never PostHog's gateway.
@@ -75,9 +77,6 @@ def __init__(self, retry_after: str | None) -> None:
 
 
 class SystemOneClient:
-    MODEL = "jev-1.13.0"
-    BASE_URL = f"{TYPESAFE_API_BASE}/v1"
-
     @staticmethod
     def normalize_base_url(base_url: str) -> str:
         parsed = urlsplit(base_url)
@@ -91,6 +90,12 @@ def normalize_base_url(base_url: str) -> str:
             or parsed.fragment
         ):
             raise ValueError("Use an HTTPS base URL without credentials, a query, or a fragment.")
+        try:
+            host = parsed.hostname.encode("idna").decode("ascii").lower().rstrip(".")
+        except UnicodeError as error:
+            raise ValueError("Use a valid HTTPS hostname.") from error
+        if host == "typesafe.ai" or host.endswith(".typesafe.ai"):
+            raise ValueError("This hosted endpoint is not available for evaluations.")
         return base_url.rstrip("/")
 
     @staticmethod
@@ -100,7 +105,7 @@ def evaluate(
         model: str,
         state: JsonValue,
         questions: Mapping[str, Question],
-        base_url: str = BASE_URL,
+        base_url: str,
         priority: Priority = Priority.BATCH,
     ) -> SystemOneResult:
         base_url = SystemOneClient.normalize_base_url(base_url)
@@ -113,8 +118,6 @@ def evaluate(
                 priority=priority,
                 timeout=60,
             ).decide(state=state, questions=questions)
-        except TypeSafeNotConfigured as error:
-            raise AuthenticationError("A TypeSafe API key is required.") from error
         except TypeSafeEgressBudgetExhausted as error:
             raise SystemOneRateLimitError(None) from error
         except SSRFBlockedError as error:
@@ -153,7 +156,7 @@ def evaluate(
             ) from error
 
     @staticmethod
-    def validate_key(api_key: str, *, base_url: str = BASE_URL, model: str = MODEL) -> tuple[str, str | None]:
+    def validate_key(api_key: str, *, base_url: str, model: str) -> tuple[str, str | None]:
         try:
             SystemOneClient.evaluate(
                 api_key=api_key,
diff --git a/products/ai_observability/backend/llm/test/test_system_one.py b/products/ai_observability/backend/llm/test/test_system_one.py
index 9fc4e4a4c5fa..501b0116d952 100644
--- a/products/ai_observability/backend/llm/test/test_system_one.py
+++ b/products/ai_observability/backend/llm/test/test_system_one.py
@@ -38,10 +38,11 @@ def isolated_egress_budget() -> Iterator[None]:
 @pytest.mark.parametrize(
     "base_url,internal,flag,enabled",
     [
-        (SystemOneClient.BASE_URL, False, True, True),
+        ("https://api.typesafe.ai/v1", False, True, False),
+        ("", False, True, False),
         ("https://decisions.example.com/v1", False, True, True),
         ("https://decisions.example.com/v1", False, False, False),
-        (SystemOneClient.BASE_URL, False, None, False),
+        ("https://api.typesafe.ai/v1", False, None, False),
         ("https://ai-gateway.us.posthog.com/v1", False, True, False),
         ("https://ai-gateway.eu.posthog.com/v1", False, True, False),
         ("https://AI-GATEWAY.US.POSTHOG.COM.:443/v1", False, True, False),
@@ -64,27 +65,29 @@ def test_system_one_connections_require_flag_and_reserve_posthog_gateway_for_int
 
 
 @pytest.mark.parametrize("status, expected_state", [(200, "ok"), (401, "invalid"), (403, "invalid"), (500, "error")])
-def test_typesafe_key_validation(status: int, expected_state: str) -> None:
+def test_system_one_key_validation(status: int, expected_state: str) -> None:
     response = Mock(status_code=status)
     response.json.return_value = {
-        "model": "jev-1.13.0",
+        "model": "example-judge-v1",
         "answers": {"verdict": {"type": "noul", "noul": 0.9}, "applicable": {"type": "noul", "noul": 0.9}},
         "usage": {"input_tokens": 12, "output_tokens": 0},
     }
     with patch("requests.Session.request", return_value=response) as request:
-        state, message = Client.validate_key("typesafe", "test-typesafe-key")
+        state, message = Client.validate_key(
+            "system_one", "example-token", base_url="https://decisions.example.com/v1", model="custom-model"
+        )
 
     assert state == expected_state
     assert (message is None) == (expected_state == "ok")
-    assert request.call_args.args == ("POST", "https://api.typesafe.ai/v1/systemone")
-    assert request.call_args.kwargs["headers"]["Authorization"] == "Bearer test-typesafe-key"
+    assert request.call_args.args == ("POST", "https://decisions.example.com/v1/systemone")
+    assert request.call_args.kwargs["headers"]["Authorization"] == "Bearer example-token"
 
 
 @pytest.mark.parametrize("probability", [-0.1, 1.1, float("nan"), float("inf"), "0.8", True, None])
-def test_typesafe_rejects_invalid_probabilities(probability: object) -> None:
+def test_system_one_rejects_invalid_probabilities(probability: object) -> None:
     response = Mock(status_code=200)
     response.json.return_value = {
-        "model": "jev-1.13.0",
+        "model": "example-judge-v1",
         "answers": {"verdict": {"type": "noul", "noul": probability}},
         "usage": {"input_tokens": 120, "output_tokens": 10},
     }
@@ -93,23 +96,25 @@ def test_typesafe_rejects_invalid_probabilities(probability: object) -> None:
         pytest.raises(StructuredOutputParseError),
     ):
         SystemOneClient.evaluate(
-            api_key="test-typesafe-key",
-            model="jev-1.13.0",
+            api_key="example-token",
+            base_url="https://decisions.example.com/v1",
+            model="example-judge-v1",
             state="Hello!",
             questions={"verdict": NoulQuestion(instructions="Is the response polite?")},
         )
 
 
 @pytest.mark.parametrize("status", [429, 503, 529])
-def test_typesafe_rate_limits_are_retryable(status: int) -> None:
+def test_system_one_rate_limits_are_retryable(status: int) -> None:
     response = Mock(status_code=status, headers={"Retry-After": "15"})
     with (
         patch("requests.Session.request", return_value=response),
         pytest.raises(SystemOneRateLimitError) as error,
     ):
         SystemOneClient.evaluate(
-            api_key="test-typesafe-key",
-            model="jev-1.13.0",
+            api_key="example-token",
+            base_url="https://decisions.example.com/v1",
+            model="example-judge-v1",
             state="Hello!",
             questions={"verdict": NoulQuestion(instructions="Is the response polite?")},
         )
@@ -129,14 +134,15 @@ def test_unavailable_usage_does_not_discard_a_valid_answer(
 ) -> None:
     response = Mock(status_code=200)
     response.json.return_value = {
-        "model": "jev-1.13.0",
+        "model": "example-judge-v1",
         "answers": {"verdict": {"type": "noul", "noul": 0.9}},
         "usage": usage,
     }
     with patch("requests.Session.request", return_value=response):
         result = SystemOneClient.evaluate(
             api_key="example-token",
-            model="jev-1.13.0",
+            base_url="https://decisions.example.com/v1",
+            model="example-judge-v1",
             state="Hello!",
             questions={"verdict": NoulQuestion(instructions="Polite?")},
         )
@@ -145,23 +151,30 @@ def test_unavailable_usage_does_not_discard_a_valid_answer(
     assert result.output_tokens == expected_output
 
 
-@override_settings(TYPESAFE_API_KEY="example-instance-key")
-def test_typesafe_requires_a_key() -> None:
+@pytest.mark.parametrize(
+    "base_url",
+    ["https://api.typesafe.ai/v1", "https://API.TYPESAFE.AI.:443/v1", "https://ａｐｉ.typesafe.ai/v1"],
+)
+def test_official_endpoint_is_blocked(base_url: str) -> None:
     with (
         patch("requests.Session.request") as request,
-        pytest.raises(AuthenticationError),
+        pytest.raises(ValueError, match="hosted endpoint is not available"),
     ):
         SystemOneClient.evaluate(
-            api_key="", model="jev-1.13.0", state="Hello!", questions={"verdict": NoulQuestion(instructions="Polite?")}
+            api_key="example-token",
+            base_url=base_url,
+            model="example-judge-v1",
+            state="Hello!",
+            questions={"verdict": NoulQuestion(instructions="Polite?")},
         )
     request.assert_not_called()
 
 
 @pytest.mark.parametrize("answers", [{}, {"verdict": {"type": "noul", "noul": 0.9}}])
-def test_typesafe_requires_every_requested_answer(answers: dict[str, object]) -> None:
+def test_system_one_requires_every_requested_answer(answers: dict[str, object]) -> None:
     response = Mock(status_code=200)
     response.json.return_value = {
-        "model": "jev-1.13.0",
+        "model": "example-judge-v1",
         "answers": answers,
         "usage": {"input_tokens": 12, "output_tokens": 2},
     }
@@ -170,8 +183,9 @@ def test_typesafe_requires_every_requested_answer(answers: dict[str, object]) ->
         pytest.raises(StructuredOutputParseError),
     ):
         SystemOneClient.evaluate(
-            api_key="test-typesafe-key",
-            model="jev-1.13.0",
+            api_key="example-token",
+            base_url="https://decisions.example.com/v1",
+            model="example-judge-v1",
             state="Hello!",
             questions={
                 "verdict": NoulQuestion(instructions="Polite?"),
@@ -191,15 +205,16 @@ def test_typesafe_requires_every_requested_answer(answers: dict[str, object]) ->
         (422, "Invalid question", SystemOneRequestRejectedError),
     ],
 )
-def test_typesafe_preserves_error_categories(status: int, message: str, error_type: type[Exception]) -> None:
+def test_system_one_preserves_error_categories(status: int, message: str, error_type: type[Exception]) -> None:
     response = Mock(status_code=status, text=message)
     with (
         patch("requests.Session.request", return_value=response),
         pytest.raises(error_type),
     ):
         SystemOneClient.evaluate(
-            api_key="test-typesafe-key",
-            model="jev-1.13.0",
+            api_key="example-token",
+            base_url="https://decisions.example.com/v1",
+            model="example-judge-v1",
             state="Hello!",
             questions={"verdict": NoulQuestion(instructions="Polite?")},
         )
@@ -252,7 +267,7 @@ def test_custom_endpoint_and_model(api_key: str) -> None:
 )
 def test_invalid_endpoint_is_rejected_before_sending_credentials(base_url: str) -> None:
     with patch("requests.Session.request") as request:
-        state, _ = SystemOneClient.validate_key("example-token", base_url=base_url)
+        state, _ = SystemOneClient.validate_key("example-token", base_url=base_url, model="custom-model")
     assert state == "error"
     request.assert_not_called()
 
@@ -263,6 +278,6 @@ def test_private_endpoint_is_blocked() -> None:
         patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("127.0.0.1")}),
         patch("requests.Session.request") as request,
     ):
-        state, _ = SystemOneClient.validate_key("example-token", base_url="https://127.0.0.1/v1")
+        state, _ = SystemOneClient.validate_key("example-token", base_url="https://127.0.0.1/v1", model="custom-model")
     assert state == "error"
     request.assert_not_called()
diff --git a/products/ai_observability/backend/models/provider_keys.py b/products/ai_observability/backend/models/provider_keys.py
index f2077b51f7b1..39f0fa30b012 100644
--- a/products/ai_observability/backend/models/provider_keys.py
+++ b/products/ai_observability/backend/models/provider_keys.py
@@ -22,7 +22,7 @@ class LLMProvider(models.TextChoices):
     TOGETHER_AI = "together_ai", "Together AI"
     MINIMAX = "minimax", "MiniMax"
     ZEABUR = "zeabur", "Zeabur AI Hub"
-    TYPESAFE = "typesafe", "System One (Jev)"
+    SYSTEM_ONE = "system_one", "System One"
 
 
 def llm_provider_choices() -> list[tuple[str, str | Promise]]:
@@ -31,7 +31,7 @@ def llm_provider_choices() -> list[tuple[str, str | Promise]]:
 
 
 def llm_completion_provider_choices() -> list[tuple[str, str | Promise]]:
-    return [(provider, label) for provider, label in LLMProvider.choices if provider != LLMProvider.TYPESAFE]
+    return [(provider, label) for provider, label in LLMProvider.choices if provider != LLMProvider.SYSTEM_ONE]
 
 
 class LLMProviderKey(UUIDTModel):
@@ -67,7 +67,7 @@ def provider_extra_kwargs(self) -> dict[str, Any]:
         extra config (e.g. Azure's ``azure_endpoint`` and ``api_version``). Most
         providers return an empty dict.
         """
-        if self.provider == LLMProvider.TYPESAFE:
+        if self.provider == LLMProvider.SYSTEM_ONE:
             return {
                 field: self.encrypted_config[field] for field in ("base_url", "model") if field in self.encrypted_config
             }
diff --git a/products/ai_observability/frontend/ByokModelPickerNotice.stories.tsx b/products/ai_observability/frontend/ByokModelPickerNotice.stories.tsx
index a16df5161d64..64ff6f187dd7 100644
--- a/products/ai_observability/frontend/ByokModelPickerNotice.stories.tsx
+++ b/products/ai_observability/frontend/ByokModelPickerNotice.stories.tsx
@@ -11,14 +11,14 @@ import { LLMProviderKey, LLMProviderKeyState } from './settings/llmProviderKeysL
 interface StoryArgs {
     keyState: LLMProviderKeyState | null
     modelsFail: boolean
-    typesafe?: boolean
+    systemOne?: boolean
 }
 
 function providerKey(state: LLMProviderKeyState): Partial<LLMProviderKey> {
     return { id: 'key-1', provider: 'openai', name: 'Production', state }
 }
 
-function PickerWithNotice({ typesafe = false }: { typesafe?: boolean }): JSX.Element {
+function PickerWithNotice({ systemOne = false }: { systemOne?: boolean }): JSX.Element {
     const { providerModelGroups, evaluationProviderModelGroups, hasByokKeys, byokModelsLoading, providerKeysLoading } =
         useValues(modelPickerLogic)
 
@@ -29,20 +29,20 @@ function PickerWithNotice({ typesafe = false }: { typesafe?: boolean }): JSX.Ele
                 model=""
                 selectedProviderKeyId={null}
                 onSelect={() => {}}
-                groups={typesafe ? evaluationProviderModelGroups : providerModelGroups}
+                groups={systemOne ? evaluationProviderModelGroups : providerModelGroups}
                 loading={byokModelsLoading || providerKeysLoading}
                 footerLink={getModelPickerFooterLink(
-                    typesafe ? evaluationProviderModelGroups.some((group) => !group.disabledReason) : hasByokKeys
+                    systemOne ? evaluationProviderModelGroups.some((group) => !group.disabledReason) : hasByokKeys
                 )}
             />
-            <ByokModelPickerNotice forEvaluation={typesafe} />
+            <ByokModelPickerNotice forEvaluation={systemOne} />
         </div>
     )
 }
 
 const meta: Meta<StoryArgs> = {
     title: 'Scenes-App/AI observability/BYOK model picker notice',
-    render: ({ keyState, modelsFail, typesafe }) => {
+    render: ({ keyState, modelsFail, systemOne }) => {
         useStorybookMocks({
             get: {
                 '/api/environments/:team_id/llm_analytics/provider_keys/': {
@@ -50,7 +50,7 @@ const meta: Meta<StoryArgs> = {
                         ? [
                               {
                                   ...providerKey(keyState),
-                                  ...(typesafe ? { provider: 'typesafe', name: 'TypeSafe' } : {}),
+                                  ...(systemOne ? { provider: 'system_one', name: 'System One' } : {}),
                               },
                           ]
                         : [],
@@ -62,13 +62,13 @@ const meta: Meta<StoryArgs> = {
                         ? [500, { error: 'Internal error' }]
                         : [
                               200,
-                              typesafe
+                              systemOne
                                   ? [
                                         {
-                                            id: 'jev-1.13.0',
-                                            name: 'jev-1.13.0',
-                                            provider: 'TypeSafe',
-                                            is_recommended: true,
+                                            id: 'example-judge-v1',
+                                            name: 'example-judge-v1',
+                                            provider: 'System One',
+                                            is_recommended: false,
                                         },
                                     ]
                                   : [],
@@ -76,7 +76,7 @@ const meta: Meta<StoryArgs> = {
             },
         })
 
-        return <PickerWithNotice typesafe={typesafe} />
+        return <PickerWithNotice systemOne={systemOne} />
     },
 }
 export default meta
@@ -93,6 +93,6 @@ export const ModelsFailedToLoad: StoryObj<StoryArgs> = {
     args: { keyState: 'ok', modelsFail: true },
 }
 
-export const Jev: StoryObj<StoryArgs> = {
-    args: { keyState: 'ok', modelsFail: false, typesafe: true },
+export const SystemOne: StoryObj<StoryArgs> = {
+    args: { keyState: 'ok', modelsFail: false, systemOne: true },
 }
diff --git a/products/ai_observability/frontend/LLMProviderIcon.test.ts b/products/ai_observability/frontend/LLMProviderIcon.test.ts
index a88be4a18611..ebb2025e72f0 100644
--- a/products/ai_observability/frontend/LLMProviderIcon.test.ts
+++ b/products/ai_observability/frontend/LLMProviderIcon.test.ts
@@ -1,4 +1,7 @@
-import { LLM_PROVIDER_SELECT_OPTIONS } from './LLMProviderIcon'
+import { createElement } from 'react'
+import { renderToStaticMarkup } from 'react-dom/server'
+
+import { LLMProviderIcon, LLM_PROVIDER_SELECT_OPTIONS } from './LLMProviderIcon'
 import { LLM_PROVIDER_LABELS } from './settings/llmProviderKeysLogic'
 
 describe('LLM_PROVIDER_SELECT_OPTIONS', () => {
@@ -10,4 +13,9 @@ describe('LLM_PROVIDER_SELECT_OPTIONS', () => {
             expect(option.icon).toBeTruthy()
         }
     })
+
+    it('shows a generic icon for System One', () => {
+        expect(LLM_PROVIDER_LABELS.system_one).toBe('System One')
+        expect(renderToStaticMarkup(createElement(LLMProviderIcon, { provider: 'system_one' }))).toContain('<svg')
+    })
 })
diff --git a/products/ai_observability/frontend/LLMProviderIcon.tsx b/products/ai_observability/frontend/LLMProviderIcon.tsx
index 721a607d4bd0..24e973d6ac1e 100644
--- a/products/ai_observability/frontend/LLMProviderIcon.tsx
+++ b/products/ai_observability/frontend/LLMProviderIcon.tsx
@@ -1,3 +1,5 @@
+import { IconCloud } from '@posthog/icons'
+
 import { AnthropicLogo } from 'scenes/onboarding/shared/logos/AnthropicLogo'
 import geminiImage from 'scenes/onboarding/shared/logos/gemini.svg'
 import { OpenAILogo } from 'scenes/onboarding/shared/logos/OpenAILogo'
@@ -21,6 +23,7 @@ const PROVIDER_COMPONENTS: Partial<Record<LLMProvider, React.ComponentType>> = {
     openai: OpenAILogo,
     openrouter: OpenRouterLogo,
     anthropic: AnthropicLogo,
+    system_one: IconCloud,
 }
 
 export function LLMProviderIcon({
diff --git a/products/ai_observability/frontend/components/EvaluationExplanation.stories.tsx b/products/ai_observability/frontend/components/EvaluationExplanation.stories.tsx
index 308817da1549..4ccb761c6186 100644
--- a/products/ai_observability/frontend/components/EvaluationExplanation.stories.tsx
+++ b/products/ai_observability/frontend/components/EvaluationExplanation.stories.tsx
@@ -12,10 +12,10 @@ export const Reasoning: StoryObj<typeof EvaluationExplanation> = {
     args: { reasoning: 'The response answers the question and includes a polite greeting.' },
 }
 
-export const Jev: StoryObj<typeof EvaluationExplanation> = {
+export const SystemOne: StoryObj<typeof EvaluationExplanation> = {
     args: { probability: 0.84 },
 }
 
-export const JevZeroProbability: StoryObj<typeof EvaluationExplanation> = {
+export const SystemOneZeroProbability: StoryObj<typeof EvaluationExplanation> = {
     args: { probability: 0 },
 }
diff --git a/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx b/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx
index dbd2f01f9b74..6b52c05db9d5 100644
--- a/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx
+++ b/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx
@@ -136,7 +136,7 @@ export function AIObservabilityEvaluation(): JSX.Element {
     const openInPlaygroundUrl =
         evaluationTypeUsesModelConfiguration(evaluation.evaluation_type) &&
         evaluation.id &&
-        evaluation.model_configuration?.provider !== 'typesafe'
+        evaluation.model_configuration?.provider !== 'system_one'
             ? combineUrl(urls.aiObservabilityPlayground(), { source_evaluation_id: evaluation.id }).url
             : null
 
@@ -930,7 +930,7 @@ function EvaluationModelPicker(): JSX.Element {
     // Evals always run on the team's own provider key, so only BYOK models are offered.
     const selectedModelName = byokModels.find((m) => m.id === selectedModel)?.name
     const groups = evaluationProviderModelGroups.filter(
-        (group) => evaluation?.output_type === 'boolean' || group.provider !== 'typesafe'
+        (group) => evaluation?.output_type === 'boolean' || group.provider !== 'system_one'
     )
     const loading = byokModelsLoading || providerKeysLoading
 
@@ -957,7 +957,7 @@ function EvaluationModelPicker(): JSX.Element {
                             data-attr="evaluation-model-selector"
                         />
                         <ByokModelPickerNotice forEvaluation />
-                        {evaluation?.model_configuration?.provider === 'typesafe' && (
+                        {evaluation?.model_configuration?.provider === 'system_one' && (
                             <p className="text-sm text-muted mt-2">
                                 This judge returns a probability without written reasoning. A probability of 50% or
                                 higher produces a true result.
diff --git a/products/ai_observability/frontend/generated/api.schemas.ts b/products/ai_observability/frontend/generated/api.schemas.ts
index 524c9731498e..df8d99dc1b7b 100644
--- a/products/ai_observability/frontend/generated/api.schemas.ts
+++ b/products/ai_observability/frontend/generated/api.schemas.ts
@@ -871,7 +871,7 @@ export const EvaluationTargetEnumApi = {
  * * `together_ai` - Together AI
  * * `minimax` - MiniMax
  * * `zeabur` - Zeabur AI Hub
- * * `typesafe` - System One (Jev)
+ * * `system_one` - System One
  */
 export type LLMProviderEnumApi = (typeof LLMProviderEnumApi)[keyof typeof LLMProviderEnumApi]
 
@@ -885,7 +885,7 @@ export const LLMProviderEnumApi = {
     TogetherAi: 'together_ai',
     Minimax: 'minimax',
     Zeabur: 'zeabur',
-    Typesafe: 'typesafe',
+    SystemOne: 'system_one',
 } as const
 
 /**
@@ -1730,10 +1730,10 @@ export interface LLMProviderKeyApi {
     readonly error_message: string | null
     api_key?: string
     readonly api_key_masked: string
-    /** System One API base URL, including /v1. Defaults to TypeSafe. */
+    /** System One API base URL, including /v1. */
     base_url?: string
     /**
-     * Model ID served by the System One endpoint. Defaults to jev-1.13.0.
+     * Model ID served by the System One endpoint.
      * @maxLength 100
      */
     system_one_model?: string
@@ -2332,10 +2332,10 @@ export interface PatchedLLMProviderKeyApi {
     readonly error_message?: string | null
     api_key?: string
     readonly api_key_masked?: string
-    /** System One API base URL, including /v1. Defaults to TypeSafe. */
+    /** System One API base URL, including /v1. */
     base_url?: string
     /**
-     * Model ID served by the System One endpoint. Defaults to jev-1.13.0.
+     * Model ID served by the System One endpoint.
      * @maxLength 100
      */
     system_one_model?: string
@@ -3810,8 +3810,8 @@ export const LlmAnalyticsModelsRetrieveProvider = {
     Minimax: 'minimax',
     Openai: 'openai',
     Openrouter: 'openrouter',
+    SystemOne: 'system_one',
     TogetherAi: 'together_ai',
-    Typesafe: 'typesafe',
     Zeabur: 'zeabur',
 } as const
 
diff --git a/products/ai_observability/frontend/generated/api.zod.ts b/products/ai_observability/frontend/generated/api.zod.ts
index 28729b394c69..ff55f94fea4a 100644
--- a/products/ai_observability/frontend/generated/api.zod.ts
+++ b/products/ai_observability/frontend/generated/api.zod.ts
@@ -460,10 +460,10 @@ export const EvaluationsCreateBody = /* @__PURE__ */ zod
                                 'together_ai',
                                 'minimax',
                                 'zeabur',
-                                'typesafe',
+                                'system_one',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `system_one` - System One'
                             ),
                         model: zod.string().max(evaluationsCreateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -787,10 +787,10 @@ export const EvaluationsUpdateBody = /* @__PURE__ */ zod
                                 'together_ai',
                                 'minimax',
                                 'zeabur',
-                                'typesafe',
+                                'system_one',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `system_one` - System One'
                             ),
                         model: zod.string().max(evaluationsUpdateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -1018,10 +1018,10 @@ export const EvaluationsPartialUpdateBody = /* @__PURE__ */ zod
                                 'together_ai',
                                 'minimax',
                                 'zeabur',
-                                'typesafe',
+                                'system_one',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `system_one` - System One'
                             ),
                         model: zod.string().max(evaluationsPartialUpdateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -1643,19 +1643,19 @@ export const LlmAnalyticsProviderKeysCreateBody = /* @__PURE__ */ zod.object({
             'together_ai',
             'minimax',
             'zeabur',
-            'typesafe',
+            'system_one',
         ])
         .describe(
-            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
+            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `system_one` - System One'
         ),
     name: zod.string().max(llmAnalyticsProviderKeysCreateBodyNameMax),
     api_key: zod.string().optional(),
-    base_url: zod.url().optional().describe('System One API base URL, including \/v1. Defaults to TypeSafe.'),
+    base_url: zod.url().optional().describe('System One API base URL, including \/v1.'),
     system_one_model: zod
         .string()
         .max(llmAnalyticsProviderKeysCreateBodySystemOneModelMax)
         .optional()
-        .describe('Model ID served by the System One endpoint. Defaults to jev-1.13.0.'),
+        .describe('Model ID served by the System One endpoint.'),
     azure_endpoint: zod.url().optional().describe('Azure OpenAI endpoint URL'),
     api_version: zod
         .string()
@@ -1685,19 +1685,19 @@ export const LlmAnalyticsProviderKeysUpdateBody = /* @__PURE__ */ zod.object({
             'together_ai',
             'minimax',
             'zeabur',
-            'typesafe',
+            'system_one',
         ])
         .describe(
-            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
+            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `system_one` - System One'
         ),
     name: zod.string().max(llmAnalyticsProviderKeysUpdateBodyNameMax),
     api_key: zod.string().optional(),
-    base_url: zod.url().optional().describe('System One API base URL, including \/v1. Defaults to TypeSafe.'),
+    base_url: zod.url().optional().describe('System One API base URL, including \/v1.'),
     system_one_model: zod
         .string()
         .max(llmAnalyticsProviderKeysUpdateBodySystemOneModelMax)
         .optional()
-        .describe('Model ID served by the System One endpoint. Defaults to jev-1.13.0.'),
+        .describe('Model ID served by the System One endpoint.'),
     azure_endpoint: zod.url().optional().describe('Azure OpenAI endpoint URL'),
     api_version: zod
         .string()
@@ -1727,20 +1727,20 @@ export const LlmAnalyticsProviderKeysPartialUpdateBody = /* @__PURE__ */ zod.obj
             'together_ai',
             'minimax',
             'zeabur',
-            'typesafe',
+            'system_one',
         ])
         .optional()
         .describe(
-            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
+            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `system_one` - System One'
         ),
     name: zod.string().max(llmAnalyticsProviderKeysPartialUpdateBodyNameMax).optional(),
     api_key: zod.string().optional(),
-    base_url: zod.url().optional().describe('System One API base URL, including \/v1. Defaults to TypeSafe.'),
+    base_url: zod.url().optional().describe('System One API base URL, including \/v1.'),
     system_one_model: zod
         .string()
         .max(llmAnalyticsProviderKeysPartialUpdateBodySystemOneModelMax)
         .optional()
-        .describe('Model ID served by the System One endpoint. Defaults to jev-1.13.0.'),
+        .describe('Model ID served by the System One endpoint.'),
     azure_endpoint: zod.url().optional().describe('Azure OpenAI endpoint URL'),
     api_version: zod
         .string()
@@ -1770,19 +1770,19 @@ export const LlmAnalyticsProviderKeysValidateCreateBody = /* @__PURE__ */ zod.ob
             'together_ai',
             'minimax',
             'zeabur',
-            'typesafe',
+            'system_one',
         ])
         .describe(
-            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
+            '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `system_one` - System One'
         ),
     name: zod.string().max(llmAnalyticsProviderKeysValidateCreateBodyNameMax),
     api_key: zod.string().optional(),
-    base_url: zod.url().optional().describe('System One API base URL, including \/v1. Defaults to TypeSafe.'),
+    base_url: zod.url().optional().describe('System One API base URL, including \/v1.'),
     system_one_model: zod
         .string()
         .max(llmAnalyticsProviderKeysValidateCreateBodySystemOneModelMax)
         .optional()
-        .describe('Model ID served by the System One endpoint. Defaults to jev-1.13.0.'),
+        .describe('Model ID served by the System One endpoint.'),
     azure_endpoint: zod.url().optional().describe('Azure OpenAI endpoint URL'),
     api_version: zod
         .string()
diff --git a/products/ai_observability/frontend/modelPickerLogic.test.ts b/products/ai_observability/frontend/modelPickerLogic.test.ts
index d997dd864bc8..75dc526f2c2f 100644
--- a/products/ai_observability/frontend/modelPickerLogic.test.ts
+++ b/products/ai_observability/frontend/modelPickerLogic.test.ts
@@ -45,13 +45,15 @@ describe('modelPickerLogic', () => {
     })
 
     describe('loadByokModels', () => {
-        it.each(['jev-1.13.0', 'custom-model'])(
+        it.each(['example-judge-v1', 'custom-model'])(
             'offers System One model %s only to evaluation model pickers',
             async (model) => {
                 useMocks({
                     get: {
                         '/api/environments/:team_id/llm_analytics/provider_keys/': {
-                            results: [{ id: 'key-typesafe', provider: 'typesafe', name: 'TypeSafe', state: 'ok' }],
+                            results: [
+                                { id: 'key-system-one', provider: 'system_one', name: 'System One', state: 'ok' },
+                            ],
                         },
                         '/api/environments/:team_id/llm_analytics/evaluation_config/': { active_provider_key: null },
                         '/api/llm_proxy/models/': ({ request }) =>
@@ -62,7 +64,7 @@ describe('modelPickerLogic', () => {
                                           {
                                               id: model,
                                               name: model,
-                                              provider: 'System One (Jev)',
+                                              provider: 'System One',
                                               is_recommended: true,
                                           },
                                       ],
diff --git a/products/ai_observability/frontend/modelPickerLogic.ts b/products/ai_observability/frontend/modelPickerLogic.ts
index 484b4c01fa9c..870fdf3bb9d2 100644
--- a/products/ai_observability/frontend/modelPickerLogic.ts
+++ b/products/ai_observability/frontend/modelPickerLogic.ts
@@ -347,13 +347,13 @@ export const modelPickerLogic = kea<modelPickerLogicType>([
         hasByokKeys: [
             (s) => [s.providerKeys],
             (providerKeys: LLMProviderKey[]): boolean =>
-                providerKeys.some((k) => k.state === 'ok' && k.provider !== 'typesafe'),
+                providerKeys.some((k) => k.state === 'ok' && k.provider !== 'system_one'),
         ],
         generativeByokModels: [
             (s) => [s.byokModels, s.providerKeys],
             (models: ModelOption[], keys: LLMProviderKey[]): ModelOption[] =>
                 models.filter(
-                    (model) => !keys.some((key) => key.id === model.providerKeyId && key.provider === 'typesafe')
+                    (model) => !keys.some((key) => key.id === model.providerKeyId && key.provider === 'system_one')
                 ),
         ],
         playgroundProviderModelGroups: [
@@ -421,7 +421,7 @@ export const modelPickerLogic = kea<modelPickerLogicType>([
         providerModelGroups: [
             (s) => [s.evaluationProviderModelGroups],
             (groups: ProviderModelGroup[]): ProviderModelGroup[] =>
-                groups.filter((group) => group.provider !== 'typesafe'),
+                groups.filter((group) => group.provider !== 'system_one'),
         ],
         evaluationModelNotice: [
             (s) => [
@@ -462,7 +462,7 @@ export const modelPickerLogic = kea<modelPickerLogicType>([
                 providerModelGroups: ProviderModelGroup[]
             ): ByokModelNotice | null => {
                 return getByokModelNotice(
-                    providerKeys.filter((key) => key.provider !== 'typesafe'),
+                    providerKeys.filter((key) => key.provider !== 'system_one'),
                     providerKeysLoading,
                     byokModelsLoading,
                     failedByokProviderKeyIds,
diff --git a/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx b/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
index 53e15f1d01df..0dccd9c52d5b 100644
--- a/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
+++ b/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
@@ -106,7 +106,7 @@ function getKeyPlaceholder(provider: LLMProvider): string {
             return 'Enter your MiniMax API key'
         case 'zeabur':
             return 'sk-...'
-        case 'typesafe':
+        case 'system_one':
             return 'Enter your endpoint’s bearer token'
     }
 }
@@ -139,7 +139,7 @@ function KeyValidationStatus({
         <ul className="text-xs text-muted mt-1 list-disc pl-4 space-y-0.5">
             <li>Your key will be encrypted and stored securely</li>
             <li>
-                {provider === 'typesafe'
+                {provider === 'system_one'
                     ? 'Model usage is billed by your endpoint provider'
                     : `You pay ${LLM_PROVIDER_LABELS[provider]} directly for model usage`}
             </li>
@@ -182,14 +182,10 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
 
     const isAzure = provider === 'azure_openai'
     const keyValidated = preValidationResult?.state === 'ok'
-    const isSystemOne = provider === 'typesafe'
+    const isSystemOne = provider === 'system_one'
     const isValid =
         name.length > 0 &&
-        (isSystemOne
-            ? systemOneBaseUrl.length > 0 &&
-              systemOneModel.length > 0 &&
-              (apiKey.length > 0 || systemOneBaseUrl !== DEFAULT_SYSTEM_ONE_BASE_URL)
-            : apiKey.length > 0) &&
+        (isSystemOne ? systemOneBaseUrl.length > 0 && systemOneModel.length > 0 : apiKey.length > 0) &&
         (!isAzure || azureEndpoint.length > 0)
     const validationFailed = !!preValidationResult && preValidationResult.state !== 'ok'
     const azureErrorField = isAzure && validationFailed ? azureErrorFieldFromResult(preValidationResult) : null
@@ -215,7 +211,7 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
                     provider,
                     name,
                     api_key: apiKey,
-                    set_as_active: provider !== 'typesafe' && !evaluationConfig?.active_provider_key,
+                    set_as_active: provider !== 'system_one' && !evaluationConfig?.active_provider_key,
                 }
                 if (isAzure) {
                     payload.azure_endpoint = azureEndpoint
@@ -337,7 +333,7 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
                         onChange={handleProviderChange}
                         options={LLM_PROVIDER_SELECT_OPTIONS.filter(
                             ({ value }) =>
-                                value !== 'typesafe' ||
+                                value !== 'system_one' ||
                                 featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_SYSTEM_ONE_EVALUATIONS] === true
                         )}
                         className="mt-1"
@@ -435,7 +431,7 @@ function EditKeyModal({
     const { systemOneBaseUrl, systemOneModel } = useValues(llmProviderKeysLogic)
     const { setEditingKey, updateProviderKey, preValidateKey, clearPreValidation } = useActions(llmProviderKeysLogic)
     const isAzureEdit = keyToEdit.provider === 'azure_openai'
-    const isSystemOne = keyToEdit.provider === 'typesafe'
+    const isSystemOne = keyToEdit.provider === 'system_one'
     const endpointChanged = systemOneBaseUrl !== (keyToEdit.base_url_display ?? DEFAULT_SYSTEM_ONE_BASE_URL)
 
     const [name, setName] = useState(keyToEdit.name)
diff --git a/products/ai_observability/frontend/settings/SystemOneConnectionFields.tsx b/products/ai_observability/frontend/settings/SystemOneConnectionFields.tsx
index 6c42142c414f..273a4b8312cb 100644
--- a/products/ai_observability/frontend/settings/SystemOneConnectionFields.tsx
+++ b/products/ai_observability/frontend/settings/SystemOneConnectionFields.tsx
@@ -11,39 +11,35 @@ export function SystemOneConnectionFields(): JSX.Element {
     const { setSystemOneBaseUrl, setSystemOneModel } = useActions(llmProviderKeysLogic)
 
     return (
-        <details>
-            <summary className="cursor-pointer text-sm font-medium">Advanced configuration</summary>
-            <div className="space-y-3 mt-3">
-                <div>
-                    <LemonLabel htmlFor="system-one-base-url">Base URL</LemonLabel>
-                    <LemonInput
-                        id="system-one-base-url"
-                        value={systemOneBaseUrl}
-                        onChange={setSystemOneBaseUrl}
-                        disabled={providerKeysLoading}
-                        placeholder="https://api.typesafe.ai/v1"
-                        data-attr="system-one-base-url"
-                        fullWidth
-                    />
-                </div>
-                <div>
-                    <LemonLabel htmlFor="system-one-model">Model ID</LemonLabel>
-                    <LemonInput
-                        id="system-one-model"
-                        value={systemOneModel}
-                        onChange={setSystemOneModel}
-                        disabled={providerKeysLoading}
-                        placeholder="jev-1.13.0"
-                        data-attr="system-one-model"
-                        fullWidth
-                    />
-                </div>
-                <p className="text-xs text-muted">
-                    Defaults to TypeSafe's hosted service and the Jev model. Use a public HTTPS endpoint that supports
-                    the System One API. Include /v1 in the base URL if required by your service. Validation sends a
-                    short synthetic example to the selected model.
-                </p>
+        <div className="space-y-3">
+            <div>
+                <LemonLabel htmlFor="system-one-base-url">Base URL</LemonLabel>
+                <LemonInput
+                    id="system-one-base-url"
+                    value={systemOneBaseUrl}
+                    onChange={setSystemOneBaseUrl}
+                    disabled={providerKeysLoading}
+                    placeholder="https://decisions.example.com/v1"
+                    data-attr="system-one-base-url"
+                    fullWidth
+                />
             </div>
-        </details>
+            <div>
+                <LemonLabel htmlFor="system-one-model">Model ID</LemonLabel>
+                <LemonInput
+                    id="system-one-model"
+                    value={systemOneModel}
+                    onChange={setSystemOneModel}
+                    disabled={providerKeysLoading}
+                    placeholder="Your model ID"
+                    data-attr="system-one-model"
+                    fullWidth
+                />
+            </div>
+            <p className="text-xs text-muted">
+                Use a public HTTPS endpoint that supports the System One API. Include /v1 in the base URL if your
+                service requires it. Validation sends a short synthetic example to the selected model.
+            </p>
+        </div>
     )
 }
diff --git a/products/ai_observability/frontend/settings/llmProviderKeysLogic.test.ts b/products/ai_observability/frontend/settings/llmProviderKeysLogic.test.ts
index 574ccf3f2432..7d656209b497 100644
--- a/products/ai_observability/frontend/settings/llmProviderKeysLogic.test.ts
+++ b/products/ai_observability/frontend/settings/llmProviderKeysLogic.test.ts
@@ -17,9 +17,8 @@ describe('normalizeLLMProvider', () => {
         ['zeabur-ai-hub', 'zeabur'],
         ['zeabur', 'zeabur'],
         ['openai', 'openai'],
-        ['System One (Jev)', 'typesafe'],
-        ['System One', 'typesafe'],
-        ['typesafe', 'typesafe'],
+        ['System One', 'system_one'],
+        ['system_one', 'system_one'],
     ])('maps %s to %s', (input, expected) => {
         expect(normalizeLLMProvider(input)).toBe(expected)
     })
diff --git a/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts b/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts
index 1406f2cae877..cf100a81b97a 100644
--- a/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts
+++ b/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts
@@ -12,8 +12,8 @@ export type LLMProvider = LLMProviderEnumApi
 
 /** Default Azure OpenAI API version — keep in sync with backend DEFAULT_API_VERSION. */
 export const DEFAULT_AZURE_API_VERSION = '2024-10-21'
-export const DEFAULT_SYSTEM_ONE_BASE_URL: string = 'https://api.typesafe.ai/v1'
-export const DEFAULT_SYSTEM_ONE_MODEL: string = 'jev-1.13.0'
+export const DEFAULT_SYSTEM_ONE_BASE_URL: string = ''
+export const DEFAULT_SYSTEM_ONE_MODEL: string = ''
 
 export const LLM_PROVIDER_LABELS: Record<LLMProvider, string> = {
     openai: 'OpenAI',
@@ -25,7 +25,7 @@ export const LLM_PROVIDER_LABELS: Record<LLMProvider, string> = {
     together_ai: 'Together AI',
     minimax: 'MiniMax',
     zeabur: 'Zeabur AI Hub',
-    typesafe: 'System One (Jev)',
+    system_one: 'System One',
 }
 
 const LLM_PROVIDERS = new Set<string>(Object.keys(LLM_PROVIDER_LABELS))
@@ -59,8 +59,8 @@ export function normalizeLLMProvider(provider: string | undefined): LLMProvider
     }
 
     const normalized = provider.trim().toLowerCase()
-    if (normalized === 'system one' || normalized === 'system one (jev)') {
-        return 'typesafe'
+    if (normalized === 'system one') {
+        return 'system_one'
     }
     if (normalized === 'google' || normalized === 'google-ai-studio') {
         return 'gemini'
diff --git a/products/ai_observability/frontend/utils.test.ts b/products/ai_observability/frontend/utils.test.ts
index 8081ba8a2412..eae8209d6853 100644
--- a/products/ai_observability/frontend/utils.test.ts
+++ b/products/ai_observability/frontend/utils.test.ts
@@ -75,7 +75,7 @@ function makeEvaluationRunRow({
 }
 
 describe('mapEvaluationRunRow', () => {
-    it.each([0, 0.49, 1])('preserves a Jev probability of %s without inventing reasoning', (probability) => {
+    it.each([0, 0.49, 1])('preserves a System One probability of %s without inventing reasoning', (probability) => {
         const row = makeEvaluationRunRow()
         row[7] = ''
         row[18] = probability
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index ec9609a76083..220b0ecd8406 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -38455,7 +38455,7 @@ export namespace Schemas {
      * * `together_ai` - Together AI
      * * `minimax` - MiniMax
      * * `zeabur` - Zeabur AI Hub
-     * * `typesafe` - System One (Jev)
+     * * `system_one` - System One
      */
     export type LLMProviderEnum = typeof LLMProviderEnum[keyof typeof LLMProviderEnum];
 
@@ -38470,7 +38470,7 @@ export namespace Schemas {
       TogetherAi: 'together_ai',
       Minimax: 'minimax',
       Zeabur: 'zeabur',
-      Typesafe: 'typesafe',
+      SystemOne: 'system_one',
     } as const;
 
     /**
@@ -38681,10 +38681,10 @@ export namespace Schemas {
       readonly error_message: string | null;
       api_key?: string;
       readonly api_key_masked: string;
-      /** System One API base URL, including /v1. Defaults to TypeSafe. */
+      /** System One API base URL, including /v1. */
       base_url?: string;
       /**
-         * Model ID served by the System One endpoint. Defaults to jev-1.13.0.
+         * Model ID served by the System One endpoint.
          * @maxLength 100
          */
       system_one_model?: string;
@@ -74028,10 +74028,10 @@ export namespace Schemas {
       readonly error_message?: string | null;
       api_key?: string;
       readonly api_key_masked?: string;
-      /** System One API base URL, including /v1. Defaults to TypeSafe. */
+      /** System One API base URL, including /v1. */
       base_url?: string;
       /**
-         * Model ID served by the System One endpoint. Defaults to jev-1.13.0.
+         * Model ID served by the System One endpoint.
          * @maxLength 100
          */
       system_one_model?: string;
@@ -112669,8 +112669,8 @@ export namespace Schemas {
       Minimax: 'minimax',
       Openai: 'openai',
       Openrouter: 'openrouter',
+      SystemOne: 'system_one',
       TogetherAi: 'together_ai',
-      Typesafe: 'typesafe',
       Zeabur: 'zeabur',
     } as const;
 
diff --git a/services/mcp/src/generated/ai_observability/api.ts b/services/mcp/src/generated/ai_observability/api.ts
index ff6bb3457533..1ab979fcf23c 100644
--- a/services/mcp/src/generated/ai_observability/api.ts
+++ b/services/mcp/src/generated/ai_observability/api.ts
@@ -762,10 +762,10 @@ export const EvaluationsCreateBody = () => zod
                                 'together_ai',
                                 'minimax',
                                 'zeabur',
-                                'typesafe',
+                                'system_one',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `system_one` - System One'
                             ),
                         model: zod.string().max(evaluationsCreateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -1011,10 +1011,10 @@ export const EvaluationsPartialUpdateBody = () => zod
                                 'together_ai',
                                 'minimax',
                                 'zeabur',
-                                'typesafe',
+                                'system_one',
                             ])
                             .describe(
-                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `typesafe` - System One (Jev)'
+                                '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `zeabur` - Zeabur AI Hub\n\* `system_one` - System One'
                             ),
                         model: zod.string().max(evaluationsPartialUpdateBodyModelConfigurationOneModelMax),
                         provider_key_id: zod
@@ -1565,8 +1565,8 @@ export const LlmAnalyticsModelsRetrieveQueryParams = () => zod.object({
             'minimax',
             'openai',
             'openrouter',
+            'system_one',
             'together_ai',
-            'typesafe',
             'zeabur',
         ])
         .optional()

From d22259e8d158006f804b512888fe5db895d391a0 Mon Sep 17 00:00:00 2001
From: "tests-posthog[bot]"
 <250237707+tests-posthog[bot]@users.noreply.github.com>
Date: Fri, 25 Sep 2026 17:09:35 +0000
Subject: [PATCH 17/30] test(mcp): update unit test snapshots

---
 .../__snapshots__/tool-schemas/llma-evaluation-create.json    | 4 ++--
 .../tool-schemas/llma-evaluation-judge-models.json            | 2 +-
 .../__snapshots__/tool-schemas/llma-evaluation-update.json    | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json
index c7acc821350a..095eab630486 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json
@@ -112,7 +112,7 @@
                             "type": "string"
                         },
                         "provider": {
-                            "description": "* `openai` - Openai\n* `anthropic` - Anthropic\n* `gemini` - Gemini\n* `openrouter` - Openrouter\n* `fireworks` - Fireworks\n* `azure_openai` - Azure OpenAI\n* `together_ai` - Together AI\n* `minimax` - MiniMax\n* `zeabur` - Zeabur AI Hub\n* `typesafe` - System One (Jev)",
+                            "description": "* `openai` - Openai\n* `anthropic` - Anthropic\n* `gemini` - Gemini\n* `openrouter` - Openrouter\n* `fireworks` - Fireworks\n* `azure_openai` - Azure OpenAI\n* `together_ai` - Together AI\n* `minimax` - MiniMax\n* `zeabur` - Zeabur AI Hub\n* `system_one` - System One",
                             "enum": [
                                 "openai",
                                 "anthropic",
@@ -123,7 +123,7 @@
                                 "together_ai",
                                 "minimax",
                                 "zeabur",
-                                "typesafe"
+                                "system_one"
                             ],
                             "type": "string"
                         },
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-judge-models.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-judge-models.json
index cef5749759f5..6a6a7ad99424 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-judge-models.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-judge-models.json
@@ -15,8 +15,8 @@
                 "minimax",
                 "openai",
                 "openrouter",
+                "system_one",
                 "together_ai",
-                "typesafe",
                 "zeabur"
             ],
             "type": "string"
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json
index 69dc4c521d7a..cc04887a49ff 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json
@@ -115,7 +115,7 @@
                             "type": "string"
                         },
                         "provider": {
-                            "description": "* `openai` - Openai\n* `anthropic` - Anthropic\n* `gemini` - Gemini\n* `openrouter` - Openrouter\n* `fireworks` - Fireworks\n* `azure_openai` - Azure OpenAI\n* `together_ai` - Together AI\n* `minimax` - MiniMax\n* `zeabur` - Zeabur AI Hub\n* `typesafe` - System One (Jev)",
+                            "description": "* `openai` - Openai\n* `anthropic` - Anthropic\n* `gemini` - Gemini\n* `openrouter` - Openrouter\n* `fireworks` - Fireworks\n* `azure_openai` - Azure OpenAI\n* `together_ai` - Together AI\n* `minimax` - MiniMax\n* `zeabur` - Zeabur AI Hub\n* `system_one` - System One",
                             "enum": [
                                 "openai",
                                 "anthropic",
@@ -126,7 +126,7 @@
                                 "together_ai",
                                 "minimax",
                                 "zeabur",
-                                "typesafe"
+                                "system_one"
                             ],
                             "type": "string"
                         },

From e10bc10ae0f37558b8ceaca9b2eafe069b14a460 Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Fri, 25 Sep 2026 19:18:30 +0200
Subject: [PATCH 18/30] chore(aio): clarify compatible endpoint guidance

---
 posthog/egress/typesafe/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/posthog/egress/typesafe/README.md b/posthog/egress/typesafe/README.md
index fd1dd06205d2..8ad21289af30 100644
--- a/posthog/egress/typesafe/README.md
+++ b/posthog/egress/typesafe/README.md
@@ -56,10 +56,10 @@ The default reserve ladder applies, and `typesafe_request` defaults to `NORMAL`.
 Give every caller an explicit lane: `NORMAL` when a person waits for the answer, `BATCH` for background work.
 AI observability uses `llma_evaluations` as its source, `BATCH` for evaluation runs, and `NORMAL` for synthetic connection validation.
 The evaluation caller checks the project-group flag `llm-analytics-system-one-evaluations` before connection validation and every evaluation.
-Customer connections require their own TypeSafe credential or compatible deployment; the evaluation adapter never falls back to instance credentials.
+AI observability connects to compatible deployments and rejects TypeSafe's hosted endpoint.
+The evaluation adapter never falls back to instance credentials.
 PostHog's regional AI gateway endpoints additionally require an organization in `POSTHOG_INTERNAL_ORG_IDS`.
-Keep the experimental flag limited to PostHog staff projects until customer rollout is approved.
-Customer-data launch still requires the opt-in and sign-off above.
+Keep the experimental flag limited to PostHog staff projects during rollout.
 
 ## Rate-limit headers
 

From bc8e42e073e22817d526e0e326a62534acad13ab Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Sat, 26 Sep 2026 13:49:55 +0200
Subject: [PATCH 19/30] fix(aio): clarify system one base url field

---
 .../frontend/settings/SystemOneConnectionFields.tsx           | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/products/ai_observability/frontend/settings/SystemOneConnectionFields.tsx b/products/ai_observability/frontend/settings/SystemOneConnectionFields.tsx
index 273a4b8312cb..a6d834f65dd1 100644
--- a/products/ai_observability/frontend/settings/SystemOneConnectionFields.tsx
+++ b/products/ai_observability/frontend/settings/SystemOneConnectionFields.tsx
@@ -37,8 +37,8 @@ export function SystemOneConnectionFields(): JSX.Element {
                 />
             </div>
             <p className="text-xs text-muted">
-                Use a public HTTPS endpoint that supports the System One API. Include /v1 in the base URL if your
-                service requires it. Validation sends a short synthetic example to the selected model.
+                Enter the public HTTPS base URL, ending before /systemone. Validation sends a short synthetic example to
+                the selected model.
             </p>
         </div>
     )

From 83cc3bc6fa6e3fd69bf58fb4ee36f27118cbff4f Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Sat, 26 Sep 2026 16:34:47 +0200
Subject: [PATCH 20/30] fix(aio): address system one evaluation review findings

---
 .../internal/ai-observability-judge-inputs.md | 11 +++---
 posthog/taxonomy/taxonomy.py                  |  6 +++
 .../ai_observability/evaluation_llm_judge.py  | 37 ++++++++++++++-----
 .../ai_observability/test_run_evaluation.py   | 31 +++++++++-------
 .../backend/api/provider_keys.py              |  7 ++--
 .../backend/api/test/test_provider_keys.py    | 13 +++++++
 .../backend/llm/system_one.py                 | 25 ++++++++++---
 .../backend/llm/test/test_system_one.py       |  6 ++-
 .../frontend/generated/api.schemas.ts         |  4 +-
 .../frontend/generated/api.zod.ts             |  8 ++--
 .../settings/LLMProviderKeysSettings.tsx      |  9 +++--
 .../settings/llmProviderKeysLogic.test.ts     | 15 +++++++-
 .../frontend/settings/llmProviderKeysLogic.ts | 12 ++++++
 services/mcp/src/api/generated.ts             |  4 +-
 14 files changed, 134 insertions(+), 54 deletions(-)

diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index c6539c72f7c8..130ea37ff9fb 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -57,7 +57,7 @@ The client appends `/systemone` to the base URL and sends the API key as a beare
 An empty key selects no authentication.
 Changing the endpoint requires entering its credential again, or explicitly choosing no authentication, so an existing key is not forwarded to a new host.
 Private network destinations and redirects are blocked by the shared DNS-pinned HTTP transport.
-Saving a connection validates it with a short synthetic input and a Noul question, without sending evaluation data.
+Saving a connection validates it with a short synthetic input and a Noul question, without sending evaluation data, using a 10-second request timeout.
 Select the connection and configured model on each evaluation; these connections cannot become the shared active provider key used by other AI features.
 Provider keys keep the provider they were created with; switching providers requires a new key.
 The evaluation integration uses Noul for boolean outputs, with the same formatted text for generation, trace, and session targets.
@@ -71,9 +71,9 @@ Compare results on representative inputs when changing models.
 
 For boolean evaluations, the prompt becomes a [Noul question](https://docs.typesafe.ai/primitives/noul).
 A probability of at least 0.5 produces `true`; the evaluation's existing pass/fail polarity still applies.
-The raw probability is stored in `$ai_evaluation_probability`, with available token usage and the resolved model version.
+The raw probability is stored in `$ai_evaluation_probability` when the criteria apply, with available token usage and the configured model ID.
 Missing or invalid token counts remain unknown and do not discard a valid answer.
-Ingestion estimates cost from the reported model and token usage when the model appears in the existing pricing catalog.
+Ingestion estimates cost from the configured model and token usage when the model appears in the existing pricing catalog.
 Models without a catalog match retain their usage with cost left unknown.
 A custom deployment reporting a recognized model name can inherit that model's catalog estimate; this does not measure its hosting cost.
 
@@ -83,9 +83,10 @@ System One answers contain no written reasoning, so reports inspect the original
 
 Each endpoint and credential pair has a separate, hashed rate-limit scope shared across workers.
 Evaluations use the batch lane; connection validation uses the normal lane.
-Local budget exhaustion, rate limits, and overload responses are retried through Temporal, honoring `Retry-After` up to five minutes.
+Local budget exhaustion, rate limits, and overload responses are retried through Temporal, honoring `Retry-After` up to one minute.
 If retries fail, the run fails and the evaluation stays enabled.
-Blocked endpoints and rejected requests disable the evaluation and mark the connection for revalidation, without recording model usage.
+Blocked endpoints and redirects disable the evaluation and mark the connection for revalidation, without recording model usage.
+Requests rejected because of an individual input skip that run without changing the shared connection.
 Invalid probabilities, missing answers, and mismatched answer types skip the item as an unparsable response.
 Inputs rejected for exceeding the model's context window are skipped.
 See TypeSafe's [API reference](https://docs.typesafe.ai/api) for the System One protocol.
diff --git a/posthog/taxonomy/taxonomy.py b/posthog/taxonomy/taxonomy.py
index 5146d4b72921..c62a8577744c 100644
--- a/posthog/taxonomy/taxonomy.py
+++ b/posthog/taxonomy/taxonomy.py
@@ -2546,6 +2546,12 @@ def visible_definitions(group: str) -> Iterator[tuple[str, CoreFilterDefinition]
             "description": "The boolean verdict of the evaluation (true = pass, false = fail).",
             "examples": [True, False],
         },
+        "$ai_evaluation_probability": {
+            "label": "AI evaluation probability",
+            "description": "The probability of a true verdict for a boolean evaluation.",
+            "examples": [0.9],
+            "type": "Numeric",
+        },
         "$ai_evaluation_numeric_result": {
             "label": "AI evaluation numeric result",
             "description": "The raw numeric score returned by an online evaluation.",
diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.py
index e33ed205d346..f1fce2aae59d 100644
--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py
+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py
@@ -44,13 +44,13 @@
     ModelPermissionError,
     OutputTokenLimitError,
     ProviderConnectionError,
-    ProviderMismatchError,
     QuotaExceededError,
     RateLimitError,
     StructuredOutputParseError,
 )
 from products.ai_observability.backend.llm.system_one import (
     SystemOneClient,
+    SystemOneEndpointBlockedError,
     SystemOneRateLimitError,
     SystemOneRequestRejectedError,
     system_one_evaluations_enabled,
@@ -453,10 +453,20 @@ def call_llm_judge(
 
     if provider == "system_one":
         if output_type != "boolean":
-            raise ApplicationError("This System One evaluation output type is not supported.", non_retryable=True)
+            return build_skipped_evaluation_result(
+                output_type=output_type,
+                allows_na=allows_na,
+                reasoning="System One currently supports boolean evaluations only.",
+                skip_reason="unsupported_output_type",
+            )
         base_url = provider_key.encrypted_config.get("base_url", "") if provider_key else ""
         if not system_one_evaluations_enabled(team_id, base_url=base_url):
-            raise ApplicationError("System One evaluations are not available for this project.", non_retryable=True)
+            return build_skipped_evaluation_result(
+                output_type=output_type,
+                allows_na=allows_na,
+                reasoning="System One evaluations are not available for this project.",
+                skip_reason="system_one_unavailable",
+            )
 
     type_config = get_output_type_config(allows_na, output_type=output_type, output_config=output_config)
     response_format = type_config.response_format
@@ -473,8 +483,6 @@ def call_llm_judge(
     system_one_result = None
     try:
         if provider == "system_one":
-            if provider_key is not None and provider_key.provider != provider:
-                raise ProviderMismatchError(provider_key.provider, provider)
             prompt = evaluation["evaluation_config"]["prompt"]
             questions: dict[str, Question] = {"verdict": NoulQuestion(instructions=prompt)}
             if allows_na:
@@ -495,10 +503,12 @@ def call_llm_judge(
             applicable = True
             if allows_na:
                 applicability_answer = system_one_result.answers["applicable"]
-                assert isinstance(applicability_answer, NoulAnswer)
+                if not isinstance(applicability_answer, NoulAnswer):
+                    raise StructuredOutputParseError("The endpoint returned an invalid applicability answer.")
                 applicable = applicability_answer.probability >= 0.5
             verdict_answer = system_one_result.answers["verdict"]
-            assert isinstance(verdict_answer, NoulAnswer)
+            if not isinstance(verdict_answer, NoulAnswer):
+                raise StructuredOutputParseError("The endpoint returned an invalid verdict answer.")
             probability = verdict_answer.probability
             parsed = (
                 BooleanWithNAEvalResult(
@@ -508,7 +518,6 @@ def call_llm_judge(
                 if allows_na
                 else BooleanEvalResult(reasoning="", verdict=probability >= 0.5)
             )
-            model = system_one_result.model
             response = CompletionResponse(
                 content="",
                 model=model,
@@ -529,7 +538,7 @@ def call_llm_judge(
                     response_format=response_format,
                 )
             )
-    except SystemOneRequestRejectedError as e:
+    except SystemOneEndpointBlockedError as e:
         increment_user_errors("request_rejected", provider=provider)
         return terminal_user_error_result(
             spec=require_user_error_spec("request_rejected", is_byok=is_byok),
@@ -539,6 +548,14 @@ def call_llm_judge(
             key_id=key_id,
             is_byok=is_byok,
         )
+    except SystemOneRequestRejectedError as e:
+        increment_user_errors("request_rejected", provider=provider)
+        return build_skipped_evaluation_result(
+            output_type=output_type,
+            allows_na=allows_na,
+            reasoning=str(e),
+            skip_reason="request_rejected",
+        )
     except SystemOneRateLimitError as e:
         increment_errors("rate_limit", provider=provider)
         raise ApplicationError(
@@ -771,7 +788,7 @@ def call_llm_judge(
     else:
         raise ValueError(f"Unexpected result type: {type(parsed_result)}")
 
-    if probability is not None:
+    if probability is not None and result_dict["verdict"] is not None:
         result_dict["probability"] = probability
     if system_one_result is not None:
         result_dict["input_tokens"] = system_one_result.input_tokens
diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index 68a5701e7c49..ac2605bd963c 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -135,7 +135,7 @@ def test_system_one_judge_emits_boolean_probability_without_reasoning(
     resolved = MagicMock(provider="system_one", model=model, provider_key=key, is_byok=True)
     response = MagicMock(status_code=200)
     response.json.return_value = {
-        "model": "example-judge-v1",
+        "model": "endpoint-controlled-model",
         "answers": {
             "verdict": {"type": "noul", "noul": probability},
             "applicable": {"type": "noul", "noul": applicability},
@@ -169,15 +169,16 @@ def test_system_one_judge_emits_boolean_probability_without_reasoning(
     assert request.call_args.kwargs["json"]["model"] == model
     assert result["verdict"] is verdict
     assert result["reasoning"] == ""
-    assert result["probability"] == probability
+    assert result.get("probability") == (probability if verdict is not None else None)
+    assert result["model"] == model
     assert result["total_tokens"] == sum(usage.values())
     if allows_na:
         assert result["applicable"] is (applicability >= 0.5)
     properties = build_evaluation_event_properties(evaluation, result, datetime(2026, 1, 1, tzinfo=UTC))
     assert properties["$ai_input_tokens"] == usage.get("input_tokens")
     assert properties["$ai_output_tokens"] == usage.get("output_tokens")
-    assert properties["$ai_evaluation_probability"] == probability
-    assert properties["$ai_model"] == "example-judge-v1"
+    assert properties.get("$ai_evaluation_probability") == (probability if verdict is not None else None)
+    assert properties["$ai_model"] == model
     assert properties["$ai_evaluation_key_type"] == "byok"
 
 
@@ -190,16 +191,15 @@ def test_system_one_numeric_mapping_is_not_enabled() -> None:
             "posthog.temporal.ai_observability.evaluation_llm_judge.system_one_evaluations_enabled", return_value=True
         ),
         patch("requests.Session.request") as request,
-        pytest.raises(ApplicationError) as error,
     ):
         spec.return_value.resolve.return_value = MagicMock(provider="system_one")
-        call_llm_judge(
+        result = call_llm_judge(
             evaluation={"team_id": 1, "output_type": "numeric"},
             system_prompt="",
             user_prompt="Hello!",
             allows_na=False,
         )
-    assert error.value.non_retryable
+    assert result["skipped"] is True
     request.assert_not_called()
 
 
@@ -214,7 +214,6 @@ def test_system_one_restricted_connection_does_not_send_evaluation_data(base_url
         patch("products.ai_observability.backend.llm.system_one.Team.objects.only") as teams,
         patch("products.ai_observability.backend.llm.system_one.get_feature_flag_or_none", return_value=flag),
         patch("requests.Session.request") as request,
-        pytest.raises(ApplicationError) as error,
     ):
         teams.return_value.get.return_value = Team(id=1, organization_id=uuid.uuid4(), uuid=uuid.uuid4())
         spec.return_value.resolve.return_value = MagicMock(
@@ -225,18 +224,18 @@ def test_system_one_restricted_connection_does_not_send_evaluation_data(base_url
             ),
             is_byok=True,
         )
-        call_llm_judge(
+        result = call_llm_judge(
             evaluation={"team_id": 1, "evaluation_config": {"prompt": "Is this a greeting?"}},
             system_prompt="",
             user_prompt="Hello!",
             allows_na=False,
         )
-    assert error.value.non_retryable
+    assert result["skipped"] is True
     request.assert_not_called()
 
 
 @pytest.mark.parametrize("status", [301, 400, 422])
-def test_system_one_rejected_requests_disable_without_model_cost_attribution(status: int) -> None:
+def test_system_one_rejections_preserve_shared_key_for_input_errors(status: int) -> None:
     key = MagicMock(
         provider="system_one",
         encrypted_config={"api_key": "example-token", "base_url": "https://decisions.example.com/v1"},
@@ -262,10 +261,14 @@ def test_system_one_rejected_requests_disable_without_model_cost_attribution(sta
             user_prompt="Hello!",
             allows_na=False,
         )
-    assert result["terminal_user_error"] is True
     assert result["skip_reason"] == "request_rejected"
-    assert result["provider_key_state"] == "error"
-    assert result["status_reason"] == "provider_key_invalid"
+    if status == 301:
+        assert result["terminal_user_error"] is True
+        assert result["provider_key_state"] == "error"
+    else:
+        assert result["skipped"] is True
+        assert "terminal_user_error" not in result
+        assert "provider_key_state" not in result
     assert "model" not in result
     assert "provider" not in result
 
diff --git a/products/ai_observability/backend/api/provider_keys.py b/products/ai_observability/backend/api/provider_keys.py
index d5948266d608..1de4b63c22fa 100644
--- a/products/ai_observability/backend/api/provider_keys.py
+++ b/products/ai_observability/backend/api/provider_keys.py
@@ -97,7 +97,7 @@ def _validation_error_field(provider: str, error_message: str | None) -> str:
 class LLMProviderKeySerializer(serializers.ModelSerializer):
     api_key = serializers.CharField(write_only=True, required=False, allow_blank=True)
     base_url = serializers.URLField(
-        write_only=True, required=False, help_text="System One API base URL, including /v1."
+        write_only=True, required=False, help_text="System One API base URL, ending before /systemone."
     )
     system_one_model = serializers.CharField(
         write_only=True,
@@ -197,9 +197,8 @@ def validate(self, data):
                     raise serializers.ValidationError({field: "This field is required for System One connections."})
         if provider != LLMProvider.SYSTEM_ONE:
             if "base_url" in data or "system_one_model" in data:
-                raise serializers.ValidationError(
-                    {"base_url": "These settings are only available for System One connections."}
-                )
+                field = "system_one_model" if "system_one_model" in data else "base_url"
+                raise serializers.ValidationError({field: "This setting is only available for System One connections."})
             if data.get("api_key") == "":
                 raise serializers.ValidationError({"api_key": "An API key is required."})
         elif self.instance is not None and "base_url" in data:
diff --git a/products/ai_observability/backend/api/test/test_provider_keys.py b/products/ai_observability/backend/api/test/test_provider_keys.py
index 25733ee21133..ac7918208959 100644
--- a/products/ai_observability/backend/api/test/test_provider_keys.py
+++ b/products/ai_observability/backend/api/test/test_provider_keys.py
@@ -90,6 +90,19 @@ def test_system_one_connection_rejects_hosted_typesafe_endpoint(self) -> None:
         self.assertFalse(serializer.is_valid())
         self.assertIn("base_url", serializer.errors)
 
+    def test_non_system_one_model_error_uses_model_field(self) -> None:
+        serializer = LLMProviderKeySerializer(
+            data={
+                "provider": "openai",
+                "name": "Example key",
+                "api_key": "sk-example",
+                "system_one_model": "custom-model",
+            }
+        )
+        self.assertFalse(serializer.is_valid())
+        self.assertIn("system_one_model", serializer.errors)
+        self.assertNotIn("base_url", serializer.errors)
+
     @parameterized.expand([("https://other.example.com/v1", False), ("https://decisions.example.com/v1", True)])
     def test_endpoint_change_requires_explicit_credentials(self, base_url: str, valid: bool) -> None:
         key = LLMProviderKey(
diff --git a/products/ai_observability/backend/llm/system_one.py b/products/ai_observability/backend/llm/system_one.py
index e5baf031bbe8..11eaadc92d4d 100644
--- a/products/ai_observability/backend/llm/system_one.py
+++ b/products/ai_observability/backend/llm/system_one.py
@@ -60,9 +60,13 @@ class SystemOneRequestRejectedError(LLMError):
     pass
 
 
+class SystemOneEndpointBlockedError(SystemOneRequestRejectedError):
+    pass
+
+
 class SystemOneRateLimitError(RateLimitError):
     def __init__(self, retry_after: str | None) -> None:
-        super().__init__("The System One endpoint is busy. Try again later.")
+        super().__init__("The System One endpoint is temporarily unavailable. Try again later.")
         self.retry_after: float | None = None
         if retry_after:
             try:
@@ -73,7 +77,7 @@ def __init__(self, retry_after: str | None) -> None:
                 except (ValueError, TypeError, OverflowError):
                     return
             if math.isfinite(delay):
-                self.retry_after = max(1, min(delay, 300))
+                self.retry_after = max(1, min(delay, 60))
 
 
 class SystemOneClient:
@@ -107,8 +111,12 @@ def evaluate(
         questions: Mapping[str, Question],
         base_url: str,
         priority: Priority = Priority.BATCH,
+        timeout: float = 60,
     ) -> SystemOneResult:
-        base_url = SystemOneClient.normalize_base_url(base_url)
+        try:
+            base_url = SystemOneClient.normalize_base_url(base_url)
+        except ValueError as error:
+            raise SystemOneEndpointBlockedError(str(error)) from error
         try:
             return TypeSafeSystemOneClient(
                 api_key=api_key,
@@ -116,12 +124,12 @@ def evaluate(
                 model=model,
                 source="llma_evaluations",
                 priority=priority,
-                timeout=60,
+                timeout=timeout,
             ).decide(state=state, questions=questions)
         except TypeSafeEgressBudgetExhausted as error:
             raise SystemOneRateLimitError(None) from error
         except SSRFBlockedError as error:
-            raise SystemOneRequestRejectedError("This endpoint is not allowed. Use a public HTTPS endpoint.") from error
+            raise SystemOneEndpointBlockedError("This endpoint is not allowed. Use a public HTTPS endpoint.") from error
         except requests.RequestException as error:
             raise ProviderConnectionError("Could not reach the System One endpoint. Try again.") from error
         except TypeSafeRequestFailed as error:
@@ -137,7 +145,7 @@ def evaluate(
                 raise ModelPermissionError(model) from error
             if status == 404:
                 raise ModelNotFoundError(model) from error
-            if status in (429, 503, 529):
+            if status in (408, 429, 503, 529):
                 raise SystemOneRateLimitError(
                     response.headers.get("Retry-After") if response is not None else None
                 ) from error
@@ -145,6 +153,10 @@ def evaluate(
                 raise ProviderConnectionError(
                     "The System One endpoint is temporarily unavailable. Try again."
                 ) from error
+            if 300 <= status < 400:
+                raise SystemOneEndpointBlockedError(
+                    "The endpoint redirected the request. Use its final HTTPS URL."
+                ) from error
             if status == 413 or (
                 status == 422 and response is not None and is_context_window_error_message(response.text)
             ):
@@ -164,6 +176,7 @@ def validate_key(api_key: str, *, base_url: str, model: str) -> tuple[str, str |
                 model=model,
                 state="Hello!",
                 priority=Priority.NORMAL,
+                timeout=10,
                 questions={"verdict": NoulQuestion(instructions="Does the text contain a greeting?")},
             )
         except (AuthenticationError, ModelPermissionError) as error:
diff --git a/products/ai_observability/backend/llm/test/test_system_one.py b/products/ai_observability/backend/llm/test/test_system_one.py
index 501b0116d952..4d5872fb8167 100644
--- a/products/ai_observability/backend/llm/test/test_system_one.py
+++ b/products/ai_observability/backend/llm/test/test_system_one.py
@@ -20,6 +20,7 @@
 )
 from products.ai_observability.backend.llm.system_one import (
     SystemOneClient,
+    SystemOneEndpointBlockedError,
     SystemOneRateLimitError,
     SystemOneRequestRejectedError,
     system_one_evaluations_enabled,
@@ -81,6 +82,7 @@ def test_system_one_key_validation(status: int, expected_state: str) -> None:
     assert (message is None) == (expected_state == "ok")
     assert request.call_args.args == ("POST", "https://decisions.example.com/v1/systemone")
     assert request.call_args.kwargs["headers"]["Authorization"] == "Bearer example-token"
+    assert request.call_args.kwargs["timeout"] == 10
 
 
 @pytest.mark.parametrize("probability", [-0.1, 1.1, float("nan"), float("inf"), "0.8", True, None])
@@ -104,7 +106,7 @@ def test_system_one_rejects_invalid_probabilities(probability: object) -> None:
         )
 
 
-@pytest.mark.parametrize("status", [429, 503, 529])
+@pytest.mark.parametrize("status", [408, 429, 503, 529])
 def test_system_one_rate_limits_are_retryable(status: int) -> None:
     response = Mock(status_code=status, headers={"Retry-After": "15"})
     with (
@@ -158,7 +160,7 @@ def test_unavailable_usage_does_not_discard_a_valid_answer(
 def test_official_endpoint_is_blocked(base_url: str) -> None:
     with (
         patch("requests.Session.request") as request,
-        pytest.raises(ValueError, match="hosted endpoint is not available"),
+        pytest.raises(SystemOneEndpointBlockedError, match="hosted endpoint is not available"),
     ):
         SystemOneClient.evaluate(
             api_key="example-token",
diff --git a/products/ai_observability/frontend/generated/api.schemas.ts b/products/ai_observability/frontend/generated/api.schemas.ts
index df8d99dc1b7b..f3c5386de425 100644
--- a/products/ai_observability/frontend/generated/api.schemas.ts
+++ b/products/ai_observability/frontend/generated/api.schemas.ts
@@ -1730,7 +1730,7 @@ export interface LLMProviderKeyApi {
     readonly error_message: string | null
     api_key?: string
     readonly api_key_masked: string
-    /** System One API base URL, including /v1. */
+    /** System One API base URL, ending before /systemone. */
     base_url?: string
     /**
      * Model ID served by the System One endpoint.
@@ -2332,7 +2332,7 @@ export interface PatchedLLMProviderKeyApi {
     readonly error_message?: string | null
     api_key?: string
     readonly api_key_masked?: string
-    /** System One API base URL, including /v1. */
+    /** System One API base URL, ending before /systemone. */
     base_url?: string
     /**
      * Model ID served by the System One endpoint.
diff --git a/products/ai_observability/frontend/generated/api.zod.ts b/products/ai_observability/frontend/generated/api.zod.ts
index ff55f94fea4a..5d7ce1b0ad3c 100644
--- a/products/ai_observability/frontend/generated/api.zod.ts
+++ b/products/ai_observability/frontend/generated/api.zod.ts
@@ -1650,7 +1650,7 @@ export const LlmAnalyticsProviderKeysCreateBody = /* @__PURE__ */ zod.object({
         ),
     name: zod.string().max(llmAnalyticsProviderKeysCreateBodyNameMax),
     api_key: zod.string().optional(),
-    base_url: zod.url().optional().describe('System One API base URL, including \/v1.'),
+    base_url: zod.url().optional().describe('System One API base URL, ending before \/systemone.'),
     system_one_model: zod
         .string()
         .max(llmAnalyticsProviderKeysCreateBodySystemOneModelMax)
@@ -1692,7 +1692,7 @@ export const LlmAnalyticsProviderKeysUpdateBody = /* @__PURE__ */ zod.object({
         ),
     name: zod.string().max(llmAnalyticsProviderKeysUpdateBodyNameMax),
     api_key: zod.string().optional(),
-    base_url: zod.url().optional().describe('System One API base URL, including \/v1.'),
+    base_url: zod.url().optional().describe('System One API base URL, ending before \/systemone.'),
     system_one_model: zod
         .string()
         .max(llmAnalyticsProviderKeysUpdateBodySystemOneModelMax)
@@ -1735,7 +1735,7 @@ export const LlmAnalyticsProviderKeysPartialUpdateBody = /* @__PURE__ */ zod.obj
         ),
     name: zod.string().max(llmAnalyticsProviderKeysPartialUpdateBodyNameMax).optional(),
     api_key: zod.string().optional(),
-    base_url: zod.url().optional().describe('System One API base URL, including \/v1.'),
+    base_url: zod.url().optional().describe('System One API base URL, ending before \/systemone.'),
     system_one_model: zod
         .string()
         .max(llmAnalyticsProviderKeysPartialUpdateBodySystemOneModelMax)
@@ -1777,7 +1777,7 @@ export const LlmAnalyticsProviderKeysValidateCreateBody = /* @__PURE__ */ zod.ob
         ),
     name: zod.string().max(llmAnalyticsProviderKeysValidateCreateBodyNameMax),
     api_key: zod.string().optional(),
-    base_url: zod.url().optional().describe('System One API base URL, including \/v1.'),
+    base_url: zod.url().optional().describe('System One API base URL, ending before \/systemone.'),
     system_one_model: zod
         .string()
         .max(llmAnalyticsProviderKeysValidateCreateBodySystemOneModelMax)
diff --git a/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx b/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
index 0dccd9c52d5b..aa7a9b0b4072 100644
--- a/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
+++ b/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
@@ -35,6 +35,7 @@ import {
     LLM_PROVIDER_LABELS,
     UpdateLLMProviderKeyPayload,
     llmProviderKeysLogic,
+    normalizeSystemOneBaseUrlForComparison,
     sortProviderKeys,
 } from './llmProviderKeysLogic'
 import { SystemOneConnectionFields } from './SystemOneConnectionFields'
@@ -107,7 +108,7 @@ function getKeyPlaceholder(provider: LLMProvider): string {
         case 'zeabur':
             return 'sk-...'
         case 'system_one':
-            return 'Enter your endpoint’s bearer token'
+            return "Enter your endpoint's bearer token"
     }
 }
 
@@ -432,7 +433,9 @@ function EditKeyModal({
     const { setEditingKey, updateProviderKey, preValidateKey, clearPreValidation } = useActions(llmProviderKeysLogic)
     const isAzureEdit = keyToEdit.provider === 'azure_openai'
     const isSystemOne = keyToEdit.provider === 'system_one'
-    const endpointChanged = systemOneBaseUrl !== (keyToEdit.base_url_display ?? DEFAULT_SYSTEM_ONE_BASE_URL)
+    const endpointChanged =
+        normalizeSystemOneBaseUrlForComparison(systemOneBaseUrl) !==
+        normalizeSystemOneBaseUrlForComparison(keyToEdit.base_url_display ?? DEFAULT_SYSTEM_ONE_BASE_URL)
 
     const [name, setName] = useState(keyToEdit.name)
     const [apiKey, setApiKey] = useState('')
@@ -571,7 +574,7 @@ function EditKeyModal({
                         onBlur={handleApiKeyBlur}
                         placeholder={
                             isSystemOne && endpointChanged
-                                ? 'Enter the new endpoint’s bearer token'
+                                ? "Enter the new endpoint's bearer token"
                                 : `Leave empty to keep current (${keyToEdit.api_key_masked})`
                         }
                         type="password"
diff --git a/products/ai_observability/frontend/settings/llmProviderKeysLogic.test.ts b/products/ai_observability/frontend/settings/llmProviderKeysLogic.test.ts
index 7d656209b497..cc4d66b289ba 100644
--- a/products/ai_observability/frontend/settings/llmProviderKeysLogic.test.ts
+++ b/products/ai_observability/frontend/settings/llmProviderKeysLogic.test.ts
@@ -1,6 +1,6 @@
-import { normalizeLLMProvider } from './llmProviderKeysLogic'
+import { normalizeLLMProvider, normalizeSystemOneBaseUrlForComparison, toLLMProvider } from './llmProviderKeysLogic'
 
-describe('normalizeLLMProvider', () => {
+describe('llmProviderKeysLogic', () => {
     it.each([
         ['google', 'gemini'],
         ['google-ai-studio', 'gemini'],
@@ -31,4 +31,15 @@ describe('normalizeLLMProvider', () => {
         expect(normalizeLLMProvider(undefined)).toBeNull()
         expect(normalizeLLMProvider('unknown-provider')).toBeNull()
     })
+
+    it('recognizes the System One display name in model pickers', () => {
+        expect(toLLMProvider('System One')).toBe('system_one')
+    })
+
+    it.each([
+        ['https://DECISIONS.example.com/v1/', 'https://decisions.example.com/v1'],
+        ['https://decisions.example.com:443/v1', 'https://decisions.example.com/v1'],
+    ])('treats %s as the same endpoint as %s', (input, expected) => {
+        expect(normalizeSystemOneBaseUrlForComparison(input)).toBe(expected)
+    })
 })
diff --git a/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts b/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts
index cf100a81b97a..78c4d1e3dddc 100644
--- a/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts
+++ b/products/ai_observability/frontend/settings/llmProviderKeysLogic.ts
@@ -37,6 +37,9 @@ export function isLLMProvider(value: string): value is LLMProvider {
 /** Normalize a raw provider string to an LLMProvider, or null if unrecognized. */
 export function toLLMProvider(raw: string): LLMProvider | null {
     const normalized = raw.toLowerCase()
+    if (normalized === 'system one') {
+        return 'system_one'
+    }
     if (isLLMProvider(normalized)) {
         return normalized
     }
@@ -44,6 +47,15 @@ export function toLLMProvider(raw: string): LLMProvider | null {
     return null
 }
 
+export function normalizeSystemOneBaseUrlForComparison(input: string): string {
+    try {
+        const url = new URL(input.trim())
+        return `${url.origin}${url.pathname.replace(/\/+$/, '')}${url.search}${url.hash}`
+    } catch {
+        return input.trim().replace(/\/+$/, '')
+    }
+}
+
 const PROVIDER_ORDER = Object.keys(LLM_PROVIDER_LABELS) as LLMProvider[]
 
 /** Sort index for a provider string. Unknown providers sort last. */
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 220b0ecd8406..3b09a23bee27 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -38681,7 +38681,7 @@ export namespace Schemas {
       readonly error_message: string | null;
       api_key?: string;
       readonly api_key_masked: string;
-      /** System One API base URL, including /v1. */
+      /** System One API base URL, ending before /systemone. */
       base_url?: string;
       /**
          * Model ID served by the System One endpoint.
@@ -74028,7 +74028,7 @@ export namespace Schemas {
       readonly error_message?: string | null;
       api_key?: string;
       readonly api_key_masked?: string;
-      /** System One API base URL, including /v1. */
+      /** System One API base URL, ending before /systemone. */
       base_url?: string;
       /**
          * Model ID served by the System One endpoint.

From badbc00f3a4858eba32f2abe9b5736e99d856718 Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Sat, 26 Sep 2026 16:38:05 +0200
Subject: [PATCH 21/30] chore(aio): regenerate evaluation probability taxonomy

---
 .../core-filter-definitions-by-group.json        | 16 ++++++++++++++++
 .../lib/trace-property-allowlist.generated.ts    |  1 +
 2 files changed, 17 insertions(+)

diff --git a/frontend/src/taxonomy/core-filter-definitions-by-group.json b/frontend/src/taxonomy/core-filter-definitions-by-group.json
index 297e1991d1be..e398158787ae 100644
--- a/frontend/src/taxonomy/core-filter-definitions-by-group.json
+++ b/frontend/src/taxonomy/core-filter-definitions-by-group.json
@@ -373,6 +373,14 @@
             "label": "AI evaluation numeric result minimum",
             "type": "Numeric"
         },
+        "$ai_evaluation_probability": {
+            "description": "The probability of a true verdict for a boolean evaluation.",
+            "examples": [
+                0.9
+            ],
+            "label": "AI evaluation probability",
+            "type": "Numeric"
+        },
         "$ai_evaluation_provider": {
             "description": "The LLM provider used as the judge (e.g., openai, anthropic, gemini).",
             "examples": [
@@ -5236,6 +5244,14 @@
             "label": "AI evaluation numeric result minimum",
             "type": "Numeric"
         },
+        "$ai_evaluation_probability": {
+            "description": "The probability of a true verdict for a boolean evaluation.",
+            "examples": [
+                0.9
+            ],
+            "label": "AI evaluation probability",
+            "type": "Numeric"
+        },
         "$ai_evaluation_provider": {
             "description": "The LLM provider used as the judge (e.g., openai, anthropic, gemini).",
             "examples": [
diff --git a/services/mcp/src/lib/trace-property-allowlist.generated.ts b/services/mcp/src/lib/trace-property-allowlist.generated.ts
index c53d58cfdbcd..617358af686b 100644
--- a/services/mcp/src/lib/trace-property-allowlist.generated.ts
+++ b/services/mcp/src/lib/trace-property-allowlist.generated.ts
@@ -39,6 +39,7 @@ export const AI_TAXONOMY_EVENT_PROPERTIES = [
     '$ai_evaluation_numeric_result',
     '$ai_evaluation_numeric_result_max',
     '$ai_evaluation_numeric_result_min',
+    '$ai_evaluation_probability',
     '$ai_evaluation_provider',
     '$ai_evaluation_reasoning',
     '$ai_evaluation_result',

From 09b0bba1fde5cdcf0dea820f5dceee8f9872f26d Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Sat, 26 Sep 2026 17:01:25 +0200
Subject: [PATCH 22/30] fix(aio): restore evaluation badge reasoning fallback

---
 .../frontend/components/EvalResultBadges.test.tsx  | 14 +++++++++++++-
 .../frontend/components/EvalResultBadges.tsx       |  2 +-
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/products/ai_observability/frontend/components/EvalResultBadges.test.tsx b/products/ai_observability/frontend/components/EvalResultBadges.test.tsx
index b691926cf043..581c3910d479 100644
--- a/products/ai_observability/frontend/components/EvalResultBadges.test.tsx
+++ b/products/ai_observability/frontend/components/EvalResultBadges.test.tsx
@@ -8,7 +8,13 @@ import { initKeaTests } from '~/test/init'
 import { llmEvaluationsLogic } from '../evaluations/llmEvaluationsLogic'
 import { EvaluationConfig, EvaluationRun } from '../evaluations/types'
 import { generationEvaluationRunsLogic } from '../generationEvaluationRunsLogic'
-import { EvalResultBadges, getEvalBadgeProps, getEvalSummaries, scopeRunsToTarget } from './EvalResultBadges'
+import {
+    EvalResultBadges,
+    EvalTooltipContent,
+    getEvalBadgeProps,
+    getEvalSummaries,
+    scopeRunsToTarget,
+} from './EvalResultBadges'
 import {
     compareEvaluationResults,
     getEvaluationResultDisplay,
@@ -32,6 +38,12 @@ function makeRun(overrides: Partial<EvaluationRun> = {}): EvaluationRun {
 }
 
 describe('EvalResultBadges', () => {
+    it('shows a fallback when a run has no reasoning', () => {
+        render(<EvalTooltipContent latestRun={makeRun()} runCount={1} />)
+
+        expect(screen.getByText('No reasoning provided')).toBeInTheDocument()
+    })
+
     it('sorts numeric scores together without colliding with status ranks', () => {
         const rows = [
             makeRun({ id: 'negative', result_type: 'numeric', score: -10 }),
diff --git a/products/ai_observability/frontend/components/EvalResultBadges.tsx b/products/ai_observability/frontend/components/EvalResultBadges.tsx
index f09872c6144d..1d41fb6fcfc1 100644
--- a/products/ai_observability/frontend/components/EvalResultBadges.tsx
+++ b/products/ai_observability/frontend/components/EvalResultBadges.tsx
@@ -58,7 +58,7 @@ export function EvalTooltipContent({ latestRun, runCount }: EvalSummary): JSX.El
                 {dayjs(latestRun.timestamp).fromNow()}
                 {runCount > 1 && <> &middot; {pluralize(runCount, 'run', 'runs', true)} total</>}
             </div>
-            {latestRun.reasoning && <div className="text-sm">{latestRun.reasoning}</div>}
+            <div className="text-sm">{latestRun.reasoning || 'No reasoning provided'}</div>
         </div>
     )
 }

From 086a7ee56487a39b610de8ff7609b83ada4d4c9a Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Sun, 27 Sep 2026 22:17:55 +0200
Subject: [PATCH 23/30] fix(aio): distinguish blocked endpoints from rejected
 inputs

---
 .../temporal/ai_observability/evaluation_errors.py    |  6 +++---
 .../temporal/ai_observability/evaluation_llm_judge.py |  4 ++--
 .../temporal/ai_observability/test_run_evaluation.py  | 11 ++++++++---
 products/ai_observability/backend/llm/system_one.py   |  3 ++-
 4 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/posthog/temporal/ai_observability/evaluation_errors.py b/posthog/temporal/ai_observability/evaluation_errors.py
index 884cfa425bd1..0d6267d0702d 100644
--- a/posthog/temporal/ai_observability/evaluation_errors.py
+++ b/posthog/temporal/ai_observability/evaluation_errors.py
@@ -31,10 +31,10 @@ class EvaluationErrorSpec:
 
 
 USER_ERROR_SPECS: dict[str, EvaluationErrorSpec] = {
-    "request_rejected": EvaluationErrorSpec(
-        error_type="request_rejected",
+    "endpoint_blocked": EvaluationErrorSpec(
+        error_type="endpoint_blocked",
         owner="user",
-        safe_message="The judge endpoint rejected the request. Check the connection settings and evaluation criteria.",
+        safe_message="The judge endpoint is not allowed. Check the connection settings.",
         status_reason=EvaluationStatusReason.PROVIDER_KEY_INVALID,
         disables_evaluation=True,
         provider_key_state=LLMProviderKey.State.ERROR,
diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.py
index f1fce2aae59d..8a3e295345ee 100644
--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py
+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py
@@ -539,9 +539,9 @@ def call_llm_judge(
                 )
             )
     except SystemOneEndpointBlockedError as e:
-        increment_user_errors("request_rejected", provider=provider)
+        increment_user_errors("endpoint_blocked", provider=provider)
         return terminal_user_error_result(
-            spec=require_user_error_spec("request_rejected", is_byok=is_byok),
+            spec=require_user_error_spec("endpoint_blocked", is_byok=is_byok),
             message=str(e),
             allows_na=allows_na,
             output_type=output_type,
diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index ac2605bd963c..a25f7a3cbce9 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -234,8 +234,13 @@ def test_system_one_restricted_connection_does_not_send_evaluation_data(base_url
     request.assert_not_called()
 
 
-@pytest.mark.parametrize("status", [301, 400, 422])
-def test_system_one_rejections_preserve_shared_key_for_input_errors(status: int) -> None:
+@pytest.mark.parametrize(
+    "status, expected_skip_reason",
+    [(301, "endpoint_blocked"), (400, "request_rejected"), (422, "request_rejected")],
+)
+def test_system_one_rejections_distinguish_blocked_endpoints_from_bad_inputs(
+    status: int, expected_skip_reason: str
+) -> None:
     key = MagicMock(
         provider="system_one",
         encrypted_config={"api_key": "example-token", "base_url": "https://decisions.example.com/v1"},
@@ -261,7 +266,7 @@ def test_system_one_rejections_preserve_shared_key_for_input_errors(status: int)
             user_prompt="Hello!",
             allows_na=False,
         )
-    assert result["skip_reason"] == "request_rejected"
+    assert result["skip_reason"] == expected_skip_reason
     if status == 301:
         assert result["terminal_user_error"] is True
         assert result["provider_key_state"] == "error"
diff --git a/products/ai_observability/backend/llm/system_one.py b/products/ai_observability/backend/llm/system_one.py
index 11eaadc92d4d..c13f8e2a6125 100644
--- a/products/ai_observability/backend/llm/system_one.py
+++ b/products/ai_observability/backend/llm/system_one.py
@@ -60,7 +60,7 @@ class SystemOneRequestRejectedError(LLMError):
     pass
 
 
-class SystemOneEndpointBlockedError(SystemOneRequestRejectedError):
+class SystemOneEndpointBlockedError(LLMError):
     pass
 
 
@@ -187,6 +187,7 @@ def validate_key(api_key: str, *, base_url: str, model: str) -> tuple[str, str |
             ProviderConnectionError,
             RateLimitError,
             StructuredOutputParseError,
+            SystemOneEndpointBlockedError,
             SystemOneRequestRejectedError,
             ContextWindowExceededError,
         ) as error:

From ff3bf972bec9217d86d4f63eeb2de0c92aa53a39 Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Sun, 27 Sep 2026 22:31:15 +0200
Subject: [PATCH 24/30] fix(ci): update pytest-split fork pin

---
 pyproject.toml | 2 +-
 uv.lock        | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 120704a81f70..586c5fdc9369 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -323,7 +323,7 @@ infi-clickhouse-orm = { git = "https://github.com/PostHog/infi.clickhouse_orm",
 # optimal_chunks split. Fork: PostHog/pytest-split (of jerry-git/pytest-split).
 # To pull upstream fixes: rebase the fork's feat branch onto upstream, cut the next
 # v0.11.0-posthog.N tag, and bump this pin.
-pytest-split = { git = "https://github.com/PostHog/pytest-split", tag = "v0.11.0-posthog.7" }
+pytest-split = { git = "https://github.com/PostHog/pytest-split", tag = "v0.11.0-posthog.8" }
 
 # xgboost's Linux wheel declares nvidia-nccl-cu12 (322 MB) for multi-GPU training. We use xgboost on
 # CPU, so drop it to keep the wheel out of the image.
diff --git a/uv.lock b/uv.lock
index 9a6068eca853..5479c42306f4 100644
--- a/uv.lock
+++ b/uv.lock
@@ -6127,7 +6127,7 @@ dev = [
     { name = "pytest-icdiff", specifier = "~=0.9" },
     { name = "pytest-mock", specifier = "~=3.15.1" },
     { name = "pytest-rerunfailures", specifier = "~=16.1" },
-    { name = "pytest-split", git = "https://github.com/PostHog/pytest-split?tag=v0.11.0-posthog.7" },
+    { name = "pytest-split", git = "https://github.com/PostHog/pytest-split?tag=v0.11.0-posthog.8" },
     { name = "pytest-timeout", specifier = "~=2.4.0" },
     { name = "pytest-xdist", specifier = "~=3.8.0" },
     { name = "python-dateutil", specifier = "~=2.9.0" },
@@ -6946,7 +6946,7 @@ wheels = [
 [[package]]
 name = "pytest-split"
 version = "0.11.0"
-source = { git = "https://github.com/PostHog/pytest-split?tag=v0.11.0-posthog.7#17ee53be4aaa2f27012da31b69656b4b45881cd5" }
+source = { git = "https://github.com/PostHog/pytest-split?tag=v0.11.0-posthog.8#559e2197cbb36b99810d95cf470aba4e69910776" }
 dependencies = [
     { name = "pytest" },
 ]

From 12cd175fc7ae385d97801ecece3b978acc42e8f7 Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Sun, 27 Sep 2026 23:09:46 +0200
Subject: [PATCH 25/30] fix(aio): restore model picker story width

---
 .../ai_observability/frontend/ByokModelPickerNotice.stories.tsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/products/ai_observability/frontend/ByokModelPickerNotice.stories.tsx b/products/ai_observability/frontend/ByokModelPickerNotice.stories.tsx
index 64ff6f187dd7..a0c0cf117bb1 100644
--- a/products/ai_observability/frontend/ByokModelPickerNotice.stories.tsx
+++ b/products/ai_observability/frontend/ByokModelPickerNotice.stories.tsx
@@ -24,7 +24,7 @@ function PickerWithNotice({ systemOne = false }: { systemOne?: boolean }): JSX.E
 
     return (
         // The story root has no width of its own, so the picker column is sized here to match the form it sits in.
-        <div className="w-full max-w-[560px]">
+        <div className="w-[560px]">
             <ModelPicker
                 model=""
                 selectedProviderKeyId={null}

From 0d2c2204adc519f1045216f9676c052f35060b26 Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Sun, 27 Sep 2026 23:44:40 +0200
Subject: [PATCH 26/30] fix(aio): show one add-key action in empty state

---
 .../settings/LLMProviderKeysSettings.tsx      | 22 ++++++++++---------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx b/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
index aa7a9b0b4072..4f1c98919445 100644
--- a/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
+++ b/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
@@ -840,16 +840,18 @@ export function LLMProviderKeysSettings(): JSX.Element {
                             </LemonBanner>
                         )}
 
-                        <div className="flex justify-between items-start">
-                            <LemonButton
-                                type="primary"
-                                icon={<IconPlus />}
-                                onClick={() => setNewKeyModalOpen(true)}
-                                disabledReason={restrictionReason}
-                            >
-                                Add API key
-                            </LemonButton>
-                        </div>
+                        {providerKeys.length > 0 && (
+                            <div className="flex justify-between items-start">
+                                <LemonButton
+                                    type="primary"
+                                    icon={<IconPlus />}
+                                    onClick={() => setNewKeyModalOpen(true)}
+                                    disabledReason={restrictionReason}
+                                >
+                                    Add API key
+                                </LemonButton>
+                            </div>
+                        )}
 
                         {providerKeys.length === 0 ? (
                             <div className="border rounded-lg p-8 flex flex-col items-center">

From da9f9a76a175bfc9e1f1fca26b83690019b80421 Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Mon, 28 Sep 2026 11:08:53 +0200
Subject: [PATCH 27/30] fix(aio): bound System One endpoint responses

---
 posthog/egress/test/test_typesafe.py          | 55 +++++++++++-
 posthog/egress/typesafe/README.md             |  1 +
 posthog/egress/typesafe/client.py             | 45 +++++++---
 posthog/egress/typesafe/transport.py          |  2 +-
 .../backend/llm/test/test_system_one.py       | 85 ++++++++++++-------
 5 files changed, 142 insertions(+), 46 deletions(-)

diff --git a/posthog/egress/test/test_typesafe.py b/posthog/egress/test/test_typesafe.py
index c8cc8d5addf1..34b995cd105b 100644
--- a/posthog/egress/test/test_typesafe.py
+++ b/posthog/egress/test/test_typesafe.py
@@ -1,4 +1,5 @@
 import json
+from io import BytesIO
 from typing import Any
 
 from unittest.mock import patch
@@ -11,7 +12,7 @@
 
 from posthog.egress.limiter.policies import Priority, resolve_policy
 from posthog.egress.observability.observability import scope_fingerprint
-from posthog.egress.typesafe.client import TypeSafeNotConfigured, TypeSafeRequestFailed, system_one
+from posthog.egress.typesafe.client import MAX_RESPONSE_BYTES, TypeSafeNotConfigured, TypeSafeRequestFailed, system_one
 from posthog.egress.typesafe.limiter import typesafe_account_key
 from posthog.llm.system_one import ChoiceAnswer, ChoiceQuestion, NoulAnswer, NoulQuestion, Question
 
@@ -45,10 +46,12 @@
 }
 
 
-def _response(status: int, body: str) -> requests.Response:
+def _response(status: int, body: str, *, content_length: str | None = None) -> requests.Response:
     response = requests.models.Response()
     response.status_code = status
-    response._content = body.encode()
+    response.raw = BytesIO(body.encode())
+    if content_length is not None:
+        response.headers["Content-Length"] = content_length
     return response
 
 
@@ -93,7 +96,9 @@ def test_sends_the_documented_request_and_records_it_without_the_key(
         assert request.call_args.args == ("POST", f"{base_url}/systemone")
         kwargs = request.call_args.kwargs
         assert kwargs["headers"].get("Authorization") == (f"Bearer {resolved_key}" if resolved_key else None)
+        assert kwargs["headers"]["Accept-Encoding"] == "identity"
         assert kwargs["allow_redirects"] is False
+        assert kwargs["stream"] is True
         assert kwargs["json"] == {
             "model": "jev-latest",
             "state": {"ticket": "Payouts fail"},
@@ -170,6 +175,50 @@ def test_raises_rather_than_returning_a_partial_answer(
             system_one(state="Payouts fail", questions=_QUESTIONS, source="test")
         assert raised.exception.status_code == expected_status_code
 
+    @parameterized.expand([("declared", True), ("chunked", False)])
+    def test_rejects_oversized_endpoint_responses(self, _name: str, declared: bool) -> None:
+        response = _response(
+            200,
+            "x" * (MAX_RESPONSE_BYTES + 1),
+            content_length=str(MAX_RESPONSE_BYTES + 1) if declared else None,
+        )
+        with (
+            patch("posthog.egress.typesafe.transport.consume_typesafe_sync", return_value=True),
+            patch("requests.request", return_value=response),
+            patch.object(response.raw, "read", wraps=response.raw.read) as read,
+            self.assertRaisesRegex(TypeSafeRequestFailed, "oversized body"),
+        ):
+            system_one(state="hello", questions=_QUESTIONS, source="test")
+        assert response.raw.closed
+        if declared:
+            read.assert_not_called()
+
+    def test_stops_reading_a_response_after_the_total_time_limit(self) -> None:
+        response = _response(200, json.dumps(_ANSWERS))
+        with (
+            patch("posthog.egress.typesafe.transport.consume_typesafe_sync", return_value=True),
+            patch("requests.request", return_value=response),
+            patch.object(response.raw, "read", wraps=response.raw.read) as read,
+            patch("posthog.egress.typesafe.client.monotonic", side_effect=[0, 0, 2]),
+            self.assertRaisesRegex(TypeSafeRequestFailed, "time limit"),
+        ):
+            system_one(state="hello", questions=_QUESTIONS, source="test", timeout=1)
+        assert response.raw.closed
+        assert read.call_count == 2
+
+    def test_rejects_compressed_responses_before_reading(self) -> None:
+        response = _response(200, json.dumps(_ANSWERS))
+        response.headers["Content-Encoding"] = "gzip"
+        with (
+            patch("posthog.egress.typesafe.transport.consume_typesafe_sync", return_value=True),
+            patch("requests.request", return_value=response),
+            patch.object(response.raw, "read", wraps=response.raw.read) as read,
+            self.assertRaisesRegex(TypeSafeRequestFailed, "compressed body"),
+        ):
+            system_one(state="hello", questions=_QUESTIONS, source="test")
+        assert response.raw.closed
+        read.assert_not_called()
+
     @parameterized.expand(
         [
             ("no_configured_key", "", "https://api.typesafe.ai/v1", Priority.NORMAL, TypeSafeNotConfigured),
diff --git a/posthog/egress/typesafe/README.md b/posthog/egress/typesafe/README.md
index 8ad21289af30..f0ec91250081 100644
--- a/posthog/egress/typesafe/README.md
+++ b/posthog/egress/typesafe/README.md
@@ -85,6 +85,7 @@ Callers use `client.py` rather than `typesafe_request`.
 `system_one(state=..., questions=..., source=..., model=...)` sends one state with a map of `NoulQuestion` and `ChoiceQuestion` entries.
 It returns a `SystemOneResult` with one `NoulAnswer` or `ChoiceAnswer` per question id, the versioned model that answered, and available input and output token counts.
 It raises `TypeSafeRequestFailed` on an HTTP error, and on a body that lacks the answering model or a complete answer for any question.
+The client requests uncompressed responses, streams up to 1 MiB, and checks the request's total time budget while reading, including for a 422 error body.
 A choice outside the options the caller sent, or a choice without a probability for every option, counts as incomplete.
 `model` defaults to the `jev-latest` alias. A caller that tunes thresholds against one version pins that version's id, such as `jev-1.13.0`.
 Only `POST /v1/systemone` is wired up, and score questions are not.
diff --git a/posthog/egress/typesafe/client.py b/posthog/egress/typesafe/client.py
index 94c2f9c5f80e..5f009f74812c 100644
--- a/posthog/egress/typesafe/client.py
+++ b/posthog/egress/typesafe/client.py
@@ -10,6 +10,7 @@
 """
 
 from collections.abc import Mapping
+from time import monotonic
 
 from django.conf import settings
 
@@ -32,6 +33,7 @@
 
 TYPESAFE_API_BASE = "https://api.typesafe.ai"
 SYSTEM_ONE_ENDPOINT = SYSTEM_ONE_PATH
+MAX_RESPONSE_BYTES = 1_048_576
 
 # The alias moves to each new release. A caller that tunes thresholds against one version pins that
 # version's id instead, so a release cannot shift its answers without a code change.
@@ -87,6 +89,7 @@ def system_one(
         else scope_fingerprint(base_url, resolved_api_key)
     )
 
+    deadline = monotonic() + (sum(timeout) if isinstance(timeout, tuple) else timeout)
     response = typesafe_request(
         "POST",
         f"{base_url}/systemone",
@@ -97,20 +100,42 @@ def system_one(
         priority=priority,
         timeout=timeout,
         allow_redirects=False,
+        stream=True,
         session=session,
         json=build_system_one_body(state=state, questions=questions, model=model),
     )
-
-    if response.status_code != 200:
-        # A 422 body echoes the offending field, which can carry the state, so keep the body out of
-        # the exception that gets logged.
-        raise TypeSafeRequestFailed(
-            f"TypeSafe returned HTTP {response.status_code}", status_code=response.status_code, response=response
-        )
     try:
-        payload: object = response.json()
-    except ValueError as exc:
-        raise TypeSafeRequestFailed("TypeSafe returned a non-JSON body") from exc
+        if response.status_code not in (200, 422):
+            raise TypeSafeRequestFailed(
+                f"TypeSafe returned HTTP {response.status_code}", status_code=response.status_code, response=response
+            )
+        if response.headers.get("Content-Encoding", "identity").lower() != "identity":
+            raise TypeSafeRequestFailed("TypeSafe returned a compressed body")
+        content_length = response.headers.get("Content-Length")
+        if content_length:
+            try:
+                if int(content_length) > MAX_RESPONSE_BYTES:
+                    raise TypeSafeRequestFailed("TypeSafe returned an oversized body")
+            except ValueError:
+                pass
+        body = bytearray()
+        # boffin: Check each byte so a trickling endpoint cannot keep a larger read open indefinitely.
+        for chunk in response.iter_content(chunk_size=1):
+            if monotonic() >= deadline:
+                raise TypeSafeRequestFailed("TypeSafe response exceeded the time limit")
+            if len(body) + len(chunk) > MAX_RESPONSE_BYTES:
+                raise TypeSafeRequestFailed("TypeSafe returned an oversized body")
+            body.extend(chunk)
+        # A 422 body may describe a context limit, but it can also echo state; keep it out of logs.
+        response._content = bytes(body)
+        if response.status_code != 200:
+            raise TypeSafeRequestFailed("TypeSafe returned HTTP 422", status_code=422, response=response)
+        try:
+            payload: object = response.json()
+        except ValueError as exc:
+            raise TypeSafeRequestFailed("TypeSafe returned a non-JSON body") from exc
+    finally:
+        response.close()
     try:
         return parse_system_one_response(payload, questions)
     except SystemOneRequestFailed as exc:
diff --git a/posthog/egress/typesafe/transport.py b/posthog/egress/typesafe/transport.py
index 196d99327b40..56b789c66587 100644
--- a/posthog/egress/typesafe/transport.py
+++ b/posthog/egress/typesafe/transport.py
@@ -28,7 +28,7 @@ class TypeSafeClient(EgressClient):
     observability = typesafe_egress
 
     def _standard_headers(self) -> dict[str, str]:
-        return {"Accept": "application/json", "Content-Type": "application/json"}
+        return {"Accept": "application/json", "Accept-Encoding": "identity", "Content-Type": "application/json"}
 
     def _consume(self, scope: str, priority: Priority, source: str, url: str) -> bool:
         return consume_typesafe_sync(scope=scope, priority=priority, source=source)
diff --git a/products/ai_observability/backend/llm/test/test_system_one.py b/products/ai_observability/backend/llm/test/test_system_one.py
index 4d5872fb8167..bd025e281e36 100644
--- a/products/ai_observability/backend/llm/test/test_system_one.py
+++ b/products/ai_observability/backend/llm/test/test_system_one.py
@@ -1,4 +1,6 @@
+import json
 from collections.abc import Iterator
+from io import BytesIO
 from ipaddress import ip_address
 from uuid import uuid4
 
@@ -7,6 +9,8 @@
 
 from django.test import override_settings
 
+import requests
+
 from posthog.llm.system_one import NoulAnswer, NoulQuestion
 from posthog.models import Team
 
@@ -27,6 +31,13 @@
 )
 
 
+def _response(status: int, body: dict[str, object] | str = "") -> requests.Response:
+    response = requests.Response()
+    response.status_code = status
+    response.raw = BytesIO((json.dumps(body) if isinstance(body, dict) else body).encode())
+    return response
+
+
 @pytest.fixture(autouse=True)
 def isolated_egress_budget() -> Iterator[None]:
     with (
@@ -67,12 +78,14 @@ def test_system_one_connections_require_flag_and_reserve_posthog_gateway_for_int
 
 @pytest.mark.parametrize("status, expected_state", [(200, "ok"), (401, "invalid"), (403, "invalid"), (500, "error")])
 def test_system_one_key_validation(status: int, expected_state: str) -> None:
-    response = Mock(status_code=status)
-    response.json.return_value = {
-        "model": "example-judge-v1",
-        "answers": {"verdict": {"type": "noul", "noul": 0.9}, "applicable": {"type": "noul", "noul": 0.9}},
-        "usage": {"input_tokens": 12, "output_tokens": 0},
-    }
+    response = _response(
+        status,
+        {
+            "model": "example-judge-v1",
+            "answers": {"verdict": {"type": "noul", "noul": 0.9}, "applicable": {"type": "noul", "noul": 0.9}},
+            "usage": {"input_tokens": 12, "output_tokens": 0},
+        },
+    )
     with patch("requests.Session.request", return_value=response) as request:
         state, message = Client.validate_key(
             "system_one", "example-token", base_url="https://decisions.example.com/v1", model="custom-model"
@@ -87,12 +100,14 @@ def test_system_one_key_validation(status: int, expected_state: str) -> None:
 
 @pytest.mark.parametrize("probability", [-0.1, 1.1, float("nan"), float("inf"), "0.8", True, None])
 def test_system_one_rejects_invalid_probabilities(probability: object) -> None:
-    response = Mock(status_code=200)
-    response.json.return_value = {
-        "model": "example-judge-v1",
-        "answers": {"verdict": {"type": "noul", "noul": probability}},
-        "usage": {"input_tokens": 120, "output_tokens": 10},
-    }
+    response = _response(
+        200,
+        {
+            "model": "example-judge-v1",
+            "answers": {"verdict": {"type": "noul", "noul": probability}},
+            "usage": {"input_tokens": 120, "output_tokens": 10},
+        },
+    )
     with (
         patch("requests.Session.request", return_value=response),
         pytest.raises(StructuredOutputParseError),
@@ -134,12 +149,14 @@ def test_system_one_rate_limits_are_retryable(status: int) -> None:
 def test_unavailable_usage_does_not_discard_a_valid_answer(
     usage: dict[str, object], expected_input: int | None, expected_output: int | None
 ) -> None:
-    response = Mock(status_code=200)
-    response.json.return_value = {
-        "model": "example-judge-v1",
-        "answers": {"verdict": {"type": "noul", "noul": 0.9}},
-        "usage": usage,
-    }
+    response = _response(
+        200,
+        {
+            "model": "example-judge-v1",
+            "answers": {"verdict": {"type": "noul", "noul": 0.9}},
+            "usage": usage,
+        },
+    )
     with patch("requests.Session.request", return_value=response):
         result = SystemOneClient.evaluate(
             api_key="example-token",
@@ -174,12 +191,14 @@ def test_official_endpoint_is_blocked(base_url: str) -> None:
 
 @pytest.mark.parametrize("answers", [{}, {"verdict": {"type": "noul", "noul": 0.9}}])
 def test_system_one_requires_every_requested_answer(answers: dict[str, object]) -> None:
-    response = Mock(status_code=200)
-    response.json.return_value = {
-        "model": "example-judge-v1",
-        "answers": answers,
-        "usage": {"input_tokens": 12, "output_tokens": 2},
-    }
+    response = _response(
+        200,
+        {
+            "model": "example-judge-v1",
+            "answers": answers,
+            "usage": {"input_tokens": 12, "output_tokens": 2},
+        },
+    )
     with (
         patch("requests.Session.request", return_value=response),
         pytest.raises(StructuredOutputParseError),
@@ -208,7 +227,7 @@ def test_system_one_requires_every_requested_answer(answers: dict[str, object])
     ],
 )
 def test_system_one_preserves_error_categories(status: int, message: str, error_type: type[Exception]) -> None:
-    response = Mock(status_code=status, text=message)
+    response = _response(status, message)
     with (
         patch("requests.Session.request", return_value=response),
         pytest.raises(error_type),
@@ -229,13 +248,15 @@ def test_system_one_preserves_error_categories(status: int, message: str, error_
 )
 @pytest.mark.parametrize("api_key", ["example-token", ""])
 def test_custom_endpoint_and_model(api_key: str) -> None:
-    response = Mock(status_code=200)
-    response.json.return_value = {
-        "model": "custom-model-revision",
-        "answers": {"verdict": {"noul": 0.7}, "applicable": {"noul": 1.0}},
-        "usage": {"input_tokens": 15},
-        "latency_ms": 42,
-    }
+    response = _response(
+        200,
+        {
+            "model": "custom-model-revision",
+            "answers": {"verdict": {"noul": 0.7}, "applicable": {"noul": 1.0}},
+            "usage": {"input_tokens": 15},
+            "latency_ms": 42,
+        },
+    )
     with patch("requests.Session.request", return_value=response) as request:
         result = SystemOneClient.evaluate(
             api_key=api_key,

From f8360e83ae5ff09d618ffa2f16fba29929d636de Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Mon, 28 Sep 2026 11:38:23 +0200
Subject: [PATCH 28/30] fix(aio): simplify bounded System One responses

---
 posthog/egress/test/test_typesafe.py | 42 ++++++++---------------
 posthog/egress/typesafe/client.py    | 50 +++++++++++++---------------
 2 files changed, 36 insertions(+), 56 deletions(-)

diff --git a/posthog/egress/test/test_typesafe.py b/posthog/egress/test/test_typesafe.py
index 34b995cd105b..385e189cfc25 100644
--- a/posthog/egress/test/test_typesafe.py
+++ b/posthog/egress/test/test_typesafe.py
@@ -46,12 +46,10 @@
 }
 
 
-def _response(status: int, body: str, *, content_length: str | None = None) -> requests.Response:
+def _response(status: int, body: str) -> requests.Response:
     response = requests.models.Response()
     response.status_code = status
     response.raw = BytesIO(body.encode())
-    if content_length is not None:
-        response.headers["Content-Length"] = content_length
     return response
 
 
@@ -175,23 +173,22 @@ def test_raises_rather_than_returning_a_partial_answer(
             system_one(state="Payouts fail", questions=_QUESTIONS, source="test")
         assert raised.exception.status_code == expected_status_code
 
-    @parameterized.expand([("declared", True), ("chunked", False)])
-    def test_rejects_oversized_endpoint_responses(self, _name: str, declared: bool) -> None:
-        response = _response(
-            200,
-            "x" * (MAX_RESPONSE_BYTES + 1),
-            content_length=str(MAX_RESPONSE_BYTES + 1) if declared else None,
-        )
+    @parameterized.expand(
+        [
+            ("declared", "{}", {"Content-Length": str(MAX_RESPONSE_BYTES + 1)}),
+            ("streamed", "x" * (MAX_RESPONSE_BYTES + 1), {}),
+            ("compressed", json.dumps(_ANSWERS), {"Content-Encoding": "gzip"}),
+        ]
+    )
+    def test_rejects_endpoint_responses_over_its_limits(self, _name: str, body: str, headers: dict[str, str]) -> None:
+        response = _response(200, body)
+        response.headers.update(headers)
         with (
             patch("posthog.egress.typesafe.transport.consume_typesafe_sync", return_value=True),
             patch("requests.request", return_value=response),
-            patch.object(response.raw, "read", wraps=response.raw.read) as read,
-            self.assertRaisesRegex(TypeSafeRequestFailed, "oversized body"),
+            self.assertRaisesRegex(TypeSafeRequestFailed, "exceeded its limits"),
         ):
             system_one(state="hello", questions=_QUESTIONS, source="test")
-        assert response.raw.closed
-        if declared:
-            read.assert_not_called()
 
     def test_stops_reading_a_response_after_the_total_time_limit(self) -> None:
         response = _response(200, json.dumps(_ANSWERS))
@@ -200,25 +197,12 @@ def test_stops_reading_a_response_after_the_total_time_limit(self) -> None:
             patch("requests.request", return_value=response),
             patch.object(response.raw, "read", wraps=response.raw.read) as read,
             patch("posthog.egress.typesafe.client.monotonic", side_effect=[0, 0, 2]),
-            self.assertRaisesRegex(TypeSafeRequestFailed, "time limit"),
+            self.assertRaisesRegex(TypeSafeRequestFailed, "exceeded its limits"),
         ):
             system_one(state="hello", questions=_QUESTIONS, source="test", timeout=1)
         assert response.raw.closed
         assert read.call_count == 2
 
-    def test_rejects_compressed_responses_before_reading(self) -> None:
-        response = _response(200, json.dumps(_ANSWERS))
-        response.headers["Content-Encoding"] = "gzip"
-        with (
-            patch("posthog.egress.typesafe.transport.consume_typesafe_sync", return_value=True),
-            patch("requests.request", return_value=response),
-            patch.object(response.raw, "read", wraps=response.raw.read) as read,
-            self.assertRaisesRegex(TypeSafeRequestFailed, "compressed body"),
-        ):
-            system_one(state="hello", questions=_QUESTIONS, source="test")
-        assert response.raw.closed
-        read.assert_not_called()
-
     @parameterized.expand(
         [
             ("no_configured_key", "", "https://api.typesafe.ai/v1", Priority.NORMAL, TypeSafeNotConfigured),
diff --git a/posthog/egress/typesafe/client.py b/posthog/egress/typesafe/client.py
index 5f009f74812c..79edc1bff20a 100644
--- a/posthog/egress/typesafe/client.py
+++ b/posthog/egress/typesafe/client.py
@@ -105,37 +105,33 @@ def system_one(
         json=build_system_one_body(state=state, questions=questions, model=model),
     )
     try:
-        if response.status_code not in (200, 422):
-            raise TypeSafeRequestFailed(
-                f"TypeSafe returned HTTP {response.status_code}", status_code=response.status_code, response=response
-            )
-        if response.headers.get("Content-Encoding", "identity").lower() != "identity":
-            raise TypeSafeRequestFailed("TypeSafe returned a compressed body")
-        content_length = response.headers.get("Content-Length")
-        if content_length:
+        if response.status_code in (200, 422):
+            length = response.headers.get("Content-Length")
             try:
-                if int(content_length) > MAX_RESPONSE_BYTES:
-                    raise TypeSafeRequestFailed("TypeSafe returned an oversized body")
+                oversized = length is not None and int(length) > MAX_RESPONSE_BYTES
             except ValueError:
-                pass
-        body = bytearray()
-        # boffin: Check each byte so a trickling endpoint cannot keep a larger read open indefinitely.
-        for chunk in response.iter_content(chunk_size=1):
-            if monotonic() >= deadline:
-                raise TypeSafeRequestFailed("TypeSafe response exceeded the time limit")
-            if len(body) + len(chunk) > MAX_RESPONSE_BYTES:
-                raise TypeSafeRequestFailed("TypeSafe returned an oversized body")
-            body.extend(chunk)
-        # A 422 body may describe a context limit, but it can also echo state; keep it out of logs.
-        response._content = bytes(body)
-        if response.status_code != 200:
-            raise TypeSafeRequestFailed("TypeSafe returned HTTP 422", status_code=422, response=response)
-        try:
-            payload: object = response.json()
-        except ValueError as exc:
-            raise TypeSafeRequestFailed("TypeSafe returned a non-JSON body") from exc
+                oversized = False
+            if oversized or response.headers.get("Content-Encoding", "identity").lower() != "identity":
+                raise TypeSafeRequestFailed("TypeSafe response exceeded its limits")
+            body = bytearray()
+            # boffin: Check each byte so a trickling endpoint cannot keep a larger read open indefinitely.
+            for chunk in response.iter_content(chunk_size=1):
+                if monotonic() >= deadline or len(body) + len(chunk) > MAX_RESPONSE_BYTES:
+                    raise TypeSafeRequestFailed("TypeSafe response exceeded its limits")
+                body.extend(chunk)
+            response._content = bytes(body)
     finally:
         response.close()
+
+    if response.status_code != 200:
+        # A 422 body can echo the state, so keep it out of the exception that gets logged.
+        raise TypeSafeRequestFailed(
+            f"TypeSafe returned HTTP {response.status_code}", status_code=response.status_code, response=response
+        )
+    try:
+        payload: object = response.json()
+    except ValueError as exc:
+        raise TypeSafeRequestFailed("TypeSafe returned a non-JSON body") from exc
     try:
         return parse_system_one_response(payload, questions)
     except SystemOneRequestFailed as exc:

From f59fec8f06aead0d896beb8a03aba83ec55ef9a5 Mon Sep 17 00:00:00 2001
From: Bernat Torres <bernat.t@posthog.com>
Date: Mon, 28 Sep 2026 12:58:10 +0200
Subject: [PATCH 29/30] fix(aio): use streamed responses in System One tests

---
 .../ai_observability/test_run_evaluation.py   | 16 ++++++----
 .../backend/api/test/test_provider_keys.py    | 29 ++++++++++++++-----
 2 files changed, 31 insertions(+), 14 deletions(-)

diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index a25f7a3cbce9..ac1d6ef0d51c 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -1,6 +1,7 @@
 import json
 import uuid
 from datetime import UTC, datetime, timedelta
+from io import BytesIO
 from ipaddress import ip_address
 from typing import Any, cast
 
@@ -9,6 +10,7 @@
 
 from django.test import override_settings
 
+import requests
 import posthoganalytics
 from asgiref.sync import async_to_sync, sync_to_async
 from parameterized import parameterized
@@ -133,8 +135,7 @@ def test_system_one_judge_emits_boolean_probability_without_reasoning(
 ) -> None:
     key = MagicMock(provider="system_one", encrypted_config=connection_config)
     resolved = MagicMock(provider="system_one", model=model, provider_key=key, is_byok=True)
-    response = MagicMock(status_code=200)
-    response.json.return_value = {
+    response_body = {
         "model": "endpoint-controlled-model",
         "answers": {
             "verdict": {"type": "noul", "noul": probability},
@@ -142,6 +143,9 @@ def test_system_one_judge_emits_boolean_probability_without_reasoning(
         },
         "usage": usage,
     }
+    response = requests.Response()
+    response.status_code = 200
+    response.raw = BytesIO(json.dumps(response_body).encode())
     evaluation = {
         "id": "test-evaluation",
         "name": "Politeness",
@@ -245,6 +249,9 @@ def test_system_one_rejections_distinguish_blocked_endpoints_from_bad_inputs(
         provider="system_one",
         encrypted_config={"api_key": "example-token", "base_url": "https://decisions.example.com/v1"},
     )
+    response = requests.Response()
+    response.status_code = status
+    response.raw = BytesIO(b"Invalid request")
     with (
         patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),
         patch("posthog.egress.limiter.backends.LimitsBackend.consume_sync", return_value=True),
@@ -252,10 +259,7 @@ def test_system_one_rejections_distinguish_blocked_endpoints_from_bad_inputs(
         patch(
             "posthog.temporal.ai_observability.evaluation_llm_judge.system_one_evaluations_enabled", return_value=True
         ),
-        patch(
-            "requests.Session.request",
-            return_value=MagicMock(status_code=status, text="Invalid request"),
-        ),
+        patch("requests.Session.request", return_value=response),
     ):
         spec.return_value.resolve.return_value = MagicMock(
             provider="system_one", model="example-judge-v1", provider_key=key, is_byok=True
diff --git a/products/ai_observability/backend/api/test/test_provider_keys.py b/products/ai_observability/backend/api/test/test_provider_keys.py
index ac7918208959..b30c1f7ba3bc 100644
--- a/products/ai_observability/backend/api/test/test_provider_keys.py
+++ b/products/ai_observability/backend/api/test/test_provider_keys.py
@@ -1,3 +1,5 @@
+import json
+from io import BytesIO
 from ipaddress import ip_address
 from uuid import uuid4
 
@@ -8,6 +10,7 @@
 from django.test import SimpleTestCase
 from django.utils import timezone
 
+import requests
 from parameterized import parameterized
 from rest_framework import serializers, status
 
@@ -264,12 +267,22 @@ def test_can_create_provider_key(self, provider: str, mock_validate: Mock) -> No
     def test_custom_system_one_connection_round_trip(
         self, _flag: Mock, request: Mock, _budget: Mock, _dns: Mock
     ) -> None:
-        request.return_value = Mock(status_code=200)
-        request.return_value.json.return_value = {
-            "model": "custom-model",
-            "answers": {"verdict": {"type": "noul", "noul": 1.0}, "applicable": {"type": "noul", "noul": 1.0}},
-            "usage": {"input_tokens": 12, "output_tokens": 0},
-        }
+        body = json.dumps(
+            {
+                "model": "custom-model",
+                "answers": {"verdict": {"type": "noul", "noul": 1.0}, "applicable": {"type": "noul", "noul": 1.0}},
+                "usage": {"input_tokens": 12, "output_tokens": 0},
+            }
+        ).encode()
+        response_status = 200
+
+        def respond(*_args: object, **_kwargs: object) -> requests.Response:
+            response = requests.Response()
+            response.status_code = response_status
+            response.raw = BytesIO(body)
+            return response
+
+        request.side_effect = respond
         url = f"/api/environments/{self.team.id}/llm_analytics/provider_keys/"
         response = self.client.post(
             url,
@@ -294,7 +307,7 @@ def test_custom_system_one_connection_round_trip(
         request.assert_not_called()
         key.refresh_from_db()
         self.assertEqual(key.encrypted_config["base_url"], "https://decisions.example.com/v1")
-        request.return_value.status_code = 401
+        response_status = 401
         response = self.client.patch(
             f"{url}{key.id}/", {"base_url": "https://other.example.com/v1", "api_key": "fake-token"}
         )
@@ -302,7 +315,7 @@ def test_custom_system_one_connection_round_trip(
         key.refresh_from_db()
         self.assertEqual(key.encrypted_config["base_url"], "https://decisions.example.com/v1")
         self.assertEqual(key.encrypted_config["api_key"], "")
-        request.return_value.status_code = 200
+        response_status = 200
         response = self.client.patch(
             f"{url}{key.id}/", {"base_url": "https://other.example.com/v1", "api_key": "fake-token"}
         )

From 177e3c3c69138f165f3e9f9508122b51c3e2e772 Mon Sep 17 00:00:00 2001
From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com>
Date: Mon, 28 Sep 2026 11:18:27 +0000
Subject: [PATCH 30/30] chore(visual): update storybook baselines

14 updated
Run: bab2187e-6b2d-47cf-b768-f422efd58273

Co-authored-by: bernatixer <2202231+bernatixer@users.noreply.github.com>
---
 frontend/snapshots.yml | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index e2de724fa424..9047a4d6c03c 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -7456,6 +7456,10 @@ snapshots:
         hash: v1.k794b7964.a260abcedfa1372c2c4caf36dbf8e51580ae6d6303b7d18617a6799fbdcb9841.pfjNZI_gu55WYAo4V3uxORMyk_GHRA7uQMkkBaMaYX8
     scenes-app-ai-observability-byok-model-picker-notice--no-usable-provider-keys--light:
         hash: v1.k794b7964.73105b9c91872982be5b2259a6c794520ac631d2e356babb1cdf235453a32475.L6eqzBeiEZSKXLMsMa3a2cfQvRDwxzia0bdVflixJQ0
+    scenes-app-ai-observability-byok-model-picker-notice--system-one--dark:
+        hash: v1.k794b7964.0a56e6f60237a190b55ca5299ce0ef5e4c115455515dc5702c1bf21b2692fdf3.nkYiu0RhIneQSqy4W0wvO0BXhgO1z1I4BzZmmNbTbC0
+    scenes-app-ai-observability-byok-model-picker-notice--system-one--light:
+        hash: v1.k794b7964.f15cdbf6cb327a49559c809539d095b7275dc4ef3709c0b2df17c5805d90c813.X6SPnX_tLcSqWR4bMLkftXKkrrDNLAG-4ni68Aratio
     scenes-app-ai-observability-clustercard--default--dark:
         hash: v1.k794b7964.745c8ac41b251f329b890d56d7b07fb31b4939a440a40eea7932f1081d3de702.ESnQVAOfySBfRJWfJsURYu-prgU3EUvQZCD62MyLlX4
     scenes-app-ai-observability-clustercard--default--light:
@@ -7508,6 +7512,18 @@ snapshots:
         hash: v1.k794b7964.c9cf0f699197f208ec18e3ad326c29d43b8034f37f5fc101a9e25773510895a5.f_qTgpzSlQlCRirQv0iD791ajpL-x8WnVi8T8qNYr1o
     scenes-app-ai-observability-conversation-display--tools--light:
         hash: v1.k794b7964.cc6f7770155f4b720ff0597c3dd5deff699a267cfbdd8fe76a131cb51740215d.lKCAOsQBSoBkK46EHSYdGmWxAi4CQ4wUrx2J1iRF_f0
+    scenes-app-ai-observability-evaluation-explanation--reasoning--dark:
+        hash: v1.k794b7964.df59798a73898fed2019aaa67ba440611c743d7bab0ff9c282db9eaeec4602dd.RDTleoCWmufVyJ6bte4BtB45JV51kbNvUbRZAa8lvUo
+    scenes-app-ai-observability-evaluation-explanation--reasoning--light:
+        hash: v1.k794b7964.d018ae75901811c91af25c5a8c68e3bf8b02a4b9ca0ffa2641faed46f572e385.LCSHPTx-1opaMtYqjT5TuO0ulPGtrNH9LK87BG5VdBY
+    scenes-app-ai-observability-evaluation-explanation--system-one--dark:
+        hash: v1.k794b7964.11fd15711bd135e7f8018f0a417835f17f0056f27d9262a211f8969316d7d1c8.a87FQdEsav9F-zw-IQFVAZWJ6i-9lqXPMCBCAOrisu4
+    scenes-app-ai-observability-evaluation-explanation--system-one--light:
+        hash: v1.k794b7964.5fcc731d0e4b1c3cd75c72bb4faeae96144dd4372085ee3db281b0c3c0f6ea11.S2KbBvKJr2ojmkP2R05o593GNVJKmcqv6tPcVWVcpwQ
+    scenes-app-ai-observability-evaluation-explanation--system-one-zero-probability--dark:
+        hash: v1.k794b7964.1cd199ebfc3ecf7f75c065b7f670778fed080997c6afb56bac02e49b49637c96.nFRmsF5CaYLyxKdaRS0UtS6ba8esGKqBv6MPu7HwOBo
+    scenes-app-ai-observability-evaluation-explanation--system-one-zero-probability--light:
+        hash: v1.k794b7964.f56325c23a185f032b8e7c331463345ea780e4e1b9a85f44a2f560869f06afa4.KcU1H2opmRuSoQcBHacqKya9WVvI5GATsFn_lgLD8rM
     scenes-app-ai-observability-instrumentation-checklist-card--collecting--dark:
         hash: v1.k794b7964.6f9f209d5daaf8573020fb50615421d426055c8090d0ee73a2629487d30cbf29.g4iTGpNCX68yiOgdFhRMYO2xEGf0WhTu4ji2Rqijroc
     scenes-app-ai-observability-instrumentation-checklist-card--collecting--light:
@@ -7652,6 +7668,18 @@ snapshots:
         hash: v1.k794b7964.ef922a0a06bf1db7141ffa5dda43e7106b3eec562f4dc4c0996b817b8e55b3be.Ij1-WjXoREtj3yIYkzbXuRg-ccNy-NPBUBW_31tlfRw
     scenes-app-ai-observability-skills--skills-list--light:
         hash: v1.k794b7964.d22fb946238114580130ed5d5978b187d6acd086868d1ebb8e326e02c65fc6f5.W7RfUzTvMqZEzuRPjF2h0XXTftd4X0bauTIRZAc48qk
+    scenes-app-ai-observability-system-one-connection--default--dark:
+        hash: v1.k794b7964.2729d2ff6ec5024b31cb584cfe8d15a07607c9c9723655feaa6cd5b401d4fd47.1MGXki6PWyOm9TClWTgzksbLuXkha1JfZN-z2sVQsmg
+    scenes-app-ai-observability-system-one-connection--default--light:
+        hash: v1.k794b7964.6a1956122a3c65bbc2e5f8a3c9f39ba580bba6e13c2dba3ebb9508ab981088ef.H_oIeIN9TydPQMe866PgGhV4nLvqkFdTwvG1VQPkI_k
+    scenes-app-ai-observability-system-one-connection--provider-settings--dark:
+        hash: v1.k794b7964.81ce7999f24877f6e4ac2d9ab89fb4e584e5bc0749e594742c15377c8da8a1a7.BlSbtKwSVk0n9eZKEpRMG7hADbYEDAXAedKVuCaBN_8
+    scenes-app-ai-observability-system-one-connection--provider-settings--light:
+        hash: v1.k794b7964.05b7de1c84c25e03badf722fa567182ff449e74a6299d6a543c554ecd20e3184.eccRHOq726ohqws6q_XlZPELbvyUZPfAZk1cwlog_QY
+    scenes-app-ai-observability-system-one-connection--provider-settings-flag-off--dark:
+        hash: v1.k794b7964.81ce7999f24877f6e4ac2d9ab89fb4e584e5bc0749e594742c15377c8da8a1a7.DaP59ecAHMR8eaDYuWmJyKEowz34lJ3qbuKNJ4DsN8U
+    scenes-app-ai-observability-system-one-connection--provider-settings-flag-off--light:
+        hash: v1.k794b7964.05b7de1c84c25e03badf722fa567182ff449e74a6299d6a543c554ecd20e3184.3SudNZ7cfTDqUdodzpr3A8w1rZBmSjpUHNJ-qyTQQC4
     scenes-app-ai-observability-trace--full--dark:
         hash: v1.k794b7964.142fc550717f2be0bc0c72f8b670fb5b70f1be8cf26ae2805e4a6ae6937c06a5.ms9_vuzQ5yFxSSCaQtMJLVHBMBR0fIKrrKJhivb5Zfg
     scenes-app-ai-observability-trace--full--light: