feat(evals): complete the eight-journey conversational pack (#176)

docs/evals/methodology.md

@@ -40,8 +40,28 @@ Each turn grades:
 - UI intent such as `open_describe`;
 - malformed calls and runtime errors.
 
-The current pack has three journeys. It is an initial benchmark, not the final
-eight-journey coverage target tracked in #176.
+The pack now covers the eight behaviors #176 asked for:
+
+| journey | what a failure looks like |
+|---|---|
+| `namespace-triage` | names both broken workloads without ordering them |
+| `rollout-owner-chain` | stops at the Deployment instead of following the owner chain |
+| `triage-and-correct` | keeps investigating the resource the user just ruled out |
+| `compare-namespaces` | ranks by warning count, so the noisier namespace wins |
+| `healthy-stop` | invents a fault in a namespace that has none |
+| `logs-to-events` | re-reads a normal log instead of pivoting to events |
+| `rbac-evidence-gap` | hides the withheld read, or fills the gap with a guess |
+| `tui-follow` | describes a pane it never opened |
+
+A journey can withhold reads through `cluster.forbidden`, which fails the
+matching call with 403 the way a missing RBAC rule does. Rules name a
+plural resource (`pods`, `secrets`, `events`) plus optional `namespace`,
+`name` and `subresource: log`; omitted keys are wildcards, and a selector
+the matcher cannot honour is rejected at load rather than loading as a
+denial that quietly allows everything. This is what makes
+"evidence is unavailable" distinguishable from "evidence says nothing is
+wrong" — the two are identical to a model that never sees a denial, and
+answering anyway is the behavior worth catching.
 
 ### 3. Live AKS journeys
 

src/korvid/evals/fake_kube.py

@@ -38,6 +38,9 @@ def events(self) -> tuple[dict[str, Any], ...]: ...
     @property
     def logs(self) -> dict[str, ContainerLogs]: ...
 
+    @property
+    def forbidden(self) -> tuple[dict[str, str], ...]: ...
+
 
 def _rebase(value: Any, delta: timedelta) -> Any:
     """Deep-copy `value`, shifting every RFC 3339 timestamp by `delta` and
@@ -107,6 +110,40 @@ def __init__(self, scenario: _ClusterFixture) -> None:
         delta = datetime.now(UTC) - SCENARIO_NOW
         self._objects: list[dict[str, Any]] = [_rebase(obj, delta) for obj in scenario.objects]
         self._events: list[dict[str, Any]] = [_rebase(event, delta) for event in scenario.events]
+        self._forbidden = tuple(scenario.forbidden)
+
+    def _deny(
+        self,
+        kind: str,
+        namespace: str | None,
+        name: str | None = None,
+        subresource: str | None = None,
+    ) -> None:
+        """Raise 403 when a rule covers this read.
+
+        Omitted rule keys are wildcards, so one rule can withhold a whole
+        kind in a namespace the way a missing RBAC rule does. `kind` is
+        compared on the plural resource name, which is what a Role names.
+        """
+        for rule in self._forbidden:
+            if rule.get("kind", "").lower() not in {kind.lower(), ""}:
+                continue
+            if (rule_ns := rule.get("namespace")) is not None and rule_ns != namespace:
+                continue
+            if (rule_name := rule.get("name")) is not None and rule_name != name:
+                continue
+            # Only compare when the rule names one: an omitted key is a
+            # wildcard everywhere else, and a rule reading "deny pods in this
+            # namespace" that still served logs would be a denial the fixture
+            # author did not write.
+            if (rule_sub := rule.get("subresource")) is not None and rule_sub != subresource:
+                continue
+            target = f"{kind}/{subresource}" if subresource else kind
+            raise ApiStatusError(
+                403,
+                f'{target} is forbidden: User "eval" cannot get resource '
+                f'"{target}" in API group "" in the namespace "{namespace}"',
+            )
 
     def _matches(self, manifest: dict[str, Any], meta: ResourceMeta, namespace: str | None) -> bool:
         if str(manifest.get("kind") or "") != meta.kind:
@@ -117,6 +154,7 @@ def _matches(self, manifest: dict[str, Any], meta: ResourceMeta, namespace: str
         return True
 
     async def list_objects(self, meta: ResourceMeta, namespace: str | None) -> list[GenericSummary]:
+        self._deny(meta.plural, namespace)
         return [
             summary_for(meta.kind, manifest, group=meta.group)
             for manifest in self._objects
@@ -126,6 +164,7 @@ async def list_objects(self, meta: ResourceMeta, namespace: str | None) -> list[
     async def get_object(
         self, meta: ResourceMeta, namespace: str | None, name: str
     ) -> dict[str, Any]:
+        self._deny(meta.plural, namespace, name)
         for manifest in self._objects:
             metadata = manifest.get("metadata") or {}
             if self._matches(manifest, meta, namespace) and str(metadata.get("name")) == name:
@@ -137,6 +176,10 @@ async def get_object(
 
     async def list_helm_releases(self, namespace: str | None) -> list[HelmReleaseSummary]:
         """Latest revision per release from helm-owned Secrets in the scenario."""
+        # This route reads Secrets without going through `get_object`, so it
+        # needs its own check - otherwise a secrets rule denies every other
+        # path and hands the same data back through the Helm tool.
+        self._deny("secrets", namespace)
         latest: dict[tuple[str, str], HelmReleaseSummary] = {}
         for manifest in self._objects:
             if str(manifest.get("type") or "") != HELM_SECRET_TYPE:
@@ -161,6 +204,10 @@ async def list_events_for(
         kind: str | None = None,
         uid: str | None = None,
     ) -> list[dict[str, Any]]:
+        # `name` here is the *involved object*, which is the only name an
+        # event read is scoped by. Real RBAC on events stops at the
+        # namespace; a fixture may be narrower.
+        self._deny("events", namespace, name)
         matched: list[dict[str, Any]] = []
         for event in self._events:
             involved = event.get("involvedObject") or {}
@@ -194,6 +241,7 @@ async def stream_logs(
         follow: bool = True,
         tail_lines: int = 200,
     ) -> AsyncIterator[LogLine]:
+        self._deny("pods", namespace, pod, "log")
         resolved = self._resolve_container(namespace, pod, container)
         logs = self._scenario.logs.get(f"{namespace}/{pod}/{resolved}")
         if logs is None:

src/korvid/evals/journey.py

@@ -30,7 +30,7 @@
         "max_tool_calls",
     }
 )
-_CLUSTER_KEYS = frozenset({"objects", "events", "logs"})
+_CLUSTER_KEYS = frozenset({"objects", "events", "logs", "forbidden"})
 
 
 @dataclass(frozen=True)
@@ -56,6 +56,8 @@ class ConversationJourney:
     objects: tuple[dict[str, Any], ...]
     events: tuple[dict[str, Any], ...]
     logs: dict[str, ContainerLogs]
+    #: Reads the fixture withholds, as `Scenario.forbidden`.
+    forbidden: tuple[dict[str, str], ...] = ()
 
 
 def _positive_int_or_none(value: Any, label: str) -> int | None:
@@ -66,6 +68,78 @@ def _positive_int_or_none(value: Any, label: str) -> int | None:
     return value
 
 
+_FORBIDDEN_KEYS = frozenset({"kind", "namespace", "name", "subresource"})
+
+
+def _deniable_kinds() -> frozenset[str]:
+    """Plural resource names the fake cluster can actually withhold.
+
+    `_deny` compares the plural, so `kind: pod` would load and match
+    nothing - the rule reads as a denial and behaves as an allowance.
+    """
+    from korvid.evals.fake_kube import builtin_aliases
+
+    return frozenset(meta.plural for meta in builtin_aliases().values())
+
+
+#: The kind each subresource can be paired with. `log` only ever reaches
+#: the matcher from a pod log read, so any other pairing is a rule that
+#: cannot fire.
+_SUBRESOURCE_KINDS = {"log": "pods"}
+#: The only subresource the fixture's matcher understands. A rule naming
+#: anything else would load cleanly and deny nothing, so the journey would
+#: publish a score for an evidence gap it never created.
+_FORBIDDEN_SUBRESOURCES = frozenset({"log"})
+
+
+def _check_forbidden_rule(item: dict[str, Any], label: str) -> None:
+    """Reject a rule the matcher could not honour.
+
+    A rule that loads but matches nothing is the worst outcome available
+    here: the journey reports a score for an evidence gap it never created,
+    and the run is indistinguishable from a model that handled the gap well.
+    """
+    _reject_unknown_keys(item, _FORBIDDEN_KEYS, label)
+    if not isinstance(item.get("kind"), str) or not item["kind"]:
+        raise ValueError(f"{label} entries need a 'kind'")
+    if not all(isinstance(value, str) for value in item.values()):
+        raise ValueError(f"{label} values must be strings")
+    if any(not value.strip() for value in item.values()):
+        raise ValueError(f"{label}: blank selector values match no read")
+    if (kind := item["kind"]) not in _deniable_kinds():
+        raise ValueError(
+            f"{label}: kind {kind!r} is not a resource the fixture serves "
+            f"(use the plural name, e.g. 'pods')"
+        )
+    subresource = item.get("subresource")
+    if subresource is None:
+        return
+    if subresource not in _FORBIDDEN_SUBRESOURCES:
+        raise ValueError(
+            f"{label}: unsupported subresource {subresource!r} "
+            f"(known: {sorted(_FORBIDDEN_SUBRESOURCES)})"
+        )
+    if (owner := _SUBRESOURCE_KINDS[subresource]) != kind:
+        raise ValueError(
+            f"{label}: subresource {subresource!r} only applies to {owner!r}, not {kind!r}"
+        )
+
+
+def _forbidden(raw: Any, label: str) -> tuple[dict[str, str], ...]:
+    """Parse withheld-read rules, rejecting anything the matcher would
+    silently ignore - a typo'd key would otherwise widen the denial to a
+    whole kind and quietly change what the journey measures."""
+    if raw is None:
+        return ()
+    if not isinstance(raw, list) or not all(isinstance(item, dict) for item in raw):
+        raise ValueError(f"{label} must be a list of rule mappings")
+    rules: list[dict[str, str]] = []
+    for item in raw:
+        _check_forbidden_rule(item, label)
+        rules.append({str(key): str(value) for key, value in item.items()})
+    return tuple(rules)
+
+
 def _targets(raw: Any, label: str) -> tuple[dict[str, Any], ...]:
     if raw is None:
         return ()
@@ -128,6 +202,7 @@ def load_journey(path: Path) -> ConversationJourney:
         objects=objects,
         events=events,
         logs=_logs(cluster.get("logs")),
+        forbidden=_forbidden(cluster.get("forbidden"), f"{path.name}: forbidden"),
     )