diff --git a/docs/evals/methodology.md b/docs/evals/methodology.md index 2b639c1d..ccf3dd1e 100644 --- a/docs/evals/methodology.md +++ b/docs/evals/methodology.md @@ -40,8 +40,28 @@ Each turn grades: - UI intent such as `open_describe`; - malformed calls and runtime errors. -The current pack has three journeys. It is an initial benchmark, not the final -eight-journey coverage target tracked in #176. +The pack now covers the eight behaviors #176 asked for: + +| journey | what a failure looks like | +|---|---| +| `namespace-triage` | names both broken workloads without ordering them | +| `rollout-owner-chain` | stops at the Deployment instead of following the owner chain | +| `triage-and-correct` | keeps investigating the resource the user just ruled out | +| `compare-namespaces` | ranks by warning count, so the noisier namespace wins | +| `healthy-stop` | invents a fault in a namespace that has none | +| `logs-to-events` | re-reads a normal log instead of pivoting to events | +| `rbac-evidence-gap` | hides the withheld read, or fills the gap with a guess | +| `tui-follow` | describes a pane it never opened | + +A journey can withhold reads through `cluster.forbidden`, which fails the +matching call with 403 the way a missing RBAC rule does. Rules name a +plural resource (`pods`, `secrets`, `events`) plus optional `namespace`, +`name` and `subresource: log`; omitted keys are wildcards, and a selector +the matcher cannot honour is rejected at load rather than loading as a +denial that quietly allows everything. This is what makes +"evidence is unavailable" distinguishable from "evidence says nothing is +wrong" — the two are identical to a model that never sees a denial, and +answering anyway is the behavior worth catching. ### 3. Live AKS journeys diff --git a/src/korvid/evals/fake_kube.py b/src/korvid/evals/fake_kube.py index 547466e9..bc558403 100644 --- a/src/korvid/evals/fake_kube.py +++ b/src/korvid/evals/fake_kube.py @@ -38,6 +38,9 @@ def events(self) -> tuple[dict[str, Any], ...]: ... @property def logs(self) -> dict[str, ContainerLogs]: ... + @property + def forbidden(self) -> tuple[dict[str, str], ...]: ... + def _rebase(value: Any, delta: timedelta) -> Any: """Deep-copy `value`, shifting every RFC 3339 timestamp by `delta` and @@ -107,6 +110,40 @@ def __init__(self, scenario: _ClusterFixture) -> None: delta = datetime.now(UTC) - SCENARIO_NOW self._objects: list[dict[str, Any]] = [_rebase(obj, delta) for obj in scenario.objects] self._events: list[dict[str, Any]] = [_rebase(event, delta) for event in scenario.events] + self._forbidden = tuple(scenario.forbidden) + + def _deny( + self, + kind: str, + namespace: str | None, + name: str | None = None, + subresource: str | None = None, + ) -> None: + """Raise 403 when a rule covers this read. + + Omitted rule keys are wildcards, so one rule can withhold a whole + kind in a namespace the way a missing RBAC rule does. `kind` is + compared on the plural resource name, which is what a Role names. + """ + for rule in self._forbidden: + if rule.get("kind", "").lower() not in {kind.lower(), ""}: + continue + if (rule_ns := rule.get("namespace")) is not None and rule_ns != namespace: + continue + if (rule_name := rule.get("name")) is not None and rule_name != name: + continue + # Only compare when the rule names one: an omitted key is a + # wildcard everywhere else, and a rule reading "deny pods in this + # namespace" that still served logs would be a denial the fixture + # author did not write. + if (rule_sub := rule.get("subresource")) is not None and rule_sub != subresource: + continue + target = f"{kind}/{subresource}" if subresource else kind + raise ApiStatusError( + 403, + f'{target} is forbidden: User "eval" cannot get resource ' + f'"{target}" in API group "" in the namespace "{namespace}"', + ) def _matches(self, manifest: dict[str, Any], meta: ResourceMeta, namespace: str | None) -> bool: if str(manifest.get("kind") or "") != meta.kind: @@ -117,6 +154,7 @@ def _matches(self, manifest: dict[str, Any], meta: ResourceMeta, namespace: str return True async def list_objects(self, meta: ResourceMeta, namespace: str | None) -> list[GenericSummary]: + self._deny(meta.plural, namespace) return [ summary_for(meta.kind, manifest, group=meta.group) for manifest in self._objects @@ -126,6 +164,7 @@ async def list_objects(self, meta: ResourceMeta, namespace: str | None) -> list[ async def get_object( self, meta: ResourceMeta, namespace: str | None, name: str ) -> dict[str, Any]: + self._deny(meta.plural, namespace, name) for manifest in self._objects: metadata = manifest.get("metadata") or {} if self._matches(manifest, meta, namespace) and str(metadata.get("name")) == name: @@ -137,6 +176,10 @@ async def get_object( async def list_helm_releases(self, namespace: str | None) -> list[HelmReleaseSummary]: """Latest revision per release from helm-owned Secrets in the scenario.""" + # This route reads Secrets without going through `get_object`, so it + # needs its own check - otherwise a secrets rule denies every other + # path and hands the same data back through the Helm tool. + self._deny("secrets", namespace) latest: dict[tuple[str, str], HelmReleaseSummary] = {} for manifest in self._objects: if str(manifest.get("type") or "") != HELM_SECRET_TYPE: @@ -161,6 +204,10 @@ async def list_events_for( kind: str | None = None, uid: str | None = None, ) -> list[dict[str, Any]]: + # `name` here is the *involved object*, which is the only name an + # event read is scoped by. Real RBAC on events stops at the + # namespace; a fixture may be narrower. + self._deny("events", namespace, name) matched: list[dict[str, Any]] = [] for event in self._events: involved = event.get("involvedObject") or {} @@ -194,6 +241,7 @@ async def stream_logs( follow: bool = True, tail_lines: int = 200, ) -> AsyncIterator[LogLine]: + self._deny("pods", namespace, pod, "log") resolved = self._resolve_container(namespace, pod, container) logs = self._scenario.logs.get(f"{namespace}/{pod}/{resolved}") if logs is None: diff --git a/src/korvid/evals/journey.py b/src/korvid/evals/journey.py index f013edf6..7696af25 100644 --- a/src/korvid/evals/journey.py +++ b/src/korvid/evals/journey.py @@ -30,7 +30,7 @@ "max_tool_calls", } ) -_CLUSTER_KEYS = frozenset({"objects", "events", "logs"}) +_CLUSTER_KEYS = frozenset({"objects", "events", "logs", "forbidden"}) @dataclass(frozen=True) @@ -56,6 +56,8 @@ class ConversationJourney: objects: tuple[dict[str, Any], ...] events: tuple[dict[str, Any], ...] logs: dict[str, ContainerLogs] + #: Reads the fixture withholds, as `Scenario.forbidden`. + forbidden: tuple[dict[str, str], ...] = () def _positive_int_or_none(value: Any, label: str) -> int | None: @@ -66,6 +68,78 @@ def _positive_int_or_none(value: Any, label: str) -> int | None: return value +_FORBIDDEN_KEYS = frozenset({"kind", "namespace", "name", "subresource"}) + + +def _deniable_kinds() -> frozenset[str]: + """Plural resource names the fake cluster can actually withhold. + + `_deny` compares the plural, so `kind: pod` would load and match + nothing - the rule reads as a denial and behaves as an allowance. + """ + from korvid.evals.fake_kube import builtin_aliases + + return frozenset(meta.plural for meta in builtin_aliases().values()) + + +#: The kind each subresource can be paired with. `log` only ever reaches +#: the matcher from a pod log read, so any other pairing is a rule that +#: cannot fire. +_SUBRESOURCE_KINDS = {"log": "pods"} +#: The only subresource the fixture's matcher understands. A rule naming +#: anything else would load cleanly and deny nothing, so the journey would +#: publish a score for an evidence gap it never created. +_FORBIDDEN_SUBRESOURCES = frozenset({"log"}) + + +def _check_forbidden_rule(item: dict[str, Any], label: str) -> None: + """Reject a rule the matcher could not honour. + + A rule that loads but matches nothing is the worst outcome available + here: the journey reports a score for an evidence gap it never created, + and the run is indistinguishable from a model that handled the gap well. + """ + _reject_unknown_keys(item, _FORBIDDEN_KEYS, label) + if not isinstance(item.get("kind"), str) or not item["kind"]: + raise ValueError(f"{label} entries need a 'kind'") + if not all(isinstance(value, str) for value in item.values()): + raise ValueError(f"{label} values must be strings") + if any(not value.strip() for value in item.values()): + raise ValueError(f"{label}: blank selector values match no read") + if (kind := item["kind"]) not in _deniable_kinds(): + raise ValueError( + f"{label}: kind {kind!r} is not a resource the fixture serves " + f"(use the plural name, e.g. 'pods')" + ) + subresource = item.get("subresource") + if subresource is None: + return + if subresource not in _FORBIDDEN_SUBRESOURCES: + raise ValueError( + f"{label}: unsupported subresource {subresource!r} " + f"(known: {sorted(_FORBIDDEN_SUBRESOURCES)})" + ) + if (owner := _SUBRESOURCE_KINDS[subresource]) != kind: + raise ValueError( + f"{label}: subresource {subresource!r} only applies to {owner!r}, not {kind!r}" + ) + + +def _forbidden(raw: Any, label: str) -> tuple[dict[str, str], ...]: + """Parse withheld-read rules, rejecting anything the matcher would + silently ignore - a typo'd key would otherwise widen the denial to a + whole kind and quietly change what the journey measures.""" + if raw is None: + return () + if not isinstance(raw, list) or not all(isinstance(item, dict) for item in raw): + raise ValueError(f"{label} must be a list of rule mappings") + rules: list[dict[str, str]] = [] + for item in raw: + _check_forbidden_rule(item, label) + rules.append({str(key): str(value) for key, value in item.items()}) + return tuple(rules) + + def _targets(raw: Any, label: str) -> tuple[dict[str, Any], ...]: if raw is None: return () @@ -128,6 +202,7 @@ def load_journey(path: Path) -> ConversationJourney: objects=objects, events=events, logs=_logs(cluster.get("logs")), + forbidden=_forbidden(cluster.get("forbidden"), f"{path.name}: forbidden"), ) diff --git a/src/korvid/evals/journeys/compare-namespaces.yaml b/src/korvid/evals/journeys/compare-namespaces.yaml new file mode 100644 index 00000000..5d261e45 --- /dev/null +++ b/src/korvid/evals/journeys/compare-namespaces.yaml @@ -0,0 +1,202 @@ +id: compare-namespaces +# Both namespaces have a warning. Only one has an outage. The failure this +# journey exists to catch is severity-by-count: `staging` has more warning +# events, and `prod` is the one that is down. +root_cause: service_selector_mismatch +turns: + - user: >- + Compare namespaces prod and staging. Which one needs attention first, + and why? + screen: "resource view: pods, all namespaces" + grading: + must_mention: + - [prod] + - [staging] + # The ordering, stated. `prod` is not merely mentioned first - it + # has to be chosen. + - [prod first, prod needs attention, prod is worse, prioritize prod, + prod is more urgent, start with prod, prod takes priority, + prod is the priority, address prod first] + # The reason, not just the pick. The turn asks "and why?", and the + # fixture hands staging every warning event - so "prod first because + # it has more warnings" is both the wrong reason and a factual + # inversion, and it passed on the name alone. + # Bare "endpoints" is not a reason - a model can name the object it + # looked at while giving the warning-count rationale. Each + # alternative states a consequence or the mismatch itself. + - [no endpoints, without endpoints, has no endpoints, not reachable, + unreachable, cannot serve, not serving, traffic fails, outage, + is down, nothing backs, no backing pods, no pods behind, + selector does not match, label mismatch, selects no pods] + must_not_mention: + # Choosing staging, or refusing to choose, are the two ways to fail + # a comparison. + - [staging first, prioritize staging, staging is worse, + staging is more urgent, start with staging] + - [both equally, equally urgent, same severity, hard to say which, + cannot determine which] + # The inverted rationale itself, which the right pick would + # otherwise carry through unnoticed: staging holds every warning + # event here, so ranking prod above it *by warnings* is a false + # statement about the cluster. + - [prod has more warning, prod has more events, + more warnings than staging, more warning events than staging, + noisier than staging] + expected_evidence: + - - tool: list_resources + args: {kind: pods, namespace: prod} + contains: checkout-1 + - - tool: list_resources + args: {kind: pods, namespace: staging} + contains: sandbox-1 + # The prod outage itself has to have been observed, or the right + # pick could rest on a guess. + - - tool: get_resource + args: {kind: endpoints, name: checkout, namespace: prod} + contains: "subsets: []" + - tool: diagnose_service + args: {service: checkout, namespace: prod} + contains: EndpointSlice + max_tool_calls: 4 + - user: >- + Give me the specific cause in the namespace you picked, and say what + is actually wrong in the other one. + screen: "resource view: services, namespace prod" + grading: + must_mention: + # prod: the Service selects a label no pod carries, so it has no + # endpoints - traffic fails while every pod looks healthy. + # The turn asks for the *specific cause*, and the symptom is not + # it: "prod has no endpoints" is the thing to be explained. Every + # alternative here names the mismatch. + - [label mismatch, labels do not match, label does not match, + selector does not match, does not match the pod, + does not match any pod, no matching pods, selects no pods, + selector is wrong, wrong selector, "checkout-v2"] + # staging: a restart that recovered. Naming it as recovered, rather + # than as a second outage, is the discrimination being measured. + - [recovered, resolved, restarted once, back to ready, now ready, + now running, healthy again, not currently failing, no longer] + must_not_mention: + - [staging is down, staging outage, staging is broken] + - [oomkilled, image pull, imagepullbackoff] + expected_evidence: + # The mismatch itself, by either route: the selector the Service + # declares, or the empty endpoints that result from it. + - - tool: get_resource + args: {kind: services, name: checkout, namespace: prod} + contains: "app: checkout-v2" + - tool: diagnose_service + args: {service: checkout, namespace: prod} + contains: selector + - - tool: get_resource + args: {kind: endpoints, name: checkout, namespace: prod} + contains: "subsets: []" + # The other half of a mismatch claim: the label the pod actually + # carries. Empty endpoints alone could also mean matching but + # unready pods, so without this the claim rests on an assumption. + - - tool: get_resource + args: {kind: pods, name: checkout-1, namespace: prod} + contains: "app: checkout" + max_tool_calls: 4 +cluster: + objects: + # prod: pods are healthy, the Service points at a label none of them has. + - kind: Service + apiVersion: v1 + metadata: + name: checkout + namespace: prod + uid: svc-checkout + creationTimestamp: "2026-07-27T05:00:00Z" + spec: + selector: {app: checkout-v2} + ports: [{port: 80, targetPort: 8080, protocol: TCP}] + # Empty on purpose: the Service selects `app: checkout-v2` and the pod + # carries `app: checkout`, so nothing backs it. + - kind: Endpoints + apiVersion: v1 + metadata: + name: checkout + namespace: prod + uid: ep-checkout + creationTimestamp: "2026-07-27T05:00:00Z" + subsets: [] + - kind: Pod + apiVersion: v1 + metadata: + name: checkout-1 + namespace: prod + uid: pod-checkout + labels: {app: checkout} + creationTimestamp: "2026-07-27T05:00:00Z" + spec: + nodeName: node-a + containers: [{name: checkout, image: shop/checkout:v1}] + status: + phase: Running + podIP: 10.0.0.11 + conditions: [{type: Ready, status: "True"}] + containerStatuses: + - name: checkout + ready: true + restartCount: 0 + state: {running: {startedAt: "2026-07-27T05:00:05Z"}} + # staging: one restart, hours ago, already back to Ready. + - kind: Pod + apiVersion: v1 + metadata: + name: sandbox-1 + namespace: staging + uid: pod-sandbox + labels: {app: sandbox} + creationTimestamp: "2026-07-27T02:00:00Z" + spec: + nodeName: node-b + containers: [{name: sandbox, image: lab/sandbox:v5}] + status: + phase: Running + podIP: 10.0.1.11 + conditions: [{type: Ready, status: "True"}] + containerStatuses: + - name: sandbox + ready: true + restartCount: 1 + state: {running: {startedAt: "2026-07-27T03:00:00Z"}} + lastState: + terminated: + reason: Error + exitCode: 1 + startedAt: "2026-07-27T02:00:00Z" + finishedAt: "2026-07-27T02:59:00Z" + - kind: Node + apiVersion: v1 + metadata: {name: node-a, uid: node-a} + status: {conditions: [{type: Ready, status: "True"}]} + - kind: Node + apiVersion: v1 + metadata: {name: node-b, uid: node-b} + status: {conditions: [{type: Ready, status: "True"}]} + events: + # Deliberately lopsided: staging carries more warnings than prod, and + # prod is still the answer. A model counting events picks wrong. + - type: Warning + reason: BackOff + count: 3 + lastTimestamp: "2026-07-27T02:50:00Z" + message: Back-off restarting failed container sandbox in pod sandbox-1 + involvedObject: + {kind: Pod, name: sandbox-1, namespace: staging, uid: pod-sandbox} + - type: Warning + reason: Unhealthy + count: 5 + lastTimestamp: "2026-07-27T02:55:00Z" + message: 'Readiness probe failed: connection refused' + involvedObject: + {kind: Pod, name: sandbox-1, namespace: staging, uid: pod-sandbox} + logs: + prod/checkout-1/checkout: + current: ["checkout listening on 8080"] + staging/sandbox-1/sandbox: + current: ["sandbox ready"] + previous: ["sandbox: transient startup error"] diff --git a/src/korvid/evals/journeys/namespace-triage.yaml b/src/korvid/evals/journeys/namespace-triage.yaml new file mode 100644 index 00000000..358809b5 --- /dev/null +++ b/src/korvid/evals/journeys/namespace-triage.yaml @@ -0,0 +1,213 @@ +id: namespace-triage +# Two workloads are abnormal at once and they are not equally bad: one is +# down, the other is degraded but still serving. Naming both is easy; +# ordering them is the measurement. +root_cause: crashloop_missing_env +turns: + - user: What looks wrong in namespace orders? + screen: "resource view: pods, namespace orders" + grading: + must_mention: + - [invoicer] + - [shipper] + # The ordering claim itself. Without this group a model that lists + # both problems and stops would pass a triage journey, which is the + # one thing triage is not. + - [invoicer first, start with invoicer, invoicer is worse, + invoicer is more urgent, prioritize invoicer, invoicer is the + priority, invoicer takes priority, address invoicer first, + invoicer is the more severe, invoicer matters more] + must_not_mention: + # A namespace with two broken workloads is not healthy, and neither + # problem is a pull failure. + - [healthy namespace, nothing is wrong, no issues, all healthy] + # The reverse order, prohibited outright. The positive group above + # only requires that "invoicer first" appears somewhere, so an + # answer could concede it and then rank shipper above it - the exact + # wrong prioritization, in the journey built to measure ordering. + - [prioritize shipper, shipper first, shipper is worse, + shipper is more urgent, start with shipper, + shipper takes priority, address shipper first] + - [image pull, imagepullbackoff] + expected_evidence: + # Two groups, one per workload. Evidence matching is route + # independent, so a single invoicer-targeted read satisfied the + # first group and `shipper` could then be named without ever having + # been observed - in the journey whose subject is discovering both. + # The same namespace list satisfies both groups at once. + - - tool: list_resources + args: {kind: pods, namespace: orders} + contains: invoicer-1 + - - tool: list_resources + args: {kind: pods, namespace: orders} + contains: shipper-3 + max_tool_calls: 3 + - user: >- + Confirm the reason for the one you picked, and say whether the other + one is still serving traffic. + screen: "resource view: pods, namespace orders, selected: invoicer-1" + grading: + must_mention: + # The cause of the crash loop, from the log the fixture does supply. + - [dsn, database url, environment variable, env var, missing config, + configuration] + # The comparative judgement the second turn exists to force: shipper + # is degraded, not down - 2 of 3 replicas are ready. + - [still serving, still available, partially, reduced capacity, + degraded, some replicas, two of three, 2 of 3, 2/3] + must_not_mention: + - [shipper is down, shipper is unavailable, complete outage, + total outage] + - [oomkilled, out of memory] + expected_evidence: + # The container is crashlooping, so the line that names the cause is + # in the *previous* log. `get_logs` has no `previous` parameter, so + # the diagnostic is the only route to it - which is itself part of + # what this turn measures. + - - tool: diagnose_pod + args: {pod: invoicer-1, namespace: orders} + contains: DATABASE_DSN + - - tool: get_resource + args: {kind: deployments, name: shipper, namespace: orders} + contains: "readyReplicas: 2" + - tool: diagnose_workload + args: {kind: deployments, name: shipper, namespace: orders} + contains: "ready=2" + max_tool_calls: 4 +cluster: + objects: + # Down: every replica is crashlooping. + - kind: Pod + apiVersion: v1 + metadata: + name: invoicer-1 + namespace: orders + uid: pod-invoicer + creationTimestamp: "2026-07-27T06:00:00Z" + spec: + nodeName: node-a + containers: [{name: invoicer, image: orders/invoicer:v3}] + status: + phase: Running + conditions: [{type: Ready, status: "False"}] + containerStatuses: + - name: invoicer + ready: false + restartCount: 9 + state: + waiting: + reason: CrashLoopBackOff + message: back-off 5m0s restarting failed container=invoicer + lastState: + terminated: + reason: Error + exitCode: 1 + startedAt: "2026-07-27T07:10:00Z" + finishedAt: "2026-07-27T07:10:02Z" + - kind: Deployment + apiVersion: apps/v1 + metadata: + name: invoicer + namespace: orders + uid: dep-invoicer + creationTimestamp: "2026-07-27T05:00:00Z" + spec: {replicas: 1} + status: {replicas: 1, readyReplicas: 0, unavailableReplicas: 1} + # Degraded: one replica of three is not ready, the service still serves. + - kind: Deployment + apiVersion: apps/v1 + metadata: + name: shipper + namespace: orders + uid: dep-shipper + creationTimestamp: "2026-07-27T05:00:00Z" + spec: {replicas: 3} + status: {replicas: 3, readyReplicas: 2, unavailableReplicas: 1} + - kind: Pod + apiVersion: v1 + metadata: + name: shipper-1 + namespace: orders + uid: pod-shipper-1 + creationTimestamp: "2026-07-27T05:00:00Z" + spec: + nodeName: node-a + containers: [{name: shipper, image: orders/shipper:v7}] + status: + phase: Running + conditions: [{type: Ready, status: "True"}] + containerStatuses: + - name: shipper + ready: true + restartCount: 0 + state: {running: {startedAt: "2026-07-27T05:00:10Z"}} + - kind: Pod + apiVersion: v1 + metadata: + name: shipper-2 + namespace: orders + uid: pod-shipper-2 + creationTimestamp: "2026-07-27T05:00:00Z" + spec: + nodeName: node-b + containers: [{name: shipper, image: orders/shipper:v7}] + status: + phase: Running + conditions: [{type: Ready, status: "True"}] + containerStatuses: + - name: shipper + ready: true + restartCount: 0 + state: {running: {startedAt: "2026-07-27T05:00:11Z"}} + - kind: Pod + apiVersion: v1 + metadata: + name: shipper-3 + namespace: orders + uid: pod-shipper-3 + creationTimestamp: "2026-07-27T05:00:00Z" + spec: + nodeName: node-b + containers: [{name: shipper, image: orders/shipper:v7}] + status: + phase: Running + conditions: [{type: Ready, status: "False"}] + containerStatuses: + - name: shipper + ready: false + restartCount: 2 + state: {running: {startedAt: "2026-07-27T07:00:00Z"}} + - kind: Node + apiVersion: v1 + metadata: {name: node-a, uid: node-a} + status: {conditions: [{type: Ready, status: "True"}]} + - kind: Node + apiVersion: v1 + metadata: {name: node-b, uid: node-b} + status: {conditions: [{type: Ready, status: "True"}]} + events: + - type: Warning + reason: BackOff + count: 9 + lastTimestamp: "2026-07-27T07:15:00Z" + message: Back-off restarting failed container invoicer in pod invoicer-1 + involvedObject: + {kind: Pod, name: invoicer-1, namespace: orders, uid: pod-invoicer} + - type: Warning + reason: Unhealthy + count: 4 + lastTimestamp: "2026-07-27T07:12:00Z" + message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' + involvedObject: + {kind: Pod, name: shipper-3, namespace: orders, uid: pod-shipper-3} + logs: + orders/invoicer-1/invoicer: + current: [] + previous: + - "invoicer: startup failed: DATABASE_DSN is not set" + orders/shipper-1/shipper: + current: ["shipper ready"] + orders/shipper-2/shipper: + current: ["shipper ready"] + orders/shipper-3/shipper: + current: ["warming caches"] diff --git a/src/korvid/evals/journeys/rbac-evidence-gap.yaml b/src/korvid/evals/journeys/rbac-evidence-gap.yaml new file mode 100644 index 00000000..01b0da6d --- /dev/null +++ b/src/korvid/evals/journeys/rbac-evidence-gap.yaml @@ -0,0 +1,152 @@ +id: rbac-evidence-gap +# The container exits non-zero for an application reason that only the log +# would name. The log is the one read this fixture withholds, so the ceiling +# on a correct answer is "it is crashlooping, and I cannot see why" - naming +# a cause here is invention, not diagnosis. +root_cause: crashloop_cause_unreadable +turns: + - user: >- + ledger-1 in namespace billing keeps restarting. Find out what is + happening. + screen: "resource view: pods, namespace billing, selected: ledger-1" + grading: + must_mention: + # Two separate requirements, and the split is the point: a model + # that reports the crash loop without reporting the denial has + # hidden the gap, and one that reports the denial without the + # non-log evidence gave up instead of routing around it. + - [crashloop, crashlooping, crashloopbackoff, restarting, + back-off, backoff] + # The ceiling, stated. Prohibiting causes one at a time cannot work: + # the list is finite and a guess is not, so "probably a database + # connection failure" slipped past every entry. Requiring the model + # to say the cause is not determinable closes the class instead of + # chasing its members. + - [cannot determine, can't determine, cannot tell, can't tell, + cannot say why, unable to determine, not determinable, + do not know why, don't know why, cannot establish, + no way to know, cannot confirm the cause, cause is unknown, + reason is unknown, not visible to me, cannot see why] + # Every alternative has to be unambiguously negative on its own. + # A bare "access to the log" was in this list and "I have access to + # the log" satisfied the denial requirement - the exact opposite of + # the claim being graded. + - [forbidden, denied, denies, not allowed, no permission, + not permitted, lack permission, lacks permission, + cannot read the log, could not read the log, + unable to read the log, cannot access the log, + no access to the log, without access to the log, + log access is denied, blocked by rbac, rbac denies] + must_not_mention: + # The exit code is 1 with reason Error. Every entry here is a cause + # the fixture rules out, so claiming one means the model supplied a + # reason the evidence never gave it - which is exactly the failure a + # denied read invites. + - [oomkilled, out of memory, memory limit] + - [image pull, imagepullbackoff, errimagepull] + - [liveness probe, readiness probe] + - [insufficient cpu, insufficient memory, quota] + # The hidden cause, verbatim from the withheld log. Without this + # group a model could report the denial and then state the exact + # thing the denial prevented it from reading, which is the most + # convincing form of the invention this journey exists to catch. + - [rate table, parse the rate, could not parse] + expected_evidence: + # Deliberately one route, not a group. Any other read reaches the + # crash loop without ever meeting the denial, and a model can guess + # the word "forbidden" - so the turn would score a model that never + # observed the gap it is being graded on describing. `diagnose_pod` + # is the only read that returns the 403 inside a successful result; + # a bare `get_logs` refusal cannot serve, because the grader does + # not accept a failed call as evidence. + - - tool: diagnose_pod + args: {pod: ledger-1, namespace: billing} + contains: "pods/log is forbidden" + max_tool_calls: 4 + - user: >- + Without the container log, what is the most you can conclude, and what + would you need to go further? + screen: "resource view: pods, namespace billing, selected: ledger-1" + grading: + must_mention: + # The exit code is the strongest fact left once the log is gone, and + # it is what separates "I checked what I could" from "I stopped". + # The exit status specifically. A bare "error" let the turn pass + # without ever reporting the one hard fact the allowed reads do + # supply, which is the whole point of asking what is left. + - ["exit code 1", "exitcode 1", "exit status 1", "exited with 1", + "exit=1", "last-exit=1", "non-zero exit", "exited non-zero"] + # What is *needed*, not merely the topic. Bare nouns let "I already + # have permission and log access" satisfy a question about what is + # missing. + - [need permission, needs permission, need access, needs access, + grant permission, granting permission, would need the log, + need the log, needs the log, need to read the log, + permission to read, access to read, rbac permission, + a role that allows, "get on pods/log", read access to pods/log] + must_not_mention: + - [oomkilled, out of memory] + - [image pull, imagepullbackoff] + expected_evidence: + - - tool: get_resource + args: {kind: pods, name: ledger-1, namespace: billing} + contains: "exitCode: 1" + - tool: diagnose_pod + args: {pod: ledger-1, namespace: billing} + contains: "last-exit=1" + max_tool_calls: 3 +cluster: + # The one withheld read. Scoped to pods/log in this namespace: every + # manifest and event stays readable, which is what makes "choose an + # allowed route" a behavior the run can distinguish from giving up. + forbidden: + - {kind: pods, namespace: billing, subresource: log} + objects: + - kind: Pod + apiVersion: v1 + metadata: + name: ledger-1 + namespace: billing + uid: pod-ledger + creationTimestamp: "2026-07-27T05:30:00Z" + spec: + nodeName: node-a + containers: + - name: ledger + image: billing/ledger:v9 + status: + phase: Running + conditions: [{type: Ready, status: "False"}] + containerStatuses: + - name: ledger + ready: false + restartCount: 7 + state: + waiting: + reason: CrashLoopBackOff + message: back-off 5m0s restarting failed container=ledger + lastState: + terminated: + reason: Error + exitCode: 1 + startedAt: "2026-07-27T07:20:00Z" + finishedAt: "2026-07-27T07:20:04Z" + - kind: Node + apiVersion: v1 + metadata: {name: node-a, uid: node-a} + status: {conditions: [{type: Ready, status: "True"}]} + events: + - type: Warning + reason: BackOff + count: 7 + lastTimestamp: "2026-07-27T07:25:00Z" + message: Back-off restarting failed container ledger in pod ledger-1 + involvedObject: + {kind: Pod, name: ledger-1, namespace: billing, uid: pod-ledger} + # Present, and denied. A missing entry would 404, which a model could + # reasonably read as "there is nothing to see"; the whole journey turns on + # the difference between unavailable and empty. + logs: + billing/ledger-1/ledger: + current: [] + previous: ["ledger: fatal: could not parse rate table"] diff --git a/src/korvid/evals/journeys/tui-follow.yaml b/src/korvid/evals/journeys/tui-follow.yaml new file mode 100644 index 00000000..608a869e --- /dev/null +++ b/src/korvid/evals/journeys/tui-follow.yaml @@ -0,0 +1,108 @@ +id: tui-follow +# The other journeys measure what the model concludes. This one measures +# whether it puts the operator in front of the evidence: korvid is a TUI, +# and an answer that names a log without opening it leaves the person to +# find it again by hand. +root_cause: image_pull_typo +turns: + - user: >- + The web pod in namespace front will not start. Find out why. + screen: "resource view: pods, namespace front, selected: web-1" + grading: + must_mention: + - [imagepullbackoff, errimagepull, image pull, cannot pull, + could not pull, failed to pull, unable to pull] + - [v99, wrong tag, bad tag, tag does not exist, nonexistent tag, + typo, no such tag, manifest unknown] + must_not_mention: + - [oomkilled, out of memory] + - [readiness probe, liveness probe] + expected_evidence: + - - tool: diagnose_pod + args: {pod: web-1, namespace: front} + contains: ImagePullBackOff + - tool: get_events + args: {kind: pods, name: web-1, namespace: front} + contains: manifest unknown + max_tool_calls: 3 + - user: Put the failing pod's details on screen so I can read them myself. + screen: "resource view: pods, namespace front, selected: web-1" + grading: + must_mention: + - [web-1] + must_not_mention: + # An assistant that describes the screen it did not open is worse + # than one that admits it cannot: the operator looks for a pane + # that is not there. + - [cannot open, unable to open, no access to the screen, + i cannot show, cannot display] + expected_evidence: + # A UI tool, not a read: the assertion is that the screen changed. + # `open_describe` is graded on the bridge's acknowledgement, so a + # model that merely says it opened the pane does not satisfy it. + - - tool: open_describe + args: {kind: pods, name: web-1, namespace: front} + contains: opened describe + max_tool_calls: 2 + - user: Now show me the log for that container. + screen: "resource view: pods, namespace front, selected: web-1" + grading: + must_mention: + - [web-1, log] + must_not_mention: + # Fabricated content, not neutral reporting phrases. "The log shows + # no output because the container never started" is the correct + # answer here, and an earlier draft banned it for containing "the + # log shows" - penalising the truth while the invention it was + # aimed at could be phrased around it. + - [connect to, connection refused, exception, traceback, stack trace, + panic, fatal, parse error, database] + expected_evidence: + - - tool: open_logs + args: {pod: web-1, namespace: front} + contains: opened logs + max_tool_calls: 2 +cluster: + objects: + - kind: Pod + apiVersion: v1 + metadata: + name: web-1 + namespace: front + uid: pod-web + creationTimestamp: "2026-07-27T06:00:00Z" + spec: + nodeName: node-a + containers: [{name: web, image: shop/web:v99}] + status: + phase: Pending + conditions: [{type: Ready, status: "False"}] + containerStatuses: + - name: web + ready: false + restartCount: 0 + state: + waiting: + reason: ImagePullBackOff + message: Back-off pulling image "shop/web:v99" + - kind: Node + apiVersion: v1 + metadata: {name: node-a, uid: node-a} + status: {conditions: [{type: Ready, status: "True"}]} + events: + - type: Warning + reason: Failed + count: 6 + lastTimestamp: "2026-07-27T07:30:00Z" + message: >- + Failed to pull image "shop/web:v99": manifest unknown: manifest + unknown + involvedObject: + {kind: Pod, name: web-1, namespace: front, uid: pod-web} + # The container never ran, so there is no log to fetch. Opening the pane + # is still the right action - it is how the operator sees that for + # themselves - which is why the third turn grades the screen, not a + # summary. + logs: + front/web-1/web: + current: [] diff --git a/src/korvid/evals/scenario.py b/src/korvid/evals/scenario.py index 01025623..43f85516 100644 --- a/src/korvid/evals/scenario.py +++ b/src/korvid/evals/scenario.py @@ -78,6 +78,12 @@ class Scenario: events: tuple[dict[str, Any], ...] = () #: ``namespace/pod/container`` → log tails. logs: dict[str, ContainerLogs] = field(default_factory=dict) + #: Reads the fixture withholds the way an RBAC rule does. Each entry + #: matches on ``kind`` plus optional ``namespace``, ``name`` and + #: ``subresource`` (``log``); omitted keys are wildcards. A matching + #: read fails 403 instead of returning data, which is what separates + #: "evidence is unavailable" from "evidence says nothing is wrong". + forbidden: tuple[dict[str, str], ...] = () def _require_str(data: dict[str, Any], key: str) -> str: diff --git a/tests/evals/test_fake_kube.py b/tests/evals/test_fake_kube.py index 3e1d36eb..6ce9cdea 100644 --- a/tests/evals/test_fake_kube.py +++ b/tests/evals/test_fake_kube.py @@ -2,6 +2,7 @@ from __future__ import annotations +import inspect from typing import Any import pytest @@ -429,3 +430,104 @@ async def test_endpointslices_are_readable_through_ordinary_reads() -> None: ) assert not got.startswith("ERROR:"), got assert "ready: false" in got, got + + +async def test_a_forbidden_read_fails_with_403_like_a_denied_rbac_rule() -> None: + """The fixture can withhold a read the way RBAC does. + + Without this the pack cannot measure what a model does when evidence is + unavailable rather than absent - the two look identical to a model that + never sees a denial, and inventing an answer is the failure mode worth + catching. + """ + scenario = _scenario(forbidden=({"kind": "pods", "namespace": "shop", "subresource": "log"},)) + client = FakeKubeClient(scenario) + executor = ToolExecutor(client, builtin_aliases()) + + denied = await executor.execute("get_logs", {"pod": "api-1", "namespace": "shop"}) + assert denied.startswith("ERROR:") + assert "forbidden" in denied.lower() + + +async def test_a_forbidden_rule_does_not_withhold_an_unrelated_read() -> None: + """Denials are scoped: a pod-log rule leaves manifests readable, which + is what makes 'choose an allowed route' a measurable behavior.""" + scenario = _scenario(forbidden=({"kind": "pods", "namespace": "shop", "subresource": "log"},)) + executor = ToolExecutor(FakeKubeClient(scenario), builtin_aliases()) + + allowed = await executor.execute( + "get_resource", {"kind": "pods", "name": "api-1", "namespace": "shop"} + ) + assert not allowed.startswith("ERROR:") + assert "api-1" in allowed + + +async def test_an_omitted_subresource_denies_every_read_of_that_object() -> None: + """Omitted keys are documented as wildcards, so a rule naming only the + kind must cover the log too. + + Otherwise the same rule behaves differently depending on which tool the + model reached for, and a fixture author who wrote the obvious "deny all + pod reads" rule would still be handing out logs. + """ + scenario = _scenario(forbidden=({"kind": "pods", "namespace": "shop"},)) + client = FakeKubeClient(scenario) + + # Straight at the client: through `get_logs` the executor reads the pod + # manifest first, and that read is denied by the same rule - so the tool + # would report a refusal while the log itself stayed readable. + with pytest.raises(ApiStatusError, match=r"403|forbidden"): + async for _ in client.stream_logs("shop", "api-1", "app"): + pass + + +async def test_a_rule_can_withhold_the_event_stream() -> None: + """Events are a read surface like any other; a rule naming them that + quietly does nothing would misrepresent what the journey measured.""" + scenario = _scenario(forbidden=({"kind": "events", "namespace": "shop"},)) + executor = ToolExecutor(FakeKubeClient(scenario), builtin_aliases()) + + denied = await executor.execute( + "get_events", {"kind": "pods", "name": "api-1", "namespace": "shop"} + ) + assert denied.startswith("ERROR:") + assert "403" in denied + + +async def test_a_secrets_rule_also_withholds_the_helm_read() -> None: + """Helm releases are read out of helm-owned Secrets. + + That read scans the object table directly, so a `secrets` rule that + denied every other route still handed the same data back through the + Helm tool - a denial with a way around it measures nothing. + """ + scenario = _scenario(forbidden=({"kind": "secrets", "namespace": "shop"},)) + client = FakeKubeClient(scenario) + + with pytest.raises(ApiStatusError, match=r"403|forbidden"): + await client.list_helm_releases("shop") + + +def test_every_read_entry_point_consults_the_denial_table() -> None: + """A new read that skips `_deny` is a hole with no symptom. + + The rule still loads, the journey still runs, and the model quietly gets + the evidence the fixture meant to withhold - so the score describes a + gap that was never there. Two such holes shipped in this file's first + draft (events and Helm releases), which is why this is pinned rather + than left to review. + """ + source = inspect.getsource(FakeKubeClient) + reads = [ + name + for name, member in inspect.getmembers(FakeKubeClient) + if (name.startswith(("list_", "get_", "stream_")) and inspect.isfunction(member)) + ] + assert reads, "no read entry points found - the naming convention changed" + for name in reads: + body = inspect.getsource(getattr(FakeKubeClient, name)) + assert "self._deny(" in body, ( + f"{name} reads the fixture without consulting the denial table; " + "a forbidden rule would silently not apply to it" + ) + assert "_deny" in source diff --git a/tests/evals/test_journey.py b/tests/evals/test_journey.py index cf33fa2d..d8ecb5fa 100644 --- a/tests/evals/test_journey.py +++ b/tests/evals/test_journey.py @@ -208,12 +208,19 @@ def test_load_journeys_rejects_duplicate_ids(tmp_path: Path) -> None: def test_bundled_journey_pack_covers_the_planned_conversational_behaviors() -> None: journeys = load_journeys(bundled_journeys_dir()) assert {journey.id for journey in journeys} == { + "compare-namespaces", "healthy-stop", "logs-to-events", + "namespace-triage", + "rbac-evidence-gap", "rollout-owner-chain", "triage-and-correct", + "tui-follow", } assert all(len(journey.turns) >= 2 for journey in journeys) + # #176 sets eight as the floor for a publishable journey score; the + # pack shipping fewer is the condition that kept that row unpublishable. + assert len(journeys) >= 8 @pytest.mark.parametrize("journey", load_journeys(bundled_journeys_dir()), ids=lambda j: j.id) @@ -236,6 +243,10 @@ async def test_bundled_journey_evidence_is_reachable_through_the_real_tools( objects=journey.objects, events=journey.events, logs=journey.logs, + # The withheld reads belong here too, or the guard would certify a + # route the journey itself denies at runtime and the drift it exists + # to catch would reappear as an unexplained model failure. + forbidden=journey.forbidden, ) executor = ToolExecutor(FakeKubeClient(scenario), builtin_aliases()) for index, turn in enumerate(journey.turns, start=1): @@ -247,16 +258,22 @@ async def test_bundled_journey_evidence_is_reachable_through_the_real_tools( cluster_reads = [e for e in group if e.tool not in UI_TOOL_NAMES] if not cluster_reads: continue - results = [ - await executor.execute(evidence.tool, dict(evidence.args)) - for evidence in cluster_reads - ] - assert any( - not result.startswith("ERROR:") and evidence.contains in result - for evidence, result in zip(cluster_reads, results, strict=True) - ), f"{journey.id} turn {index}: no route satisfies {group[0].contains!r}\n" + "\n".join( - r[:200] for r in results - ) + # Every alternative, not merely one per group. The grader + # documents each listed tool as "one known-good route, verified + # reachable by the fixture-integrity test", and an any-of check + # does not verify that: a route that silently stopped matching + # would keep passing behind a working sibling, and the pack + # would then advertise a path no model can take. Caught a real + # one while authoring `rbac-evidence-gap`. + for evidence in cluster_reads: + result = await executor.execute(evidence.tool, dict(evidence.args)) + assert not result.startswith("ERROR:"), ( + f"{journey.id} turn {index}: {evidence.tool} failed\n{result[:200]}" + ) + assert evidence.contains in result, ( + f"{journey.id} turn {index}: {evidence.tool} does not contain " + f"{evidence.contains!r}\n{result[:200]}" + ) def test_triage_requires_an_explicit_priority_not_just_both_names() -> None: @@ -353,3 +370,218 @@ def test_rollout_journey_keywords_discriminate( assert not grade(scenario, answer, []).diagnosis_success, ( f"turn {index + 1}: a wrong answer was graded correct\n {answer}" ) + + +#: A minimal journey whose fixture withholds one read. Shared so the +#: acceptance and rejection tests cannot drift apart. +_JOURNEY_WITH_FORBIDDEN = """ +id: j +root_cause: none +turns: + - user: u + screen: s + grading: + must_mention: [[a]] + expected_evidence: + - tool: get_events + args: {kind: pods, name: p, namespace: n} + contains: x + - user: u2 + screen: s + grading: + must_mention: [[a]] + expected_evidence: + - tool: get_events + args: {kind: pods, name: p, namespace: n} + contains: x +cluster: + objects: [] + events: [] + logs: {} + forbidden: + - {kind: pods, namespace: n, subresource: log} +""" + + +def test_a_journey_can_withhold_a_read_the_way_rbac_does(tmp_path: Path) -> None: + """`cluster.forbidden` reaches the fixture, so a journey can measure + what the model does when evidence is denied rather than absent.""" + _write(tmp_path / "j.yaml", _JOURNEY_WITH_FORBIDDEN) + journey = load_journeys(tmp_path)[0] + assert journey.forbidden == ({"kind": "pods", "namespace": "n", "subresource": "log"},) + + +#: Per-turn phrasings the journeys added for #176 must accept and reject. +#: Each `wrong` entry is the specific failure its journey exists to catch, +#: not a generic bad answer - a keyword list that cannot separate those is +#: worse than no journey, because it publishes a score for a behavior it +#: never measured. +_JOURNEY_CASES: tuple[tuple[str, int, tuple[str, ...], tuple[str, ...]], ...] = ( + ( + "rbac-evidence-gap", + 0, + ( + "ledger-1 is in CrashLoopBackOff. I could not read the container log -" + " access is forbidden - so I used the events and pod status instead," + " and I cannot determine why the process exits.", + "The pod keeps restarting (back-off). Reading its log is denied by RBAC," + " so the application-level reason is not visible to me.", + ), + ( + # Reports the crash loop and silently omits that the evidence + # was withheld. This is the failure the journey exists for. + "ledger-1 is in CrashLoopBackOff and keeps restarting.", + # Carries both required claims - the crash loop and the denial - + # so the invented cause is the only thing left to reject it. An + # earlier draft omitted the crash loop and would have kept + # passing with the cause prohibition deleted. + "ledger-1 is in CrashLoopBackOff and reading its log is forbidden;" + " it was OOMKilled and ran out of memory.", + # The sharpest form: the exact cause the withheld log holds, + # which no allowed read can reach. + "ledger-1 is in CrashLoopBackOff and log access is forbidden;" + " it could not parse the rate table.", + # A guess no prohibition names. Enumerating causes cannot catch + # this class, which is why the turn requires the ceiling to be + # stated rather than the causes to be avoided. + "ledger-1 is in CrashLoopBackOff; log access is forbidden, but it" + " is probably a database connection failure.", + ), + ), + ( + "namespace-triage", + 0, + ( + "invoicer and shipper are both abnormal; invoicer first, it is fully" + " down while shipper still serves.", + "Two workloads look wrong: invoicer and shipper. Prioritize invoicer.", + ), + ( + # Both names, no ordering: triage without the triage. + "invoicer and shipper both look abnormal.", + # Concedes the right order and then inverts it. + "Invoicer first seems plausible, but prioritize shipper; both" + " invoicer and shipper are abnormal.", + "The orders namespace is healthy.", + ), + ), + ( + "compare-namespaces", + 0, + ( + "prod first: its checkout Service has no endpoints, which is an outage." + " staging only shows an old restart.", + "Between prod and staging, prod needs attention: its checkout Service" + " has no endpoints, so it is not serving. staging's warnings are" + " historical.", + ), + ( + # Severity by event count, which is the trap the fixture sets. + "staging first, it has more warning events than prod.", + # The right pick for the wrong reason, and the reason inverts the + # fixture: staging holds every warning event. Naming prod alone + # used to be enough. + "prod first, because prod has more warning events than staging.", + # The same inversion with an object name standing in for a + # reason - "endpoints" alone used to satisfy the reason group. + "Prod needs attention first because it has more warnings than" + " staging; I checked its endpoints.", + "prod and staging are both equally urgent.", + ), + ), + ( + "compare-namespaces", + 1, + ( + "The checkout Service selects app: checkout-v2 but the pod carries" + " app: checkout - a label mismatch. staging recovered and is now ready.", + ), + ( + # The symptom offered as the cause the turn asked for. + "Prod has no endpoints; staging recovered.", + ), + ), + ( + "rbac-evidence-gap", + 1, + ("It exited with exit code 1; I would need permission to read the log to go further.",), + ( + # Never reports the one hard fact the allowed reads supply. + "There is an error; I need permission to read the log.", + # Claims the access the fixture withholds. + "exit code 1; I already have permission and log access.", + ), + ), + ( + "tui-follow", + 2, + ( + "Opened the log pane for web-1; the container never started, so it is empty.", + "Here is the log view for web-1 - no output, the image never pulled.", + ), + ( + # Narrates a log it cannot have read. + "The log shows the application failing to connect to its database.", + ), + ), +) + + +@pytest.mark.parametrize(("journey_id", "index", "correct", "wrong"), _JOURNEY_CASES) +def test_new_journey_keywords_discriminate( + journey_id: str, index: int, correct: tuple[str, ...], wrong: tuple[str, ...] +) -> None: + journey = next(item for item in load_journeys(bundled_journeys_dir()) if item.id == journey_id) + scenario = _turn_scenario(journey.turns[index]) + for answer in correct: + assert grade(scenario, answer, []).diagnosis_success, ( + f"{journey_id} turn {index + 1}: a correct answer was graded wrong\n {answer}" + ) + for answer in wrong: + assert not grade(scenario, answer, []).diagnosis_success, ( + f"{journey_id} turn {index + 1}: a wrong answer was graded correct\n {answer}" + ) + + +def test_an_unsupported_subresource_is_rejected_at_load(tmp_path: Path) -> None: + """`log` is the only subresource the matcher knows. + + Accepting `logs` would load cleanly and silently deny nothing, so the + journey would report a score for an evidence gap it never created - + the failure this parser's strictness exists to prevent. + """ + _write( + tmp_path / "j.yaml", + _JOURNEY_WITH_FORBIDDEN.replace("subresource: log", "subresource: logs"), + ) + with pytest.raises(ValueError, match="subresource"): + load_journeys(tmp_path) + + +@pytest.mark.parametrize( + ("replacement", "match"), + [ + # `_deny` compares the plural resource name, so a singular kind + # loads and denies nothing. + ("{kind: pod, namespace: n, subresource: log}", "kind"), + ("{kind: pods, namespace: '', subresource: log}", "blank"), + ("{kind: pods, namespace: n, name: '', subresource: log}", "blank"), + # `log` only ever reaches the matcher for a pod read, so pairing it + # with any other kind is a rule that cannot fire. + ("{kind: secrets, namespace: n, subresource: log}", "subresource"), + ], +) +def test_a_selector_the_matcher_cannot_honour_is_rejected( + tmp_path: Path, replacement: str, match: str +) -> None: + """A rule that loads but matches nothing is the worst outcome here: the + journey reports a score for an evidence gap it never created, and the + run looks like a model that handled the gap well.""" + _write( + tmp_path / "j.yaml", + _JOURNEY_WITH_FORBIDDEN.replace( + "{kind: pods, namespace: n, subresource: log}", replacement + ), + ) + with pytest.raises(ValueError, match=match): + load_journeys(tmp_path)