From 4a1a1581324eee33d3a146d221989a080faecd67 Mon Sep 17 00:00:00 2001 From: minion1227 Date: Thu, 30 Jul 2026 15:53:19 -0700 Subject: [PATCH 1/2] =?UTF-8?q?feat(bench):=20composite=20guards=20?= =?UTF-8?q?=E2=80=94=20efficiency,=20consistency,=20canary?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vouch scored a mean over per-category means and nothing else. a retrieval strategy can score well on category means while being right by luck: brittle under paraphrase, or dragging in half the kb to reach the answer. the koth ladder measures strategies against each other, so an unguarded score is exactly what a ladder optimises into. the compatibility problem the issue raises is real: changing the composite formula makes every recorded score and ladder entry incomparable, which is a season-boundary decision, not a merge-day one. so the guards are reported *beside* the composite rather than inside it. `composite` keeps its exact formula and meaning; `composite_guarded` is the new measurement; and `bench_version` on every report is what stops the two being compared by accident. the ladder adopts the guarded number when a maintainer decides, not when this merges. run_seeds also degrades cleanly on a version-1 report with no guard block. efficiency is a bounded penalty on how much budget the answers consumed, floored so a wasteful run is penalised and never zeroed — it is a tiebreak, not a verdict. consistency is metamorphic: each pack-graded case is re-asked through a template paraphrase and must grade the same. template-driven, never llm-written, so a run stays reproducible from the seed alone. canary plants a coined value that answers no question; any pack surfacing it halves the composite. the wording took two passes. the first draft read "the retired access code was ..." and tripped on "what was the project codename before it changed?" — lexical overlap on code/retired, not a dump, so the guard was measuring the generator's word choice. a test now asserts the bait shares no vocabulary with any generated question. the binary multiplier is joined by a leak rate, because "one stray pack in twenty-six" and "half of them" are different engineering problems and a 0.5 that cannot tell them apart is not actionable. measured over seeds 1-3: efficiency 0.88, consistency 1.00, canary 0.50 — tripped on every seed, leak 0.08-0.24. a ten-item pack over this corpus carries about a quarter of it, which is the lever the guard exists to expose. consistency at 1.00 is a floor rather than proof: prefix-form paraphrases keep every content word, so they do not move lexical retrieval. both numbers are in the reference table rather than tuned away. Closes #616 --- CHANGELOG.md | 10 +++ src/vouch/bench.py | 200 +++++++++++++++++++++++++++++++++++++++++++- tests/test_bench.py | 195 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 404 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e5a72cb..6b8a6c24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ All notable changes to vouch are documented here. Format follows ## [Unreleased] ### Added +- **bench: composite guards** (#616): `efficiency`, `consistency` and `canary` + as bounded multipliers over the composite, plus a `bench_version` stamp on + every report. Reported **beside** the composite, never folded into it — + `composite` keeps its exact formula and meaning, so no recorded score or + ladder entry becomes incomparable, and `composite_guarded` is the new + measurement the ladder can adopt at a season boundary of the maintainer's + choosing. `run_seeds` tolerates a version-1 report with no guard block. + Measured on seeds 1-3: efficiency 0.88, consistency 1.00, canary **0.50 — + tripped on every seed** (leak rate 0.08-0.24), because a 10-item pack over + this corpus carries ~24% of it. That is the lever the guard exists to expose. - **explicit pins — a working set that always enters the pack** (#615): `vouch pin ` / `vouch pins list` / `vouch unpin `. Pinned claims and pages lead every context pack instead of having to win the query each turn, diff --git a/src/vouch/bench.py b/src/vouch/bench.py index 6cfa90c3..c3b7d97c 100644 --- a/src/vouch/bench.py +++ b/src/vouch/bench.py @@ -87,6 +87,26 @@ carried the parts. multi-hop-relational is where the derivation axes actually bite today. +Composite guards (#616), measured over seeds 1,2,3 @ 2026-07-30. Reported +beside the composite, never folded into it, and stamped with ``bench_version`` +so a guarded score is never silently compared to a recorded legacy one: + +====================== ===================================================== +composite 0.64 — unchanged formula, unchanged meaning +composite_guarded 0.28 — composite x efficiency x consistency x canary +efficiency 0.88 — packs run ~12% of budget below the cap +consistency 1.00 — prefix-form paraphrases do not move retrieval, + so this is a floor on robustness, not proof + of it; a content-rewriting paraphrase would + test harder but risks changing the question +canary 0.50 — TRIPPED on every seed, leak rate 0.08-0.24. + the planted line reaches a pack because a + 10-item pack over this corpus carries ~24% of + it. this is the lever the guard exists to + expose, and the reason the guarded composite + is not yet the ladder's number +====================== ===================================================== + For calibration only (different benchmarks, not directly comparable): ditto's stock production-mirroring harness reports memory_mean 0.200-0.226 on its own v5/v6 contracts, with 0.00 on its consolidation/multi-hop @@ -146,6 +166,44 @@ # somewhere. See the note at the top of their block in ``generate``. _DERIVED_SALT = 0x9E3779B9 +# Bench report version. "1" is every score recorded before the composite +# guards existed. The stamp is the whole compatibility story: a guarded score +# and a legacy one are different measurements, and a version on the report is +# what stops them being compared by accident. +BENCH_VERSION = 2 + +# Guards (#616). Bounded multipliers over the composite, never categories: a +# guard averaged in with everything else stops being a guard — canary at zero +# should be alarming, not a ten-percent dent. +GUARD_MIN_EFFICIENCY = 0.5 # a maximally wasteful pack still scores half +CANARY_PENALTY = 0.5 # echoing planted bait halves the composite +PARAPHRASE_COUNT = 2 # the original plus one metamorphic rephrasing + +# Semantically identical rewrites. Template-generated, never LLM-written, so a +# run stays reproducible from the seed alone. +_PARAPHRASE_TEMPLATES = ( + "quick one — {q}", + "remind me: {q}", + "sorry, asking again: {q}", +) + +# The canary is a coined value planted in the corpus that answers nothing. Any +# pack that surfaces it is dragging in content no question asked for. +# +# The wording matters as much as the value: the sentence must share no +# vocabulary with any generated question, or the guard measures the +# generator's word choice instead of the ranker. A first draft used "the +# retired access code was ..." and tripped on "what was the project codename +# before it changed?" — lexical overlap on code/retired, not a dump. +# test_canary_shares_no_vocabulary_with_any_question keeps it honest. +_CANARY_TEMPLATE = "{value} surfaced in a thread nobody followed up on." + +# Graded against the store's receipts and lifecycle rather than the pack text, +# so a rephrased question cannot change their score. +_STORE_GRADED_CATEGORIES = frozenset( + ("citation-correctness", "receipt-coverage", "supersede-hygiene") +) + # A superseded, archived, or redacted claim is not a live memory (mirrors # the set the context pack excludes). _RETIRED_STATUSES = frozenset( @@ -333,6 +391,8 @@ class Dataset: seed: int sessions: tuple[tuple[str, str], ...] # (title, text) cases: tuple[MemoryCase, ...] + # Planted bait no question asks for; surfacing it trips the canary guard. + canary: str = "" @dataclass @@ -573,6 +633,12 @@ def coin_distinct(n: int, syllables: int = 3) -> list[str]: "aggregation", sub.choice(set_asks), None, required=tuple(set_items), )) + # The canary: a coined value that answers no question in the dataset. It + # sits on the derived rng like the derivation categories, so categories + # 1-10 stay byte-identical. + canary = coin_distinct(1)[0] + docs.add(sub_spread(1)[0], _CANARY_TEMPLATE.format(value=canary)) + # Filler prose in every session, shuffled placement. for idx in range(sessions): for _ in range(rng.randrange(2, 5)): @@ -583,7 +649,9 @@ def coin_distinct(n: int, syllables: int = 3) -> list[str]: (f"bench session {i + 1}", "\n".join(lines)) for i, lines in enumerate(docs.buckets) ) - return Dataset(seed=seed, sessions=session_docs, cases=tuple(cases)) + return Dataset( + seed=seed, sessions=session_docs, cases=tuple(cases), canary=canary, + ) def grade_case(case: MemoryCase, pack_text: str) -> tuple[float, str | None]: @@ -734,6 +802,51 @@ def _pack_text(pack: dict[str, Any]) -> str: return text.replace("«", "").replace("»", "") +def paraphrase(question: str, index: int) -> str: + """A semantically identical rewrite of ``question``. + + Index 0 is the original. Deterministic and template-driven — an LLM + rewrite would make the metamorphic check unreproducible from the seed, + which is the one property the whole benchmark rests on. + """ + if index <= 0: + return question + template = _PARAPHRASE_TEMPLATES[(index - 1) % len(_PARAPHRASE_TEMPLATES)] + return template.format(q=question) + + +def efficiency_multiplier(used_chars: list[int], budget_chars: int) -> float: + """Bounded penalty on how much budget the answers consumed. + + A strategy that hits the answer by dragging in half the KB is right by + luck, not by ranking; this is what makes the dump-guard cost something + even on categories with no forbidden value. Bounded below so a wasteful + run is penalised, never zeroed — efficiency is a tiebreak, not a verdict. + """ + if not used_chars or budget_chars <= 0: + return 1.0 + mean_used = statistics.mean(used_chars) + share = min(1.0, mean_used / budget_chars) + return round(GUARD_MIN_EFFICIENCY + (1.0 - GUARD_MIN_EFFICIENCY) * (1.0 - share), 4) + + +def consistency_multiplier(agreements: list[bool]) -> float: + """Fraction of cases whose paraphrases all graded the same. + + Metamorphic: a question asked two ways is the same question. Scoring well + on one phrasing and badly on its rewrite is brittleness, and a category + mean cannot see it. + """ + if not agreements: + return 1.0 + return round(sum(1 for a in agreements if a) / len(agreements), 4) + + +def canary_multiplier(tripped: bool) -> float: + """Halve the composite when planted bait reached a pack.""" + return CANARY_PENALTY if tripped else 1.0 + + def run( seed: int, *, @@ -791,12 +904,21 @@ def run( per_category: dict[str, list[float]] = {c: [] for c in CATEGORIES} failures: list[dict[str, Any]] = [] + used_chars: list[int] = [] + agreements: list[bool] = [] + canary_hits = 0 + packs_seen = 0 for case in dataset.cases: pack = build_context_pack( store, query=case.question, limit=limit, max_chars=budget_chars, strategy=strategy, ) pack_dict = dict(pack) + text = _pack_text(pack_dict) + used_chars.append(len(text)) + packs_seen += 1 + if dataset.canary and dataset.canary in text: + canary_hits += 1 if case.category == "citation-correctness": score, reason = grade_citation_correctness(store, case, pack_dict) elif case.category == "receipt-coverage": @@ -806,6 +928,29 @@ def run( else: score, reason = grade_case(case, _pack_text(pack_dict)) per_category[case.category].append(score) + + # Metamorphic check: the same question, asked another way, must + # grade the same. Only the pack-text categories are re-asked — + # the three verifiability graders read the store, not the query, + # so rephrasing cannot move them. + if case.category not in _STORE_GRADED_CATEGORIES: + agreed = True + for i in range(1, PARAPHRASE_COUNT): + alt = build_context_pack( + store, query=paraphrase(case.question, i), limit=limit, + max_chars=budget_chars, strategy=strategy, + ) + alt_dict = dict(alt) + alt_text = _pack_text(alt_dict) + used_chars.append(len(alt_text)) + packs_seen += 1 + if dataset.canary and dataset.canary in alt_text: + canary_hits += 1 + alt_score, _ = grade_case(case, alt_text) + if alt_score != score: + agreed = False + agreements.append(agreed) + if reason is not None: failures.append({ "category": case.category, @@ -823,7 +968,33 @@ def run( } means = [statistics.mean(s) for s in per_category.values() if s] composite = round(statistics.mean(means), 4) if means else 0.0 + + # The guards are reported beside the composite, never folded into it. An + # opaque composite is worse than no composite, and — the compatibility + # point — `composite` keeps meaning exactly what every recorded score and + # ladder entry already means. `composite_guarded` is the new measurement, + # and `bench_version` is what stops the two being compared by accident. + # The multiplier is binary per the spec — bait in the pack is bait in the + # pack. The leak *rate* is reported beside it because "one stray pack out + # of twenty-six" and "half of them" are very different engineering + # problems, and a 0.5 that cannot tell them apart is not actionable. + canary_tripped = canary_hits > 0 + guards = { + "efficiency": efficiency_multiplier(used_chars, budget_chars), + "consistency": consistency_multiplier(agreements), + "canary": canary_multiplier(canary_tripped), + "canary_tripped": canary_tripped, + "canary_leak_rate": ( + round(canary_hits / packs_seen, 4) if packs_seen else 0.0 + ), + } + guarded = round( + composite + * guards["efficiency"] * guards["consistency"] * guards["canary"], + 4, + ) return { + "bench_version": BENCH_VERSION, "seed": seed, "budget_chars": budget_chars, "limit": limit, @@ -831,6 +1002,8 @@ def run( "cases": len(dataset.cases), "categories": categories, "composite": composite, + "guards": guards, + "composite_guarded": guarded, "failures": failures, } @@ -874,11 +1047,36 @@ def run_seeds( ] if vals: category_means[name] = round(statistics.mean(vals), 4) + # Degrade rather than crash on a report without guards: a bench_version 1 + # run (or a cached one) has no guard block, and the aggregate should still + # produce the legacy numbers. Missing guards read as neutral 1.0. + guarded = [ + r.get("composite_guarded", r["composite"]) for r in reports + ] + guarded_mean = statistics.mean(guarded) + guarded_se = ( + statistics.stdev(guarded) / (len(guarded) ** 0.5) + if len(guarded) > 1 else 0.0 + ) + guard_means = { + name: round( + statistics.mean( + [float(r.get("guards", {}).get(name, 1.0)) for r in reports] + ), 4, + ) + for name in ("efficiency", "consistency", "canary") + } return { + "bench_version": BENCH_VERSION, "seeds": seeds, "budget_chars": budget_chars, "composite_mean": round(mean, 4), "composite_se": round(se, 4), + # The guarded score is reported alongside so the ladder can adopt it at + # a season boundary of the maintainer's choosing, not on merge day. + "composite_guarded_mean": round(guarded_mean, 4), + "composite_guarded_se": round(guarded_se, 4), + "guards": guard_means, "categories": category_means, "runs": reports, } diff --git a/tests/test_bench.py b/tests/test_bench.py index 4f7969d5..49e9893c 100644 --- a/tests/test_bench.py +++ b/tests/test_bench.py @@ -391,3 +391,198 @@ def test_run_scores_every_derivation_category() -> None: assert entry["n"] == 1 assert entry["mean"] is not None assert 0.0 <= entry["mean"] <= 1.0 + + +# --- composite guards (#616) ---------------------------------------------- + + +def test_the_report_carries_a_bench_version() -> None: + """The whole compatibility story: a guarded score must not be silently + compared to a legacy one.""" + report = run(1, sessions=3) + assert report["bench_version"] == bench.BENCH_VERSION + assert bench.BENCH_VERSION >= 2 + + +def test_composite_keeps_its_original_meaning() -> None: + """`composite` is still the plain mean over category means. + + Every recorded score and ladder entry depends on this. The guards ride + beside it in `composite_guarded`, never folded into it. + """ + report = run(1, sessions=3) + means = [ + entry["mean"] for entry in report["categories"].values() + if entry["mean"] is not None + ] + assert report["composite"] == pytest.approx( + round(sum(means) / len(means), 4), abs=1e-4 + ) + + +def test_guarded_composite_is_the_product_of_the_guards() -> None: + report = run(1, sessions=3) + g = report["guards"] + assert report["composite_guarded"] == pytest.approx( + round(report["composite"] * g["efficiency"] * g["consistency"] + * g["canary"], 4), + abs=1e-4, + ) + + +def test_guards_are_reported_separately_not_folded_in() -> None: + """An opaque composite is worse than no composite.""" + report = run(1, sessions=3) + for key in ("efficiency", "consistency", "canary", "canary_leak_rate"): + assert key in report["guards"] + + +# --- efficiency ----------------------------------------------------------- + + +def test_efficiency_penalises_a_fuller_pack() -> None: + lean = bench.efficiency_multiplier([100, 120], 2000) + fat = bench.efficiency_multiplier([1900, 2000], 2000) + assert lean > fat + + +def test_efficiency_is_bounded_below() -> None: + """A wasteful run is penalised, never zeroed — this is a tiebreak.""" + assert bench.efficiency_multiplier([5000], 2000) == bench.GUARD_MIN_EFFICIENCY + assert bench.efficiency_multiplier([0], 2000) == 1.0 + + +@pytest.mark.parametrize(("used", "budget"), [([], 2000), ([100], 0)]) +def test_efficiency_degrades_to_one_without_data( + used: list[int], budget: int +) -> None: + assert bench.efficiency_multiplier(used, budget) == 1.0 + + +# --- consistency ---------------------------------------------------------- + + +def test_paraphrase_zero_is_the_original() -> None: + assert bench.paraphrase("what is the deploy day?", 0) == ( + "what is the deploy day?" + ) + + +def test_paraphrases_are_deterministic_and_rewrite_the_question() -> None: + """Template-driven: an LLM rewrite would break seed reproducibility.""" + for i in range(1, 4): + first = bench.paraphrase("what is the deploy day?", i) + assert first == bench.paraphrase("what is the deploy day?", i) + assert first != "what is the deploy day?" + assert "what is the deploy day?" in first + + +def test_consistency_counts_agreeing_cases() -> None: + assert bench.consistency_multiplier([True, True]) == 1.0 + assert bench.consistency_multiplier([True, False]) == 0.5 + assert bench.consistency_multiplier([False]) == 0.0 + assert bench.consistency_multiplier([]) == 1.0 + + +# --- canary --------------------------------------------------------------- + + +def test_canary_halves_only_when_tripped() -> None: + assert bench.canary_multiplier(False) == 1.0 + assert bench.canary_multiplier(True) == bench.CANARY_PENALTY + + +def test_every_seed_plants_a_distinct_canary() -> None: + values = {generate(seed).canary for seed in range(1, 6)} + assert all(values) + assert len(values) == 5 + + +def test_the_canary_is_in_the_corpus_but_answers_nothing() -> None: + dataset = generate(3) + corpus = "\n".join(text for _title, text in dataset.sessions) + assert dataset.canary in corpus + for case in dataset.cases: + assert case.expected != dataset.canary + assert dataset.canary not in (case.required or ()) + + +def test_canary_shares_no_vocabulary_with_any_question() -> None: + """Otherwise the guard measures the generator's word choice, not the ranker. + + A first draft read "the retired access code was ..." and tripped on + "what was the project codename before it changed?" — lexical overlap on + code/retired, not a dump. + """ + import re + + dataset = generate(1) + sentence = bench._CANARY_TEMPLATE.format(value=dataset.canary) + bait_words = { + w for w in re.findall(r"[a-z]{4,}", sentence.lower()) + } - {dataset.canary} + for case in dataset.cases: + asked = set(re.findall(r"[a-z]{4,}", case.question.lower())) + assert not (bait_words & asked), ( + f"canary shares {bait_words & asked} with {case.question!r}" + ) + + +def test_canary_leak_rate_accompanies_the_binary_trip() -> None: + """0.5 cannot distinguish one stray pack from half of them; the rate can.""" + report = run(1, sessions=3) + g = report["guards"] + assert 0.0 <= g["canary_leak_rate"] <= 1.0 + assert (g["canary_leak_rate"] > 0) is g["canary_tripped"] + + +# --- run_seeds ------------------------------------------------------------ + + +def test_run_seeds_reports_both_composites_and_the_guards() -> None: + report = run_seeds([1, 2], sessions=3) + assert report["bench_version"] == bench.BENCH_VERSION + assert "composite_mean" in report + assert "composite_guarded_mean" in report + assert set(report["guards"]) == {"efficiency", "consistency", "canary"} + assert report["composite_guarded_mean"] <= report["composite_mean"] + + +def test_run_seeds_tolerates_a_report_without_guards(monkeypatch) -> None: + """A bench_version 1 report has no guard block; the aggregate must not crash. + + This is the compatibility case the version stamp exists for — an old + recorded run still aggregates, and its guards read as neutral. + """ + def legacy_run(seed: int, **kwargs: object) -> dict: + return { + "seed": seed, "composite": 0.5, + "categories": {name: {"mean": 0.5} for name in CATEGORIES}, + } + + monkeypatch.setattr(bench, "run", legacy_run) + report = run_seeds([1, 2]) + assert report["composite_mean"] == 0.5 + assert report["composite_guarded_mean"] == 0.5 + assert report["guards"] == { + "efficiency": 1.0, "consistency": 1.0, "canary": 1.0, + } + + +def test_consistency_drops_when_a_rephrasing_changes_the_answer( + monkeypatch +) -> None: + """The guard's failure path. + + Stock retrieval agrees with its own paraphrases, so this forces a + disagreement: a rewrite that retrieves nothing must cost consistency, + which is the brittleness a category mean cannot see. + """ + def destructive(question: str, index: int) -> str: + return question if index <= 0 else "zzzz nonexistent query zzzz" + + monkeypatch.setattr(bench, "paraphrase", destructive) + report = run(1, sessions=3) + + assert report["guards"]["consistency"] < 1.0 + assert report["composite_guarded"] < report["composite"] From ab422a395c65dc8b202c6506213f805869f078d3 Mon Sep 17 00:00:00 2001 From: minion1227 Date: Fri, 31 Jul 2026 02:21:10 -0700 Subject: [PATCH 2/2] fix(capture): apply coerce_numeric to the numeric config fields #686 added coerce_numeric and routed capture.py's two boolean fields through coerce_bool, but left min_observations and dedup_window_seconds on bare int()/float(). a typo'd value raised out of load_config instead of falling back to the default, which is the exact case the helper's own docstring cites (`min_observations: "three"`), and the resulting unused import tripped ruff F401. surfaced by merging test into this branch: the branch-push workflows on test don't run pytest/mypy/ruff, so the gate never ran on the merge that landed it. Co-authored-by: Cursor --- src/vouch/capture.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/vouch/capture.py b/src/vouch/capture.py index 9aeb98b8..ec99b654 100644 --- a/src/vouch/capture.py +++ b/src/vouch/capture.py @@ -75,9 +75,15 @@ def load_config(store: KBStore) -> CaptureConfig: return CaptureConfig( enabled=coerce_bool(raw.get("enabled", DEFAULT_ENABLED), DEFAULT_ENABLED), realtime=coerce_bool(raw.get("realtime", DEFAULT_REALTIME), DEFAULT_REALTIME), - min_observations=int(raw.get("min_observations", DEFAULT_MIN_OBSERVATIONS)), - dedup_window_seconds=float( - raw.get("dedup_window_seconds", DEFAULT_DEDUP_WINDOW_SECONDS) + min_observations=coerce_numeric( + raw.get("min_observations", DEFAULT_MIN_OBSERVATIONS), + DEFAULT_MIN_OBSERVATIONS, + int, + ), + dedup_window_seconds=coerce_numeric( + raw.get("dedup_window_seconds", DEFAULT_DEDUP_WINDOW_SECONDS), + DEFAULT_DEDUP_WINDOW_SECONDS, + float, ), answer_mode=answer_mode, )