diff --git a/CHANGELOG.md b/CHANGELOG.md index 05eef387..2b0113b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ All notable changes to vouch are documented here. Format follows ## [Unreleased] ### Added +- **bench: composite guards** (#616): `efficiency`, `consistency` and `canary` + as bounded multipliers over the composite, plus a `bench_version` stamp on + every report. Reported **beside** the composite, never folded into it — + `composite` keeps its exact formula and meaning, so no recorded score or + ladder entry becomes incomparable, and `composite_guarded` is the new + measurement the ladder can adopt at a season boundary of the maintainer's + choosing. `run_seeds` tolerates a version-1 report with no guard block. + Measured on seeds 1-3: efficiency 0.88, consistency 1.00, canary **0.50 — + tripped on every seed** (leak rate 0.08-0.24), because a 10-item pack over + this corpus carries ~24% of it. That is the lever the guard exists to expose. - **pdf and audio sources — page and timestamp receipts** (#613): a spec, a paper, a recorded call could not become citable material, because a receipt is a byte span into a source's stored bytes and the bytes of a pdf or an mp3 do diff --git a/src/vouch/bench.py b/src/vouch/bench.py index 6cfa90c3..c3b7d97c 100644 --- a/src/vouch/bench.py +++ b/src/vouch/bench.py @@ -87,6 +87,26 @@ carried the parts. multi-hop-relational is where the derivation axes actually bite today. +Composite guards (#616), measured over seeds 1,2,3 @ 2026-07-30. Reported +beside the composite, never folded into it, and stamped with ``bench_version`` +so a guarded score is never silently compared to a recorded legacy one: + +====================== ===================================================== +composite 0.64 — unchanged formula, unchanged meaning +composite_guarded 0.28 — composite x efficiency x consistency x canary +efficiency 0.88 — packs run ~12% of budget below the cap +consistency 1.00 — prefix-form paraphrases do not move retrieval, + so this is a floor on robustness, not proof + of it; a content-rewriting paraphrase would + test harder but risks changing the question +canary 0.50 — TRIPPED on every seed, leak rate 0.08-0.24. + the planted line reaches a pack because a + 10-item pack over this corpus carries ~24% of + it. this is the lever the guard exists to + expose, and the reason the guarded composite + is not yet the ladder's number +====================== ===================================================== + For calibration only (different benchmarks, not directly comparable): ditto's stock production-mirroring harness reports memory_mean 0.200-0.226 on its own v5/v6 contracts, with 0.00 on its consolidation/multi-hop @@ -146,6 +166,44 @@ # somewhere. See the note at the top of their block in ``generate``. _DERIVED_SALT = 0x9E3779B9 +# Bench report version. "1" is every score recorded before the composite +# guards existed. The stamp is the whole compatibility story: a guarded score +# and a legacy one are different measurements, and a version on the report is +# what stops them being compared by accident. +BENCH_VERSION = 2 + +# Guards (#616). Bounded multipliers over the composite, never categories: a +# guard averaged in with everything else stops being a guard — canary at zero +# should be alarming, not a ten-percent dent. +GUARD_MIN_EFFICIENCY = 0.5 # a maximally wasteful pack still scores half +CANARY_PENALTY = 0.5 # echoing planted bait halves the composite +PARAPHRASE_COUNT = 2 # the original plus one metamorphic rephrasing + +# Semantically identical rewrites. Template-generated, never LLM-written, so a +# run stays reproducible from the seed alone. +_PARAPHRASE_TEMPLATES = ( + "quick one — {q}", + "remind me: {q}", + "sorry, asking again: {q}", +) + +# The canary is a coined value planted in the corpus that answers nothing. Any +# pack that surfaces it is dragging in content no question asked for. +# +# The wording matters as much as the value: the sentence must share no +# vocabulary with any generated question, or the guard measures the +# generator's word choice instead of the ranker. A first draft used "the +# retired access code was ..." and tripped on "what was the project codename +# before it changed?" — lexical overlap on code/retired, not a dump. +# test_canary_shares_no_vocabulary_with_any_question keeps it honest. +_CANARY_TEMPLATE = "{value} surfaced in a thread nobody followed up on." + +# Graded against the store's receipts and lifecycle rather than the pack text, +# so a rephrased question cannot change their score. +_STORE_GRADED_CATEGORIES = frozenset( + ("citation-correctness", "receipt-coverage", "supersede-hygiene") +) + # A superseded, archived, or redacted claim is not a live memory (mirrors # the set the context pack excludes). _RETIRED_STATUSES = frozenset( @@ -333,6 +391,8 @@ class Dataset: seed: int sessions: tuple[tuple[str, str], ...] # (title, text) cases: tuple[MemoryCase, ...] + # Planted bait no question asks for; surfacing it trips the canary guard. + canary: str = "" @dataclass @@ -573,6 +633,12 @@ def coin_distinct(n: int, syllables: int = 3) -> list[str]: "aggregation", sub.choice(set_asks), None, required=tuple(set_items), )) + # The canary: a coined value that answers no question in the dataset. It + # sits on the derived rng like the derivation categories, so categories + # 1-10 stay byte-identical. + canary = coin_distinct(1)[0] + docs.add(sub_spread(1)[0], _CANARY_TEMPLATE.format(value=canary)) + # Filler prose in every session, shuffled placement. for idx in range(sessions): for _ in range(rng.randrange(2, 5)): @@ -583,7 +649,9 @@ def coin_distinct(n: int, syllables: int = 3) -> list[str]: (f"bench session {i + 1}", "\n".join(lines)) for i, lines in enumerate(docs.buckets) ) - return Dataset(seed=seed, sessions=session_docs, cases=tuple(cases)) + return Dataset( + seed=seed, sessions=session_docs, cases=tuple(cases), canary=canary, + ) def grade_case(case: MemoryCase, pack_text: str) -> tuple[float, str | None]: @@ -734,6 +802,51 @@ def _pack_text(pack: dict[str, Any]) -> str: return text.replace("«", "").replace("»", "") +def paraphrase(question: str, index: int) -> str: + """A semantically identical rewrite of ``question``. + + Index 0 is the original. Deterministic and template-driven — an LLM + rewrite would make the metamorphic check unreproducible from the seed, + which is the one property the whole benchmark rests on. + """ + if index <= 0: + return question + template = _PARAPHRASE_TEMPLATES[(index - 1) % len(_PARAPHRASE_TEMPLATES)] + return template.format(q=question) + + +def efficiency_multiplier(used_chars: list[int], budget_chars: int) -> float: + """Bounded penalty on how much budget the answers consumed. + + A strategy that hits the answer by dragging in half the KB is right by + luck, not by ranking; this is what makes the dump-guard cost something + even on categories with no forbidden value. Bounded below so a wasteful + run is penalised, never zeroed — efficiency is a tiebreak, not a verdict. + """ + if not used_chars or budget_chars <= 0: + return 1.0 + mean_used = statistics.mean(used_chars) + share = min(1.0, mean_used / budget_chars) + return round(GUARD_MIN_EFFICIENCY + (1.0 - GUARD_MIN_EFFICIENCY) * (1.0 - share), 4) + + +def consistency_multiplier(agreements: list[bool]) -> float: + """Fraction of cases whose paraphrases all graded the same. + + Metamorphic: a question asked two ways is the same question. Scoring well + on one phrasing and badly on its rewrite is brittleness, and a category + mean cannot see it. + """ + if not agreements: + return 1.0 + return round(sum(1 for a in agreements if a) / len(agreements), 4) + + +def canary_multiplier(tripped: bool) -> float: + """Halve the composite when planted bait reached a pack.""" + return CANARY_PENALTY if tripped else 1.0 + + def run( seed: int, *, @@ -791,12 +904,21 @@ def run( per_category: dict[str, list[float]] = {c: [] for c in CATEGORIES} failures: list[dict[str, Any]] = [] + used_chars: list[int] = [] + agreements: list[bool] = [] + canary_hits = 0 + packs_seen = 0 for case in dataset.cases: pack = build_context_pack( store, query=case.question, limit=limit, max_chars=budget_chars, strategy=strategy, ) pack_dict = dict(pack) + text = _pack_text(pack_dict) + used_chars.append(len(text)) + packs_seen += 1 + if dataset.canary and dataset.canary in text: + canary_hits += 1 if case.category == "citation-correctness": score, reason = grade_citation_correctness(store, case, pack_dict) elif case.category == "receipt-coverage": @@ -806,6 +928,29 @@ def run( else: score, reason = grade_case(case, _pack_text(pack_dict)) per_category[case.category].append(score) + + # Metamorphic check: the same question, asked another way, must + # grade the same. Only the pack-text categories are re-asked — + # the three verifiability graders read the store, not the query, + # so rephrasing cannot move them. + if case.category not in _STORE_GRADED_CATEGORIES: + agreed = True + for i in range(1, PARAPHRASE_COUNT): + alt = build_context_pack( + store, query=paraphrase(case.question, i), limit=limit, + max_chars=budget_chars, strategy=strategy, + ) + alt_dict = dict(alt) + alt_text = _pack_text(alt_dict) + used_chars.append(len(alt_text)) + packs_seen += 1 + if dataset.canary and dataset.canary in alt_text: + canary_hits += 1 + alt_score, _ = grade_case(case, alt_text) + if alt_score != score: + agreed = False + agreements.append(agreed) + if reason is not None: failures.append({ "category": case.category, @@ -823,7 +968,33 @@ def run( } means = [statistics.mean(s) for s in per_category.values() if s] composite = round(statistics.mean(means), 4) if means else 0.0 + + # The guards are reported beside the composite, never folded into it. An + # opaque composite is worse than no composite, and — the compatibility + # point — `composite` keeps meaning exactly what every recorded score and + # ladder entry already means. `composite_guarded` is the new measurement, + # and `bench_version` is what stops the two being compared by accident. + # The multiplier is binary per the spec — bait in the pack is bait in the + # pack. The leak *rate* is reported beside it because "one stray pack out + # of twenty-six" and "half of them" are very different engineering + # problems, and a 0.5 that cannot tell them apart is not actionable. + canary_tripped = canary_hits > 0 + guards = { + "efficiency": efficiency_multiplier(used_chars, budget_chars), + "consistency": consistency_multiplier(agreements), + "canary": canary_multiplier(canary_tripped), + "canary_tripped": canary_tripped, + "canary_leak_rate": ( + round(canary_hits / packs_seen, 4) if packs_seen else 0.0 + ), + } + guarded = round( + composite + * guards["efficiency"] * guards["consistency"] * guards["canary"], + 4, + ) return { + "bench_version": BENCH_VERSION, "seed": seed, "budget_chars": budget_chars, "limit": limit, @@ -831,6 +1002,8 @@ def run( "cases": len(dataset.cases), "categories": categories, "composite": composite, + "guards": guards, + "composite_guarded": guarded, "failures": failures, } @@ -874,11 +1047,36 @@ def run_seeds( ] if vals: category_means[name] = round(statistics.mean(vals), 4) + # Degrade rather than crash on a report without guards: a bench_version 1 + # run (or a cached one) has no guard block, and the aggregate should still + # produce the legacy numbers. Missing guards read as neutral 1.0. + guarded = [ + r.get("composite_guarded", r["composite"]) for r in reports + ] + guarded_mean = statistics.mean(guarded) + guarded_se = ( + statistics.stdev(guarded) / (len(guarded) ** 0.5) + if len(guarded) > 1 else 0.0 + ) + guard_means = { + name: round( + statistics.mean( + [float(r.get("guards", {}).get(name, 1.0)) for r in reports] + ), 4, + ) + for name in ("efficiency", "consistency", "canary") + } return { + "bench_version": BENCH_VERSION, "seeds": seeds, "budget_chars": budget_chars, "composite_mean": round(mean, 4), "composite_se": round(se, 4), + # The guarded score is reported alongside so the ladder can adopt it at + # a season boundary of the maintainer's choosing, not on merge day. + "composite_guarded_mean": round(guarded_mean, 4), + "composite_guarded_se": round(guarded_se, 4), + "guards": guard_means, "categories": category_means, "runs": reports, } diff --git a/tests/test_bench.py b/tests/test_bench.py index 4f7969d5..49e9893c 100644 --- a/tests/test_bench.py +++ b/tests/test_bench.py @@ -391,3 +391,198 @@ def test_run_scores_every_derivation_category() -> None: assert entry["n"] == 1 assert entry["mean"] is not None assert 0.0 <= entry["mean"] <= 1.0 + + +# --- composite guards (#616) ---------------------------------------------- + + +def test_the_report_carries_a_bench_version() -> None: + """The whole compatibility story: a guarded score must not be silently + compared to a legacy one.""" + report = run(1, sessions=3) + assert report["bench_version"] == bench.BENCH_VERSION + assert bench.BENCH_VERSION >= 2 + + +def test_composite_keeps_its_original_meaning() -> None: + """`composite` is still the plain mean over category means. + + Every recorded score and ladder entry depends on this. The guards ride + beside it in `composite_guarded`, never folded into it. + """ + report = run(1, sessions=3) + means = [ + entry["mean"] for entry in report["categories"].values() + if entry["mean"] is not None + ] + assert report["composite"] == pytest.approx( + round(sum(means) / len(means), 4), abs=1e-4 + ) + + +def test_guarded_composite_is_the_product_of_the_guards() -> None: + report = run(1, sessions=3) + g = report["guards"] + assert report["composite_guarded"] == pytest.approx( + round(report["composite"] * g["efficiency"] * g["consistency"] + * g["canary"], 4), + abs=1e-4, + ) + + +def test_guards_are_reported_separately_not_folded_in() -> None: + """An opaque composite is worse than no composite.""" + report = run(1, sessions=3) + for key in ("efficiency", "consistency", "canary", "canary_leak_rate"): + assert key in report["guards"] + + +# --- efficiency ----------------------------------------------------------- + + +def test_efficiency_penalises_a_fuller_pack() -> None: + lean = bench.efficiency_multiplier([100, 120], 2000) + fat = bench.efficiency_multiplier([1900, 2000], 2000) + assert lean > fat + + +def test_efficiency_is_bounded_below() -> None: + """A wasteful run is penalised, never zeroed — this is a tiebreak.""" + assert bench.efficiency_multiplier([5000], 2000) == bench.GUARD_MIN_EFFICIENCY + assert bench.efficiency_multiplier([0], 2000) == 1.0 + + +@pytest.mark.parametrize(("used", "budget"), [([], 2000), ([100], 0)]) +def test_efficiency_degrades_to_one_without_data( + used: list[int], budget: int +) -> None: + assert bench.efficiency_multiplier(used, budget) == 1.0 + + +# --- consistency ---------------------------------------------------------- + + +def test_paraphrase_zero_is_the_original() -> None: + assert bench.paraphrase("what is the deploy day?", 0) == ( + "what is the deploy day?" + ) + + +def test_paraphrases_are_deterministic_and_rewrite_the_question() -> None: + """Template-driven: an LLM rewrite would break seed reproducibility.""" + for i in range(1, 4): + first = bench.paraphrase("what is the deploy day?", i) + assert first == bench.paraphrase("what is the deploy day?", i) + assert first != "what is the deploy day?" + assert "what is the deploy day?" in first + + +def test_consistency_counts_agreeing_cases() -> None: + assert bench.consistency_multiplier([True, True]) == 1.0 + assert bench.consistency_multiplier([True, False]) == 0.5 + assert bench.consistency_multiplier([False]) == 0.0 + assert bench.consistency_multiplier([]) == 1.0 + + +# --- canary --------------------------------------------------------------- + + +def test_canary_halves_only_when_tripped() -> None: + assert bench.canary_multiplier(False) == 1.0 + assert bench.canary_multiplier(True) == bench.CANARY_PENALTY + + +def test_every_seed_plants_a_distinct_canary() -> None: + values = {generate(seed).canary for seed in range(1, 6)} + assert all(values) + assert len(values) == 5 + + +def test_the_canary_is_in_the_corpus_but_answers_nothing() -> None: + dataset = generate(3) + corpus = "\n".join(text for _title, text in dataset.sessions) + assert dataset.canary in corpus + for case in dataset.cases: + assert case.expected != dataset.canary + assert dataset.canary not in (case.required or ()) + + +def test_canary_shares_no_vocabulary_with_any_question() -> None: + """Otherwise the guard measures the generator's word choice, not the ranker. + + A first draft read "the retired access code was ..." and tripped on + "what was the project codename before it changed?" — lexical overlap on + code/retired, not a dump. + """ + import re + + dataset = generate(1) + sentence = bench._CANARY_TEMPLATE.format(value=dataset.canary) + bait_words = { + w for w in re.findall(r"[a-z]{4,}", sentence.lower()) + } - {dataset.canary} + for case in dataset.cases: + asked = set(re.findall(r"[a-z]{4,}", case.question.lower())) + assert not (bait_words & asked), ( + f"canary shares {bait_words & asked} with {case.question!r}" + ) + + +def test_canary_leak_rate_accompanies_the_binary_trip() -> None: + """0.5 cannot distinguish one stray pack from half of them; the rate can.""" + report = run(1, sessions=3) + g = report["guards"] + assert 0.0 <= g["canary_leak_rate"] <= 1.0 + assert (g["canary_leak_rate"] > 0) is g["canary_tripped"] + + +# --- run_seeds ------------------------------------------------------------ + + +def test_run_seeds_reports_both_composites_and_the_guards() -> None: + report = run_seeds([1, 2], sessions=3) + assert report["bench_version"] == bench.BENCH_VERSION + assert "composite_mean" in report + assert "composite_guarded_mean" in report + assert set(report["guards"]) == {"efficiency", "consistency", "canary"} + assert report["composite_guarded_mean"] <= report["composite_mean"] + + +def test_run_seeds_tolerates_a_report_without_guards(monkeypatch) -> None: + """A bench_version 1 report has no guard block; the aggregate must not crash. + + This is the compatibility case the version stamp exists for — an old + recorded run still aggregates, and its guards read as neutral. + """ + def legacy_run(seed: int, **kwargs: object) -> dict: + return { + "seed": seed, "composite": 0.5, + "categories": {name: {"mean": 0.5} for name in CATEGORIES}, + } + + monkeypatch.setattr(bench, "run", legacy_run) + report = run_seeds([1, 2]) + assert report["composite_mean"] == 0.5 + assert report["composite_guarded_mean"] == 0.5 + assert report["guards"] == { + "efficiency": 1.0, "consistency": 1.0, "canary": 1.0, + } + + +def test_consistency_drops_when_a_rephrasing_changes_the_answer( + monkeypatch +) -> None: + """The guard's failure path. + + Stock retrieval agrees with its own paraphrases, so this forces a + disagreement: a rewrite that retrieves nothing must cost consistency, + which is the brittleness a category mean cannot see. + """ + def destructive(question: str, index: int) -> str: + return question if index <= 0 else "zzzz nonexistent query zzzz" + + monkeypatch.setattr(bench, "paraphrase", destructive) + report = run(1, sessions=3) + + assert report["guards"]["consistency"] < 1.0 + assert report["composite_guarded"] < report["composite"]