From 74ce320967057d6ff66a495513769a7e3ff1050a Mon Sep 17 00:00:00 2001 From: dripsmvcp <138900956+dripsmvcp@users.noreply.github.com> Date: Fri, 31 Jul 2026 05:25:25 +0900 Subject: [PATCH 1/2] feat(capture): auto-propose a draft claim when the user pushes back MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the adapter captures tool outcomes passively, but the single highest-signal event in a session — the user correcting the agent ("no, we deploy from main not release") — evaporated unless someone remembered to propose a claim afterwards. that is exactly the knowledge worth keeping, and the knowledge most reliably lost. detection is a regex on the turn boundary: a pushback opener that also asserts something. no llm call, so it costs nothing per turn and stays deterministic. deliberately conservative — a false negative loses one correction, a false positive spends a reviewer's attention, and the second is what makes an ambient feature get switched off. the opener itself is stripped, so "no, we deploy from main" is kept as "we deploy from main": the disagreement is context, the assertion is the knowledge. the whole design constraint is that this proposes and never writes. the module routes exclusively through propose_quoted_claim, does not import approve, and a test asserts there is no import of it. the pending queue is the draft state; a human still drains it. the claim carries a receipt rather than a paraphrase: the user's message is registered as a `message` source and the corrective sentence is quoted verbatim out of it, so verify_receipt confirms it by string comparison. three guards bound the heuristic: - a per-session cap (capture.correction.max_per_session, default 3), counted from the pending queue rather than in memory so it holds across a process restart — the unattended case it exists for. - dedup against approved claims and pending corrections. lexical, with the #147 embedding hits folded in on top: that path needs the [embeddings] extra, and dedup that silently stops working on a base install is precisely how an unattended capture floods a queue. - secret masking before anything durable is written. a correction is free-form text typed in a hurry, which is where a pasted credential shows up. wired into the existing UserPromptSubmit hook via maybe_capture, which swallows its own failures — the hook contract is that a broken kb drops the correction rather than breaking the turn. registered as kb.capture_correction across the four sites, and added to the admission gate's auto-capture actors so the same deterministic floor applies to it as to every other passive firehose. closes #430 --- CHANGELOG.md | 20 ++ src/vouch/admission.py | 2 +- src/vouch/capabilities.py | 1 + src/vouch/cli.py | 21 ++ src/vouch/correction.py | 318 +++++++++++++++++++++++++++++++ src/vouch/hooks.py | 10 + src/vouch/hot_memory.py | 3 + src/vouch/jsonl_server.py | 12 ++ src/vouch/server.py | 20 ++ tests/test_capture_correction.py | 277 +++++++++++++++++++++++++++ 10 files changed, 683 insertions(+), 1 deletion(-) create mode 100644 src/vouch/correction.py create mode 100644 tests/test_capture_correction.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f1ef062c..f66aed81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,26 @@ All notable changes to vouch are documented here. Format follows ## [Unreleased] ### Added +- **correction capture — the pushback becomes a proposal** (#430): the adapter + captured tool *outcomes* passively but never the single highest-signal event + in a session, the user correcting the agent ("no, we deploy from `main` not + `release`"). That evaporated unless someone remembered to propose a claim + afterwards. `kb.capture_correction` detects pushback on the turn boundary + with a cheap regex heuristic — no LLM call, deterministic — and files it as a + **pending** claim proposal tagged `auto:correction`, wired into the existing + `UserPromptSubmit` hook so it needs no new plumbing. It proposes and never + writes: the module routes exclusively through `proposals.propose_quoted_claim` + and has no import of `approve` at all. The claim cites a receipt — the user's + message is registered as a `message` source and the corrective sentence is + quoted verbatim out of it — so what reaches the queue is mechanically + verifiable rather than a paraphrase. Three guards bound an over-eager + heuristic: a per-session cap (`capture.correction.max_per_session`, default + 3) counted from the queue so it survives a restart, lexical dedup against + approved claims and pending corrections folded together with the #147 + embedding path, and secret masking before anything durable is written. + `capture.correction.enabled` (default true) gates it; declines report + `{"captured": false, "reason": ...}` rather than failing silently. + `vouch capture-correction`, plus MCP and JSONL. - **explicit pins — a working set that always enters the pack** (#615): `vouch pin ` / `vouch pins list` / `vouch unpin `. Pinned claims and pages lead every context pack instead of having to win the query each turn, diff --git a/src/vouch/admission.py b/src/vouch/admission.py index b7e3abba..4ddfe299 100644 --- a/src/vouch/admission.py +++ b/src/vouch/admission.py @@ -50,7 +50,7 @@ # Passive session-capture actors whose proposals are auto-rejected on a failed # admission check. Deliberate / human / downstream actors are advisory-only. AUTO_CAPTURE_ACTORS: frozenset[str] = frozenset( - {"vouch-capture", "session-split", "codex"} + {"vouch-capture", "session-split", "codex", "auto:correction"} ) # ``session`` / ``log`` pages are raw material, not topics — a mirror of diff --git a/src/vouch/capabilities.py b/src/vouch/capabilities.py index 5fecb9dc..a7c1690a 100644 --- a/src/vouch/capabilities.py +++ b/src/vouch/capabilities.py @@ -79,6 +79,7 @@ "kb.session_transcript", "kb.volunteer_context", "kb.crystallize", + "kb.capture_correction", "kb.summarize_session", "kb.index_rebuild", "kb.lint", diff --git a/src/vouch/cli.py b/src/vouch/cli.py index dfcaeb0f..3c2417d4 100644 --- a/src/vouch/cli.py +++ b/src/vouch/cli.py @@ -33,6 +33,7 @@ from . import codex_rollout as codex_rollout_mod from . import compile as compile_mod from . import contradictions as contradictions_mod +from . import correction as correction_mod from . import digest as digest_mod from . import fetch as fetch_mod from . import hub as hub_mod @@ -2568,6 +2569,26 @@ def notify_test(url: str, secret: str | None) -> None: sys.exit(1) +# --- correction capture --------------------------------------------------- + + +@cli.command(name="capture-correction") +@click.argument("prompt") +@click.option("--session-id", default=None) +@click.option("--context", default=None, help="what the agent had just done") +def capture_correction_cmd( + prompt: str, session_id: str | None, context: str | None +) -> None: + """File a user correction as a pending claim proposal, if it is one.""" + store = _load_store() + with _cli_errors(): + report = correction_mod.capture( + store, prompt=prompt, session_id=session_id, + agent=_whoami(), context=context, + ) + click.echo(json.dumps(report, indent=2)) + + # --- lifecycle ------------------------------------------------------------ diff --git a/src/vouch/correction.py b/src/vouch/correction.py new file mode 100644 index 00000000..87b661e3 --- /dev/null +++ b/src/vouch/correction.py @@ -0,0 +1,318 @@ +"""Correction capture — the highest-signal event in a session (#430). + +The adapter already captures tool *outcomes* passively (`PostToolUse`). The +one thing it never captured is the user pushing back — "no, we deploy from +`main` not `release`" — which is simultaneously the knowledge most worth +keeping and the knowledge most reliably lost, because keeping it required +someone to remember to propose a claim afterwards. + +This turns a detected correction into a **proposal**. Never a write. + +The whole design constraint is in that sentence. This module routes +exclusively through `proposals.propose_quoted_claim`, it has no import of +`proposals.approve`, and the pending queue *is* the draft state — a human +still drains it. Three guards keep an over-eager heuristic from becoming a +reviewer's problem: + +* **a cheap trigger** — regex pushback detection on the turn boundary, no + LLM call, so this costs nothing per turn and stays deterministic. +* **dedup** — a repeated correction does not re-file. Checked against + approved claims and against this session's own pending queue. +* **a per-session cap** — `capture.correction.max_per_session` bounds how + much one run can put in front of a reviewer, counted from the queue itself + so it survives a process restart. + +The claim cites a receipt: the user's message is registered as a `message` +source and the corrective sentence is quoted verbatim out of it, so what +lands in the queue is mechanically verifiable rather than a paraphrase the +reviewer has to take on faith. +""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass +from typing import Any + +import yaml + +from .config_coerce import coerce_bool +from .models import ProposalStatus +from .secrets import mask_secrets +from .storage import KBStore + +_log = logging.getLogger(__name__) + +CORRECTION_ACTOR = "auto:correction" +CORRECTION_TAG = "auto:correction" +CORRECTION_RATIONALE = "captured from user correction" + +DEFAULT_ENABLED = True +DEFAULT_MAX_PER_SESSION = 3 +DEFAULT_MIN_CHARS = 12 +# Above this token overlap with an existing approved claim or a pending +# correction, the correction is treated as already known and dropped. +DEFAULT_DEDUP_THRESHOLD = 0.6 + +# A correction is longer than a token of disagreement but shorter than a +# fresh instruction; past this it is a new task, not a fix to the last one. +MAX_CORRECTION_CHARS = 400 + +# Openers that mark the turn as a correction of what just happened. Anchored +# at the start of the prompt (or of a sentence in it) so "no" inside ordinary +# prose — "there is no config file" — does not trip the heuristic. +_PUSHBACK = re.compile( + r"""^\s*(?: + no[,.\s!]+ | + nope[,.\s!]+ | + wrong[,.\s!]+ | + that'?s\s+(?:not\s+right|wrong|incorrect) | + not\s+(?:quite|right|correct) | + actually[,\s] | + incorrect[,.\s!]+ | + don'?t\s+do\s+that | + i\s+(?:said|meant|told\s+you) + )""", + re.IGNORECASE | re.VERBOSE, +) + +# The correction is only meaningful if it also *asserts* something. A bare +# "no." is disagreement without content and is not worth a reviewer's time. +_ASSERTION = re.compile( + r"\b(?:is|are|was|were|use|uses|should|must|always|never|it'?s|we|the)\b", + re.IGNORECASE, +) + + +# Words too common to say anything about whether two corrections match. +_STOPWORDS = frozenset({ + "a", "an", "and", "are", "as", "at", "be", "but", "by", "do", "dont", + "for", "from", "in", "is", "it", "its", "must", "not", "of", "on", "or", + "should", "that", "the", "this", "to", "use", "was", "we", "were", "you", +}) + +_TOKEN_RE = re.compile(r"[a-z0-9]+") + + +class CorrectionError(RuntimeError): + pass + + +def _tokens(text: str) -> set[str]: + return { + t for t in _TOKEN_RE.findall(text.lower()) + if t not in _STOPWORDS and len(t) > 2 + } + + +def overlap(a: str, b: str) -> float: + """Jaccard overlap of the two texts' significant tokens, 0.0 to 1.0.""" + ta, tb = _tokens(a), _tokens(b) + if not ta or not tb: + return 0.0 + return len(ta & tb) / len(ta | tb) + + +@dataclass(frozen=True) +class CorrectionConfig: + enabled: bool = DEFAULT_ENABLED + max_per_session: int = DEFAULT_MAX_PER_SESSION + min_chars: int = DEFAULT_MIN_CHARS + dedup_threshold: float = DEFAULT_DEDUP_THRESHOLD + + +def load_config(store: KBStore) -> CorrectionConfig: + """Read ``capture.correction:`` from config.yaml; fall back to defaults.""" + try: + loaded = yaml.safe_load(store.config_path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, yaml.YAMLError): + return CorrectionConfig() + if not isinstance(loaded, dict): + return CorrectionConfig() + capture = loaded.get("capture") + if not isinstance(capture, dict): + return CorrectionConfig() + raw = capture.get("correction") + if not isinstance(raw, dict): + return CorrectionConfig() + try: + max_per_session = int(raw.get("max_per_session", DEFAULT_MAX_PER_SESSION)) + except (TypeError, ValueError): + max_per_session = DEFAULT_MAX_PER_SESSION + try: + min_chars = int(raw.get("min_chars", DEFAULT_MIN_CHARS)) + except (TypeError, ValueError): + min_chars = DEFAULT_MIN_CHARS + try: + threshold = float(raw.get("dedup_threshold", DEFAULT_DEDUP_THRESHOLD)) + except (TypeError, ValueError): + threshold = DEFAULT_DEDUP_THRESHOLD + return CorrectionConfig( + enabled=coerce_bool(raw.get("enabled", DEFAULT_ENABLED), DEFAULT_ENABLED), + max_per_session=max(0, max_per_session), + min_chars=max(0, min_chars), + dedup_threshold=threshold, + ) + + +def detect(prompt: str, *, min_chars: int = DEFAULT_MIN_CHARS) -> str | None: + """The corrective statement inside `prompt`, or None if it isn't one. + + Deliberately cheap and deliberately conservative: a false negative costs + one lost correction, a false positive costs a reviewer's attention, and + the second is the one that makes an ambient feature get turned off. + """ + text = (prompt or "").strip() + if not text or len(text) > MAX_CORRECTION_CHARS: + return None + if not _PUSHBACK.match(text): + return None + if len(text) < min_chars: + return None + # Strip the opener itself: "no, we deploy from main" is worth keeping as + # "we deploy from main" — the disagreement is context, not knowledge. + stripped = _PUSHBACK.sub("", text, count=1).strip(" ,.;:!-—") + if not stripped or len(stripped) < min_chars: + return None + if not _ASSERTION.search(stripped): + return None + return stripped + + +def _session_pending(store: KBStore, session_id: str | None) -> list[Any]: + return [ + p for p in store.list_proposals(ProposalStatus.PENDING) + if p.proposed_by == CORRECTION_ACTOR + and (session_id is None or p.session_id == session_id) + ] + + +def _already_known(store: KBStore, text: str, *, threshold: float) -> str | None: + """The id of an approved claim or pending correction that already says this. + + Lexical on purpose, matching the lesson repeat guard: the embedding path + (#147) needs the `[embeddings]` extra, and dedup that silently stops + working on a base install is precisely how an unattended capture floods a + queue. The embedding hits are folded in on top when available. + """ + for claim in store.list_claims(): + if overlap(text, claim.text) >= threshold: + return claim.id + for proposal in _session_pending(store, None): + existing = str(proposal.payload.get("text", "")) + if overlap(text, existing) >= threshold: + return proposal.id + try: + from .embeddings.similarity import find_similar_on_propose + + for warning in find_similar_on_propose(store, text): + artifact_id = warning.get("artifact_id") + if isinstance(artifact_id, str): + return artifact_id + except ImportError: + pass + return None + + +def _skip(reason: str, **extra: Any) -> dict[str, Any]: + return {"captured": False, "reason": reason, **extra} + + +def capture( + store: KBStore, + *, + prompt: str, + session_id: str | None = None, + agent: str | None = None, + context: str | None = None, +) -> dict[str, Any]: + """File a detected correction as a pending claim proposal. + + Returns a small report either way — `{"captured": False, "reason": ...}` + when a guard declined, so a caller (or a test) can see *why* nothing was + filed instead of inferring it from silence. Never raises on a normal + decline; the only errors are a broken KB. + """ + cfg = load_config(store) + if not cfg.enabled: + return _skip("disabled") + + corrective = detect(prompt, min_chars=cfg.min_chars) + if corrective is None: + return _skip("not_a_correction") + + # Mask before anything durable is written. A correction is free-form user + # text typed in a hurry — exactly where a pasted credential shows up. + corrective = mask_secrets(corrective) + + filed = len(_session_pending(store, session_id)) + if filed >= cfg.max_per_session: + return _skip("session_cap", cap=cfg.max_per_session, filed=filed) + + duplicate = _already_known(store, corrective, threshold=cfg.dedup_threshold) + if duplicate is not None: + return _skip("duplicate", duplicate_of=duplicate) + + # The user's own message is the source, and the corrective sentence is + # quoted verbatim out of it, so the proposal carries a receipt the gate + # can verify by string comparison rather than a paraphrase. + source = store.put_source( + corrective.encode("utf-8"), + title="user correction", + source_type="message", + tags=[CORRECTION_TAG], + metadata={ + "session_id": session_id, + "agent": agent, + "context": context, + }, + ) + + # Imported here rather than at module scope: proposals imports lessons, + # and a module-scope import back into proposals would be circular. + from .proposals import propose_quoted_claim + + result = propose_quoted_claim( + store, + text=corrective, + source_id=source.id, + quote=corrective, + proposed_by=CORRECTION_ACTOR, + claim_type="preference", + confidence=0.5, + tags=[CORRECTION_TAG], + rationale=CORRECTION_RATIONALE, + session_id=session_id, + ) + if result is None: # pragma: no cover - quote is the source, always found + return _skip("no_receipt") + return { + "captured": True, + "proposal_id": result.proposal.id, + "status": result.proposal.status.value, + "text": corrective, + "source_id": source.id, + "warnings": result.warnings, + } + + +def maybe_capture( + store: KBStore, + *, + prompt: str, + session_id: str | None = None, + agent: str | None = None, +) -> dict[str, Any] | None: + """`capture` for hook callers: swallows every failure, returns None on one. + + The UserPromptSubmit hook's contract is that it must never break a turn, + so an unwritable KB or a malformed config drops the correction rather + than raising into the host. + """ + try: + report = capture(store, prompt=prompt, session_id=session_id, agent=agent) + except Exception: + _log.warning("correction capture failed", exc_info=True) + return None + return report if report.get("captured") else None diff --git a/src/vouch/hooks.py b/src/vouch/hooks.py index 9b3ea3df..80ec97c9 100644 --- a/src/vouch/hooks.py +++ b/src/vouch/hooks.py @@ -24,6 +24,7 @@ import yaml +from . import correction as correction_mod from . import salience as salience_mod from .config_coerce import coerce_bool from .context import build_context_pack @@ -275,6 +276,15 @@ def build_claude_prompt_hook( except Exception: cfg = {} + # Correction capture (#430): the hook already sees the prompt, and the + # turn boundary is the only place a pushback is visible. Files a PENDING + # proposal for a human to drain — there is no path from here to approve. + # Best-effort by contract: maybe_capture swallows its own failures so a + # broken KB drops the correction instead of breaking the turn. + correction_mod.maybe_capture( + store, prompt=prompt, session_id=session_id, + ) + # Feed the entity-salience reflex (#223) so repeated mentions of an # entity within a session sharpen ranking on subsequent turns -- this # was previously computed but never actually recorded from the hook diff --git a/src/vouch/hot_memory.py b/src/vouch/hot_memory.py index 026db86c..4176c1b0 100644 --- a/src/vouch/hot_memory.py +++ b/src/vouch/hot_memory.py @@ -188,6 +188,9 @@ def mark_volunteered(session_id: str, claim_id: str, *, pushed_at: float) -> Non "kb.session_end": "session control — not a KB read", "kb.volunteer_context": "push channel — already surfaces hot claims", "kb.crystallize": "write path — proposal intake", + "kb.capture_correction": ( + "write path — review gate (files a pending claim proposal)" + ), "kb.index_rebuild": "maintenance — mutates derived index", "kb.lint": "diagnostics — no claim payload", "kb.doctor": "diagnostics — no claim payload", diff --git a/src/vouch/jsonl_server.py b/src/vouch/jsonl_server.py index 6e45e8a3..074f081c 100644 --- a/src/vouch/jsonl_server.py +++ b/src/vouch/jsonl_server.py @@ -30,6 +30,7 @@ from . import audit, bundle, health, volunteer_context from . import compile as compile_mod +from . import correction as correction_mod from . import digest as digest_mod from . import hot_memory as hot_mod from . import lifecycle as life @@ -560,6 +561,16 @@ def _h_propose_delete(p: dict) -> dict: } +def _h_capture_correction(p: dict) -> dict: + return correction_mod.capture( + _store(), + prompt=p["prompt"], + session_id=p.get("session_id"), + agent=_agent(), + context=p.get("context"), + ) + + def _h_approve(p: dict) -> dict: a = approve(_store(), p["proposal_id"], approved_by=_agent(), reason=p.get("reason"), @@ -981,6 +992,7 @@ def _h_propose_theme(p: dict) -> dict: "kb.session_end": _h_session_end, "kb.volunteer_context": _h_volunteer_context, "kb.crystallize": _h_crystallize, + "kb.capture_correction": _h_capture_correction, "kb.index_rebuild": _h_index_rebuild, "kb.lint": _h_lint, "kb.doctor": _h_doctor, diff --git a/src/vouch/server.py b/src/vouch/server.py index 000fa632..94aeac66 100644 --- a/src/vouch/server.py +++ b/src/vouch/server.py @@ -21,6 +21,7 @@ from . import audit, bundle, health, mcp_profiles, volunteer_context from . import compile as compile_mod +from . import correction as correction_mod from . import digest as digest_mod from . import hot_memory as hot_mod from . import lifecycle as life @@ -817,6 +818,25 @@ def kb_propose_delete( return _proposal_response(pr, dry_run) +@mcp.tool() +def kb_capture_correction( + prompt: str, session_id: str | None = None, context: str | None = None +) -> dict[str, Any]: + """Turn a user correction into a PENDING claim proposal. + + Detects pushback ("no, we deploy from main not release") and files it for + review, tagged `auto:correction` so the reviewer sees where it came from. + Proposes only — there is no path from here to approve. Bounded by + `capture.correction.max_per_session` and deduped against what the KB + already knows; returns `{"captured": false, "reason": ...}` when a guard + declines, so a caller can see why nothing was filed. + """ + return correction_mod.capture( + _store(), prompt=prompt, session_id=session_id, + agent=_agent(), context=context, + ) + + def _proposal_response(result, dry_run: bool) -> dict[str, Any]: pr = result.proposal if hasattr(result, "proposal") else result out: dict[str, Any] = { diff --git a/tests/test_capture_correction.py b/tests/test_capture_correction.py new file mode 100644 index 00000000..c938ab01 --- /dev/null +++ b/tests/test_capture_correction.py @@ -0,0 +1,277 @@ +"""Correction capture — propose-only, bounded, deduped (#430). + +The load-bearing invariant is in the module docstring: a detected correction +becomes a *proposal*, never a write. Everything else here is the guards that +keep an ambient heuristic from becoming a reviewer's problem. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from vouch import correction +from vouch.capabilities import capabilities +from vouch.jsonl_server import HANDLERS +from vouch.models import ProposalKind, ProposalStatus +from vouch.storage import KBStore + +CORRECTION = "no, we deploy from main not release" + + +@pytest.fixture +def store(tmp_path: Path) -> KBStore: + s = KBStore.init(tmp_path) + s.config_path.write_text( + "review:\n approver_role: trusted-agent\n", encoding="utf-8", + ) + return s + + +# --- the invariant --------------------------------------------------------- + + +def test_a_correction_lands_as_a_pending_proposal(store: KBStore) -> None: + report = correction.capture(store, prompt=CORRECTION, session_id="s1") + + assert report["captured"] is True + proposal = store.get_proposal(report["proposal_id"]) + assert proposal.status is ProposalStatus.PENDING + assert proposal.kind is ProposalKind.CLAIM + assert proposal.proposed_by == correction.CORRECTION_ACTOR + assert proposal.rationale == correction.CORRECTION_RATIONALE + assert proposal.session_id == "s1" + # nothing durable was written + assert store.list_claims() == [] + + +def test_the_origin_is_visible_in_the_queue(store: KBStore) -> None: + report = correction.capture(store, prompt=CORRECTION) + proposal = store.get_proposal(report["proposal_id"]) + assert correction.CORRECTION_TAG in proposal.payload["tags"] + assert proposal.proposed_by == correction.CORRECTION_ACTOR + + +def test_the_module_has_no_path_to_approve() -> None: + """The whole design constraint. An import of `approve` here would be a + write path that skips the human at the gate.""" + source = Path(correction.__file__).read_text(encoding="utf-8") + assert "propose_quoted_claim" in source + assert not hasattr(correction, "approve") + assert "approve" not in { + line.split()[-1] for line in source.splitlines() + if line.strip().startswith(("import ", "from ")) + } + + +def test_the_claim_carries_a_verifiable_receipt(store: KBStore) -> None: + """The correction is quoted verbatim out of its own source, so the gate + can check it by string comparison instead of trusting a paraphrase.""" + from vouch import receipts + + report = correction.capture(store, prompt=CORRECTION) + proposal = store.get_proposal(report["proposal_id"]) + evidence_id = proposal.payload["evidence"][0] + evidence = store.get_evidence(evidence_id) + assert evidence.source_id == report["source_id"] + result = receipts.verify_receipt( + evidence, store.read_source_content(evidence.source_id) + ) + assert result.status is receipts.ReceiptStatus.VERIFIED + + +# --- detection ------------------------------------------------------------- + + +@pytest.mark.parametrize( + "prompt", + [ + "no, we deploy from main not release", + "Nope, the config lives in etc not var", + "actually, the retry limit should be 5", + "that's wrong — the worker uses redis", + "not quite, we always run mypy first", + "I said the timeout is 30 seconds", + ], +) +def test_pushback_openers_are_detected(prompt: str) -> None: + assert correction.detect(prompt) is not None + + +@pytest.mark.parametrize( + "prompt", + [ + "please add a retry to the worker", + "there is no config file in this repo", # "no" mid-sentence + "no.", # disagreement without content + "nope", + "", + " ", + ], +) +def test_ordinary_prompts_do_not_trip_the_heuristic(prompt: str) -> None: + assert correction.detect(prompt) is None + + +def test_the_opener_is_stripped_from_the_captured_text(store: KBStore) -> None: + """"no, we deploy from main" is worth keeping as "we deploy from main" — + the disagreement is context, the assertion is the knowledge.""" + assert correction.detect(CORRECTION) == "we deploy from main not release" + report = correction.capture(store, prompt=CORRECTION) + assert report["text"] == "we deploy from main not release" + + +def test_a_long_prompt_is_a_new_task_not_a_correction() -> None: + assert correction.detect("no, " + "x y z " * 200) is None + + +def test_a_non_correction_reports_why_nothing_was_filed(store: KBStore) -> None: + report = correction.capture(store, prompt="please add a retry to the worker") + assert report == {"captured": False, "reason": "not_a_correction"} + assert store.list_proposals(ProposalStatus.PENDING) == [] + + +# --- guards ---------------------------------------------------------------- + + +def test_the_per_session_cap_bounds_one_run(store: KBStore) -> None: + store.config_path.write_text( + "review:\n approver_role: trusted-agent\n" + "capture:\n correction:\n max_per_session: 2\n", + encoding="utf-8", + ) + assert correction.capture( + store, prompt="no, we deploy from main", session_id="s1" + )["captured"] + assert correction.capture( + store, prompt="actually the retry limit is five attempts", session_id="s1" + )["captured"] + + third = correction.capture( + store, prompt="nope, the worker queue lives in redis", session_id="s1" + ) + assert third == { + "captured": False, "reason": "session_cap", "cap": 2, "filed": 2, + } + assert len(store.list_proposals(ProposalStatus.PENDING)) == 2 + + # the cap is per session — a different session starts fresh + assert correction.capture( + store, prompt="nope, the worker queue lives in redis", session_id="s2" + )["captured"] + + +def test_the_cap_is_counted_from_the_queue_not_from_memory(store: KBStore) -> None: + """So it survives a process restart — the guard has to hold for an + unattended capture, which is the case it exists for.""" + store.config_path.write_text( + "review:\n approver_role: trusted-agent\n" + "capture:\n correction:\n max_per_session: 1\n", + encoding="utf-8", + ) + correction.capture(store, prompt="no, we deploy from main", session_id="s1") + fresh = KBStore(store.kb_dir.parent) + assert correction.capture( + store=fresh, prompt="actually the retry limit is five", session_id="s1", + )["reason"] == "session_cap" + + +def test_a_repeated_correction_is_suppressed(store: KBStore) -> None: + first = correction.capture(store, prompt=CORRECTION, session_id="s1") + assert first["captured"] is True + + again = correction.capture( + store, prompt="no, we deploy from main, not from release", session_id="s2" + ) + assert again["captured"] is False + assert again["reason"] == "duplicate" + assert again["duplicate_of"] == first["proposal_id"] + + +def test_an_unrelated_correction_still_files(store: KBStore) -> None: + correction.capture(store, prompt=CORRECTION, session_id="s1") + other = correction.capture( + store, prompt="actually the retry limit should be five attempts", + session_id="s1", + ) + assert other["captured"] is True + + +def test_config_can_turn_it_off(store: KBStore) -> None: + store.config_path.write_text( + "review:\n approver_role: trusted-agent\n" + "capture:\n correction:\n enabled: false\n", + encoding="utf-8", + ) + assert correction.load_config(store).enabled is False + assert correction.capture(store, prompt=CORRECTION) == { + "captured": False, "reason": "disabled", + } + assert store.list_proposals(ProposalStatus.PENDING) == [] + + +def test_defaults_when_no_capture_block_is_configured(store: KBStore) -> None: + cfg = correction.load_config(store) + assert cfg.enabled is True + assert cfg.max_per_session == correction.DEFAULT_MAX_PER_SESSION + + +def test_a_quoted_true_does_not_read_as_off(store: KBStore) -> None: + store.config_path.write_text( + "review:\n approver_role: trusted-agent\n" + 'capture:\n correction:\n enabled: "true"\n', + encoding="utf-8", + ) + assert correction.load_config(store).enabled is True + + +def test_a_pasted_credential_is_masked_before_it_is_filed(store: KBStore) -> None: + report = correction.capture( + store, + prompt="no, the token is AKIAIOSFODNN7EXAMPLE and it must be rotated", + ) + if report["captured"]: + assert "AKIAIOSFODNN7EXAMPLE" not in report["text"] + + +# --- hook integration ------------------------------------------------------ + + +def test_maybe_capture_never_raises_into_the_hook(store: KBStore) -> None: + """The UserPromptSubmit contract: a broken KB drops the correction, it + does not break the turn.""" + assert correction.maybe_capture(store, prompt="hello there") is None + assert correction.maybe_capture(store, prompt=CORRECTION) is not None + + broken = KBStore(store.kb_dir.parent) + broken.kb_dir = Path("/nonexistent/vouch") # type: ignore[misc] + assert correction.maybe_capture(broken, prompt=CORRECTION) is None + + +def test_the_prompt_hook_files_a_correction(store: KBStore) -> None: + import json + + from vouch import hooks + + hooks.build_claude_prompt_hook( + store, json.dumps({"prompt": CORRECTION, "session_id": "hook-1"}), + ) + pending = store.list_proposals(ProposalStatus.PENDING) + assert [p.proposed_by for p in pending] == [correction.CORRECTION_ACTOR] + + +# --- registration ---------------------------------------------------------- + + +def test_capture_correction_registered_on_every_surface() -> None: + method = "kb.capture_correction" + assert method in set(capabilities().methods) + assert method in HANDLERS + from vouch.server import mcp + + assert mcp._tool_manager.get_tool("kb_capture_correction") is not None + + from vouch.cli import cli + + assert "capture-correction" in cli.commands From f2fd3ba3f711acf7044f7c693372384d5991d832 Mon Sep 17 00:00:00 2001 From: dripsmvcp <138900956+dripsmvcp@users.noreply.github.com> Date: Fri, 31 Jul 2026 07:41:48 +0900 Subject: [PATCH 2/2] test(capture): cover the correction guards and both dedup paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the diff-coverage gate wants 100% of changed python. what was uncovered was the cli and mcp bodies (registered, never called) and the guards — which are the part of an ambient heuristic worth pinning, since they are what keeps it from becoming a reviewer's problem. added: the cli and mcp surfaces filing a real proposal; overlap with no shared signal; every load_config fallback and the three numeric-typo coercions; pushback that strips to nothing, strips below min_chars, or carries no assertion; dedup against an already-approved claim as well as a pending correction from another session. the embedding half of dedup is exercised both ways on purpose — stubbed in one test so the fold-in runs, and with the import forced to fail in another. ci installs `[dev,web]`, so without the second test the base install's lexical-only path is the one nobody checks, and dedup silently not working is exactly how an unattended capture floods a queue. --- tests/test_capture_correction.py | 168 +++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) diff --git a/tests/test_capture_correction.py b/tests/test_capture_correction.py index c938ab01..1b4ad967 100644 --- a/tests/test_capture_correction.py +++ b/tests/test_capture_correction.py @@ -275,3 +275,171 @@ def test_capture_correction_registered_on_every_surface() -> None: from vouch.cli import cli assert "capture-correction" in cli.commands + + +# --- the surfaces, exercised rather than merely registered ----------------- + + +def test_cli_capture_correction_files_a_proposal( + store: KBStore, monkeypatch: pytest.MonkeyPatch +) -> None: + import json as _json + + from click.testing import CliRunner + + from vouch.cli import cli + + monkeypatch.chdir(store.root) + result = CliRunner().invoke( + cli, + ["capture-correction", CORRECTION, "--session-id", "cli-1", + "--context", "the agent said release"], + ) + assert result.exit_code == 0, result.output + report = _json.loads(result.output) + assert report["captured"] is True + pending = store.list_proposals(ProposalStatus.PENDING) + assert [p.kind for p in pending] == [ProposalKind.CLAIM] + + +def test_mcp_capture_correction_files_a_proposal( + store: KBStore, monkeypatch: pytest.MonkeyPatch +) -> None: + from vouch import server + + monkeypatch.chdir(store.root) + report = server.kb_capture_correction(prompt=CORRECTION, session_id="mcp-1") + assert report["captured"] is True + assert store.list_proposals(ProposalStatus.PENDING) + + +# --- config coercion and the guards ---------------------------------------- + + +def test_overlap_is_zero_when_either_side_has_no_signal() -> None: + # all stopwords / too-short tokens on one side -> no shared signal to score + assert correction.overlap("we do it", "we deploy from main") == 0.0 + assert correction.overlap("we deploy from main", "") == 0.0 + + +def test_config_falls_back_on_an_unreadable_or_odd_document( + store: KBStore +) -> None: + store.config_path.write_text("capture: [unclosed\n", encoding="utf-8") + assert correction.load_config(store) == correction.CorrectionConfig() + store.config_path.write_text("just-a-string\n", encoding="utf-8") + assert correction.load_config(store) == correction.CorrectionConfig() + store.config_path.write_text("capture: not-a-mapping\n", encoding="utf-8") + assert correction.load_config(store) == correction.CorrectionConfig() + store.config_path.write_text( + "capture:\n correction: not-a-mapping\n", encoding="utf-8" + ) + assert correction.load_config(store) == correction.CorrectionConfig() + + +def test_config_typos_coerce_to_defaults(store: KBStore) -> None: + # A config mistake must not take down an ambient capture path. + store.config_path.write_text( + "capture:\n correction:\n" + " max_per_session: many\n" + " min_chars: lots\n" + " dedup_threshold: highish\n", + encoding="utf-8", + ) + cfg = correction.load_config(store) + assert cfg.max_per_session == correction.DEFAULT_MAX_PER_SESSION + assert cfg.min_chars == correction.DEFAULT_MIN_CHARS + assert cfg.dedup_threshold == correction.DEFAULT_DEDUP_THRESHOLD + + +def test_a_correction_that_is_only_its_opener_is_not_knowledge( + store: KBStore +) -> None: + # "no, actually" strips to nothing; "no, it is" strips below min_chars. + for prompt in ("no, actually", "no, wrong"): + report = correction.capture(store, prompt=prompt, session_id="s1", agent="a") + assert report["captured"] is False + assert store.list_proposals(ProposalStatus.PENDING) == [] + + +def test_pushback_without_an_assertion_is_not_knowledge() -> None: + # Disagreement with no claim in it — nothing here belongs in a KB. + assert correction.detect("no, that one over there instead please") is None + + +def test_dedup_catches_an_already_approved_claim(store: KBStore) -> None: + from vouch.proposals import approve, propose_claim + + src = store.put_source(b"we deploy from main not release") + pr = propose_claim( + store, text="we deploy from main not release", evidence=[src.id], + proposed_by="agent-a", + ) + approve(store, pr.id, approved_by="human-b") + report = correction.capture(store, prompt=CORRECTION, session_id="s1") + assert report["captured"] is False + assert report["reason"] == "duplicate" + + +def test_a_short_prompt_is_ignored_before_anything_else(store: KBStore) -> None: + assert correction.capture(store, prompt="no", session_id="s1", agent="a")[ + "captured" + ] is False + + +def test_dedup_catches_a_pending_correction_from_another_session( + store: KBStore +) -> None: + first = correction.capture(store, prompt=CORRECTION, session_id="s1", agent="a") + assert first["captured"] is True + again = correction.capture( + store, prompt="no, we deploy from main not release branch", + session_id="s2", agent="a", + ) + assert again["captured"] is False + assert again["reason"] == "duplicate" + assert len(store.list_proposals(ProposalStatus.PENDING)) == 1 + + +def test_dedup_folds_in_the_embedding_hits_when_the_extra_is_present( + store: KBStore, monkeypatch: pytest.MonkeyPatch +) -> None: + """The embedding path is additive on top of the lexical guard, and is + stubbed here so the branch is exercised with or without the extra + installed — a base install must still dedup, which is why the lexical + pass runs first and this one only adds.""" + import sys + import types + + module = types.ModuleType("vouch.embeddings.similarity") + module.find_similar_on_propose = lambda store, text: [ # type: ignore[attr-defined] + {"artifact_id": None}, # ignored: not a string + {"artifact_id": "semantic-twin"}, + ] + monkeypatch.setitem(sys.modules, "vouch.embeddings.similarity", module) + report = correction.capture( + store, prompt="no, the release train leaves on thursdays now", + session_id="s3", agent="a", + ) + assert report["captured"] is False + assert report["reason"] == "duplicate" + assert report["duplicate_of"] == "semantic-twin" + + +def test_dedup_still_works_on_a_base_install( + store: KBStore, monkeypatch: pytest.MonkeyPatch +) -> None: + """Without the `[embeddings]` extra the import fails and the lexical guard + is all there is — which is the whole reason it runs first. Pinned, because + dedup that silently stops working is how an unattended capture floods a + review queue.""" + import sys + + monkeypatch.setitem(sys.modules, "vouch.embeddings.similarity", None) + assert correction.capture( + store, prompt=CORRECTION, session_id="s1" + )["captured"] is True + again = correction.capture( + store, prompt="no, we deploy from main not release", session_id="s2" + ) + assert again["reason"] == "duplicate"