diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index 42ebbca4..80f9522d 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -118,7 +118,7 @@ Make failure useful. | F7 | Add calibration link from negative-result entries to intake reports (complete). — evidence/negative_result_calibration_link.py: links NRR- rejection records to calibration intake reports for closed-loop learning; tests/evidence/test_negative_result_calibration_link.py. | Closes learning loop. | C | | F8 | Add benchmark for whether rejected candidates resemble known failure modes (complete). — FMS- schema: 14 fields, 11 validation rules, PATTERN_REPEATED_THRESHOLD=0.80, pattern_repeated_flag enforcement, calibration_action_recommended; 63 tests. | Improves rejection logic. | C | | F9 | Add negative-result dashboard schema (complete). | NRD- aggregates NRR- rejection statistics: rejection rate consistency check, all_rejections_have_nrr enforced, top stage/reason controlled vocabulary, 100% rejection warning. | B | -| F10 | Add policy that public claims must mention relevant negative results. | Prevents cherry-picking. | D | +| F10 | Add policy that public claims must mention relevant negative results. | Prevents cherry-picking. | D | DONE | ## Phase G — Calibration and active-learning rigor diff --git a/src/openamp_foundry/evidence/negative_result_citation_policy.py b/src/openamp_foundry/evidence/negative_result_citation_policy.py new file mode 100644 index 00000000..9354eba7 --- /dev/null +++ b/src/openamp_foundry/evidence/negative_result_citation_policy.py @@ -0,0 +1,153 @@ +"""NRC- negative-result citation policy schema. + +Enforces that any public positive claim references all known relevant negative +results. Cherry-picking — highlighting successes while silently omitting +failures — is the most common way an AI pipeline overstates its case. + +Every public claim about pipeline outputs must be paired with an NRC- record +that lists relevant NRR- records and confirms they are cited. A claim with +uncited negatives is machine-flagged as non_compliant. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +VALID_NRC_VERDICTS: frozenset[str] = frozenset({ + "compliant", + "non_compliant", + "no_relevant_negatives", +}) + + +@dataclass +class NegativeResultCitationPolicy: + nrc_id: str + claim_id: str + pipeline_version: str + relevant_nrr_ids: list[str] + cited_nrr_ids: list[str] + uncited_nrr_ids: list[str] + uncited_count: int + all_relevant_cited: bool + policy_verdict: str + dry_lab_only: bool + limitations: list[str] + created_at: str + + +def validate_negative_result_citation_policy(nrc: NegativeResultCitationPolicy) -> None: + if not nrc.nrc_id.startswith("NRC-"): + raise ValueError(f"nrc_id must start with 'NRC-': {nrc.nrc_id!r}") + if not nrc.claim_id: + raise ValueError("claim_id must be non-empty") + if not nrc.pipeline_version: + raise ValueError("pipeline_version must be non-empty") + for nrr_id in nrc.relevant_nrr_ids: + if not nrr_id.startswith("NRR-"): + raise ValueError(f"relevant_nrr_id must start with 'NRR-': {nrr_id!r}") + for nrr_id in nrc.cited_nrr_ids: + if not nrr_id.startswith("NRR-"): + raise ValueError(f"cited_nrr_id must start with 'NRR-': {nrr_id!r}") + relevant_set = set(nrc.relevant_nrr_ids) + for nrr_id in nrc.cited_nrr_ids: + if nrr_id not in relevant_set: + raise ValueError( + f"cited_nrr_id {nrr_id!r} not in relevant_nrr_ids" + ) + expected_uncited = [r for r in nrc.relevant_nrr_ids if r not in set(nrc.cited_nrr_ids)] + if sorted(nrc.uncited_nrr_ids) != sorted(expected_uncited): + raise ValueError("uncited_nrr_ids inconsistent with relevant_nrr_ids and cited_nrr_ids") + if nrc.uncited_count != len(nrc.uncited_nrr_ids): + raise ValueError("uncited_count must equal len(uncited_nrr_ids)") + expected_all_cited = nrc.uncited_count == 0 + if nrc.all_relevant_cited != expected_all_cited: + raise ValueError("all_relevant_cited inconsistent with uncited_count") + if nrc.policy_verdict not in VALID_NRC_VERDICTS: + raise ValueError( + f"policy_verdict {nrc.policy_verdict!r} not in VALID_NRC_VERDICTS" + ) + if not nrc.relevant_nrr_ids and nrc.policy_verdict != "no_relevant_negatives": + raise ValueError( + "policy_verdict must be 'no_relevant_negatives' when relevant_nrr_ids is empty" + ) + if nrc.relevant_nrr_ids and nrc.all_relevant_cited and nrc.policy_verdict != "compliant": + raise ValueError( + "policy_verdict must be 'compliant' when all relevant negatives are cited" + ) + if nrc.relevant_nrr_ids and not nrc.all_relevant_cited and nrc.policy_verdict != "non_compliant": + raise ValueError( + "policy_verdict must be 'non_compliant' when relevant negatives are uncited" + ) + if not nrc.dry_lab_only: + raise ValueError("dry_lab_only must be True") + if not nrc.limitations: + raise ValueError("limitations must be non-empty") + if not nrc.created_at: + raise ValueError("created_at must be non-empty") + + +def _compute_verdict(relevant: list[str], cited_set: set[str]) -> str: + if not relevant: + return "no_relevant_negatives" + if all(r in cited_set for r in relevant): + return "compliant" + return "non_compliant" + + +def build_negative_result_citation_policy( + *, + nrc_id: str, + claim_id: str, + pipeline_version: str, + relevant_nrr_ids: list[str], + cited_nrr_ids: list[str], + limitations: list[str], + created_at: str, +) -> NegativeResultCitationPolicy: + """Build a NegativeResultCitationPolicy. + + uncited_nrr_ids, uncited_count, all_relevant_cited, and policy_verdict + are all auto-computed from relevant_nrr_ids and cited_nrr_ids. + """ + relevant = list(relevant_nrr_ids) + cited = list(cited_nrr_ids) + cited_set = set(cited) + uncited = [r for r in relevant if r not in cited_set] + uncited_count = len(uncited) + all_cited = uncited_count == 0 + verdict = _compute_verdict(relevant, cited_set) + nrc = NegativeResultCitationPolicy( + nrc_id=nrc_id, + claim_id=claim_id, + pipeline_version=pipeline_version, + relevant_nrr_ids=relevant, + cited_nrr_ids=cited, + uncited_nrr_ids=uncited, + uncited_count=uncited_count, + all_relevant_cited=all_cited, + policy_verdict=verdict, + dry_lab_only=True, + limitations=limitations, + created_at=created_at, + ) + validate_negative_result_citation_policy(nrc) + return nrc + + +def format_negative_result_citation_policy(nrc: NegativeResultCitationPolicy) -> str: + lines = [ + f"Negative-Result Citation Policy — {nrc.nrc_id}", + f"Claim: {nrc.claim_id} | Pipeline: {nrc.pipeline_version}", + f"Verdict: {nrc.policy_verdict}", + f"Relevant negatives: {len(nrc.relevant_nrr_ids)} " + f"Cited: {len(nrc.cited_nrr_ids)} " + f"Uncited: {nrc.uncited_count}", + ] + if nrc.uncited_nrr_ids: + lines.append(f"Uncited negatives: {', '.join(nrc.uncited_nrr_ids)}") + lines.append(f"All relevant cited: {nrc.all_relevant_cited}") + lines.append(f"Limitations: {'; '.join(nrc.limitations)}") + lines.append(f"Created: {nrc.created_at}") + lines.append(f"dry_lab_only: {nrc.dry_lab_only}") + return "\n".join(lines) diff --git a/tests/evidence/test_negative_result_citation_policy.py b/tests/evidence/test_negative_result_citation_policy.py new file mode 100644 index 00000000..d085c863 --- /dev/null +++ b/tests/evidence/test_negative_result_citation_policy.py @@ -0,0 +1,314 @@ +"""Tests for NRC- negative-result citation policy schema.""" + +import pytest +from openamp_foundry.evidence.negative_result_citation_policy import ( + NegativeResultCitationPolicy, + VALID_NRC_VERDICTS, + build_negative_result_citation_policy, + format_negative_result_citation_policy, + validate_negative_result_citation_policy, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _build(**kwargs): + defaults = dict( + nrc_id="NRC-001", + claim_id="CLM-001", + pipeline_version="v1.0", + relevant_nrr_ids=["NRR-001", "NRR-002"], + cited_nrr_ids=["NRR-001", "NRR-002"], + limitations=["dry-lab only"], + created_at="2026-07-10", + ) + defaults.update(kwargs) + return build_negative_result_citation_policy(**defaults) + + +# --------------------------------------------------------------------------- +# 1. Constants +# --------------------------------------------------------------------------- + + +def test_valid_nrc_verdicts_is_frozenset(): + assert isinstance(VALID_NRC_VERDICTS, frozenset) + + +def test_valid_nrc_verdicts_contains_compliant(): + assert "compliant" in VALID_NRC_VERDICTS + + +def test_valid_nrc_verdicts_contains_non_compliant(): + assert "non_compliant" in VALID_NRC_VERDICTS + + +def test_valid_nrc_verdicts_contains_no_relevant_negatives(): + assert "no_relevant_negatives" in VALID_NRC_VERDICTS + + +def test_valid_nrc_verdicts_has_three_values(): + assert len(VALID_NRC_VERDICTS) == 3 + + +# --------------------------------------------------------------------------- +# 2. build happy paths +# --------------------------------------------------------------------------- + + +def test_build_returns_negative_result_citation_policy(): + assert isinstance(_build(), NegativeResultCitationPolicy) + + +def test_build_nrc_id_stored(): + assert _build().nrc_id == "NRC-001" + + +def test_build_claim_id_stored(): + assert _build().claim_id == "CLM-001" + + +def test_build_pipeline_version_stored(): + assert _build().pipeline_version == "v1.0" + + +def test_build_dry_lab_only_true(): + assert _build().dry_lab_only is True + + +def test_build_all_cited_gives_compliant(): + r = _build( + relevant_nrr_ids=["NRR-001", "NRR-002"], + cited_nrr_ids=["NRR-001", "NRR-002"], + ) + assert r.policy_verdict == "compliant" + + +def test_build_no_relevant_gives_no_relevant_negatives(): + r = _build(relevant_nrr_ids=[], cited_nrr_ids=[]) + assert r.policy_verdict == "no_relevant_negatives" + + +def test_build_uncited_gives_non_compliant(): + r = _build( + relevant_nrr_ids=["NRR-001", "NRR-002"], + cited_nrr_ids=["NRR-001"], + ) + assert r.policy_verdict == "non_compliant" + + +def test_build_all_relevant_cited_true_when_compliant(): + assert _build().all_relevant_cited is True + + +def test_build_all_relevant_cited_false_when_non_compliant(): + r = _build( + relevant_nrr_ids=["NRR-001", "NRR-002"], + cited_nrr_ids=["NRR-001"], + ) + assert r.all_relevant_cited is False + + +def test_build_uncited_count_zero_when_compliant(): + assert _build().uncited_count == 0 + + +def test_build_uncited_count_correct_when_non_compliant(): + r = _build( + relevant_nrr_ids=["NRR-001", "NRR-002"], + cited_nrr_ids=["NRR-001"], + ) + assert r.uncited_count == 1 + + +def test_build_uncited_nrr_ids_correct(): + r = _build( + relevant_nrr_ids=["NRR-001", "NRR-002"], + cited_nrr_ids=["NRR-001"], + ) + assert r.uncited_nrr_ids == ["NRR-002"] + + +def test_build_uncited_nrr_ids_empty_when_compliant(): + assert _build().uncited_nrr_ids == [] + + +def test_build_no_relevant_uncited_count_zero(): + r = _build(relevant_nrr_ids=[], cited_nrr_ids=[]) + assert r.uncited_count == 0 + + +def test_build_relevant_nrr_ids_stored(): + r = _build(relevant_nrr_ids=["NRR-001"], cited_nrr_ids=["NRR-001"]) + assert "NRR-001" in r.relevant_nrr_ids + + +def test_build_cited_nrr_ids_stored(): + r = _build(relevant_nrr_ids=["NRR-001"], cited_nrr_ids=["NRR-001"]) + assert "NRR-001" in r.cited_nrr_ids + + +def test_build_limitations_stored(): + assert _build().limitations == ["dry-lab only"] + + +def test_build_created_at_stored(): + assert _build().created_at == "2026-07-10" + + +def test_build_single_relevant_single_cited_compliant(): + r = _build(relevant_nrr_ids=["NRR-010"], cited_nrr_ids=["NRR-010"]) + assert r.policy_verdict == "compliant" + + +def test_build_many_relevant_none_cited_non_compliant(): + relevant = [f"NRR-{i:03d}" for i in range(1, 6)] + r = _build(relevant_nrr_ids=relevant, cited_nrr_ids=[]) + assert r.policy_verdict == "non_compliant" + assert r.uncited_count == 5 + + +def test_build_many_relevant_some_cited_non_compliant(): + relevant = ["NRR-001", "NRR-002", "NRR-003"] + r = _build(relevant_nrr_ids=relevant, cited_nrr_ids=["NRR-001"]) + assert r.uncited_count == 2 + + +# --------------------------------------------------------------------------- +# 3. validate rejection cases +# --------------------------------------------------------------------------- + + +def test_validate_rejects_bad_nrc_id_prefix(): + with pytest.raises(ValueError, match="NRC-"): + _build(nrc_id="BAD-001") + + +def test_validate_rejects_empty_claim_id(): + with pytest.raises(ValueError, match="claim_id"): + _build(claim_id="") + + +def test_validate_rejects_empty_pipeline_version(): + with pytest.raises(ValueError): + _build(pipeline_version="") + + +def test_validate_rejects_bad_relevant_nrr_id_prefix(): + with pytest.raises(ValueError, match="NRR-"): + _build(relevant_nrr_ids=["BAD-001"], cited_nrr_ids=[]) + + +def test_validate_rejects_bad_cited_nrr_id_prefix(): + with pytest.raises(ValueError, match="NRR-"): + _build( + relevant_nrr_ids=["NRR-001"], + cited_nrr_ids=["BAD-001"], + ) + + +def test_validate_rejects_cited_not_in_relevant(): + with pytest.raises(ValueError, match="not in relevant_nrr_ids"): + _build( + relevant_nrr_ids=["NRR-001"], + cited_nrr_ids=["NRR-999"], + ) + + +def test_validate_rejects_uncited_count_mismatch(): + nrc = _build() + nrc.uncited_count = 99 + with pytest.raises(ValueError, match="uncited_count"): + validate_negative_result_citation_policy(nrc) + + +def test_validate_rejects_all_relevant_cited_mismatch(): + nrc = _build() + nrc.all_relevant_cited = False + with pytest.raises(ValueError, match="all_relevant_cited"): + validate_negative_result_citation_policy(nrc) + + +def test_validate_rejects_invalid_policy_verdict(): + nrc = _build() + nrc.policy_verdict = "UNKNOWN" + with pytest.raises(ValueError, match="policy_verdict"): + validate_negative_result_citation_policy(nrc) + + +def test_validate_rejects_compliant_when_no_relevant(): + nrc = _build(relevant_nrr_ids=[], cited_nrr_ids=[]) + nrc.policy_verdict = "compliant" + with pytest.raises(ValueError, match="no_relevant_negatives"): + validate_negative_result_citation_policy(nrc) + + +def test_validate_rejects_non_compliant_when_all_cited(): + nrc = _build() + nrc.policy_verdict = "non_compliant" + with pytest.raises(ValueError, match="compliant"): + validate_negative_result_citation_policy(nrc) + + +def test_validate_rejects_dry_lab_only_false(): + nrc = _build() + nrc.dry_lab_only = False + with pytest.raises(ValueError, match="dry_lab_only"): + validate_negative_result_citation_policy(nrc) + + +def test_validate_rejects_empty_limitations(): + with pytest.raises(ValueError, match="limitations"): + _build(limitations=[]) + + +def test_validate_rejects_empty_created_at(): + with pytest.raises(ValueError): + _build(created_at="") + + +# --------------------------------------------------------------------------- +# 4. format +# --------------------------------------------------------------------------- + + +def test_format_contains_nrc_id(): + assert "NRC-001" in format_negative_result_citation_policy(_build()) + + +def test_format_contains_claim_id(): + assert "CLM-001" in format_negative_result_citation_policy(_build()) + + +def test_format_contains_pipeline_version(): + assert "v1.0" in format_negative_result_citation_policy(_build()) + + +def test_format_contains_verdict(): + assert "compliant" in format_negative_result_citation_policy(_build()) + + +def test_format_contains_uncited_nrr_when_non_compliant(): + r = _build( + relevant_nrr_ids=["NRR-001", "NRR-002"], + cited_nrr_ids=["NRR-001"], + ) + assert "NRR-002" in format_negative_result_citation_policy(r) + + +def test_format_contains_all_relevant_cited(): + assert "True" in format_negative_result_citation_policy(_build()) + + +def test_format_contains_limitations(): + assert "dry-lab only" in format_negative_result_citation_policy(_build()) + + +def test_format_contains_dry_lab_only(): + assert "dry_lab_only: True" in format_negative_result_citation_policy(_build()) + + +def test_format_is_string(): + assert isinstance(format_negative_result_citation_policy(_build()), str) diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index dcbb0eee..95d4e4ce 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -4,7 +4,7 @@ import sys import math -BASELINE = 10924 +BASELINE = 10974 def test_test_count_regression():