From e287314d62d26acfd06362c41bd4bf3ad4e99369 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Fri, 10 Jul 2026 15:48:33 +0700 Subject: [PATCH] feat: Phase W W5 Phase W benchmark gate schema (WBG-) -- top-level gate asserting NCH+CMC+SCH+BCR all present; VALID_WBG_VERDICTS (3: hardened/partially_hardened/not_hardened), HARDENED_REQUIRED_PRESENT=4; closes Phase W; 61 tests --- docs/research/NEXT_100_PR_MAP.md | 2 +- .../evidence/phase_w_benchmark_gate.py | 170 +++++++++ tests/evidence/test_phase_w_benchmark_gate.py | 336 ++++++++++++++++++ 3 files changed, 507 insertions(+), 1 deletion(-) create mode 100644 src/openamp_foundry/evidence/phase_w_benchmark_gate.py create mode 100644 tests/evidence/test_phase_w_benchmark_gate.py diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index a53ecd01..17fefeb5 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -280,4 +280,4 @@ Make it machine-verifiable that the pipeline produces novel candidates that beat | W2 | Add charge-matched challenge schema (CMC-) (complete). — src/openamp_foundry/evidence/charge_matched_challenge.py: VALID_CMC_VERDICTS (4: gap_meaningful/gap_marginal/gap_absent/challenge_not_run), VALID_CHARGE_BASELINE_METHODS (4), MEANINGFUL_GAP_THRESHOLD=0.05, MARGINAL_GAP_LOWER=0.02; auroc_gap auto-computed; verdict auto-derived; dry_lab_only=True enforced; 60 tests in tests/evidence/test_charge_matched_challenge.py. | Formally documents the charge-matched challenge: compares pipeline AUROC vs a charge-only baseline on the same candidate set; verdict controlled vocabulary (gap_meaningful/gap_marginal/gap_absent/not_run); blocks performance claims when the charge-only baseline explains the gap. | C | | W3 | Add similarity challenge harness schema (SCH-) (complete). — src/openamp_foundry/evidence/similarity_challenge_harness.py: VALID_SCH_VERDICTS (4: selection_adds_value/marginal_improvement/proximity_driven/challenge_not_run), VALID_SIMILARITY_METRICS (4), SELECTION_VALUE_GAP_THRESHOLD=0.10, MARGINAL_IMPROVEMENT_LOWER=0.03; SimilarityGroupStats helper; similarity_gap auto-computed; dry_lab_only=True enforced; 60 tests in tests/evidence/test_similarity_challenge_harness.py. | Documents whether pipeline-selected candidates are systematically more similar to known AMPs than random selection from the sequence space; flags selection bias from similarity clustering; prevents "novel panel" claim when selection is proximity-driven. | C | | W4 | Add benchmark challenge registry schema (BCR-) (complete). — src/openamp_foundry/evidence/benchmark_challenge_registry.py: REQUIRED_CHALLENGE_TYPES (3: NCH/CMC/SCH), VALID_CHALLENGE_VERDICTS (4: pass/marginal/fail/not_run), VALID_BCR_HARDNESS_GRADES (A-D); ChallengeEntry helper; verdict mapping (novel_batch→pass, mixed_novelty→marginal, gap_meaningful→pass, etc.); grade A=all pass, B=all pass+marginal, C=some pass/marginal, D=all fail/not_run; dry_lab_only=True; 64 tests in tests/evidence/test_benchmark_challenge_registry.py. | Machine-readable registry of which benchmark challenges (NCH/CMC/SCH) have been run and passed for a given pipeline version; aggregates challenge verdicts; overall hardness grade (A: all passed, B: most passed, C: some passed, D: none passed). | C | -| W5 | Add Phase W benchmark gate (WBG-). | Top-level gate asserting NCH + CMC + SCH + BCR all present; overall verdict: hardened/partially_hardened/not_hardened; closes Phase W; no batch-level performance claim is credible without passing this gate. | C | +| W5 | Add Phase W benchmark gate (WBG-) (complete). — src/openamp_foundry/evidence/phase_w_benchmark_gate.py: REQUIRED_W_COMPONENTS (4: NCH/CMC/SCH/BCR), VALID_WBG_VERDICTS (3: hardened/partially_hardened/not_hardened), HARDENED_REQUIRED_PRESENT=4, PARTIALLY_HARDENED_MIN_PRESENT=2; WComponentCheck helper; prefix validation for all artifact IDs; verdict: hardened=all 4 present, partially_hardened=2-3, not_hardened=0-1; dry_lab_only=True; 61 tests. Closes Phase W. | Top-level gate asserting NCH + CMC + SCH + BCR all present; overall verdict: hardened/partially_hardened/not_hardened; closes Phase W; no batch-level performance claim is credible without passing this gate. | C | diff --git a/src/openamp_foundry/evidence/phase_w_benchmark_gate.py b/src/openamp_foundry/evidence/phase_w_benchmark_gate.py new file mode 100644 index 00000000..8516e687 --- /dev/null +++ b/src/openamp_foundry/evidence/phase_w_benchmark_gate.py @@ -0,0 +1,170 @@ +"""WBG- Phase W benchmark gate schema. + +Top-level gate asserting all four Phase W benchmark challenge artifacts +(NCH, CMC, SCH, BCR) are present for a batch. Produces an overall +verdict: hardened/partially_hardened/not_hardened. No batch-level +performance claim is credible without a passing WBG-. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +REQUIRED_W_COMPONENTS: tuple[str, ...] = ("NCH", "CMC", "SCH", "BCR") + +VALID_WBG_VERDICTS: frozenset[str] = frozenset({ + "hardened", + "partially_hardened", + "not_hardened", +}) + +_COMPONENT_ID_PREFIXES: dict[str, str] = { + "NCH": "NCH-", + "CMC": "CMC-", + "SCH": "SCH-", + "BCR": "BCR-", +} + +HARDENED_REQUIRED_PRESENT: int = 4 +PARTIALLY_HARDENED_MIN_PRESENT: int = 2 + + +@dataclass +class WComponentCheck: + component_type: str + artifact_id: str + present: bool + + +@dataclass +class PhaseWBenchmarkGate: + wbg_id: str + batch_id: str + pipeline_version: str + component_checks: list[WComponentCheck] + n_components_required: int + n_components_present: int + missing_component_types: list[str] + gate_verdict: str + dry_lab_only: bool + limitations: list[str] + created_at: str + + +def validate_phase_w_benchmark_gate(gate: PhaseWBenchmarkGate) -> None: + if not gate.wbg_id.startswith("WBG-"): + raise ValueError(f"wbg_id must start with 'WBG-': {gate.wbg_id!r}") + if not gate.batch_id: + raise ValueError("batch_id must be non-empty") + if not gate.pipeline_version: + raise ValueError("pipeline_version must be non-empty") + seen: set[str] = set() + for check in gate.component_checks: + if check.component_type not in REQUIRED_W_COMPONENTS: + raise ValueError( + f"component_type {check.component_type!r} not in REQUIRED_W_COMPONENTS" + ) + if check.component_type in seen: + raise ValueError(f"duplicate component_type: {check.component_type!r}") + seen.add(check.component_type) + expected_prefix = _COMPONENT_ID_PREFIXES[check.component_type] + if check.present and not check.artifact_id.startswith(expected_prefix): + raise ValueError( + f"artifact_id for {check.component_type} must start with " + f"'{expected_prefix}': {check.artifact_id!r}" + ) + if gate.n_components_required != len(REQUIRED_W_COMPONENTS): + raise ValueError( + f"n_components_required must be {len(REQUIRED_W_COMPONENTS)}" + ) + present_count = sum(1 for c in gate.component_checks if c.present) + if gate.n_components_present != present_count: + raise ValueError("n_components_present mismatch") + expected_missing = sorted( + c.component_type for c in gate.component_checks if not c.present + ) + if gate.missing_component_types != expected_missing: + raise ValueError("missing_component_types mismatch") + if gate.gate_verdict not in VALID_WBG_VERDICTS: + raise ValueError( + f"gate_verdict {gate.gate_verdict!r} not in VALID_WBG_VERDICTS" + ) + if not gate.dry_lab_only: + raise ValueError("dry_lab_only must be True") + if not gate.limitations: + raise ValueError("limitations must be non-empty") + if not gate.created_at: + raise ValueError("created_at must be non-empty") + + +def _compute_verdict(n_present: int) -> str: + if n_present == HARDENED_REQUIRED_PRESENT: + return "hardened" + if n_present >= PARTIALLY_HARDENED_MIN_PRESENT: + return "partially_hardened" + return "not_hardened" + + +def build_phase_w_benchmark_gate( + *, + wbg_id: str, + batch_id: str, + pipeline_version: str, + nch_artifact_id: str = "", + cmc_artifact_id: str = "", + sch_artifact_id: str = "", + bcr_artifact_id: str = "", + limitations: list[str], + created_at: str, +) -> PhaseWBenchmarkGate: + artifact_map = { + "NCH": nch_artifact_id, + "CMC": cmc_artifact_id, + "SCH": sch_artifact_id, + "BCR": bcr_artifact_id, + } + checks = [ + WComponentCheck( + component_type=ctype, + artifact_id=artifact_map[ctype], + present=bool(artifact_map[ctype]), + ) + for ctype in REQUIRED_W_COMPONENTS + ] + n_present = sum(1 for c in checks if c.present) + missing = sorted(c.component_type for c in checks if not c.present) + verdict = _compute_verdict(n_present) + gate = PhaseWBenchmarkGate( + wbg_id=wbg_id, + batch_id=batch_id, + pipeline_version=pipeline_version, + component_checks=checks, + n_components_required=len(REQUIRED_W_COMPONENTS), + n_components_present=n_present, + missing_component_types=missing, + gate_verdict=verdict, + dry_lab_only=True, + limitations=limitations, + created_at=created_at, + ) + validate_phase_w_benchmark_gate(gate) + return gate + + +def format_phase_w_benchmark_gate(gate: PhaseWBenchmarkGate) -> str: + lines = [ + f"Phase W Benchmark Gate — {gate.wbg_id}", + f"Batch: {gate.batch_id} | Pipeline: {gate.pipeline_version}", + f"Verdict: {gate.gate_verdict} | Present: {gate.n_components_present}/{gate.n_components_required}", + "Components:", + ] + for check in gate.component_checks: + status = "PRESENT" if check.present else "MISSING" + aid = check.artifact_id if check.artifact_id else "(none)" + lines.append(f" {check.component_type}: {status} {aid}") + if gate.missing_component_types: + lines.append(f"Missing: {', '.join(gate.missing_component_types)}") + lines.append(f"Created: {gate.created_at}") + lines.append(f"Limitations: {'; '.join(gate.limitations)}") + lines.append(f"dry_lab_only: {gate.dry_lab_only}") + return "\n".join(lines) diff --git a/tests/evidence/test_phase_w_benchmark_gate.py b/tests/evidence/test_phase_w_benchmark_gate.py new file mode 100644 index 00000000..9df09fd7 --- /dev/null +++ b/tests/evidence/test_phase_w_benchmark_gate.py @@ -0,0 +1,336 @@ +"""Tests for WBG- Phase W benchmark gate schema.""" + +import pytest +from openamp_foundry.evidence.phase_w_benchmark_gate import ( + PhaseWBenchmarkGate, + WComponentCheck, + REQUIRED_W_COMPONENTS, + VALID_WBG_VERDICTS, + HARDENED_REQUIRED_PRESENT, + PARTIALLY_HARDENED_MIN_PRESENT, + build_phase_w_benchmark_gate, + format_phase_w_benchmark_gate, + validate_phase_w_benchmark_gate, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _build(**kwargs): + defaults = dict( + wbg_id="WBG-001", + batch_id="BATCH-01", + pipeline_version="v1.0", + nch_artifact_id="NCH-001", + cmc_artifact_id="CMC-001", + sch_artifact_id="SCH-001", + bcr_artifact_id="BCR-001", + limitations=["dry-lab only"], + created_at="2026-07-10", + ) + defaults.update(kwargs) + return build_phase_w_benchmark_gate(**defaults) + + +# --------------------------------------------------------------------------- +# 1. Constants +# --------------------------------------------------------------------------- + + +def test_required_w_components_is_tuple(): + assert isinstance(REQUIRED_W_COMPONENTS, tuple) + + +def test_required_w_components_contains_nch(): + assert "NCH" in REQUIRED_W_COMPONENTS + + +def test_required_w_components_contains_cmc(): + assert "CMC" in REQUIRED_W_COMPONENTS + + +def test_required_w_components_contains_sch(): + assert "SCH" in REQUIRED_W_COMPONENTS + + +def test_required_w_components_contains_bcr(): + assert "BCR" in REQUIRED_W_COMPONENTS + + +def test_required_w_components_count(): + assert len(REQUIRED_W_COMPONENTS) == 4 + + +def test_valid_wbg_verdicts_is_frozenset(): + assert isinstance(VALID_WBG_VERDICTS, frozenset) + + +def test_valid_wbg_verdicts_contains_hardened(): + assert "hardened" in VALID_WBG_VERDICTS + + +def test_valid_wbg_verdicts_contains_partially_hardened(): + assert "partially_hardened" in VALID_WBG_VERDICTS + + +def test_valid_wbg_verdicts_contains_not_hardened(): + assert "not_hardened" in VALID_WBG_VERDICTS + + +def test_hardened_required_present(): + assert HARDENED_REQUIRED_PRESENT == 4 + + +def test_partially_hardened_min_present(): + assert PARTIALLY_HARDENED_MIN_PRESENT == 2 + + +# --------------------------------------------------------------------------- +# 2. build – happy paths +# --------------------------------------------------------------------------- + + +def test_build_returns_phase_w_benchmark_gate(): + assert isinstance(_build(), PhaseWBenchmarkGate) + + +def test_build_wbg_id_stored(): + assert _build().wbg_id == "WBG-001" + + +def test_build_batch_id_stored(): + assert _build().batch_id == "BATCH-01" + + +def test_build_pipeline_version_stored(): + assert _build().pipeline_version == "v1.0" + + +def test_build_dry_lab_only_true(): + assert _build().dry_lab_only is True + + +def test_build_n_components_required_is_4(): + assert _build().n_components_required == 4 + + +def test_build_all_present_gives_hardened(): + assert _build().gate_verdict == "hardened" + + +def test_build_all_present_n_components_4(): + assert _build().n_components_present == 4 + + +def test_build_all_present_missing_is_empty(): + assert _build().missing_component_types == [] + + +def test_build_component_checks_length(): + assert len(_build().component_checks) == 4 + + +def test_build_component_checks_are_w_component_check(): + for c in _build().component_checks: + assert isinstance(c, WComponentCheck) + + +def test_build_nch_check_present(): + checks = {c.component_type: c for c in _build().component_checks} + assert checks["NCH"].present is True + + +def test_build_cmc_check_present(): + checks = {c.component_type: c for c in _build().component_checks} + assert checks["CMC"].present is True + + +def test_build_sch_check_present(): + checks = {c.component_type: c for c in _build().component_checks} + assert checks["SCH"].present is True + + +def test_build_bcr_check_present(): + checks = {c.component_type: c for c in _build().component_checks} + assert checks["BCR"].present is True + + +def test_build_nch_artifact_id_stored(): + checks = {c.component_type: c for c in _build().component_checks} + assert checks["NCH"].artifact_id == "NCH-001" + + +def test_build_bcr_artifact_id_stored(): + checks = {c.component_type: c for c in _build().component_checks} + assert checks["BCR"].artifact_id == "BCR-001" + + +def test_build_missing_one_gives_partially_hardened(): + r = _build(nch_artifact_id="") + assert r.gate_verdict == "partially_hardened" + + +def test_build_missing_two_gives_partially_hardened(): + r = _build(nch_artifact_id="", cmc_artifact_id="") + assert r.gate_verdict == "partially_hardened" + + +def test_build_missing_three_gives_not_hardened(): + r = _build(nch_artifact_id="", cmc_artifact_id="", sch_artifact_id="") + assert r.gate_verdict == "not_hardened" + + +def test_build_all_missing_gives_not_hardened(): + r = _build(nch_artifact_id="", cmc_artifact_id="", sch_artifact_id="", bcr_artifact_id="") + assert r.gate_verdict == "not_hardened" + + +def test_build_all_missing_n_present_0(): + r = _build(nch_artifact_id="", cmc_artifact_id="", sch_artifact_id="", bcr_artifact_id="") + assert r.n_components_present == 0 + + +def test_build_missing_nch_in_missing_list(): + r = _build(nch_artifact_id="") + assert "NCH" in r.missing_component_types + + +def test_build_missing_list_sorted(): + r = _build(nch_artifact_id="", bcr_artifact_id="") + assert r.missing_component_types == sorted(r.missing_component_types) + + +def test_build_limitations_stored(): + assert _build().limitations == ["dry-lab only"] + + +def test_build_created_at_stored(): + assert _build().created_at == "2026-07-10" + + +# --------------------------------------------------------------------------- +# 3. validate – rejection cases +# --------------------------------------------------------------------------- + + +def test_validate_rejects_bad_wbg_id_prefix(): + with pytest.raises(ValueError, match="WBG-"): + _build(wbg_id="BAD-001") + + +def test_validate_rejects_empty_batch_id(): + with pytest.raises(ValueError): + _build(batch_id="") + + +def test_validate_rejects_empty_pipeline_version(): + with pytest.raises(ValueError): + _build(pipeline_version="") + + +def test_validate_rejects_nch_artifact_without_prefix(): + gate = _build() + gate.component_checks[0].artifact_id = "BAD-001" + with pytest.raises(ValueError): + validate_phase_w_benchmark_gate(gate) + + +def test_validate_rejects_invalid_gate_verdict(): + gate = _build() + gate.gate_verdict = "UNKNOWN" + with pytest.raises(ValueError, match="gate_verdict"): + validate_phase_w_benchmark_gate(gate) + + +def test_validate_rejects_dry_lab_only_false(): + gate = _build() + gate.dry_lab_only = False + with pytest.raises(ValueError, match="dry_lab_only"): + validate_phase_w_benchmark_gate(gate) + + +def test_validate_rejects_empty_limitations(): + with pytest.raises(ValueError, match="limitations"): + _build(limitations=[]) + + +def test_validate_rejects_empty_created_at(): + with pytest.raises(ValueError): + _build(created_at="") + + +def test_validate_rejects_wrong_n_components_required(): + gate = _build() + gate.n_components_required = 3 + with pytest.raises(ValueError, match="n_components_required"): + validate_phase_w_benchmark_gate(gate) + + +def test_validate_rejects_n_present_mismatch(): + gate = _build() + gate.n_components_present = 99 + with pytest.raises(ValueError, match="n_components_present"): + validate_phase_w_benchmark_gate(gate) + + +def test_validate_rejects_missing_list_mismatch(): + gate = _build() + gate.missing_component_types = ["NCH"] + with pytest.raises(ValueError, match="missing_component_types"): + validate_phase_w_benchmark_gate(gate) + + +# --------------------------------------------------------------------------- +# 4. format +# --------------------------------------------------------------------------- + + +def test_format_contains_wbg_id(): + assert "WBG-001" in format_phase_w_benchmark_gate(_build()) + + +def test_format_contains_batch_id(): + assert "BATCH-01" in format_phase_w_benchmark_gate(_build()) + + +def test_format_contains_hardened_verdict(): + assert "hardened" in format_phase_w_benchmark_gate(_build()) + + +def test_format_contains_nch(): + assert "NCH" in format_phase_w_benchmark_gate(_build()) + + +def test_format_contains_cmc(): + assert "CMC" in format_phase_w_benchmark_gate(_build()) + + +def test_format_contains_sch(): + assert "SCH" in format_phase_w_benchmark_gate(_build()) + + +def test_format_contains_bcr(): + assert "BCR" in format_phase_w_benchmark_gate(_build()) + + +def test_format_contains_present_label(): + assert "PRESENT" in format_phase_w_benchmark_gate(_build()) + + +def test_format_contains_missing_label_when_absent(): + r = _build(nch_artifact_id="") + assert "MISSING" in format_phase_w_benchmark_gate(r) + + +def test_format_contains_limitations(): + assert "dry-lab only" in format_phase_w_benchmark_gate(_build()) + + +def test_format_contains_dry_lab_only(): + assert "dry_lab_only: True" in format_phase_w_benchmark_gate(_build()) + + +def test_format_is_string(): + assert isinstance(format_phase_w_benchmark_gate(_build()), str)