diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index 92bde6ab..eee12414 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -312,7 +312,8 @@ Make per-family performance gaps visible and machine-checkable, so the pipeline | PR | Task | Why it matters | Review class | |---:|---|---|---| -| Z1 | Add family blindness challenge harness schema (FBH-). | Per-family AUROC + panel representation check; flags when weak AMP classes (AUROC<0.55) are excluded from selected panel; prevents aggregate-metric hiding of family blind spots. | C | -| Z2 | Add batch explanation report schema (BXR-). | Per-candidate selection reason tracking (winner_exploit/uncertainty_probe/diversity_anchor/etc.); safety_cleared flag per candidate; verdict (explained/partially_explained/unexplained) based on safety clearance fraction; makes multi-batch selection auditable. | C | -| Z3 | Add adapter registry schema (ARG-). | Machine-readable registry of all external scoring/simulation adapters; adapter_type/status/evidence_level/can_affect_ranking per entry; only active+baseline_verified adapters may affect ranking; blocks experimental/pending adapters from influencing candidate selection. | C | -| Z4 | Add cheap baseline flag schema (CBF-). | Per-scorer gate ensuring every external adapter declares its cheapest meaningful baseline before influencing candidate ranking; blocks_ranking=True when baseline missing or AUROC delta <0.05; creates permanent anti-hype infrastructure. | C | +| Z1 | Add family blindness challenge harness schema (FBH-) (complete). | Per-family AUROC + panel representation check; flags when weak AMP classes (AUROC<0.55) are excluded from selected panel; prevents aggregate-metric hiding of family blind spots. | C | +| Z2 | Add batch explanation report schema (BXR-) (complete). | Per-candidate selection reason tracking (winner_exploit/uncertainty_probe/diversity_anchor/etc.); safety_cleared flag per candidate; verdict (explained/partially_explained/unexplained) based on safety clearance fraction; makes multi-batch selection auditable. | C | +| Z3 | Add adapter registry schema (ARG-) (complete). | Machine-readable registry of all external scoring/simulation adapters; adapter_type/status/evidence_level/can_affect_ranking per entry; only active+baseline_verified adapters may affect ranking; blocks experimental/pending adapters from influencing candidate selection. | C | +| Z4 | Add cheap baseline flag schema (CBF-) (complete). | Per-scorer gate ensuring every external adapter declares its cheapest meaningful baseline before influencing candidate ranking; blocks_ranking=True when baseline missing or AUROC delta <0.05; creates permanent anti-hype infrastructure. | C | +| Z5 | Add Phase Z accountability gate (ZAG-). | Top-level gate asserting FBH+BXR+ARG+CBF all present; verdict (accountability_verified/accountability_partial/accountability_not_established); closes Phase Z; no external pilot claim or adapter governance claim is credible without passing this gate. | C | diff --git a/src/openamp_foundry/evidence/phase_z_accountability_gate.py b/src/openamp_foundry/evidence/phase_z_accountability_gate.py new file mode 100644 index 00000000..e30e2bab --- /dev/null +++ b/src/openamp_foundry/evidence/phase_z_accountability_gate.py @@ -0,0 +1,167 @@ +"""ZAG- Phase Z accountability gate schema. + +Top-level gate asserting all four Phase Z components are +present: FBH + BXR + ARG + CBF. + +No external pilot claim or adapter governance claim is credible +without passing this gate. +Verdict: accountability_verified / accountability_partial / accountability_not_established. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +REQUIRED_Z_COMPONENTS: tuple[str, ...] = ("FBH", "BXR", "ARG", "CBF") + +VALID_ZAG_VERDICTS: frozenset[str] = frozenset({ + "accountability_verified", + "accountability_partial", + "accountability_not_established", +}) + +ACCOUNTABILITY_VERIFIED_REQUIRED_PRESENT: int = 4 +ACCOUNTABILITY_PARTIAL_MIN_PRESENT: int = 2 + + +@dataclass +class ZComponentCheck: + component_type: str + artifact_id: str + is_present: bool + + +@dataclass +class PhaseZAccountabilityGate: + zag_id: str + pipeline_version: str + fbh_id: str + bxr_id: str + arg_id: str + cbf_id: str + component_checks: list[ZComponentCheck] + n_components_present: int + verdict: str + dry_lab_only: bool + created_at: str + + +def _compute_verdict(n_present: int) -> str: + if n_present >= ACCOUNTABILITY_VERIFIED_REQUIRED_PRESENT: + return "accountability_verified" + if n_present >= ACCOUNTABILITY_PARTIAL_MIN_PRESENT: + return "accountability_partial" + return "accountability_not_established" + + +def build_phase_z_accountability_gate( + *, + zag_id: str, + pipeline_version: str, + fbh_id: str = "", + bxr_id: str = "", + arg_id: str = "", + cbf_id: str = "", + created_at: str, +) -> PhaseZAccountabilityGate: + checks = [ + ZComponentCheck( + component_type="FBH", + artifact_id=fbh_id, + is_present=bool(fbh_id) and fbh_id.startswith("FBH-"), + ), + ZComponentCheck( + component_type="BXR", + artifact_id=bxr_id, + is_present=bool(bxr_id) and bxr_id.startswith("BXR-"), + ), + ZComponentCheck( + component_type="ARG", + artifact_id=arg_id, + is_present=bool(arg_id) and arg_id.startswith("ARG-"), + ), + ZComponentCheck( + component_type="CBF", + artifact_id=cbf_id, + is_present=bool(cbf_id) and cbf_id.startswith("CBF-"), + ), + ] + n_present = sum(1 for c in checks if c.is_present) + verdict = _compute_verdict(n_present) + zag = PhaseZAccountabilityGate( + zag_id=zag_id, + pipeline_version=pipeline_version, + fbh_id=fbh_id, + bxr_id=bxr_id, + arg_id=arg_id, + cbf_id=cbf_id, + component_checks=checks, + n_components_present=n_present, + verdict=verdict, + dry_lab_only=True, + created_at=created_at, + ) + validate_phase_z_accountability_gate(zag) + return zag + + +def validate_phase_z_accountability_gate(zag: PhaseZAccountabilityGate) -> None: + if not zag.zag_id.startswith("ZAG-"): + raise ValueError(f"zag_id must start with 'ZAG-': {zag.zag_id!r}") + if not zag.pipeline_version: + raise ValueError("pipeline_version must be non-empty") + if len(zag.component_checks) != len(REQUIRED_Z_COMPONENTS): + raise ValueError( + f"component_checks must have exactly {len(REQUIRED_Z_COMPONENTS)} entries" + ) + for check in zag.component_checks: + if check.component_type not in REQUIRED_Z_COMPONENTS: + raise ValueError( + f"component_type {check.component_type!r} not in REQUIRED_Z_COMPONENTS" + ) + expected_prefix = f"{check.component_type}-" + if check.artifact_id and not check.artifact_id.startswith(expected_prefix): + raise ValueError( + f"artifact_id {check.artifact_id!r} must start with {expected_prefix!r}" + ) + for component_type, id_field, expected_prefix in [ + ("FBH", zag.fbh_id, "FBH-"), + ("BXR", zag.bxr_id, "BXR-"), + ("ARG", zag.arg_id, "ARG-"), + ("CBF", zag.cbf_id, "CBF-"), + ]: + if id_field and not id_field.startswith(expected_prefix): + raise ValueError( + f"{component_type.lower()}_id {id_field!r} must start with {expected_prefix!r}" + ) + n_present = sum(1 for c in zag.component_checks if c.is_present) + if zag.n_components_present != n_present: + raise ValueError("n_components_present mismatch") + if zag.verdict not in VALID_ZAG_VERDICTS: + raise ValueError(f"verdict {zag.verdict!r} not in VALID_ZAG_VERDICTS") + expected_verdict = _compute_verdict(zag.n_components_present) + if zag.verdict != expected_verdict: + raise ValueError( + f"verdict {zag.verdict!r} inconsistent with n_components_present={zag.n_components_present}" + ) + if not zag.dry_lab_only: + raise ValueError("dry_lab_only must be True") + if not zag.created_at: + raise ValueError("created_at must be non-empty") + + +def format_phase_z_accountability_gate(zag: PhaseZAccountabilityGate) -> str: + lines = [ + f"Phase Z Accountability Gate — {zag.zag_id}", + f"Pipeline: {zag.pipeline_version}", + f"Verdict: {zag.verdict}", + f"Components present: {zag.n_components_present}/{len(REQUIRED_Z_COMPONENTS)}", + ] + lines.append("Component checks:") + for check in zag.component_checks: + status = "PRESENT" if check.is_present else "ABSENT" + artifact = f" [{check.artifact_id}]" if check.artifact_id else "" + lines.append(f" {check.component_type}: {status}{artifact}") + lines.append(f"Created: {zag.created_at}") + lines.append(f"dry_lab_only: {zag.dry_lab_only}") + return "\n".join(lines) diff --git a/tests/evidence/test_phase_z_accountability_gate.py b/tests/evidence/test_phase_z_accountability_gate.py new file mode 100644 index 00000000..1ee98145 --- /dev/null +++ b/tests/evidence/test_phase_z_accountability_gate.py @@ -0,0 +1,298 @@ +"""Tests for ZAG- Phase Z accountability gate schema.""" + +import pytest +from openamp_foundry.evidence.phase_z_accountability_gate import ( + PhaseZAccountabilityGate, + ZComponentCheck, + REQUIRED_Z_COMPONENTS, + VALID_ZAG_VERDICTS, + build_phase_z_accountability_gate, + format_phase_z_accountability_gate, + validate_phase_z_accountability_gate, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _build(**kwargs): + defaults = dict( + zag_id="ZAG-001", + pipeline_version="v1.0", + fbh_id="FBH-001", + bxr_id="BXR-001", + arg_id="ARG-001", + cbf_id="CBF-001", + created_at="2026-07-10", + ) + defaults.update(kwargs) + return build_phase_z_accountability_gate(**defaults) + + +# --------------------------------------------------------------------------- +# 1. Constants +# --------------------------------------------------------------------------- + + +def test_required_z_components_is_tuple(): + assert isinstance(REQUIRED_Z_COMPONENTS, tuple) + + +def test_required_z_components_has_four(): + assert len(REQUIRED_Z_COMPONENTS) == 4 + + +def test_required_z_components_contains_fbh(): + assert "FBH" in REQUIRED_Z_COMPONENTS + + +def test_required_z_components_contains_bxr(): + assert "BXR" in REQUIRED_Z_COMPONENTS + + +def test_required_z_components_contains_arg(): + assert "ARG" in REQUIRED_Z_COMPONENTS + + +def test_required_z_components_contains_cbf(): + assert "CBF" in REQUIRED_Z_COMPONENTS + + +def test_valid_zag_verdicts_is_frozenset(): + assert isinstance(VALID_ZAG_VERDICTS, frozenset) + + +def test_valid_zag_verdicts_has_three(): + assert len(VALID_ZAG_VERDICTS) == 3 + + +# --------------------------------------------------------------------------- +# 2. build – happy paths +# --------------------------------------------------------------------------- + + +def test_build_returns_phase_z_accountability_gate(): + assert isinstance(_build(), PhaseZAccountabilityGate) + + +def test_build_zag_id_stored(): + assert _build().zag_id == "ZAG-001" + + +def test_build_pipeline_version_stored(): + assert _build().pipeline_version == "v1.0" + + +def test_build_dry_lab_only_true(): + assert _build().dry_lab_only is True + + +def test_build_all_present_gives_accountability_verified(): + assert _build().verdict == "accountability_verified" + + +def test_build_all_present_n_components_4(): + assert _build().n_components_present == 4 + + +def test_build_three_present_gives_accountability_partial(): + r = _build(cbf_id="") + assert r.verdict == "accountability_partial" + + +def test_build_two_present_gives_accountability_partial(): + r = _build(arg_id="", cbf_id="") + assert r.verdict == "accountability_partial" + + +def test_build_one_present_gives_accountability_not_established(): + r = _build(bxr_id="", arg_id="", cbf_id="") + assert r.verdict == "accountability_not_established" + + +def test_build_none_present_gives_accountability_not_established(): + r = _build(fbh_id="", bxr_id="", arg_id="", cbf_id="") + assert r.verdict == "accountability_not_established" + + +def test_build_none_present_n_components_0(): + r = _build(fbh_id="", bxr_id="", arg_id="", cbf_id="") + assert r.n_components_present == 0 + + +def test_build_component_checks_are_z_component_check(): + for c in _build().component_checks: + assert isinstance(c, ZComponentCheck) + + +def test_build_component_checks_count(): + assert len(_build().component_checks) == 4 + + +def test_build_fbh_present_true(): + c = next(c for c in _build().component_checks if c.component_type == "FBH") + assert c.is_present is True + + +def test_build_fbh_absent_false(): + c = next( + c + for c in _build(fbh_id="").component_checks + if c.component_type == "FBH" + ) + assert c.is_present is False + + +def test_build_fbh_artifact_id_stored(): + c = next(c for c in _build().component_checks if c.component_type == "FBH") + assert c.artifact_id == "FBH-001" + + +def test_build_bxr_artifact_id_stored(): + c = next(c for c in _build().component_checks if c.component_type == "BXR") + assert c.artifact_id == "BXR-001" + + +def test_build_created_at_stored(): + assert _build().created_at == "2026-07-10" + + +# --------------------------------------------------------------------------- +# 3. validate – rejection cases +# --------------------------------------------------------------------------- + + +def test_validate_rejects_bad_zag_id_prefix(): + with pytest.raises(ValueError, match="ZAG-"): + _build(zag_id="BAD-001") + + +def test_validate_rejects_empty_pipeline_version(): + with pytest.raises(ValueError): + _build(pipeline_version="") + + +def test_validate_rejects_wrong_fbh_prefix(): + with pytest.raises(ValueError, match="FBH-"): + _build(fbh_id="BAD-001") + + +def test_validate_rejects_wrong_bxr_prefix(): + with pytest.raises(ValueError, match="BXR-"): + _build(bxr_id="BAD-001") + + +def test_validate_rejects_wrong_arg_prefix(): + with pytest.raises(ValueError, match="ARG-"): + _build(arg_id="BAD-001") + + +def test_validate_rejects_wrong_cbf_prefix(): + with pytest.raises(ValueError, match="CBF-"): + _build(cbf_id="BAD-001") + + +def test_validate_rejects_n_components_mismatch(): + zag = _build() + zag.n_components_present = 99 + with pytest.raises(ValueError, match="n_components_present"): + validate_phase_z_accountability_gate(zag) + + +def test_validate_rejects_verdict_mismatch_4_present(): + zag = _build() + zag.verdict = "accountability_partial" + with pytest.raises(ValueError, match="inconsistent"): + validate_phase_z_accountability_gate(zag) + + +def test_validate_rejects_verdict_mismatch_3_present(): + zag = _build(cbf_id="") + zag.verdict = "accountability_not_established" + with pytest.raises(ValueError, match="inconsistent"): + validate_phase_z_accountability_gate(zag) + + +def test_validate_rejects_verdict_mismatch_2_present(): + zag = _build(arg_id="", cbf_id="") + zag.verdict = "accountability_not_established" + with pytest.raises(ValueError, match="inconsistent"): + validate_phase_z_accountability_gate(zag) + + +def test_validate_rejects_verdict_mismatch_1_present(): + zag = _build(bxr_id="", arg_id="", cbf_id="") + zag.verdict = "accountability_verified" + with pytest.raises(ValueError, match="inconsistent"): + validate_phase_z_accountability_gate(zag) + + +def test_validate_rejects_verdict_mismatch_0_present(): + zag = _build(fbh_id="", bxr_id="", arg_id="", cbf_id="") + zag.verdict = "accountability_verified" + with pytest.raises(ValueError, match="inconsistent"): + validate_phase_z_accountability_gate(zag) + + +def test_validate_rejects_dry_lab_only_false(): + zag = _build() + zag.dry_lab_only = False + with pytest.raises(ValueError, match="dry_lab_only"): + validate_phase_z_accountability_gate(zag) + + +def test_validate_rejects_empty_created_at(): + with pytest.raises(ValueError): + _build(created_at="") + + +def test_validate_rejects_invalid_verdict(): + zag = _build() + zag.verdict = "UNKNOWN" + with pytest.raises(ValueError, match="verdict"): + validate_phase_z_accountability_gate(zag) + + +def test_validate_rejects_wrong_component_count(): + zag = _build() + zag.component_checks = zag.component_checks[:2] + with pytest.raises(ValueError, match="component_checks"): + validate_phase_z_accountability_gate(zag) + + +# --------------------------------------------------------------------------- +# 4. format +# --------------------------------------------------------------------------- + + +def test_format_contains_zag_id(): + assert "ZAG-001" in format_phase_z_accountability_gate(_build()) + + +def test_format_contains_pipeline_version(): + assert "v1.0" in format_phase_z_accountability_gate(_build()) + + +def test_format_contains_verdict(): + assert "accountability_verified" in format_phase_z_accountability_gate(_build()) + + +def test_format_contains_component_name(): + assert "FBH" in format_phase_z_accountability_gate(_build()) + + +def test_format_contains_present_status(): + assert "PRESENT" in format_phase_z_accountability_gate(_build()) + + +def test_format_contains_absent_when_missing(): + assert "ABSENT" in format_phase_z_accountability_gate(_build(cbf_id="")) + + +def test_format_contains_dry_lab_only(): + assert "dry_lab_only: True" in format_phase_z_accountability_gate(_build()) + + +def test_format_is_string(): + assert isinstance(format_phase_z_accountability_gate(_build()), str) diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index c6a1ed4c..994a5b6e 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -4,7 +4,7 @@ import sys import math -BASELINE = 11469 +BASELINE = 11519 def test_test_count_regression():