diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index eccfec44..4cd120dc 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -303,5 +303,5 @@ Track and publish structured comparisons between pipeline selections and cheap b | Y1 | Add cheap baseline comparison record schema (CBR-) (complete). — src/openamp_foundry/evidence/cheap_baseline_comparison_record.py: VALID_CBR_VERDICTS (4: pipeline_superior/tied/baseline_superior/insufficient_data), VALID_BASELINE_METHODS (5: charge_only_rank/length_only_rank/random_selection/charge_length_combined/hydrophobicity_only_rank), VALID_CBR_METRICS (4: auroc/hit_rate/top_k_precision/ndcg), SUPERIORITY_THRESHOLD=0.05, MIN_SAMPLE_SIZE=5; metric_delta auto-computed; verdict auto-derived; dry_lab_only=True; 62 tests. | Structured record: pipeline metric vs charge-only/random/length-only baseline; pre-registered threshold; verdict (pipeline_superior/tied/baseline_superior/insufficient_data). Forces every performance claim to cite the baseline it beat. | C | | Y2 | Add feature importance audit schema (FIA-) (complete). — src/openamp_foundry/evidence/feature_importance_audit.py: VALID_FIA_VERDICTS (5), VALID_FEATURE_IMPORTANCE_LEVELS (4), VALID_AUDIT_FEATURES (8), DOMINATION_THRESHOLD=0.80; importance_level auto-assigned; top_feature/charge_score/length_score auto-extracted; verdict: charge_dominated when charge_explains_fraction>=0.80; dry_lab_only=True; 50 tests. | Documents which features drove selections and whether charge/length alone explains the result; anti-cheap-explanation gate. | C | | Y3 | Add selection diversity audit schema (SDA-) (complete). — src/openamp_foundry/evidence/selection_diversity_audit.py: VALID_SDA_VERDICTS (4: diverse_panel/moderately_diverse/proximity_driven/insufficient_data), VALID_DIVERSITY_METRICS (4), DIVERSE_PANEL_THRESHOLD=0.10, PROXIMITY_DRIVEN_THRESHOLD=-0.05, MIN_PANEL_SIZE=3; diversity_delta auto-computed; verdict: diverse_panel (delta>=0.10), proximity_driven (delta<=-0.05); dry_lab_only=True; 46 tests. | Tracks sequence diversity of selected panel vs random draw; detects proximity-driven selection masquerading as discovery; required before any novelty claim. | C | -| Y4 | Add pipeline maturity certificate schema (PMC-). | Aggregates CBR/FIA/SDA results into A/B/C/D maturity grade; anchors pre-registration; prevents retroactive interpretation of results. | C | +| Y4 | Add pipeline maturity certificate schema (PMC-) (complete). — src/openamp_foundry/evidence/pipeline_maturity_certificate.py: VALID_PMC_GRADES (A-D), VALID_PMC_VERDICTS (4: pipeline_validated/pipeline_provisional/pipeline_unvalidated/insufficient_evidence), REQUIRED_PMC_COMPONENTS=(CBR,FIA,SDA); PMCComponentCheck helper; grade A (all 3 superior), B (2), C (1), D (0/none assessed); contributes_to_grade auto-derived from verdict; dry_lab_only=True; 54 tests. | Aggregates CBR/FIA/SDA results into A/B/C/D maturity grade; anchors pre-registration; prevents retroactive interpretation. | C | | Y5 | Add Phase Y accountability gate (YAG-). | Top-level gate asserting CBR+FIA+SDA+PMC all present; verdict: accountability_verified/accountability_partial/accountability_not_established; closes Phase Y; no external pilot claim is credible without passing this gate. | C | diff --git a/src/openamp_foundry/evidence/pipeline_maturity_certificate.py b/src/openamp_foundry/evidence/pipeline_maturity_certificate.py new file mode 100644 index 00000000..0ffe594d --- /dev/null +++ b/src/openamp_foundry/evidence/pipeline_maturity_certificate.py @@ -0,0 +1,198 @@ +"""PMC- pipeline maturity certificate schema. + +Aggregates CBR (baseline comparison), FIA (feature importance audit), and +SDA (selection diversity audit) results into an A/B/C/D maturity grade. +Anchors pre-registration: the grade is frozen before results are interpreted +so it cannot be retroactively adjusted. No "mature pipeline" claim is +credible without this certificate. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +VALID_PMC_GRADES: frozenset[str] = frozenset({ + "A", + "B", + "C", + "D", +}) + +VALID_PMC_VERDICTS: frozenset[str] = frozenset({ + "pipeline_validated", + "pipeline_provisional", + "pipeline_unvalidated", + "insufficient_evidence", +}) + +REQUIRED_PMC_COMPONENTS: tuple[str, ...] = ("CBR", "FIA", "SDA") + +GRADE_A_REQUIRED_SUPERIOR_VERDICTS: int = 3 +GRADE_B_REQUIRED_SUPERIOR_VERDICTS: int = 2 +GRADE_C_REQUIRED_SUPERIOR_VERDICTS: int = 1 + + +@dataclass +class PMCComponentCheck: + component_type: str + artifact_id: str + verdict: str + contributes_to_grade: bool + + +@dataclass +class PipelineMaturityCertificate: + pmc_id: str + pipeline_version: str + component_checks: list[PMCComponentCheck] + n_components_assessed: int + n_superior_verdicts: int + pmc_grade: str + pmc_verdict: str + dry_lab_only: bool + limitations: list[str] + created_at: str + + +def validate_pipeline_maturity_certificate(pmc: PipelineMaturityCertificate) -> None: + if not pmc.pmc_id.startswith("PMC-"): + raise ValueError(f"pmc_id must start with 'PMC-': {pmc.pmc_id!r}") + if not pmc.pipeline_version: + raise ValueError("pipeline_version must be non-empty") + if len(pmc.component_checks) != len(REQUIRED_PMC_COMPONENTS): + raise ValueError( + f"component_checks must have exactly {len(REQUIRED_PMC_COMPONENTS)} entries" + ) + for check in pmc.component_checks: + if check.component_type not in REQUIRED_PMC_COMPONENTS: + raise ValueError( + f"component_type {check.component_type!r} not in REQUIRED_PMC_COMPONENTS" + ) + expected_prefix = f"{check.component_type}-" + if check.artifact_id and not check.artifact_id.startswith(expected_prefix): + raise ValueError( + f"artifact_id {check.artifact_id!r} must start with {expected_prefix!r}" + ) + n_assessed = sum(1 for c in pmc.component_checks if c.artifact_id) + if pmc.n_components_assessed != n_assessed: + raise ValueError("n_components_assessed mismatch") + n_superior = sum(1 for c in pmc.component_checks if c.contributes_to_grade) + if pmc.n_superior_verdicts != n_superior: + raise ValueError("n_superior_verdicts mismatch") + if pmc.pmc_grade not in VALID_PMC_GRADES: + raise ValueError(f"pmc_grade {pmc.pmc_grade!r} not in VALID_PMC_GRADES") + if pmc.pmc_verdict not in VALID_PMC_VERDICTS: + raise ValueError( + f"pmc_verdict {pmc.pmc_verdict!r} not in VALID_PMC_VERDICTS" + ) + if not pmc.dry_lab_only: + raise ValueError("dry_lab_only must be True") + if not pmc.limitations: + raise ValueError("limitations must be non-empty") + if not pmc.created_at: + raise ValueError("created_at must be non-empty") + + +_SUPERIOR_VERDICTS: frozenset[str] = frozenset({ + "pipeline_superior", + "multi_feature_signal", + "diverse_panel", +}) + + +def _contributes_to_grade(component_type: str, verdict: str) -> bool: + return verdict in _SUPERIOR_VERDICTS + + +def _compute_grade_and_verdict( + n_assessed: int, + n_superior: int, +) -> tuple[str, str]: + if n_assessed == 0: + return "D", "insufficient_evidence" + if n_superior >= GRADE_A_REQUIRED_SUPERIOR_VERDICTS: + return "A", "pipeline_validated" + if n_superior >= GRADE_B_REQUIRED_SUPERIOR_VERDICTS: + return "B", "pipeline_provisional" + if n_superior >= GRADE_C_REQUIRED_SUPERIOR_VERDICTS: + return "C", "pipeline_unvalidated" + return "D", "pipeline_unvalidated" + + +def build_pipeline_maturity_certificate( + *, + pmc_id: str, + pipeline_version: str, + cbr_artifact_id: str = "", + cbr_verdict: str = "", + fia_artifact_id: str = "", + fia_verdict: str = "", + sda_artifact_id: str = "", + sda_verdict: str = "", + limitations: list[str], + created_at: str, +) -> PipelineMaturityCertificate: + """Build a PipelineMaturityCertificate. + + Pass non-empty artifact_id + verdict for each assessed component. + Empty artifact_id means the component was not assessed. + """ + checks = [ + PMCComponentCheck( + component_type="CBR", + artifact_id=cbr_artifact_id, + verdict=cbr_verdict, + contributes_to_grade=_contributes_to_grade("CBR", cbr_verdict), + ), + PMCComponentCheck( + component_type="FIA", + artifact_id=fia_artifact_id, + verdict=fia_verdict, + contributes_to_grade=_contributes_to_grade("FIA", fia_verdict), + ), + PMCComponentCheck( + component_type="SDA", + artifact_id=sda_artifact_id, + verdict=sda_verdict, + contributes_to_grade=_contributes_to_grade("SDA", sda_verdict), + ), + ] + n_assessed = sum(1 for c in checks if c.artifact_id) + n_superior = sum(1 for c in checks if c.contributes_to_grade) + grade, verdict = _compute_grade_and_verdict(n_assessed, n_superior) + pmc = PipelineMaturityCertificate( + pmc_id=pmc_id, + pipeline_version=pipeline_version, + component_checks=checks, + n_components_assessed=n_assessed, + n_superior_verdicts=n_superior, + pmc_grade=grade, + pmc_verdict=verdict, + dry_lab_only=True, + limitations=limitations, + created_at=created_at, + ) + validate_pipeline_maturity_certificate(pmc) + return pmc + + +def format_pipeline_maturity_certificate(pmc: PipelineMaturityCertificate) -> str: + lines = [ + f"Pipeline Maturity Certificate — {pmc.pmc_id}", + f"Pipeline: {pmc.pipeline_version}", + f"Grade: {pmc.pmc_grade} | Verdict: {pmc.pmc_verdict}", + f"Components assessed: {pmc.n_components_assessed}/{len(REQUIRED_PMC_COMPONENTS)} " + f"| Superior verdicts: {pmc.n_superior_verdicts}", + ] + lines.append("Component checks:") + for check in pmc.component_checks: + status = "ASSESSED" if check.artifact_id else "NOT_ASSESSED" + contrib = " [contributes]" if check.contributes_to_grade else "" + lines.append( + f" {check.component_type}: {status} " + f"verdict={check.verdict or 'N/A'}{contrib}" + ) + lines.append(f"Created: {pmc.created_at}") + lines.append(f"Limitations: {'; '.join(pmc.limitations)}") + lines.append(f"dry_lab_only: {pmc.dry_lab_only}") + return "\n".join(lines) diff --git a/tests/evidence/test_pipeline_maturity_certificate.py b/tests/evidence/test_pipeline_maturity_certificate.py new file mode 100644 index 00000000..9704231a --- /dev/null +++ b/tests/evidence/test_pipeline_maturity_certificate.py @@ -0,0 +1,325 @@ +"""Tests for PMC- pipeline maturity certificate schema.""" + +import pytest +from openamp_foundry.evidence.pipeline_maturity_certificate import ( + PipelineMaturityCertificate, + PMCComponentCheck, + VALID_PMC_GRADES, + VALID_PMC_VERDICTS, + REQUIRED_PMC_COMPONENTS, + GRADE_A_REQUIRED_SUPERIOR_VERDICTS, + GRADE_B_REQUIRED_SUPERIOR_VERDICTS, + GRADE_C_REQUIRED_SUPERIOR_VERDICTS, + build_pipeline_maturity_certificate, + format_pipeline_maturity_certificate, + validate_pipeline_maturity_certificate, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _build(**kwargs): + defaults = dict( + pmc_id="PMC-001", + pipeline_version="v1.0", + cbr_artifact_id="CBR-001", + cbr_verdict="pipeline_superior", + fia_artifact_id="FIA-001", + fia_verdict="multi_feature_signal", + sda_artifact_id="SDA-001", + sda_verdict="diverse_panel", + limitations=["dry-lab only"], + created_at="2026-07-10", + ) + defaults.update(kwargs) + return build_pipeline_maturity_certificate(**defaults) + + +# --------------------------------------------------------------------------- +# 1. Constants +# --------------------------------------------------------------------------- + + +def test_valid_pmc_grades_is_frozenset(): + assert isinstance(VALID_PMC_GRADES, frozenset) + + +def test_valid_pmc_grades_contains_a(): + assert "A" in VALID_PMC_GRADES + + +def test_valid_pmc_grades_contains_d(): + assert "D" in VALID_PMC_GRADES + + +def test_valid_pmc_verdicts_is_frozenset(): + assert isinstance(VALID_PMC_VERDICTS, frozenset) + + +def test_valid_pmc_verdicts_contains_pipeline_validated(): + assert "pipeline_validated" in VALID_PMC_VERDICTS + + +def test_valid_pmc_verdicts_contains_insufficient_evidence(): + assert "insufficient_evidence" in VALID_PMC_VERDICTS + + +def test_required_pmc_components_contains_cbr(): + assert "CBR" in REQUIRED_PMC_COMPONENTS + + +def test_required_pmc_components_contains_fia(): + assert "FIA" in REQUIRED_PMC_COMPONENTS + + +def test_required_pmc_components_contains_sda(): + assert "SDA" in REQUIRED_PMC_COMPONENTS + + +def test_grade_a_required_superior_verdicts(): + assert GRADE_A_REQUIRED_SUPERIOR_VERDICTS == 3 + + +def test_grade_b_required_superior_verdicts(): + assert GRADE_B_REQUIRED_SUPERIOR_VERDICTS == 2 + + +def test_grade_c_required_superior_verdicts(): + assert GRADE_C_REQUIRED_SUPERIOR_VERDICTS == 1 + + +# --------------------------------------------------------------------------- +# 2. build – happy paths +# --------------------------------------------------------------------------- + + +def test_build_returns_pipeline_maturity_certificate(): + assert isinstance(_build(), PipelineMaturityCertificate) + + +def test_build_pmc_id_stored(): + assert _build().pmc_id == "PMC-001" + + +def test_build_pipeline_version_stored(): + assert _build().pipeline_version == "v1.0" + + +def test_build_dry_lab_only_true(): + assert _build().dry_lab_only is True + + +def test_build_all_superior_gives_grade_a(): + assert _build().pmc_grade == "A" + + +def test_build_all_superior_gives_pipeline_validated(): + assert _build().pmc_verdict == "pipeline_validated" + + +def test_build_two_superior_gives_grade_b(): + r = _build(sda_verdict="proximity_driven") + assert r.pmc_grade == "B" + + +def test_build_two_superior_gives_pipeline_provisional(): + r = _build(sda_verdict="proximity_driven") + assert r.pmc_verdict == "pipeline_provisional" + + +def test_build_one_superior_gives_grade_c(): + r = _build(fia_verdict="charge_dominated", sda_verdict="proximity_driven") + assert r.pmc_grade == "C" + + +def test_build_no_superior_gives_grade_d(): + r = _build( + cbr_verdict="tied", + fia_verdict="charge_dominated", + sda_verdict="proximity_driven", + ) + assert r.pmc_grade == "D" + + +def test_build_empty_gives_grade_d_insufficient(): + r = _build( + cbr_artifact_id="", + cbr_verdict="", + fia_artifact_id="", + fia_verdict="", + sda_artifact_id="", + sda_verdict="", + ) + assert r.pmc_grade == "D" + assert r.pmc_verdict == "insufficient_evidence" + + +def test_build_n_components_assessed_all(): + assert _build().n_components_assessed == 3 + + +def test_build_n_components_assessed_partial(): + r = _build(sda_artifact_id="", sda_verdict="") + assert r.n_components_assessed == 2 + + +def test_build_n_superior_verdicts_all(): + assert _build().n_superior_verdicts == 3 + + +def test_build_n_superior_verdicts_none(): + r = _build( + cbr_verdict="tied", + fia_verdict="charge_dominated", + sda_verdict="proximity_driven", + ) + assert r.n_superior_verdicts == 0 + + +def test_build_component_checks_are_pmc_component_check(): + for c in _build().component_checks: + assert isinstance(c, PMCComponentCheck) + + +def test_build_component_checks_count(): + assert len(_build().component_checks) == 3 + + +def test_build_cbr_check_artifact_id_stored(): + r = _build() + cbr_check = next(c for c in r.component_checks if c.component_type == "CBR") + assert cbr_check.artifact_id == "CBR-001" + + +def test_build_contributes_to_grade_true_for_superior(): + r = _build() + cbr_check = next(c for c in r.component_checks if c.component_type == "CBR") + assert cbr_check.contributes_to_grade is True + + +def test_build_contributes_to_grade_false_for_tied(): + r = _build(cbr_verdict="tied") + cbr_check = next(c for c in r.component_checks if c.component_type == "CBR") + assert cbr_check.contributes_to_grade is False + + +def test_build_limitations_stored(): + assert _build().limitations == ["dry-lab only"] + + +def test_build_created_at_stored(): + assert _build().created_at == "2026-07-10" + + +# --------------------------------------------------------------------------- +# 3. validate – rejection cases +# --------------------------------------------------------------------------- + + +def test_validate_rejects_bad_pmc_id_prefix(): + with pytest.raises(ValueError, match="PMC-"): + _build(pmc_id="BAD-001") + + +def test_validate_rejects_empty_pipeline_version(): + with pytest.raises(ValueError): + _build(pipeline_version="") + + +def test_validate_rejects_wrong_cbr_artifact_prefix(): + with pytest.raises(ValueError, match="CBR-"): + _build(cbr_artifact_id="BAD-001") + + +def test_validate_rejects_wrong_fia_artifact_prefix(): + with pytest.raises(ValueError, match="FIA-"): + _build(fia_artifact_id="BAD-001") + + +def test_validate_rejects_wrong_sda_artifact_prefix(): + with pytest.raises(ValueError, match="SDA-"): + _build(sda_artifact_id="BAD-001") + + +def test_validate_rejects_n_components_mismatch(): + pmc = _build() + pmc.n_components_assessed = 99 + with pytest.raises(ValueError, match="n_components_assessed"): + validate_pipeline_maturity_certificate(pmc) + + +def test_validate_rejects_n_superior_mismatch(): + pmc = _build() + pmc.n_superior_verdicts = 99 + with pytest.raises(ValueError, match="n_superior_verdicts"): + validate_pipeline_maturity_certificate(pmc) + + +def test_validate_rejects_invalid_pmc_grade(): + pmc = _build() + pmc.pmc_grade = "X" + with pytest.raises(ValueError, match="pmc_grade"): + validate_pipeline_maturity_certificate(pmc) + + +def test_validate_rejects_invalid_pmc_verdict(): + pmc = _build() + pmc.pmc_verdict = "UNKNOWN" + with pytest.raises(ValueError, match="pmc_verdict"): + validate_pipeline_maturity_certificate(pmc) + + +def test_validate_rejects_dry_lab_only_false(): + pmc = _build() + pmc.dry_lab_only = False + with pytest.raises(ValueError, match="dry_lab_only"): + validate_pipeline_maturity_certificate(pmc) + + +def test_validate_rejects_empty_limitations(): + with pytest.raises(ValueError, match="limitations"): + _build(limitations=[]) + + +def test_validate_rejects_empty_created_at(): + with pytest.raises(ValueError): + _build(created_at="") + + +# --------------------------------------------------------------------------- +# 4. format +# --------------------------------------------------------------------------- + + +def test_format_contains_pmc_id(): + assert "PMC-001" in format_pipeline_maturity_certificate(_build()) + + +def test_format_contains_pipeline_version(): + assert "v1.0" in format_pipeline_maturity_certificate(_build()) + + +def test_format_contains_grade(): + assert "A" in format_pipeline_maturity_certificate(_build()) + + +def test_format_contains_verdict(): + assert "pipeline_validated" in format_pipeline_maturity_certificate(_build()) + + +def test_format_contains_cbr(): + assert "CBR" in format_pipeline_maturity_certificate(_build()) + + +def test_format_contains_limitations(): + assert "dry-lab only" in format_pipeline_maturity_certificate(_build()) + + +def test_format_contains_dry_lab_only(): + assert "dry_lab_only: True" in format_pipeline_maturity_certificate(_build()) + + +def test_format_is_string(): + assert isinstance(format_pipeline_maturity_certificate(_build()), str) diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 26022df3..811291bc 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -4,7 +4,7 @@ import sys import math -BASELINE = 10656 +BASELINE = 10710 def test_test_count_regression():