|
| 1 | +"""PMC- pipeline maturity certificate schema. |
| 2 | +
|
| 3 | +Aggregates CBR (baseline comparison), FIA (feature importance audit), and |
| 4 | +SDA (selection diversity audit) results into an A/B/C/D maturity grade. |
| 5 | +Anchors pre-registration: the grade is frozen before results are interpreted |
| 6 | +so it cannot be retroactively adjusted. No "mature pipeline" claim is |
| 7 | +credible without this certificate. |
| 8 | +""" |
| 9 | + |
| 10 | +from __future__ import annotations |
| 11 | + |
| 12 | +from dataclasses import dataclass |
| 13 | + |
| 14 | +VALID_PMC_GRADES: frozenset[str] = frozenset({ |
| 15 | + "A", |
| 16 | + "B", |
| 17 | + "C", |
| 18 | + "D", |
| 19 | +}) |
| 20 | + |
| 21 | +VALID_PMC_VERDICTS: frozenset[str] = frozenset({ |
| 22 | + "pipeline_validated", |
| 23 | + "pipeline_provisional", |
| 24 | + "pipeline_unvalidated", |
| 25 | + "insufficient_evidence", |
| 26 | +}) |
| 27 | + |
| 28 | +REQUIRED_PMC_COMPONENTS: tuple[str, ...] = ("CBR", "FIA", "SDA") |
| 29 | + |
| 30 | +GRADE_A_REQUIRED_SUPERIOR_VERDICTS: int = 3 |
| 31 | +GRADE_B_REQUIRED_SUPERIOR_VERDICTS: int = 2 |
| 32 | +GRADE_C_REQUIRED_SUPERIOR_VERDICTS: int = 1 |
| 33 | + |
| 34 | + |
| 35 | +@dataclass |
| 36 | +class PMCComponentCheck: |
| 37 | + component_type: str |
| 38 | + artifact_id: str |
| 39 | + verdict: str |
| 40 | + contributes_to_grade: bool |
| 41 | + |
| 42 | + |
| 43 | +@dataclass |
| 44 | +class PipelineMaturityCertificate: |
| 45 | + pmc_id: str |
| 46 | + pipeline_version: str |
| 47 | + component_checks: list[PMCComponentCheck] |
| 48 | + n_components_assessed: int |
| 49 | + n_superior_verdicts: int |
| 50 | + pmc_grade: str |
| 51 | + pmc_verdict: str |
| 52 | + dry_lab_only: bool |
| 53 | + limitations: list[str] |
| 54 | + created_at: str |
| 55 | + |
| 56 | + |
| 57 | +def validate_pipeline_maturity_certificate(pmc: PipelineMaturityCertificate) -> None: |
| 58 | + if not pmc.pmc_id.startswith("PMC-"): |
| 59 | + raise ValueError(f"pmc_id must start with 'PMC-': {pmc.pmc_id!r}") |
| 60 | + if not pmc.pipeline_version: |
| 61 | + raise ValueError("pipeline_version must be non-empty") |
| 62 | + if len(pmc.component_checks) != len(REQUIRED_PMC_COMPONENTS): |
| 63 | + raise ValueError( |
| 64 | + f"component_checks must have exactly {len(REQUIRED_PMC_COMPONENTS)} entries" |
| 65 | + ) |
| 66 | + for check in pmc.component_checks: |
| 67 | + if check.component_type not in REQUIRED_PMC_COMPONENTS: |
| 68 | + raise ValueError( |
| 69 | + f"component_type {check.component_type!r} not in REQUIRED_PMC_COMPONENTS" |
| 70 | + ) |
| 71 | + expected_prefix = f"{check.component_type}-" |
| 72 | + if check.artifact_id and not check.artifact_id.startswith(expected_prefix): |
| 73 | + raise ValueError( |
| 74 | + f"artifact_id {check.artifact_id!r} must start with {expected_prefix!r}" |
| 75 | + ) |
| 76 | + n_assessed = sum(1 for c in pmc.component_checks if c.artifact_id) |
| 77 | + if pmc.n_components_assessed != n_assessed: |
| 78 | + raise ValueError("n_components_assessed mismatch") |
| 79 | + n_superior = sum(1 for c in pmc.component_checks if c.contributes_to_grade) |
| 80 | + if pmc.n_superior_verdicts != n_superior: |
| 81 | + raise ValueError("n_superior_verdicts mismatch") |
| 82 | + if pmc.pmc_grade not in VALID_PMC_GRADES: |
| 83 | + raise ValueError(f"pmc_grade {pmc.pmc_grade!r} not in VALID_PMC_GRADES") |
| 84 | + if pmc.pmc_verdict not in VALID_PMC_VERDICTS: |
| 85 | + raise ValueError( |
| 86 | + f"pmc_verdict {pmc.pmc_verdict!r} not in VALID_PMC_VERDICTS" |
| 87 | + ) |
| 88 | + if not pmc.dry_lab_only: |
| 89 | + raise ValueError("dry_lab_only must be True") |
| 90 | + if not pmc.limitations: |
| 91 | + raise ValueError("limitations must be non-empty") |
| 92 | + if not pmc.created_at: |
| 93 | + raise ValueError("created_at must be non-empty") |
| 94 | + |
| 95 | + |
| 96 | +_SUPERIOR_VERDICTS: frozenset[str] = frozenset({ |
| 97 | + "pipeline_superior", |
| 98 | + "multi_feature_signal", |
| 99 | + "diverse_panel", |
| 100 | +}) |
| 101 | + |
| 102 | + |
| 103 | +def _contributes_to_grade(component_type: str, verdict: str) -> bool: |
| 104 | + return verdict in _SUPERIOR_VERDICTS |
| 105 | + |
| 106 | + |
| 107 | +def _compute_grade_and_verdict( |
| 108 | + n_assessed: int, |
| 109 | + n_superior: int, |
| 110 | +) -> tuple[str, str]: |
| 111 | + if n_assessed == 0: |
| 112 | + return "D", "insufficient_evidence" |
| 113 | + if n_superior >= GRADE_A_REQUIRED_SUPERIOR_VERDICTS: |
| 114 | + return "A", "pipeline_validated" |
| 115 | + if n_superior >= GRADE_B_REQUIRED_SUPERIOR_VERDICTS: |
| 116 | + return "B", "pipeline_provisional" |
| 117 | + if n_superior >= GRADE_C_REQUIRED_SUPERIOR_VERDICTS: |
| 118 | + return "C", "pipeline_unvalidated" |
| 119 | + return "D", "pipeline_unvalidated" |
| 120 | + |
| 121 | + |
| 122 | +def build_pipeline_maturity_certificate( |
| 123 | + *, |
| 124 | + pmc_id: str, |
| 125 | + pipeline_version: str, |
| 126 | + cbr_artifact_id: str = "", |
| 127 | + cbr_verdict: str = "", |
| 128 | + fia_artifact_id: str = "", |
| 129 | + fia_verdict: str = "", |
| 130 | + sda_artifact_id: str = "", |
| 131 | + sda_verdict: str = "", |
| 132 | + limitations: list[str], |
| 133 | + created_at: str, |
| 134 | +) -> PipelineMaturityCertificate: |
| 135 | + """Build a PipelineMaturityCertificate. |
| 136 | +
|
| 137 | + Pass non-empty artifact_id + verdict for each assessed component. |
| 138 | + Empty artifact_id means the component was not assessed. |
| 139 | + """ |
| 140 | + checks = [ |
| 141 | + PMCComponentCheck( |
| 142 | + component_type="CBR", |
| 143 | + artifact_id=cbr_artifact_id, |
| 144 | + verdict=cbr_verdict, |
| 145 | + contributes_to_grade=_contributes_to_grade("CBR", cbr_verdict), |
| 146 | + ), |
| 147 | + PMCComponentCheck( |
| 148 | + component_type="FIA", |
| 149 | + artifact_id=fia_artifact_id, |
| 150 | + verdict=fia_verdict, |
| 151 | + contributes_to_grade=_contributes_to_grade("FIA", fia_verdict), |
| 152 | + ), |
| 153 | + PMCComponentCheck( |
| 154 | + component_type="SDA", |
| 155 | + artifact_id=sda_artifact_id, |
| 156 | + verdict=sda_verdict, |
| 157 | + contributes_to_grade=_contributes_to_grade("SDA", sda_verdict), |
| 158 | + ), |
| 159 | + ] |
| 160 | + n_assessed = sum(1 for c in checks if c.artifact_id) |
| 161 | + n_superior = sum(1 for c in checks if c.contributes_to_grade) |
| 162 | + grade, verdict = _compute_grade_and_verdict(n_assessed, n_superior) |
| 163 | + pmc = PipelineMaturityCertificate( |
| 164 | + pmc_id=pmc_id, |
| 165 | + pipeline_version=pipeline_version, |
| 166 | + component_checks=checks, |
| 167 | + n_components_assessed=n_assessed, |
| 168 | + n_superior_verdicts=n_superior, |
| 169 | + pmc_grade=grade, |
| 170 | + pmc_verdict=verdict, |
| 171 | + dry_lab_only=True, |
| 172 | + limitations=limitations, |
| 173 | + created_at=created_at, |
| 174 | + ) |
| 175 | + validate_pipeline_maturity_certificate(pmc) |
| 176 | + return pmc |
| 177 | + |
| 178 | + |
| 179 | +def format_pipeline_maturity_certificate(pmc: PipelineMaturityCertificate) -> str: |
| 180 | + lines = [ |
| 181 | + f"Pipeline Maturity Certificate — {pmc.pmc_id}", |
| 182 | + f"Pipeline: {pmc.pipeline_version}", |
| 183 | + f"Grade: {pmc.pmc_grade} | Verdict: {pmc.pmc_verdict}", |
| 184 | + f"Components assessed: {pmc.n_components_assessed}/{len(REQUIRED_PMC_COMPONENTS)} " |
| 185 | + f"| Superior verdicts: {pmc.n_superior_verdicts}", |
| 186 | + ] |
| 187 | + lines.append("Component checks:") |
| 188 | + for check in pmc.component_checks: |
| 189 | + status = "ASSESSED" if check.artifact_id else "NOT_ASSESSED" |
| 190 | + contrib = " [contributes]" if check.contributes_to_grade else "" |
| 191 | + lines.append( |
| 192 | + f" {check.component_type}: {status} " |
| 193 | + f"verdict={check.verdict or 'N/A'}{contrib}" |
| 194 | + ) |
| 195 | + lines.append(f"Created: {pmc.created_at}") |
| 196 | + lines.append(f"Limitations: {'; '.join(pmc.limitations)}") |
| 197 | + lines.append(f"dry_lab_only: {pmc.dry_lab_only}") |
| 198 | + return "\n".join(lines) |
0 commit comments