|
| 1 | +"""CBR- cheap baseline comparison record schema. |
| 2 | +
|
| 3 | +Structured record comparing the pipeline's performance metric against a |
| 4 | +cheap baseline (charge-only, length-only, random) on the same candidate set. |
| 5 | +Forces every performance claim to name the specific baseline it beat and |
| 6 | +provide a machine-readable verdict. No performance claim is credible without |
| 7 | +this record. |
| 8 | +""" |
| 9 | + |
| 10 | +from __future__ import annotations |
| 11 | + |
| 12 | +from dataclasses import dataclass |
| 13 | + |
| 14 | +VALID_CBR_VERDICTS: frozenset[str] = frozenset({ |
| 15 | + "pipeline_superior", |
| 16 | + "tied", |
| 17 | + "baseline_superior", |
| 18 | + "insufficient_data", |
| 19 | +}) |
| 20 | + |
| 21 | +VALID_BASELINE_METHODS: frozenset[str] = frozenset({ |
| 22 | + "charge_only_rank", |
| 23 | + "length_only_rank", |
| 24 | + "random_selection", |
| 25 | + "charge_length_combined", |
| 26 | + "hydrophobicity_only_rank", |
| 27 | +}) |
| 28 | + |
| 29 | +VALID_CBR_METRICS: frozenset[str] = frozenset({ |
| 30 | + "auroc", |
| 31 | + "hit_rate", |
| 32 | + "top_k_precision", |
| 33 | + "ndcg", |
| 34 | +}) |
| 35 | + |
| 36 | +SUPERIORITY_THRESHOLD: float = 0.05 |
| 37 | +INFERIORITY_THRESHOLD: float = -0.05 |
| 38 | +MIN_SAMPLE_SIZE: int = 5 |
| 39 | + |
| 40 | + |
| 41 | +@dataclass |
| 42 | +class CheapBaselineComparisonRecord: |
| 43 | + cbr_id: str |
| 44 | + pipeline_version: str |
| 45 | + baseline_method: str |
| 46 | + metric_name: str |
| 47 | + pipeline_metric_value: float |
| 48 | + baseline_metric_value: float |
| 49 | + metric_delta: float |
| 50 | + n_candidates_evaluated: int |
| 51 | + pre_registered_threshold: float |
| 52 | + cbr_verdict: str |
| 53 | + comparison_notes: str |
| 54 | + dry_lab_only: bool |
| 55 | + limitations: list[str] |
| 56 | + created_at: str |
| 57 | + |
| 58 | + |
| 59 | +def validate_cheap_baseline_comparison_record( |
| 60 | + cbr: CheapBaselineComparisonRecord, |
| 61 | +) -> None: |
| 62 | + if not cbr.cbr_id.startswith("CBR-"): |
| 63 | + raise ValueError(f"cbr_id must start with 'CBR-': {cbr.cbr_id!r}") |
| 64 | + if not cbr.pipeline_version: |
| 65 | + raise ValueError("pipeline_version must be non-empty") |
| 66 | + if cbr.baseline_method not in VALID_BASELINE_METHODS: |
| 67 | + raise ValueError( |
| 68 | + f"baseline_method {cbr.baseline_method!r} not in VALID_BASELINE_METHODS" |
| 69 | + ) |
| 70 | + if cbr.metric_name not in VALID_CBR_METRICS: |
| 71 | + raise ValueError( |
| 72 | + f"metric_name {cbr.metric_name!r} not in VALID_CBR_METRICS" |
| 73 | + ) |
| 74 | + if not (0.0 <= cbr.pipeline_metric_value <= 1.0): |
| 75 | + raise ValueError( |
| 76 | + f"pipeline_metric_value must be in [0, 1]: {cbr.pipeline_metric_value}" |
| 77 | + ) |
| 78 | + if not (0.0 <= cbr.baseline_metric_value <= 1.0): |
| 79 | + raise ValueError( |
| 80 | + f"baseline_metric_value must be in [0, 1]: {cbr.baseline_metric_value}" |
| 81 | + ) |
| 82 | + expected_delta = round(cbr.pipeline_metric_value - cbr.baseline_metric_value, 6) |
| 83 | + if abs(cbr.metric_delta - expected_delta) > 1e-5: |
| 84 | + raise ValueError( |
| 85 | + f"metric_delta mismatch: expected {expected_delta}, got {cbr.metric_delta}" |
| 86 | + ) |
| 87 | + if cbr.n_candidates_evaluated < 0: |
| 88 | + raise ValueError("n_candidates_evaluated must be non-negative") |
| 89 | + if cbr.cbr_verdict not in VALID_CBR_VERDICTS: |
| 90 | + raise ValueError( |
| 91 | + f"cbr_verdict {cbr.cbr_verdict!r} not in VALID_CBR_VERDICTS" |
| 92 | + ) |
| 93 | + if not cbr.dry_lab_only: |
| 94 | + raise ValueError("dry_lab_only must be True") |
| 95 | + if not cbr.limitations: |
| 96 | + raise ValueError("limitations must be non-empty") |
| 97 | + if not cbr.created_at: |
| 98 | + raise ValueError("created_at must be non-empty") |
| 99 | + |
| 100 | + |
| 101 | +def _compute_verdict( |
| 102 | + delta: float, |
| 103 | + n_candidates: int, |
| 104 | + superiority_threshold: float, |
| 105 | +) -> str: |
| 106 | + if n_candidates < MIN_SAMPLE_SIZE: |
| 107 | + return "insufficient_data" |
| 108 | + if delta >= superiority_threshold: |
| 109 | + return "pipeline_superior" |
| 110 | + if delta <= INFERIORITY_THRESHOLD: |
| 111 | + return "baseline_superior" |
| 112 | + return "tied" |
| 113 | + |
| 114 | + |
| 115 | +def build_cheap_baseline_comparison_record( |
| 116 | + *, |
| 117 | + cbr_id: str, |
| 118 | + pipeline_version: str, |
| 119 | + baseline_method: str, |
| 120 | + metric_name: str, |
| 121 | + pipeline_metric_value: float, |
| 122 | + baseline_metric_value: float, |
| 123 | + n_candidates_evaluated: int, |
| 124 | + pre_registered_threshold: float = SUPERIORITY_THRESHOLD, |
| 125 | + comparison_notes: str = "", |
| 126 | + limitations: list[str], |
| 127 | + created_at: str, |
| 128 | +) -> CheapBaselineComparisonRecord: |
| 129 | + """Build a CheapBaselineComparisonRecord. |
| 130 | +
|
| 131 | + metric_delta = pipeline_metric_value - baseline_metric_value (auto-computed). |
| 132 | + cbr_verdict is auto-derived from delta vs pre_registered_threshold and n_candidates. |
| 133 | + """ |
| 134 | + delta = round(pipeline_metric_value - baseline_metric_value, 6) |
| 135 | + verdict = _compute_verdict(delta, n_candidates_evaluated, pre_registered_threshold) |
| 136 | + cbr = CheapBaselineComparisonRecord( |
| 137 | + cbr_id=cbr_id, |
| 138 | + pipeline_version=pipeline_version, |
| 139 | + baseline_method=baseline_method, |
| 140 | + metric_name=metric_name, |
| 141 | + pipeline_metric_value=float(pipeline_metric_value), |
| 142 | + baseline_metric_value=float(baseline_metric_value), |
| 143 | + metric_delta=delta, |
| 144 | + n_candidates_evaluated=n_candidates_evaluated, |
| 145 | + pre_registered_threshold=float(pre_registered_threshold), |
| 146 | + cbr_verdict=verdict, |
| 147 | + comparison_notes=comparison_notes, |
| 148 | + dry_lab_only=True, |
| 149 | + limitations=limitations, |
| 150 | + created_at=created_at, |
| 151 | + ) |
| 152 | + validate_cheap_baseline_comparison_record(cbr) |
| 153 | + return cbr |
| 154 | + |
| 155 | + |
| 156 | +def format_cheap_baseline_comparison_record( |
| 157 | + cbr: CheapBaselineComparisonRecord, |
| 158 | +) -> str: |
| 159 | + lines = [ |
| 160 | + f"Cheap Baseline Comparison Record — {cbr.cbr_id}", |
| 161 | + f"Pipeline: {cbr.pipeline_version}", |
| 162 | + f"Baseline: {cbr.baseline_method} | Metric: {cbr.metric_name}", |
| 163 | + f"Verdict: {cbr.cbr_verdict}", |
| 164 | + f"Pipeline {cbr.metric_name}: {cbr.pipeline_metric_value:.4f}", |
| 165 | + f"Baseline {cbr.metric_name}: {cbr.baseline_metric_value:.4f}", |
| 166 | + f"Delta: {cbr.metric_delta:+.4f} " |
| 167 | + f"(threshold: {cbr.pre_registered_threshold:+.4f})", |
| 168 | + f"Candidates evaluated: {cbr.n_candidates_evaluated}", |
| 169 | + ] |
| 170 | + if cbr.comparison_notes: |
| 171 | + lines.append(f"Notes: {cbr.comparison_notes}") |
| 172 | + lines.append(f"Created: {cbr.created_at}") |
| 173 | + lines.append(f"Limitations: {'; '.join(cbr.limitations)}") |
| 174 | + lines.append(f"dry_lab_only: {cbr.dry_lab_only}") |
| 175 | + return "\n".join(lines) |
0 commit comments