From a45e92ccb6592f2437667c09b49543887711f335 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Fri, 10 Jul 2026 16:06:22 +0700 Subject: [PATCH] feat: Phase Y Y1 cheap baseline comparison record schema (CBR-) -- machine-verifiable verdict comparing pipeline vs cheap baseline; prevents performance claims without named baseline (#854) VALID_CBR_VERDICTS (4: pipeline_superior/tied/baseline_superior/insufficient_data), VALID_BASELINE_METHODS (5: charge_only_rank/length_only_rank/random_selection/charge_length_combined/hydrophobicity_only_rank), VALID_CBR_METRICS (4: auroc/hit_rate/top_k_precision/ndcg). SUPERIORITY_THRESHOLD=0.05, MIN_SAMPLE_SIZE=5; metric_delta auto-computed. Adds Phase Y baseline-vs-pipeline accountability section to NEXT_100_PR_MAP.md. 62 tests; BASELINE updated 10498->10560. --- docs/research/NEXT_100_PR_MAP.md | 12 + .../cheap_baseline_comparison_record.py | 175 +++++++++ .../test_cheap_baseline_comparison_record.py | 340 ++++++++++++++++++ tests/test_test_count_regression.py | 2 +- 4 files changed, 528 insertions(+), 1 deletion(-) create mode 100644 src/openamp_foundry/evidence/cheap_baseline_comparison_record.py create mode 100644 tests/evidence/test_cheap_baseline_comparison_record.py diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index b367bfdf..64bd0a65 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -293,3 +293,15 @@ Track whether the pipeline actually improves across batches by capturing per-bat | X3 | Add learning progress report schema (LPR-) (complete). — src/openamp_foundry/evidence/learning_progress_report.py: VALID_LPR_VERDICTS (4: learning_confirmed/learning_inconclusive/no_learning_signal/insufficient_data), VALID_FEATURE_PREDICTIVITY (3: predictive/not_predictive/uncertain), VALID_FEATURE_CATEGORIES (8); FeatureLearningEntry helper; verdict auto-computed: learning_confirmed (n_pred>n_non), learning_inconclusive (equal+>0), no_learning_signal (n_pred None: + if not cbr.cbr_id.startswith("CBR-"): + raise ValueError(f"cbr_id must start with 'CBR-': {cbr.cbr_id!r}") + if not cbr.pipeline_version: + raise ValueError("pipeline_version must be non-empty") + if cbr.baseline_method not in VALID_BASELINE_METHODS: + raise ValueError( + f"baseline_method {cbr.baseline_method!r} not in VALID_BASELINE_METHODS" + ) + if cbr.metric_name not in VALID_CBR_METRICS: + raise ValueError( + f"metric_name {cbr.metric_name!r} not in VALID_CBR_METRICS" + ) + if not (0.0 <= cbr.pipeline_metric_value <= 1.0): + raise ValueError( + f"pipeline_metric_value must be in [0, 1]: {cbr.pipeline_metric_value}" + ) + if not (0.0 <= cbr.baseline_metric_value <= 1.0): + raise ValueError( + f"baseline_metric_value must be in [0, 1]: {cbr.baseline_metric_value}" + ) + expected_delta = round(cbr.pipeline_metric_value - cbr.baseline_metric_value, 6) + if abs(cbr.metric_delta - expected_delta) > 1e-5: + raise ValueError( + f"metric_delta mismatch: expected {expected_delta}, got {cbr.metric_delta}" + ) + if cbr.n_candidates_evaluated < 0: + raise ValueError("n_candidates_evaluated must be non-negative") + if cbr.cbr_verdict not in VALID_CBR_VERDICTS: + raise ValueError( + f"cbr_verdict {cbr.cbr_verdict!r} not in VALID_CBR_VERDICTS" + ) + if not cbr.dry_lab_only: + raise ValueError("dry_lab_only must be True") + if not cbr.limitations: + raise ValueError("limitations must be non-empty") + if not cbr.created_at: + raise ValueError("created_at must be non-empty") + + +def _compute_verdict( + delta: float, + n_candidates: int, + superiority_threshold: float, +) -> str: + if n_candidates < MIN_SAMPLE_SIZE: + return "insufficient_data" + if delta >= superiority_threshold: + return "pipeline_superior" + if delta <= INFERIORITY_THRESHOLD: + return "baseline_superior" + return "tied" + + +def build_cheap_baseline_comparison_record( + *, + cbr_id: str, + pipeline_version: str, + baseline_method: str, + metric_name: str, + pipeline_metric_value: float, + baseline_metric_value: float, + n_candidates_evaluated: int, + pre_registered_threshold: float = SUPERIORITY_THRESHOLD, + comparison_notes: str = "", + limitations: list[str], + created_at: str, +) -> CheapBaselineComparisonRecord: + """Build a CheapBaselineComparisonRecord. + + metric_delta = pipeline_metric_value - baseline_metric_value (auto-computed). + cbr_verdict is auto-derived from delta vs pre_registered_threshold and n_candidates. + """ + delta = round(pipeline_metric_value - baseline_metric_value, 6) + verdict = _compute_verdict(delta, n_candidates_evaluated, pre_registered_threshold) + cbr = CheapBaselineComparisonRecord( + cbr_id=cbr_id, + pipeline_version=pipeline_version, + baseline_method=baseline_method, + metric_name=metric_name, + pipeline_metric_value=float(pipeline_metric_value), + baseline_metric_value=float(baseline_metric_value), + metric_delta=delta, + n_candidates_evaluated=n_candidates_evaluated, + pre_registered_threshold=float(pre_registered_threshold), + cbr_verdict=verdict, + comparison_notes=comparison_notes, + dry_lab_only=True, + limitations=limitations, + created_at=created_at, + ) + validate_cheap_baseline_comparison_record(cbr) + return cbr + + +def format_cheap_baseline_comparison_record( + cbr: CheapBaselineComparisonRecord, +) -> str: + lines = [ + f"Cheap Baseline Comparison Record — {cbr.cbr_id}", + f"Pipeline: {cbr.pipeline_version}", + f"Baseline: {cbr.baseline_method} | Metric: {cbr.metric_name}", + f"Verdict: {cbr.cbr_verdict}", + f"Pipeline {cbr.metric_name}: {cbr.pipeline_metric_value:.4f}", + f"Baseline {cbr.metric_name}: {cbr.baseline_metric_value:.4f}", + f"Delta: {cbr.metric_delta:+.4f} " + f"(threshold: {cbr.pre_registered_threshold:+.4f})", + f"Candidates evaluated: {cbr.n_candidates_evaluated}", + ] + if cbr.comparison_notes: + lines.append(f"Notes: {cbr.comparison_notes}") + lines.append(f"Created: {cbr.created_at}") + lines.append(f"Limitations: {'; '.join(cbr.limitations)}") + lines.append(f"dry_lab_only: {cbr.dry_lab_only}") + return "\n".join(lines) diff --git a/tests/evidence/test_cheap_baseline_comparison_record.py b/tests/evidence/test_cheap_baseline_comparison_record.py new file mode 100644 index 00000000..e2570663 --- /dev/null +++ b/tests/evidence/test_cheap_baseline_comparison_record.py @@ -0,0 +1,340 @@ +"""Tests for CBR- cheap baseline comparison record schema.""" + +import pytest +from openamp_foundry.evidence.cheap_baseline_comparison_record import ( + CheapBaselineComparisonRecord, + VALID_CBR_VERDICTS, + VALID_BASELINE_METHODS, + VALID_CBR_METRICS, + SUPERIORITY_THRESHOLD, + INFERIORITY_THRESHOLD, + MIN_SAMPLE_SIZE, + build_cheap_baseline_comparison_record, + format_cheap_baseline_comparison_record, + validate_cheap_baseline_comparison_record, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _build(**kwargs): + defaults = dict( + cbr_id="CBR-001", + pipeline_version="v1.0", + baseline_method="charge_only_rank", + metric_name="auroc", + pipeline_metric_value=0.75, + baseline_metric_value=0.60, + n_candidates_evaluated=20, + limitations=["dry-lab only"], + created_at="2026-07-10", + ) + defaults.update(kwargs) + return build_cheap_baseline_comparison_record(**defaults) + + +# --------------------------------------------------------------------------- +# 1. Constants +# --------------------------------------------------------------------------- + + +def test_valid_cbr_verdicts_is_frozenset(): + assert isinstance(VALID_CBR_VERDICTS, frozenset) + + +def test_valid_cbr_verdicts_contains_pipeline_superior(): + assert "pipeline_superior" in VALID_CBR_VERDICTS + + +def test_valid_cbr_verdicts_contains_tied(): + assert "tied" in VALID_CBR_VERDICTS + + +def test_valid_cbr_verdicts_contains_baseline_superior(): + assert "baseline_superior" in VALID_CBR_VERDICTS + + +def test_valid_cbr_verdicts_contains_insufficient_data(): + assert "insufficient_data" in VALID_CBR_VERDICTS + + +def test_valid_baseline_methods_is_frozenset(): + assert isinstance(VALID_BASELINE_METHODS, frozenset) + + +def test_valid_baseline_methods_contains_charge_only(): + assert "charge_only_rank" in VALID_BASELINE_METHODS + + +def test_valid_baseline_methods_contains_length_only(): + assert "length_only_rank" in VALID_BASELINE_METHODS + + +def test_valid_baseline_methods_contains_random(): + assert "random_selection" in VALID_BASELINE_METHODS + + +def test_valid_cbr_metrics_is_frozenset(): + assert isinstance(VALID_CBR_METRICS, frozenset) + + +def test_valid_cbr_metrics_contains_auroc(): + assert "auroc" in VALID_CBR_METRICS + + +def test_valid_cbr_metrics_contains_hit_rate(): + assert "hit_rate" in VALID_CBR_METRICS + + +def test_superiority_threshold(): + assert SUPERIORITY_THRESHOLD == 0.05 + + +def test_inferiority_threshold(): + assert INFERIORITY_THRESHOLD == -0.05 + + +def test_min_sample_size(): + assert MIN_SAMPLE_SIZE == 5 + + +# --------------------------------------------------------------------------- +# 2. build – happy paths +# --------------------------------------------------------------------------- + + +def test_build_returns_cheap_baseline_comparison_record(): + assert isinstance(_build(), CheapBaselineComparisonRecord) + + +def test_build_cbr_id_stored(): + assert _build().cbr_id == "CBR-001" + + +def test_build_pipeline_version_stored(): + assert _build().pipeline_version == "v1.0" + + +def test_build_dry_lab_only_true(): + assert _build().dry_lab_only is True + + +def test_build_pipeline_superior_verdict(): + r = _build(pipeline_metric_value=0.75, baseline_metric_value=0.60) + assert r.cbr_verdict == "pipeline_superior" + + +def test_build_baseline_superior_verdict(): + r = _build(pipeline_metric_value=0.50, baseline_metric_value=0.70) + assert r.cbr_verdict == "baseline_superior" + + +def test_build_tied_verdict(): + r = _build(pipeline_metric_value=0.62, baseline_metric_value=0.60) + assert r.cbr_verdict == "tied" + + +def test_build_insufficient_data_few_candidates(): + r = _build(n_candidates_evaluated=3) + assert r.cbr_verdict == "insufficient_data" + + +def test_build_metric_delta_auto_computed(): + r = _build(pipeline_metric_value=0.75, baseline_metric_value=0.60) + assert abs(r.metric_delta - 0.15) < 1e-5 + + +def test_build_negative_delta(): + r = _build(pipeline_metric_value=0.50, baseline_metric_value=0.70) + assert r.metric_delta < 0 + + +def test_build_baseline_method_stored(): + assert _build().baseline_method == "charge_only_rank" + + +def test_build_metric_name_stored(): + assert _build().metric_name == "auroc" + + +def test_build_pipeline_metric_stored(): + assert abs(_build().pipeline_metric_value - 0.75) < 1e-6 + + +def test_build_baseline_metric_stored(): + assert abs(_build().baseline_metric_value - 0.60) < 1e-6 + + +def test_build_n_candidates_stored(): + assert _build().n_candidates_evaluated == 20 + + +def test_build_pre_registered_threshold_default(): + assert abs(_build().pre_registered_threshold - SUPERIORITY_THRESHOLD) < 1e-6 + + +def test_build_pre_registered_threshold_custom(): + r = _build(pre_registered_threshold=0.10) + assert abs(r.pre_registered_threshold - 0.10) < 1e-6 + + +def test_build_comparison_notes_stored(): + r = _build(comparison_notes="charge baseline used APD3") + assert r.comparison_notes == "charge baseline used APD3" + + +def test_build_comparison_notes_default_empty(): + assert _build().comparison_notes == "" + + +def test_build_limitations_stored(): + assert _build().limitations == ["dry-lab only"] + + +def test_build_created_at_stored(): + assert _build().created_at == "2026-07-10" + + +def test_build_pipeline_superior_at_exact_threshold(): + # delta == threshold → pipeline_superior (>= comparison) + r = _build(pipeline_metric_value=0.65, baseline_metric_value=0.60) + assert r.cbr_verdict == "pipeline_superior" + + +def test_build_pipeline_superior_just_over_threshold(): + r = _build( + pipeline_metric_value=0.651, + baseline_metric_value=0.60, + pre_registered_threshold=0.05, + ) + assert r.cbr_verdict == "pipeline_superior" + + +def test_build_random_selection_baseline(): + r = _build(baseline_method="random_selection") + assert r.baseline_method == "random_selection" + + +def test_build_hit_rate_metric(): + r = _build(metric_name="hit_rate") + assert r.metric_name == "hit_rate" + + +def test_build_five_candidates_is_not_insufficient(): + r = _build(n_candidates_evaluated=5) + assert r.cbr_verdict != "insufficient_data" + + +# --------------------------------------------------------------------------- +# 3. validate – rejection cases +# --------------------------------------------------------------------------- + + +def test_validate_rejects_bad_cbr_id_prefix(): + with pytest.raises(ValueError, match="CBR-"): + _build(cbr_id="BAD-001") + + +def test_validate_rejects_empty_pipeline_version(): + with pytest.raises(ValueError): + _build(pipeline_version="") + + +def test_validate_rejects_invalid_baseline_method(): + with pytest.raises(ValueError, match="baseline_method"): + _build(baseline_method="UNKNOWN") + + +def test_validate_rejects_invalid_metric_name(): + with pytest.raises(ValueError, match="metric_name"): + _build(metric_name="UNKNOWN") + + +def test_validate_rejects_pipeline_metric_above_one(): + with pytest.raises(ValueError, match="pipeline_metric_value"): + _build(pipeline_metric_value=1.5) + + +def test_validate_rejects_pipeline_metric_below_zero(): + with pytest.raises(ValueError, match="pipeline_metric_value"): + _build(pipeline_metric_value=-0.1) + + +def test_validate_rejects_baseline_metric_above_one(): + with pytest.raises(ValueError, match="baseline_metric_value"): + _build(baseline_metric_value=1.5) + + +def test_validate_rejects_metric_delta_mismatch(): + cbr = _build() + cbr.metric_delta = 99.0 + with pytest.raises(ValueError, match="metric_delta"): + validate_cheap_baseline_comparison_record(cbr) + + +def test_validate_rejects_negative_n_candidates(): + with pytest.raises(ValueError, match="n_candidates_evaluated"): + _build(n_candidates_evaluated=-1) + + +def test_validate_rejects_invalid_verdict(): + cbr = _build() + cbr.cbr_verdict = "UNKNOWN" + with pytest.raises(ValueError, match="cbr_verdict"): + validate_cheap_baseline_comparison_record(cbr) + + +def test_validate_rejects_dry_lab_only_false(): + cbr = _build() + cbr.dry_lab_only = False + with pytest.raises(ValueError, match="dry_lab_only"): + validate_cheap_baseline_comparison_record(cbr) + + +def test_validate_rejects_empty_limitations(): + with pytest.raises(ValueError, match="limitations"): + _build(limitations=[]) + + +def test_validate_rejects_empty_created_at(): + with pytest.raises(ValueError): + _build(created_at="") + + +# --------------------------------------------------------------------------- +# 4. format +# --------------------------------------------------------------------------- + + +def test_format_contains_cbr_id(): + assert "CBR-001" in format_cheap_baseline_comparison_record(_build()) + + +def test_format_contains_pipeline_version(): + assert "v1.0" in format_cheap_baseline_comparison_record(_build()) + + +def test_format_contains_baseline_method(): + assert "charge_only_rank" in format_cheap_baseline_comparison_record(_build()) + + +def test_format_contains_metric_name(): + assert "auroc" in format_cheap_baseline_comparison_record(_build()) + + +def test_format_contains_verdict(): + assert "pipeline_superior" in format_cheap_baseline_comparison_record(_build()) + + +def test_format_contains_limitations(): + assert "dry-lab only" in format_cheap_baseline_comparison_record(_build()) + + +def test_format_contains_dry_lab_only(): + assert "dry_lab_only: True" in format_cheap_baseline_comparison_record(_build()) + + +def test_format_is_string(): + assert isinstance(format_cheap_baseline_comparison_record(_build()), str) diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 3ee3db32..60bf2db2 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -4,7 +4,7 @@ import sys import math -BASELINE = 10498 +BASELINE = 10560 def test_test_count_regression():