diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index b5d86e19..1ac13382 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -290,6 +290,6 @@ Track whether the pipeline actually improves across batches by capturing per-bat |----|------|----------------|----------| | X1 | Add multi-batch learning record schema (MBL-) (complete). — src/openamp_foundry/evidence/multi_batch_learning_record.py: VALID_MBL_QUALITY_GRADES (5: A-D/N/A), VALID_BATCH_LEARNING_STATUSES (3), GRADE_A_HIT_RATE=0.40; hit_rate auto-computed from n_confirmed_hits/n_candidates_tested; grade auto-derived; no_wet_lab_data forces N/A grade; whr_ids list; dry_lab_only=True; 61 tests in tests/evidence/test_multi_batch_learning_record.py. | Per-batch snapshot of prediction quality (hit rate, AUROC, n_confirmed_hits) after wet-lab feedback; enables cross-batch comparison of whether the pipeline is learning; feeds into calibration improvement tracker. | C | | X2 | Add calibration improvement tracker schema (CIT-) (complete). — src/openamp_foundry/evidence/calibration_improvement_tracker.py: VALID_CIT_TREND_DIRECTIONS (4: improving/stable/degrading/insufficient_data), VALID_CIT_SUMMARY_GRADES (5: A-D/N/A), MIN_BATCHES_FOR_TREND=2, IMPROVEMENT_THRESHOLD=0.05; BatchHitRateEntry helper; trend auto-computed from first→latest hit_rate delta; insufficient_data forces N/A grade; dry_lab_only=True; 49 tests. | Aggregates MBL records across batches; computes hit-rate trend direction (improving/stable/degrading/insufficient_data); minimum 2 batches required; flags when calibration is not producing measurable improvement. | C | -| X3 | Add learning progress report schema (LPR-). | Human-readable summary of what the pipeline has learned from all batches to date; references CIT- for trend data; includes which candidate features proved predictive vs not; links to calibration decision logs. | C | +| X3 | Add learning progress report schema (LPR-) (complete). — src/openamp_foundry/evidence/learning_progress_report.py: VALID_LPR_VERDICTS (4: learning_confirmed/learning_inconclusive/no_learning_signal/insufficient_data), VALID_FEATURE_PREDICTIVITY (3: predictive/not_predictive/uncertain), VALID_FEATURE_CATEGORIES (8); FeatureLearningEntry helper; verdict auto-computed: learning_confirmed (n_pred>n_non), learning_inconclusive (equal+>0), no_learning_signal (n_pred None: + if not lpr.lpr_id.startswith("LPR-"): + raise ValueError(f"lpr_id must start with 'LPR-': {lpr.lpr_id!r}") + if not lpr.pipeline_version: + raise ValueError("pipeline_version must be non-empty") + if not lpr.cit_id.startswith("CIT-"): + raise ValueError(f"cit_id must start with 'CIT-': {lpr.cit_id!r}") + if lpr.n_batches_summarized < 0: + raise ValueError("n_batches_summarized must be non-negative") + for entry in lpr.feature_entries: + if entry.feature_category not in VALID_FEATURE_CATEGORIES: + raise ValueError( + f"feature_category {entry.feature_category!r} not in VALID_FEATURE_CATEGORIES" + ) + if entry.predictivity not in VALID_FEATURE_PREDICTIVITY: + raise ValueError( + f"predictivity {entry.predictivity!r} not in VALID_FEATURE_PREDICTIVITY" + ) + n_pred = sum(1 for e in lpr.feature_entries if e.predictivity == "predictive") + n_non = sum(1 for e in lpr.feature_entries if e.predictivity == "not_predictive") + if lpr.n_predictive_features != n_pred: + raise ValueError("n_predictive_features mismatch") + if lpr.n_non_predictive_features != n_non: + raise ValueError("n_non_predictive_features mismatch") + if lpr.lpr_verdict not in VALID_LPR_VERDICTS: + raise ValueError( + f"lpr_verdict {lpr.lpr_verdict!r} not in VALID_LPR_VERDICTS" + ) + if not lpr.key_findings: + raise ValueError("key_findings must be non-empty") + if not lpr.dry_lab_only: + raise ValueError("dry_lab_only must be True") + if not lpr.limitations: + raise ValueError("limitations must be non-empty") + if not lpr.created_at: + raise ValueError("created_at must be non-empty") + + +def _compute_verdict( + n_batches: int, + n_predictive: int, + n_non_predictive: int, +) -> str: + if n_batches == 0: + return "insufficient_data" + if n_predictive == 0 and n_non_predictive == 0: + return "insufficient_data" + if n_predictive > n_non_predictive: + return "learning_confirmed" + if n_predictive == n_non_predictive and n_predictive > 0: + return "learning_inconclusive" + return "no_learning_signal" + + +def build_learning_progress_report( + *, + lpr_id: str, + pipeline_version: str, + cit_id: str, + n_batches_summarized: int, + feature_entry_dicts: list[dict], + key_findings: list[str], + limitations: list[str], + created_at: str, +) -> LearningProgressReport: + """Build a LearningProgressReport. + + feature_entry_dicts: list of dicts with keys: + feature_category, predictivity, evidence_summary (optional, default "") + """ + entries = [ + FeatureLearningEntry( + feature_category=d["feature_category"], + predictivity=d["predictivity"], + evidence_summary=d.get("evidence_summary", ""), + ) + for d in feature_entry_dicts + ] + n_pred = sum(1 for e in entries if e.predictivity == "predictive") + n_non = sum(1 for e in entries if e.predictivity == "not_predictive") + verdict = _compute_verdict(n_batches_summarized, n_pred, n_non) + lpr = LearningProgressReport( + lpr_id=lpr_id, + pipeline_version=pipeline_version, + cit_id=cit_id, + n_batches_summarized=n_batches_summarized, + feature_entries=entries, + n_predictive_features=n_pred, + n_non_predictive_features=n_non, + lpr_verdict=verdict, + key_findings=list(key_findings), + dry_lab_only=True, + limitations=limitations, + created_at=created_at, + ) + validate_learning_progress_report(lpr) + return lpr + + +def format_learning_progress_report(lpr: LearningProgressReport) -> str: + lines = [ + f"Learning Progress Report — {lpr.lpr_id}", + f"Pipeline: {lpr.pipeline_version} | CIT: {lpr.cit_id}", + f"Verdict: {lpr.lpr_verdict}", + f"Batches summarized: {lpr.n_batches_summarized}", + f"Features: {lpr.n_predictive_features} predictive, " + f"{lpr.n_non_predictive_features} not predictive", + ] + if lpr.feature_entries: + lines.append("Feature categories:") + for entry in lpr.feature_entries: + lines.append(f" {entry.feature_category}: {entry.predictivity}") + lines.append("Key findings:") + for finding in lpr.key_findings: + lines.append(f" - {finding}") + lines.append(f"Created: {lpr.created_at}") + lines.append(f"Limitations: {'; '.join(lpr.limitations)}") + lines.append(f"dry_lab_only: {lpr.dry_lab_only}") + return "\n".join(lines) diff --git a/tests/evidence/test_learning_progress_report.py b/tests/evidence/test_learning_progress_report.py new file mode 100644 index 00000000..99e2ee37 --- /dev/null +++ b/tests/evidence/test_learning_progress_report.py @@ -0,0 +1,332 @@ +"""Tests for LPR- learning progress report schema.""" + +import pytest +from openamp_foundry.evidence.learning_progress_report import ( + LearningProgressReport, + FeatureLearningEntry, + VALID_LPR_VERDICTS, + VALID_FEATURE_PREDICTIVITY, + VALID_FEATURE_CATEGORIES, + build_learning_progress_report, + format_learning_progress_report, + validate_learning_progress_report, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_PREDICTIVE_FEATURES = [ + {"feature_category": "charge", "predictivity": "predictive", "evidence_summary": "strong signal"}, + {"feature_category": "hydrophobicity", "predictivity": "predictive"}, + {"feature_category": "length", "predictivity": "not_predictive"}, +] + + +def _build(**kwargs): + defaults = dict( + lpr_id="LPR-001", + pipeline_version="v1.0", + cit_id="CIT-001", + n_batches_summarized=3, + feature_entry_dicts=_PREDICTIVE_FEATURES, + key_findings=["charge is the strongest predictor", "length not informative"], + limitations=["dry-lab only"], + created_at="2026-07-10", + ) + defaults.update(kwargs) + return build_learning_progress_report(**defaults) + + +# --------------------------------------------------------------------------- +# 1. Constants +# --------------------------------------------------------------------------- + + +def test_valid_lpr_verdicts_is_frozenset(): + assert isinstance(VALID_LPR_VERDICTS, frozenset) + + +def test_valid_lpr_verdicts_contains_learning_confirmed(): + assert "learning_confirmed" in VALID_LPR_VERDICTS + + +def test_valid_lpr_verdicts_contains_learning_inconclusive(): + assert "learning_inconclusive" in VALID_LPR_VERDICTS + + +def test_valid_lpr_verdicts_contains_no_learning_signal(): + assert "no_learning_signal" in VALID_LPR_VERDICTS + + +def test_valid_lpr_verdicts_contains_insufficient_data(): + assert "insufficient_data" in VALID_LPR_VERDICTS + + +def test_valid_feature_predictivity_is_frozenset(): + assert isinstance(VALID_FEATURE_PREDICTIVITY, frozenset) + + +def test_valid_feature_predictivity_contains_predictive(): + assert "predictive" in VALID_FEATURE_PREDICTIVITY + + +def test_valid_feature_predictivity_contains_not_predictive(): + assert "not_predictive" in VALID_FEATURE_PREDICTIVITY + + +def test_valid_feature_predictivity_contains_uncertain(): + assert "uncertain" in VALID_FEATURE_PREDICTIVITY + + +def test_valid_feature_categories_is_frozenset(): + assert isinstance(VALID_FEATURE_CATEGORIES, frozenset) + + +def test_valid_feature_categories_contains_charge(): + assert "charge" in VALID_FEATURE_CATEGORIES + + +def test_valid_feature_categories_contains_hydrophobicity(): + assert "hydrophobicity" in VALID_FEATURE_CATEGORIES + + +def test_valid_feature_categories_contains_length(): + assert "length" in VALID_FEATURE_CATEGORIES + + +# --------------------------------------------------------------------------- +# 2. build – happy paths +# --------------------------------------------------------------------------- + + +def test_build_returns_learning_progress_report(): + assert isinstance(_build(), LearningProgressReport) + + +def test_build_lpr_id_stored(): + assert _build().lpr_id == "LPR-001" + + +def test_build_pipeline_version_stored(): + assert _build().pipeline_version == "v1.0" + + +def test_build_cit_id_stored(): + assert _build().cit_id == "CIT-001" + + +def test_build_dry_lab_only_true(): + assert _build().dry_lab_only is True + + +def test_build_n_batches_summarized_stored(): + assert _build().n_batches_summarized == 3 + + +def test_build_n_predictive_features_counted(): + assert _build().n_predictive_features == 2 + + +def test_build_n_non_predictive_features_counted(): + assert _build().n_non_predictive_features == 1 + + +def test_build_learning_confirmed_when_more_predictive(): + assert _build().lpr_verdict == "learning_confirmed" + + +def test_build_no_learning_signal_when_more_non_predictive(): + features = [ + {"feature_category": "charge", "predictivity": "not_predictive"}, + {"feature_category": "hydrophobicity", "predictivity": "not_predictive"}, + {"feature_category": "length", "predictivity": "predictive"}, + ] + r = _build(feature_entry_dicts=features) + assert r.lpr_verdict == "no_learning_signal" + + +def test_build_learning_inconclusive_when_equal(): + features = [ + {"feature_category": "charge", "predictivity": "predictive"}, + {"feature_category": "length", "predictivity": "not_predictive"}, + ] + r = _build(feature_entry_dicts=features) + assert r.lpr_verdict == "learning_inconclusive" + + +def test_build_insufficient_data_when_n_batches_zero(): + r = _build(n_batches_summarized=0) + assert r.lpr_verdict == "insufficient_data" + + +def test_build_insufficient_data_when_no_features(): + r = _build(feature_entry_dicts=[]) + assert r.lpr_verdict == "insufficient_data" + + +def test_build_feature_entries_are_feature_learning_entry(): + for e in _build().feature_entries: + assert isinstance(e, FeatureLearningEntry) + + +def test_build_evidence_summary_stored(): + r = _build() + assert r.feature_entries[0].evidence_summary == "strong signal" + + +def test_build_evidence_summary_defaults_empty(): + r = _build() + assert r.feature_entries[1].evidence_summary == "" + + +def test_build_key_findings_stored(): + assert len(_build().key_findings) == 2 + + +def test_build_all_features_uncertain_gives_insufficient(): + features = [ + {"feature_category": "charge", "predictivity": "uncertain"}, + {"feature_category": "length", "predictivity": "uncertain"}, + ] + r = _build(feature_entry_dicts=features) + assert r.lpr_verdict == "insufficient_data" + + +def test_build_limitations_stored(): + assert _build().limitations == ["dry-lab only"] + + +def test_build_created_at_stored(): + assert _build().created_at == "2026-07-10" + + +def test_build_amphipathicity_feature(): + features = [{"feature_category": "amphipathicity", "predictivity": "predictive"}] + r = _build(feature_entry_dicts=features) + assert r.feature_entries[0].feature_category == "amphipathicity" + + +def test_build_helicity_feature(): + features = [{"feature_category": "helicity", "predictivity": "uncertain"}] + r = _build(feature_entry_dicts=features) + assert r.feature_entries[0].feature_category == "helicity" + + +# --------------------------------------------------------------------------- +# 3. validate – rejection cases +# --------------------------------------------------------------------------- + + +def test_validate_rejects_bad_lpr_id_prefix(): + with pytest.raises(ValueError, match="LPR-"): + _build(lpr_id="BAD-001") + + +def test_validate_rejects_empty_pipeline_version(): + with pytest.raises(ValueError): + _build(pipeline_version="") + + +def test_validate_rejects_bad_cit_id_prefix(): + with pytest.raises(ValueError, match="CIT-"): + _build(cit_id="BAD-001") + + +def test_validate_rejects_negative_n_batches(): + lpr = _build() + lpr.n_batches_summarized = -1 + with pytest.raises(ValueError, match="n_batches_summarized"): + validate_learning_progress_report(lpr) + + +def test_validate_rejects_invalid_feature_category(): + features = [{"feature_category": "UNKNOWN", "predictivity": "predictive"}] + with pytest.raises(ValueError, match="feature_category"): + _build(feature_entry_dicts=features) + + +def test_validate_rejects_invalid_predictivity(): + features = [{"feature_category": "charge", "predictivity": "UNKNOWN"}] + with pytest.raises(ValueError, match="predictivity"): + _build(feature_entry_dicts=features) + + +def test_validate_rejects_n_predictive_mismatch(): + lpr = _build() + lpr.n_predictive_features = 99 + with pytest.raises(ValueError, match="n_predictive_features"): + validate_learning_progress_report(lpr) + + +def test_validate_rejects_n_non_predictive_mismatch(): + lpr = _build() + lpr.n_non_predictive_features = 99 + with pytest.raises(ValueError, match="n_non_predictive_features"): + validate_learning_progress_report(lpr) + + +def test_validate_rejects_invalid_lpr_verdict(): + lpr = _build() + lpr.lpr_verdict = "UNKNOWN" + with pytest.raises(ValueError, match="lpr_verdict"): + validate_learning_progress_report(lpr) + + +def test_validate_rejects_empty_key_findings(): + with pytest.raises(ValueError, match="key_findings"): + _build(key_findings=[]) + + +def test_validate_rejects_dry_lab_only_false(): + lpr = _build() + lpr.dry_lab_only = False + with pytest.raises(ValueError, match="dry_lab_only"): + validate_learning_progress_report(lpr) + + +def test_validate_rejects_empty_limitations(): + with pytest.raises(ValueError, match="limitations"): + _build(limitations=[]) + + +def test_validate_rejects_empty_created_at(): + with pytest.raises(ValueError): + _build(created_at="") + + +# --------------------------------------------------------------------------- +# 4. format +# --------------------------------------------------------------------------- + + +def test_format_contains_lpr_id(): + assert "LPR-001" in format_learning_progress_report(_build()) + + +def test_format_contains_pipeline_version(): + assert "v1.0" in format_learning_progress_report(_build()) + + +def test_format_contains_cit_id(): + assert "CIT-001" in format_learning_progress_report(_build()) + + +def test_format_contains_verdict(): + assert "learning_confirmed" in format_learning_progress_report(_build()) + + +def test_format_contains_key_finding(): + assert "charge is the strongest predictor" in format_learning_progress_report(_build()) + + +def test_format_contains_limitations(): + assert "dry-lab only" in format_learning_progress_report(_build()) + + +def test_format_contains_dry_lab_only(): + assert "dry_lab_only: True" in format_learning_progress_report(_build()) + + +def test_format_is_string(): + assert isinstance(format_learning_progress_report(_build()), str) diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 80411d1f..d5bb840f 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -4,7 +4,7 @@ import sys import math -BASELINE = 9830 +BASELINE = 10367 def test_test_count_regression():