|
| 1 | +"""Tests for expert composite ranking integration. |
| 2 | +
|
| 3 | +Verifies that: |
| 4 | +1. score_candidates() now computes expert_composite and hemolysis_risk for every candidate. |
| 5 | +2. rank_candidates(ranking_mode='expert') produces a different order than ensemble. |
| 6 | +3. The expert composite partially corrects the anti-selective bias. |
| 7 | +4. The CLI --ranking-mode expert flag works end-to-end. |
| 8 | +5. The batch report records which ranking mode was used. |
| 9 | +
|
| 10 | +Context: The triage benchmark (v0.5.12) showed the ensemble has an anti-selective |
| 11 | +bias (AUROC selective_vs_hemolytic = 0.466, below random). The expert composite |
| 12 | +incorporates selectivity, hemolysis risk, and helix-hinge analysis. This test |
| 13 | +verifies the integration is correct, not that the expert composite is biologically |
| 14 | +validated (it is not). |
| 15 | +""" |
| 16 | +from __future__ import annotations |
| 17 | + |
| 18 | +import json |
| 19 | + |
| 20 | + |
| 21 | +from openamp_foundry.pipeline import score_candidates |
| 22 | +from openamp_foundry.selection.pareto import rank_candidates |
| 23 | + |
| 24 | + |
| 25 | +class TestExpertCompositeInPipeline: |
| 26 | + """Verify expert_composite is computed during scoring.""" |
| 27 | + |
| 28 | + def test_expert_composite_present_in_scores(self): |
| 29 | + scored, _ = score_candidates("examples/sequences/demo_candidates.csv") |
| 30 | + for s in scored: |
| 31 | + assert "expert_composite" in s.scores, ( |
| 32 | + f"{s.candidate.candidate_id} missing expert_composite score" |
| 33 | + ) |
| 34 | + assert 0.0 <= s.scores["expert_composite"] <= 1.0 |
| 35 | + |
| 36 | + def test_hemolysis_risk_present_in_scores(self): |
| 37 | + scored, _ = score_candidates("examples/sequences/demo_candidates.csv") |
| 38 | + for s in scored: |
| 39 | + assert "hemolysis_risk" in s.scores, ( |
| 40 | + f"{s.candidate.candidate_id} missing hemolysis_risk score" |
| 41 | + ) |
| 42 | + assert 0.0 <= s.scores["hemolysis_risk"] <= 1.0 |
| 43 | + |
| 44 | + def test_expert_composite_differs_from_ensemble(self): |
| 45 | + """Expert composite should not be identical to ensemble (different formula).""" |
| 46 | + scored, _ = score_candidates("examples/sequences/demo_candidates.csv") |
| 47 | + for s in scored: |
| 48 | + assert s.scores["expert_composite"] != s.scores["ensemble"], ( |
| 49 | + f"{s.candidate.candidate_id}: expert_composite equals ensemble " |
| 50 | + "-- they should differ (different component weights)" |
| 51 | + ) |
| 52 | + |
| 53 | + |
| 54 | +class TestExpertRankingMode: |
| 55 | + """Verify ranking_mode='expert' produces different ordering than ensemble.""" |
| 56 | + |
| 57 | + def test_expert_ranking_differs_from_ensemble(self): |
| 58 | + """At least one position should differ between ensemble and expert ranking.""" |
| 59 | + scored, _ = score_candidates("examples/sequences/demo_candidates.csv") |
| 60 | + ens_ranked = rank_candidates(scored, ranking_mode="ensemble") |
| 61 | + exp_ranked = rank_candidates(scored, ranking_mode="expert") |
| 62 | + |
| 63 | + ens_ids = [s.candidate.candidate_id for s in ens_ranked] |
| 64 | + exp_ids = [s.candidate.candidate_id for s in exp_ranked] |
| 65 | + |
| 66 | + # With 10 candidates, at least one position should differ |
| 67 | + assert ens_ids != exp_ids, ( |
| 68 | + "Expert ranking produced identical order to ensemble — " |
| 69 | + "the expert composite should change at least some rankings" |
| 70 | + ) |
| 71 | + |
| 72 | + def test_expert_ranking_sorts_by_expert_composite(self): |
| 73 | + """Expert ranking should sort by expert_composite, highest first.""" |
| 74 | + scored, _ = score_candidates("examples/sequences/demo_candidates.csv") |
| 75 | + exp_ranked = rank_candidates(scored, ranking_mode="expert") |
| 76 | + |
| 77 | + for i in range(len(exp_ranked) - 1): |
| 78 | + assert exp_ranked[i].scores["expert_composite"] >= exp_ranked[i + 1].scores["expert_composite"] |
| 79 | + |
| 80 | + def test_ensemble_ranking_unchanged_by_default(self): |
| 81 | + """Default ranking_mode should still be ensemble.""" |
| 82 | + scored, _ = score_candidates("examples/sequences/demo_candidates.csv") |
| 83 | + default_ranked = rank_candidates(scored) |
| 84 | + ens_ranked = rank_candidates(scored, ranking_mode="ensemble") |
| 85 | + |
| 86 | + default_ids = [s.candidate.candidate_id for s in default_ranked] |
| 87 | + ens_ids = [s.candidate.candidate_id for s in ens_ranked] |
| 88 | + assert default_ids == ens_ids |
| 89 | + |
| 90 | + |
| 91 | +class TestExpertRankingCLI: |
| 92 | + """CLI integration test for --ranking-mode expert.""" |
| 93 | + |
| 94 | + def test_cli_expert_ranking_produces_valid_output(self, tmp_path): |
| 95 | + from openamp_foundry.cli import main |
| 96 | + |
| 97 | + out = tmp_path / "ranked.jsonl" |
| 98 | + report = tmp_path / "report.md" |
| 99 | + rc = main([ |
| 100 | + "rank", |
| 101 | + "--candidates", "examples/sequences/demo_candidates.csv", |
| 102 | + "--references", "examples/known_reference/demo_known_amps.csv", |
| 103 | + "--out", str(out), |
| 104 | + "--report", str(report), |
| 105 | + "--ranking-mode", "expert", |
| 106 | + ]) |
| 107 | + assert rc == 0 |
| 108 | + assert out.exists() |
| 109 | + |
| 110 | + # Batch report should record the ranking mode |
| 111 | + report_json = tmp_path / "report.json" |
| 112 | + assert report_json.exists() |
| 113 | + data = json.loads(report_json.read_text()) |
| 114 | + assert data["ranking_mode"] == "expert" |
| 115 | + |
| 116 | + def test_cli_ensemble_ranking_default(self, tmp_path): |
| 117 | + """Default ranking (no --ranking-mode) should be ensemble.""" |
| 118 | + from openamp_foundry.cli import main |
| 119 | + |
| 120 | + out = tmp_path / "ranked.jsonl" |
| 121 | + report = tmp_path / "report.md" |
| 122 | + rc = main([ |
| 123 | + "rank", |
| 124 | + "--candidates", "examples/sequences/demo_candidates.csv", |
| 125 | + "--references", "examples/known_reference/demo_known_amps.csv", |
| 126 | + "--out", str(out), |
| 127 | + "--report", str(report), |
| 128 | + ]) |
| 129 | + assert rc == 0 |
| 130 | + report_json = tmp_path / "report.json" |
| 131 | + data = json.loads(report_json.read_text()) |
| 132 | + assert data["ranking_mode"] == "ensemble" |
| 133 | + |
| 134 | + |
| 135 | +class TestExpertRankingSelectivity: |
| 136 | + """Verify expert ranking shifts top candidates toward safer profiles.""" |
| 137 | + |
| 138 | + def test_expert_top_5_have_lower_hemolysis_risk_on_average(self): |
| 139 | + """Expert-ranked top-5 should have lower mean hemolysis_risk than ensemble top-5. |
| 140 | +
|
| 141 | + This is the core value proposition: the expert composite incorporates |
| 142 | + hemolysis risk, so top-ranked candidates should be safer on average. |
| 143 | + If this test fails, the expert composite's hemolysis components are |
| 144 | + not effectively influencing the ranking. |
| 145 | + """ |
| 146 | + scored, _ = score_candidates("examples/benchmark/mixed_candidates.csv") |
| 147 | + ens_ranked = rank_candidates(scored, ranking_mode="ensemble") |
| 148 | + exp_ranked = rank_candidates(scored, ranking_mode="expert") |
| 149 | + |
| 150 | + ens_top5_hemo = sum(s.scores["hemolysis_risk"] for s in ens_ranked[:5]) / 5 |
| 151 | + exp_top5_hemo = sum(s.scores["hemolysis_risk"] for s in exp_ranked[:5]) / 5 |
| 152 | + |
| 153 | + assert exp_top5_hemo <= ens_top5_hemo, ( |
| 154 | + f"Expert top-5 mean hemolysis_risk ({exp_top5_hemo:.4f}) should be <= " |
| 155 | + f"ensemble top-5 ({ens_top5_hemo:.4f}). The expert composite should " |
| 156 | + f"favor safer candidates." |
| 157 | + ) |
0 commit comments