feat: confidence-gap safeguards before wet-lab synthesis

Makefile

@@ -1,4 +1,4 @@
-.PHONY: demo test lint clean bench-leakage bench-baseline bench-hidden-active generate phase3 pilot
+.PHONY: demo test lint clean bench-leakage bench-baseline bench-hidden-active generate phase3 pilot validate-scoring external-predict pilot-confident
 
 PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3)
 PYTEST  := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest)
@@ -68,5 +68,27 @@ pilot: phase3
 		--out-csv outputs/pilot_panel.csv \
 		--out-md outputs/pilot_panel.md
 
+validate-scoring:
+	PYTHONPATH=src $(PYTHON) -m openamp_foundry.cli validate-scoring \
+		--amp-csv examples/validation/known_amps.csv \
+		--decoy-csv examples/validation/scrambled_decoys.csv \
+		--out outputs/validate_scoring_report.json
+
+external-predict: pilot
+	PYTHONPATH=src $(PYTHON) -m openamp_foundry.cli external-predict \
+		--pilot-csv outputs/pilot_panel.csv \
+		--out-fasta outputs/pilot_panel.fasta \
+		--out-checklist outputs/external_predict_checklist.md
+
+pilot-confident:
+	@if [ -z "$(KEEP)" ]; then \
+		echo "Usage: make pilot-confident KEEP=ID1,ID2,..."; \
+		exit 1; \
+	fi
+	PYTHONPATH=src $(PYTHON) -m openamp_foundry.cli pilot-confident \
+		--pilot-csv outputs/pilot_panel.csv \
+		--keep "$(KEEP)" \
+		--out outputs/confident_panel
+
 clean:
 	rm -rf outputs/*.jsonl outputs/*.md outputs/*.json outputs/evidence outputs/phase3_evidence .pytest_cache .ruff_cache

examples/validation/known_amps.csv

@@ -0,0 +1,45 @@
+id,sequence,family,reference,label
+REF-MAG-001,GIGKFLHSAKKFGKAFVGEIMNS,magainin,Zasloff_1987_PNAS,1
+REF-MAG-002,GIGKFLHSAGKFGKAFVGEIMKS,magainin,Zasloff_1987_PNAS,1
+REF-MAG-003,GIGKFLHSAKKFGKAFVGQIMNS,magainin_analog,Chen_1988,1
+REF-PEX-001,GIGKFLKKAKKFGKAFVKILKK,pexiganan_analog,Ge_1999_Antimicrob,1
+REF-BUF-001,TRSSRAGLQFPVGRVHRLLRK,buforin,Kim_1996_Eur_J_Biochem,1
+REF-BUF-002,RAGLQFPVGRVHRLLRK,buforin_ii,Park_2000_J_Biol_Chem,1
+REF-IND-001,ILPWKWPWWPWRR,indolicidin,Selsted_1992_J_Biol_Chem,1
+REF-IND-002,ILPWKWPWWPWRRRR,indolicidin_analog,Falla_1996_J_Biol_Chem,1
+REF-TMP-001,FLPLIGRVLSGIL,temporin_a,Mangoni_2001_FEBS,1
+REF-TMP-002,FVQWFSKFLGRIL,temporin_l,Mangoni_2001_FEBS,1
+REF-TMP-003,LLPIVGNLLKSLL,temporin_b,Simmaco_1996_Eur_J_Biochem,1
+REF-TMP-004,FLPLLAGLAANFLPKIF,brevinin_like,Conlon_2004_Peptides,1
+REF-AUR-001,GLFDIIKKIAESF,aurein_1,Rozek_2000_Eur_J_Biochem,1
+REF-AUR-002,GLFDIVKKVVGALGSL,aurein_2,Rozek_2000_Eur_J_Biochem,1
+REF-AUR-003,GLFDIIKKIAGSFS,aurein_3,Rozek_2000_Eur_J_Biochem,1
+REF-ESC-001,GIFSKLAGKKIKNLLISGLKG,esculentin_fragment,Simmaco_1993_J_Biol_Chem,1
+REF-UPR-001,GVGDLIRKAIASVAGKELG,uperin,Jackway_2008_Peptides,1
+REF-BOM-001,IIGPVLGLVGSALGGLLKKI,bombinin_h,Halverson_1990,1
+REF-MAC-001,GLFGVLAKVAAHVVPAIAEHF,maculatin,Chia_2000_Eur_J_Biochem,1
+REF-CAE-001,GLLSVLGSVAKHVLPHVVPVIAEH,caerin,Rozek_2000_Biochemistry,1
+REF-RRW-001,RRWQWRMKKLG,tachyplesin_like,Tam_2002_J_Biol_Chem,1
+REF-CEC-001,KWKLFKKIEKVGQNIRDGIIKAGPAV,cecropin_a_fragment,Steiner_1981_PNAS,1
+REF-CEC-002,KWKIFKKIEKVGRNVRDGIIKAGP,cecropin_b_fragment,Hultmark_1980_Eur_J_Biochem,1
+REF-MEL-001,GIGAVLKVLTTGLPALISWIKRKRQQ,melittin,Habermann_1972,1
+REF-KWK-001,KWKLFKKIGAVLKVL,template_seed_1,Oren_1997_Biochemistry,1
+REF-GIG-001,GIGKFLHSAKKFGKAFVGEIMNS,template_seed_2,Zasloff_1987_PNAS,1
+REF-MAG-004,KKLKKFGLRIINKIVAAL,magainin_helix,Wieprecht_1997,1
+REF-BMA-001,GRFKRFRKKFKKLFKKLSP,bmap_fragment,Skerlavaj_1999,1
+REF-LLL-001,LLRWPWWPWRRK,omiganan_analog,Sader_2004,1
+REF-TAC-001,KWKKLFKKI,cecropin_core,Andrieu_1996,1
+REF-GRM-001,GRMKKVFFKSLRQH,gramicidin_s_like,Fontenot_1993,1
+REF-SMA-001,SMKQLAHKLKSEKLELSDREQMR,smap_fragment,Tossi_2000,1
+REF-HLP-001,HLPKHFKAFVARLIRNAMSN,hlp_fragment,Hiemstra_1993,1
+REF-FKK-001,FKKLKKSLKL,klaklak_analog,Javadpour_1996,1
+REF-KAA-001,KAAAKAAAKAAAK,ala_lys_repeat,Braunstein_2004,1
+REF-GKK-001,GKKLFSKLKK,cecropin_melittin_analog,Gazit_1994,1
+REF-RLK-001,RLKKVWKKWKK,cationic_tryptophan,Parisien_2008,1
+REF-ALA-001,ALWKTMLKKLGTMALHAGKAALGAAADTISQGTQ,dermaseptin_s3_fragment,Mor_1994_Biochemistry,1
+REF-ILK-001,ILKKWPWWPWRRK,indolicidin_lys_analog,Lawyer_1996,1
+REF-VRG-001,VRGRPIPICSRGGYCFCLPFLPGACHRPSGMC,defensin_like_linear,skipped_disulfide,1
+REF-PLA-001,KKVVFKVKFK,short_cationic,Lee_2004,1
+REF-GKA-001,GKAFVGEIMNS,magainin_c_fragment,Westerhoff_1989,1
+REF-WKL-001,WKLFKKIGAVLKVL,magainin_analog,Dathe_1997,1
+REF-KFL-001,KFLHSAKKFGKAFV,magainin_core,Maloy_1995,1

examples/validation/scrambled_decoys.csv

@@ -0,0 +1,45 @@
+id,sequence,family,source_id,label
+DECOY-MAG-001,VFGALKGGSHKIIFKNFESAGKM,shuffled_magainin,REF-MAG-001,0
+DECOY-MAG-002,GKKVFEIIFLHFGKGGAASKGMS,shuffled_magainin,REF-MAG-002,0
+DECOY-MAG-003,FKHGSNGGVLKKFMQGKAIAISF,shuffled_magainin_analog,REF-MAG-003,0
+DECOY-PEX-001,KGFLIKKKGIKFKKVKLFAAGK,shuffled_pexiganan_analog,REF-PEX-001,0
+DECOY-BUF-001,VQTSRVRGLHAFSLRGKRLRP,shuffled_buforin,REF-BUF-001,0
+DECOY-BUF-002,VQRAHRRVKGLFPRLLG,shuffled_buforin_ii,REF-BUF-002,0
+DECOY-IND-001,WIWPWKWRLPPWR,shuffled_indolicidin,REF-IND-001,0
+DECOY-IND-002,PPIRWWPRWKWRLRW,shuffled_indolicidin_analog,REF-IND-002,0
+DECOY-TMP-001,VGGFRISLILLPL,shuffled_temporin_a,REF-TMP-001,0
+DECOY-TMP-002,QGLKLRIWFSVFF,shuffled_temporin_l,REF-TMP-002,0
+DECOY-TMP-003,LNLIGVLKLSPLL,shuffled_temporin_b,REF-TMP-003,0
+DECOY-TMP-004,GFLIPFPLLFALAKNLA,shuffled_brevinin_like,REF-TMP-004,0
+DECOY-AUR-001,FKIIFKISELGDA,shuffled_aurein_1,REF-AUR-001,0
+DECOY-AUR-002,ISGLVAKLGVFLDKGV,shuffled_aurein_2,REF-AUR-002,0
+DECOY-AUR-003,DFLIIASGKSFIKG,shuffled_aurein_3,REF-AUR-003,0
+DECOY-ESC-001,KLISNLGKKIGKIFGKLLAGS,shuffled_esculentin_fragment,REF-ESC-001,0
+DECOY-UPR-001,AVLGKGAISVGGRLDEKAI,shuffled_uperin,REF-UPR-001,0
+DECOY-BOM-001,SPGLLLILGGLIVAKIKGGV,shuffled_bombinin_h,REF-BOM-001,0
+DECOY-MAC-001,IAVEAVFALGHVAAPGVKHFL,shuffled_maculatin,REF-MAC-001,0
+DECOY-CAE-001,SHHLGHVVLVIGLPLPAVVVSKEA,shuffled_caerin,REF-CAE-001,0
+DECOY-RRW-001,MRKWKLGWRQR,shuffled_tachyplesin_like,REF-RRW-001,0
+DECOY-CEC-001,KRWNVKKIIQKGAPDIKIGEFLVKGA,shuffled_cecropin_a_fragment,REF-CEC-001,0
+DECOY-CEC-002,KAGIKDIRIRIKGGWPNVKEKFVK,shuffled_cecropin_b_fragment,REF-CEC-002,0
+DECOY-MEL-001,ILTLIKTLIGQALGWKVPSQARVRKG,shuffled_melittin,REF-MEL-001,0
+DECOY-KWK-001,FALKGWKKLVLIKVK,shuffled_template_seed_1,REF-KWK-001,0
+DECOY-GIG-001,KKGHASFIFIGLEKNSVFMGAKG,shuffled_template_seed_2,REF-GIG-001,0
+DECOY-MAG-004,ALLKLAFKKKVGKRIIIN,shuffled_magainin_helix,REF-MAG-004,0
+DECOY-BMA-001,KKGRKPLFFRLFKRKKSKF,shuffled_bmap_fragment,REF-BMA-001,0
+DECOY-LLL-001,WLWPRLWKPRWR,shuffled_omiganan_analog,REF-LLL-001,0
+DECOY-TAC-001,LKKIWKKKF,shuffled_cecropin_core,REF-TAC-001,0
+DECOY-GRM-001,KRGHMFFQVSKKRL,shuffled_gramicidin_s_like,REF-GRM-001,0
+DECOY-SMA-001,LLAMKSSRRQKQEDEHLEKLKMS,shuffled_smap_fragment,REF-SMA-001,0
+DECOY-HLP-001,HNNAKVHIFRFASLPRKAML,shuffled_hlp_fragment,REF-HLP-001,0
+DECOY-FKK-001,KFKLSKLKLK,shuffled_klaklak_analog,REF-FKK-001,0
+DECOY-KAA-001,AAKKAAAAAKAAK,shuffled_ala_lys_repeat,REF-KAA-001,0
+DECOY-GKK-001,FLKKGKSKKL,shuffled_cecropin_melittin_analog,REF-GKK-001,0
+DECOY-RLK-001,VKKKWKLRKKW,shuffled_cationic_tryptophan,REF-RLK-001,0
+DECOY-ALA-001,GKQLITLKWATGMLTLAKAHKASAGADTMALGAQ,shuffled_dermaseptin_s3_fragment,REF-ALA-001,0
+DECOY-ILK-001,WKRPKPIWWKWRL,shuffled_indolicidin_lys_analog,REF-ILK-001,0
+DECOY-VRG-001,IPCASPPPFHRCCMPGIGRLVFGRYGCGSCRL,shuffled_defensin_like_linear,REF-VRG-001,0
+DECOY-PLA-001,VVVKKFKKFK,shuffled_short_cationic,REF-PLA-001,0
+DECOY-GKA-001,SMAEFGIVGNK,shuffled_magainin_c_fragment,REF-GKA-001,0
+DECOY-WKL-001,AVLWGKIKLFLKVK,shuffled_magainin_analog,REF-WKL-001,0
+DECOY-KFL-001,SLFAKHKGFVKKFA,shuffled_magainin_core,REF-KFL-001,0

src/openamp_foundry/benchmark/retrospective.py

@@ -0,0 +1,155 @@
+"""Retrospective AUROC benchmark against known AMPs vs composition-matched decoys.
+
+Tests whether our scoring model ranks known antimicrobial peptides higher than
+randomly shuffled decoys with the same amino acid composition.
+
+Benchmark design:
+  Positives — 44 sequences from amp_curated_references.csv (confirmed literature AMPs)
+  Negatives — 44 per-sequence shuffled decoys (RNG seed=42, same composition)
+
+Key metric: AUROC (area under receiver operating characteristic curve)
+  = P(random AMP scores higher than random decoy)
+  < 0.55 → model is near-random; do NOT proceed to synthesis
+  0.55–0.70 → model has weak but real signal; treat $10k budget with caution
+  > 0.70 → model has meaningful discriminative power; proceed to synthesis
+
+IMPORTANT: This benchmark ONLY tests discrimination of order-dependent features
+(hydrophobic moment) since composition-based features are identical by design.
+A high AUROC here means the amphipathicity signal is real; it does NOT test
+whether our nominees are actually antimicrobial.
+"""
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+
+
+def _auc_wilcoxon(pos_scores: list[float], neg_scores: list[float]) -> float:
+    """Compute AUROC via the Wilcoxon-Mann-Whitney statistic (O(n*m) but n is small)."""
+    n_pos = len(pos_scores)
+    n_neg = len(neg_scores)
+    if n_pos == 0 or n_neg == 0:
+        return 0.5
+    concordant = sum(
+        1 for p in pos_scores for n in neg_scores if p > n
+    ) + 0.5 * sum(
+        1 for p in pos_scores for n in neg_scores if p == n
+    )
+    return concordant / (n_pos * n_neg)
+
+
+def _recall_at_k(labels: list[int], k: int) -> float:
+    """Fraction of true positives in the top-k ranked items."""
+    n_pos = sum(labels)
+    if n_pos == 0:
+        return 0.0
+    top_k_pos = sum(labels[:k])
+    return top_k_pos / n_pos
+
+
+def run_retrospective_benchmark(
+    amp_csv: str | Path,
+    decoy_csv: str | Path,
+    config_path: str | Path = "configs/pipeline.yaml",
+    recall_ks: list[int] | None = None,
+) -> dict:
+    """Score known AMPs and shuffled decoys and compute AUROC + recall@k.
+
+    Returns a dict with AUROC, per-k recall, and the full ranked list.
+    """
+    from openamp_foundry.features.physchem import compute_features
+    from openamp_foundry.scoring.activity import activity_likeness_score
+    from openamp_foundry.scoring.boman import boman_activity_score, gravy_score
+    from openamp_foundry.scoring.ensemble import ensemble_score
+    from openamp_foundry.scoring.novelty import novelty_score
+    from openamp_foundry.scoring.safety import safety_score
+    from openamp_foundry.scoring.synthesis import synthesis_feasibility_score
+    from openamp_foundry.config import load_config
+
+    config = load_config(config_path)
+    weights = config["weights"]
+
+    rows = []
+
+    for path, true_label in [(amp_csv, 1), (decoy_csv, 0)]:
+        with open(path, newline="", encoding="utf-8") as f:
+            for row in csv.DictReader(f):
+                seq = row["sequence"].strip().upper()
+                seq_id = row["id"]
+                features = compute_features(seq)
+                act = activity_likeness_score(features)
+                safe = safety_score(features)
+                synth = synthesis_feasibility_score(features, valid_sequence=True)
+                nov, _ = novelty_score(seq, [])
+                boman_act = boman_activity_score(seq)
+                raw_scores = {
+                    "activity": act, "safety": safe,
+                    "synthesis": synth, "novelty": nov,
+                    "boman_activity": boman_act,
+                    "disagreement": abs(act - boman_act),
+                }
+                raw_scores["ensemble"] = ensemble_score(raw_scores, weights)
+                rows.append({
+                    "id": seq_id,
+                    "sequence": seq,
+                    "label": true_label,
+                    "ensemble": raw_scores["ensemble"],
+                    "activity": act,
+                    "safety": safe,
+                    "boman_activity": boman_act,
+                    "hydrophobic_moment": features.get("hydrophobic_moment", 0.0),
+                })
+
+    rows.sort(key=lambda r: r["ensemble"], reverse=True)
+
+    pos_scores = [r["ensemble"] for r in rows if r["label"] == 1]
+    neg_scores = [r["ensemble"] for r in rows if r["label"] == 0]
+    auroc = round(_auc_wilcoxon(pos_scores, neg_scores), 4)
+
+    n_total = len(rows)
+    n_pos = sum(r["label"] for r in rows)
+    labels_ranked = [r["label"] for r in rows]
+
+    if recall_ks is None:
+        recall_ks = [10, 20, 44]
+    recall = {f"recall_at_{k}": round(_recall_at_k(labels_ranked, k), 4) for k in recall_ks}
+
+    random_auroc = 0.5
+    interpretation = (
+        "STRONG — model has meaningful discriminative power (AUROC > 0.70)"
+        if auroc >= 0.70
+        else "WEAK — model has modest signal; proceed with caution (AUROC 0.55–0.70)"
+        if auroc >= 0.55
+        else "POOR — model is near-random (AUROC < 0.55); do NOT proceed to synthesis"
+    )
+
+    return {
+        "benchmark": "retrospective_auroc",
+        "n_positives": n_pos,
+        "n_negatives": n_total - n_pos,
+        "n_total": n_total,
+        "auroc": auroc,
+        "random_auroc": random_auroc,
+        "auroc_above_random": round(auroc - random_auroc, 4),
+        **recall,
+        "interpretation": interpretation,
+        "design_note": (
+            "Negatives are amino-acid-composition-matched shuffled decoys (RNG seed=42). "
+            "AUROC reflects discrimination by ORDER-DEPENDENT features only "
+            "(primarily hydrophobic moment). Composition-based features "
+            "(charge, hydrophobic fraction, Boman index, GRAVY) are identical "
+            "for each AMP/decoy pair and do NOT contribute to discrimination."
+        ),
+        "known_blind_spots": [
+            "Melittin-like bent-helix peptides: hemolytic character not captured "
+            "by simple 1D hydrophobic moment (Habermann 1972).",
+            "Proline-rich AMPs (PR-39): activity relies on intracellular targets, "
+            "not membrane disruption; hydrophobic moment is low but activity is real.",
+        ],
+        "top_ranked": rows[:10],
+        "disclaimer": (
+            "AUROC > 0.70 does NOT imply the nominated candidates are antimicrobial. "
+            "It implies the model has some discriminative power over composition-matched "
+            "controls. Wet-lab validation remains mandatory."
+        ),
+    }