feat: cluster split validation — Phase 2 benchmark honesty

examples/benchmark/cluster_split_pool.csv

@@ -0,0 +1,14 @@
+id,sequence,source
+CS-POS-001,KWKLFKKIGAVLKFL,cluster_split
+CS-POS-002,KWKLFKRIGAVLKVL,cluster_split
+CS-POS-003,GLFDIVKKVVGALGAL,cluster_split
+CS-NEG-001,AAAAAAAAAAAA,cluster_split
+CS-NEG-002,DEDEDEDEDEDE,cluster_split
+CS-NEG-003,GGGGGGGGGGGG,cluster_split
+CS-NEG-004,EEEEEEEEEEEE,cluster_split
+CS-NEG-005,SSSSSSSSSSSS,cluster_split
+CS-NEG-006,PPPPPPPPPPPP,cluster_split
+CS-NEG-007,TTTTTTTTTTTT,cluster_split
+CS-NEG-008,NNNNNNNNNNNN,cluster_split
+CS-NEG-009,QQQQQQQQQQQQ,cluster_split
+CS-NEG-010,LLLLLLLLLLL,cluster_split

examples/benchmark/cluster_split_refs.csv

@@ -0,0 +1,4 @@
+id,sequence,source
+CSREF-001,KWKLFKKIGAVLKVL,reference
+CSREF-002,GIGKFLHSAKKFGKAFVGEIMNS,reference
+CSREF-003,GLFDIVKKVVGALGSL,reference

src/openamp_foundry/benchmark/evaluate.py

@@ -1,8 +1,130 @@
 from __future__ import annotations
 
+from openamp_foundry.scoring.novelty import normalized_similarity
 from openamp_foundry.types import ScoredCandidate
 
 
 def top_k_ids(scored: list[ScoredCandidate], k: int) -> set[str]:
     ranked = sorted(scored, key=lambda x: x.scores.get("ensemble", 0.0), reverse=True)
     return {item.candidate.candidate_id for item in ranked[:k]}
+
+
+def recall_at_k(
+    scored: list[ScoredCandidate],
+    positive_ids: set[str],
+    k: int,
+) -> float:
+    """Fraction of known positives recovered in the top-k ranked candidates.
+
+    Recall@k = |positives in top-k| / |total positives|
+
+    A random baseline would recover roughly k / |all candidates| of the positives.
+    The pipeline is meaningful if recall@k significantly exceeds the random baseline.
+    """
+    if not positive_ids:
+        return 0.0
+    top = top_k_ids(scored, k)
+    recovered = len(top & positive_ids)
+    return round(recovered / len(positive_ids), 4)
+
+
+def random_recall_at_k(n_candidates: int, n_positives: int, k: int) -> float:
+    """Expected recall@k for a random ranker.
+
+    E[recall@k] = min(k, n_positives) / n_positives when sampling without replacement.
+    """
+    if n_positives == 0 or n_candidates == 0:
+        return 0.0
+    expected_hits = min(k, n_positives) * min(k, n_candidates) / max(n_candidates, 1)
+    return round(min(1.0, expected_hits / n_positives), 4)
+
+
+def enrichment_factor(
+    scored: list[ScoredCandidate],
+    positive_ids: set[str],
+    k: int,
+) -> float:
+    """Enrichment factor at k: recall@k relative to random baseline recall@k.
+
+    EF > 1.0 means the pipeline outperforms random.
+    EF = 1.0 is random.
+    EF < 1.0 is worse than random (anti-enrichment).
+    """
+    n = len(scored)
+    n_pos = len(positive_ids)
+    rc = recall_at_k(scored, positive_ids, k)
+    random_rc = random_recall_at_k(n, n_pos, k)
+    if random_rc == 0.0:
+        return 0.0
+    return round(rc / random_rc, 4)
+
+
+def benchmark_summary(
+    scored: list[ScoredCandidate],
+    positive_ids: set[str],
+    ks: list[int] | None = None,
+) -> dict:
+    """Produce a benchmark summary comparing pipeline vs random ranker.
+
+    Returns recall@k and enrichment factor at each k, plus a verdict.
+    All results are computational only — they do not prove biological activity.
+    """
+    if ks is None:
+        n = len(scored)
+        ks = sorted({max(1, n // 10), max(1, n // 5), max(1, n // 2), n})
+
+    results = []
+    for k in ks:
+        rc = recall_at_k(scored, positive_ids, k)
+        rrc = random_recall_at_k(len(scored), len(positive_ids), k)
+        ef = enrichment_factor(scored, positive_ids, k)
+        results.append(
+            {
+                "k": k,
+                "recall_at_k": rc,
+                "random_recall_at_k": rrc,
+                "enrichment_factor": ef,
+            }
+        )
+
+    any_enrichment = any(r["enrichment_factor"] > 1.0 for r in results)
+    verdict = "pipeline outperforms random" if any_enrichment else "pipeline does not outperform random"
+
+    return {
+        "disclaimer": (
+            "These are retrospective benchmark results on demo data. "
+            "They do not prove biological efficacy. "
+            "They only measure whether the pipeline recovers known positives "
+            "better than a random ranker would."
+        ),
+        "n_candidates": len(scored),
+        "n_positives": len(positive_ids),
+        "results": results,
+        "verdict": verdict,
+    }
+
+
+def find_contaminated_references(
+    candidate_sequences: list[str],
+    reference_sequences: list[str],
+    positive_ids: set[str],
+    candidate_ids: list[str],
+    threshold: float = 0.70,
+) -> set[int]:
+    """Return indices of references that are near-duplicates of test positives.
+
+    Used to build a cluster-split reference set: removing contaminated references
+    ensures benchmark performance is not inflated by reference-set memorization.
+    """
+    contaminated: set[int] = set()
+    positive_seqs = {
+        seq
+        for seq, cid in zip(candidate_sequences, candidate_ids)
+        if cid in positive_ids
+    }
+    for ref_idx, ref_seq in enumerate(reference_sequences):
+        for pos_seq in positive_seqs:
+            if normalized_similarity(ref_seq, pos_seq) >= threshold:
+                contaminated.add(ref_idx)
+                break
+    return contaminated

src/openamp_foundry/benchmark/splits.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from openamp_foundry.scoring.novelty import normalized_similarity
 from openamp_foundry.types import PeptideCandidate
 
 
@@ -14,3 +15,53 @@ def deterministic_split(
         else:
             train.append(cand)
     return train, holdout
+
+
+def cluster_by_similarity(
+    sequences: list[str],
+    threshold: float = 0.70,
+) -> list[list[int]]:
+    """Greedy single-linkage clustering: sequences within threshold go to the same cluster.
+
+    Returns a list of clusters, where each cluster is a list of indices into `sequences`.
+    The first sequence assigned to a cluster becomes its center for subsequent comparisons.
+    threshold: sequences with normalized_similarity >= threshold are co-clustered.
+    """
+    clusters: list[list[int]] = []
+    centers: list[str] = []
+
+    for idx, seq in enumerate(sequences):
+        assigned = False
+        for ci, center in enumerate(centers):
+            if normalized_similarity(seq, center) >= threshold:
+                clusters[ci].append(idx)
+                assigned = True
+                break
+        if not assigned:
+            clusters.append([idx])
+            centers.append(seq)
+
+    return clusters
+
+
+def cluster_split(
+    sequences: list[str],
+    threshold: float = 0.70,
+) -> tuple[list[int], list[int]]:
+    """Split sequences into reference and test partitions by cluster membership.
+
+    The first member of each cluster becomes the reference representative.
+    All subsequent cluster members become test (held-out) sequences.
+
+    This ensures no test sequence has a near-duplicate in the reference set,
+    preventing benchmark inflation from reference-set memorization.
+
+    Returns (reference_indices, test_indices).
+    """
+    clusters = cluster_by_similarity(sequences, threshold)
+    reference_indices: list[int] = []
+    test_indices: list[int] = []
+    for cluster in clusters:
+        reference_indices.append(cluster[0])
+        test_indices.extend(cluster[1:])
+    return reference_indices, test_indices

src/openamp_foundry/pipeline.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import hashlib
 import uuid
 from datetime import datetime, timezone
 from pathlib import Path

tests/test_cli.py

@@ -3,7 +3,6 @@
 
 import json
 
-import pytest
 
 from openamp_foundry.cli import main
 
@@ -45,7 +44,6 @@ def test_validate_command_success(tmp_path):
         "--out", out,
         "--cert-dir", certs,
     ])
-    import os
     cert_files = list((tmp_path / "certs").glob("*.json"))
     assert cert_files, "No certificates were generated"
     ret = main([