From 3b915fafefe8a156a9eb70b9c205cca7a473d064 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Fri, 10 Jul 2026 15:45:07 +0700 Subject: [PATCH] feat: Phase W W3 similarity challenge harness schema (SCH-) -- flags proximity-driven selection vs random draw; VALID_SCH_VERDICTS (4), SELECTION_VALUE_GAP_THRESHOLD=0.10, SimilarityGroupStats helper; gap auto-computed; dry_lab_only=True; 60 tests --- docs/research/NEXT_100_PR_MAP.md | 2 +- .../evidence/similarity_challenge_harness.py | 167 +++++++++ .../test_similarity_challenge_harness.py | 325 ++++++++++++++++++ 3 files changed, 493 insertions(+), 1 deletion(-) create mode 100644 src/openamp_foundry/evidence/similarity_challenge_harness.py create mode 100644 tests/evidence/test_similarity_challenge_harness.py diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index 7c135fd7..70346f42 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -278,6 +278,6 @@ Make it machine-verifiable that the pipeline produces novel candidates that beat |----|------|----------------|----------| | W1 | Add novelty challenge harness schema (NCH-) (complete). — src/openamp_foundry/evidence/novelty_challenge_harness.py: VALID_NCH_VERDICTS (4: novel_batch/mixed_novelty/near_neighbor_dominated/challenge_not_run), VALID_REFERENCE_DATABASES (6), NEAR_NEIGHBOR_IDENTITY_THRESHOLD=0.80, NOVEL_BATCH_CEILING=0.20, NEAR_NEIGHBOR_DOMINATED_FLOOR=0.60; NCHCandidateResult helper; build() auto-computes is_near_neighbor from identity vs threshold, fraction, verdict; dry_lab_only=True enforced; 63 tests in tests/evidence/test_novelty_challenge_harness.py. | Batch-level novelty challenge: documents the fraction of top candidates with ≥80% sequence identity to a known AMP in a reference database (APD3/DRAMP/etc.); blocks novel_batch claim when near-neighbor fraction exceeds 20%; prevents pipeline from advancing near-copies of known AMPs under a novelty label. | C | | W2 | Add charge-matched challenge schema (CMC-) (complete). — src/openamp_foundry/evidence/charge_matched_challenge.py: VALID_CMC_VERDICTS (4: gap_meaningful/gap_marginal/gap_absent/challenge_not_run), VALID_CHARGE_BASELINE_METHODS (4), MEANINGFUL_GAP_THRESHOLD=0.05, MARGINAL_GAP_LOWER=0.02; auroc_gap auto-computed; verdict auto-derived; dry_lab_only=True enforced; 60 tests in tests/evidence/test_charge_matched_challenge.py. | Formally documents the charge-matched challenge: compares pipeline AUROC vs a charge-only baseline on the same candidate set; verdict controlled vocabulary (gap_meaningful/gap_marginal/gap_absent/not_run); blocks performance claims when the charge-only baseline explains the gap. | C | -| W3 | Add similarity challenge harness schema (SCH-). | Documents whether pipeline-selected candidates are systematically more similar to known AMPs than random selection from the sequence space; flags selection bias from similarity clustering; prevents "novel panel" claim when selection is proximity-driven. | C | +| W3 | Add similarity challenge harness schema (SCH-) (complete). — src/openamp_foundry/evidence/similarity_challenge_harness.py: VALID_SCH_VERDICTS (4: selection_adds_value/marginal_improvement/proximity_driven/challenge_not_run), VALID_SIMILARITY_METRICS (4), SELECTION_VALUE_GAP_THRESHOLD=0.10, MARGINAL_IMPROVEMENT_LOWER=0.03; SimilarityGroupStats helper; similarity_gap auto-computed; dry_lab_only=True enforced; 60 tests in tests/evidence/test_similarity_challenge_harness.py. | Documents whether pipeline-selected candidates are systematically more similar to known AMPs than random selection from the sequence space; flags selection bias from similarity clustering; prevents "novel panel" claim when selection is proximity-driven. | C | | W4 | Add benchmark challenge registry schema (BCR-). | Machine-readable registry of which benchmark challenges (NCH/CMC/SCH) have been run and passed for a given pipeline version; aggregates challenge verdicts; overall hardness grade (A: all passed, B: most passed, C: some passed, D: none passed). | C | | W5 | Add Phase W benchmark gate (WBG-). | Top-level gate asserting NCH + CMC + SCH + BCR all present; overall verdict: hardened/partially_hardened/not_hardened; closes Phase W; no batch-level performance claim is credible without passing this gate. | C | diff --git a/src/openamp_foundry/evidence/similarity_challenge_harness.py b/src/openamp_foundry/evidence/similarity_challenge_harness.py new file mode 100644 index 00000000..63109f36 --- /dev/null +++ b/src/openamp_foundry/evidence/similarity_challenge_harness.py @@ -0,0 +1,167 @@ +"""SCH- similarity challenge harness schema. + +Documents whether pipeline-selected candidates are systematically more similar +to known AMPs than a random draw from the same sequence space. Flags selection +bias from proximity clustering: if the pipeline is essentially selecting +near-neighbors of known AMPs rather than exploring new space, novelty claims +are not credible. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +VALID_SCH_VERDICTS: frozenset[str] = frozenset({ + "selection_adds_value", + "marginal_improvement", + "proximity_driven", + "challenge_not_run", +}) + +VALID_SIMILARITY_METRICS: frozenset[str] = frozenset({ + "sequence_identity", + "blosum62_score", + "edit_distance", + "physicochemical_distance", +}) + +SELECTION_VALUE_GAP_THRESHOLD: float = 0.10 +MARGINAL_IMPROVEMENT_LOWER: float = 0.03 + + +@dataclass +class SimilarityGroupStats: + group_label: str + mean_similarity_to_known: float + n_sequences: int + + +@dataclass +class SimilarityChallengeHarness: + sch_id: str + batch_id: str + pipeline_version: str + similarity_metric: str + pipeline_group: SimilarityGroupStats + random_group: SimilarityGroupStats + similarity_gap: float + sch_verdict: str + dry_lab_only: bool + limitations: list[str] + created_at: str + + +def validate_similarity_challenge_harness(sch: SimilarityChallengeHarness) -> None: + if not sch.sch_id.startswith("SCH-"): + raise ValueError(f"sch_id must start with 'SCH-': {sch.sch_id!r}") + if not sch.batch_id: + raise ValueError("batch_id must be non-empty") + if not sch.pipeline_version: + raise ValueError("pipeline_version must be non-empty") + if sch.similarity_metric not in VALID_SIMILARITY_METRICS: + raise ValueError( + f"similarity_metric {sch.similarity_metric!r} not in VALID_SIMILARITY_METRICS" + ) + for group in (sch.pipeline_group, sch.random_group): + if not (0.0 <= group.mean_similarity_to_known <= 1.0): + raise ValueError( + f"mean_similarity_to_known must be in [0, 1]: {group.mean_similarity_to_known}" + ) + if group.n_sequences < 0: + raise ValueError( + f"n_sequences must be non-negative: {group.n_sequences}" + ) + expected_gap = round( + sch.pipeline_group.mean_similarity_to_known + - sch.random_group.mean_similarity_to_known, + 6, + ) + if abs(sch.similarity_gap - expected_gap) > 1e-4: + raise ValueError( + f"similarity_gap {sch.similarity_gap} does not match computed " + f"{expected_gap}" + ) + if sch.sch_verdict not in VALID_SCH_VERDICTS: + raise ValueError( + f"sch_verdict {sch.sch_verdict!r} not in VALID_SCH_VERDICTS" + ) + if not sch.dry_lab_only: + raise ValueError("dry_lab_only must be True") + if not sch.limitations: + raise ValueError("limitations must be non-empty") + if not sch.created_at: + raise ValueError("created_at must be non-empty") + + +def _compute_verdict( + n_pipeline: int, + n_random: int, + similarity_gap: float, +) -> str: + if n_pipeline == 0 or n_random == 0: + return "challenge_not_run" + if similarity_gap >= SELECTION_VALUE_GAP_THRESHOLD: + return "selection_adds_value" + if similarity_gap >= MARGINAL_IMPROVEMENT_LOWER: + return "marginal_improvement" + return "proximity_driven" + + +def build_similarity_challenge_harness( + *, + sch_id: str, + batch_id: str, + pipeline_version: str, + similarity_metric: str, + pipeline_mean_similarity: float, + pipeline_n_sequences: int, + random_mean_similarity: float, + random_n_sequences: int, + limitations: list[str], + created_at: str, +) -> SimilarityChallengeHarness: + pipeline_group = SimilarityGroupStats( + group_label="pipeline_selected", + mean_similarity_to_known=pipeline_mean_similarity, + n_sequences=pipeline_n_sequences, + ) + random_group = SimilarityGroupStats( + group_label="random_draw", + mean_similarity_to_known=random_mean_similarity, + n_sequences=random_n_sequences, + ) + gap = round(pipeline_mean_similarity - random_mean_similarity, 6) + verdict = _compute_verdict(pipeline_n_sequences, random_n_sequences, gap) + sch = SimilarityChallengeHarness( + sch_id=sch_id, + batch_id=batch_id, + pipeline_version=pipeline_version, + similarity_metric=similarity_metric, + pipeline_group=pipeline_group, + random_group=random_group, + similarity_gap=gap, + sch_verdict=verdict, + dry_lab_only=True, + limitations=limitations, + created_at=created_at, + ) + validate_similarity_challenge_harness(sch) + return sch + + +def format_similarity_challenge_harness(sch: SimilarityChallengeHarness) -> str: + lines = [ + f"Similarity Challenge Harness — {sch.sch_id}", + f"Batch: {sch.batch_id} | Pipeline: {sch.pipeline_version}", + f"Similarity metric: {sch.similarity_metric}", + f"Verdict: {sch.sch_verdict}", + f"Pipeline selected: mean={sch.pipeline_group.mean_similarity_to_known:.4f} " + f"(n={sch.pipeline_group.n_sequences})", + f"Random draw: mean={sch.random_group.mean_similarity_to_known:.4f} " + f"(n={sch.random_group.n_sequences})", + f"Gap (pipeline - random): {sch.similarity_gap:+.4f}", + f"Created: {sch.created_at}", + f"Limitations: {'; '.join(sch.limitations)}", + f"dry_lab_only: {sch.dry_lab_only}", + ] + return "\n".join(lines) diff --git a/tests/evidence/test_similarity_challenge_harness.py b/tests/evidence/test_similarity_challenge_harness.py new file mode 100644 index 00000000..25acba0f --- /dev/null +++ b/tests/evidence/test_similarity_challenge_harness.py @@ -0,0 +1,325 @@ +"""Tests for SCH- similarity challenge harness schema.""" + +import pytest +from openamp_foundry.evidence.similarity_challenge_harness import ( + SimilarityChallengeHarness, + SimilarityGroupStats, + VALID_SCH_VERDICTS, + VALID_SIMILARITY_METRICS, + SELECTION_VALUE_GAP_THRESHOLD, + MARGINAL_IMPROVEMENT_LOWER, + build_similarity_challenge_harness, + format_similarity_challenge_harness, + validate_similarity_challenge_harness, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _build(**kwargs): + defaults = dict( + sch_id="SCH-001", + batch_id="BATCH-01", + pipeline_version="v1.0", + similarity_metric="sequence_identity", + pipeline_mean_similarity=0.55, + pipeline_n_sequences=50, + random_mean_similarity=0.30, + random_n_sequences=50, + limitations=["dry-lab only", "similarity metric is approximate"], + created_at="2026-07-10", + ) + defaults.update(kwargs) + return build_similarity_challenge_harness(**defaults) + + +# --------------------------------------------------------------------------- +# 1. Constants +# --------------------------------------------------------------------------- + + +def test_valid_sch_verdicts_is_frozenset(): + assert isinstance(VALID_SCH_VERDICTS, frozenset) + + +def test_valid_sch_verdicts_contains_selection_adds_value(): + assert "selection_adds_value" in VALID_SCH_VERDICTS + + +def test_valid_sch_verdicts_contains_marginal_improvement(): + assert "marginal_improvement" in VALID_SCH_VERDICTS + + +def test_valid_sch_verdicts_contains_proximity_driven(): + assert "proximity_driven" in VALID_SCH_VERDICTS + + +def test_valid_sch_verdicts_contains_challenge_not_run(): + assert "challenge_not_run" in VALID_SCH_VERDICTS + + +def test_valid_similarity_metrics_is_frozenset(): + assert isinstance(VALID_SIMILARITY_METRICS, frozenset) + + +def test_valid_similarity_metrics_contains_sequence_identity(): + assert "sequence_identity" in VALID_SIMILARITY_METRICS + + +def test_valid_similarity_metrics_contains_blosum62(): + assert "blosum62_score" in VALID_SIMILARITY_METRICS + + +def test_valid_similarity_metrics_contains_edit_distance(): + assert "edit_distance" in VALID_SIMILARITY_METRICS + + +def test_valid_similarity_metrics_contains_physicochemical(): + assert "physicochemical_distance" in VALID_SIMILARITY_METRICS + + +def test_selection_value_gap_threshold(): + assert SELECTION_VALUE_GAP_THRESHOLD == 0.10 + + +def test_marginal_improvement_lower(): + assert MARGINAL_IMPROVEMENT_LOWER == 0.03 + + +# --------------------------------------------------------------------------- +# 2. build – happy paths +# --------------------------------------------------------------------------- + + +def test_build_returns_similarity_challenge_harness(): + assert isinstance(_build(), SimilarityChallengeHarness) + + +def test_build_sch_id_stored(): + assert _build().sch_id == "SCH-001" + + +def test_build_batch_id_stored(): + assert _build().batch_id == "BATCH-01" + + +def test_build_pipeline_version_stored(): + assert _build().pipeline_version == "v1.0" + + +def test_build_dry_lab_only_true(): + assert _build().dry_lab_only is True + + +def test_build_similarity_metric_stored(): + assert _build().similarity_metric == "sequence_identity" + + +def test_build_pipeline_group_is_group_stats(): + assert isinstance(_build().pipeline_group, SimilarityGroupStats) + + +def test_build_random_group_is_group_stats(): + assert isinstance(_build().random_group, SimilarityGroupStats) + + +def test_build_pipeline_group_label(): + assert _build().pipeline_group.group_label == "pipeline_selected" + + +def test_build_random_group_label(): + assert _build().random_group.group_label == "random_draw" + + +def test_build_pipeline_mean_similarity_stored(): + assert abs(_build().pipeline_group.mean_similarity_to_known - 0.55) < 1e-9 + + +def test_build_random_mean_similarity_stored(): + assert abs(_build().random_group.mean_similarity_to_known - 0.30) < 1e-9 + + +def test_build_pipeline_n_sequences_stored(): + assert _build().pipeline_group.n_sequences == 50 + + +def test_build_random_n_sequences_stored(): + assert _build().random_group.n_sequences == 50 + + +def test_build_similarity_gap_computed(): + r = _build() + assert abs(r.similarity_gap - (0.55 - 0.30)) < 1e-4 + + +def test_build_selection_adds_value_verdict(): + r = _build(pipeline_mean_similarity=0.55, random_mean_similarity=0.30) + assert r.sch_verdict == "selection_adds_value" + + +def test_build_marginal_improvement_verdict(): + r = _build(pipeline_mean_similarity=0.33, random_mean_similarity=0.30) + assert r.sch_verdict == "marginal_improvement" + + +def test_build_proximity_driven_verdict(): + r = _build(pipeline_mean_similarity=0.30, random_mean_similarity=0.30) + assert r.sch_verdict == "proximity_driven" + + +def test_build_challenge_not_run_when_pipeline_n_zero(): + r = _build(pipeline_n_sequences=0) + assert r.sch_verdict == "challenge_not_run" + + +def test_build_challenge_not_run_when_random_n_zero(): + r = _build(random_n_sequences=0) + assert r.sch_verdict == "challenge_not_run" + + +def test_build_proximity_driven_when_random_higher(): + r = _build(pipeline_mean_similarity=0.30, random_mean_similarity=0.35) + assert r.sch_verdict == "proximity_driven" + assert r.similarity_gap < 0 + + +def test_build_blosum62_metric(): + r = _build(similarity_metric="blosum62_score") + assert r.similarity_metric == "blosum62_score" + + +def test_build_edit_distance_metric(): + r = _build(similarity_metric="edit_distance") + assert r.similarity_metric == "edit_distance" + + +def test_build_limitations_stored(): + assert "dry-lab only" in _build().limitations + + +def test_build_created_at_stored(): + assert _build().created_at == "2026-07-10" + + +# --------------------------------------------------------------------------- +# 3. validate – rejection cases +# --------------------------------------------------------------------------- + + +def test_validate_rejects_bad_sch_id_prefix(): + with pytest.raises(ValueError, match="SCH-"): + _build(sch_id="BAD-001") + + +def test_validate_rejects_empty_batch_id(): + with pytest.raises(ValueError): + _build(batch_id="") + + +def test_validate_rejects_empty_pipeline_version(): + with pytest.raises(ValueError): + _build(pipeline_version="") + + +def test_validate_rejects_invalid_similarity_metric(): + with pytest.raises(ValueError, match="VALID_SIMILARITY_METRICS"): + _build(similarity_metric="UNKNOWN_METRIC") + + +def test_validate_rejects_pipeline_mean_above_one(): + with pytest.raises(ValueError, match="mean_similarity_to_known"): + _build(pipeline_mean_similarity=1.1) + + +def test_validate_rejects_pipeline_mean_below_zero(): + with pytest.raises(ValueError, match="mean_similarity_to_known"): + _build(pipeline_mean_similarity=-0.01) + + +def test_validate_rejects_random_mean_above_one(): + with pytest.raises(ValueError, match="mean_similarity_to_known"): + _build(random_mean_similarity=1.1) + + +def test_validate_rejects_negative_pipeline_n(): + with pytest.raises(ValueError, match="n_sequences"): + _build(pipeline_n_sequences=-1) + + +def test_validate_rejects_negative_random_n(): + with pytest.raises(ValueError, match="n_sequences"): + _build(random_n_sequences=-1) + + +def test_validate_rejects_similarity_gap_mismatch(): + sch = _build() + sch.similarity_gap = 0.999 + with pytest.raises(ValueError, match="similarity_gap"): + validate_similarity_challenge_harness(sch) + + +def test_validate_rejects_invalid_verdict(): + sch = _build() + sch.sch_verdict = "UNKNOWN" + with pytest.raises(ValueError, match="sch_verdict"): + validate_similarity_challenge_harness(sch) + + +def test_validate_rejects_dry_lab_only_false(): + sch = _build() + sch.dry_lab_only = False + with pytest.raises(ValueError, match="dry_lab_only"): + validate_similarity_challenge_harness(sch) + + +def test_validate_rejects_empty_limitations(): + with pytest.raises(ValueError, match="limitations"): + _build(limitations=[]) + + +def test_validate_rejects_empty_created_at(): + with pytest.raises(ValueError): + _build(created_at="") + + +# --------------------------------------------------------------------------- +# 4. format +# --------------------------------------------------------------------------- + + +def test_format_contains_sch_id(): + assert "SCH-001" in format_similarity_challenge_harness(_build()) + + +def test_format_contains_batch_id(): + assert "BATCH-01" in format_similarity_challenge_harness(_build()) + + +def test_format_contains_similarity_metric(): + assert "sequence_identity" in format_similarity_challenge_harness(_build()) + + +def test_format_contains_verdict(): + assert "selection_adds_value" in format_similarity_challenge_harness(_build()) + + +def test_format_contains_pipeline_mean(): + assert "0.5500" in format_similarity_challenge_harness(_build()) + + +def test_format_contains_random_mean(): + assert "0.3000" in format_similarity_challenge_harness(_build()) + + +def test_format_contains_limitations(): + assert "dry-lab only" in format_similarity_challenge_harness(_build()) + + +def test_format_contains_dry_lab_only(): + assert "dry_lab_only: True" in format_similarity_challenge_harness(_build()) + + +def test_format_is_string(): + assert isinstance(format_similarity_challenge_harness(_build()), str)