|
| 1 | +"""NCH- novelty challenge harness schema. |
| 2 | +
|
| 3 | +Batch-level novelty challenge record: documents what fraction of top |
| 4 | +candidates have high sequence identity (≥ threshold) to known AMPs in a |
| 5 | +reference database. Blocks 'novel family' claims at the batch level when |
| 6 | +the near-neighbor fraction is above the allowed ceiling. |
| 7 | +
|
| 8 | +This schema operates at batch level; per-family novelty is captured in |
| 9 | +the FNR- (family novelty report) schema. NCH- provides the aggregate |
| 10 | +machine-verifiable gate. |
| 11 | +""" |
| 12 | + |
| 13 | +from __future__ import annotations |
| 14 | + |
| 15 | +from dataclasses import dataclass |
| 16 | + |
| 17 | +VALID_NCH_VERDICTS: frozenset[str] = frozenset({ |
| 18 | + "novel_batch", |
| 19 | + "mixed_novelty", |
| 20 | + "near_neighbor_dominated", |
| 21 | + "challenge_not_run", |
| 22 | +}) |
| 23 | + |
| 24 | +VALID_REFERENCE_DATABASES: frozenset[str] = frozenset({ |
| 25 | + "APD3", |
| 26 | + "DRAMP", |
| 27 | + "DBAASP", |
| 28 | + "CAMP", |
| 29 | + "LAMP", |
| 30 | + "custom", |
| 31 | +}) |
| 32 | + |
| 33 | +NEAR_NEIGHBOR_IDENTITY_THRESHOLD: float = 0.80 |
| 34 | +NOVEL_BATCH_CEILING: float = 0.20 |
| 35 | +NEAR_NEIGHBOR_DOMINATED_FLOOR: float = 0.60 |
| 36 | + |
| 37 | + |
| 38 | +@dataclass |
| 39 | +class NCHCandidateResult: |
| 40 | + candidate_id: str |
| 41 | + max_identity_to_known: float |
| 42 | + is_near_neighbor: bool |
| 43 | + closest_known_amp_id: str |
| 44 | + |
| 45 | + |
| 46 | +@dataclass |
| 47 | +class NoveltyChallengeHarness: |
| 48 | + nch_id: str |
| 49 | + batch_id: str |
| 50 | + pipeline_version: str |
| 51 | + reference_database: str |
| 52 | + identity_threshold: float |
| 53 | + n_candidates_checked: int |
| 54 | + n_near_neighbors: int |
| 55 | + near_neighbor_fraction: float |
| 56 | + candidate_results: list[NCHCandidateResult] |
| 57 | + nch_verdict: str |
| 58 | + dry_lab_only: bool |
| 59 | + limitations: list[str] |
| 60 | + created_at: str |
| 61 | + |
| 62 | + |
| 63 | +def validate_novelty_challenge_harness(nch: NoveltyChallengeHarness) -> None: |
| 64 | + if not nch.nch_id.startswith("NCH-"): |
| 65 | + raise ValueError(f"nch_id must start with 'NCH-': {nch.nch_id!r}") |
| 66 | + if not nch.batch_id: |
| 67 | + raise ValueError("batch_id must be non-empty") |
| 68 | + if not nch.pipeline_version: |
| 69 | + raise ValueError("pipeline_version must be non-empty") |
| 70 | + if nch.reference_database not in VALID_REFERENCE_DATABASES: |
| 71 | + raise ValueError( |
| 72 | + f"reference_database {nch.reference_database!r} not in VALID_REFERENCE_DATABASES" |
| 73 | + ) |
| 74 | + if not (0.0 < nch.identity_threshold <= 1.0): |
| 75 | + raise ValueError( |
| 76 | + f"identity_threshold must be in (0, 1]: {nch.identity_threshold}" |
| 77 | + ) |
| 78 | + if nch.n_candidates_checked < 0: |
| 79 | + raise ValueError("n_candidates_checked must be non-negative") |
| 80 | + if nch.n_near_neighbors < 0: |
| 81 | + raise ValueError("n_near_neighbors must be non-negative") |
| 82 | + if nch.n_near_neighbors > nch.n_candidates_checked: |
| 83 | + raise ValueError("n_near_neighbors cannot exceed n_candidates_checked") |
| 84 | + for cr in nch.candidate_results: |
| 85 | + if not (0.0 <= cr.max_identity_to_known <= 1.0): |
| 86 | + raise ValueError( |
| 87 | + f"max_identity_to_known must be in [0, 1]: {cr.max_identity_to_known}" |
| 88 | + ) |
| 89 | + expected_nn = cr.max_identity_to_known >= nch.identity_threshold |
| 90 | + if cr.is_near_neighbor != expected_nn: |
| 91 | + raise ValueError( |
| 92 | + f"is_near_neighbor mismatch for {cr.candidate_id!r}: " |
| 93 | + f"identity={cr.max_identity_to_known}, threshold={nch.identity_threshold}" |
| 94 | + ) |
| 95 | + if nch.n_candidates_checked != len(nch.candidate_results): |
| 96 | + raise ValueError("n_candidates_checked must equal len(candidate_results)") |
| 97 | + expected_nn_count = sum(1 for cr in nch.candidate_results if cr.is_near_neighbor) |
| 98 | + if nch.n_near_neighbors != expected_nn_count: |
| 99 | + raise ValueError("n_near_neighbors mismatch") |
| 100 | + if nch.n_candidates_checked == 0: |
| 101 | + expected_fraction = 0.0 |
| 102 | + else: |
| 103 | + expected_fraction = round( |
| 104 | + nch.n_near_neighbors / nch.n_candidates_checked, 6 |
| 105 | + ) |
| 106 | + if abs(nch.near_neighbor_fraction - expected_fraction) > 1e-4: |
| 107 | + raise ValueError( |
| 108 | + f"near_neighbor_fraction {nch.near_neighbor_fraction} does not match " |
| 109 | + f"computed {expected_fraction}" |
| 110 | + ) |
| 111 | + if nch.nch_verdict not in VALID_NCH_VERDICTS: |
| 112 | + raise ValueError( |
| 113 | + f"nch_verdict {nch.nch_verdict!r} not in VALID_NCH_VERDICTS" |
| 114 | + ) |
| 115 | + if not nch.dry_lab_only: |
| 116 | + raise ValueError("dry_lab_only must be True") |
| 117 | + if not nch.limitations: |
| 118 | + raise ValueError("limitations must be non-empty") |
| 119 | + if not nch.created_at: |
| 120 | + raise ValueError("created_at must be non-empty") |
| 121 | + |
| 122 | + |
| 123 | +def _compute_verdict( |
| 124 | + n_candidates: int, |
| 125 | + near_neighbor_fraction: float, |
| 126 | +) -> str: |
| 127 | + if n_candidates == 0: |
| 128 | + return "challenge_not_run" |
| 129 | + if near_neighbor_fraction <= NOVEL_BATCH_CEILING: |
| 130 | + return "novel_batch" |
| 131 | + if near_neighbor_fraction >= NEAR_NEIGHBOR_DOMINATED_FLOOR: |
| 132 | + return "near_neighbor_dominated" |
| 133 | + return "mixed_novelty" |
| 134 | + |
| 135 | + |
| 136 | +def build_novelty_challenge_harness( |
| 137 | + *, |
| 138 | + nch_id: str, |
| 139 | + batch_id: str, |
| 140 | + pipeline_version: str, |
| 141 | + reference_database: str, |
| 142 | + identity_threshold: float = NEAR_NEIGHBOR_IDENTITY_THRESHOLD, |
| 143 | + candidate_result_dicts: list[dict], |
| 144 | + limitations: list[str], |
| 145 | + created_at: str, |
| 146 | +) -> NoveltyChallengeHarness: |
| 147 | + """Build a NoveltyChallengeHarness. |
| 148 | +
|
| 149 | + candidate_result_dicts: list of dicts with keys: |
| 150 | + candidate_id (str), max_identity_to_known (float), |
| 151 | + closest_known_amp_id (str, optional, default "") |
| 152 | + """ |
| 153 | + candidate_results = [] |
| 154 | + for d in candidate_result_dicts: |
| 155 | + identity = float(d["max_identity_to_known"]) |
| 156 | + is_nn = identity >= identity_threshold |
| 157 | + candidate_results.append( |
| 158 | + NCHCandidateResult( |
| 159 | + candidate_id=d["candidate_id"], |
| 160 | + max_identity_to_known=identity, |
| 161 | + is_near_neighbor=is_nn, |
| 162 | + closest_known_amp_id=d.get("closest_known_amp_id", ""), |
| 163 | + ) |
| 164 | + ) |
| 165 | + n = len(candidate_results) |
| 166 | + n_nn = sum(1 for cr in candidate_results if cr.is_near_neighbor) |
| 167 | + fraction = round(n_nn / n, 6) if n > 0 else 0.0 |
| 168 | + verdict = _compute_verdict(n, fraction) |
| 169 | + nch = NoveltyChallengeHarness( |
| 170 | + nch_id=nch_id, |
| 171 | + batch_id=batch_id, |
| 172 | + pipeline_version=pipeline_version, |
| 173 | + reference_database=reference_database, |
| 174 | + identity_threshold=identity_threshold, |
| 175 | + n_candidates_checked=n, |
| 176 | + n_near_neighbors=n_nn, |
| 177 | + near_neighbor_fraction=fraction, |
| 178 | + candidate_results=candidate_results, |
| 179 | + nch_verdict=verdict, |
| 180 | + dry_lab_only=True, |
| 181 | + limitations=limitations, |
| 182 | + created_at=created_at, |
| 183 | + ) |
| 184 | + validate_novelty_challenge_harness(nch) |
| 185 | + return nch |
| 186 | + |
| 187 | + |
| 188 | +def format_novelty_challenge_harness(nch: NoveltyChallengeHarness) -> str: |
| 189 | + lines = [ |
| 190 | + f"Novelty Challenge Harness — {nch.nch_id}", |
| 191 | + f"Batch: {nch.batch_id} | Pipeline: {nch.pipeline_version}", |
| 192 | + f"Reference DB: {nch.reference_database} | Identity threshold: {nch.identity_threshold:.0%}", |
| 193 | + f"Verdict: {nch.nch_verdict}", |
| 194 | + f"Near-neighbors: {nch.n_near_neighbors}/{nch.n_candidates_checked} " |
| 195 | + f"({nch.near_neighbor_fraction:.1%})", |
| 196 | + ] |
| 197 | + if nch.candidate_results: |
| 198 | + lines.append("Top candidates:") |
| 199 | + for cr in nch.candidate_results[:5]: |
| 200 | + nn_label = "NEAR-NEIGHBOR" if cr.is_near_neighbor else "novel" |
| 201 | + lines.append( |
| 202 | + f" {cr.candidate_id}: {cr.max_identity_to_known:.1%} identity [{nn_label}]" |
| 203 | + ) |
| 204 | + if len(nch.candidate_results) > 5: |
| 205 | + lines.append(f" ... ({len(nch.candidate_results) - 5} more)") |
| 206 | + lines.append(f"Created: {nch.created_at}") |
| 207 | + lines.append(f"Limitations: {'; '.join(nch.limitations)}") |
| 208 | + lines.append(f"dry_lab_only: {nch.dry_lab_only}") |
| 209 | + return "\n".join(lines) |
0 commit comments