|
1 | | -"""External review packet schema — Phase E E1. |
| 1 | +"""ERP- external review packet schema. |
2 | 2 |
|
3 | | -Machine-checkable ERP- schema for complete external review packets. |
4 | | -Enables lab partners and reviewers to verify what a review packet contains |
5 | | -before running experiments. |
6 | | -
|
7 | | -Key safety invariants: |
8 | | -- dry_lab_only_attestation must always be True |
9 | | -- proof_ladder_level capped at 2 (computationally nominated) for dry-lab packets |
10 | | -- safety_attestations.no_known_toxicity_claim must be True |
11 | | -- limitations list must be non-empty |
| 3 | +Assembles BRC + ECI + FET + PTR + SRS into a single record listing everything |
| 4 | +a scientist needs to review the batch's computational evidence. Dry-lab-only |
| 5 | +constraint explicit. Closes the "what do I send to a reviewer?" question. |
12 | 6 | """ |
| 7 | + |
13 | 8 | from __future__ import annotations |
14 | 9 |
|
15 | | -import re |
16 | | -from dataclasses import dataclass, field |
| 10 | +from dataclasses import dataclass |
17 | 11 |
|
| 12 | +REQUIRED_PACKET_COMPONENTS: tuple[str, ...] = ( |
| 13 | + "BRC", "ECI", "FET", "PTR", "SRS", |
| 14 | +) |
18 | 15 |
|
19 | | -VALID_CALIBRATION_ASSESSMENTS: frozenset[str] = frozenset({ |
20 | | - "well_calibrated", |
21 | | - "moderately_calibrated", |
22 | | - "poorly_calibrated", |
23 | | - "uninformative", |
| 16 | +VALID_PACKET_STATUSES: frozenset[str] = frozenset({ |
| 17 | + "ready", "incomplete", "draft", |
24 | 18 | }) |
25 | 19 |
|
26 | | -DRY_LAB_MAX_PROOF_LADDER_LEVEL: int = 2 |
27 | | -_SEMVER_RE = re.compile(r"^\d+\.\d+\.\d+$") |
28 | | -_GIT_SHA_RE = re.compile(r"^[a-f0-9]{7,40}$") |
29 | | -_AA_SEQ_RE = re.compile(r"^[ACDEFGHIKLMNPQRSTVWY]+$") |
30 | | - |
31 | | - |
32 | | -@dataclass |
33 | | -class CandidateEntry: |
34 | | - candidate_id: str |
35 | | - sequence: str |
36 | | - ensemble_score: float |
37 | | - proof_ladder_level: int |
38 | | - family: str = "" |
39 | | - selection_rationale: str = "" |
40 | | - safety_notes: str = "" |
41 | | - |
42 | | - |
43 | | -@dataclass |
44 | | -class BenchmarkSummary: |
45 | | - auroc: float |
46 | | - benchmark_name: str |
47 | | - n_positives: int |
48 | | - n_negatives: int |
49 | | - |
50 | | - |
51 | | -@dataclass |
52 | | -class CalibrationSummary: |
53 | | - calibration_assessment: str |
54 | | - n_calibration_samples: int = 0 |
55 | | - |
56 | 20 |
|
57 | 21 | @dataclass |
58 | | -class SafetyAttestations: |
59 | | - reviewed_by_human: bool |
60 | | - safety_gate_passed: bool |
61 | | - no_known_toxicity_claim: bool |
62 | | - hemolysis_assay_mandatory: bool = True |
| 22 | +class PacketComponent: |
| 23 | + component_type: str |
| 24 | + artifact_id: str |
| 25 | + present: bool |
63 | 26 |
|
64 | 27 |
|
65 | 28 | @dataclass |
66 | 29 | class ExternalReviewPacket: |
67 | | - packet_id: str |
68 | | - version: str |
69 | | - generated_at: str |
| 30 | + erp_id: str |
| 31 | + batch_id: str |
70 | 32 | pipeline_version: str |
71 | | - git_sha: str |
72 | | - candidate_count: int |
73 | | - candidates: list[CandidateEntry] = field(default_factory=list) |
74 | | - benchmark_summary: BenchmarkSummary | None = None |
75 | | - calibration_summary: CalibrationSummary | None = None |
76 | | - limitations: list[str] = field(default_factory=list) |
77 | | - safety_attestations: SafetyAttestations | None = None |
78 | | - dry_lab_only_attestation: bool = True |
79 | | - proof_ladder_level: int = 1 |
80 | | - contact: str = "" |
81 | | - notes: str = "" |
82 | | - |
83 | | - |
84 | | -@dataclass |
85 | | -class PacketValidationResult: |
86 | | - is_valid: bool |
87 | | - violations: list[str] |
88 | | - warnings: list[str] |
89 | | - packet_id: str |
90 | | - validation_summary: str |
91 | | - |
92 | | - |
93 | | -def validate_external_review_packet( |
94 | | - packet: ExternalReviewPacket, |
95 | | -) -> PacketValidationResult: |
96 | | - """Validate an ExternalReviewPacket against ERP- schema rules. |
97 | | -
|
98 | | - Returns a PacketValidationResult with all violations and warnings. |
99 | | - A packet is valid only when violations is empty. |
100 | | - """ |
101 | | - violations: list[str] = [] |
102 | | - warnings: list[str] = [] |
103 | | - |
104 | | - # Rule 1: packet_id prefix |
105 | | - if not packet.packet_id.startswith("ERP-"): |
106 | | - violations.append( |
107 | | - f"packet_id must start with 'ERP-', got '{packet.packet_id}'" |
| 33 | + components: list[PacketComponent] |
| 34 | + n_components_required: int |
| 35 | + n_components_present: int |
| 36 | + missing_component_types: list[str] |
| 37 | + packet_status: str |
| 38 | + dry_lab_only: bool |
| 39 | + limitations: list[str] |
| 40 | + created_at: str |
| 41 | + |
| 42 | + |
| 43 | +def _compute_status(n_present: int, n_required: int) -> str: |
| 44 | + if n_present == n_required: |
| 45 | + return "ready" |
| 46 | + if n_present == 0: |
| 47 | + return "draft" |
| 48 | + return "incomplete" |
| 49 | + |
| 50 | + |
| 51 | +def validate_external_review_packet(erp: ExternalReviewPacket) -> None: |
| 52 | + if not erp.erp_id.startswith("ERP-"): |
| 53 | + raise ValueError(f"erp_id must start with 'ERP-': {erp.erp_id!r}") |
| 54 | + if not erp.batch_id: |
| 55 | + raise ValueError("batch_id must be non-empty") |
| 56 | + if not erp.pipeline_version: |
| 57 | + raise ValueError("pipeline_version must be non-empty") |
| 58 | + component_types = [c.component_type for c in erp.components] |
| 59 | + for req in REQUIRED_PACKET_COMPONENTS: |
| 60 | + if component_types.count(req) != 1: |
| 61 | + raise ValueError(f"Exactly one component required for {req!r}") |
| 62 | + if erp.packet_status not in VALID_PACKET_STATUSES: |
| 63 | + raise ValueError( |
| 64 | + f"packet_status {erp.packet_status!r} not in VALID_PACKET_STATUSES" |
108 | 65 | ) |
109 | | - |
110 | | - # Rule 2: version semver format |
111 | | - if not _SEMVER_RE.match(packet.version): |
112 | | - violations.append( |
113 | | - f"version must be semver (X.Y.Z), got '{packet.version}'" |
114 | | - ) |
115 | | - |
116 | | - # Rule 3: git_sha hex format |
117 | | - if not _GIT_SHA_RE.match(packet.git_sha): |
118 | | - violations.append( |
119 | | - f"git_sha must be 7-40 lowercase hex chars, got '{packet.git_sha}'" |
| 66 | + if not erp.dry_lab_only: |
| 67 | + raise ValueError("dry_lab_only must be True") |
| 68 | + if not erp.limitations: |
| 69 | + raise ValueError("limitations must be non-empty") |
| 70 | + if not erp.created_at: |
| 71 | + raise ValueError("created_at must be non-empty") |
| 72 | + n_req = len(REQUIRED_PACKET_COMPONENTS) |
| 73 | + if erp.n_components_required != n_req: |
| 74 | + raise ValueError( |
| 75 | + f"n_components_required must be {n_req}, got {erp.n_components_required}" |
120 | 76 | ) |
121 | | - |
122 | | - # Rule 4: candidate_count consistency |
123 | | - if packet.candidate_count != len(packet.candidates): |
124 | | - violations.append( |
125 | | - f"candidate_count ({packet.candidate_count}) does not match " |
126 | | - f"len(candidates) ({len(packet.candidates)})" |
127 | | - ) |
128 | | - |
129 | | - # Rule 5: candidates not empty |
130 | | - if not packet.candidates: |
131 | | - violations.append("candidates must not be empty (minItems: 1)") |
132 | | - |
133 | | - # Rule 6: dry_lab_only_attestation must be True |
134 | | - if not packet.dry_lab_only_attestation: |
135 | | - violations.append( |
136 | | - "dry_lab_only_attestation must be True — " |
137 | | - "this packet contains dry-lab computational outputs only" |
138 | | - ) |
139 | | - |
140 | | - # Rule 7: packet-level proof_ladder_level cap |
141 | | - if packet.proof_ladder_level > DRY_LAB_MAX_PROOF_LADDER_LEVEL: |
142 | | - violations.append( |
143 | | - f"proof_ladder_level ({packet.proof_ladder_level}) exceeds dry-lab cap " |
144 | | - f"of {DRY_LAB_MAX_PROOF_LADDER_LEVEL}. " |
145 | | - "Dry-lab outputs cannot claim beyond 'computationally nominated' (Level 2)." |
146 | | - ) |
147 | | - |
148 | | - # Rule 8: proof_ladder_level must be >= 1 |
149 | | - if packet.proof_ladder_level < 1: |
150 | | - violations.append( |
151 | | - f"proof_ladder_level must be >= 1, got {packet.proof_ladder_level}" |
152 | | - ) |
153 | | - |
154 | | - # Rule 9: safety_attestations |
155 | | - if packet.safety_attestations is None: |
156 | | - violations.append("safety_attestations is required") |
157 | | - else: |
158 | | - if not packet.safety_attestations.no_known_toxicity_claim: |
159 | | - violations.append( |
160 | | - "safety_attestations.no_known_toxicity_claim must be True — " |
161 | | - "dry-lab scores do not prove biological safety" |
162 | | - ) |
163 | | - |
164 | | - # Rule 10: limitations non-empty |
165 | | - if not packet.limitations: |
166 | | - violations.append( |
167 | | - "limitations must contain at least one entry. " |
168 | | - "Minimum: 'Computational outputs are hypotheses and review aids. " |
169 | | - "They are not biological proof.'" |
170 | | - ) |
171 | | - |
172 | | - # Rule 11: benchmark_summary auroc range |
173 | | - if packet.benchmark_summary is not None: |
174 | | - if not (0.0 <= packet.benchmark_summary.auroc <= 1.0): |
175 | | - violations.append( |
176 | | - f"benchmark_summary.auroc must be in [0, 1], " |
177 | | - f"got {packet.benchmark_summary.auroc}" |
178 | | - ) |
179 | | - |
180 | | - # Rule 12: calibration_assessment vocabulary |
181 | | - if packet.calibration_summary is not None: |
182 | | - if ( |
183 | | - packet.calibration_summary.calibration_assessment |
184 | | - not in VALID_CALIBRATION_ASSESSMENTS |
185 | | - ): |
186 | | - violations.append( |
187 | | - f"calibration_summary.calibration_assessment must be one of " |
188 | | - f"{sorted(VALID_CALIBRATION_ASSESSMENTS)}, " |
189 | | - f"got '{packet.calibration_summary.calibration_assessment}'" |
190 | | - ) |
191 | | - |
192 | | - # Rule 13+14: per-candidate validation |
193 | | - for i, cand in enumerate(packet.candidates): |
194 | | - if not (0.0 <= cand.ensemble_score <= 1.0): |
195 | | - violations.append( |
196 | | - f"candidates[{i}].ensemble_score must be in [0, 1], " |
197 | | - f"got {cand.ensemble_score}" |
198 | | - ) |
199 | | - if cand.proof_ladder_level > DRY_LAB_MAX_PROOF_LADDER_LEVEL: |
200 | | - violations.append( |
201 | | - f"candidates[{i}].proof_ladder_level ({cand.proof_ladder_level}) " |
202 | | - f"exceeds dry-lab cap of {DRY_LAB_MAX_PROOF_LADDER_LEVEL}" |
203 | | - ) |
204 | | - |
205 | | - is_valid = len(violations) == 0 |
206 | | - |
207 | | - if is_valid: |
208 | | - summary = ( |
209 | | - f"ERP- packet '{packet.packet_id}' is valid: " |
210 | | - f"{len(packet.candidates)} candidate(s), " |
211 | | - f"proof_ladder_level={packet.proof_ladder_level}, " |
212 | | - "dry_lab_only=True." |
213 | | - ) |
214 | | - else: |
215 | | - summary = ( |
216 | | - f"ERP- packet '{packet.packet_id}' has {len(violations)} violation(s). " |
217 | | - "Fix all violations before sending to external reviewers." |
| 77 | + n_present = sum(1 for c in erp.components if c.present) |
| 78 | + if erp.n_components_present != n_present: |
| 79 | + raise ValueError("n_components_present mismatch") |
| 80 | + expected_missing = sorted( |
| 81 | + c.component_type for c in erp.components if not c.present |
| 82 | + ) |
| 83 | + if erp.missing_component_types != expected_missing: |
| 84 | + raise ValueError("missing_component_types mismatch") |
| 85 | + |
| 86 | + |
| 87 | +def build_external_review_packet( |
| 88 | + *, |
| 89 | + erp_id: str, |
| 90 | + batch_id: str, |
| 91 | + pipeline_version: str, |
| 92 | + brc_artifact_id: str = "", |
| 93 | + eci_artifact_id: str = "", |
| 94 | + fet_artifact_id: str = "", |
| 95 | + ptr_artifact_id: str = "", |
| 96 | + srs_artifact_id: str = "", |
| 97 | + limitations: list[str], |
| 98 | + created_at: str, |
| 99 | +) -> ExternalReviewPacket: |
| 100 | + artifact_map = { |
| 101 | + "BRC": brc_artifact_id, |
| 102 | + "ECI": eci_artifact_id, |
| 103 | + "FET": fet_artifact_id, |
| 104 | + "PTR": ptr_artifact_id, |
| 105 | + "SRS": srs_artifact_id, |
| 106 | + } |
| 107 | + components = [ |
| 108 | + PacketComponent( |
| 109 | + component_type=ct, |
| 110 | + artifact_id=artifact_map[ct], |
| 111 | + present=bool(artifact_map[ct]), |
218 | 112 | ) |
219 | | - |
220 | | - return PacketValidationResult( |
221 | | - is_valid=is_valid, |
222 | | - violations=violations, |
223 | | - warnings=warnings, |
224 | | - packet_id=packet.packet_id, |
225 | | - validation_summary=summary, |
| 113 | + for ct in REQUIRED_PACKET_COMPONENTS |
| 114 | + ] |
| 115 | + n_req = len(REQUIRED_PACKET_COMPONENTS) |
| 116 | + n_present = sum(1 for c in components if c.present) |
| 117 | + missing = sorted(c.component_type for c in components if not c.present) |
| 118 | + status = _compute_status(n_present, n_req) |
| 119 | + |
| 120 | + erp = ExternalReviewPacket( |
| 121 | + erp_id=erp_id, |
| 122 | + batch_id=batch_id, |
| 123 | + pipeline_version=pipeline_version, |
| 124 | + components=components, |
| 125 | + n_components_required=n_req, |
| 126 | + n_components_present=n_present, |
| 127 | + missing_component_types=missing, |
| 128 | + packet_status=status, |
| 129 | + dry_lab_only=True, |
| 130 | + limitations=limitations, |
| 131 | + created_at=created_at, |
226 | 132 | ) |
| 133 | + validate_external_review_packet(erp) |
| 134 | + return erp |
227 | 135 |
|
228 | 136 |
|
229 | | -def format_external_review_packet(packet: ExternalReviewPacket) -> str: |
230 | | - """Format an ExternalReviewPacket as a human-readable string.""" |
231 | | - result = validate_external_review_packet(packet) |
| 137 | +def format_external_review_packet(erp: ExternalReviewPacket) -> str: |
232 | 138 | lines = [ |
233 | | - "=== EXTERNAL REVIEW PACKET ===", |
234 | | - f"Packet ID: {packet.packet_id}", |
235 | | - f"Version: {packet.version}", |
236 | | - f"Pipeline: {packet.pipeline_version} ({packet.git_sha})", |
237 | | - f"Generated: {packet.generated_at}", |
238 | | - f"Candidates: {packet.candidate_count}", |
239 | | - f"Proof ladder level: {packet.proof_ladder_level}", |
240 | | - f"Dry-lab only: {'YES' if packet.dry_lab_only_attestation else 'NO'}", |
241 | | - "", |
242 | | - "-- SAFETY ATTESTATIONS --", |
| 139 | + f"External Review Packet — {erp.erp_id}", |
| 140 | + f"Batch: {erp.batch_id} | Pipeline: {erp.pipeline_version}", |
| 141 | + f"Status: {erp.packet_status} | Components: {erp.n_components_present}/{erp.n_components_required}", |
243 | 142 | ] |
244 | | - if packet.safety_attestations: |
245 | | - lines.extend([ |
246 | | - f" Reviewed by human: {packet.safety_attestations.reviewed_by_human}", |
247 | | - f" Safety gate passed: {packet.safety_attestations.safety_gate_passed}", |
248 | | - f" No toxicity claim: {packet.safety_attestations.no_known_toxicity_claim}", |
249 | | - f" Hemolysis assay mandatory: {packet.safety_attestations.hemolysis_assay_mandatory}", |
250 | | - ]) |
251 | | - lines.extend(["", "-- LIMITATIONS --"]) |
252 | | - for lim in packet.limitations: |
253 | | - lines.append(f" • {lim}") |
254 | | - lines.extend(["", "-- VALIDATION --"]) |
255 | | - if result.is_valid: |
256 | | - lines.append(" STATUS: VALID") |
257 | | - else: |
258 | | - lines.append(f" STATUS: INVALID ({len(result.violations)} violation(s))") |
259 | | - for v in result.violations: |
260 | | - lines.append(f" ✗ {v}") |
261 | | - lines.extend(["", result.validation_summary]) |
262 | | - if not result.is_valid: |
263 | | - lines.extend([ |
264 | | - "", |
265 | | - "NOTICE: Do not send this packet to external reviewers until all violations are resolved.", |
266 | | - ]) |
| 143 | + if erp.missing_component_types: |
| 144 | + lines.append(f"Missing: {', '.join(erp.missing_component_types)}") |
| 145 | + lines.append("Components:") |
| 146 | + for comp in erp.components: |
| 147 | + status_label = "PRESENT" if comp.present else "MISSING" |
| 148 | + artifact_str = comp.artifact_id if comp.artifact_id else "(none)" |
| 149 | + lines.append(f" [{status_label}] {comp.component_type}: {artifact_str}") |
| 150 | + lines.append(f"Created: {erp.created_at}") |
| 151 | + lines.append(f"Limitations: {'; '.join(erp.limitations)}") |
| 152 | + lines.append(f"dry_lab_only: {erp.dry_lab_only}") |
267 | 153 | return "\n".join(lines) |
0 commit comments