Skip to content

Commit 022bb5f

Browse files
cschanhniemOpenCode
andauthored
feat: Phase V V4 external review packet schema (ERP-) -- assembles BRC+ECI+FET+PTR+SRS for scientist reviewer; REQUIRED_PACKET_COMPONENTS (5), VALID_PACKET_STATUSES (ready/incomplete/draft); PacketComponent per type; ready when all 5 present; dry_lab_only=True; 45 tests (#1001) (#1001)
Co-authored-by: OpenCode <opencode@example.com>
1 parent 9719f8b commit 022bb5f

3 files changed

Lines changed: 350 additions & 598 deletions

File tree

Lines changed: 129 additions & 243 deletions
Original file line numberDiff line numberDiff line change
@@ -1,267 +1,153 @@
1-
"""External review packet schema — Phase E E1.
1+
"""ERP- external review packet schema.
22
3-
Machine-checkable ERP- schema for complete external review packets.
4-
Enables lab partners and reviewers to verify what a review packet contains
5-
before running experiments.
6-
7-
Key safety invariants:
8-
- dry_lab_only_attestation must always be True
9-
- proof_ladder_level capped at 2 (computationally nominated) for dry-lab packets
10-
- safety_attestations.no_known_toxicity_claim must be True
11-
- limitations list must be non-empty
3+
Assembles BRC + ECI + FET + PTR + SRS into a single record listing everything
4+
a scientist needs to review the batch's computational evidence. Dry-lab-only
5+
constraint explicit. Closes the "what do I send to a reviewer?" question.
126
"""
7+
138
from __future__ import annotations
149

15-
import re
16-
from dataclasses import dataclass, field
10+
from dataclasses import dataclass
1711

12+
REQUIRED_PACKET_COMPONENTS: tuple[str, ...] = (
13+
"BRC", "ECI", "FET", "PTR", "SRS",
14+
)
1815

19-
VALID_CALIBRATION_ASSESSMENTS: frozenset[str] = frozenset({
20-
"well_calibrated",
21-
"moderately_calibrated",
22-
"poorly_calibrated",
23-
"uninformative",
16+
VALID_PACKET_STATUSES: frozenset[str] = frozenset({
17+
"ready", "incomplete", "draft",
2418
})
2519

26-
DRY_LAB_MAX_PROOF_LADDER_LEVEL: int = 2
27-
_SEMVER_RE = re.compile(r"^\d+\.\d+\.\d+$")
28-
_GIT_SHA_RE = re.compile(r"^[a-f0-9]{7,40}$")
29-
_AA_SEQ_RE = re.compile(r"^[ACDEFGHIKLMNPQRSTVWY]+$")
30-
31-
32-
@dataclass
33-
class CandidateEntry:
34-
candidate_id: str
35-
sequence: str
36-
ensemble_score: float
37-
proof_ladder_level: int
38-
family: str = ""
39-
selection_rationale: str = ""
40-
safety_notes: str = ""
41-
42-
43-
@dataclass
44-
class BenchmarkSummary:
45-
auroc: float
46-
benchmark_name: str
47-
n_positives: int
48-
n_negatives: int
49-
50-
51-
@dataclass
52-
class CalibrationSummary:
53-
calibration_assessment: str
54-
n_calibration_samples: int = 0
55-
5620

5721
@dataclass
58-
class SafetyAttestations:
59-
reviewed_by_human: bool
60-
safety_gate_passed: bool
61-
no_known_toxicity_claim: bool
62-
hemolysis_assay_mandatory: bool = True
22+
class PacketComponent:
23+
component_type: str
24+
artifact_id: str
25+
present: bool
6326

6427

6528
@dataclass
6629
class ExternalReviewPacket:
67-
packet_id: str
68-
version: str
69-
generated_at: str
30+
erp_id: str
31+
batch_id: str
7032
pipeline_version: str
71-
git_sha: str
72-
candidate_count: int
73-
candidates: list[CandidateEntry] = field(default_factory=list)
74-
benchmark_summary: BenchmarkSummary | None = None
75-
calibration_summary: CalibrationSummary | None = None
76-
limitations: list[str] = field(default_factory=list)
77-
safety_attestations: SafetyAttestations | None = None
78-
dry_lab_only_attestation: bool = True
79-
proof_ladder_level: int = 1
80-
contact: str = ""
81-
notes: str = ""
82-
83-
84-
@dataclass
85-
class PacketValidationResult:
86-
is_valid: bool
87-
violations: list[str]
88-
warnings: list[str]
89-
packet_id: str
90-
validation_summary: str
91-
92-
93-
def validate_external_review_packet(
94-
packet: ExternalReviewPacket,
95-
) -> PacketValidationResult:
96-
"""Validate an ExternalReviewPacket against ERP- schema rules.
97-
98-
Returns a PacketValidationResult with all violations and warnings.
99-
A packet is valid only when violations is empty.
100-
"""
101-
violations: list[str] = []
102-
warnings: list[str] = []
103-
104-
# Rule 1: packet_id prefix
105-
if not packet.packet_id.startswith("ERP-"):
106-
violations.append(
107-
f"packet_id must start with 'ERP-', got '{packet.packet_id}'"
33+
components: list[PacketComponent]
34+
n_components_required: int
35+
n_components_present: int
36+
missing_component_types: list[str]
37+
packet_status: str
38+
dry_lab_only: bool
39+
limitations: list[str]
40+
created_at: str
41+
42+
43+
def _compute_status(n_present: int, n_required: int) -> str:
44+
if n_present == n_required:
45+
return "ready"
46+
if n_present == 0:
47+
return "draft"
48+
return "incomplete"
49+
50+
51+
def validate_external_review_packet(erp: ExternalReviewPacket) -> None:
52+
if not erp.erp_id.startswith("ERP-"):
53+
raise ValueError(f"erp_id must start with 'ERP-': {erp.erp_id!r}")
54+
if not erp.batch_id:
55+
raise ValueError("batch_id must be non-empty")
56+
if not erp.pipeline_version:
57+
raise ValueError("pipeline_version must be non-empty")
58+
component_types = [c.component_type for c in erp.components]
59+
for req in REQUIRED_PACKET_COMPONENTS:
60+
if component_types.count(req) != 1:
61+
raise ValueError(f"Exactly one component required for {req!r}")
62+
if erp.packet_status not in VALID_PACKET_STATUSES:
63+
raise ValueError(
64+
f"packet_status {erp.packet_status!r} not in VALID_PACKET_STATUSES"
10865
)
109-
110-
# Rule 2: version semver format
111-
if not _SEMVER_RE.match(packet.version):
112-
violations.append(
113-
f"version must be semver (X.Y.Z), got '{packet.version}'"
114-
)
115-
116-
# Rule 3: git_sha hex format
117-
if not _GIT_SHA_RE.match(packet.git_sha):
118-
violations.append(
119-
f"git_sha must be 7-40 lowercase hex chars, got '{packet.git_sha}'"
66+
if not erp.dry_lab_only:
67+
raise ValueError("dry_lab_only must be True")
68+
if not erp.limitations:
69+
raise ValueError("limitations must be non-empty")
70+
if not erp.created_at:
71+
raise ValueError("created_at must be non-empty")
72+
n_req = len(REQUIRED_PACKET_COMPONENTS)
73+
if erp.n_components_required != n_req:
74+
raise ValueError(
75+
f"n_components_required must be {n_req}, got {erp.n_components_required}"
12076
)
121-
122-
# Rule 4: candidate_count consistency
123-
if packet.candidate_count != len(packet.candidates):
124-
violations.append(
125-
f"candidate_count ({packet.candidate_count}) does not match "
126-
f"len(candidates) ({len(packet.candidates)})"
127-
)
128-
129-
# Rule 5: candidates not empty
130-
if not packet.candidates:
131-
violations.append("candidates must not be empty (minItems: 1)")
132-
133-
# Rule 6: dry_lab_only_attestation must be True
134-
if not packet.dry_lab_only_attestation:
135-
violations.append(
136-
"dry_lab_only_attestation must be True — "
137-
"this packet contains dry-lab computational outputs only"
138-
)
139-
140-
# Rule 7: packet-level proof_ladder_level cap
141-
if packet.proof_ladder_level > DRY_LAB_MAX_PROOF_LADDER_LEVEL:
142-
violations.append(
143-
f"proof_ladder_level ({packet.proof_ladder_level}) exceeds dry-lab cap "
144-
f"of {DRY_LAB_MAX_PROOF_LADDER_LEVEL}. "
145-
"Dry-lab outputs cannot claim beyond 'computationally nominated' (Level 2)."
146-
)
147-
148-
# Rule 8: proof_ladder_level must be >= 1
149-
if packet.proof_ladder_level < 1:
150-
violations.append(
151-
f"proof_ladder_level must be >= 1, got {packet.proof_ladder_level}"
152-
)
153-
154-
# Rule 9: safety_attestations
155-
if packet.safety_attestations is None:
156-
violations.append("safety_attestations is required")
157-
else:
158-
if not packet.safety_attestations.no_known_toxicity_claim:
159-
violations.append(
160-
"safety_attestations.no_known_toxicity_claim must be True — "
161-
"dry-lab scores do not prove biological safety"
162-
)
163-
164-
# Rule 10: limitations non-empty
165-
if not packet.limitations:
166-
violations.append(
167-
"limitations must contain at least one entry. "
168-
"Minimum: 'Computational outputs are hypotheses and review aids. "
169-
"They are not biological proof.'"
170-
)
171-
172-
# Rule 11: benchmark_summary auroc range
173-
if packet.benchmark_summary is not None:
174-
if not (0.0 <= packet.benchmark_summary.auroc <= 1.0):
175-
violations.append(
176-
f"benchmark_summary.auroc must be in [0, 1], "
177-
f"got {packet.benchmark_summary.auroc}"
178-
)
179-
180-
# Rule 12: calibration_assessment vocabulary
181-
if packet.calibration_summary is not None:
182-
if (
183-
packet.calibration_summary.calibration_assessment
184-
not in VALID_CALIBRATION_ASSESSMENTS
185-
):
186-
violations.append(
187-
f"calibration_summary.calibration_assessment must be one of "
188-
f"{sorted(VALID_CALIBRATION_ASSESSMENTS)}, "
189-
f"got '{packet.calibration_summary.calibration_assessment}'"
190-
)
191-
192-
# Rule 13+14: per-candidate validation
193-
for i, cand in enumerate(packet.candidates):
194-
if not (0.0 <= cand.ensemble_score <= 1.0):
195-
violations.append(
196-
f"candidates[{i}].ensemble_score must be in [0, 1], "
197-
f"got {cand.ensemble_score}"
198-
)
199-
if cand.proof_ladder_level > DRY_LAB_MAX_PROOF_LADDER_LEVEL:
200-
violations.append(
201-
f"candidates[{i}].proof_ladder_level ({cand.proof_ladder_level}) "
202-
f"exceeds dry-lab cap of {DRY_LAB_MAX_PROOF_LADDER_LEVEL}"
203-
)
204-
205-
is_valid = len(violations) == 0
206-
207-
if is_valid:
208-
summary = (
209-
f"ERP- packet '{packet.packet_id}' is valid: "
210-
f"{len(packet.candidates)} candidate(s), "
211-
f"proof_ladder_level={packet.proof_ladder_level}, "
212-
"dry_lab_only=True."
213-
)
214-
else:
215-
summary = (
216-
f"ERP- packet '{packet.packet_id}' has {len(violations)} violation(s). "
217-
"Fix all violations before sending to external reviewers."
77+
n_present = sum(1 for c in erp.components if c.present)
78+
if erp.n_components_present != n_present:
79+
raise ValueError("n_components_present mismatch")
80+
expected_missing = sorted(
81+
c.component_type for c in erp.components if not c.present
82+
)
83+
if erp.missing_component_types != expected_missing:
84+
raise ValueError("missing_component_types mismatch")
85+
86+
87+
def build_external_review_packet(
88+
*,
89+
erp_id: str,
90+
batch_id: str,
91+
pipeline_version: str,
92+
brc_artifact_id: str = "",
93+
eci_artifact_id: str = "",
94+
fet_artifact_id: str = "",
95+
ptr_artifact_id: str = "",
96+
srs_artifact_id: str = "",
97+
limitations: list[str],
98+
created_at: str,
99+
) -> ExternalReviewPacket:
100+
artifact_map = {
101+
"BRC": brc_artifact_id,
102+
"ECI": eci_artifact_id,
103+
"FET": fet_artifact_id,
104+
"PTR": ptr_artifact_id,
105+
"SRS": srs_artifact_id,
106+
}
107+
components = [
108+
PacketComponent(
109+
component_type=ct,
110+
artifact_id=artifact_map[ct],
111+
present=bool(artifact_map[ct]),
218112
)
219-
220-
return PacketValidationResult(
221-
is_valid=is_valid,
222-
violations=violations,
223-
warnings=warnings,
224-
packet_id=packet.packet_id,
225-
validation_summary=summary,
113+
for ct in REQUIRED_PACKET_COMPONENTS
114+
]
115+
n_req = len(REQUIRED_PACKET_COMPONENTS)
116+
n_present = sum(1 for c in components if c.present)
117+
missing = sorted(c.component_type for c in components if not c.present)
118+
status = _compute_status(n_present, n_req)
119+
120+
erp = ExternalReviewPacket(
121+
erp_id=erp_id,
122+
batch_id=batch_id,
123+
pipeline_version=pipeline_version,
124+
components=components,
125+
n_components_required=n_req,
126+
n_components_present=n_present,
127+
missing_component_types=missing,
128+
packet_status=status,
129+
dry_lab_only=True,
130+
limitations=limitations,
131+
created_at=created_at,
226132
)
133+
validate_external_review_packet(erp)
134+
return erp
227135

228136

229-
def format_external_review_packet(packet: ExternalReviewPacket) -> str:
230-
"""Format an ExternalReviewPacket as a human-readable string."""
231-
result = validate_external_review_packet(packet)
137+
def format_external_review_packet(erp: ExternalReviewPacket) -> str:
232138
lines = [
233-
"=== EXTERNAL REVIEW PACKET ===",
234-
f"Packet ID: {packet.packet_id}",
235-
f"Version: {packet.version}",
236-
f"Pipeline: {packet.pipeline_version} ({packet.git_sha})",
237-
f"Generated: {packet.generated_at}",
238-
f"Candidates: {packet.candidate_count}",
239-
f"Proof ladder level: {packet.proof_ladder_level}",
240-
f"Dry-lab only: {'YES' if packet.dry_lab_only_attestation else 'NO'}",
241-
"",
242-
"-- SAFETY ATTESTATIONS --",
139+
f"External Review Packet — {erp.erp_id}",
140+
f"Batch: {erp.batch_id} | Pipeline: {erp.pipeline_version}",
141+
f"Status: {erp.packet_status} | Components: {erp.n_components_present}/{erp.n_components_required}",
243142
]
244-
if packet.safety_attestations:
245-
lines.extend([
246-
f" Reviewed by human: {packet.safety_attestations.reviewed_by_human}",
247-
f" Safety gate passed: {packet.safety_attestations.safety_gate_passed}",
248-
f" No toxicity claim: {packet.safety_attestations.no_known_toxicity_claim}",
249-
f" Hemolysis assay mandatory: {packet.safety_attestations.hemolysis_assay_mandatory}",
250-
])
251-
lines.extend(["", "-- LIMITATIONS --"])
252-
for lim in packet.limitations:
253-
lines.append(f" • {lim}")
254-
lines.extend(["", "-- VALIDATION --"])
255-
if result.is_valid:
256-
lines.append(" STATUS: VALID")
257-
else:
258-
lines.append(f" STATUS: INVALID ({len(result.violations)} violation(s))")
259-
for v in result.violations:
260-
lines.append(f" ✗ {v}")
261-
lines.extend(["", result.validation_summary])
262-
if not result.is_valid:
263-
lines.extend([
264-
"",
265-
"NOTICE: Do not send this packet to external reviewers until all violations are resolved.",
266-
])
143+
if erp.missing_component_types:
144+
lines.append(f"Missing: {', '.join(erp.missing_component_types)}")
145+
lines.append("Components:")
146+
for comp in erp.components:
147+
status_label = "PRESENT" if comp.present else "MISSING"
148+
artifact_str = comp.artifact_id if comp.artifact_id else "(none)"
149+
lines.append(f" [{status_label}] {comp.component_type}: {artifact_str}")
150+
lines.append(f"Created: {erp.created_at}")
151+
lines.append(f"Limitations: {'; '.join(erp.limitations)}")
152+
lines.append(f"dry_lab_only: {erp.dry_lab_only}")
267153
return "\n".join(lines)

0 commit comments

Comments
 (0)