diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index f89af834..3f0efb0a 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -95,7 +95,7 @@ Make qualified external review easier and safer. | E1 | Add external review packet schema (complete). — evidence/external_review_packet.py: ERP- schema with 14 fields, validation rules, dry_lab_only enforcement; tests/evidence/test_external_review_packet.py + test_external_review_packet_schema.py. | Packets become machine-checkable. | C/D | | E2 | Add example external review packet using toy data (complete). — evidence/example_external_review_packet.py: toy-data example ERP with MOCK-/TOY- prefix enforcement and is_example_data=True; tests/evidence/test_example_external_review_packet.py. | Partners know what to expect. | C/D | | E3 | Add reviewer questionnaire schema (complete). | Makes external review feedback machine-readable: Likert clarity ratings for activity/safety/novelty claims, synthesis recommendation, structured comments. | B/C | -| E4 | Add safety-release decision schema. | Release review becomes auditable. | D | +| E4 | Add safety-release decision schema. | Release review becomes auditable. | D | DONE | | E5 | Add non-protocol pilot pre-registration schema (complete). — evidence/pilot_preregistration.py: non-protocol pilot pre-registration schema (PPR-) freezing selection logic before batch release; tests/evidence/test_pilot_preregistration.py + test_pilot_preregistration_schema.py. | Freezes selection logic. | C/D | | E6 | Add packet generator CLI (complete). — scripts/generate_review_packet.py: generates skeleton external review packet JSON; make generate-review-packet target; validates against schemas/external_review_packet.schema.json; dry_lab_only_attestation=True enforced. | Reduces manual packaging errors. | C/D | | E7 | Add packet validator CLI (complete). — src/openamp_foundry/cli/commands/validate_packet.py: load_packet_from_json() reads ERP- JSON from disk; validate_packet_file() returns {valid, violations, packet_id, error}; _run_validate_packet() prints PASS/FAIL with violations; 45 tests in tests/cli/test_validate_packet.py. | Review readiness becomes testable. | C/D | diff --git a/src/openamp_foundry/evidence/safety_release_decision.py b/src/openamp_foundry/evidence/safety_release_decision.py new file mode 100644 index 00000000..89ed7af9 --- /dev/null +++ b/src/openamp_foundry/evidence/safety_release_decision.py @@ -0,0 +1,165 @@ +"""SRD- safety-release decision schema. + +Machine-checkable record of whether a candidate package was authorized for +external release, which safety checks were passed, what restrictions apply, +and why it was rejected if not authorized. + +Makes release review auditable: every external release of pipeline outputs +must reference an SRD- record. No release is implicit or unrecorded. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +VALID_SRD_DECISIONS: frozenset[str] = frozenset({ + "authorized", + "rejected", + "pending_review", +}) + +VALID_RELEASE_SCOPES: frozenset[str] = frozenset({ + "academic_collaboration", + "public_preprint", + "internal_only", + "restricted_partner", +}) + +VALID_SAFETY_CHECK_IDS: frozenset[str] = frozenset({ + "dual_use_screened", + "toxicity_flags_reviewed", + "hemolysis_flags_reviewed", + "novelty_claims_bounded", + "dry_lab_only_label_present", + "no_sequence_privacy_violation", + "candidate_count_within_limit", +}) + +REQUIRED_SAFETY_CHECKS: tuple[str, ...] = ( + "dual_use_screened", + "dry_lab_only_label_present", + "novelty_claims_bounded", +) + + +@dataclass +class SafetyReleaseDecision: + srd_id: str + pipeline_version: str + erp_id: str + release_decision: str + release_scope: str + safety_checks_passed: list[str] + restrictions: list[str] + rejection_reason: str + all_required_checks_passed: bool + dry_lab_only: bool + limitations: list[str] + created_at: str + + +def validate_safety_release_decision(srd: SafetyReleaseDecision) -> None: + if not srd.srd_id.startswith("SRD-"): + raise ValueError(f"srd_id must start with 'SRD-': {srd.srd_id!r}") + if not srd.pipeline_version: + raise ValueError("pipeline_version must be non-empty") + if not srd.erp_id.startswith("ERP-"): + raise ValueError(f"erp_id must start with 'ERP-': {srd.erp_id!r}") + if srd.release_decision not in VALID_SRD_DECISIONS: + raise ValueError( + f"release_decision {srd.release_decision!r} not in VALID_SRD_DECISIONS" + ) + if srd.release_scope not in VALID_RELEASE_SCOPES: + raise ValueError( + f"release_scope {srd.release_scope!r} not in VALID_RELEASE_SCOPES" + ) + for check_id in srd.safety_checks_passed: + if check_id not in VALID_SAFETY_CHECK_IDS: + raise ValueError( + f"safety check {check_id!r} not in VALID_SAFETY_CHECK_IDS" + ) + required_passed = all( + req in srd.safety_checks_passed for req in REQUIRED_SAFETY_CHECKS + ) + if srd.all_required_checks_passed != required_passed: + raise ValueError( + "all_required_checks_passed mismatch with safety_checks_passed" + ) + if srd.release_decision == "authorized" and not srd.all_required_checks_passed: + raise ValueError( + "release_decision='authorized' requires all_required_checks_passed=True" + ) + if srd.release_decision == "rejected" and not srd.rejection_reason: + raise ValueError( + "rejection_reason must be non-empty when release_decision='rejected'" + ) + if not srd.dry_lab_only: + raise ValueError("dry_lab_only must be True") + if not srd.limitations: + raise ValueError("limitations must be non-empty") + if not srd.created_at: + raise ValueError("created_at must be non-empty") + + +def build_safety_release_decision( + *, + srd_id: str, + pipeline_version: str, + erp_id: str, + release_decision: str, + release_scope: str, + safety_checks_passed: list[str], + restrictions: list[str], + rejection_reason: str = "", + limitations: list[str], + created_at: str, +) -> SafetyReleaseDecision: + """Build a SafetyReleaseDecision. + + safety_checks_passed: list of VALID_SAFETY_CHECK_IDS that were verified. + all_required_checks_passed is auto-computed from REQUIRED_SAFETY_CHECKS. + """ + required_passed = all( + req in safety_checks_passed for req in REQUIRED_SAFETY_CHECKS + ) + srd = SafetyReleaseDecision( + srd_id=srd_id, + pipeline_version=pipeline_version, + erp_id=erp_id, + release_decision=release_decision, + release_scope=release_scope, + safety_checks_passed=list(safety_checks_passed), + restrictions=list(restrictions), + rejection_reason=rejection_reason, + all_required_checks_passed=required_passed, + dry_lab_only=True, + limitations=limitations, + created_at=created_at, + ) + validate_safety_release_decision(srd) + return srd + + +def format_safety_release_decision(srd: SafetyReleaseDecision) -> str: + lines = [ + f"Safety-Release Decision — {srd.srd_id}", + f"Pipeline: {srd.pipeline_version} | ERP: {srd.erp_id}", + f"Decision: {srd.release_decision} | Scope: {srd.release_scope}", + f"All required checks passed: {srd.all_required_checks_passed}", + ] + lines.append(f"Safety checks passed ({len(srd.safety_checks_passed)}):") + for check in srd.safety_checks_passed: + lines.append(f" + {check}") + missing = [r for r in REQUIRED_SAFETY_CHECKS if r not in srd.safety_checks_passed] + if missing: + lines.append("Required checks NOT passed:") + for m in missing: + lines.append(f" ! {m}") + if srd.restrictions: + lines.append(f"Restrictions: {'; '.join(srd.restrictions)}") + if srd.rejection_reason: + lines.append(f"Rejection reason: {srd.rejection_reason}") + lines.append(f"Created: {srd.created_at}") + lines.append(f"Limitations: {'; '.join(srd.limitations)}") + lines.append(f"dry_lab_only: {srd.dry_lab_only}") + return "\n".join(lines) diff --git a/tests/evidence/test_safety_release_decision.py b/tests/evidence/test_safety_release_decision.py new file mode 100644 index 00000000..5c5bba99 --- /dev/null +++ b/tests/evidence/test_safety_release_decision.py @@ -0,0 +1,336 @@ +"""Tests for SRD- safety-release decision schema.""" + +import pytest +from openamp_foundry.evidence.safety_release_decision import ( + SafetyReleaseDecision, + VALID_SRD_DECISIONS, + VALID_RELEASE_SCOPES, + VALID_SAFETY_CHECK_IDS, + REQUIRED_SAFETY_CHECKS, + build_safety_release_decision, + format_safety_release_decision, + validate_safety_release_decision, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_ALL_REQUIRED = list(REQUIRED_SAFETY_CHECKS) +_ALL_CHECKS = list(VALID_SAFETY_CHECK_IDS) + + +def _build(**kwargs): + defaults = dict( + srd_id="SRD-001", + pipeline_version="v1.0", + erp_id="ERP-001", + release_decision="authorized", + release_scope="academic_collaboration", + safety_checks_passed=_ALL_REQUIRED, + restrictions=["do not share sequence data publicly"], + limitations=["dry-lab only"], + created_at="2026-07-10", + ) + defaults.update(kwargs) + return build_safety_release_decision(**defaults) + + +# --------------------------------------------------------------------------- +# 1. Constants +# --------------------------------------------------------------------------- + + +def test_valid_srd_decisions_is_frozenset(): + assert isinstance(VALID_SRD_DECISIONS, frozenset) + + +def test_valid_srd_decisions_contains_authorized(): + assert "authorized" in VALID_SRD_DECISIONS + + +def test_valid_srd_decisions_contains_rejected(): + assert "rejected" in VALID_SRD_DECISIONS + + +def test_valid_srd_decisions_contains_pending_review(): + assert "pending_review" in VALID_SRD_DECISIONS + + +def test_valid_release_scopes_is_frozenset(): + assert isinstance(VALID_RELEASE_SCOPES, frozenset) + + +def test_valid_release_scopes_contains_academic_collaboration(): + assert "academic_collaboration" in VALID_RELEASE_SCOPES + + +def test_valid_release_scopes_contains_public_preprint(): + assert "public_preprint" in VALID_RELEASE_SCOPES + + +def test_valid_release_scopes_contains_internal_only(): + assert "internal_only" in VALID_RELEASE_SCOPES + + +def test_valid_release_scopes_contains_restricted_partner(): + assert "restricted_partner" in VALID_RELEASE_SCOPES + + +def test_valid_safety_check_ids_is_frozenset(): + assert isinstance(VALID_SAFETY_CHECK_IDS, frozenset) + + +def test_valid_safety_check_ids_contains_dual_use(): + assert "dual_use_screened" in VALID_SAFETY_CHECK_IDS + + +def test_valid_safety_check_ids_contains_dry_lab_label(): + assert "dry_lab_only_label_present" in VALID_SAFETY_CHECK_IDS + + +def test_valid_safety_check_ids_contains_novelty_bounded(): + assert "novelty_claims_bounded" in VALID_SAFETY_CHECK_IDS + + +def test_required_safety_checks_is_tuple(): + assert isinstance(REQUIRED_SAFETY_CHECKS, tuple) + + +def test_required_safety_checks_contains_dual_use(): + assert "dual_use_screened" in REQUIRED_SAFETY_CHECKS + + +def test_required_safety_checks_contains_dry_lab_label(): + assert "dry_lab_only_label_present" in REQUIRED_SAFETY_CHECKS + + +def test_required_safety_checks_contains_novelty_bounded(): + assert "novelty_claims_bounded" in REQUIRED_SAFETY_CHECKS + + +# --------------------------------------------------------------------------- +# 2. build – happy paths +# --------------------------------------------------------------------------- + + +def test_build_returns_safety_release_decision(): + assert isinstance(_build(), SafetyReleaseDecision) + + +def test_build_srd_id_stored(): + assert _build().srd_id == "SRD-001" + + +def test_build_pipeline_version_stored(): + assert _build().pipeline_version == "v1.0" + + +def test_build_erp_id_stored(): + assert _build().erp_id == "ERP-001" + + +def test_build_dry_lab_only_true(): + assert _build().dry_lab_only is True + + +def test_build_authorized_with_all_required(): + r = _build(release_decision="authorized", safety_checks_passed=_ALL_REQUIRED) + assert r.release_decision == "authorized" + + +def test_build_all_required_checks_passed_true(): + r = _build(safety_checks_passed=_ALL_REQUIRED) + assert r.all_required_checks_passed is True + + +def test_build_all_required_checks_passed_false_when_missing(): + r = _build( + release_decision="pending_review", + safety_checks_passed=["dual_use_screened"], + ) + assert r.all_required_checks_passed is False + + +def test_build_rejected_with_reason(): + r = _build( + release_decision="rejected", + safety_checks_passed=["dual_use_screened"], + rejection_reason="toxicity flags not reviewed", + ) + assert r.release_decision == "rejected" + assert r.rejection_reason == "toxicity flags not reviewed" + + +def test_build_pending_review(): + r = _build( + release_decision="pending_review", + safety_checks_passed=_ALL_REQUIRED, + ) + assert r.release_decision == "pending_review" + + +def test_build_release_scope_stored(): + assert _build().release_scope == "academic_collaboration" + + +def test_build_public_preprint_scope(): + r = _build(release_scope="public_preprint") + assert r.release_scope == "public_preprint" + + +def test_build_restrictions_stored(): + assert _build().restrictions == ["do not share sequence data publicly"] + + +def test_build_rejection_reason_default_empty(): + assert _build().rejection_reason == "" + + +def test_build_safety_checks_stored(): + r = _build(safety_checks_passed=_ALL_REQUIRED) + for check in _ALL_REQUIRED: + assert check in r.safety_checks_passed + + +def test_build_all_checks_accepted(): + r = _build(safety_checks_passed=_ALL_CHECKS) + assert len(r.safety_checks_passed) == len(_ALL_CHECKS) + + +def test_build_limitations_stored(): + assert _build().limitations == ["dry-lab only"] + + +def test_build_created_at_stored(): + assert _build().created_at == "2026-07-10" + + +def test_build_internal_only_scope(): + r = _build(release_scope="internal_only") + assert r.release_scope == "internal_only" + + +# --------------------------------------------------------------------------- +# 3. validate – rejection cases +# --------------------------------------------------------------------------- + + +def test_validate_rejects_bad_srd_id_prefix(): + with pytest.raises(ValueError, match="SRD-"): + _build(srd_id="BAD-001") + + +def test_validate_rejects_empty_pipeline_version(): + with pytest.raises(ValueError): + _build(pipeline_version="") + + +def test_validate_rejects_bad_erp_id_prefix(): + with pytest.raises(ValueError, match="ERP-"): + _build(erp_id="BAD-001") + + +def test_validate_rejects_invalid_release_decision(): + srd = _build() + srd.release_decision = "UNKNOWN" + with pytest.raises(ValueError, match="release_decision"): + validate_safety_release_decision(srd) + + +def test_validate_rejects_invalid_release_scope(): + with pytest.raises(ValueError, match="release_scope"): + _build(release_scope="UNKNOWN") + + +def test_validate_rejects_invalid_safety_check_id(): + with pytest.raises(ValueError, match="safety check"): + _build(safety_checks_passed=["UNKNOWN_CHECK"]) + + +def test_validate_rejects_authorized_without_required_checks(): + with pytest.raises(ValueError, match="authorized"): + _build( + release_decision="authorized", + safety_checks_passed=["dual_use_screened"], + ) + + +def test_validate_rejects_rejected_without_reason(): + with pytest.raises(ValueError, match="rejection_reason"): + _build( + release_decision="rejected", + safety_checks_passed=["dual_use_screened"], + rejection_reason="", + ) + + +def test_validate_rejects_all_required_checks_mismatch(): + srd = _build() + srd.all_required_checks_passed = False + with pytest.raises(ValueError, match="all_required_checks_passed"): + validate_safety_release_decision(srd) + + +def test_validate_rejects_dry_lab_only_false(): + srd = _build() + srd.dry_lab_only = False + with pytest.raises(ValueError, match="dry_lab_only"): + validate_safety_release_decision(srd) + + +def test_validate_rejects_empty_limitations(): + with pytest.raises(ValueError, match="limitations"): + _build(limitations=[]) + + +def test_validate_rejects_empty_created_at(): + with pytest.raises(ValueError): + _build(created_at="") + + +# --------------------------------------------------------------------------- +# 4. format +# --------------------------------------------------------------------------- + + +def test_format_contains_srd_id(): + assert "SRD-001" in format_safety_release_decision(_build()) + + +def test_format_contains_pipeline_version(): + assert "v1.0" in format_safety_release_decision(_build()) + + +def test_format_contains_erp_id(): + assert "ERP-001" in format_safety_release_decision(_build()) + + +def test_format_contains_decision(): + assert "authorized" in format_safety_release_decision(_build()) + + +def test_format_contains_scope(): + assert "academic_collaboration" in format_safety_release_decision(_build()) + + +def test_format_contains_safety_check(): + assert "dual_use_screened" in format_safety_release_decision(_build()) + + +def test_format_contains_restrictions(): + assert "do not share sequence data publicly" in format_safety_release_decision( + _build() + ) + + +def test_format_contains_limitations(): + assert "dry-lab only" in format_safety_release_decision(_build()) + + +def test_format_contains_dry_lab_only(): + assert "dry_lab_only: True" in format_safety_release_decision(_build()) + + +def test_format_is_string(): + assert isinstance(format_safety_release_decision(_build()), str) diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index b9afd63c..7161c799 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -4,7 +4,7 @@ import sys import math -BASELINE = 10764 +BASELINE = 10822 def test_test_count_regression():