diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index 3f0efb0a..0fdcba97 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -99,7 +99,7 @@ Make qualified external review easier and safer. | E5 | Add non-protocol pilot pre-registration schema (complete). — evidence/pilot_preregistration.py: non-protocol pilot pre-registration schema (PPR-) freezing selection logic before batch release; tests/evidence/test_pilot_preregistration.py + test_pilot_preregistration_schema.py. | Freezes selection logic. | C/D | | E6 | Add packet generator CLI (complete). — scripts/generate_review_packet.py: generates skeleton external review packet JSON; make generate-review-packet target; validates against schemas/external_review_packet.schema.json; dry_lab_only_attestation=True enforced. | Reduces manual packaging errors. | C/D | | E7 | Add packet validator CLI (complete). — src/openamp_foundry/cli/commands/validate_packet.py: load_packet_from_json() reads ERP- JSON from disk; validate_packet_file() returns {valid, violations, packet_id, error}; _run_validate_packet() prints PASS/FAIL with violations; 45 tests in tests/cli/test_validate_packet.py. | Review readiness becomes testable. | C/D | -| E8 | Add release-summary generator that strips restricted fields. | Safer public summaries. | D | +| E8 | Add release-summary generator that strips restricted fields. | Safer public summaries. | D | DONE | | E9 | Add domain review outcome schema (complete). | Structured expert verdict on a PEP with controlled taxonomy of domains and outcomes; closes ESC→RVQ→DRO review chain. | B/C | | E10 | Add expert-review example with mock/toy candidates only (complete). | ERP- schema: 14 fields, 16 validation rules, mock candidate ID prefix enforcement (MOCK-/TOY-/EXAMPLE-/DEMO-/TEST-), is_example_data=True and dry_lab_only=True enforced; CI-checkable template cannot accidentally leak real candidates. | B/C | diff --git a/src/openamp_foundry/evidence/release_summary.py b/src/openamp_foundry/evidence/release_summary.py new file mode 100644 index 00000000..fbdd88cd --- /dev/null +++ b/src/openamp_foundry/evidence/release_summary.py @@ -0,0 +1,181 @@ +"""RSM- release-summary schema and generator. + +Produces a public-safe summary of what was released by stripping restricted +fields before any external distribution. Every external share of pipeline +outputs must go through this generator — no ad-hoc field omission. + +Restricted fields list what must NOT appear in public summaries: +individual candidate IDs, internal batch references, rejection reasons, +raw sequence data, reviewer identities, and internal restriction text. + +Makes the stripping step machine-checkable rather than relying on human +memory of what is sensitive. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +RESTRICTED_FIELDS: tuple[str, ...] = ( + "erp_id", + "rejection_reason", + "restrictions", + "candidate_ids", + "reviewer_ids", + "raw_sequences", + "batch_ids", + "internal_notes", + "collaborator_names", +) + +VALID_RSM_RELEASE_SCOPES: frozenset[str] = frozenset({ + "academic_collaboration", + "public_preprint", + "internal_only", + "restricted_partner", +}) + +VALID_RSM_DECISIONS: frozenset[str] = frozenset({ + "authorized", + "rejected", + "pending_review", +}) + + +@dataclass +class ReleaseSummary: + rsm_id: str + srd_id: str + pipeline_version: str + release_scope: str + release_decision: str + candidate_count: int + safety_checks_summary: list[str] + public_notes: str + limitations_summary: str + restricted_fields_stripped: bool + dry_lab_only: bool + created_at: str + + +def validate_release_summary(rsm: ReleaseSummary) -> None: + if not rsm.rsm_id.startswith("RSM-"): + raise ValueError(f"rsm_id must start with 'RSM-': {rsm.rsm_id!r}") + if not rsm.srd_id.startswith("SRD-"): + raise ValueError(f"srd_id must start with 'SRD-': {rsm.srd_id!r}") + if not rsm.pipeline_version: + raise ValueError("pipeline_version must be non-empty") + if rsm.release_scope not in VALID_RSM_RELEASE_SCOPES: + raise ValueError( + f"release_scope {rsm.release_scope!r} not in VALID_RSM_RELEASE_SCOPES" + ) + if rsm.release_decision not in VALID_RSM_DECISIONS: + raise ValueError( + f"release_decision {rsm.release_decision!r} not in VALID_RSM_DECISIONS" + ) + if rsm.candidate_count < 0: + raise ValueError("candidate_count must be non-negative") + if not rsm.restricted_fields_stripped: + raise ValueError("restricted_fields_stripped must be True") + if not rsm.dry_lab_only: + raise ValueError("dry_lab_only must be True") + if not rsm.limitations_summary: + raise ValueError("limitations_summary must be non-empty") + if not rsm.created_at: + raise ValueError("created_at must be non-empty") + + +def build_release_summary( + *, + rsm_id: str, + srd_id: str, + pipeline_version: str, + release_scope: str, + release_decision: str, + candidate_count: int, + safety_checks_summary: list[str], + public_notes: str = "", + limitations_summary: str, + created_at: str, +) -> ReleaseSummary: + """Build a ReleaseSummary with restricted_fields_stripped and dry_lab_only always True.""" + rsm = ReleaseSummary( + rsm_id=rsm_id, + srd_id=srd_id, + pipeline_version=pipeline_version, + release_scope=release_scope, + release_decision=release_decision, + candidate_count=candidate_count, + safety_checks_summary=list(safety_checks_summary), + public_notes=public_notes, + limitations_summary=limitations_summary, + restricted_fields_stripped=True, + dry_lab_only=True, + created_at=created_at, + ) + validate_release_summary(rsm) + return rsm + + +def strip_restricted_fields(source: dict) -> dict: + """Return a copy of *source* with all RESTRICTED_FIELDS removed.""" + return {k: v for k, v in source.items() if k not in RESTRICTED_FIELDS} + + +def generate_release_summary( + *, + rsm_id: str, + srd_id: str, + pipeline_version: str, + release_scope: str, + release_decision: str, + candidate_count: int, + safety_checks_summary: list[str], + limitations_summary: str, + public_notes: str = "", + created_at: str, + source_fields: dict | None = None, +) -> ReleaseSummary: + """Generate a public-safe ReleaseSummary, stripping any restricted fields + found in *source_fields* before building. + + *source_fields* is optional raw data that will be validated for absence of + restricted fields — raising ValueError if any restricted key is present. + This makes accidental leakage detectable at generation time. + """ + if source_fields is not None: + leaked = [k for k in RESTRICTED_FIELDS if k in source_fields] + if leaked: + raise ValueError( + f"Restricted fields present in source_fields, must be stripped first: {leaked}" + ) + return build_release_summary( + rsm_id=rsm_id, + srd_id=srd_id, + pipeline_version=pipeline_version, + release_scope=release_scope, + release_decision=release_decision, + candidate_count=candidate_count, + safety_checks_summary=safety_checks_summary, + public_notes=public_notes, + limitations_summary=limitations_summary, + created_at=created_at, + ) + + +def format_release_summary(rsm: ReleaseSummary) -> str: + lines = [ + f"Release Summary — {rsm.rsm_id}", + f"Authorization: {rsm.srd_id} | Pipeline: {rsm.pipeline_version}", + f"Decision: {rsm.release_decision} | Scope: {rsm.release_scope}", + f"Candidates in summary: {rsm.candidate_count}", + f"Restricted fields stripped: {rsm.restricted_fields_stripped}", + ] + if rsm.safety_checks_summary: + lines.append(f"Safety checks: {'; '.join(rsm.safety_checks_summary)}") + if rsm.public_notes: + lines.append(f"Notes: {rsm.public_notes}") + lines.append(f"Limitations: {rsm.limitations_summary}") + lines.append(f"Created: {rsm.created_at}") + lines.append(f"dry_lab_only: {rsm.dry_lab_only}") + return "\n".join(lines) diff --git a/tests/evidence/test_release_summary.py b/tests/evidence/test_release_summary.py new file mode 100644 index 00000000..0d9a6fa8 --- /dev/null +++ b/tests/evidence/test_release_summary.py @@ -0,0 +1,419 @@ +"""Tests for RSM- release-summary schema and generator.""" + +import pytest +from openamp_foundry.evidence.release_summary import ( + ReleaseSummary, + RESTRICTED_FIELDS, + VALID_RSM_RELEASE_SCOPES, + VALID_RSM_DECISIONS, + build_release_summary, + generate_release_summary, + strip_restricted_fields, + validate_release_summary, + format_release_summary, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _build(**kwargs): + defaults = dict( + rsm_id="RSM-001", + srd_id="SRD-001", + pipeline_version="v1.0", + release_scope="academic_collaboration", + release_decision="authorized", + candidate_count=5, + safety_checks_summary=["dual_use_screened", "novelty_claims_bounded"], + public_notes="dry-lab candidates only", + limitations_summary="computational predictions, not validated in wet lab", + created_at="2026-07-10", + ) + defaults.update(kwargs) + return build_release_summary(**defaults) + + +# --------------------------------------------------------------------------- +# 1. Constants +# --------------------------------------------------------------------------- + + +def test_restricted_fields_is_tuple(): + assert isinstance(RESTRICTED_FIELDS, tuple) + + +def test_restricted_fields_contains_erp_id(): + assert "erp_id" in RESTRICTED_FIELDS + + +def test_restricted_fields_contains_rejection_reason(): + assert "rejection_reason" in RESTRICTED_FIELDS + + +def test_restricted_fields_contains_restrictions(): + assert "restrictions" in RESTRICTED_FIELDS + + +def test_restricted_fields_contains_candidate_ids(): + assert "candidate_ids" in RESTRICTED_FIELDS + + +def test_restricted_fields_contains_reviewer_ids(): + assert "reviewer_ids" in RESTRICTED_FIELDS + + +def test_restricted_fields_contains_raw_sequences(): + assert "raw_sequences" in RESTRICTED_FIELDS + + +def test_restricted_fields_contains_batch_ids(): + assert "batch_ids" in RESTRICTED_FIELDS + + +def test_restricted_fields_contains_internal_notes(): + assert "internal_notes" in RESTRICTED_FIELDS + + +def test_valid_rsm_release_scopes_is_frozenset(): + assert isinstance(VALID_RSM_RELEASE_SCOPES, frozenset) + + +def test_valid_rsm_release_scopes_contains_academic_collaboration(): + assert "academic_collaboration" in VALID_RSM_RELEASE_SCOPES + + +def test_valid_rsm_release_scopes_contains_public_preprint(): + assert "public_preprint" in VALID_RSM_RELEASE_SCOPES + + +def test_valid_rsm_release_scopes_contains_internal_only(): + assert "internal_only" in VALID_RSM_RELEASE_SCOPES + + +def test_valid_rsm_release_scopes_contains_restricted_partner(): + assert "restricted_partner" in VALID_RSM_RELEASE_SCOPES + + +def test_valid_rsm_decisions_is_frozenset(): + assert isinstance(VALID_RSM_DECISIONS, frozenset) + + +def test_valid_rsm_decisions_contains_authorized(): + assert "authorized" in VALID_RSM_DECISIONS + + +def test_valid_rsm_decisions_contains_rejected(): + assert "rejected" in VALID_RSM_DECISIONS + + +def test_valid_rsm_decisions_contains_pending_review(): + assert "pending_review" in VALID_RSM_DECISIONS + + +# --------------------------------------------------------------------------- +# 2. build – happy paths +# --------------------------------------------------------------------------- + + +def test_build_returns_release_summary(): + assert isinstance(_build(), ReleaseSummary) + + +def test_build_rsm_id_stored(): + assert _build().rsm_id == "RSM-001" + + +def test_build_srd_id_stored(): + assert _build().srd_id == "SRD-001" + + +def test_build_pipeline_version_stored(): + assert _build().pipeline_version == "v1.0" + + +def test_build_release_scope_stored(): + assert _build().release_scope == "academic_collaboration" + + +def test_build_release_decision_stored(): + assert _build().release_decision == "authorized" + + +def test_build_candidate_count_stored(): + assert _build().candidate_count == 5 + + +def test_build_safety_checks_summary_stored(): + rsm = _build() + assert "dual_use_screened" in rsm.safety_checks_summary + assert "novelty_claims_bounded" in rsm.safety_checks_summary + + +def test_build_public_notes_stored(): + assert _build().public_notes == "dry-lab candidates only" + + +def test_build_limitations_summary_stored(): + assert "computational" in _build().limitations_summary + + +def test_build_restricted_fields_stripped_true(): + assert _build().restricted_fields_stripped is True + + +def test_build_dry_lab_only_true(): + assert _build().dry_lab_only is True + + +def test_build_created_at_stored(): + assert _build().created_at == "2026-07-10" + + +def test_build_rejected_decision(): + r = _build(release_decision="rejected") + assert r.release_decision == "rejected" + + +def test_build_pending_review_decision(): + r = _build(release_decision="pending_review") + assert r.release_decision == "pending_review" + + +def test_build_public_preprint_scope(): + r = _build(release_scope="public_preprint") + assert r.release_scope == "public_preprint" + + +def test_build_zero_candidates(): + r = _build(candidate_count=0) + assert r.candidate_count == 0 + + +def test_build_empty_public_notes_allowed(): + r = _build(public_notes="") + assert r.public_notes == "" + + +def test_build_empty_safety_checks_summary_allowed(): + r = _build(safety_checks_summary=[]) + assert r.safety_checks_summary == [] + + +# --------------------------------------------------------------------------- +# 3. validate – rejection cases +# --------------------------------------------------------------------------- + + +def test_validate_rejects_bad_rsm_id_prefix(): + with pytest.raises(ValueError, match="RSM-"): + _build(rsm_id="BAD-001") + + +def test_validate_rejects_bad_srd_id_prefix(): + with pytest.raises(ValueError, match="SRD-"): + _build(srd_id="BAD-001") + + +def test_validate_rejects_empty_pipeline_version(): + with pytest.raises(ValueError): + _build(pipeline_version="") + + +def test_validate_rejects_invalid_release_scope(): + with pytest.raises(ValueError, match="release_scope"): + _build(release_scope="UNKNOWN") + + +def test_validate_rejects_invalid_release_decision(): + with pytest.raises(ValueError, match="release_decision"): + _build(release_decision="UNKNOWN") + + +def test_validate_rejects_negative_candidate_count(): + with pytest.raises(ValueError, match="candidate_count"): + _build(candidate_count=-1) + + +def test_validate_rejects_restricted_fields_stripped_false(): + rsm = _build() + rsm.restricted_fields_stripped = False + with pytest.raises(ValueError, match="restricted_fields_stripped"): + validate_release_summary(rsm) + + +def test_validate_rejects_dry_lab_only_false(): + rsm = _build() + rsm.dry_lab_only = False + with pytest.raises(ValueError, match="dry_lab_only"): + validate_release_summary(rsm) + + +def test_validate_rejects_empty_limitations_summary(): + with pytest.raises(ValueError, match="limitations_summary"): + _build(limitations_summary="") + + +def test_validate_rejects_empty_created_at(): + with pytest.raises(ValueError): + _build(created_at="") + + +# --------------------------------------------------------------------------- +# 4. strip_restricted_fields +# --------------------------------------------------------------------------- + + +def test_strip_removes_erp_id(): + source = {"erp_id": "ERP-001", "pipeline_version": "v1.0"} + result = strip_restricted_fields(source) + assert "erp_id" not in result + + +def test_strip_removes_rejection_reason(): + source = {"rejection_reason": "toxicity", "pipeline_version": "v1.0"} + result = strip_restricted_fields(source) + assert "rejection_reason" not in result + + +def test_strip_removes_candidate_ids(): + source = {"candidate_ids": ["C001", "C002"], "candidate_count": 2} + result = strip_restricted_fields(source) + assert "candidate_ids" not in result + + +def test_strip_keeps_non_restricted_fields(): + source = {"pipeline_version": "v1.0", "erp_id": "ERP-001"} + result = strip_restricted_fields(source) + assert result["pipeline_version"] == "v1.0" + + +def test_strip_all_restricted_fields_removed(): + source = {k: "value" for k in RESTRICTED_FIELDS} + source["pipeline_version"] = "v1.0" + result = strip_restricted_fields(source) + for k in RESTRICTED_FIELDS: + assert k not in result + + +def test_strip_returns_dict(): + assert isinstance(strip_restricted_fields({}), dict) + + +def test_strip_does_not_mutate_source(): + source = {"erp_id": "ERP-001", "pipeline_version": "v1.0"} + strip_restricted_fields(source) + assert "erp_id" in source + + +# --------------------------------------------------------------------------- +# 5. generate_release_summary +# --------------------------------------------------------------------------- + + +def _generate(**kwargs): + defaults = dict( + rsm_id="RSM-001", + srd_id="SRD-001", + pipeline_version="v1.0", + release_scope="academic_collaboration", + release_decision="authorized", + candidate_count=5, + safety_checks_summary=["dual_use_screened"], + limitations_summary="dry-lab only", + created_at="2026-07-10", + ) + defaults.update(kwargs) + return generate_release_summary(**defaults) + + +def test_generate_returns_release_summary(): + assert isinstance(_generate(), ReleaseSummary) + + +def test_generate_restricted_fields_stripped_true(): + assert _generate().restricted_fields_stripped is True + + +def test_generate_dry_lab_only_true(): + assert _generate().dry_lab_only is True + + +def test_generate_raises_if_source_fields_has_erp_id(): + with pytest.raises(ValueError, match="erp_id"): + _generate(source_fields={"erp_id": "ERP-001"}) + + +def test_generate_raises_if_source_fields_has_rejection_reason(): + with pytest.raises(ValueError, match="rejection_reason"): + _generate(source_fields={"rejection_reason": "toxicity"}) + + +def test_generate_raises_if_source_fields_has_candidate_ids(): + with pytest.raises(ValueError, match="candidate_ids"): + _generate(source_fields={"candidate_ids": ["C1"]}) + + +def test_generate_raises_if_source_fields_has_raw_sequences(): + with pytest.raises(ValueError, match="raw_sequences"): + _generate(source_fields={"raw_sequences": ["ACGT"]}) + + +def test_generate_accepts_clean_source_fields(): + rsm = _generate(source_fields={"pipeline_version": "v1.0", "candidate_count": 5}) + assert rsm.candidate_count == 5 + + +def test_generate_no_source_fields_ok(): + rsm = _generate(source_fields=None) + assert isinstance(rsm, ReleaseSummary) + + +# --------------------------------------------------------------------------- +# 6. format +# --------------------------------------------------------------------------- + + +def test_format_contains_rsm_id(): + assert "RSM-001" in format_release_summary(_build()) + + +def test_format_contains_srd_id(): + assert "SRD-001" in format_release_summary(_build()) + + +def test_format_contains_pipeline_version(): + assert "v1.0" in format_release_summary(_build()) + + +def test_format_contains_decision(): + assert "authorized" in format_release_summary(_build()) + + +def test_format_contains_scope(): + assert "academic_collaboration" in format_release_summary(_build()) + + +def test_format_contains_candidate_count(): + assert "5" in format_release_summary(_build()) + + +def test_format_contains_safety_check(): + assert "dual_use_screened" in format_release_summary(_build()) + + +def test_format_contains_limitations(): + assert "computational" in format_release_summary(_build()) + + +def test_format_contains_restricted_fields_stripped(): + assert "True" in format_release_summary(_build()) + + +def test_format_contains_dry_lab_only(): + assert "dry_lab_only: True" in format_release_summary(_build()) + + +def test_format_is_string(): + assert isinstance(format_release_summary(_build()), str) diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 7161c799..cfb8ef7b 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -4,7 +4,7 @@ import sys import math -BASELINE = 10822 +BASELINE = 10896 def test_test_count_regression():