From b47aa39d3e07390697fb98eaded9f2ce1be2ac6e Mon Sep 17 00:00:00 2001 From: OpenCode Date: Fri, 10 Jul 2026 16:34:12 +0700 Subject: [PATCH] feat: Phase H H10 external-simulator review checklist -- ESC- schema with 12-item checklist, 7 required items, auto-computed verdict (approved/conditional_approval/not_approved) --- docs/research/NEXT_100_PR_MAP.md | 2 +- .../external_simulator_checklist.py | 172 ++++++++++ .../test_external_simulator_checklist.py | 317 ++++++++++++++++++ tests/test_test_count_regression.py | 2 +- 4 files changed, 491 insertions(+), 2 deletions(-) create mode 100644 src/openamp_foundry/simulation/external_simulator_checklist.py create mode 100644 tests/simulation/test_external_simulator_checklist.py diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index 3c2ecd02..e4c5ac4f 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -152,7 +152,7 @@ Make simulation useful or harmless. | H7 | Add documentation that failed simulation stays useful as negative evidence (complete). — docs/FAILED_SIMULATION_AS_NEGATIVE_EVIDENCE.md: cultural standard requiring NRR- record for every failure; agent MUST NOT rules; anti-selective-reporting enforcement. | Cultural standard. | A | | H8 | Add `weighted` integration dry-run report, still blocked by gate (complete). — WDR- schema: 15 fields, 12 validation rules, results_applied_to_ranking=False enforcement, GATE_CLOSED_DISCLAIMER required, counterfactual-only; 63 tests. | Shows what would change without applying. | C | | H9 | Add module deprecation mechanism for simulation theater (complete). — simulation/deprecation_enforcer.py: marks and enforces deprecation of simulation modules that produce theater (no real predictive value); make simulation-deprecation-check target; tests/simulation/test_deprecation_enforcer.py. | Cleanup discipline. | B/C | -| H10 | Add external-simulator review checklist. | Safer ecosystem bridges. | D | +| H10 | Add external-simulator review checklist. | Safer ecosystem bridges. | D | DONE | ## Phase I — Interoperability and adoption diff --git a/src/openamp_foundry/simulation/external_simulator_checklist.py b/src/openamp_foundry/simulation/external_simulator_checklist.py new file mode 100644 index 00000000..86181d6f --- /dev/null +++ b/src/openamp_foundry/simulation/external_simulator_checklist.py @@ -0,0 +1,172 @@ +"""ESC- external-simulator review checklist schema. + +Before any external simulation service is connected to the pipeline, +it must pass a structured review. This checklist makes the review +machine-auditable: every item is explicitly confirmed or flagged. + +The items cover: data privacy (what does the service receive?), +output trustworthiness (what does it return?), security (authentication +and TLS), license (can results be used commercially/academically?), +and operational reliability (SLA, fallback, failure modes). + +A simulator with unchecked critical items must not be used in candidate +ranking until those items are resolved. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +CHECKLIST_ITEMS: tuple[str, ...] = ( + "data_sent_to_service_documented", + "sequence_data_not_transmitted", + "output_schema_documented", + "output_schema_validated_against_schema", + "authentication_required", + "tls_enforced", + "license_permits_research_use", + "failure_mode_documented", + "timeout_policy_declared", + "network_call_declared_in_nsn_policy", + "baseline_comparison_exists", + "deprecation_policy_declared", +) + +REQUIRED_ITEMS: tuple[str, ...] = ( + "data_sent_to_service_documented", + "sequence_data_not_transmitted", + "output_schema_documented", + "tls_enforced", + "license_permits_research_use", + "failure_mode_documented", + "network_call_declared_in_nsn_policy", +) + +VALID_ESC_VERDICTS: frozenset[str] = frozenset({ + "approved", + "conditional_approval", + "not_approved", +}) + + +@dataclass +class ExternalSimulatorChecklist: + esc_id: str + simulator_name: str + pipeline_version: str + items_checked: list[str] + items_unchecked: list[str] + required_items_complete: bool + n_items_checked: int + esc_verdict: str + reviewer_notes: str + dry_lab_only: bool + limitations: list[str] + created_at: str + + +def validate_external_simulator_checklist(esc: ExternalSimulatorChecklist) -> None: + if not esc.esc_id.startswith("ESC-"): + raise ValueError(f"esc_id must start with 'ESC-': {esc.esc_id!r}") + if not esc.simulator_name: + raise ValueError("simulator_name must be non-empty") + if not esc.pipeline_version: + raise ValueError("pipeline_version must be non-empty") + for item in esc.items_checked: + if item not in CHECKLIST_ITEMS: + raise ValueError(f"checked item {item!r} not in CHECKLIST_ITEMS") + for item in esc.items_unchecked: + if item not in CHECKLIST_ITEMS: + raise ValueError(f"unchecked item {item!r} not in CHECKLIST_ITEMS") + all_items = set(esc.items_checked) | set(esc.items_unchecked) + if all_items != set(CHECKLIST_ITEMS): + raise ValueError( + "items_checked + items_unchecked must cover all CHECKLIST_ITEMS exactly" + ) + if esc.n_items_checked != len(esc.items_checked): + raise ValueError("n_items_checked must equal len(items_checked)") + expected_required_complete = all( + r in esc.items_checked for r in REQUIRED_ITEMS + ) + if esc.required_items_complete != expected_required_complete: + raise ValueError( + "required_items_complete inconsistent with items_checked and REQUIRED_ITEMS" + ) + if esc.esc_verdict not in VALID_ESC_VERDICTS: + raise ValueError( + f"esc_verdict {esc.esc_verdict!r} not in VALID_ESC_VERDICTS" + ) + if esc.esc_verdict == "approved" and not esc.required_items_complete: + raise ValueError( + "esc_verdict='approved' requires required_items_complete=True" + ) + if not esc.dry_lab_only: + raise ValueError("dry_lab_only must be True") + if not esc.limitations: + raise ValueError("limitations must be non-empty") + if not esc.created_at: + raise ValueError("created_at must be non-empty") + + +def _compute_verdict(required_complete: bool, n_checked: int) -> str: + if required_complete and n_checked == len(CHECKLIST_ITEMS): + return "approved" + if required_complete: + return "conditional_approval" + return "not_approved" + + +def build_external_simulator_checklist( + *, + esc_id: str, + simulator_name: str, + pipeline_version: str, + items_checked: list[str], + reviewer_notes: str = "", + limitations: list[str], + created_at: str, +) -> ExternalSimulatorChecklist: + """Build an ExternalSimulatorChecklist. + + items_unchecked, required_items_complete, n_items_checked, and esc_verdict + are all auto-computed from items_checked and CHECKLIST_ITEMS / REQUIRED_ITEMS. + """ + checked_set = set(items_checked) + unchecked = [item for item in CHECKLIST_ITEMS if item not in checked_set] + required_complete = all(r in checked_set for r in REQUIRED_ITEMS) + n_checked = len(items_checked) + verdict = _compute_verdict(required_complete, n_checked) + esc = ExternalSimulatorChecklist( + esc_id=esc_id, + simulator_name=simulator_name, + pipeline_version=pipeline_version, + items_checked=list(items_checked), + items_unchecked=unchecked, + required_items_complete=required_complete, + n_items_checked=n_checked, + esc_verdict=verdict, + reviewer_notes=reviewer_notes, + dry_lab_only=True, + limitations=limitations, + created_at=created_at, + ) + validate_external_simulator_checklist(esc) + return esc + + +def format_external_simulator_checklist(esc: ExternalSimulatorChecklist) -> str: + lines = [ + f"External-Simulator Checklist — {esc.esc_id}", + f"Simulator: {esc.simulator_name} | Pipeline: {esc.pipeline_version}", + f"Verdict: {esc.esc_verdict}", + f"Items checked: {esc.n_items_checked}/{len(CHECKLIST_ITEMS)}", + f"Required items complete: {esc.required_items_complete}", + ] + if esc.items_unchecked: + lines.append(f"Unchecked items: {', '.join(esc.items_unchecked)}") + if esc.reviewer_notes: + lines.append(f"Notes: {esc.reviewer_notes}") + lines.append(f"Limitations: {'; '.join(esc.limitations)}") + lines.append(f"Created: {esc.created_at}") + lines.append(f"dry_lab_only: {esc.dry_lab_only}") + return "\n".join(lines) diff --git a/tests/simulation/test_external_simulator_checklist.py b/tests/simulation/test_external_simulator_checklist.py new file mode 100644 index 00000000..d3fd1fcc --- /dev/null +++ b/tests/simulation/test_external_simulator_checklist.py @@ -0,0 +1,317 @@ +"""Tests for ESC- external-simulator review checklist schema.""" + +import pytest +from openamp_foundry.simulation.external_simulator_checklist import ( + ExternalSimulatorChecklist, + CHECKLIST_ITEMS, + REQUIRED_ITEMS, + VALID_ESC_VERDICTS, + build_external_simulator_checklist, + format_external_simulator_checklist, + validate_external_simulator_checklist, +) + +_ALL_ITEMS = list(CHECKLIST_ITEMS) +_REQUIRED = list(REQUIRED_ITEMS) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _build(**kwargs): + defaults = dict( + esc_id="ESC-001", + simulator_name="membrane-sim-v2", + pipeline_version="v1.0", + items_checked=_ALL_ITEMS, + reviewer_notes="", + limitations=["dry-lab only"], + created_at="2026-07-10", + ) + defaults.update(kwargs) + return build_external_simulator_checklist(**defaults) + + +def _build_required_only(**kwargs): + defaults = dict( + esc_id="ESC-001", + simulator_name="membrane-sim-v2", + pipeline_version="v1.0", + items_checked=_REQUIRED, + limitations=["dry-lab only"], + created_at="2026-07-10", + ) + defaults.update(kwargs) + return build_external_simulator_checklist(**defaults) + + +# --------------------------------------------------------------------------- +# 1. Constants +# --------------------------------------------------------------------------- + + +def test_checklist_items_is_tuple(): + assert isinstance(CHECKLIST_ITEMS, tuple) + + +def test_checklist_items_contains_data_sent(): + assert "data_sent_to_service_documented" in CHECKLIST_ITEMS + + +def test_checklist_items_contains_sequence_not_transmitted(): + assert "sequence_data_not_transmitted" in CHECKLIST_ITEMS + + +def test_checklist_items_contains_tls_enforced(): + assert "tls_enforced" in CHECKLIST_ITEMS + + +def test_checklist_items_contains_license(): + assert "license_permits_research_use" in CHECKLIST_ITEMS + + +def test_checklist_items_contains_failure_mode(): + assert "failure_mode_documented" in CHECKLIST_ITEMS + + +def test_checklist_items_contains_nsn_policy(): + assert "network_call_declared_in_nsn_policy" in CHECKLIST_ITEMS + + +def test_checklist_items_has_twelve_items(): + assert len(CHECKLIST_ITEMS) == 12 + + +def test_required_items_is_tuple(): + assert isinstance(REQUIRED_ITEMS, tuple) + + +def test_required_items_contains_sequence_not_transmitted(): + assert "sequence_data_not_transmitted" in REQUIRED_ITEMS + + +def test_required_items_contains_tls_enforced(): + assert "tls_enforced" in REQUIRED_ITEMS + + +def test_required_items_subset_of_checklist(): + assert set(REQUIRED_ITEMS) <= set(CHECKLIST_ITEMS) + + +def test_valid_esc_verdicts_is_frozenset(): + assert isinstance(VALID_ESC_VERDICTS, frozenset) + + +def test_valid_esc_verdicts_contains_approved(): + assert "approved" in VALID_ESC_VERDICTS + + +def test_valid_esc_verdicts_contains_conditional_approval(): + assert "conditional_approval" in VALID_ESC_VERDICTS + + +def test_valid_esc_verdicts_contains_not_approved(): + assert "not_approved" in VALID_ESC_VERDICTS + + +# --------------------------------------------------------------------------- +# 2. build happy paths +# --------------------------------------------------------------------------- + + +def test_build_returns_external_simulator_checklist(): + assert isinstance(_build(), ExternalSimulatorChecklist) + + +def test_build_esc_id_stored(): + assert _build().esc_id == "ESC-001" + + +def test_build_simulator_name_stored(): + assert _build().simulator_name == "membrane-sim-v2" + + +def test_build_pipeline_version_stored(): + assert _build().pipeline_version == "v1.0" + + +def test_build_dry_lab_only_true(): + assert _build().dry_lab_only is True + + +def test_build_all_items_gives_approved(): + assert _build().esc_verdict == "approved" + + +def test_build_required_only_gives_conditional_approval(): + r = _build_required_only() + assert r.esc_verdict == "conditional_approval" + + +def test_build_no_items_gives_not_approved(): + r = _build(items_checked=[]) + assert r.esc_verdict == "not_approved" + + +def test_build_required_items_complete_true_when_all(): + assert _build().required_items_complete is True + + +def test_build_required_items_complete_true_when_required_only(): + assert _build_required_only().required_items_complete is True + + +def test_build_required_items_complete_false_when_none(): + r = _build(items_checked=[]) + assert r.required_items_complete is False + + +def test_build_n_items_checked_all(): + assert _build().n_items_checked == 12 + + +def test_build_n_items_checked_required_only(): + assert _build_required_only().n_items_checked == 7 + + +def test_build_n_items_checked_none(): + r = _build(items_checked=[]) + assert r.n_items_checked == 0 + + +def test_build_items_unchecked_empty_when_all(): + assert _build().items_unchecked == [] + + +def test_build_items_unchecked_populated_when_partial(): + r = _build_required_only() + assert len(r.items_unchecked) == len(CHECKLIST_ITEMS) - len(REQUIRED_ITEMS) + + +def test_build_reviewer_notes_stored(): + r = _build(reviewer_notes="reviewed by safety team") + assert r.reviewer_notes == "reviewed by safety team" + + +def test_build_limitations_stored(): + assert _build().limitations == ["dry-lab only"] + + +def test_build_created_at_stored(): + assert _build().created_at == "2026-07-10" + + +# --------------------------------------------------------------------------- +# 3. validate rejection cases +# --------------------------------------------------------------------------- + + +def test_validate_rejects_bad_esc_id_prefix(): + with pytest.raises(ValueError, match="ESC-"): + _build(esc_id="BAD-001") + + +def test_validate_rejects_empty_simulator_name(): + with pytest.raises(ValueError, match="simulator_name"): + _build(simulator_name="") + + +def test_validate_rejects_empty_pipeline_version(): + with pytest.raises(ValueError): + _build(pipeline_version="") + + +def test_validate_rejects_invalid_checked_item(): + with pytest.raises(ValueError, match="checked item"): + _build(items_checked=_ALL_ITEMS + ["UNKNOWN_ITEM"]) + + +def test_validate_rejects_n_items_checked_mismatch(): + esc = _build() + esc.n_items_checked = 99 + with pytest.raises(ValueError, match="n_items_checked"): + validate_external_simulator_checklist(esc) + + +def test_validate_rejects_required_items_complete_mismatch(): + esc = _build() + esc.required_items_complete = False + with pytest.raises(ValueError, match="required_items_complete"): + validate_external_simulator_checklist(esc) + + +def test_validate_rejects_invalid_esc_verdict(): + esc = _build() + esc.esc_verdict = "UNKNOWN" + with pytest.raises(ValueError, match="esc_verdict"): + validate_external_simulator_checklist(esc) + + +def test_validate_rejects_approved_without_required_complete(): + esc = _build(items_checked=[]) + esc.esc_verdict = "approved" + esc.required_items_complete = True + with pytest.raises(ValueError): + validate_external_simulator_checklist(esc) + + +def test_validate_rejects_dry_lab_only_false(): + esc = _build() + esc.dry_lab_only = False + with pytest.raises(ValueError, match="dry_lab_only"): + validate_external_simulator_checklist(esc) + + +def test_validate_rejects_empty_limitations(): + with pytest.raises(ValueError, match="limitations"): + _build(limitations=[]) + + +def test_validate_rejects_empty_created_at(): + with pytest.raises(ValueError): + _build(created_at="") + + +# --------------------------------------------------------------------------- +# 4. format +# --------------------------------------------------------------------------- + + +def test_format_contains_esc_id(): + assert "ESC-001" in format_external_simulator_checklist(_build()) + + +def test_format_contains_simulator_name(): + assert "membrane-sim-v2" in format_external_simulator_checklist(_build()) + + +def test_format_contains_pipeline_version(): + assert "v1.0" in format_external_simulator_checklist(_build()) + + +def test_format_contains_verdict(): + assert "approved" in format_external_simulator_checklist(_build()) + + +def test_format_contains_items_checked_count(): + assert "12" in format_external_simulator_checklist(_build()) + + +def test_format_contains_unchecked_item_when_partial(): + r = _build_required_only() + output = format_external_simulator_checklist(r) + assert any(item in output for item in r.items_unchecked) + + +def test_format_contains_limitations(): + assert "dry-lab only" in format_external_simulator_checklist(_build()) + + +def test_format_contains_dry_lab_only(): + assert "dry_lab_only: True" in format_external_simulator_checklist(_build()) + + +def test_format_is_string(): + assert isinstance(format_external_simulator_checklist(_build()), str) diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 892fbb9e..4e56cc08 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -4,7 +4,7 @@ import sys import math -BASELINE = 11083 +BASELINE = 11138 def test_test_count_regression():