diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index fc79d239..c9fdc347 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -338,3 +338,4 @@ Machine-verifiable audit trail for claim downgrades, expert decisions, and exter |---:|---|---|---| | AB1 | Add claim strength downgrade record schema (CSD-). | Auditable trail for every claim downgrade; proof_ladder_steps_dropped auto-computed; is_retracted invariant; trigger_type controlled vocabulary (benchmark/reviewer/cheap_enemy/etc.); prevents silent claim drift upward after challenges. | B/C | | AB2 | Add reviewer decision record schema (RDR-). | Machine-parseable expert review: 5 dimensions (novelty/controls/safety/synthesis/claim_scope); rating per dimension (acceptable/concerns_noted/requires_revision/not_assessed); n_blocking auto-computed; "approved" blocked when any required dimension unassessed or any dimension requires_revision. | B/C | +| AB3 | Add evidence gap notification schema (EGN-). | Structured record of what evidence is missing and how to close the gap; gap_type (9 types: missing_wet_lab/baseline/novelty/safety/reproducibility/reviewer/claim_mismatch/family_benchmark/adapter_baseline); closure_artifact_type (14 types); effort_estimate/priority/verdict; is_blocking flag; makes "needs more work" actionable. | C | diff --git a/src/openamp_foundry/evidence/evidence_gap_notification.py b/src/openamp_foundry/evidence/evidence_gap_notification.py new file mode 100644 index 00000000..7cd14770 --- /dev/null +++ b/src/openamp_foundry/evidence/evidence_gap_notification.py @@ -0,0 +1,171 @@ +"""EGN- Evidence Gap Notification schema. + +Machine-readable record of what evidence is missing and how to close +the gap. Makes "needs more work" outcomes actionable. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +VALID_EGN_GAP_TYPES: frozenset[str] = frozenset({ + "missing_wet_lab_validation", + "missing_baseline_comparison", + "missing_novelty_evidence", + "missing_safety_clearance", + "missing_reproducibility_data", + "missing_reviewer_assessment", + "claim_strength_mismatch", + "missing_family_benchmark", + "missing_adapter_baseline", +}) + +VALID_EGN_CLOSURE_ARTIFACT_TYPES: frozenset[str] = frozenset({ + "WHR", "CBR", "SEG", "CFC", "FNR", "PSC", "DCR", "RMC", + "RDR", "CSD", "FBH", "CBF", "BXR", "ARG", +}) + +VALID_EGN_EFFORT_ESTIMATES: frozenset[str] = frozenset({ + "hours", + "days", + "weeks", + "months", + "unknown", +}) + +VALID_EGN_PRIORITIES: frozenset[str] = frozenset({ + "critical", "high", "medium", "low", +}) + +VALID_EGN_VERDICTS: frozenset[str] = frozenset({ + "actionable", + "blocked_on_resources", + "under_investigation", + "accepted_limitation", +}) + + +@dataclass +class EvidenceGapNotification: + egn_id: str + pipeline_version: str + artifact_id: str + gap_type: str + gap_description: str + closure_artifact_type: str + closure_description: str + effort_estimate: str + priority: str + verdict: str + is_blocking: bool + limitations: list[str] + created_at: str + dry_lab_only: bool = True + + +def build_evidence_gap_notification( + *, + egn_id: str, + pipeline_version: str, + artifact_id: str, + gap_type: str, + gap_description: str, + closure_artifact_type: str, + closure_description: str, + effort_estimate: str, + priority: str, + verdict: str, + is_blocking: bool, + limitations: list[str], + created_at: str, +) -> EvidenceGapNotification: + egn = EvidenceGapNotification( + egn_id=egn_id, + pipeline_version=pipeline_version, + artifact_id=artifact_id, + gap_type=gap_type, + gap_description=gap_description, + closure_artifact_type=closure_artifact_type, + closure_description=closure_description, + effort_estimate=effort_estimate, + priority=priority, + verdict=verdict, + is_blocking=is_blocking, + dry_lab_only=True, + limitations=limitations, + created_at=created_at, + ) + validate_evidence_gap_notification(egn) + return egn + + +def validate_evidence_gap_notification( + egn: EvidenceGapNotification, +) -> None: + if not egn.egn_id.startswith("EGN-"): + raise ValueError( + f"egn_id must start with 'EGN-': {egn.egn_id!r}" + ) + if not egn.pipeline_version: + raise ValueError("pipeline_version must be non-empty") + if not egn.artifact_id: + raise ValueError("artifact_id must be non-empty") + if not egn.gap_description: + raise ValueError("gap_description must be non-empty") + if len(egn.gap_description) > 400: + raise ValueError( + f"gap_description exceeds 400 chars: {len(egn.gap_description)}" + ) + if egn.gap_type not in VALID_EGN_GAP_TYPES: + raise ValueError( + f"gap_type {egn.gap_type!r} not in VALID_EGN_GAP_TYPES" + ) + if not egn.closure_description: + raise ValueError("closure_description must be non-empty") + if egn.closure_artifact_type not in VALID_EGN_CLOSURE_ARTIFACT_TYPES: + raise ValueError( + f"closure_artifact_type {egn.closure_artifact_type!r} " + f"not in VALID_EGN_CLOSURE_ARTIFACT_TYPES" + ) + if egn.effort_estimate not in VALID_EGN_EFFORT_ESTIMATES: + raise ValueError( + f"effort_estimate {egn.effort_estimate!r} " + f"not in VALID_EGN_EFFORT_ESTIMATES" + ) + if egn.priority not in VALID_EGN_PRIORITIES: + raise ValueError( + f"priority {egn.priority!r} not in VALID_EGN_PRIORITIES" + ) + if egn.verdict not in VALID_EGN_VERDICTS: + raise ValueError( + f"verdict {egn.verdict!r} not in VALID_EGN_VERDICTS" + ) + if egn.dry_lab_only is not True: + raise ValueError("dry_lab_only must be True") + if not egn.limitations: + raise ValueError("limitations must be non-empty") + if not egn.created_at: + raise ValueError("created_at must be non-empty") + + +def format_evidence_gap_notification( + egn: EvidenceGapNotification, +) -> str: + lines = [ + f"Evidence Gap Notification — {egn.egn_id}", + f"Pipeline: {egn.pipeline_version}", + f"Artifact: {egn.artifact_id}", + f"Gap type: {egn.gap_type}", + f"Gap description: {egn.gap_description}", + f"Closure artifact type: {egn.closure_artifact_type}", + f"Closure description: {egn.closure_description}", + f"Effort estimate: {egn.effort_estimate}", + f"Priority: {egn.priority}", + f"Verdict: {egn.verdict}", + f"Is blocking: {egn.is_blocking}", + ] + for lim in egn.limitations: + lines.append(f" Limitation: {lim}") + lines.append(f"Dry lab only: {egn.dry_lab_only}") + lines.append(f"Created: {egn.created_at}") + return "\n".join(lines) diff --git a/tests/evidence/test_evidence_gap_notification.py b/tests/evidence/test_evidence_gap_notification.py new file mode 100644 index 00000000..e6944569 --- /dev/null +++ b/tests/evidence/test_evidence_gap_notification.py @@ -0,0 +1,332 @@ +"""Tests for EGN- evidence gap notification schema.""" + +import pytest +from openamp_foundry.evidence.evidence_gap_notification import ( + EvidenceGapNotification, + VALID_EGN_GAP_TYPES, + VALID_EGN_CLOSURE_ARTIFACT_TYPES, + VALID_EGN_EFFORT_ESTIMATES, + VALID_EGN_PRIORITIES, + VALID_EGN_VERDICTS, + build_evidence_gap_notification, + format_evidence_gap_notification, + validate_evidence_gap_notification, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _build(**kwargs): + defaults = dict( + egn_id="EGN-001", + pipeline_version="v1.0", + artifact_id="fam-001", + gap_type="missing_wet_lab_validation", + gap_description="No WHR records exist for this candidate family.", + closure_artifact_type="WHR", + closure_description="Submit at least one WHR record for a member of this family.", + effort_estimate="weeks", + priority="high", + verdict="actionable", + is_blocking=True, + limitations=["Requires wet-lab partner access."], + created_at="2026-07-10", + ) + defaults.update(kwargs) + return build_evidence_gap_notification(**defaults) + + +def _make_egn(**kwargs): + defaults = dict( + egn_id="EGN-001", + pipeline_version="v1.0", + artifact_id="fam-001", + gap_type="missing_wet_lab_validation", + gap_description="No WHR records exist for this candidate family.", + closure_artifact_type="WHR", + closure_description="Submit at least one WHR record for a member of this family.", + effort_estimate="weeks", + priority="high", + verdict="actionable", + is_blocking=True, + dry_lab_only=True, + limitations=["Requires wet-lab partner access."], + created_at="2026-07-10", + ) + defaults.update(kwargs) + return EvidenceGapNotification(**defaults) + + +# --------------------------------------------------------------------------- +# 1. Constants (14 tests) +# --------------------------------------------------------------------------- + + +def test_valid_egn_gap_types_is_frozenset(): + assert isinstance(VALID_EGN_GAP_TYPES, frozenset) + + +def test_valid_egn_gap_types_has_nine(): + assert len(VALID_EGN_GAP_TYPES) == 9 + + +def test_valid_egn_gap_types_contains_missing_wet_lab(): + assert "missing_wet_lab_validation" in VALID_EGN_GAP_TYPES + + +def test_valid_egn_gap_types_contains_missing_baseline(): + assert "missing_baseline_comparison" in VALID_EGN_GAP_TYPES + + +def test_valid_egn_gap_types_contains_claim_strength(): + assert "claim_strength_mismatch" in VALID_EGN_GAP_TYPES + + +def test_valid_egn_closure_artifact_types_is_frozenset(): + assert isinstance(VALID_EGN_CLOSURE_ARTIFACT_TYPES, frozenset) + + +def test_valid_egn_closure_artifact_types_has_fourteen(): + assert len(VALID_EGN_CLOSURE_ARTIFACT_TYPES) == 14 + + +def test_valid_egn_closure_artifact_types_contains_whr(): + assert "WHR" in VALID_EGN_CLOSURE_ARTIFACT_TYPES + + +def test_valid_egn_closure_artifact_types_contains_arg(): + assert "ARG" in VALID_EGN_CLOSURE_ARTIFACT_TYPES + + +def test_valid_egn_effort_estimates_is_frozenset(): + assert isinstance(VALID_EGN_EFFORT_ESTIMATES, frozenset) + + +def test_valid_egn_effort_estimates_has_five(): + assert len(VALID_EGN_EFFORT_ESTIMATES) == 5 + + +def test_valid_egn_priorities_is_frozenset(): + assert isinstance(VALID_EGN_PRIORITIES, frozenset) + + +def test_valid_egn_priorities_has_four(): + assert len(VALID_EGN_PRIORITIES) == 4 + + +def test_valid_egn_verdicts_is_frozenset(): + assert isinstance(VALID_EGN_VERDICTS, frozenset) + assert len(VALID_EGN_VERDICTS) == 4 + + +# --------------------------------------------------------------------------- +# 2. build – happy paths (16 tests) +# --------------------------------------------------------------------------- + + +def test_build_returns_evidence_gap_notification(): + assert isinstance(_build(), EvidenceGapNotification) + + +def test_build_gap_type_stored(): + egn = _build(gap_type="missing_safety_clearance") + assert egn.gap_type == "missing_safety_clearance" + + +def test_build_closure_artifact_type_stored(): + egn = _build(closure_artifact_type="CFC") + assert egn.closure_artifact_type == "CFC" + + +def test_build_is_blocking_true(): + egn = _build(is_blocking=True) + assert egn.is_blocking is True + + +def test_build_is_blocking_false(): + egn = _build(is_blocking=False) + assert egn.is_blocking is False + + +def test_build_priority_critical(): + egn = _build(priority="critical") + assert egn.priority == "critical" + + +def test_build_effort_weeks(): + egn = _build(effort_estimate="weeks") + assert egn.effort_estimate == "weeks" + + +def test_build_verdict_actionable(): + egn = _build(verdict="actionable") + assert egn.verdict == "actionable" + + +def test_build_verdict_blocked_on_resources(): + egn = _build(verdict="blocked_on_resources") + assert egn.verdict == "blocked_on_resources" + + +def test_build_verdict_accepted_limitation(): + egn = _build(verdict="accepted_limitation") + assert egn.verdict == "accepted_limitation" + + +def test_build_verdict_under_investigation(): + egn = _build(verdict="under_investigation") + assert egn.verdict == "under_investigation" + + +def test_build_egn_id_stored(): + egn = _build(egn_id="EGN-099") + assert egn.egn_id == "EGN-099" + + +def test_build_pipeline_version_stored(): + egn = _build(pipeline_version="v2.0") + assert egn.pipeline_version == "v2.0" + + +def test_build_artifact_id_stored(): + egn = _build(artifact_id="fam-099") + assert egn.artifact_id == "fam-099" + + +def test_build_closure_description_stored(): + egn = _build(closure_description="Run a charge-matched benchmark.") + assert egn.closure_description == "Run a charge-matched benchmark." + + +def test_build_limitations_stored(): + egn = _build(limitations=["No partner available.", "Expensive assay."]) + assert len(egn.limitations) == 2 + assert "No partner available." in egn.limitations + + +# --------------------------------------------------------------------------- +# 3. validate – rejection cases (16 tests) +# --------------------------------------------------------------------------- + + +def test_validate_rejects_bad_egn_id_prefix(): + with pytest.raises(ValueError, match="EGN-"): + _build(egn_id="BAD-001") + + +def test_validate_rejects_empty_pipeline_version(): + with pytest.raises(ValueError): + _build(pipeline_version="") + + +def test_validate_rejects_empty_artifact_id(): + with pytest.raises(ValueError): + _build(artifact_id="") + + +def test_validate_rejects_empty_gap_description(): + with pytest.raises(ValueError): + _build(gap_description="") + + +def test_validate_rejects_gap_description_too_long(): + with pytest.raises(ValueError, match="gap_description"): + _build(gap_description="x" * 401) + + +def test_validate_rejects_invalid_gap_type(): + with pytest.raises(ValueError, match="gap_type"): + _build(gap_type="invalid_gap_type") + + +def test_validate_rejects_empty_closure_description(): + with pytest.raises(ValueError): + _build(closure_description="") + + +def test_validate_rejects_invalid_closure_artifact_type(): + with pytest.raises(ValueError, match="closure_artifact_type"): + _build(closure_artifact_type="INVALID") + + +def test_validate_rejects_invalid_effort_estimate(): + with pytest.raises(ValueError, match="effort_estimate"): + _build(effort_estimate="years") + + +def test_validate_rejects_invalid_priority(): + with pytest.raises(ValueError, match="priority"): + _build(priority="urgent") + + +def test_validate_rejects_invalid_verdict(): + with pytest.raises(ValueError, match="verdict"): + _build(verdict="rejected") + + +def test_validate_rejects_dry_lab_only_false(): + egn = _make_egn() + egn.dry_lab_only = False + with pytest.raises(ValueError, match="dry_lab_only"): + validate_evidence_gap_notification(egn) + + +def test_validate_rejects_empty_limitations(): + egn = _make_egn() + egn.limitations = [] + with pytest.raises(ValueError, match="limitations"): + validate_evidence_gap_notification(egn) + + +def test_validate_rejects_empty_created_at(): + with pytest.raises(ValueError): + _build(created_at="") + + +def test_validate_rejects_empty_egn_id(): + with pytest.raises(ValueError, match="EGN-"): + _build(egn_id="") + + +def test_validate_rejects_gap_description_400_boundary(): + with pytest.raises(ValueError, match="gap_description"): + _build(gap_description="x" * 401) + + +# --------------------------------------------------------------------------- +# 4. format (8 tests) +# --------------------------------------------------------------------------- + + +def test_format_contains_egn_id(): + assert "EGN-001" in format_evidence_gap_notification(_build()) + + +def test_format_contains_gap_type(): + assert "missing_wet_lab_validation" in format_evidence_gap_notification(_build()) + + +def test_format_contains_priority(): + assert "high" in format_evidence_gap_notification(_build()) + + +def test_format_contains_effort(): + assert "weeks" in format_evidence_gap_notification(_build()) + + +def test_format_contains_verdict(): + assert "actionable" in format_evidence_gap_notification(_build()) + + +def test_format_contains_is_blocking(): + assert "True" in format_evidence_gap_notification(_build()) + + +def test_format_contains_closure_artifact_type(): + assert "WHR" in format_evidence_gap_notification(_build()) + + +def test_format_is_string(): + assert isinstance(format_evidence_gap_notification(_build()), str) diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index a8399e74..66e32b57 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -4,7 +4,7 @@ import sys import math -BASELINE = 11911 +BASELINE = 11965 def test_test_count_regression():