diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index 64bd0a65..74e9cd06 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -301,7 +301,7 @@ Track and publish structured comparisons between pipeline selections and cheap b | ID | Task | Why it matters | Priority | |----|------|----------------|----------| | Y1 | Add cheap baseline comparison record schema (CBR-) (complete). — src/openamp_foundry/evidence/cheap_baseline_comparison_record.py: VALID_CBR_VERDICTS (4: pipeline_superior/tied/baseline_superior/insufficient_data), VALID_BASELINE_METHODS (5: charge_only_rank/length_only_rank/random_selection/charge_length_combined/hydrophobicity_only_rank), VALID_CBR_METRICS (4: auroc/hit_rate/top_k_precision/ndcg), SUPERIORITY_THRESHOLD=0.05, MIN_SAMPLE_SIZE=5; metric_delta auto-computed; verdict auto-derived; dry_lab_only=True; 62 tests. | Structured record: pipeline metric vs charge-only/random/length-only baseline; pre-registered threshold; verdict (pipeline_superior/tied/baseline_superior/insufficient_data). Forces every performance claim to cite the baseline it beat. | C | -| Y2 | Add feature importance audit schema (FIA-). | Documents which features drove selections and whether charge/length alone explains the result; anti-cheap-explanation gate; rejects candidate panels where charge-only ordering recovers the same top-k. | C | +| Y2 | Add feature importance audit schema (FIA-) (complete). — src/openamp_foundry/evidence/feature_importance_audit.py: VALID_FIA_VERDICTS (5), VALID_FEATURE_IMPORTANCE_LEVELS (4), VALID_AUDIT_FEATURES (8), DOMINATION_THRESHOLD=0.80; importance_level auto-assigned; top_feature/charge_score/length_score auto-extracted; verdict: charge_dominated when charge_explains_fraction>=0.80; dry_lab_only=True; 50 tests. | Documents which features drove selections and whether charge/length alone explains the result; anti-cheap-explanation gate. | C | | Y3 | Add selection diversity audit schema (SDA-). | Tracks sequence diversity of selected panel vs random draw; detects proximity-driven selection masquerading as discovery; required before any novelty claim. | C | | Y4 | Add pipeline maturity certificate schema (PMC-). | Aggregates CBR/FIA/SDA results into A/B/C/D maturity grade; anchors pre-registration; prevents retroactive interpretation of results. | C | | Y5 | Add Phase Y accountability gate (YAG-). | Top-level gate asserting CBR+FIA+SDA+PMC all present; verdict: accountability_verified/accountability_partial/accountability_not_established; closes Phase Y; no external pilot claim is credible without passing this gate. | C | diff --git a/src/openamp_foundry/evidence/feature_importance_audit.py b/src/openamp_foundry/evidence/feature_importance_audit.py new file mode 100644 index 00000000..5aec83a8 --- /dev/null +++ b/src/openamp_foundry/evidence/feature_importance_audit.py @@ -0,0 +1,211 @@ +"""FIA- feature importance audit schema. + +Documents which features drove selections and whether charge/length alone +explains the result. An anti-cheap-explanation gate: rejects candidate panels +where a single cheap feature (charge, length, hydrophobicity) recovers the +same top-k ordering as the full pipeline. Required before any novelty or +multi-feature claim. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +VALID_FIA_VERDICTS: frozenset[str] = frozenset({ + "multi_feature_signal", + "charge_dominated", + "length_dominated", + "hydrophobicity_dominated", + "insufficient_data", +}) + +VALID_FEATURE_IMPORTANCE_LEVELS: frozenset[str] = frozenset({ + "high", + "moderate", + "low", + "negligible", +}) + +VALID_AUDIT_FEATURES: frozenset[str] = frozenset({ + "charge", + "length", + "hydrophobicity", + "amphipathicity", + "helicity", + "sequence_motif", + "secondary_structure", + "physicochemical_composite", +}) + +DOMINATION_THRESHOLD: float = 0.80 +MIN_FEATURES_FOR_AUDIT: int = 2 + + +@dataclass +class FeatureImportanceEntry: + feature_name: str + importance_score: float + importance_level: str + + +@dataclass +class FeatureImportanceAudit: + fia_id: str + pipeline_version: str + feature_entries: list[FeatureImportanceEntry] + top_feature: str + top_feature_importance_score: float + charge_importance_score: float + length_importance_score: float + charge_explains_fraction: float + fia_verdict: str + dry_lab_only: bool + limitations: list[str] + created_at: str + + +def validate_feature_importance_audit(fia: FeatureImportanceAudit) -> None: + if not fia.fia_id.startswith("FIA-"): + raise ValueError(f"fia_id must start with 'FIA-': {fia.fia_id!r}") + if not fia.pipeline_version: + raise ValueError("pipeline_version must be non-empty") + for entry in fia.feature_entries: + if entry.feature_name not in VALID_AUDIT_FEATURES: + raise ValueError( + f"feature_name {entry.feature_name!r} not in VALID_AUDIT_FEATURES" + ) + if not (0.0 <= entry.importance_score <= 1.0): + raise ValueError( + f"importance_score must be in [0, 1] for {entry.feature_name!r}: " + f"{entry.importance_score}" + ) + if entry.importance_level not in VALID_FEATURE_IMPORTANCE_LEVELS: + raise ValueError( + f"importance_level {entry.importance_level!r} not in " + f"VALID_FEATURE_IMPORTANCE_LEVELS" + ) + if not (0.0 <= fia.charge_explains_fraction <= 1.0): + raise ValueError( + f"charge_explains_fraction must be in [0, 1]: {fia.charge_explains_fraction}" + ) + if fia.fia_verdict not in VALID_FIA_VERDICTS: + raise ValueError( + f"fia_verdict {fia.fia_verdict!r} not in VALID_FIA_VERDICTS" + ) + if not fia.dry_lab_only: + raise ValueError("dry_lab_only must be True") + if not fia.limitations: + raise ValueError("limitations must be non-empty") + if not fia.created_at: + raise ValueError("created_at must be non-empty") + + +def _assign_importance_level(score: float) -> str: + if score >= 0.50: + return "high" + if score >= 0.20: + return "moderate" + if score >= 0.05: + return "low" + return "negligible" + + +def _compute_verdict( + n_features: int, + charge_explains_fraction: float, + top_feature: str, +) -> str: + if n_features < MIN_FEATURES_FOR_AUDIT: + return "insufficient_data" + if charge_explains_fraction >= DOMINATION_THRESHOLD: + return "charge_dominated" + if top_feature == "length": + return "length_dominated" + if top_feature == "hydrophobicity": + return "hydrophobicity_dominated" + return "multi_feature_signal" + + +def build_feature_importance_audit( + *, + fia_id: str, + pipeline_version: str, + feature_importance_dicts: list[dict], + charge_explains_fraction: float, + limitations: list[str], + created_at: str, +) -> FeatureImportanceAudit: + """Build a FeatureImportanceAudit. + + feature_importance_dicts: list of dicts with keys: + feature_name, importance_score + importance_level is auto-assigned from score. + + charge_explains_fraction: fraction [0,1] of top-k ordering explained + by charge-only ranking (pre-computed externally). + """ + entries = [ + FeatureImportanceEntry( + feature_name=d["feature_name"], + importance_score=float(d["importance_score"]), + importance_level=_assign_importance_level(float(d["importance_score"])), + ) + for d in feature_importance_dicts + ] + if entries: + top_entry = max(entries, key=lambda e: e.importance_score) + top_feature = top_entry.feature_name + top_score = top_entry.importance_score + else: + top_feature = "" + top_score = 0.0 + + charge_entry = next( + (e for e in entries if e.feature_name == "charge"), None + ) + charge_score = charge_entry.importance_score if charge_entry else 0.0 + length_entry = next( + (e for e in entries if e.feature_name == "length"), None + ) + length_score = length_entry.importance_score if length_entry else 0.0 + + verdict = _compute_verdict(len(entries), charge_explains_fraction, top_feature) + fia = FeatureImportanceAudit( + fia_id=fia_id, + pipeline_version=pipeline_version, + feature_entries=entries, + top_feature=top_feature, + top_feature_importance_score=top_score, + charge_importance_score=charge_score, + length_importance_score=length_score, + charge_explains_fraction=float(charge_explains_fraction), + fia_verdict=verdict, + dry_lab_only=True, + limitations=limitations, + created_at=created_at, + ) + validate_feature_importance_audit(fia) + return fia + + +def format_feature_importance_audit(fia: FeatureImportanceAudit) -> str: + lines = [ + f"Feature Importance Audit — {fia.fia_id}", + f"Pipeline: {fia.pipeline_version}", + f"Verdict: {fia.fia_verdict}", + f"Top feature: {fia.top_feature} (score={fia.top_feature_importance_score:.3f})", + f"Charge importance: {fia.charge_importance_score:.3f} " + f"Length importance: {fia.length_importance_score:.3f}", + f"Charge explains fraction: {fia.charge_explains_fraction:.1%}", + ] + if fia.feature_entries: + lines.append("Feature scores:") + for entry in fia.feature_entries: + lines.append( + f" {entry.feature_name}: {entry.importance_score:.3f} " + f"({entry.importance_level})" + ) + lines.append(f"Created: {fia.created_at}") + lines.append(f"Limitations: {'; '.join(fia.limitations)}") + lines.append(f"dry_lab_only: {fia.dry_lab_only}") + return "\n".join(lines) diff --git a/tests/evidence/test_feature_importance_audit.py b/tests/evidence/test_feature_importance_audit.py new file mode 100644 index 00000000..9411260d --- /dev/null +++ b/tests/evidence/test_feature_importance_audit.py @@ -0,0 +1,327 @@ +"""Tests for FIA- feature importance audit schema.""" + +import pytest +from openamp_foundry.evidence.feature_importance_audit import ( + FeatureImportanceAudit, + FeatureImportanceEntry, + VALID_FIA_VERDICTS, + VALID_FEATURE_IMPORTANCE_LEVELS, + VALID_AUDIT_FEATURES, + DOMINATION_THRESHOLD, + MIN_FEATURES_FOR_AUDIT, + build_feature_importance_audit, + format_feature_importance_audit, + validate_feature_importance_audit, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_MULTI_FEATURES = [ + {"feature_name": "charge", "importance_score": 0.30}, + {"feature_name": "hydrophobicity", "importance_score": 0.25}, + {"feature_name": "length", "importance_score": 0.20}, + {"feature_name": "amphipathicity", "importance_score": 0.25}, +] + +_CHARGE_DOMINATED = [ + {"feature_name": "charge", "importance_score": 0.85}, + {"feature_name": "length", "importance_score": 0.15}, +] + + +def _build(**kwargs): + defaults = dict( + fia_id="FIA-001", + pipeline_version="v1.0", + feature_importance_dicts=_MULTI_FEATURES, + charge_explains_fraction=0.40, + limitations=["dry-lab only"], + created_at="2026-07-10", + ) + defaults.update(kwargs) + return build_feature_importance_audit(**defaults) + + +# --------------------------------------------------------------------------- +# 1. Constants +# --------------------------------------------------------------------------- + + +def test_valid_fia_verdicts_is_frozenset(): + assert isinstance(VALID_FIA_VERDICTS, frozenset) + + +def test_valid_fia_verdicts_contains_multi_feature_signal(): + assert "multi_feature_signal" in VALID_FIA_VERDICTS + + +def test_valid_fia_verdicts_contains_charge_dominated(): + assert "charge_dominated" in VALID_FIA_VERDICTS + + +def test_valid_fia_verdicts_contains_length_dominated(): + assert "length_dominated" in VALID_FIA_VERDICTS + + +def test_valid_fia_verdicts_contains_insufficient_data(): + assert "insufficient_data" in VALID_FIA_VERDICTS + + +def test_valid_feature_importance_levels_is_frozenset(): + assert isinstance(VALID_FEATURE_IMPORTANCE_LEVELS, frozenset) + + +def test_valid_feature_importance_levels_contains_high(): + assert "high" in VALID_FEATURE_IMPORTANCE_LEVELS + + +def test_valid_feature_importance_levels_contains_negligible(): + assert "negligible" in VALID_FEATURE_IMPORTANCE_LEVELS + + +def test_valid_audit_features_is_frozenset(): + assert isinstance(VALID_AUDIT_FEATURES, frozenset) + + +def test_valid_audit_features_contains_charge(): + assert "charge" in VALID_AUDIT_FEATURES + + +def test_valid_audit_features_contains_hydrophobicity(): + assert "hydrophobicity" in VALID_AUDIT_FEATURES + + +def test_domination_threshold(): + assert DOMINATION_THRESHOLD == 0.80 + + +def test_min_features_for_audit(): + assert MIN_FEATURES_FOR_AUDIT == 2 + + +# --------------------------------------------------------------------------- +# 2. build – happy paths +# --------------------------------------------------------------------------- + + +def test_build_returns_feature_importance_audit(): + assert isinstance(_build(), FeatureImportanceAudit) + + +def test_build_fia_id_stored(): + assert _build().fia_id == "FIA-001" + + +def test_build_pipeline_version_stored(): + assert _build().pipeline_version == "v1.0" + + +def test_build_dry_lab_only_true(): + assert _build().dry_lab_only is True + + +def test_build_multi_feature_verdict(): + r = _build(feature_importance_dicts=_MULTI_FEATURES, charge_explains_fraction=0.40) + assert r.fia_verdict == "multi_feature_signal" + + +def test_build_charge_dominated_verdict(): + r = _build( + feature_importance_dicts=_CHARGE_DOMINATED, + charge_explains_fraction=0.90, + ) + assert r.fia_verdict == "charge_dominated" + + +def test_build_length_dominated_verdict(): + features = [ + {"feature_name": "length", "importance_score": 0.70}, + {"feature_name": "charge", "importance_score": 0.30}, + ] + r = _build(feature_importance_dicts=features, charge_explains_fraction=0.30) + assert r.fia_verdict == "length_dominated" + + +def test_build_insufficient_data_one_feature(): + features = [{"feature_name": "charge", "importance_score": 0.80}] + r = _build(feature_importance_dicts=features, charge_explains_fraction=0.90) + assert r.fia_verdict == "insufficient_data" + + +def test_build_insufficient_data_empty(): + r = _build(feature_importance_dicts=[], charge_explains_fraction=0.0) + assert r.fia_verdict == "insufficient_data" + + +def test_build_feature_entries_are_entry_objects(): + for e in _build().feature_entries: + assert isinstance(e, FeatureImportanceEntry) + + +def test_build_top_feature_identified(): + r = _build(feature_importance_dicts=_MULTI_FEATURES, charge_explains_fraction=0.40) + assert r.top_feature == "charge" + + +def test_build_charge_score_extracted(): + r = _build(feature_importance_dicts=_MULTI_FEATURES) + assert abs(r.charge_importance_score - 0.30) < 1e-6 + + +def test_build_length_score_extracted(): + r = _build(feature_importance_dicts=_MULTI_FEATURES) + assert abs(r.length_importance_score - 0.20) < 1e-6 + + +def test_build_charge_explains_fraction_stored(): + r = _build(charge_explains_fraction=0.55) + assert abs(r.charge_explains_fraction - 0.55) < 1e-6 + + +def test_build_importance_level_auto_assigned_high(): + features = [ + {"feature_name": "charge", "importance_score": 0.60}, + {"feature_name": "length", "importance_score": 0.10}, + ] + r = _build(feature_importance_dicts=features) + charge_entry = next(e for e in r.feature_entries if e.feature_name == "charge") + assert charge_entry.importance_level == "high" + + +def test_build_importance_level_auto_assigned_negligible(): + features = [ + {"feature_name": "charge", "importance_score": 0.02}, + {"feature_name": "length", "importance_score": 0.01}, + ] + r = _build(feature_importance_dicts=features) + charge_entry = next(e for e in r.feature_entries if e.feature_name == "charge") + assert charge_entry.importance_level == "negligible" + + +def test_build_limitations_stored(): + assert _build().limitations == ["dry-lab only"] + + +def test_build_created_at_stored(): + assert _build().created_at == "2026-07-10" + + +def test_build_no_charge_entry_gives_zero_charge_score(): + features = [ + {"feature_name": "length", "importance_score": 0.60}, + {"feature_name": "amphipathicity", "importance_score": 0.40}, + ] + r = _build(feature_importance_dicts=features, charge_explains_fraction=0.10) + assert r.charge_importance_score == 0.0 + + +def test_build_hydrophobicity_dominated_verdict(): + features = [ + {"feature_name": "hydrophobicity", "importance_score": 0.80}, + {"feature_name": "length", "importance_score": 0.20}, + ] + r = _build(feature_importance_dicts=features, charge_explains_fraction=0.10) + assert r.fia_verdict == "hydrophobicity_dominated" + + +# --------------------------------------------------------------------------- +# 3. validate – rejection cases +# --------------------------------------------------------------------------- + + +def test_validate_rejects_bad_fia_id_prefix(): + with pytest.raises(ValueError, match="FIA-"): + _build(fia_id="BAD-001") + + +def test_validate_rejects_empty_pipeline_version(): + with pytest.raises(ValueError): + _build(pipeline_version="") + + +def test_validate_rejects_invalid_feature_name(): + features = [{"feature_name": "UNKNOWN", "importance_score": 0.5}] + with pytest.raises(ValueError, match="feature_name"): + _build(feature_importance_dicts=features) + + +def test_validate_rejects_importance_score_above_one(): + features = [ + {"feature_name": "charge", "importance_score": 1.5}, + {"feature_name": "length", "importance_score": 0.5}, + ] + with pytest.raises(ValueError, match="importance_score"): + _build(feature_importance_dicts=features) + + +def test_validate_rejects_importance_score_below_zero(): + features = [ + {"feature_name": "charge", "importance_score": -0.1}, + {"feature_name": "length", "importance_score": 0.5}, + ] + with pytest.raises(ValueError, match="importance_score"): + _build(feature_importance_dicts=features) + + +def test_validate_rejects_charge_explains_fraction_above_one(): + with pytest.raises(ValueError, match="charge_explains_fraction"): + _build(charge_explains_fraction=1.5) + + +def test_validate_rejects_invalid_fia_verdict(): + fia = _build() + fia.fia_verdict = "UNKNOWN" + with pytest.raises(ValueError, match="fia_verdict"): + validate_feature_importance_audit(fia) + + +def test_validate_rejects_dry_lab_only_false(): + fia = _build() + fia.dry_lab_only = False + with pytest.raises(ValueError, match="dry_lab_only"): + validate_feature_importance_audit(fia) + + +def test_validate_rejects_empty_limitations(): + with pytest.raises(ValueError, match="limitations"): + _build(limitations=[]) + + +def test_validate_rejects_empty_created_at(): + with pytest.raises(ValueError): + _build(created_at="") + + +# --------------------------------------------------------------------------- +# 4. format +# --------------------------------------------------------------------------- + + +def test_format_contains_fia_id(): + assert "FIA-001" in format_feature_importance_audit(_build()) + + +def test_format_contains_pipeline_version(): + assert "v1.0" in format_feature_importance_audit(_build()) + + +def test_format_contains_verdict(): + assert "multi_feature_signal" in format_feature_importance_audit(_build()) + + +def test_format_contains_top_feature(): + assert "charge" in format_feature_importance_audit(_build()) + + +def test_format_contains_limitations(): + assert "dry-lab only" in format_feature_importance_audit(_build()) + + +def test_format_contains_dry_lab_only(): + assert "dry_lab_only: True" in format_feature_importance_audit(_build()) + + +def test_format_is_string(): + assert isinstance(format_feature_importance_audit(_build()), str) diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 60bf2db2..81689709 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -4,7 +4,7 @@ import sys import math -BASELINE = 10560 +BASELINE = 10610 def test_test_count_regression():