feat: Phase Z Z1 family blindness challenge harness -- FBH- schema tracks per-family AUROC and panel representation, flags weak AMP classes excluded from selected panel (#xxxx)

Author: OpenCode

docs/research/NEXT_100_PR_MAP.md

@@ -305,3 +305,11 @@ Track and publish structured comparisons between pipeline selections and cheap b
 | Y3 | Add selection diversity audit schema (SDA-) (complete). — src/openamp_foundry/evidence/selection_diversity_audit.py: VALID_SDA_VERDICTS (4: diverse_panel/moderately_diverse/proximity_driven/insufficient_data), VALID_DIVERSITY_METRICS (4), DIVERSE_PANEL_THRESHOLD=0.10, PROXIMITY_DRIVEN_THRESHOLD=-0.05, MIN_PANEL_SIZE=3; diversity_delta auto-computed; verdict: diverse_panel (delta>=0.10), proximity_driven (delta<=-0.05); dry_lab_only=True; 46 tests. | Tracks sequence diversity of selected panel vs random draw; detects proximity-driven selection masquerading as discovery; required before any novelty claim. | C |
 | Y4 | Add pipeline maturity certificate schema (PMC-) (complete). — src/openamp_foundry/evidence/pipeline_maturity_certificate.py: VALID_PMC_GRADES (A-D), VALID_PMC_VERDICTS (4: pipeline_validated/pipeline_provisional/pipeline_unvalidated/insufficient_evidence), REQUIRED_PMC_COMPONENTS=(CBR,FIA,SDA); PMCComponentCheck helper; grade A (all 3 superior), B (2), C (1), D (0/none assessed); contributes_to_grade auto-derived from verdict; dry_lab_only=True; 54 tests. | Aggregates CBR/FIA/SDA results into A/B/C/D maturity grade; anchors pre-registration; prevents retroactive interpretation. | C |
 | Y5 | Add Phase Y accountability gate (YAG-) (complete). — src/openamp_foundry/evidence/phase_y_accountability_gate.py: REQUIRED_Y_COMPONENTS=(CBR,FIA,SDA,PMC), VALID_YAG_VERDICTS (3: accountability_verified/accountability_partial/accountability_not_established); YComponentCheck helper; verdict: accountability_verified (all 4), accountability_partial (2-3), accountability_not_established (0-1); artifact_id prefix-validated per component; dry_lab_only=True; 54 tests. Closes Phase Y. | Top-level gate asserting CBR+FIA+SDA+PMC all present; closes Phase Y; no external pilot claim is credible without passing this gate. | C |
+
+## Phase Z — Per-family benchmark accountability
+
+Make per-family performance gaps visible and machine-checkable, so the pipeline cannot hide weak AMP class coverage behind aggregate metrics.
+
+| PR | Task | Why it matters | Review class |
+|---:|---|---|---|
+| Z1 | Add family blindness challenge harness schema (FBH-). | Per-family AUROC + panel representation check; flags when weak AMP classes (AUROC&lt;0.55) are excluded from selected panel; prevents aggregate-metric hiding of family blind spots. | C |

src/openamp_foundry/evidence/family_blindness_challenge.py

@@ -0,0 +1,219 @@
+"""FBH- family blindness challenge harness schema.
+
+Per-family benchmark performance record: flags when the pipeline
+underperforms on specific AMP families. Prevents aggregate-metric
+hiding of family-blind spots.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+VALID_FBH_VERDICTS: frozenset[str] = frozenset({
+    "all_families_represented",
+    "weak_family_excluded",
+    "insufficient_data",
+})
+
+VALID_AUROC_GRADES: frozenset[str] = frozenset({
+    "strong",
+    "adequate",
+    "weak",
+    "not_evaluated",
+})
+
+WEAK_FAMILY_AUROC_THRESHOLD: float = 0.55
+WEAK_FAMILY_PANEL_FLOOR: float = 0.10
+
+
+@dataclass
+class FamilyPerformanceEntry:
+    family_name: str
+    auroc: float
+    n_candidates: int
+    panel_count: int
+    panel_fraction: float
+    auroc_grade: str
+    is_weak_class: bool
+
+
+@dataclass
+class FamilyBlingnessChallenge:
+    fbh_id: str
+    pipeline_version: str
+    family_entries: list[FamilyPerformanceEntry]
+    n_families_total: int
+    n_weak_families: int
+    n_excluded_weak_families: int
+    excluded_weak_family_names: list[str]
+    verdict: str
+    dry_lab_only: bool
+    limitations: list[str]
+    created_at: str
+
+
+def _compute_auroc_grade(auroc: float) -> str:
+    if auroc == -1.0:
+        return "not_evaluated"
+    if auroc >= 0.70:
+        return "strong"
+    if auroc >= 0.55:
+        return "adequate"
+    return "weak"
+
+
+def _compute_is_weak_class(auroc: float, n_candidates: int) -> bool:
+    return auroc < WEAK_FAMILY_AUROC_THRESHOLD and n_candidates >= 3
+
+
+def _compute_verdict(family_entries: list[FamilyPerformanceEntry]) -> str:
+    if len(family_entries) == 0:
+        return "insufficient_data"
+    for entry in family_entries:
+        if entry.is_weak_class and entry.panel_fraction < WEAK_FAMILY_PANEL_FLOOR:
+            return "weak_family_excluded"
+    return "all_families_represented"
+
+
+def validate_family_blindness_challenge(fbh: FamilyBlingnessChallenge) -> None:
+    if not fbh.fbh_id.startswith("FBH-"):
+        raise ValueError(f"fbh_id must start with 'FBH-': {fbh.fbh_id!r}")
+    if not fbh.pipeline_version:
+        raise ValueError("pipeline_version must be non-empty")
+    for entry in fbh.family_entries:
+        if not (-1.0 <= entry.auroc <= 1.0):
+            raise ValueError(f"auroc must be in [-1.0, 1.0]: {entry.auroc}")
+        if entry.n_candidates < 1:
+            raise ValueError(f"n_candidates must be >= 1: {entry.n_candidates}")
+        if entry.panel_count < 0:
+            raise ValueError(f"panel_count must be >= 0: {entry.panel_count}")
+        if entry.panel_count > entry.n_candidates:
+            raise ValueError(
+                f"panel_count ({entry.panel_count}) cannot exceed "
+                f"n_candidates ({entry.n_candidates})"
+            )
+    if fbh.n_families_total != len(fbh.family_entries):
+        raise ValueError(
+            f"n_families_total {fbh.n_families_total} != "
+            f"len(family_entries) {len(fbh.family_entries)}"
+        )
+    expected_n_weak = sum(1 for e in fbh.family_entries if e.is_weak_class)
+    if fbh.n_weak_families != expected_n_weak:
+        raise ValueError(
+            f"n_weak_families {fbh.n_weak_families} != computed {expected_n_weak}"
+        )
+    expected_excluded = [
+        e.family_name
+        for e in fbh.family_entries
+        if e.is_weak_class and e.panel_fraction < WEAK_FAMILY_PANEL_FLOOR
+    ]
+    if fbh.n_excluded_weak_families != len(expected_excluded):
+        raise ValueError(
+            f"n_excluded_weak_families {fbh.n_excluded_weak_families} != "
+            f"computed {len(expected_excluded)}"
+        )
+    if fbh.excluded_weak_family_names != expected_excluded:
+        raise ValueError("excluded_weak_family_names mismatch")
+    if fbh.verdict not in VALID_FBH_VERDICTS:
+        raise ValueError(f"verdict {fbh.verdict!r} not in VALID_FBH_VERDICTS")
+    if fbh.verdict == "weak_family_excluded" and fbh.n_excluded_weak_families < 1:
+        raise ValueError(
+            "verdict weak_family_excluded but n_excluded_weak_families < 1"
+        )
+    if fbh.verdict == "all_families_represented" and fbh.n_excluded_weak_families != 0:
+        raise ValueError(
+            "verdict all_families_represented but n_excluded_weak_families != 0"
+        )
+    if not fbh.dry_lab_only:
+        raise ValueError("dry_lab_only must be True")
+    if not fbh.limitations:
+        raise ValueError("limitations must be non-empty")
+    if not fbh.created_at:
+        raise ValueError("created_at must be non-empty")
+
+
+def build_family_blindness_challenge(
+    *,
+    fbh_id: str,
+    pipeline_version: str,
+    family_entry_dicts: list[dict],
+    limitations: list[str],
+    created_at: str,
+) -> FamilyBlingnessChallenge:
+    """Build a FamilyBlingnessChallenge.
+
+    family_entry_dicts: list of dicts with keys:
+        family_name (str), auroc (float),
+        n_candidates (int), panel_count (int)
+    """
+    family_entries = []
+    for d in family_entry_dicts:
+        auroc = float(d["auroc"])
+        n_candidates = int(d["n_candidates"])
+        panel_count = int(d["panel_count"])
+        panel_fraction = round(panel_count / n_candidates, 6) if n_candidates > 0 else 0.0
+        auroc_grade = _compute_auroc_grade(auroc)
+        is_weak_class = _compute_is_weak_class(auroc, n_candidates)
+        family_entries.append(
+            FamilyPerformanceEntry(
+                family_name=d["family_name"],
+                auroc=auroc,
+                n_candidates=n_candidates,
+                panel_count=panel_count,
+                panel_fraction=panel_fraction,
+                auroc_grade=auroc_grade,
+                is_weak_class=is_weak_class,
+            )
+        )
+    n_families_total = len(family_entries)
+    n_weak_families = sum(1 for e in family_entries if e.is_weak_class)
+    excluded_names = [
+        e.family_name
+        for e in family_entries
+        if e.is_weak_class and e.panel_fraction < WEAK_FAMILY_PANEL_FLOOR
+    ]
+    n_excluded_weak_families = len(excluded_names)
+    verdict = _compute_verdict(family_entries)
+    fbh = FamilyBlingnessChallenge(
+        fbh_id=fbh_id,
+        pipeline_version=pipeline_version,
+        family_entries=family_entries,
+        n_families_total=n_families_total,
+        n_weak_families=n_weak_families,
+        n_excluded_weak_families=n_excluded_weak_families,
+        excluded_weak_family_names=excluded_names,
+        verdict=verdict,
+        dry_lab_only=True,
+        limitations=limitations,
+        created_at=created_at,
+    )
+    validate_family_blindness_challenge(fbh)
+    return fbh
+
+
+def format_family_blindness_challenge(fbh: FamilyBlingnessChallenge) -> str:
+    lines = [
+        f"Family Blindness Challenge — {fbh.fbh_id}",
+        f"Pipeline: {fbh.pipeline_version}",
+        f"Verdict: {fbh.verdict}",
+        f"Families: {fbh.n_families_total} total, {fbh.n_weak_families} weak, "
+        f"{fbh.n_excluded_weak_families} excluded",
+    ]
+    if fbh.excluded_weak_family_names:
+        lines.append(
+            "Excluded weak families: "
+            f"{', '.join(fbh.excluded_weak_family_names)}"
+        )
+    if fbh.family_entries:
+        lines.append("Per-family performance:")
+        for entry in fbh.family_entries:
+            lines.append(
+                f"  {entry.family_name}: AUROC={entry.auroc:.3f} "
+                f"(grade={entry.auroc_grade}), "
+                f"panel={entry.panel_count}/{entry.n_candidates} "
+                f"({entry.panel_fraction:.1%}), weak={entry.is_weak_class}"
+            )
+    lines.append(f"Created: {fbh.created_at}")
+    lines.append(f"Limitations: {'; '.join(fbh.limitations)}")
+    lines.append(f"dry_lab_only: {fbh.dry_lab_only}")
+    return "\n".join(lines)