Skip to content

Commit b498b3b

Browse files
author
OpenCode
committed
feat: Phase AB AB1 claim strength downgrade -- CSD- schema audits every claim downgrade with proof_ladder_steps_dropped and is_retracted; prevents silent claim drift
1 parent 4de1f2a commit b498b3b

4 files changed

Lines changed: 650 additions & 1 deletion

File tree

‎docs/research/NEXT_100_PR_MAP.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -329,3 +329,11 @@ Machine-verifiable proof that every pipeline run carries the fields needed for e
329329
| AA3 | Add configuration fingerprint schema (CFP-) (complete). | Structured record of all config file paths and their SHA256/etc. hashes for a pipeline run; verdict all_configs_hashed/some_configs_unhashed/no_configs_recorded; complements RMC- config_hash field with full per-file audit trail. | C |
330330
| AA4 | Add synthetic boundary warning record schema (SBW-) (complete). | Machine-checkable label enforcement for synthetic lab results (mic_simulation/structure_prediction/docking/etc.); verdict labelled/partially_labelled/unlabelled; cannot_support_biological_claim=True invariant enforced; prevents synthetic data from being treated as experimental validation. | C |
331331
| AA5 | Add Phase AA reproducibility gate (AARG-). | Top-level gate asserting RMC+DCR+CFP+SBW all present; verdict reproducibility_verified/partial/not_established; closes Phase AA; no run is reproducibility-certified until this gate passes. | C |
332+
333+
## Phase AB — Claim integrity and external handoff
334+
335+
Machine-verifiable audit trail for claim downgrades, expert decisions, and external collaboration handoffs.
336+
337+
| PR | Task | Why it matters | Review class |
338+
|---:|---|---|---|
339+
| AB1 | Add claim strength downgrade record schema (CSD-). | Auditable trail for every claim downgrade; proof_ladder_steps_dropped auto-computed; is_retracted invariant; trigger_type controlled vocabulary (benchmark/reviewer/cheap_enemy/etc.); prevents silent claim drift upward after challenges. | B/C |
Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
"""CSD- Claim Strength Downgrade Record schema.
2+
3+
Auditable trail for every claim downgrade triggered by benchmark results,
4+
reviewer challenges, cheap enemy refutations, data quality issues, scope
5+
creep detection, or calibration failures. Prevents silent claim drift upward
6+
after challenges.
7+
"""
8+
9+
from __future__ import annotations
10+
11+
from dataclasses import dataclass
12+
13+
VALID_CSD_CLAIM_CLASSES: frozenset[str] = frozenset({
14+
"novelty",
15+
"activity",
16+
"selectivity",
17+
"safety",
18+
"reproducibility",
19+
"calibration_improvement",
20+
"cheap_baseline_outperformed",
21+
})
22+
23+
VALID_CSD_TRIGGER_TYPES: frozenset[str] = frozenset({
24+
"benchmark_result",
25+
"reviewer_challenge",
26+
"cheap_enemy_refutation",
27+
"data_quality_issue",
28+
"scope_creep_detected",
29+
"calibration_failure",
30+
})
31+
32+
VALID_CSD_DOWNGRADE_LEVELS: frozenset[str] = frozenset({
33+
"minor",
34+
"major",
35+
"retracted",
36+
})
37+
38+
PROOF_LADDER_LEVELS: tuple[str, ...] = (
39+
"speculative",
40+
"plausible_candidate",
41+
"multi_signal_candidate_evidence",
42+
"expert_reviewed_assay_proposal",
43+
"confirmed_in_vitro_hit",
44+
"replicated_independent_hit",
45+
"in_vivo_validated",
46+
)
47+
48+
49+
@dataclass
50+
class ClaimStrengthDowngrade:
51+
csd_id: str
52+
pipeline_version: str
53+
artifact_id: str
54+
claim_class: str
55+
trigger_type: str
56+
original_claim_text: str
57+
downgraded_claim_text: str
58+
original_proof_ladder_level: str
59+
downgraded_proof_ladder_level: str
60+
downgrade_level: str
61+
proof_ladder_steps_dropped: int
62+
trigger_evidence_ref: str
63+
is_retracted: bool
64+
dry_lab_only: bool = True
65+
limitations: list[str] | None = None
66+
created_at: str = ""
67+
68+
69+
def build_claim_strength_downgrade(
70+
*,
71+
csd_id: str,
72+
pipeline_version: str,
73+
artifact_id: str,
74+
claim_class: str,
75+
trigger_type: str,
76+
original_claim_text: str,
77+
downgraded_claim_text: str,
78+
original_proof_ladder_level: str,
79+
downgraded_proof_ladder_level: str,
80+
downgrade_level: str,
81+
trigger_evidence_ref: str,
82+
limitations: list[str],
83+
created_at: str,
84+
) -> ClaimStrengthDowngrade:
85+
is_retracted = downgrade_level == "retracted"
86+
if is_retracted:
87+
proof_ladder_steps_dropped = 0
88+
else:
89+
orig_idx = PROOF_LADDER_LEVELS.index(original_proof_ladder_level)
90+
down_idx = PROOF_LADDER_LEVELS.index(downgraded_proof_ladder_level)
91+
proof_ladder_steps_dropped = orig_idx - down_idx
92+
93+
csd = ClaimStrengthDowngrade(
94+
csd_id=csd_id,
95+
pipeline_version=pipeline_version,
96+
artifact_id=artifact_id,
97+
claim_class=claim_class,
98+
trigger_type=trigger_type,
99+
original_claim_text=original_claim_text,
100+
downgraded_claim_text=downgraded_claim_text,
101+
original_proof_ladder_level=original_proof_ladder_level,
102+
downgraded_proof_ladder_level=downgraded_proof_ladder_level,
103+
downgrade_level=downgrade_level,
104+
proof_ladder_steps_dropped=proof_ladder_steps_dropped,
105+
trigger_evidence_ref=trigger_evidence_ref,
106+
is_retracted=is_retracted,
107+
dry_lab_only=True,
108+
limitations=limitations,
109+
created_at=created_at,
110+
)
111+
validate_claim_strength_downgrade(csd)
112+
return csd
113+
114+
115+
def validate_claim_strength_downgrade(
116+
csd: ClaimStrengthDowngrade,
117+
) -> None:
118+
if not csd.csd_id.startswith("CSD-"):
119+
raise ValueError(
120+
f"csd_id must start with 'CSD-': {csd.csd_id!r}"
121+
)
122+
if not csd.pipeline_version:
123+
raise ValueError("pipeline_version must be non-empty")
124+
if not csd.artifact_id:
125+
raise ValueError("artifact_id must be non-empty")
126+
if not csd.original_claim_text:
127+
raise ValueError("original_claim_text must be non-empty")
128+
if not csd.downgraded_claim_text:
129+
raise ValueError("downgraded_claim_text must be non-empty")
130+
if csd.claim_class not in VALID_CSD_CLAIM_CLASSES:
131+
raise ValueError(
132+
f"claim_class {csd.claim_class!r} not in VALID_CSD_CLAIM_CLASSES"
133+
)
134+
if csd.trigger_type not in VALID_CSD_TRIGGER_TYPES:
135+
raise ValueError(
136+
f"trigger_type {csd.trigger_type!r} not in VALID_CSD_TRIGGER_TYPES"
137+
)
138+
if csd.downgrade_level not in VALID_CSD_DOWNGRADE_LEVELS:
139+
raise ValueError(
140+
f"downgrade_level {csd.downgrade_level!r} not in VALID_CSD_DOWNGRADE_LEVELS"
141+
)
142+
if csd.original_proof_ladder_level not in PROOF_LADDER_LEVELS:
143+
raise ValueError(
144+
f"original_proof_ladder_level {csd.original_proof_ladder_level!r} "
145+
f"not in PROOF_LADDER_LEVELS"
146+
)
147+
if csd.downgraded_proof_ladder_level not in PROOF_LADDER_LEVELS:
148+
raise ValueError(
149+
f"downgraded_proof_ladder_level {csd.downgraded_proof_ladder_level!r} "
150+
f"not in PROOF_LADDER_LEVELS"
151+
)
152+
153+
if csd.downgrade_level != "retracted":
154+
orig_idx = PROOF_LADDER_LEVELS.index(csd.original_proof_ladder_level)
155+
down_idx = PROOF_LADDER_LEVELS.index(csd.downgraded_proof_ladder_level)
156+
if down_idx > orig_idx:
157+
raise ValueError(
158+
f"downgraded_proof_ladder_level {csd.downgraded_proof_ladder_level!r} "
159+
f"(index {down_idx}) > original_proof_ladder_level "
160+
f"{csd.original_proof_ladder_level!r} (index {orig_idx}) — "
161+
f"upgrade not allowed"
162+
)
163+
else:
164+
if csd.downgraded_claim_text != "RETRACTED":
165+
raise ValueError(
166+
f"downgraded_claim_text must be 'RETRACTED' for retracted: "
167+
f"{csd.downgraded_claim_text!r}"
168+
)
169+
170+
if csd.downgrade_level != "retracted":
171+
expected_steps = (
172+
PROOF_LADDER_LEVELS.index(csd.original_proof_ladder_level)
173+
- PROOF_LADDER_LEVELS.index(csd.downgraded_proof_ladder_level)
174+
)
175+
else:
176+
expected_steps = 0
177+
if csd.proof_ladder_steps_dropped != expected_steps:
178+
raise ValueError(
179+
f"proof_ladder_steps_dropped {csd.proof_ladder_steps_dropped} != "
180+
f"{expected_steps} (expected)"
181+
)
182+
183+
expected_retracted = csd.downgrade_level == "retracted"
184+
if csd.is_retracted != expected_retracted:
185+
raise ValueError(
186+
f"is_retracted {csd.is_retracted} != {expected_retracted} "
187+
f"expected for downgrade_level={csd.downgrade_level!r}"
188+
)
189+
190+
if csd.dry_lab_only is not True:
191+
raise ValueError("dry_lab_only must be True")
192+
if not csd.trigger_evidence_ref:
193+
raise ValueError("trigger_evidence_ref must be non-empty")
194+
if not csd.limitations:
195+
raise ValueError("limitations must be non-empty")
196+
if not csd.created_at:
197+
raise ValueError("created_at must be non-empty")
198+
199+
200+
def format_claim_strength_downgrade(
201+
csd: ClaimStrengthDowngrade,
202+
) -> str:
203+
lines = [
204+
f"Claim Strength Downgrade — {csd.csd_id}",
205+
f"Pipeline: {csd.pipeline_version}",
206+
f"Artifact: {csd.artifact_id}",
207+
f"Claim class: {csd.claim_class}",
208+
f"Trigger: {csd.trigger_type}",
209+
f"Downgrade level: {csd.downgrade_level}",
210+
f"Original proof ladder level: {csd.original_proof_ladder_level}",
211+
f"Downgraded proof ladder level: {csd.downgraded_proof_ladder_level}",
212+
f"Proof ladder steps dropped: {csd.proof_ladder_steps_dropped}",
213+
f"Original claim: {csd.original_claim_text}",
214+
f"Downgraded claim: {csd.downgraded_claim_text}",
215+
f"Trigger evidence ref: {csd.trigger_evidence_ref}",
216+
f"Retracted: {csd.is_retracted}",
217+
f"Dry lab only: {csd.dry_lab_only}",
218+
]
219+
if csd.limitations:
220+
lines.append(f"Limitations: {'; '.join(csd.limitations)}")
221+
lines.append(f"Created: {csd.created_at}")
222+
return "\n".join(lines)

0 commit comments

Comments
 (0)