From 036029e310a03cc7b3a8795ba2b6b3bbe0ad9ddd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=BCneyt=20=C3=96zt=C3=BCrk?= Date: Mon, 29 Jun 2026 13:18:16 +0300 Subject: [PATCH] fix: verify_eval_log works against current Inspect logs (0.3.1) Inspect 0.3.x no longer records eval.dataset.sha, and results.scores[].name carries the scorer name, not the metric -- so the from-log verification path raised MalformedLogError ("missing fields: ['dataset_hash']") on every modern log. A real user calling verify_eval_log would have hit this. - verify_eval_log now accepts dataset_hash= and metric= overrides; both default to whatever the log carries, so logs that include those fields are unaffected. - the MalformedLogError now names exactly which fields to supply. - +2 tests: override round-trip PASS, and helpful error when omitted. - examples/offline-mockllm/: a fully offline, deterministic showcase running a real Inspect eval with mockllm (no API key/network), PASS + a TAMPERED model-swap via verify_live. - bump 0.3.0 -> 0.3.1, CHANGELOG. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 16 ++++ examples/offline-mockllm/.gitignore | 5 ++ examples/offline-mockllm/README.md | 90 +++++++++++++++++++ examples/offline-mockllm/requirements.txt | 2 + examples/offline-mockllm/run_showcase.py | 103 ++++++++++++++++++++++ falsify_inspect/__init__.py | 2 +- falsify_inspect/core.py | 26 ++++-- pyproject.toml | 2 +- tests/test_core.py | 56 ++++++++++++ 9 files changed, 291 insertions(+), 11 deletions(-) create mode 100644 examples/offline-mockllm/.gitignore create mode 100644 examples/offline-mockllm/README.md create mode 100644 examples/offline-mockllm/requirements.txt create mode 100644 examples/offline-mockllm/run_showcase.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bc9e15..08af96d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,22 @@ All notable changes to `falsify-inspect` are documented here. +## [0.3.1] — 2026-06-29 + +### Fixed +- **`verify_eval_log` now works against current Inspect logs.** Inspect 0.3.x no + longer records `eval.dataset.sha`, and `results.scores[].name` carries the + *scorer* name, not the metric. The from-log path raised `MalformedLogError` + (`missing fields: ['dataset_hash']`) on every modern log. `verify_eval_log` + now accepts `dataset_hash=` and `metric=` overrides (both default to whatever + the log carries, so older logs are unaffected), and the error message names + exactly which fields to supply. Round-trip with `preregister` is verified. + +### Added +- `examples/offline-mockllm/` — a fully offline, deterministic showcase that + runs a real Inspect eval with `mockllm` (no API key, no network), pre-registers + the claim, and demonstrates PASS plus a TAMPERED model-swap via `verify_live`. + ## [0.3.0] — 2026-06-17 ### Changed (BREAKING) diff --git a/examples/offline-mockllm/.gitignore b/examples/offline-mockllm/.gitignore new file mode 100644 index 0000000..fd1a06d --- /dev/null +++ b/examples/offline-mockllm/.gitignore @@ -0,0 +1,5 @@ +logs/ +eval_log.json +claim_A.prml.yaml +claim_B.prml.yaml +__pycache__/ diff --git a/examples/offline-mockllm/README.md b/examples/offline-mockllm/README.md new file mode 100644 index 0000000..d7fe6a3 --- /dev/null +++ b/examples/offline-mockllm/README.md @@ -0,0 +1,90 @@ +# Inspect AI × PRML — offline, deterministic showcase + +[Inspect AI](https://github.com/UKGovernmentBEIS/inspect_ai) (the UK AI Safety +Institute's eval framework) records what an evaluation *produced*. PRML records +what was *promised before it ran* — the metric, the threshold, the dataset, and +the **model identity** — sealed to a SHA-256. Run the eval on a different model +than you pre-registered, and the seal breaks: the claim reads **TAMPERED**. + +This example runs a **real Inspect evaluation with the `mockllm` model**, so it +reproduces byte-for-byte on any machine with **no API key and no network**. It +uses `falsify-inspect`'s native `verify_live` path — the same one the Inspect +hook calls on `on_task_end`. + +## Why model identity matters for a leaderboard + +A benchmark score is only meaningful next to *which model produced it*. The +quiet gaming move is to publish a strong score and attribute it to a flagship +model, when the run was actually a different (cheaper, or differently-configured) +one. Pre-registering the claim's identity makes that substitution detectable +offline, by anyone, from the manifest alone. + +## Run it + +```bash +pip install -r requirements.txt +python run_showcase.py +``` + +## Verified output + +``` +================================================================== +A. Honest run -- pre-registered {accuracy >= 0.75, model mockllm} +================================================================== +locked BEFORE run sha256=63e5ecb745b3b67f56782a166eb3249e6d0ea11d548e05f1325cc4541c06869a + bar: accuracy >= 0.75 model=mockllm/model dataset=584e4f442288b464... +ran Inspect (mockllm) observed accuracy = 0.8000 +verify_live -> PASS + +================================================================== +B. Swapped model -- sealed for claude-3.5, but the run was mockllm +================================================================== +locked BEFORE run sha256=18452dadcc6bcc76d16fe361e72079a73f03d4b7af00d0573ef27b4de4e046bf (model: anthropic/claude-3.5-sonnet) +ran Inspect (mockllm) observed accuracy = 0.8000 +...the published run was actually on mockllm, not the sealed model... +verify_live -> TAMPERED (identity does not match the sealed claim) + +================================================================== +RESULT +================================================================== +A honest run : PASS (expected PASS) +B swapped model : TAMPERED (expected TAMPERED) +``` + +Both digests (`63e5ec…`, `18452d…`) reproduce across runs — the eval is +deterministic because `mockllm` returns fixed outputs (8 of 10 correct → 0.80). + +## The integration in three lines + +```python +from falsify_inspect import preregister, verify_live + +h, _ = preregister(metric="accuracy", threshold=0.75, threshold_direction=">=", + dataset="qa-toy-10", dataset_hash=DATASET_HASH, + model_version="mockllm/model", seed=42, + output_path="claim.prml.yaml") # BEFORE the run +observed = run_your_inspect_eval() # Inspect, unchanged +verdict = verify_live(manifest_path="claim.prml.yaml", observed_value=observed, + live_model="mockllm/model") # AFTER -> PASS / FAIL / TAMPERED +``` + +In production the same check runs automatically via the `falsify_prml` Inspect +hook (`on_task_end`) when `FALSIFY_PRML` points at the committed manifest — no +glue code, the verdict lands in the eval log. + +## What this does and does not prove + +- **Does:** prove the eval's identity (metric, comparator, threshold, dataset, + model, seed) was fixed before the result was known. Swapping the model — or + any sealed field — after the fact is detectable. +- **Does not:** prove the score is correct, or that the producer ran the eval at + all, or published every claim. PRML proves the bar was *locked*, never the + *result*. + +--- + +*Part of [`studio-11-co/falsify-inspect`](https://github.com/studio-11-co/falsify-inspect). +Spec: [spec.falsify.dev/v0.1](https://spec.falsify.dev/v0.1) · CC BY 4.0; code MIT. +Inspect AI is © UK AI Safety Institute, MIT — this example depends on it and is +not affiliated with or endorsed by AISI.* diff --git a/examples/offline-mockllm/requirements.txt b/examples/offline-mockllm/requirements.txt new file mode 100644 index 0000000..1f647dd --- /dev/null +++ b/examples/offline-mockllm/requirements.txt @@ -0,0 +1,2 @@ +inspect_ai>=0.3 +falsify-inspect>=0.3 diff --git a/examples/offline-mockllm/run_showcase.py b/examples/offline-mockllm/run_showcase.py new file mode 100644 index 0000000..5d2db36 --- /dev/null +++ b/examples/offline-mockllm/run_showcase.py @@ -0,0 +1,103 @@ +"""Fully offline, deterministic Inspect AI x PRML showcase -- no API key, no network. + +Runs a *real* Inspect AI evaluation with the `mockllm` model (so it reproduces +byte-for-byte anywhere), pre-registers the claim's identity with PRML *before* +the run, then verifies the live run against the committed manifest with the +adapter's native `verify_live` path. + + A. Honest run -- pre-register {accuracy >= 0.75, model mockllm}, run, observe + 0.80, verify -> PASS. + B. Swapped model -- pre-register the SAME claim for a *different* model, run on + mockllm, verify -> TAMPERED (the run's identity does not + match what was sealed; the hash breaks). + +Run: python run_showcase.py +""" +from __future__ import annotations + +import hashlib +import json +import sys +from pathlib import Path + +from inspect_ai import Task, eval as inspect_eval +from inspect_ai.dataset import Sample +from inspect_ai.model import ModelOutput, get_model +from inspect_ai.scorer import match +from inspect_ai.solver import generate + +from falsify_inspect import preregister, verify_live + +HERE = Path(__file__).resolve().parent +PRE_REGISTERED = "2026-06-29T12:00:00Z" +MODEL = "mockllm/model" +DATASET_ID = "qa-toy-10" + +SAMPLES = [Sample(input=f"Q{i}: is the sky blue?", target="yes") for i in range(10)] +OUTPUTS = [ModelOutput.from_content(MODEL, "yes")] * 8 + [ModelOutput.from_content(MODEL, "no")] * 2 +DATASET_HASH = hashlib.sha256( + json.dumps([{"input": s.input, "target": s.target} for s in SAMPLES], sort_keys=True).encode() +).hexdigest() + + +def run_eval_observed() -> float: + """Run the real Inspect eval with mockllm; return observed accuracy.""" + task = Task(dataset=SAMPLES, solver=generate(), scorer=match()) + logs = inspect_eval(task, model=get_model(MODEL, custom_outputs=OUTPUTS), + display="none", log_dir=str(HERE / "logs")) + metrics = logs[0].results.scores[0].metrics + return float(metrics["accuracy"].value) + + +def lock(model_version: str, threshold: float, path: Path) -> str: + h, _ = preregister( + metric="accuracy", threshold=threshold, threshold_direction=">=", + dataset=DATASET_ID, dataset_hash=DATASET_HASH, model_version=model_version, + seed=42, pre_registered=PRE_REGISTERED, inspect_task="qa_toy", + inspect_scorer="match", output_path=str(path), + ) + return h + + +def banner(t: str) -> None: + print(f"\n{'=' * 66}\n{t}\n{'=' * 66}") + + +def main() -> int: + observed = run_eval_observed() + + # ---- A: honest run, pre-registered for the model actually used ---------- + banner("A. Honest run -- pre-registered {accuracy >= 0.75, model mockllm}") + claim_a = HERE / "claim_A.prml.yaml" + h_a = lock(MODEL, 0.75, claim_a) + print(f"locked BEFORE run sha256={h_a}") + print(f" bar: accuracy >= 0.75 model={MODEL} dataset={DATASET_HASH[:16]}...") + print(f"ran Inspect (mockllm) observed accuracy = {observed:.4f}") + res_a = verify_live(manifest_path=str(claim_a), observed_value=observed, + live_model=MODEL, live_task="qa_toy") + print(f"verify_live -> {res_a['status']}") + + # ---- B: same claim sealed for a DIFFERENT model than the run ------------ + banner("B. Swapped model -- sealed for claude-3.5, but the run was mockllm") + claim_b = HERE / "claim_B.prml.yaml" + h_b = lock("anthropic/claude-3.5-sonnet", 0.75, claim_b) + print(f"locked BEFORE run sha256={h_b} (model: anthropic/claude-3.5-sonnet)") + print(f"ran Inspect (mockllm) observed accuracy = {observed:.4f}") + print("...the published run was actually on mockllm, not the sealed model...") + res_b = verify_live(manifest_path=str(claim_b), observed_value=observed, + live_model=MODEL, live_task="qa_toy") + print(f"verify_live -> {res_b['status']} (identity does not match the sealed claim)") + + # ---- Result ------------------------------------------------------------- + ok = res_a["status"] == "PASS" and res_b["status"] == "TAMPERED" + banner("RESULT") + print(f"A honest run : {res_a['status']} (expected PASS)") + print(f"B swapped model : {res_b['status']} (expected TAMPERED)") + print("\nPRML tied a real Inspect eval to its pre-registered identity -- " + "a swapped model breaks the seal, checkable offline by anyone." + if ok else "\nUNEXPECTED -- see above.") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/falsify_inspect/__init__.py b/falsify_inspect/__init__.py index fc14c56..da8ebbe 100644 --- a/falsify_inspect/__init__.py +++ b/falsify_inspect/__init__.py @@ -50,7 +50,7 @@ # importable without it. Inspect loads the hook via the `inspect_ai` entry # point (falsify_inspect._registry). -__version__ = "0.3.0" +__version__ = "0.3.1" __all__ = [ "preregister", diff --git a/falsify_inspect/core.py b/falsify_inspect/core.py index 1200a2e..3f521c6 100644 --- a/falsify_inspect/core.py +++ b/falsify_inspect/core.py @@ -311,6 +311,8 @@ def verify_eval_log( sample_size: int | None = None, seed: int | None = None, inspect_scorer: str | None = None, + dataset_hash: str | None = None, + metric: str | None = None, ) -> dict[str, Any]: """Reconstruct the pre-registered manifest from log metadata + caller's threshold/seed/etc., then verify the hash matches ``expected_hash``. @@ -321,6 +323,12 @@ def verify_eval_log( no claim_id — the same ``claim_id`` used at lock, defaulting to ``"{dataset}:{metric}"`` as in :func:`preregister`). + Current Inspect logs do not record ``eval.dataset.sha`` (and the scorer name, + not the metric, is what lands in ``results.scores[].name``). Pass + ``dataset_hash=`` and ``metric=`` to supply what the log omits; both default + to whatever the log does carry, so logs that include those fields keep + working unchanged. + Returns a dict with keys: ok, hash_match, threshold_satisfied, observed_value, expected_hash, actual_hash, manifest @@ -329,14 +337,12 @@ def verify_eval_log( MalformedLogError if the log is structurally malformed. """ extracted = extract_manifest_from_log(log_path) - if not extracted.get("metric"): + metric = metric if metric is not None else extracted.get("metric") + if not metric: raise MalformedLogError( - f"could not extract primary metric from log {log_path}: " - "the log is structurally invalid (no `results.scores[].name`). " - "This is not a tamper — it means the log shape is wrong." + f"could not determine the metric for log {log_path}: the log has no " + "`results.scores[].name` and no `metric=` override was supplied." ) - - metric = extracted["metric"] dataset_id = dataset if dataset is not None else extracted["dataset_id"] producer_id = ( model_version if model_version is not None else extracted["producer_id"] @@ -346,7 +352,7 @@ def verify_eval_log( "comparator": threshold_direction, "threshold": threshold, "dataset_id": dataset_id, - "dataset_hash": extracted["dataset_hash"], + "dataset_hash": dataset_hash if dataset_hash is not None else extracted["dataset_hash"], "producer_id": producer_id, "sample_size": sample_size if sample_size is not None else extracted["sample_size"], "seed": seed if seed is not None else extracted["seed"], @@ -359,8 +365,10 @@ def verify_eval_log( missing = [k for k in required if fields[k] is None] if missing: raise MalformedLogError( - f"missing fields after extraction: {missing}; supply via kwargs. " - "This is a structurally invalid log, not a tamper." + "the log does not carry these fields and no override was supplied: " + f"{missing}. Current Inspect logs omit dataset.sha, so pass " + "dataset_hash= (and dataset=/seed=/model_version=/metric= as needed). " + "This is a missing field, not a tamper." ) manifest = InspectManifest( diff --git a/pyproject.toml b/pyproject.toml index 978e614..15529ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "falsify-inspect" -version = "0.3.0" +version = "0.3.1" description = "PRML v0.1 pre-registration integration for Inspect AI eval logs" readme = "README.md" requires-python = ">=3.11" diff --git a/tests/test_core.py b/tests/test_core.py index 1039657..c644330 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -12,6 +12,7 @@ preregister, verify_eval_log, ) +from falsify_inspect.core import MalformedLogError # A valid PRML dataset.hash: 64 lowercase hex chars. HEX = "a" * 64 @@ -236,3 +237,58 @@ def test_verify_eval_log_tamper(tmp_path: Path): ) assert r["hash_match"] is False assert r["ok"] is False + + +def _log_no_sha(tmp_path: Path, *, value: float, model="m", dataset="d", seed=1, + epochs=10, task="t", scorer="match") -> Path: + """A log shaped like CURRENT Inspect (0.3.x): no dataset.sha, and the scorer + name -- not the metric -- in results.scores[].name.""" + log = { + "eval": { + "task": task, + "model": model, + "dataset": {"name": dataset}, # note: no "sha" + "config": {"epochs": epochs, "seed": seed}, + }, + "results": {"scores": [{"name": scorer, "metrics": {"accuracy": {"value": value}}}]}, + } + p = tmp_path / "eval_nosha.json" + p.write_text(json.dumps(log)) + return p + + +def test_verify_eval_log_dataset_hash_and_metric_override(tmp_path: Path): + """Current Inspect logs omit dataset.sha and carry the scorer name, not the + metric. Supplying dataset_hash= and metric= reconstructs the locked manifest.""" + h, _ = preregister( + metric="accuracy", threshold=0.75, threshold_direction=">=", + dataset="d", dataset_hash=HEX, model_version="m", sample_size=10, seed=1, + pre_registered="2026-01-01T00:00:00Z", inspect_task="t", + ) + p = _log_no_sha(tmp_path, value=0.80, scorer="match") + r = verify_eval_log( + p, expected_hash=h, threshold=0.75, threshold_direction=">=", + pre_registered="2026-01-01T00:00:00Z", + dataset="d", dataset_hash=HEX, metric="accuracy", model_version="m", + sample_size=10, seed=1, + ) + assert r["hash_match"] is True + assert r["threshold_satisfied"] is True + assert r["ok"] is True + + +def test_verify_eval_log_missing_dataset_hash_errors(tmp_path: Path): + """No dataset.sha in the log and no dataset_hash= override -> helpful error, + not a crash and not a false TAMPERED.""" + h, _ = preregister( + metric="accuracy", threshold=0.75, threshold_direction=">=", + dataset="d", dataset_hash=HEX, model_version="m", sample_size=10, seed=1, + pre_registered="2026-01-01T00:00:00Z", inspect_task="t", + ) + p = _log_no_sha(tmp_path, value=0.80) + with pytest.raises(MalformedLogError, match="dataset_hash"): + verify_eval_log( + p, expected_hash=h, threshold=0.75, threshold_direction=">=", + pre_registered="2026-01-01T00:00:00Z", + dataset="d", metric="accuracy", model_version="m", seed=1, + )