From cb7e50e8d0dc55e08918ca0671492091d1ba9c82 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Sun, 16 Aug 2026 23:07:59 -0400 Subject: [PATCH 01/70] Generate fail-closed Comparator workspaces --- .github/workflows/build-and-docs.yml | 19 + .gitignore | 1 + comparator/README.md | 79 ++ .../arithmetic_sum_s_conjecture_1_1.toml | 4 + comparator/problems/erdos_1038_part_i.toml | 10 + .../problems/margulis_conjecture_1_1.toml | 7 + comparator/templates/WorkspaceTest.lean | 37 + comparator/tools.toml | 7 + lakefile.toml | 7 + scripts/comparator_facts.lean | 140 +++ scripts/make_comparator_workspace.py | 841 ++++++++++++++++++ scripts/test_make_comparator_workspace.py | 282 ++++++ 12 files changed, 1434 insertions(+) create mode 100644 comparator/README.md create mode 100644 comparator/problems/arithmetic_sum_s_conjecture_1_1.toml create mode 100644 comparator/problems/erdos_1038_part_i.toml create mode 100644 comparator/problems/margulis_conjecture_1_1.toml create mode 100644 comparator/templates/WorkspaceTest.lean create mode 100644 comparator/tools.toml create mode 100644 scripts/comparator_facts.lean create mode 100644 scripts/make_comparator_workspace.py create mode 100644 scripts/test_make_comparator_workspace.py diff --git a/.github/workflows/build-and-docs.yml b/.github/workflows/build-and-docs.yml index a0aa4b00cf..4b1d9b0a7a 100644 --- a/.github/workflows/build-and-docs.yml +++ b/.github/workflows/build-and-docs.yml @@ -155,6 +155,25 @@ jobs: lake --wfail build rm -f FormalConjectures/All.lean + # The elaborator-to-generator boundary, exercised on the oleans the + # build above produced: one plain theorem, a Prop answer slot, a + # non-Prop answer slot, a forall-conclusion (whose binder must not be + # applied), and explicit parameters (which must be). The comparator run + # itself needs landrun and stays in a separate Linux job. + - name: Comparator generation smoke test + if: steps.mode.outputs.website_only != 'true' + run: | + lake build comparator_facts + for d in exists_hadamard_zero erdos_940.variants.large_integers \ + erdos_1038.parts.i erdos_100.variants.strong \ + KotherConjecture.variants.le_KotherRadical; do + python3 scripts/make_comparator_workspace.py "$d" --out .comparator + done + grep -q "large_integers_answer : Prop" .comparator/erdos_940_variants_large_integers/Challenge.lean + grep -q "i_answer : ENNReal" .comparator/erdos_1038_parts_i/Challenge.lean + grep -q "Submission.erdos_100.variants.strong$" .comparator/erdos_100_variants_strong/Solution.lean + grep -q "le_KotherRadical hI" .comparator/KotherConjecture_variants_le_KotherRadical/Solution.lean + - name: Build literate source pages if: steps.mode.outputs.website_only != 'true' && steps.mode.outputs.site == 'true' run: | diff --git a/.gitignore b/.gitignore index a62e89ebd8..55ea0833b8 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ FormalConjectures/All.lean # Python bytecode from the scripts in `scripts/`. __pycache__/ +.comparator/ diff --git a/comparator/README.md b/comparator/README.md new file mode 100644 index 0000000000..85c253d11e --- /dev/null +++ b/comparator/README.md @@ -0,0 +1,79 @@ +# Comparator workspace adapter + +This directory contains a thin adapter from Formal Conjectures to +[`leanprover/lean-eval`](https://github.com/leanprover/lean-eval) and +[`leanprover/comparator`](https://github.com/leanprover/comparator). It does not +implement another evaluator. + +## How it works + +1. `scripts/comparator_facts.lean` asks Lean for the selected declaration's + source range, binders, and `answer(sorry)` slot types. +2. `scripts/make_comparator_workspace.py` creates one pinned workspace. +3. The generated project builds the challenge and submission. +4. `lake test` runs Comparator against `config.json`. + +The workspace contains: + +- `Challenge.lean`, with the trusted statement and proof hole; +- `Submission.lean` and `Submission/`, where a solver works; +- `Solution.lean`, which connects the submission to the trusted statement; +- `config.json`, with theorem targets, definition targets, and permitted axioms; +- `holes.json`, with the exact extracted declaration blocks; +- pinned Lean, Mathlib, Formal Conjectures, Comparator, and helper-tool versions. + +`Solution.lean` is fixed. It fails to build if the submission changes the +statement. Comparator also rejects `sorryAx` because it is not in the permitted +axiom list. + +## Generate one workspace + +```bash +python3 scripts/make_comparator_workspace.py erdos_940.variants.large_integers +``` + +Use `--out` to choose the parent directory. The generator refuses to overwrite +an existing workspace. It writes into a temporary directory and renames the +complete result into place. + +The generator also stops when the selected source differs from the pinned +upstream revision. This prevents a workspace from combining a working-tree +statement with an older imported context. + +## Supported inputs + +- theorem proofs; +- definition answers represented by `answer(sorry)`; +- helper modules under `Submission/`. + +Plain-statement disproofs remain out of scope until Comparator provides an +upstream interface for them. + +## Problem manifests + +Most declarations need no manifest. Add one TOML file under `problems/` only +when the source cannot select the declaration by itself. + +| Field | Meaning | +|---|---| +| `id` | Workspace name. It must match the TOML filename. | +| `declaration` | Lean declaration name. | +| `module` | Source file when the declaration name is ambiguous. | +| `answer_type` | Explicit override when slot types cannot be matched safely. | +| `source` | Optional source link for the generated README. | +| `notes` | Optional reviewer note for the generated README. | + +Run the manifest check after moving or renaming a declaration: + +```bash +python3 scripts/make_comparator_workspace.py --validate +``` + +## Tool pins + +`tools.toml` records the external tool revisions. Generated workspaces pin +Mathlib from `lake-manifest.json` and Formal Conjectures to the current upstream +revision. The workspace build fetches these dependencies; workspace generation +itself does not run Comparator. + +Issue #4930 tracks the upstream integration and execution-service decisions. diff --git a/comparator/problems/arithmetic_sum_s_conjecture_1_1.toml b/comparator/problems/arithmetic_sum_s_conjecture_1_1.toml new file mode 100644 index 0000000000..cf71b6e68b --- /dev/null +++ b/comparator/problems/arithmetic_sum_s_conjecture_1_1.toml @@ -0,0 +1,4 @@ +id = "arithmetic_sum_s_conjecture_1_1" +declaration = "conjecture_1_1" +module = "FormalConjectures/Arxiv/2501.03234/ArithmeticSumS.lean" +source = "https://arxiv.org/abs/2501.03234" diff --git a/comparator/problems/erdos_1038_part_i.toml b/comparator/problems/erdos_1038_part_i.toml new file mode 100644 index 0000000000..57d95bef01 --- /dev/null +++ b/comparator/problems/erdos_1038_part_i.toml @@ -0,0 +1,10 @@ +# The answer type is inferred from the elaborated statement; this manifest +# remains as the worked example of `id` naming a workspace, and its `notes`. +id = "erdos_1038_part_i" +declaration = "erdos_1038.parts.i" +module = "FormalConjectures/ErdosProblems/1038.lean" +source = "https://www.erdosproblems.com/1038" +notes = """ +Asks for the infimum of `|{x : |f x| < 1}|` over nonconstant monic real +polynomials whose roots all lie in `[-1,1]`. +""" diff --git a/comparator/problems/margulis_conjecture_1_1.toml b/comparator/problems/margulis_conjecture_1_1.toml new file mode 100644 index 0000000000..23e682c381 --- /dev/null +++ b/comparator/problems/margulis_conjecture_1_1.toml @@ -0,0 +1,7 @@ +# `conjecture_1_1` is declared by two files. Without `module` the generator +# refuses rather than picking one, so each needs its own manifest and its own +# `id`; the id is what names the workspace directory. +id = "margulis_conjecture_1_1" +declaration = "conjecture_1_1" +module = "FormalConjectures/Arxiv/2504.17644/Margulis.lean" +source = "https://arxiv.org/abs/2504.17644" diff --git a/comparator/templates/WorkspaceTest.lean b/comparator/templates/WorkspaceTest.lean new file mode 100644 index 0000000000..a231373e04 --- /dev/null +++ b/comparator/templates/WorkspaceTest.lean @@ -0,0 +1,37 @@ +/- +Copyright 2026 The Formal Conjectures Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +-/ + +import Lean + +open Lean + +/-- Run comparator on this workspace's `config.json`, so that `lake test` +is the check. Adapted from `leanprover/lean-eval`'s workspace test template. +The binary comes from `PATH`, or from `COMPARATOR_BIN`. -/ +def main : IO UInt32 := do + let comparatorBin := (← IO.getEnv "COMPARATOR_BIN").getD "comparator" + try + let child ← IO.Process.spawn { + cmd := "lake" + args := #["env", comparatorBin, "config.json"] + } + child.wait + catch err => + IO.eprintln s!"Failed to run comparator via `{comparatorBin}`." + IO.eprintln "Install comparator, with landrun and lean4export, and put it \ +on PATH, or set COMPARATOR_BIN. See leanprover/comparator's README." + IO.eprintln s!"Original error: {err}" + pure 1 diff --git a/comparator/tools.toml b/comparator/tools.toml new file mode 100644 index 0000000000..a04bb304e9 --- /dev/null +++ b/comparator/tools.toml @@ -0,0 +1,7 @@ +# The pinned external tools, one machine-readable source of truth. "At or +# after" prose is not a lock; CI, local setup and documentation read this. +[tools] +comparator = "71b52ec29e06d4b7d882726553b1ceb99a2499e0" +landrun = "5ed4a3db3a4ad930d577215c6b9abaa19df7f99f" +# lean4export: the tag matching the workspace's lean-toolchain, v4.27.0 today. +lean4export = "v4.27.0" diff --git a/lakefile.toml b/lakefile.toml index 22ee8e56e7..246104783f 100644 --- a/lakefile.toml +++ b/lakefile.toml @@ -77,6 +77,13 @@ weak.linter.style.imports = true weak.google.answer = "postpone" +[[lean_exe]] +name = "comparator_facts" +srcDir = "scripts" +root = "comparator_facts" +exeName = "comparator_facts" +supportInterpreter = true + [[lean_exe]] name = "extract_names" srcDir = "scripts" diff --git a/scripts/comparator_facts.lean b/scripts/comparator_facts.lean new file mode 100644 index 0000000000..b6167485b1 --- /dev/null +++ b/scripts/comparator_facts.lean @@ -0,0 +1,140 @@ +/- +Copyright 2026 The Formal Conjectures Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +-/ +import Lean +import FormalConjecturesUtil.Answer +import FormalConjecturesUtil.Attributes.Basic + +/-! +The elaborator-side facts `make_comparator_workspace.py` currently gets by +reading Lean with regular expressions. + +Given a module and a declaration name, this prints JSON with what the +elaborated environment knows exactly and the text layer can only guess: + +- the declaration's source range, for slicing its original text; +- its binders, with names and explicitness, for the Solution adapter; +- the type of each `sorry` inside the *statement*, which is the type of an + `answer(sorry)` slot. The generator's manifest `answer_type` field exists + only because surface syntax does not carry this; the environment does. + +Usage: + lake exe comparator_facts + +The declaration may be given in full or by any whole suffix, the same rule +the Python generator uses. +-/ + +open Lean Meta + +/-- A request matches a name in full, or by dropping any whole prefix. -/ +def declares (declared : Name) (requested : String) : Bool := + let s := declared.toString + s == requested || s.endsWith ("." ++ requested) + +/-- Declaration parameters, as opposed to `∀` binders in the conclusion. + +`theorem foo (n : Nat) : P n` lambda-abstracts `n` in its proof value; +`theorem foo : ∀ n : Nat, P n` does not. Only the former are applied by the +generated Solution adapter, and `forallTelescope` alone cannot tell them +apart: the lambda arity of the (sorry) value can. lean-eval's extractor +draws the same line for the same reason. -/ +partial def lambdaArity : Expr → Nat + | .lam _ _ b _ => lambdaArity b + 1 + | .mdata _ b => lambdaArity b + | _ => 0 + +def binderJson (name : Name) (bi : BinderInfo) : Json := + Json.mkObj [("name", toJson name.toString), ("explicit", toJson bi.isExplicit)] + +unsafe def runWithImports {α : Type} (moduleNames : Array Name) + (actionToRun : MetaM α) : IO α := do + initSearchPath (← getBuildDir) + let imports := moduleNames.map fun n => { module := n } + Lean.enableInitializersExecution + let env ← Lean.importModules imports {} (trustLevel := 1024) (loadExts := true) + -- Twice the default budget, in the context's raw units, which are a + -- thousand times the `maxHeartbeats` option's: 800000 here meant "800" and + -- killed the first query. Finite, so a pathological statement errors and is + -- caught rather than grinding forever, which maxHeartbeats := 0 did. + let ctx := { fileName := "", fileMap := default, maxHeartbeats := 400000000 } + let (result, _) ← Core.CoreM.toIO (actionToRun.run' {} {}) ctx { env := env } + return result + +/-- Resolve within one module. Names declared elsewhere are not candidates, +which is what lets one environment holding every module still disambiguate +`conjecture_1_1` the way a per-module import does. -/ +def resolveIn (env : Environment) (modName : Name) (declName : String) : + Except String Name := + let inModule (n : Name) : Bool := + match env.getModuleIdxFor? n with + | some idx => env.header.moduleNames[idx.toNat]? == some modName + | none => false + -- No `isInternal` filter: `erdos_340.variants._33_mem_sub` has a component + -- starting with an underscore, which that heuristic calls internal. The + -- whole-suffix rule in `declares` already keeps auxiliary declarations out, + -- since `foo.proof_1` is not a suffix match for `foo`. + let matches_ := env.constants.toList.filterMap fun (n, _) => + if declares n declName && inModule n then some n else none + match matches_ with + | [] => .error s!"{declName} not found in {modName}" + | [n] => .ok n + | _ => + match matches_.filter (·.toString == declName) with + | [n] => .ok n + | _ => .error s!"{declName} is ambiguous: {matches_}" + +unsafe def main (args : List String) : IO UInt32 := do + let [modName, declName] := args + | IO.eprintln "usage: comparator_facts "; return 1 + runWithImports #[modName.toName] do + let env ← getEnv + match resolveIn env modName.toName declName with + | .error msg => IO.eprintln msg; return 1 + | .ok n => emit env n declName +where + emit (env : Environment) (name : Name) (decl : String) : MetaM UInt32 := do + let some info := env.find? name | IO.eprintln "vanished"; return 1 + let ranges ← findDeclarationRanges? name + -- The statement's sorries are `answer(sorry)` slots; a proof's sorry is + -- not in the *type*, so everything found here is a slot. + -- `findAnswerExprs` is the repository's own detection: it reads the + -- annotation the `answer` elaborator leaves, rather than guessing from + -- `sorryAx` applications. + let answerTypes ← forallTelescope info.type fun _ body => do + let found := Google.findAnswerExprs body + found.mapM fun a => do pure (toString (← ppExpr (← inferType a))) + let arity := match info.value? with + | some v => lambdaArity v + | none => 0 + let binders ← forallTelescope info.type fun xs _ => + (xs.extract 0 arity).mapM fun x => do + let d ← x.fvarId!.getDecl + pure (binderJson d.userName d.binderInfo) + let rangeJson := match ranges with + | some r => Json.mkObj [ + ("startLine", toJson r.range.pos.line), + ("startColumn", toJson r.range.pos.column), + ("endLine", toJson r.range.endPos.line), + ("endColumn", toJson r.range.endPos.column)] + | none => Json.null + let payload := Json.mkObj [ + ("declaration", toJson decl), + ("name", toJson name.toString), + ("range", rangeJson), + ("binders", toJson binders.toList), + ("answerTypes", toJson answerTypes.toList)] + IO.println payload.pretty + return 0 diff --git a/scripts/make_comparator_workspace.py b/scripts/make_comparator_workspace.py new file mode 100644 index 0000000000..ce9946556e --- /dev/null +++ b/scripts/make_comparator_workspace.py @@ -0,0 +1,841 @@ +#!/usr/bin/env python3 +"""Generate a comparator workspace for one problem statement. + +`leanprover/lean-eval` verifies a submission by building it against a Challenge +module whose statement the maintainers trust, under a config that pins the +permitted axioms. This script generates that shape for one Formal Conjectures +declaration. + +Challenge.lean imports the problem's own module. lean-eval's generated +Challenge is one `import Mathlib` and one statement, because its problems are +authored self-contained; this repository's are not, so the import here is the +problem's module and the statement's context comes with it. Only what Lean +scopes to a file has to be copied: `open`, `variable`, `universe`, +`set_option` and `local notation`. + +Layout produced: + + // + lakefile.toml pins: this checkout's Mathlib rev and FC commit + Challenge.lean the import, the file-scoped preamble, the target + statement with attributes stripped and its proof + replaced by `sorry`, and each `answer(sorry)` hoisted + into a definition hole the solver must fill + Submission.lean where the solver works; helper modules go under + Submission/ + Solution.lean fixed: restates the statement and closes it with the + Submission theorem, so the statement cannot drift + WorkspaceTest.lean `lake test` runs comparator on config.json + README.md what the solver needs to know, cache fetch included + config.json theorem and definition names, permitted axioms + holes.json the extracted blocks, for tooling and for review + +Lean reports the type of each `answer(sorry)` slot. The generator refuses a +case when it cannot match the reported types to their source positions. + +Two things the source cannot settle live in `comparator/problems/.toml`, +one file per problem: that answer type, and which file is meant when two +declare the same name. See that directory's README. + +Usage: + python make_comparator_workspace.py (ID | DECLARATION) [--out DIR] + [--answer-type T] [--module FILE] + python make_comparator_workspace.py --validate + +The workspace's own build needs a network fetch of its pinned dependencies, so +this script does not attempt it; generation is offline and the build belongs to +the comparator run. +""" + +import argparse +import json +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile +import tomllib + +ROOT = pathlib.Path(__file__).resolve().parent.parent +SOURCE_DIRS = [ROOT / "FormalConjectures"] +COMPARATOR_DIR = ROOT / "comparator" +MANIFEST_DIR = COMPARATOR_DIR / "problems" + + +def tool_pins(): + """The locked external tool revisions; comparator/tools.toml is the one + machine-readable source, and this module refuses to restate it.""" + with (COMPARATOR_DIR / "tools.toml").open("rb") as handle: + return tomllib.load(handle)["tools"] + + +PERMITTED_AXIOMS = ["propext", "Quot.sound", "Classical.choice"] + +DECL_START = re.compile( + # `local notation` and `scoped notation` carry the modifier before the + # keyword. Without them here, Erdos 125's `local notation "A" => ...` typed + # as nothing and was dropped, and its statements lost the sets they name. + r"^(?:noncomputable\s+|private\s+|protected\s+|local\s+|scoped\s+)*" + r"(theorem|lemma|def|abbrev|structure|inductive|instance|notation)\s", +) +KEEP_LOOSE = re.compile( + r"^(open|variable|universe|section|namespace|end|attribute|set_option)\b" +) + + +def elaborator_facts(module, declaration): + """What the elaborated environment knows about a declaration. + + Runs `lake exe comparator_facts`, which imports the module and reports the + declaration's source range, its binders with real explicitness, and the + inferred type of each `answer(sorry)` slot. Every one of these used to be + reconstructed from text, and each reconstruction had failure modes the + elaborator does not. + """ + proc = subprocess.run( + ["lake", "exe", "comparator_facts", module, declaration], + capture_output=True, + text=True, + cwd=ROOT, + ) + if proc.returncode != 0: + raise SystemExit( + f"comparator_facts {declaration}: " + f"{proc.stderr.strip() or proc.stdout.strip()}" + ) + out = proc.stdout + if "{" not in out: + raise SystemExit(f"comparator_facts {declaration}: no JSON in output") + return json.loads(out[out.index("{") :]) + + +def file_scoped_preamble(lines, start_line): + """Directives in force at `start_line`, and the namespace stack there. + + Lean scopes `open`, `variable`, `universe`, `set_option` and notation to + the file, so Challenge.lean has to restate them; nothing in the olean + records them. A directive counts only if it precedes the statement and + its scope still encloses it. + """ + stack, preamble, depth = [], [], 0 + for line in lines[: start_line - 1]: + if depth == 0 and KEEP_LOOSE.match(line) and not line.rstrip().endswith(" in"): + kind = line.split()[0] + parts = line.split(None, 1) + name = parts[1].strip() if len(parts) > 1 else None + if kind in ("namespace", "section"): + stack.append((kind, name)) + elif kind == "end": + if stack and ( + stack[-1][1] == name or (name is None and stack[-1][0] == "section") + ): + stack.pop() + else: + preamble.append((line, list(stack))) + depth += len(re.findall(r"/-", line)) - len(re.findall(r"-/", line)) + depth = max(depth, 0) + scope = list(stack) + in_force = [text for text, s in preamble if s == scope[: len(s)]] + return in_force, [n for k, n in scope if k == "namespace" and n] + + +def slug(name): + """A Lake package name and directory name for a declaration. + + A Lake package name is an identifier, so the dots in a qualified + declaration cannot go into one verbatim. + """ + return re.sub(r"[^0-9A-Za-z_]", "_", name) + + +def load_manifest(problem_id): + """Read explicit choices that Lean source cannot select by itself. + + A manifest selects the module when names collide. It may also override an + answer-slot type when Lean reports several types that cannot be matched to + source positions. The generator refuses both cases without an explicit + choice. + + `leanprover/lean-eval` keeps one TOML per problem, and the reason is worth + copying: two pull requests adding different problems never touch the same + file. + + id the filename stem, and the workspace directory name + declaration the Lean name, which need not be unique across the repository + module the file declaring it, relative to the repository root + answer_type the type of a non-`Prop` answer slot + notes free text for a reviewer + source a citation or URL + """ + path = MANIFEST_DIR / f"{problem_id}.toml" + if not path.exists(): + return {} + with path.open("rb") as handle: + data = tomllib.load(handle) + if data.get("id") != problem_id: + raise SystemExit( + f"{path} declares id {data.get('id')!r}, but its filename says " + f"{problem_id!r}; the two must agree" + ) + if "declaration" not in data: + raise SystemExit(f"{path} has no `declaration` field") + return data + + +def manifest_ids(): + return sorted(p.stem for p in MANIFEST_DIR.glob("*.toml")) + + +def module_name(rel_path): + """The Lean module name for a path under `FormalConjectures/`. + + Most problem files are named for a number, which is not an identifier, so + the component is written in guillemets: + `FormalConjectures.ErdosProblems.«940»`. + """ + parts = [ + c if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", c) else f"«{c}»" + for c in str(rel_path)[: -len(".lean")].split("/") + ] + return ".".join(parts) + + +def find_declaration(basename, module=None): + """Locate the file declaring `basename`. Returns (path, module_docstring, body). + + `module` names the file when more than one declares the name, and comes + from the problem's manifest. + """ + if module is not None: + named = ROOT / module + if not named.exists(): + raise SystemExit(f"manifest names {module}, which does not exist") + return _read_source(named) + hits = [] + for src in SOURCE_DIRS: + for path in sorted(src.rglob("*.lean")): + text = path.read_text(encoding="utf-8") + if re.search( + rf"(?:theorem|lemma)\s+(?:[\w.«»]*\.)?{re.escape(basename)}[\s:]", text + ): + hits.append(path) + if not hits: + raise SystemExit( + f"no declaration named {basename!r} found under FormalConjectures/" + ) + if len(hits) > 1: + raise SystemExit( + f"{basename!r} is ambiguous: " + + ", ".join(str(h.relative_to(ROOT)) for h in hits) + + "; pass --module to choose one, or record the choice in " + "comparator/problems/.toml" + ) + return _read_source(hits[0]) + + +def _read_source(path): + text = path.read_text(encoding="utf-8") + # Drop the license header; keep the module docstring; the rest is the body. + text = re.sub(r"\A/-.*?-/\s*", "", text, flags=re.DOTALL) + doc = "" + m = re.match(r"\s*(/-!.*?-/)\s*", text, flags=re.DOTALL) + if m: + doc = m.group(1) + text = text[m.end() :] + # Imports precede the docstring in source order; recover them from the original. + imports = re.findall( + r"^import\s+(\S+)", path.read_text(encoding="utf-8"), re.MULTILINE + ) + return path, imports, doc, text + + +def strip_decorations(block_text): + """Remove the docstring, line comments and attributes from a declaration. + + These interleave. Erdos 918 puts a `--` formalisation note between its + docstring and its `@[category ...]` line, and one anchored pass each left + the attribute in place. `@[category research open, AMS 5]` then reached + Challenge.lean, where the workspace has no such attribute, and Lean parsed + as far as the `open` inside it before giving up. + """ + # `open X in` binds to the declaration and has to survive, but it sits + # above the docstring, so stripping anchored at the start would stop dead + # on it. + prefix = "" + m = re.match(r"\A\s*(open\b[^\n]*\bin)\n", block_text) + if m: + prefix = m.group(1) + "\n" + block_text = block_text[m.end() :] + while True: + stripped = re.sub(r"\A\s*/--.*?-/\s*", "", block_text, flags=re.DOTALL) + stripped = re.sub(r"\A\s*--[^\n]*\n", "", stripped) + stripped = re.sub(r"\A\s*@\[[^\]]*\]\s*", "", stripped, flags=re.DOTALL) + if stripped == block_text: + return prefix + stripped + block_text = stripped + + +def replace_proof_with_sorry(text): + """Cut the proof body after `:=`, keeping the statement. + + A tactic proof is found by `:= by`, which a statement cannot contain, + `by` being a keyword. A term proof leaves only a bare `:=` to cut at, and + a statement can contain one of those: a structure literal `{ a := b }` + inside the statement would be cut in half. With more than one candidate + the script refuses, as everywhere else it cannot decide. + """ + m = re.search(r":=\s*by\b", text) + if m: + return text[: m.start()].rstrip() + " := by\n sorry" + if text.count(":=") > 1: + raise SystemExit( + "the declaration has a term-mode proof and more than one `:=`, so " + "the start of the proof cannot be read off the text" + ) + m = re.search(r":=", text) + if m: + return text[: m.start()].rstrip() + " := by\n sorry" + return text.rstrip() + " := by\n sorry" + + +def answer_spans(text): + """Return the source spans of syntactic `answer(...)` calls. + + This small lexer skips strings and nested line/block comments and balances + parentheses, so an answer term may itself contain parentheses. It is not a + Lean parser; malformed or unterminated syntax is refused. + """ + spans = [] + i = 0 + block_depth = 0 + in_string = False + escaped = False + while i < len(text): + pair = text[i : i + 2] + if block_depth: + if pair == "/-": + block_depth += 1 + i += 2 + elif pair == "-/": + block_depth -= 1 + i += 2 + else: + i += 1 + continue + if in_string: + if escaped: + escaped = False + elif text[i] == "\\": + escaped = True + elif text[i] == '"': + in_string = False + i += 1 + continue + if pair == "/-": + block_depth = 1 + i += 2 + continue + if pair == "--": + newline = text.find("\n", i + 2) + i = len(text) if newline < 0 else newline + 1 + continue + if text[i] == '"': + in_string = True + i += 1 + continue + if text.startswith("answer", i) and ( + i == 0 or not (text[i - 1].isalnum() or text[i - 1] in "_.'") + ): + j = i + len("answer") + while j < len(text) and text[j].isspace(): + j += 1 + if j < len(text) and text[j] == "(": + depth = 1 + k = j + 1 + nested_string = False + nested_escaped = False + nested_comment = 0 + while k < len(text) and depth: + nested_pair = text[k : k + 2] + if nested_comment: + if nested_pair == "/-": + nested_comment += 1 + k += 2 + elif nested_pair == "-/": + nested_comment -= 1 + k += 2 + else: + k += 1 + continue + if nested_string: + if nested_escaped: + nested_escaped = False + elif text[k] == "\\": + nested_escaped = True + elif text[k] == '"': + nested_string = False + k += 1 + continue + if nested_pair == "/-": + nested_comment = 1 + k += 2 + elif nested_pair == "--": + newline = text.find("\n", k + 2) + k = len(text) if newline < 0 else newline + 1 + elif text[k] == '"': + nested_string = True + k += 1 + else: + if text[k] == "(": + depth += 1 + elif text[k] == ")": + depth -= 1 + k += 1 + if depth: + raise SystemExit("unterminated answer(...) term") + spans.append((i, k, text[j + 1 : k - 1])) + i = k + continue + i += 1 + if block_depth or in_string: + raise SystemExit("unterminated comment or string while reading answers") + return spans + + +def hoist_answers(statement, basename, slot_types, override=None): + """Replace each `answer(sorry)` with a named definition hole. + + The slot types come from the elaborated environment, where the `answer` + elaborator ran with the expected type in hand; the old surface-syntax + guess (an `↔` beside the slot means `Prop`) and the manifest's hand-kept + `answer_type` both survive only as overrides. Slots of different types in + one statement are refused: the environment reports the types as a set, + and matching them to positions would be a guess. + """ + holes = [] + calls = answer_spans(statement) + selected = [call for call in calls if call[2].strip() == "sorry"] + count = len(selected) + if count == 0: + return statement, holes + # Under the default `alwaysTrue` setting, the `answer` elaborator erases a + # slot to `True` if and only if its expected type is `Prop` + # (FormalConjecturesUtil/Answer.lean). So a slot the environment carries + # no annotation for is a `Prop` slot by the elaborator's own rule, not by + # guesswork, and no postpone build is needed. + missing = count - len(slot_types) + if override: + types = [override] * count + elif missing == count: + types = ["Prop"] * count + elif missing == 0 and len(set(slot_types)) == 1: + types = [slot_types[0]] * count + elif missing == 0: + raise SystemExit( + f"{basename} has {count} answer slots of differing types " + f"{slot_types}; pass --answer-type" + ) + else: + # Some slots are Prop and some are not: which positions are which + # cannot be read off an unordered set, so refuse rather than assign. + raise SystemExit( + f"{basename}: {missing} Prop slot(s) and {len(slot_types)} typed " + f"slot(s) {slot_types} cannot be matched to positions; pass " + "--answer-type" + ) + replacements = [] + for i, (start, end, _argument) in enumerate(selected): + hole = f"{basename}_answer" if count == 1 else f"{basename}_answer_{i + 1}" + holes.append(f"noncomputable def {hole} : {types[i]} := sorry") + replacements.append((start, end, hole)) + for start, end, hole in reversed(replacements): + statement = statement[:start] + hole + statement[end:] + return statement, holes + + +def pins(source_path=None): + """Revisions the workspace's own build can actually fetch. + + The FC pin must be reachable from the upstream repository the lakefile + names, so it is the merge-base with `origin/main`, not HEAD: a local + branch commit would generate a workspace whose build fails at fetch time. + The generator stops if the selected source differs from that revision. + Otherwise it could combine a working-tree statement with an older imported + context. + """ + manifest = json.loads((ROOT / "lake-manifest.json").read_text()) + mathlib_rev = next(p["rev"] for p in manifest["packages"] if p["name"] == "mathlib") + merge_base = subprocess.run( + ["git", "-C", str(ROOT), "merge-base", "HEAD", "origin/main"], + capture_output=True, + text=True, + ) + if merge_base.returncode != 0 or not merge_base.stdout.strip(): + raise SystemExit("cannot resolve the Formal Conjectures source revision") + fc_rev = merge_base.stdout.strip() + if source_path is not None: + comparison = subprocess.run( + ["git", "-C", str(ROOT), "diff", "--quiet", fc_rev, "--", str(source_path)] + ) + if comparison.returncode not in (0, 1): + raise SystemExit(f"cannot compare {source_path} with {fc_rev[:12]}") + if comparison.returncode == 1: + raise SystemExit( + f"{source_path} differs from pinned revision {fc_rev[:12]}; " + "land the source on upstream main before generating" + ) + return mathlib_rev, fc_rev + + +def lakefile(workspace_id, mathlib_rev, fc_rev): + return f"""name = "{workspace_id}" +testDriver = "workspace_test" +defaultTargets = ["Challenge", "Solution", "Submission"] + +[leanOptions] +autoImplicit = false + +[[require]] +name = "mathlib" +git = "https://github.com/leanprover-community/mathlib4.git" +rev = "{mathlib_rev}" + +[[require]] +name = "formal_conjectures" +git = "https://github.com/google-deepmind/formal-conjectures.git" +rev = "{fc_rev}" + +[[lean_lib]] +name = "Challenge" + +[[lean_lib]] +name = "Solution" + +[[lean_lib]] +name = "Submission" + +[[lean_exe]] +name = "workspace_test" +root = "WorkspaceTest" +""" + + +def hole_names_of(holes): + return [h.split()[2] for h in holes] + + +def write_workspace(target, files): + """Write a complete workspace without overwriting or leaving a partial one.""" + target = pathlib.Path(target) + if target.exists(): + raise SystemExit(f"refusing to overwrite existing workspace: {target}") + target.parent.mkdir(parents=True, exist_ok=True) + staging = pathlib.Path( + tempfile.mkdtemp(prefix=f".{target.name}.", dir=target.parent) + ) + try: + for relative, content in files.items(): + destination = staging / relative + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(content, encoding="utf-8") + staging.rename(target) + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise + + +def generate(basename, out_dir, answer_type=None, module=None): + """Write a comparator workspace for one declaration. + + Challenge.lean imports the problem's own module rather than restating its + dependencies. `leanprover/lean-eval` generates a Challenge that is one + `import` and one statement, and reconstructing the surrounding definitions + by hand instead cost six defects that only Lean could find: file-scoped + `open` and `variable` lost, `local notation` unrecognised, a `namespace` + swallowing the declaration below it, `section` lines left unclosed. An + import has none of those failure modes. + + Importing a repository full of `sorry` is safe here because comparator + checks axioms. A solution closing the goal with the imported statement + reports `sorryAx`, which `permitted_axioms` does not allow. + """ + manifest = load_manifest(basename) + declaration = manifest.get("declaration", basename) + # An argument given on the command line is explicit, so it wins over the + # manifest; the manifest is the durable record of the same choice. + answer_type = answer_type or manifest.get("answer_type") + module = module or manifest.get("module") + path, _imports, _module_doc, body = find_declaration(declaration, module) + fc_module = module_name(path.relative_to(ROOT)) + facts = elaborator_facts(fc_module, declaration) + if facts["range"] is None: + raise SystemExit(f"{declaration}: no source range recorded") + + source_lines = path.read_text(encoding="utf-8").split("\n") + lo, hi = facts["range"]["startLine"], facts["range"]["endLine"] + end_col = facts["range"].get("endColumn") + # `open X in` is part of the command but sits above what the range covers + # in some toolchains; pull it in when the line above ends with ` in`. + while ( + lo > 1 + and source_lines[lo - 2].rstrip().endswith(" in") + and KEEP_LOOSE.match(source_lines[lo - 2]) + ): + lo -= 1 + sliced = source_lines[lo - 1 : hi] + if end_col is not None and sliced: + sliced = sliced[:-1] + [sliced[-1][:end_col]] + original = "\n".join(sliced) + statement = original + + preamble, namespaces_at_target = file_scoped_preamble(source_lines, lo) + + statement = strip_decorations(statement) + statement = replace_proof_with_sorry(statement) + declared = None + for line in statement.split("\n"): + dm = DECL_START.match(line) + if dm: + declared = re.match(r"\s*([\w.«»]+)", line[dm.end() :]).group(1) + break + if declared is None: + raise SystemExit(f"{declaration}: no declaration line in the slice") + statement, holes = hoist_answers( + statement, declared, facts.get("answerTypes", []), answer_type + ) + + args = [b["name"] for b in facts["binders"] if b["explicit"]] + bad = [a for a in args if "✝" in a or "._" in a] + if bad: + raise SystemExit( + f"{declared} has inaccessible explicit binders {bad}; the " + "Solution adapter cannot apply them by name" + ) + + # `open A`, then `open A.B`: opening the inner namespace does not open the + # outer one, and a statement may name siblings from either. + opens = [ + f"open {'.'.join(namespaces_at_target[:i + 1])}" + for i in range(len(namespaces_at_target)) + ] + + # One header shared by all three Lean files: the statement's text is + # identical in each, so what it needs to elaborate must be too. + header = ( + ("\n".join(opens) + "\n" if opens else "") + + ("\n".join(preamble) + "\n" if preamble else "") + + ("\n" if opens or preamble else "") + ) + suffix = ":= by\n sorry" + signature = statement.rstrip() + if signature.endswith(suffix): + signature = signature[: -len(suffix)].rstrip() + + challenge = ( + f"import {fc_module}\n\n" + + header + + "\n\n".join(holes) + + ("\n\n" if holes else "") + + statement + + "\n" + ) + + # The participant's file. The statement sits inside `namespace Submission` + # so nothing here can collide with, or stand in for, the trusted names. + submission = ( + f"import {fc_module}\nimport Submission.Helpers\n\n" + + header + + "namespace Submission\n\n" + + "\n\n".join(holes) + + ("\n\n" if holes else "") + + statement + + "\n\n" + + "end Submission\n" + ) + + # The fixed adapter, lean-eval's shape: it restates the trusted statement + # and closes it with the Submission theorem, so it fails to compile the + # moment the submission proves anything else. The participant never edits + # it, which is what keeps the statement pinned. + delegated = [ + h.rsplit(":= sorry", 1)[0] + ":= Submission." + hn + for h, hn in zip(holes, hole_names_of(holes)) + ] + solution = ( + f"import {fc_module}\nimport Submission\n\n" + + header + + "\n\n".join(delegated) + + ("\n\n" if delegated else "") + + signature + + " :=\n Submission." + + declared + + ("".join(" " + a for a in args)) + + "\n" + ) + + mathlib_rev, fc_rev = pins(path.relative_to(ROOT)) + full_name = ".".join(namespaces_at_target + [declared]) + hole_names = hole_names_of(holes) + + workspace_id = slug(manifest.get("id", declared)) + ws = pathlib.Path(out_dir) / workspace_id + holes_line = ( + "\nFill each definition hole in `Submission.lean` too. Hole answers " + "also get a\nhuman check, because a hole can be gamed in ways the " + "comparator cannot see.\nChecking holes needs a comparator built at " + f"commit `{tool_pins()['comparator'][:8]}`, which\nadded definition " + "support.\n" + if holes + else "" + ) + manifest_lines = "".join( + f"- {field.capitalize()}: {' '.join(str(manifest[field]).split())}\n" + for field in ("source", "notes") + if manifest.get(field) + ) + workspace_readme = ( + f"# {workspace_id}\n\n" + f"A comparator challenge for `{declared}`, generated from\n" + f"`{path.relative_to(ROOT)}` in google-deepmind/formal-conjectures.\n\n" + + manifest_lines + + "\nProve the statement in `Submission.lean`, keeping it as it stands; " + "put helper\nmodules under `Submission/` if you need them. Do not " + "modify `Challenge.lean` or\n`Solution.lean`: the trusted statement " + "lives there, and `Solution.lean` closes it\nwith your `Submission` " + "theorem, so it fails to compile if the submission proves\nanything " + "else.\n" + "\nComparator accepts the workspace only if the statement is proved " + "under the\naxioms in `config.json`. `sorry` adds `sorryAx`, which is " + "not permitted, and\nclosing the goal with the imported original " + "fails the same way, since that is\n`sorry` too. `lake test` runs " + "comparator, from `PATH` or `COMPARATOR_BIN`.\n" + "\nIf comparator fails with `incompatible header` on an `.olean`, the " + "mismatch is\nbetween this workspace's toolchain and the one " + "`lean4export` was built with,\nnever a problem with the proof: copy " + "this workspace's `lean-toolchain` into\nyour `lean4export` checkout, " + "rebuild it, and clear `.lake/build` here.\n" + + holes_line + + "\nFetch the Mathlib cache before the first build; a cold build takes " + "the best\npart of an hour without it:\n\n" + " lake exe cache get\n" + " lake build\n" + ) + helper = ( + "import Mathlib\n\n" + "/-! Helper lemmas for the submission go here, or in further modules\n" + "under `Submission/`, each imported from `Submission.lean`. -/\n\n" + "namespace Submission\n\nend Submission\n" + ) + config = { + "challenge_module": "Challenge", + "solution_module": "Solution", + "theorem_names": [declared], + "permitted_axioms": PERMITTED_AXIOMS, + "enable_nanoda": False, + } + if hole_names: + # Comparator's documented no-hole config carries no such field. + config["definition_names"] = hole_names + holes_payload = { + "id": manifest.get("id", declared), + "module": str(path.relative_to(ROOT)), + "holes": [ + { + "name": ".".join(namespaces_at_target + [hn]), + "basename": hn, + "kind": "def", + "body": body_, + } + for hn, body_ in zip(hole_names, holes) + ] + + [ + { + "name": full_name, + "basename": declared, + "kind": "theorem", + "body": original, + } + ], + } + write_workspace( + ws, + { + "lakefile.toml": lakefile(workspace_id, mathlib_rev, fc_rev), + "lean-toolchain": (ROOT / "lean-toolchain").read_text(encoding="utf-8"), + "README.md": workspace_readme, + "Challenge.lean": challenge, + "Solution.lean": solution, + "Submission.lean": submission, + "Submission/Helpers.lean": helper, + "WorkspaceTest.lean": ( + COMPARATOR_DIR / "templates" / "WorkspaceTest.lean" + ).read_text(encoding="utf-8"), + "config.json": json.dumps(config, indent=2) + "\n", + "holes.json": json.dumps(holes_payload, indent=2, ensure_ascii=False) + + "\n", + }, + ) + return ws + + +def validate(): + """Check every manifest resolves to exactly one declaration. + + Run this rather than discovering a stale `module` field when someone + generates the workspace months later. + """ + bad = 0 + for problem_id in manifest_ids(): + try: + manifest = load_manifest(problem_id) + declaration = manifest["declaration"] + path, _i, _d, _b = find_declaration(declaration, manifest.get("module")) + elaborator_facts(module_name(path.relative_to(ROOT)), declaration) + except SystemExit as exc: + print(f"{problem_id}: {exc}", file=sys.stderr) + bad += 1 + continue + print(f"{problem_id}: {declaration} in {path.relative_to(ROOT)}") + if bad: + print(f"{bad} manifest(s) do not resolve", file=sys.stderr) + return 1 if bad else 0 + + +def main(argv): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument( + "declaration", + nargs="?", + help="a manifest id, or a declaration name such as erdos_940", + ) + ap.add_argument("--out", default=str(ROOT / ".comparator")) + ap.add_argument( + "--answer-type", + default=None, + help="type of a non-Prop answer(sorry) slot; " + "the manifest's `answer_type` is used when absent", + ) + ap.add_argument( + "--module", + default=None, + help="the file declaring it, when more than one does; " + "overrides the manifest's `module`", + ) + ap.add_argument( + "--validate", + action="store_true", + help="check every manifest resolves, and generate nothing", + ) + args = ap.parse_args(argv) + if args.validate: + return validate() + if not args.declaration: + ap.error("give a declaration, or --validate") + ws = generate(args.declaration, args.out, args.answer_type, args.module) + print(ws) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/test_make_comparator_workspace.py b/scripts/test_make_comparator_workspace.py new file mode 100644 index 0000000000..a12bd71cdd --- /dev/null +++ b/scripts/test_make_comparator_workspace.py @@ -0,0 +1,282 @@ +# Copyright 2026 The Formal Conjectures Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline tests for `make_comparator_workspace.py`. + +Every case here pins a failure the first real workspace build produced, or a +rule whose violation would generate a workspace that builds but poses the +wrong problem. The build itself is the comparator's job, not these tests'. +""" + +import json +import pathlib +import subprocess +import tempfile +import unittest +from unittest import mock + +import make_comparator_workspace as mcw +from make_comparator_workspace import ( + answer_spans, + file_scoped_preamble, + hoist_answers, + load_manifest, + pins, + replace_proof_with_sorry, + strip_decorations, + write_workspace, +) + + +class HoistTest(unittest.TestCase): + """Slot types come from the elaborated environment.""" + + def test_slot_takes_the_environment_type(self): + stmt, holes = hoist_answers( + "theorem t : answer(sorry) ↔ ∀ n, n ≤ n := by\n sorry", "t", ["Prop"] + ) + self.assertIn("t_answer", stmt) + self.assertIn("noncomputable def t_answer : Prop := sorry", holes) + + def test_erased_slot_is_prop_by_the_elaborators_rule(self): + # The default `alwaysTrue` setting erases a slot iff its expected + # type is Prop, so a missing annotation names the type exactly. + _, holes = hoist_answers( + "theorem t : answer(sorry) ↔ P := by\n sorry", "t", [] + ) + self.assertIn("noncomputable def t_answer : Prop := sorry", holes) + + def test_mixed_prop_and_typed_slots_are_refused(self): + with self.assertRaises(SystemExit): + hoist_answers( + "theorem t : answer(sorry) ∧ (answer(sorry) = 3) := by\n sorry", + "t", + ["Nat"], + ) + + def test_non_prop_type_is_read_not_guessed(self): + _, holes = hoist_answers( + "theorem t : sSup S = answer(sorry) := by\n sorry", "t", ["ENNReal"] + ) + self.assertIn("t_answer : ENNReal", holes[0]) + + def test_override_wins(self): + _, holes = hoist_answers( + "theorem t : sSup S = answer(sorry) := by\n sorry", "t", ["ENNReal"], "ℝ" + ) + self.assertIn("t_answer : ℝ", holes[0]) + + def test_differing_slot_types_are_refused(self): + # Matching types to positions would be a guess. + with self.assertRaises(SystemExit): + hoist_answers( + "theorem t : answer(sorry) = answer(sorry) := by\n sorry", + "t", + ["Nat", "Int"], + ) + + def test_no_slot_is_left_alone(self): + stmt, holes = hoist_answers("theorem t : True := by\n sorry", "t", []) + self.assertEqual(holes, []) + + def test_fixed_answer_is_not_turned_into_a_hole(self): + original = "theorem t : IsGLB S answer(2) := by\n sorry" + unchanged, holes = hoist_answers(original, "t", ["ENNReal"]) + self.assertEqual(unchanged, original) + self.assertEqual(holes, []) + + def test_nested_answer_term_is_one_balanced_slot(self): + calls = answer_spans("theorem t : f answer((fun x => x) (g 2)) := by\n sorry") + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0][2], "(fun x => x) (g 2)") + + def test_answer_text_in_comments_and_strings_is_ignored(self): + calls = answer_spans( + '-- answer(1)\ntheorem t : p "answer(2)" answer(3) := by sorry' + ) + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0][2], "3") + + +class PreambleTest(unittest.TestCase): + """Only directives in force at the statement are carried.""" + + def test_variable_in_a_closed_section_is_dropped(self): + lines = [ + "section S", + "variable {n : Nat}", + "end S", + "", + "open Nat", + "", + "theorem t : True := trivial", + ] + pre, ns = file_scoped_preamble(lines, 7) + self.assertEqual(pre, ["open Nat"]) + self.assertEqual(ns, []) + + def test_namespace_stack_is_reported(self): + lines = ["namespace A", "open Nat", "theorem t : True := trivial"] + pre, ns = file_scoped_preamble(lines, 3) + self.assertEqual(pre, ["open Nat"]) + self.assertEqual(ns, ["A"]) + + def test_directive_inside_a_comment_is_not_a_directive(self): + lines = ["/--", "open the door", "-/", "theorem t : True := trivial"] + pre, _ = file_scoped_preamble(lines, 4) + self.assertEqual(pre, []) + + +class StatementTest(unittest.TestCase): + def test_proof_is_replaced_but_statement_kept(self): + out = replace_proof_with_sorry( + "theorem t : True := by\n have h := trivial\n exact h" + ) + self.assertIn("theorem t : True", out) + self.assertNotIn("have h", out) + self.assertTrue(out.rstrip().endswith("sorry")) + + def test_term_mode_proof_is_replaced_too(self): + out = replace_proof_with_sorry("theorem t : True := trivial") + self.assertNotIn("trivial", out) + self.assertTrue(out.rstrip().endswith("sorry")) + + def test_term_proof_with_a_structure_literal_is_refused(self): + # The statement's own `:=` cannot be told from the proof's, and + # cutting at the wrong one truncates the statement. + with self.assertRaises(SystemExit): + replace_proof_with_sorry("theorem t : F { a := 1 } := ⟨rfl⟩") + + def test_a_line_comment_between_docstring_and_attribute_is_stripped(self): + # Erdos 918 writes a `--` formalisation note there. One anchored pass + # each left `@[category research open]` on the statement, and Lean + # parsed as far as the `open` inside it. + out = strip_decorations( + "/-- doc -/\n-- note\n@[category research open, AMS 5]\n" + "theorem t : True := by\n sorry" + ) + self.assertTrue(out.startswith("theorem")) + + def test_open_in_survives_stripping(self): + # It binds to the declaration, and it sits above the docstring. + out = strip_decorations( + "open scoped Classical in\n/-- doc -/\n@[category research open]\n" + "theorem t : True := by\n sorry" + ) + self.assertTrue(out.startswith("open scoped Classical in\ntheorem")) + + def test_decorations_are_stripped_from_the_target(self): + out = strip_decorations( + "/-- doc -/\n@[category research open]\ntheorem t : True := by\n sorry" + ) + self.assertTrue(out.startswith("theorem")) + + +class TemplateTest(unittest.TestCase): + def test_workspace_test_template_exists_and_is_the_runner(self): + # The generator copies this file into every workspace; a missing or + # gutted template would only surface at `lake test` time, elsewhere. + text = (mcw.COMPARATOR_DIR / "templates" / "WorkspaceTest.lean").read_text() + self.assertIn("def main", text) + self.assertIn("COMPARATOR_BIN", text) + + +class ManifestTest(unittest.TestCase): + """A manifest supplies what the Lean source cannot.""" + + def setUp(self): + self._dir = tempfile.TemporaryDirectory() + self._saved = mcw.MANIFEST_DIR + mcw.MANIFEST_DIR = pathlib.Path(self._dir.name) + + def tearDown(self): + mcw.MANIFEST_DIR = self._saved + self._dir.cleanup() + + def write(self, name, body): + (mcw.MANIFEST_DIR / name).write_text(body) + + def test_absent_manifest_is_not_an_error(self): + # Most statements need none, and the generator works without one. + self.assertEqual(load_manifest("no_such_problem"), {}) + + def test_fields_are_read(self): + self.write("p.toml", 'id = "p"\ndeclaration = "d"\nanswer_type = "ENNReal"\n') + self.assertEqual(load_manifest("p")["answer_type"], "ENNReal") + + def test_id_must_match_the_filename(self): + # The filename is what the generator looks up, so a disagreeing `id` + # would silently name a workspace directory nobody asked for. + self.write("p.toml", 'id = "other"\ndeclaration = "d"\n') + with self.assertRaises(SystemExit): + load_manifest("p") + + def test_declaration_is_required(self): + self.write("p.toml", 'id = "p"\n') + with self.assertRaises(SystemExit): + load_manifest("p") + + +class OutputTest(unittest.TestCase): + def test_existing_workspace_is_not_overwritten(self): + with tempfile.TemporaryDirectory() as tmp: + target = pathlib.Path(tmp) / "workspace" + target.mkdir() + sentinel = target / "keep.txt" + sentinel.write_text("keep", encoding="utf-8") + with self.assertRaisesRegex(SystemExit, "refusing to overwrite"): + write_workspace(target, {"Challenge.lean": "theorem t : True"}) + self.assertEqual(sentinel.read_text(encoding="utf-8"), "keep") + + def test_failed_write_leaves_no_partial_workspace(self): + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + target = root / "workspace" + with mock.patch.object( + pathlib.Path, "write_text", side_effect=OSError("disk error") + ): + with self.assertRaisesRegex(OSError, "disk error"): + write_workspace(target, {"Challenge.lean": "theorem t : True"}) + self.assertFalse(target.exists()) + self.assertEqual(list(root.iterdir()), []) + + +class PinTest(unittest.TestCase): + def test_changed_source_is_refused(self): + saved_root = mcw.ROOT + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + mcw.ROOT = root + (root / "lake-manifest.json").write_text( + json.dumps( + { + "packages": [{"name": "mathlib", "rev": "b" * 40}], + } + ), + encoding="utf-8", + ) + results = [ + subprocess.CompletedProcess([], 0, stdout="a" * 40 + "\n"), + subprocess.CompletedProcess([], 1), + ] + try: + with mock.patch.object(mcw.subprocess, "run", side_effect=results): + with self.assertRaisesRegex(SystemExit, "differs from pinned"): + pins(pathlib.Path("FormalConjectures/Example.lean")) + finally: + mcw.ROOT = saved_root + + +if __name__ == "__main__": + unittest.main() From fc7583b423e09a50b84b2e0e60bb2e7ed2a22915 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:09:06 -0400 Subject: [PATCH 02/70] docs: align comparator adapter with LeanEval 4.33 architecture --- comparator/README.md | 124 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 99 insertions(+), 25 deletions(-) diff --git a/comparator/README.md b/comparator/README.md index 85c253d11e..3f3cd0ee59 100644 --- a/comparator/README.md +++ b/comparator/README.md @@ -1,19 +1,90 @@ -# Comparator workspace adapter +# Formal Conjectures to LeanEval adapter -This directory contains a thin adapter from Formal Conjectures to +This directory contains the Formal Conjectures side of the integration with [`leanprover/lean-eval`](https://github.com/leanprover/lean-eval) and -[`leanprover/comparator`](https://github.com/leanprover/comparator). It does not -implement another evaluator. - -## How it works - -1. `scripts/comparator_facts.lean` asks Lean for the selected declaration's - source range, binders, and `answer(sorry)` slot types. -2. `scripts/make_comparator_workspace.py` creates one pinned workspace. -3. The generated project builds the challenge and submission. -4. `lake test` runs Comparator against `config.json`. - -The workspace contains: +[`leanprover/comparator`](https://github.com/leanprover/comparator). + +## Status and version boundary + +The code in this draft is a **conformance prototype**, not a second permanent +workspace generator. + +- Formal Conjectures currently elaborates its source under its own pinned + toolchain. +- LeanEval is the benchmark host and target environment. Imported problems must + compile under LeanEval's pinned **Lean 4.33** toolchain and matching Mathlib + revision. +- Formal Conjectures does not need a repository-wide toolchain upgrade merely + to support the integration. +- LeanEval owns the shared Challenge/Solution/Submission generator and + Comparator execution path. +- Formal Conjectures owns an importer that resolves declarations, preserves + provenance, maps `answer(sorry)` semantics, and emits reviewable LeanEval + source and manifests. + +The standalone workspace writer in this draft exercises the hard FC-side +extraction cases while the shared generator interface is being separated from +LeanEval's `EvalTools`. Once that interface exists, the importer must call it +rather than retain parallel generation logic. + +This follows the ownership split proposed in +[`lean-eval#536`](https://github.com/leanprover/lean-eval/pull/536), with +coordination tracked in +[`lean-eval#533`](https://github.com/leanprover/lean-eval/issues/533) and +[`formal-conjectures#4930`](https://github.com/google-deepmind/formal-conjectures/issues/4930). + +## Final integration flow + +1. The FC importer resolves a declaration against an exact Formal Conjectures + commit and obtains its source range, binders, namespace, dependencies, and + `answer(sorry)` slot types from Lean. +2. It emits vendored LeanEval source, one LeanEval problem manifest, and + immutable provenance containing at least the FC repository, commit, source + path, fully qualified declaration name, and frozen-set identity. +3. LeanEval builds the vendored source under Lean 4.33 and its matching Mathlib + pin. +4. The shared LeanEval generator creates `Challenge`, `ChallengeDeps`, + `Submission`, `Solution`, and Comparator configuration. +5. LeanEval CI builds the generated workspace and runs Comparator with + `sorryAx` rejected. +6. A deterministic trusted-statement fingerprint links the imported source, + generated challenge, result record, and later upstream corrections. + +The importer must fail closed on ambiguous declarations, source drift, +inaccessible binders, unsupported dependencies, answer-slot types that cannot +be matched safely, and existing output. + +## Conformance suite before a public import + +The adapter should cover these boundary cases before importing a frozen set: + +- a plain theorem proof; +- a `Prop`-valued `answer(sorry)` slot; +- a non-`Prop` answer slot; +- explicit declaration parameters versus `∀` binders in the conclusion; +- trusted helper dependencies requiring `ChallengeDeps` or multiple trusted + files. + +The smoke cases in this draft exercise those distinctions. They validate +extraction and adapter behavior, not mathematical correctness or maintainer +acceptance. + +The first public open-conjectures import also needs a corrected source set. +`FC100OpenSet1` currently verifies itself as 92 `research open` entries and 8 +`research solved` entries, so it must not be imported wholesale as one hundred +open conjectures. + +## Current prototype + +`scripts/comparator_facts.lean` asks Lean for the selected declaration's source +range, binders, and `answer(sorry)` slot types. + +`scripts/make_comparator_workspace.py` then creates one pinned standalone +workspace as a conformance harness. The generated workspace uses the Formal +Conjectures toolchain and dependency pins. It is **not** the final LeanEval 4.33 +artifact. + +The prototype workspace contains: - `Challenge.lean`, with the trusted statement and proof hole; - `Submission.lean` and `Submission/`, where a solver works; @@ -26,7 +97,7 @@ The workspace contains: statement. Comparator also rejects `sorryAx` because it is not in the permitted axiom list. -## Generate one workspace +### Generate one prototype workspace ```bash python3 scripts/make_comparator_workspace.py erdos_940.variants.large_integers @@ -34,13 +105,13 @@ python3 scripts/make_comparator_workspace.py erdos_940.variants.large_integers Use `--out` to choose the parent directory. The generator refuses to overwrite an existing workspace. It writes into a temporary directory and renames the -complete result into place. +complete workspace into place. The generator also stops when the selected source differs from the pinned upstream revision. This prevents a workspace from combining a working-tree statement with an older imported context. -## Supported inputs +### Supported prototype inputs - theorem proofs; - definition answers represented by `answer(sorry)`; @@ -49,10 +120,10 @@ statement with an older imported context. Plain-statement disproofs remain out of scope until Comparator provides an upstream interface for them. -## Problem manifests +## Prototype problem manifests -Most declarations need no manifest. Add one TOML file under `problems/` only -when the source cannot select the declaration by itself. +Most declarations need no prototype manifest. Add one TOML file under +`problems/` only when the source cannot select the declaration by itself. | Field | Meaning | |---|---| @@ -71,9 +142,12 @@ python3 scripts/make_comparator_workspace.py --validate ## Tool pins -`tools.toml` records the external tool revisions. Generated workspaces pin -Mathlib from `lake-manifest.json` and Formal Conjectures to the current upstream -revision. The workspace build fetches these dependencies; workspace generation -itself does not run Comparator. +`tools.toml` records the external tool revisions used by the prototype. +Generated prototype workspaces pin Mathlib from `lake-manifest.json` and Formal +Conjectures to the current upstream revision. Workspace generation itself does +not run Comparator. -Issue #4930 tracks the upstream integration and execution-service decisions. +The final importer instead targets LeanEval's Lean 4.33 toolchain, Mathlib pin, +manifest schema, shared generator revision, and CI policy. Those target pins +belong on the LeanEval side and must be recorded in every generated import or +its provenance record. From b06bdbb27f5ca56f84699802ace02825f0e35b81 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:21:11 -0400 Subject: [PATCH 03/70] test: add Lean 4.33 Comparator integration pilot --- .github/workflows/comparator-lean-4-33.yml | 76 +++++++++++++++++++ .../fc_sum_of_three_cubes/Challenge.lean | 41 ++++++++++ .../fc_sum_of_three_cubes/ChallengeDeps.lean | 30 ++++++++ .../pilots/fc_sum_of_three_cubes/README.md | 66 ++++++++++++++++ .../fc_sum_of_three_cubes/Solution.lean | 36 +++++++++ .../fc_sum_of_three_cubes/Submission.lean | 48 ++++++++++++ .../Submission/Helpers.lean | 23 ++++++ .../fc_sum_of_three_cubes/WorkspaceTest.lean | 36 +++++++++ .../pilots/fc_sum_of_three_cubes/config.json | 17 +++++ .../fc_sum_of_three_cubes/lakefile.toml | 27 +++++++ .../fc_sum_of_three_cubes/lean-toolchain | 1 + .../fc_sum_of_three_cubes/provenance.json | 35 +++++++++ 12 files changed, 436 insertions(+) create mode 100644 .github/workflows/comparator-lean-4-33.yml create mode 100644 comparator/pilots/fc_sum_of_three_cubes/Challenge.lean create mode 100644 comparator/pilots/fc_sum_of_three_cubes/ChallengeDeps.lean create mode 100644 comparator/pilots/fc_sum_of_three_cubes/README.md create mode 100644 comparator/pilots/fc_sum_of_three_cubes/Solution.lean create mode 100644 comparator/pilots/fc_sum_of_three_cubes/Submission.lean create mode 100644 comparator/pilots/fc_sum_of_three_cubes/Submission/Helpers.lean create mode 100644 comparator/pilots/fc_sum_of_three_cubes/WorkspaceTest.lean create mode 100644 comparator/pilots/fc_sum_of_three_cubes/config.json create mode 100644 comparator/pilots/fc_sum_of_three_cubes/lakefile.toml create mode 100644 comparator/pilots/fc_sum_of_three_cubes/lean-toolchain create mode 100644 comparator/pilots/fc_sum_of_three_cubes/provenance.json diff --git a/.github/workflows/comparator-lean-4-33.yml b/.github/workflows/comparator-lean-4-33.yml new file mode 100644 index 0000000000..fd38d1339f --- /dev/null +++ b/.github/workflows/comparator-lean-4-33.yml @@ -0,0 +1,76 @@ +name: Comparator Lean 4.33 pilot + +on: + push: + branches: + - comparator-workspaces + paths: + - 'comparator/pilots/fc_sum_of_three_cubes/**' + - '.github/workflows/comparator-lean-4-33.yml' + pull_request: + paths: + - 'comparator/pilots/fc_sum_of_three_cubes/**' + - '.github/workflows/comparator-lean-4-33.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + build-and-compare: + runs-on: ubuntu-latest + name: Build and run Comparator + steps: + - name: Checkout Formal Conjectures + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + + - name: Install elan + run: | + set -o pipefail + curl -sSfL https://github.com/leanprover/elan/releases/download/v1.4.2/elan-x86_64-unknown-linux-gnu.tar.gz | tar xz + ./elan-init -y --default-toolchain none + echo "$HOME/.elan/bin" >> "$GITHUB_PATH" + + - name: Verify pins and trusted-source fingerprint + run: | + python3 - <<'PY' + import hashlib + import json + from pathlib import Path + + root = Path('comparator/pilots/fc_sum_of_three_cubes') + provenance = json.loads((root / 'provenance.json').read_text()) + assert (root / 'lean-toolchain').read_text().strip() == provenance['target']['lean_toolchain'] + lakefile = (root / 'lakefile.toml').read_text() + assert provenance['target']['mathlib_revision'] in lakefile + trusted = (root / 'ChallengeDeps.lean').read_text() + '\n' + (root / 'Challenge.lean').read_text() + actual = hashlib.sha256(trusted.encode()).hexdigest() + assert actual == provenance['trusted_files_sha256'], (actual, provenance['trusted_files_sha256']) + print(actual) + PY + + - name: Build the Lean 4.33 workspace + working-directory: comparator/pilots/fc_sum_of_three_cubes + run: | + lake update + lake exe cache get + lake build + + - name: Build pinned Comparator and Lean 4.33 exporter + env: + COMPARATOR_REV: 575674928e239f5bc452aab72d1dd7b0f1326494 + run: | + git clone https://github.com/leanprover/comparator.git "$RUNNER_TEMP/comparator" + git -C "$RUNNER_TEMP/comparator" checkout "$COMPARATOR_REV" + git clone https://github.com/leanprover/lean4export.git "$RUNNER_TEMP/lean4export" + git -C "$RUNNER_TEMP/lean4export" checkout v4.33.0 + (cd "$RUNNER_TEMP/comparator" && lake build comparator) + (cd "$RUNNER_TEMP/lean4export" && lake build lean4export) + + - name: Run Comparator smoke test + working-directory: comparator/pilots/fc_sum_of_three_cubes + run: | + export COMPARATOR_BIN="$RUNNER_TEMP/comparator/.lake/build/bin/comparator" + export COMPARATOR_LANDRUN="$RUNNER_TEMP/comparator/scripts/fake-landrun.sh" + export COMPARATOR_LEAN4EXPORT="$RUNNER_TEMP/lean4export/.lake/build/bin/lean4export" + lake test diff --git a/comparator/pilots/fc_sum_of_three_cubes/Challenge.lean b/comparator/pilots/fc_sum_of_three_cubes/Challenge.lean new file mode 100644 index 0000000000..7aa5487475 --- /dev/null +++ b/comparator/pilots/fc_sum_of_three_cubes/Challenge.lean @@ -0,0 +1,41 @@ +/- +Copyright 2026 The Formal Conjectures Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +-/ + +import ChallengeDeps + +/-! +Lean 4.33 conformance pilot for the Formal Conjectures to LeanEval importer. + +The first target is a solved smoke theorem. The second preserves the +`answer(sorry)` shape of an open conjecture as a Comparator definition hole. +-/ + +namespace SumOfThreeCubes + +def isSumOfThreeCubes_iff_mod_9_answer : Prop := sorry + +theorem isSumOfThreeCubes_2 : + IsSumOfThreeCubes (2 : ℤ) := by + sorry + +theorem isSumOfThreeCubes_iff_mod_9 : + isSumOfThreeCubes_iff_mod_9_answer ↔ + ∀ n : ℤ, + IsSumOfThreeCubes n ↔ + ¬(n ≡ 4 [ZMOD 9] ∨ n ≡ 5 [ZMOD 9]) := by + sorry + +end SumOfThreeCubes diff --git a/comparator/pilots/fc_sum_of_three_cubes/ChallengeDeps.lean b/comparator/pilots/fc_sum_of_three_cubes/ChallengeDeps.lean new file mode 100644 index 0000000000..2e0c388af8 --- /dev/null +++ b/comparator/pilots/fc_sum_of_three_cubes/ChallengeDeps.lean @@ -0,0 +1,30 @@ +/- +Copyright 2026 The Formal Conjectures Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +-/ + +import Mathlib + +/-! +Trusted dependency copied from +`FormalConjectures/Wikipedia/SumOfThreeCubes.lean`. +-/ + +namespace SumOfThreeCubes + +/-- The predicate that `n : R` is a sum of three cubes. -/ +def IsSumOfThreeCubes {R : Type*} [Ring R] (n : R) : Prop := + ∃ x y z : R, n = x ^ 3 + y ^ 3 + z ^ 3 + +end SumOfThreeCubes diff --git a/comparator/pilots/fc_sum_of_three_cubes/README.md b/comparator/pilots/fc_sum_of_three_cubes/README.md new file mode 100644 index 0000000000..f4ffb02dc3 --- /dev/null +++ b/comparator/pilots/fc_sum_of_three_cubes/README.md @@ -0,0 +1,66 @@ +# FC Sum of Three Cubes: Lean 4.33 Comparator pilot + +This is a vendored, reviewable conformance pilot for the Formal Conjectures to +LeanEval boundary. It does not upgrade the Formal Conjectures repository and it +does not claim to solve the open sum-of-three-cubes conjecture. + +## Pins + +- Formal Conjectures source commit: `9f5ee773841921f460b4a26a3552f5eca4accaa0` +- LeanEval reference commit: `7699436464052268e6c04b41554bfbc2c6908ec5` +- Lean: `leanprover/lean4:v4.33.0` +- Mathlib: `6f1ef4e5dd604a435bddba4747b13970cd65d2a1` + +`provenance.json` records the source declarations, transformation log, target +pins, and a SHA-256 fingerprint over the exact trusted `ChallengeDeps.lean` and +`Challenge.lean` files. + +## What the workspace checks + +The workspace follows LeanEval's generated structure: + +- `ChallengeDeps.lean` contains the trusted predicate copied from Formal + Conjectures. +- `Challenge.lean` contains a solved smoke theorem and the open conjecture's + `answer(sorry)` slot, hoisted into a `Prop`-valued definition hole. +- `Submission.lean` supplies an actual proof of the solved smoke theorem. +- `Solution.lean` is fixed and delegates the challenge names to the submission. +- `config.json` asks Comparator to check both theorem names and the definition + hole under LeanEval's permitted-axiom policy. + +The open-conjecture smoke submission intentionally defines the answer hole to +be the proposition itself and proves the bridge by `Iff.rfl`. Comparator should +accept that declaration-level shape. It is not a mathematical resolution and a +human semantic reviewer must reject it. Keeping this case explicit tests the +reason definition answers require a separate review stage. + +## Build + +```bash +lake exe cache get +lake build +``` + +## Comparator smoke test + +Build Comparator and a `lean4export` compatible with Lean 4.33, then run: + +```bash +COMPARATOR_BIN=/absolute/path/to/comparator \ +COMPARATOR_LANDRUN=/absolute/path/to/comparator/scripts/fake-landrun.sh \ +COMPARATOR_LEAN4EXPORT=/absolute/path/to/lean4export \ +lake test +``` + +The fake landrun is acceptable only for this trusted development smoke test. A +real submission service must use the production sandbox and preserve +Comparator's clean-build assumptions. + +## What this does not establish + +A passing run establishes that one vendored FC problem shape builds under +LeanEval's Lean 4.33 and Mathlib pins and crosses Comparator's theorem and +definition-hole interfaces. It does not establish that the general importer is +complete, that arbitrary FC dependencies port automatically, that the open +conjecture is solved, or that maintainers should accept the problem into a +frozen release. diff --git a/comparator/pilots/fc_sum_of_three_cubes/Solution.lean b/comparator/pilots/fc_sum_of_three_cubes/Solution.lean new file mode 100644 index 0000000000..e67b9d07fb --- /dev/null +++ b/comparator/pilots/fc_sum_of_three_cubes/Solution.lean @@ -0,0 +1,36 @@ +/- +Copyright 2026 The Formal Conjectures Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +-/ + +import ChallengeDeps +import Submission + +namespace SumOfThreeCubes + +@[reducible] noncomputable def isSumOfThreeCubes_iff_mod_9_answer : Prop := + Submission.SumOfThreeCubes.isSumOfThreeCubes_iff_mod_9_answer + +theorem isSumOfThreeCubes_2 : + IsSumOfThreeCubes (2 : ℤ) := + Submission.SumOfThreeCubes.isSumOfThreeCubes_2 + +theorem isSumOfThreeCubes_iff_mod_9 : + isSumOfThreeCubes_iff_mod_9_answer ↔ + ∀ n : ℤ, + IsSumOfThreeCubes n ↔ + ¬(n ≡ 4 [ZMOD 9] ∨ n ≡ 5 [ZMOD 9]) := + Submission.SumOfThreeCubes.isSumOfThreeCubes_iff_mod_9 + +end SumOfThreeCubes diff --git a/comparator/pilots/fc_sum_of_three_cubes/Submission.lean b/comparator/pilots/fc_sum_of_three_cubes/Submission.lean new file mode 100644 index 0000000000..c037dfb478 --- /dev/null +++ b/comparator/pilots/fc_sum_of_three_cubes/Submission.lean @@ -0,0 +1,48 @@ +/- +Copyright 2026 The Formal Conjectures Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +-/ + +import ChallengeDeps +import Submission.Helpers + +/-! +Trusted smoke submission for the integration harness. + +The open-conjecture target deliberately uses the proposition itself as the +definition-hole value and proves the bridge by reflexivity. Comparator should +accept this shape, while a human semantic reviewer must reject it as a +mathematical resolution. That is the definition-hole threat model this pilot +is meant to preserve. +-/ + +namespace Submission.SumOfThreeCubes + +def isSumOfThreeCubes_iff_mod_9_answer : Prop := + ∀ n : ℤ, + _root_.SumOfThreeCubes.IsSumOfThreeCubes n ↔ + ¬(n ≡ 4 [ZMOD 9] ∨ n ≡ 5 [ZMOD 9]) + +theorem isSumOfThreeCubes_2 : + _root_.SumOfThreeCubes.IsSumOfThreeCubes (2 : ℤ) := by + exact ⟨1, 1, 0, by norm_num⟩ + +theorem isSumOfThreeCubes_iff_mod_9 : + isSumOfThreeCubes_iff_mod_9_answer ↔ + ∀ n : ℤ, + _root_.SumOfThreeCubes.IsSumOfThreeCubes n ↔ + ¬(n ≡ 4 [ZMOD 9] ∨ n ≡ 5 [ZMOD 9]) := + Iff.rfl + +end Submission.SumOfThreeCubes diff --git a/comparator/pilots/fc_sum_of_three_cubes/Submission/Helpers.lean b/comparator/pilots/fc_sum_of_three_cubes/Submission/Helpers.lean new file mode 100644 index 0000000000..4d2e18725c --- /dev/null +++ b/comparator/pilots/fc_sum_of_three_cubes/Submission/Helpers.lean @@ -0,0 +1,23 @@ +/- +Copyright 2026 The Formal Conjectures Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +-/ + +import Mathlib + +/-! Helper lemmas for the trusted smoke submission. -/ + +namespace Submission + +end Submission diff --git a/comparator/pilots/fc_sum_of_three_cubes/WorkspaceTest.lean b/comparator/pilots/fc_sum_of_three_cubes/WorkspaceTest.lean new file mode 100644 index 0000000000..7ce5489c7a --- /dev/null +++ b/comparator/pilots/fc_sum_of_three_cubes/WorkspaceTest.lean @@ -0,0 +1,36 @@ +/- +Copyright 2026 The Formal Conjectures Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +-/ + +import Lean + +open Lean + +/-- Run Comparator on this workspace's `config.json`, so that `lake test` +is the integration check. The binary comes from `PATH`, or from +`COMPARATOR_BIN`. -/ +def main : IO UInt32 := do + let comparatorBin := (← IO.getEnv "COMPARATOR_BIN").getD "comparator" + try + let child ← IO.Process.spawn { + cmd := "lake" + args := #["env", comparatorBin, "config.json"] + } + child.wait + catch err => + IO.eprintln s!"Failed to run Comparator via `{comparatorBin}`." + IO.eprintln "Install Comparator with a Lean 4.33-compatible lean4export, or set COMPARATOR_BIN." + IO.eprintln s!"Original error: {err}" + pure 1 diff --git a/comparator/pilots/fc_sum_of_three_cubes/config.json b/comparator/pilots/fc_sum_of_three_cubes/config.json new file mode 100644 index 0000000000..cb99776f93 --- /dev/null +++ b/comparator/pilots/fc_sum_of_three_cubes/config.json @@ -0,0 +1,17 @@ +{ + "challenge_module": "Challenge", + "solution_module": "Solution", + "theorem_names": [ + "SumOfThreeCubes.isSumOfThreeCubes_2", + "SumOfThreeCubes.isSumOfThreeCubes_iff_mod_9" + ], + "definition_names": [ + "SumOfThreeCubes.isSumOfThreeCubes_iff_mod_9_answer" + ], + "permitted_axioms": [ + "propext", + "Quot.sound", + "Classical.choice" + ], + "enable_nanoda": false +} diff --git a/comparator/pilots/fc_sum_of_three_cubes/lakefile.toml b/comparator/pilots/fc_sum_of_three_cubes/lakefile.toml new file mode 100644 index 0000000000..18ef8135ed --- /dev/null +++ b/comparator/pilots/fc_sum_of_three_cubes/lakefile.toml @@ -0,0 +1,27 @@ +name = "fc_sum_of_three_cubes_lean433" +testDriver = "workspace_test" +defaultTargets = ["ChallengeDeps", "Challenge", "Solution", "Submission"] + +[leanOptions] +autoImplicit = false + +[[require]] +name = "mathlib" +git = "https://github.com/leanprover-community/mathlib4.git" +rev = "6f1ef4e5dd604a435bddba4747b13970cd65d2a1" + +[[lean_lib]] +name = "ChallengeDeps" + +[[lean_lib]] +name = "Challenge" + +[[lean_lib]] +name = "Solution" + +[[lean_lib]] +name = "Submission" + +[[lean_exe]] +name = "workspace_test" +root = "WorkspaceTest" diff --git a/comparator/pilots/fc_sum_of_three_cubes/lean-toolchain b/comparator/pilots/fc_sum_of_three_cubes/lean-toolchain new file mode 100644 index 0000000000..025e59548e --- /dev/null +++ b/comparator/pilots/fc_sum_of_three_cubes/lean-toolchain @@ -0,0 +1 @@ +leanprover/lean4:v4.33.0 diff --git a/comparator/pilots/fc_sum_of_three_cubes/provenance.json b/comparator/pilots/fc_sum_of_three_cubes/provenance.json new file mode 100644 index 0000000000..fdc618c65a --- /dev/null +++ b/comparator/pilots/fc_sum_of_three_cubes/provenance.json @@ -0,0 +1,35 @@ +{ + "schema_version": 1, + "source": { + "repository": "google-deepmind/formal-conjectures", + "commit": "9f5ee773841921f460b4a26a3552f5eca4accaa0", + "path": "FormalConjectures/Wikipedia/SumOfThreeCubes.lean", + "blob_sha": "f36a362c53f254a14ed0e0fb9239abefb4b762f1", + "declarations": [ + "SumOfThreeCubes.IsSumOfThreeCubes", + "SumOfThreeCubes.isSumOfThreeCubes_2", + "SumOfThreeCubes.isSumOfThreeCubes_iff_mod_9" + ], + "subset": "Subsets.FC100OpenSet1.problems", + "category": "research open" + }, + "target": { + "repository": "leanprover/lean-eval", + "commit": "7699436464052268e6c04b41554bfbc2c6908ec5", + "lean_toolchain": "leanprover/lean4:v4.33.0", + "mathlib_revision": "6f1ef4e5dd604a435bddba4747b13970cd65d2a1", + "workspace_shape": "ChallengeDeps/Challenge/Submission/Solution" + }, + "transformations": [ + "removed Formal Conjectures category attributes", + "copied the trusted IsSumOfThreeCubes dependency", + "hoisted answer(sorry) into a Prop-valued definition hole", + "added a fixed Solution adapter delegating to Submission" + ], + "trusted_files_sha256": "ad2ddda464477fd13661730acda71e4e13eaf2e3f90c97a3821290ae9b82b4b3", + "review": { + "mathematical_status": "open", + "smoke_submission_is_not_a_resolution": true, + "human_definition_value_review_required": true + } +} From 9044c9a835808122c53ab4973c0f53b78fb9cbe3 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:23:49 -0400 Subject: [PATCH 04/70] ci: pin Lean 4.33 exporter commit --- .github/workflows/comparator-lean-4-33.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/comparator-lean-4-33.yml b/.github/workflows/comparator-lean-4-33.yml index fd38d1339f..9b00f64370 100644 --- a/.github/workflows/comparator-lean-4-33.yml +++ b/.github/workflows/comparator-lean-4-33.yml @@ -59,11 +59,12 @@ jobs: - name: Build pinned Comparator and Lean 4.33 exporter env: COMPARATOR_REV: 575674928e239f5bc452aab72d1dd7b0f1326494 + LEAN4EXPORT_REV: 15f6055e299ad5b89345e533cc2192f4cc00f659 run: | git clone https://github.com/leanprover/comparator.git "$RUNNER_TEMP/comparator" git -C "$RUNNER_TEMP/comparator" checkout "$COMPARATOR_REV" git clone https://github.com/leanprover/lean4export.git "$RUNNER_TEMP/lean4export" - git -C "$RUNNER_TEMP/lean4export" checkout v4.33.0 + git -C "$RUNNER_TEMP/lean4export" checkout "$LEAN4EXPORT_REV" (cd "$RUNNER_TEMP/comparator" && lake build comparator) (cd "$RUNNER_TEMP/lean4export" && lake build lean4export) From 3311503d6734b36cb799125c44f8a732b6578df3 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:24:10 -0400 Subject: [PATCH 05/70] docs: record exact Comparator verifier pins --- comparator/pilots/fc_sum_of_three_cubes/provenance.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/comparator/pilots/fc_sum_of_three_cubes/provenance.json b/comparator/pilots/fc_sum_of_three_cubes/provenance.json index fdc618c65a..d559d2c6af 100644 --- a/comparator/pilots/fc_sum_of_three_cubes/provenance.json +++ b/comparator/pilots/fc_sum_of_three_cubes/provenance.json @@ -18,6 +18,8 @@ "commit": "7699436464052268e6c04b41554bfbc2c6908ec5", "lean_toolchain": "leanprover/lean4:v4.33.0", "mathlib_revision": "6f1ef4e5dd604a435bddba4747b13970cd65d2a1", + "comparator_commit": "575674928e239f5bc452aab72d1dd7b0f1326494", + "lean4export_commit": "15f6055e299ad5b89345e533cc2192f4cc00f659", "workspace_shape": "ChallengeDeps/Challenge/Submission/Solution" }, "transformations": [ From 92e97ead99036ad644065ae3b6dec9630b5d4275 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:24:35 -0400 Subject: [PATCH 06/70] docs: list exact Lean 4.33 verifier pins --- comparator/pilots/fc_sum_of_three_cubes/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/comparator/pilots/fc_sum_of_three_cubes/README.md b/comparator/pilots/fc_sum_of_three_cubes/README.md index f4ffb02dc3..673ac59fe2 100644 --- a/comparator/pilots/fc_sum_of_three_cubes/README.md +++ b/comparator/pilots/fc_sum_of_three_cubes/README.md @@ -10,6 +10,8 @@ does not claim to solve the open sum-of-three-cubes conjecture. - LeanEval reference commit: `7699436464052268e6c04b41554bfbc2c6908ec5` - Lean: `leanprover/lean4:v4.33.0` - Mathlib: `6f1ef4e5dd604a435bddba4747b13970cd65d2a1` +- Comparator: `575674928e239f5bc452aab72d1dd7b0f1326494` +- Lean 4.33 exporter: `15f6055e299ad5b89345e533cc2192f4cc00f659` `provenance.json` records the source declarations, transformation log, target pins, and a SHA-256 fingerprint over the exact trusted `ChallengeDeps.lean` and @@ -43,7 +45,7 @@ lake build ## Comparator smoke test -Build Comparator and a `lean4export` compatible with Lean 4.33, then run: +Build the pinned Comparator and Lean 4.33 exporter recorded above, then run: ```bash COMPARATOR_BIN=/absolute/path/to/comparator \ From 10ea6d0f658ffd5a6c63ad878495e5d64f55eb61 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:30:59 -0400 Subject: [PATCH 07/70] ci: pin Comparator to its final Lean 4.33 commit --- .github/workflows/comparator-lean-4-33.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/comparator-lean-4-33.yml b/.github/workflows/comparator-lean-4-33.yml index 9b00f64370..6627abaf93 100644 --- a/.github/workflows/comparator-lean-4-33.yml +++ b/.github/workflows/comparator-lean-4-33.yml @@ -56,22 +56,18 @@ jobs: lake exe cache get lake build - - name: Build pinned Comparator and Lean 4.33 exporter + - name: Build the pinned Lean 4.33 verifier stack env: - COMPARATOR_REV: 575674928e239f5bc452aab72d1dd7b0f1326494 - LEAN4EXPORT_REV: 15f6055e299ad5b89345e533cc2192f4cc00f659 + COMPARATOR_REV: c0c5a52d2aff92b457c3e5ed4a68c1ebc5795809 run: | git clone https://github.com/leanprover/comparator.git "$RUNNER_TEMP/comparator" git -C "$RUNNER_TEMP/comparator" checkout "$COMPARATOR_REV" - git clone https://github.com/leanprover/lean4export.git "$RUNNER_TEMP/lean4export" - git -C "$RUNNER_TEMP/lean4export" checkout "$LEAN4EXPORT_REV" - (cd "$RUNNER_TEMP/comparator" && lake build comparator) - (cd "$RUNNER_TEMP/lean4export" && lake build lean4export) + (cd "$RUNNER_TEMP/comparator" && lake build comparator lean4export) - name: Run Comparator smoke test working-directory: comparator/pilots/fc_sum_of_three_cubes run: | export COMPARATOR_BIN="$RUNNER_TEMP/comparator/.lake/build/bin/comparator" export COMPARATOR_LANDRUN="$RUNNER_TEMP/comparator/scripts/fake-landrun.sh" - export COMPARATOR_LEAN4EXPORT="$RUNNER_TEMP/lean4export/.lake/build/bin/lean4export" + export COMPARATOR_LEAN4EXPORT="$RUNNER_TEMP/comparator/.lake/packages/lean4export/.lake/build/bin/lean4export" lake test From 3a94f9b8bf6d5bb64e23069901818199691d0e01 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:31:22 -0400 Subject: [PATCH 08/70] docs: record final Lean 4.33 Comparator commit --- comparator/pilots/fc_sum_of_three_cubes/provenance.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comparator/pilots/fc_sum_of_three_cubes/provenance.json b/comparator/pilots/fc_sum_of_three_cubes/provenance.json index d559d2c6af..763cf0f2ef 100644 --- a/comparator/pilots/fc_sum_of_three_cubes/provenance.json +++ b/comparator/pilots/fc_sum_of_three_cubes/provenance.json @@ -18,7 +18,7 @@ "commit": "7699436464052268e6c04b41554bfbc2c6908ec5", "lean_toolchain": "leanprover/lean4:v4.33.0", "mathlib_revision": "6f1ef4e5dd604a435bddba4747b13970cd65d2a1", - "comparator_commit": "575674928e239f5bc452aab72d1dd7b0f1326494", + "comparator_commit": "c0c5a52d2aff92b457c3e5ed4a68c1ebc5795809", "lean4export_commit": "15f6055e299ad5b89345e533cc2192f4cc00f659", "workspace_shape": "ChallengeDeps/Challenge/Submission/Solution" }, From 1c2a1a822b4ff6d53158cbdc67f331d5276f4617 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:31:47 -0400 Subject: [PATCH 09/70] docs: identify the final Lean 4.33 Comparator revision --- comparator/pilots/fc_sum_of_three_cubes/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/comparator/pilots/fc_sum_of_three_cubes/README.md b/comparator/pilots/fc_sum_of_three_cubes/README.md index 673ac59fe2..af19a78df0 100644 --- a/comparator/pilots/fc_sum_of_three_cubes/README.md +++ b/comparator/pilots/fc_sum_of_three_cubes/README.md @@ -10,7 +10,7 @@ does not claim to solve the open sum-of-three-cubes conjecture. - LeanEval reference commit: `7699436464052268e6c04b41554bfbc2c6908ec5` - Lean: `leanprover/lean4:v4.33.0` - Mathlib: `6f1ef4e5dd604a435bddba4747b13970cd65d2a1` -- Comparator: `575674928e239f5bc452aab72d1dd7b0f1326494` +- Comparator, final Lean 4.33 commit: `c0c5a52d2aff92b457c3e5ed4a68c1ebc5795809` - Lean 4.33 exporter: `15f6055e299ad5b89345e533cc2192f4cc00f659` `provenance.json` records the source declarations, transformation log, target @@ -45,7 +45,8 @@ lake build ## Comparator smoke test -Build the pinned Comparator and Lean 4.33 exporter recorded above, then run: +Build the pinned Comparator commit recorded above. Its manifest pins the matching +Lean 4.33 exporter, then run: ```bash COMPARATOR_BIN=/absolute/path/to/comparator \ From 7fbcac025647ae83e2ed246ba5cb5f774c948c94 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:31:09 -0400 Subject: [PATCH 10/70] Generate Mathlib-only Challenges The generated workspace required this repository so Challenge.lean could import the problem's own module. lean-eval vendors its problems and cannot fetch Formal Conjectures at evaluation time, so the statement's dependencies have to travel with the workspace. The hand-built pilot in this branch already had the right shape, `import ChallengeDeps` against Mathlib alone; the generator did not, and the two disagreeing was this branch's main gap. comparator_facts now reports the FC-local closure of a statement in dependency order, and the generator copies it into ChallengeDeps.lean, each declaration inside its own section carrying the open, variable, universe, set_option and local notation in force where it was written. Copying is a construction and its failure modes are ones Lean sees and a reader does not, so --verify elaborates the generated Challenge against this checkout's Mathlib, which is the revision the workspace pins. It found four defects that no amount of reading would have: - Elaborator artifacts. _proof_N and .match_N have no source range because they have no source; copying the parent regenerates them. They are reported separately so a generated constant with no copied ancestor is an error rather than a silent omission. The heaviest FC100 closure is 10 real declarations, not 19. - Constructors and where-auxiliaries carry ranges inside their parent's. EdgeN.mk covers line 88 of a structure spanning 83 to 93, and pmSumListAux._sparseCasesOn_1 has exactly its parent's range, so copying either duplicated a declaration or sliced a fragment of one. - An inductive's fields live in its constructor, so the closure ordered EdgeN before the V it uses and the copy did not elaborate. - answer(False) reached Challenge.lean verbatim. answer is this repository's elaborator, and a statement that already carries its answer has no sorry slot for hoist_answers to remove. Unwrapping to the bare term is faithful: in the default postpone mode the elaborator elaborates the term and attaches an annotation. Copied dependencies keep every attribute except this repository's own, since dropping simp or reducible changes how the declarations after them elaborate. provenance.json is now generated rather than only hand-written for the pilot. The lakefile's requirement was the only record of which commit a statement came from, and removing it would otherwise have left the workspace untraceable. Verified on the ten declarations covering every FC100 dependency pattern and both hole kinds: all ten generate and elaborate. 35 adapter tests pass, 58 across scripts/, ruff no worse than before, lake --wfail build comparator_facts clean. --- scripts/comparator_facts.lean | 80 ++++- scripts/make_comparator_workspace.py | 375 ++++++++++++++++++++-- scripts/test_make_comparator_workspace.py | 141 ++++++++ 3 files changed, 562 insertions(+), 34 deletions(-) diff --git a/scripts/comparator_facts.lean b/scripts/comparator_facts.lean index b6167485b1..2287b6fe65 100644 --- a/scripts/comparator_facts.lean +++ b/scripts/comparator_facts.lean @@ -59,6 +59,45 @@ partial def lambdaArity : Expr → Nat def binderJson (name : Name) (bi : BinderInfo) : Json := Json.mkObj [("name", toJson name.toString), ("explicit", toJson bi.isExplicit)] +def moduleOf (env : Environment) (n : Name) : String := + match env.getModuleIdxFor? n with + | some idx => (env.header.moduleNames[idx.toNat]?.getD Name.anonymous).toString + | none => "" + +/-- Declared by this repository, as opposed to arriving with `import Mathlib`. -/ +def isFCLocal (env : Environment) (n : Name) : Bool := + (moduleOf env n).startsWith "FormalConjectures" + +/-- The FC-local constants a declaration needs, dependencies before dependents. + +Post-order over the dependency graph, expanding through both the type and the +value of each FC-local constant: a definition's body names constants its type +does not, and `ChallengeDeps` has to carry them or the copy will not elaborate. +Mathlib and core constants are not expanded, since they arrive with +`import Mathlib`. -/ +partial def fcOrder (env : Environment) (n : Name) + (seen : Std.HashSet Name) (acc : Array Name) : Std.HashSet Name × Array Name := + if seen.contains n then (seen, acc) else + let seen := seen.insert n + match env.find? n with + | none => (seen, acc) + | some info => + let fromValue := match info.value? with + | some v => v.getUsedConstants + | none => #[] + -- An inductive has no value, and its fields live in the constructor + -- rather than in its own type: `structure EdgeN (N D : Nat) where u : V N` + -- has type `Nat → Nat → Type`, which never mentions `V`. Without the + -- constructors here the closure still contains `V`, reached some other + -- way, but orders it after `EdgeN`, and the copy does not elaborate. + let fromCtors := match info with + | .inductInfo val => val.ctors.toArray + | _ => #[] + let children := (info.type.getUsedConstants ++ fromValue ++ fromCtors).filter + fun c => isFCLocal env c && c != n + let (seen, acc) := children.foldl (fun p c => fcOrder env c p.1 p.2) (seen, acc) + (seen, acc.push n) + unsafe def runWithImports {α : Type} (moduleNames : Array Name) (actionToRun : MetaM α) : IO α := do initSearchPath (← getBuildDir) @@ -123,18 +162,43 @@ where (xs.extract 0 arity).mapM fun x => do let d ← x.fvarId!.getDecl pure (binderJson d.userName d.binderInfo) - let rangeJson := match ranges with - | some r => Json.mkObj [ - ("startLine", toJson r.range.pos.line), - ("startColumn", toJson r.range.pos.column), - ("endLine", toJson r.range.endPos.line), - ("endColumn", toJson r.range.endPos.column)] - | none => Json.null + let rangeJson := rangeToJson ranges + -- Only the statement's dependencies: the proof is replaced by `sorry` in + -- the generated Challenge, so nothing the value names has to be carried. + let direct := info.type.getUsedConstants.filter (isFCLocal env) + let (_, ordered) := direct.foldl (fun p c => fcOrder env c p.1 p.2) + (({} : Std.HashSet Name), (#[] : Array Name)) + -- The equation compiler and `decide` leave constants like + -- `Finset.greedySidon.aux._proof_1` and `.match_1` in the closure. They + -- have no source range because they have no source: copying the parent + -- declaration's text regenerates them. Emit them separately so the + -- generator can check each one has an ancestor that is being copied, + -- rather than dropping them silently. + let mut deps := #[] + let mut generated := #[] + for d in ordered.filter (· != name) do + match ← findDeclarationRanges? d with + | some r => + deps := deps.push <| Json.mkObj [ + ("name", toJson d.toString), + ("module", toJson (moduleOf env d)), + ("range", rangeToJson (some r))] + | none => generated := generated.push (toJson d.toString) let payload := Json.mkObj [ ("declaration", toJson decl), ("name", toJson name.toString), ("range", rangeJson), ("binders", toJson binders.toList), - ("answerTypes", toJson answerTypes.toList)] + ("answerTypes", toJson answerTypes.toList), + ("dependencies", toJson deps.toList), + ("generatedDependencies", toJson generated.toList)] IO.println payload.pretty return 0 + rangeToJson (ranges : Option DeclarationRanges) : Json := + match ranges with + | some r => Json.mkObj [ + ("startLine", toJson r.range.pos.line), + ("startColumn", toJson r.range.pos.column), + ("endLine", toJson r.range.endPos.line), + ("endColumn", toJson r.range.endPos.column)] + | none => Json.null diff --git a/scripts/make_comparator_workspace.py b/scripts/make_comparator_workspace.py index ce9946556e..42a93102dc 100644 --- a/scripts/make_comparator_workspace.py +++ b/scripts/make_comparator_workspace.py @@ -6,17 +6,23 @@ permitted axioms. This script generates that shape for one Formal Conjectures declaration. -Challenge.lean imports the problem's own module. lean-eval's generated -Challenge is one `import Mathlib` and one statement, because its problems are -authored self-contained; this repository's are not, so the import here is the -problem's module and the statement's context comes with it. Only what Lean -scopes to a file has to be copied: `open`, `variable`, `universe`, -`set_option` and `local notation`. +The generated workspace requires Mathlib and nothing else. lean-eval vendors +its problems, so a Challenge cannot fetch this repository at evaluation time, +which rules out importing the problem's own module. This repository's +statements are not authored self-contained, so the declarations a statement +needs are copied into `ChallengeDeps.lean` instead, dependencies first, each +carrying the `open`, `variable`, `universe`, `set_option` and `local notation` +in force where it was written. + +Copying is a construction and it can be wrong in ways only Lean sees, so +`--verify` builds the generated workspace before you trust it. Layout produced: // - lakefile.toml pins: this checkout's Mathlib rev and FC commit + lakefile.toml pins: this checkout's Mathlib rev + ChallengeDeps.lean the statement's Formal Conjectures closure, copied, + importing Mathlib alone Challenge.lean the import, the file-scoped preamble, the target statement with attributes stripped and its proof replaced by `sorry`, and each `answer(sorry)` hoisted @@ -276,6 +282,179 @@ def strip_decorations(block_text): block_text = stripped +# Attributes this repository defines. A generated workspace requires Mathlib +# and nothing else, so these have to go; everything else has to stay. +FC_ATTRIBUTES = ("category", "AMS", "formal_proof") + + +def strip_fc_attributes(block_text): + """Remove this repository's own attributes from a copied declaration. + + Unlike `strip_decorations`, which clears every attribute off the target + statement, this keeps the rest. A dependency is copied to be elaborated, + not restated, and dropping `simp`, `reducible` or `instance` attributes + changes how the declarations after it in the same closure elaborate. + """ + + def replace(match): + inner = match.group(1) + # Nested brackets mean an argument this simple split would cut in + # half, so leave the whole attribute alone rather than mangle it. + if "[" in inner: + return match.group(0) + kept = [ + part.strip() + for part in inner.split(",") + if part.strip() and part.strip().split()[0] not in FC_ATTRIBUTES + ] + return f"@[{', '.join(kept)}]" if kept else "" + + text = re.sub(r"@\[([^\]]*)\]", replace, block_text) + # An attribute line that emptied out leaves a blank line behind. + return re.sub(r"^[ \t]*\n", "", text, flags=re.MULTILINE) + + +def module_source_path(module): + """The file declaring a dotted Lean module name, undoing guillemets.""" + parts = [ + component[1:-1] if component.startswith("«") else component + for component in module.split(".") + ] + path = ROOT.joinpath(*parts).with_suffix(".lean") + if not path.is_file(): + raise SystemExit(f"{module}: no source file at {path}") + return path + + +def slice_range(lines, source_range): + """The source text a declaration range covers, and the line it starts on. + + `open X in` binds to the declaration below it but sits above what the + range covers in some toolchains, so it is pulled in when present. + """ + lo, hi = source_range["startLine"], source_range["endLine"] + end_column = source_range.get("endColumn") + while ( + lo > 1 + and lines[lo - 2].rstrip().endswith(" in") + and KEEP_LOOSE.match(lines[lo - 2]) + ): + lo -= 1 + sliced = lines[lo - 1 : hi] + if end_column is not None and sliced: + sliced = sliced[:-1] + [sliced[-1][:end_column]] + return "\n".join(sliced), lo + + +def challenge_deps(dependencies, generated, declaration, opened_namespaces=()): + """One Mathlib-only module carrying a declaration's FC-local closure. + + lean-eval vendors problems, so a generated Challenge cannot fetch this + repository at evaluation time and has to stand on Mathlib alone. That + rules out importing the problem's own module, and brings back the failure + modes an import does not have: file-scoped `open` and `variable` lost, + `local notation` unrecognised, a namespace swallowing what follows. + + So each declaration is emitted inside its own `section`, carrying the + preamble in force where it was written and reopening the namespace it was + written in. That is a construction, not a proof, and the only check that + covers every one of those failure modes at once is building the generated + workspace, which `--verify` does. + """ + copied = [dep["name"] for dep in dependencies] + orphans = [ + name + for name in generated + if not any(name.startswith(parent + ".") for parent in copied) + ] + if orphans: + raise SystemExit( + f"{declaration}: {len(orphans)} elaborator-generated constant(s) " + "have no copied ancestor, so copying the closure would not " + f"reproduce them: {', '.join(orphans[:5])}" + ) + + # A constructor, a `where` auxiliary and a `_sparseCasesOn` all carry a + # source range inside the declaration that produces them, so copying them + # in their own right either duplicates a declaration or slices a fragment + # of one. `MonochromaticQuantumGraph.EdgeN.mk` covers line 88 of a + # structure spanning 83 to 93; `pmSumListAux._sparseCasesOn_1` has exactly + # its parent's range. Copying the outer declaration reproduces both. + def covered_by_another(dep): + inner = dep["range"] + for other in dependencies: + if other is dep or other["module"] != dep["module"]: + continue + outer = other["range"] + if outer is None or inner is None: + continue + if not ( + outer["startLine"] <= inner["startLine"] + and outer["endLine"] >= inner["endLine"] + ): + continue + same_span = ( + outer["startLine"] == inner["startLine"] + and outer["endLine"] == inner["endLine"] + ) + # A tie on the span is broken by name: the parent is the prefix. + if not same_span or len(other["name"]) < len(dep["name"]): + return True + return False + + subsumed = [dep["name"] for dep in dependencies if covered_by_another(dep)] + dependencies = [dep for dep in dependencies if dep["name"] not in subsumed] + + blocks, provenance = [], [] + for dep in dependencies: + if dep["range"] is None: + raise SystemExit(f"{declaration}: {dep['name']} has no source range") + path = module_source_path(dep["module"]) + lines = path.read_text(encoding="utf-8").split("\n") + text, start = slice_range(lines, dep["range"]) + preamble, namespaces = file_scoped_preamble(lines, start) + body = strip_fc_attributes(text).strip("\n") + if not body: + raise SystemExit(f"{declaration}: {dep['name']} sliced to nothing") + namespace = ".".join(namespaces) + chunk = [f"-- {dep['name']}, from {path.relative_to(ROOT)}", "section"] + chunk += preamble + if namespace: + chunk.append(f"namespace {namespace}") + chunk += ["", body, ""] + if namespace: + chunk.append(f"end {namespace}") + chunk.append("end") + blocks.append("\n".join(chunk)) + provenance.append(dep["name"]) + + # Challenge.lean reopens the namespace stack the target sat in, so its + # statement can name siblings short. `open` on a namespace nothing has + # declared is an error, and with the problem's module no longer imported + # only the copied declarations can declare one. An empty namespace block + # is enough to make the name exist. + declared_namespaces = { + name.rsplit(".", 1)[0] for name in provenance if "." in name + } + for depth in range(len(opened_namespaces)): + prefix = ".".join(opened_namespaces[: depth + 1]) + if not any( + ns == prefix or ns.startswith(prefix + ".") for ns in declared_namespaces + ): + blocks.append(f"namespace {prefix}\nend {prefix}") + + listing = "\n".join(f"* `{name}`" for name in provenance) + return ( + "import Mathlib\n\n" + "/-!\n" + f"The Formal Conjectures declarations `{declaration}` needs, copied so\n" + "that `Challenge.lean` requires Mathlib and nothing else. Dependencies\n" + "come before the declarations that use them:\n\n" + f"{listing}\n" + "-/\n\n" + "\n\n".join(blocks) + "\n" + ) + + def replace_proof_with_sorry(text): """Cut the proof body after `:=`, keeping the statement. @@ -403,6 +582,27 @@ def answer_spans(text): return spans +def unwrap_answers(statement): + """Replace any surviving `answer(t)` with `(t)`. + + `answer` is this repository's own elaborator, so a Mathlib-only workspace + cannot parse it. `hoist_answers` removes the `answer(sorry)` slots by + turning them into definition holes; a slot that already carries its answer, + which is how a `research solved` statement is written, is left behind and + used to reach Challenge.lean as literal text that does not parse. + + Unwrapping is faithful. In the default `postpone` mode the elaborator + elaborates the term and attaches an annotation + (`FormalConjecturesUtil/Answer.lean`), so `answer(t)` and `t` denote the + same term and only the annotation is lost. The annotation is what marks + which part of the statement was the question, and a generated Challenge + records that in `holes.json` instead. + """ + for start, end, argument in reversed(answer_spans(statement)): + statement = statement[:start] + f"({argument.strip()})" + statement[end:] + return statement + + def hoist_answers(statement, basename, slot_types, override=None): """Replace each `answer(sorry)` with a named definition hole. @@ -488,10 +688,19 @@ def pins(source_path=None): return mathlib_rev, fc_rev -def lakefile(workspace_id, mathlib_rev, fc_rev): +def lakefile(workspace_id, mathlib_rev): + """Mathlib and nothing else. + + The workspace used to require this repository too, so that Challenge.lean + could import the problem's module. lean-eval vendors its problems and + cannot fetch Formal Conjectures at evaluation time, so the closure travels + in `ChallengeDeps.lean` instead and the require is gone. The Formal + Conjectures commit the copy came from is recorded in `provenance.json`, + which is where a reader should look for it. + """ return f"""name = "{workspace_id}" testDriver = "workspace_test" -defaultTargets = ["Challenge", "Solution", "Submission"] +defaultTargets = ["ChallengeDeps", "Challenge", "Solution", "Submission"] [leanOptions] autoImplicit = false @@ -501,10 +710,8 @@ def lakefile(workspace_id, mathlib_rev, fc_rev): git = "https://github.com/leanprover-community/mathlib4.git" rev = "{mathlib_rev}" -[[require]] -name = "formal_conjectures" -git = "https://github.com/google-deepmind/formal-conjectures.git" -rev = "{fc_rev}" +[[lean_lib]] +name = "ChallengeDeps" [[lean_lib]] name = "Challenge" @@ -548,17 +755,23 @@ def write_workspace(target, files): def generate(basename, out_dir, answer_type=None, module=None): """Write a comparator workspace for one declaration. - Challenge.lean imports the problem's own module rather than restating its - dependencies. `leanprover/lean-eval` generates a Challenge that is one - `import` and one statement, and reconstructing the surrounding definitions - by hand instead cost six defects that only Lean could find: file-scoped + Challenge.lean imports `ChallengeDeps`, which carries the statement's + Formal Conjectures closure and requires Mathlib alone. Importing the + problem's own module would be safer to construct and was what this script + did first, but lean-eval vendors its problems and cannot fetch this + repository at evaluation time, so the closure has to travel with the + workspace. + + That brings back the failure modes an import does not have. Reconstructing + definitions by hand cost six defects that only Lean could find: file-scoped `open` and `variable` lost, `local notation` unrecognised, a `namespace` - swallowing the declaration below it, `section` lines left unclosed. An - import has none of those failure modes. + swallowing the declaration below it, `section` lines left unclosed. The + answer is not to construct more carefully but to check: `--verify` builds + the workspace, which catches all six at once. - Importing a repository full of `sorry` is safe here because comparator - checks axioms. A solution closing the goal with the imported statement - reports `sorryAx`, which `permitted_axioms` does not allow. + Copying a closure out of a repository full of `sorry` is safe because + comparator checks axioms. A solution closing the goal with a copied + statement reports `sorryAx`, which `permitted_axioms` does not allow. """ manifest = load_manifest(basename) declaration = manifest.get("declaration", basename) @@ -572,6 +785,7 @@ def generate(basename, out_dir, answer_type=None, module=None): if facts["range"] is None: raise SystemExit(f"{declaration}: no source range recorded") + source_lines = path.read_text(encoding="utf-8").split("\n") lo, hi = facts["range"]["startLine"], facts["range"]["endLine"] end_col = facts["range"].get("endColumn") @@ -590,6 +804,12 @@ def generate(basename, out_dir, answer_type=None, module=None): statement = original preamble, namespaces_at_target = file_scoped_preamble(source_lines, lo) + deps_module = challenge_deps( + facts.get("dependencies", []), + facts.get("generatedDependencies", []), + declaration, + namespaces_at_target, + ) statement = strip_decorations(statement) statement = replace_proof_with_sorry(statement) @@ -604,6 +824,10 @@ def generate(basename, out_dir, answer_type=None, module=None): statement, holes = hoist_answers( statement, declared, facts.get("answerTypes", []), answer_type ) + # A `research solved` statement carries its answer rather than a `sorry` + # slot, so nothing above removed it and `answer(` would reach a workspace + # that cannot parse it. + statement = unwrap_answers(statement) args = [b["name"] for b in facts["binders"] if b["explicit"]] bad = [a for a in args if "✝" in a or "._" in a] @@ -633,7 +857,7 @@ def generate(basename, out_dir, answer_type=None, module=None): signature = signature[: -len(suffix)].rstrip() challenge = ( - f"import {fc_module}\n\n" + "import ChallengeDeps\n\n" + header + "\n\n".join(holes) + ("\n\n" if holes else "") @@ -644,7 +868,7 @@ def generate(basename, out_dir, answer_type=None, module=None): # The participant's file. The statement sits inside `namespace Submission` # so nothing here can collide with, or stand in for, the trusted names. submission = ( - f"import {fc_module}\nimport Submission.Helpers\n\n" + "import ChallengeDeps\nimport Submission.Helpers\n\n" + header + "namespace Submission\n\n" + "\n\n".join(holes) @@ -663,7 +887,7 @@ def generate(basename, out_dir, answer_type=None, module=None): for h, hn in zip(holes, hole_names_of(holes)) ] solution = ( - f"import {fc_module}\nimport Submission\n\n" + "import ChallengeDeps\nimport Submission\n\n" + header + "\n\n".join(delegated) + ("\n\n" if delegated else "") @@ -761,9 +985,10 @@ def generate(basename, out_dir, answer_type=None, module=None): write_workspace( ws, { - "lakefile.toml": lakefile(workspace_id, mathlib_rev, fc_rev), + "lakefile.toml": lakefile(workspace_id, mathlib_rev), "lean-toolchain": (ROOT / "lean-toolchain").read_text(encoding="utf-8"), "README.md": workspace_readme, + "ChallengeDeps.lean": deps_module, "Challenge.lean": challenge, "Solution.lean": solution, "Submission.lean": submission, @@ -771,6 +996,17 @@ def generate(basename, out_dir, answer_type=None, module=None): "WorkspaceTest.lean": ( COMPARATOR_DIR / "templates" / "WorkspaceTest.lean" ).read_text(encoding="utf-8"), + "provenance.json": json.dumps( + provenance( + declared, + fc_module, + path.relative_to(ROOT), + fc_rev, + [dep["name"] for dep in facts.get("dependencies", [])], + ), + indent=2, + ) + + "\n", "config.json": json.dumps(config, indent=2) + "\n", "holes.json": json.dumps(holes_payload, indent=2, ensure_ascii=False) + "\n", @@ -802,6 +1038,85 @@ def validate(): return 1 if bad else 0 +def provenance(declaration, module, source_path, fc_rev, dependencies): + """Where the copied statement and its dependencies came from. + + Until this workspace carried its own copy of the closure, the lakefile's + `formal_conjectures` requirement named the commit and that was the record. + Nothing else did, so removing the requirement would have left a workspace + whose statement cannot be traced back to a revision of this repository. + """ + blob = subprocess.run( + ["git", "-C", str(ROOT), "rev-parse", f"{fc_rev}:{source_path}"], + capture_output=True, + text=True, + check=False, + ) + return { + "source_repository": "https://github.com/google-deepmind/formal-conjectures", + "source_commit": fc_rev, + "source_path": str(source_path), + "source_blob_sha": blob.stdout.strip() or None, + "declaration": declaration, + "module": module, + "copied_dependencies": dependencies, + "toolchain": (ROOT / "lean-toolchain").read_text(encoding="utf-8").strip(), + "tools": tool_pins(), + } + + +def verify(workspace): + """Elaborate the generated Challenge against its copied dependencies. + + Copying a closure is a construction, and its failure modes are the ones + Lean sees and a reader does not: a lost `open`, an unrecognised + `local notation`, a namespace that no longer exists because nothing + declares it any more. Each of those is a clean build away from being + caught and a long review away from being spotted. + + This concatenates `ChallengeDeps.lean` and `Challenge.lean` and elaborates + them with this checkout's Mathlib, which is the revision the workspace + pins, so the check is offline and does not fetch a second Mathlib. It + checks elaboration and not the lakefile; a comparator run is what + exercises the build. + """ + deps = (workspace / "ChallengeDeps.lean").read_text(encoding="utf-8") + challenge = (workspace / "Challenge.lean").read_text(encoding="utf-8") + challenge = "\n".join( + line + for line in challenge.split("\n") + if line.strip() != "import ChallengeDeps" + ) + with tempfile.NamedTemporaryFile( + "w", suffix=".lean", delete=False, encoding="utf-8" + ) as handle: + handle.write(deps + "\n" + challenge) + combined = handle.name + try: + proc = subprocess.run( + ["lake", "env", "lean", combined], + capture_output=True, + text=True, + cwd=ROOT, + check=False, + ) + finally: + pathlib.Path(combined).unlink(missing_ok=True) + output = (proc.stdout + proc.stderr).replace(combined, "Challenge") + # Only errors fail the check. The target statement's proof is `sorry` by + # construction and each `answer(sorry)` hole is one the solver fills, so + # those warnings are the generator working. Linter warnings such as + # `unused variable` come from the copied source and say nothing about + # whether the copy is faithful. + errors = [line for line in output.splitlines() if "error:" in line] + if proc.returncode != 0 or errors: + raise SystemExit( + f"{workspace.name}: the generated workspace does not elaborate:\n" + + "\n".join(errors or output.splitlines()[-10:]) + ) + return 0 + + def main(argv): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument( @@ -822,6 +1137,12 @@ def main(argv): help="the file declaring it, when more than one does; " "overrides the manifest's `module`", ) + ap.add_argument( + "--verify", + action="store_true", + help="elaborate the generated workspace against this checkout's " + "Mathlib before accepting it", + ) ap.add_argument( "--validate", action="store_true", @@ -833,6 +1154,8 @@ def main(argv): if not args.declaration: ap.error("give a declaration, or --validate") ws = generate(args.declaration, args.out, args.answer_type, args.module) + if args.verify: + verify(pathlib.Path(ws)) print(ws) return 0 diff --git a/scripts/test_make_comparator_workspace.py b/scripts/test_make_comparator_workspace.py index a12bd71cdd..9687b3b7d2 100644 --- a/scripts/test_make_comparator_workspace.py +++ b/scripts/test_make_comparator_workspace.py @@ -19,6 +19,7 @@ wrong problem. The build itself is the comparator's job, not these tests'. """ +import contextlib import json import pathlib import subprocess @@ -29,12 +30,15 @@ import make_comparator_workspace as mcw from make_comparator_workspace import ( answer_spans, + challenge_deps, file_scoped_preamble, hoist_answers, load_manifest, pins, replace_proof_with_sorry, strip_decorations, + strip_fc_attributes, + unwrap_answers, write_workspace, ) @@ -278,5 +282,142 @@ def test_changed_source_is_refused(self): mcw.ROOT = saved_root +@contextlib.contextmanager +def _root_at(directory): + """Point the module's ROOT at a fixture tree. + + `challenge_deps` records each copied declaration's path relative to ROOT, + so a fixture written outside it cannot be described. + """ + saved = mcw.ROOT + mcw.ROOT = pathlib.Path(directory) + try: + yield + finally: + mcw.ROOT = saved + + +class MathlibOnlyChallengeTest(unittest.TestCase): + """The closure travels with the workspace, so copying has to be right. + + Each case here is a defect a generated workspace actually had, found by + elaborating it rather than by reading it. + """ + + def test_answer_with_a_value_is_unwrapped(self): + # `answer` is this repository's elaborator. `hoist_answers` removes the + # `answer(sorry)` slots; `conjecture327` is `research solved` and + # carries `answer(False)`, which reached Challenge.lean verbatim and + # failed to parse against Mathlib alone. + self.assertEqual( + unwrap_answers("theorem t : answer(False) ↔ P := by\n sorry"), + "theorem t : (False) ↔ P := by\n sorry", + ) + + def test_unwrapping_keeps_a_parenthesised_argument_whole(self): + self.assertEqual(unwrap_answers("answer(f (n + 1))"), "(f (n + 1))") + + def test_only_this_repository_s_attributes_are_dropped(self): + # `strip_decorations` clears every attribute off the target statement. + # A copied dependency keeps the rest: dropping `simp` or `reducible` + # changes how the declarations after it in the closure elaborate. + self.assertEqual( + strip_fc_attributes("@[simp, category API, AMS 11]\ntheorem t : P"), + "@[simp]\ntheorem t : P", + ) + self.assertEqual( + strip_fc_attributes("@[category API]\ntheorem t : P"), "theorem t : P" + ) + self.assertEqual(strip_fc_attributes("@[simp]\ndef f := 1"), "@[simp]\ndef f := 1") + + def test_a_generated_constant_with_no_copied_ancestor_is_refused(self): + with self.assertRaisesRegex(SystemExit, "no copied ancestor"): + challenge_deps([], ["Foo.bar._proof_1"], "t") + + def test_a_generated_constant_under_a_copied_parent_is_accepted(self): + # `_proof_1` and `.match_1` have no source: copying the parent + # declaration regenerates them, so they are not an error. + deps = [ + { + "name": "Foo.bar", + "module": "FormalConjectures.Example", + "range": {"startLine": 1, "endLine": 1, "endColumn": None}, + } + ] + with ( + mock.patch.object(mcw, "module_source_path") as resolve, + tempfile.TemporaryDirectory() as tmp, + _root_at(tmp), + ): + source = pathlib.Path(tmp) / "Example.lean" + source.write_text("def Foo.bar := 1\n", encoding="utf-8") + resolve.return_value = source + out = challenge_deps(deps, ["Foo.bar._proof_1"], "t") + self.assertIn("import Mathlib", out) + self.assertIn("def Foo.bar := 1", out) + + def test_a_declaration_inside_another_s_range_is_not_copied_twice(self): + # `EdgeN.mk` covers line 88 of a structure spanning 83 to 93, and + # `pmSumListAux._sparseCasesOn_1` has exactly its parent's range. + # Copying either in its own right duplicated a declaration or sliced a + # fragment of one. + def span(name, lo, hi): + return { + "name": name, + "module": "FormalConjectures.Example", + "range": {"startLine": lo, "endLine": hi, "endColumn": None}, + } + + deps = [ + span("Foo.EdgeN.mk", 2, 2), + span("Foo.EdgeN", 1, 3), + span("Foo.aux._sparseCasesOn_1", 5, 5), + span("Foo.aux", 5, 5), + ] + with ( + mock.patch.object(mcw, "module_source_path") as resolve, + tempfile.TemporaryDirectory() as tmp, + _root_at(tmp), + ): + source = pathlib.Path(tmp) / "Example.lean" + source.write_text( + "structure EdgeN where\n u : Nat\n deriving DecidableEq\n" + "\ndef aux := 1\n", + encoding="utf-8", + ) + resolve.return_value = source + out = challenge_deps(deps, [], "t") + self.assertIn("Foo.EdgeN`", out) + self.assertNotIn("Foo.EdgeN.mk`", out) + self.assertIn("Foo.aux`", out) + self.assertNotIn("_sparseCasesOn_1`", out) + + def test_an_opened_namespace_no_dependency_declares_is_created(self): + # Challenge.lean reopens the namespace stack its target sat in. With + # the problem's module no longer imported, `open Grimm` is an error + # unless something declares that namespace. + out = challenge_deps([], [], "grimm_conjecture", ["Grimm"]) + self.assertIn("namespace Grimm\nend Grimm", out) + + def test_a_namespace_a_dependency_declares_is_not_restated(self): + deps = [ + { + "name": "Grimm.helper", + "module": "FormalConjectures.Example", + "range": {"startLine": 1, "endLine": 1, "endColumn": None}, + } + ] + with ( + mock.patch.object(mcw, "module_source_path") as resolve, + tempfile.TemporaryDirectory() as tmp, + _root_at(tmp), + ): + source = pathlib.Path(tmp) / "Example.lean" + source.write_text("def Grimm.helper := 1\n", encoding="utf-8") + resolve.return_value = source + out = challenge_deps(deps, [], "t", ["Grimm"]) + self.assertNotIn("namespace Grimm\nend Grimm", out) + + if __name__ == "__main__": unittest.main() From 8eab377516256dbb7fb84d7e85a1f5694cf3f289 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Wed, 19 Aug 2026 19:42:13 -0400 Subject: [PATCH 11/70] Split the FC importer from the workspace generator lean-eval#536 divides this integration in two: lean-eval's generator core is extracted into leanprover/lean-eval-generator and consumed as a pinned dependency, and the Formal Conjectures side owns an importer that maps FC declarations and metadata to LeanEval modules and manifests. The FC importer does not fork the generation logic. make_comparator_workspace.py did both halves in one 1,164-line module, so the extraction would have been a rewrite. Split it along that line: fc_leaneval_importer.py resolves a declaration against an exact FC commit, reads the elaborated environment, copies the FC-local closure, types each answer(sorry) slot, records the provenance leaneval_interface.py the two values that cross: one marked-up Mathlib-only Lean module in labelled regions, and one problem manifest leaneval_generator.py the workspace: layout, import graph, scope placement, lakefile, Solution adapter, config make_comparator_workspace.py the command that runs one then the other The generator imports the interface and never the importer, so it can be replaced by a pinned package without touching the FC side. comparator/OWNERSHIP.md gives the line counts either side of that deletion, and the five things lean-eval has to settle before the interface is real. The manifest now records the FC source commit and declaration id, as lean-eval#536 requires, and travels into the workspace as manifest.json; it subsumes the previous provenance.json and holes.json. Every other generated file is byte-identical to what the pre-split script produced. --emit-import writes only the pair this repository owns. A new CI step feeds those bytes back through the generator and requires them to reproduce the workspace exactly, which is what says the pair is the whole interface. --verify now elaborates the marked-up module before anything is written, so a copying defect fails here rather than in lean-eval CI. Python tests: 91, up from 58. --- .github/workflows/build-and-docs.yml | 38 +- .gitignore | 1 + comparator/OWNERSHIP.md | 131 +++ comparator/README.md | 67 +- scripts/comparator_facts.lean | 13 +- scripts/fc_leaneval_importer.py | 843 +++++++++++++++ scripts/leaneval_generator.py | 218 ++++ scripts/leaneval_interface.py | 270 +++++ scripts/make_comparator_workspace.py | 1137 ++------------------- scripts/test_fc_leaneval_importer.py | 400 ++++++++ scripts/test_leaneval_generator.py | 153 +++ scripts/test_leaneval_interface.py | 175 ++++ scripts/test_make_comparator_workspace.py | 416 +------- 13 files changed, 2403 insertions(+), 1459 deletions(-) create mode 100644 comparator/OWNERSHIP.md create mode 100644 scripts/fc_leaneval_importer.py create mode 100644 scripts/leaneval_generator.py create mode 100644 scripts/leaneval_interface.py create mode 100644 scripts/test_fc_leaneval_importer.py create mode 100644 scripts/test_leaneval_generator.py create mode 100644 scripts/test_leaneval_interface.py diff --git a/.github/workflows/build-and-docs.yml b/.github/workflows/build-and-docs.yml index 4b1d9b0a7a..f47c412e7c 100644 --- a/.github/workflows/build-and-docs.yml +++ b/.github/workflows/build-and-docs.yml @@ -155,7 +155,7 @@ jobs: lake --wfail build rm -f FormalConjectures/All.lean - # The elaborator-to-generator boundary, exercised on the oleans the + # The elaborator-to-importer boundary, exercised on the oleans the # build above produced: one plain theorem, a Prop answer slot, a # non-Prop answer slot, a forall-conclusion (whose binder must not be # applied), and explicit parameters (which must be). The comparator run @@ -174,6 +174,42 @@ jobs: grep -q "Submission.erdos_100.variants.strong$" .comparator/erdos_100_variants_strong/Solution.lean grep -q "le_KotherRadical hI" .comparator/KotherConjecture_variants_le_KotherRadical/Solution.lean + # The importer-to-generator seam, on a real declaration. `--emit-import` + # writes only what this repository owns, and feeding those bytes back + # through the generator has to reproduce the workspace exactly; if it + # does not, the pair is not the whole interface and a pinned + # `lean-eval-generator` could not be dropped in. See comparator/OWNERSHIP.md. + - name: Importer to generator seam + if: steps.mode.outputs.website_only != 'true' + run: | + python3 scripts/make_comparator_workspace.py erdos_1038.parts.i \ + --emit-import .comparator-import + python3 - <<'PY' + import pathlib + import sys + + sys.path.insert(0, "scripts") + import leaneval_generator as generator + from leaneval_interface import MarkedUpModule, ProblemManifest + + handed_over = pathlib.Path(".comparator-import/erdos_1038_parts_i") + module = MarkedUpModule.parse( + (handed_over / "Problem.lean").read_text(encoding="utf-8") + ) + manifest = ProblemManifest.from_json( + (handed_over / "manifest.json").read_text(encoding="utf-8") + ) + assert len(manifest.source.commit) == 40, manifest.source.commit + assert manifest.source.declaration, "no FC declaration id" + + workspace = pathlib.Path(".comparator/erdos_1038_parts_i") + regenerated = generator.generate(module, manifest) + for name, content in regenerated.items(): + expected = (workspace / name).read_text(encoding="utf-8") + assert content == expected, name + print(f"{len(regenerated)} files reproduced from the module and manifest") + PY + - name: Build literate source pages if: steps.mode.outputs.website_only != 'true' && steps.mode.outputs.site == 'true' run: | diff --git a/.gitignore b/.gitignore index 55ea0833b8..628dd087f7 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ FormalConjectures/All.lean # Python bytecode from the scripts in `scripts/`. __pycache__/ .comparator/ +.comparator-import/ diff --git a/comparator/OWNERSHIP.md b/comparator/OWNERSHIP.md new file mode 100644 index 0000000000..175d654233 --- /dev/null +++ b/comparator/OWNERSHIP.md @@ -0,0 +1,131 @@ +# What this repository owns, and what it hands over + +[`lean-eval#536`](https://github.com/leanprover/lean-eval/pull/536) §10 divides +this integration in two. lean-eval's generator core — the part that turns a +marked-up Lean module plus a manifest into a Challenge / Solution / Submission +workspace, with the import and scope fidelity work from +[`lean-eval#531`](https://github.com/leanprover/lean-eval/pull/531) — is being +extracted into `leanprover/lean-eval-generator` and consumed as a pinned +dependency. **The Formal Conjectures importer does not fork the generation +logic.** It maps FC declarations and metadata to LeanEval modules and manifests, +and each manifest records the FC source commit and declaration id. + +The code here is arranged along that line so the handover is a deletion rather +than a rewrite. This file says exactly what goes. + +## The seam + + scripts/fc_leaneval_importer.py FC declaration -> (module, manifest) + scripts/leaneval_interface.py the two values, and nothing else + scripts/leaneval_generator.py (module, manifest) -> workspace files + +`scripts/make_comparator_workspace.py` is the command that runs one after the +other. The arrow points one way: the generator imports the interface and never +the importer, and a test asserts that. + +### What crosses it + +`MarkedUpModule` is one Lean module that requires Mathlib and nothing else, +divided into four labelled regions: + +| Region | Contents | +|---|---| +| `dependencies` | the statement's FC-local closure, copied, each declaration carrying the `open`, `variable`, `universe`, `set_option` and `local notation` in force where it was written | +| `scope` | the directives the statement itself needs, and the namespaces it is stated in | +| `holes` | one `noncomputable def : := sorry` for each `answer(sorry)` slot | +| `statement` | the target statement, decorations stripped, proof replaced by `sorry` | + +The module is not pre-split into Challenge and ChallengeDeps, because deciding +which generated file imports which, and where the scope has to be restated so +that the same statement text elaborates in all three, is the generator's work. +It is one module rather than four strings because the importer can then +elaborate exactly what it is about to hand over: `--verify` runs the module +through this checkout's Mathlib, so an FC-side defect — a lost `open`, an +unrecognised `local notation`, a namespace nothing declares any more — fails +here and not in lean-eval's CI. + +`ProblemManifest` carries what the Lean text does not say: the theorem's name +and its explicit parameters, the hole types Lean reported, the permitted +axioms, the toolchain and Mathlib pins, and a `source` record with the FC +repository, commit, blob, module and declaration id. lean-eval#536 requires the +commit and the declaration id by name, and they are FC-side by necessity: the +generator sees a Lean module, not a repository. They are also what makes +regeneration possible when Formal Conjectures corrects a misformalisation +upstream. The generator writes the manifest into the workspace unaltered, as +`manifest.json`. + +## What is deleted when `lean-eval-generator` lands + +| File | Lines | Then | +|---|---|---| +| `scripts/leaneval_generator.py` | 218 | deleted; `generate` becomes a call into the pinned package | +| `scripts/test_leaneval_generator.py` | 153 | deleted, less whatever remains useful as a contract test against the pinned generator | +| `comparator/templates/WorkspaceTest.lean` | 37 | deleted; the generator supplies its own workspace test | +| `scripts/leaneval_interface.py` | 270 | replaced by an import from the pinned package, to the extent its types match | + +That is 408 lines deleted outright and 270 more replaced. Nothing in +`scripts/fc_leaneval_importer.py` changes, and `make_comparator_workspace.py` +changes by one import. + +## What stays Formal Conjectures' permanently + +| File | Lines | Why it cannot move | +|---|---|---| +| `scripts/fc_leaneval_importer.py` | 843 | resolves a declaration against an exact FC commit, reads the elaborated environment, copies the FC-local closure, types each `answer(sorry)` slot, and records the provenance | +| `scripts/comparator_facts.lean` | 205 | the Lean extractor: source ranges, binder explicitness, and answer-slot types, all of which only this repository's elaborated environment knows | +| `scripts/test_fc_leaneval_importer.py` | 400 | every case pins a real extraction defect | +| `scripts/make_comparator_workspace.py` | 175 | the command, and the directory write that belongs to neither side | +| `scripts/test_make_comparator_workspace.py` | 99 | asserts the emitted pair rebuilds the workspace exactly | +| `comparator/problems/*.toml` | — | the choices FC source cannot make for itself: which module, and an answer type Lean reports ambiguously | + +Nothing in the importer names a workspace file, a workspace layout, or an +import graph. If a change to it would, the change belongs on the other side. + +## Not built, on purpose + +**Disproof support.** Blocked upstream: Comparator has no interface for a +plain-statement disproof. Nothing here anticipates one. + +**Multi-file Challenge support.** The generator already carries a statement's +whole closure in `ChallengeDeps`, which is one file. Splitting that closure +across several trusted files is a generator-side change: the importer would +hand over the same declarations, and only the `dependencies` region's shape +would have to say how they group. lean-eval#536 asks for this to be scoped +against the actual FC100 statements rather than in the abstract, so it is not +built here. + +**Lifecycle.** Result records, resubmission, and revision tracking are +LeanEval's, per lean-eval#536. This repository regenerates and opens a pull +request; it keeps no state about what happened to one. + +## What this side cannot settle alone + +Each of these is a place where the interface above is a guess that lean-eval +has to confirm or replace. None of them is blocking the FC work; all of them +would change bytes at the seam. + +1. **The markup convention is invented here.** `-- @region ` and the four + region names are local. The generator core is the natural owner of the + convention, since it is the reader. +2. **The manifest schema is invented here.** `schema_version = 1` and the field + names are this repository's. lean-eval#536 says the importer emits PRs that + lean-eval CI validates like any other problem PR, which needs a published + schema to validate against. The two fields the plan does name — the FC + source commit and the declaration id — are present under + `source.commit` and `source.declaration`. +3. **The `definition_names` config field is undocumented.** Comparator's + published no-hole config does not carry it, and hole support depends on the + comparator commit pinned in `tools.toml`. A generated workspace with an + `answer(sorry)` hole is only checkable against that build. +4. **Answer-slot types are read under this repository's toolchain.** The + importer asks Formal Conjectures' elaborated environment, at FC's Lean and + Mathlib pins, for the type of each slot; LeanEval builds at Lean 4.33 and + its own Mathlib. A type whose name or elaboration differs between the two + revisions would be wrong in a way `--verify` cannot see, because `--verify` + also runs at FC's pins. Only a build on the LeanEval side closes this. +5. **Who triggers regeneration is unassigned.** The plan gives the importer the + duty to regenerate and re-PR when Formal Conjectures fixes a + misformalisation upstream, and gives lifecycle to LeanEval. Nothing yet says + which side watches FC commits for a change to an imported declaration. The + manifest records what is needed to answer the question — commit, path, blob + and declaration — but nobody is asking it. diff --git a/comparator/README.md b/comparator/README.md index 3f3cd0ee59..ad153545ed 100644 --- a/comparator/README.md +++ b/comparator/README.md @@ -22,10 +22,11 @@ workspace generator. provenance, maps `answer(sorry)` semantics, and emits reviewable LeanEval source and manifests. -The standalone workspace writer in this draft exercises the hard FC-side -extraction cases while the shared generator interface is being separated from -LeanEval's `EvalTools`. Once that interface exists, the importer must call it -rather than retain parallel generation logic. +The code is already arranged along that line. `scripts/fc_leaneval_importer.py` +is the FC side and stays; `scripts/leaneval_generator.py` stands in for the +shared generator and is written to be deleted, not rewritten, once +`leanprover/lean-eval-generator` exists. [`OWNERSHIP.md`](OWNERSHIP.md) gives +the interface between them and the line counts either side of that deletion. This follows the ownership split proposed in [`lean-eval#536`](https://github.com/leanprover/lean-eval/pull/536), with @@ -38,9 +39,11 @@ coordination tracked in 1. The FC importer resolves a declaration against an exact Formal Conjectures commit and obtains its source range, binders, namespace, dependencies, and `answer(sorry)` slot types from Lean. -2. It emits vendored LeanEval source, one LeanEval problem manifest, and - immutable provenance containing at least the FC repository, commit, source - path, fully qualified declaration name, and frozen-set identity. +2. It emits one marked-up LeanEval module and one problem manifest. The + manifest records at least the FC repository, commit, source path, blob, and + fully qualified declaration name, so that a workspace can be traced back to + a revision of this repository and regenerated when that revision is + corrected. 3. LeanEval builds the vendored source under Lean 4.33 and its matching Mathlib pin. 4. The shared LeanEval generator creates `Challenge`, `ChallengeDeps`, @@ -79,18 +82,24 @@ open conjectures. `scripts/comparator_facts.lean` asks Lean for the selected declaration's source range, binders, and `answer(sorry)` slot types. -`scripts/make_comparator_workspace.py` then creates one pinned standalone -workspace as a conformance harness. The generated workspace uses the Formal -Conjectures toolchain and dependency pins. It is **not** the final LeanEval 4.33 -artifact. +`scripts/fc_leaneval_importer.py` maps that declaration to the pair the +generator consumes: one marked-up Mathlib-only Lean module, and one manifest. +`scripts/leaneval_generator.py` turns the pair into a pinned standalone +workspace as a conformance harness, and +`scripts/make_comparator_workspace.py` runs the two in order. The generated +workspace uses the Formal Conjectures toolchain and dependency pins. It is +**not** the final LeanEval 4.33 artifact. The prototype workspace contains: +- `ChallengeDeps.lean`, with the statement's copied Formal Conjectures closure; - `Challenge.lean`, with the trusted statement and proof hole; - `Submission.lean` and `Submission/`, where a solver works; - `Solution.lean`, which connects the submission to the trusted statement; - `config.json`, with theorem targets, definition targets, and permitted axioms; -- `holes.json`, with the exact extracted declaration blocks; +- `manifest.json`, the manifest the importer handed the generator: the Formal + Conjectures source commit and declaration id, the copied closure, the exact + original declaration text, the hole types, and the pins; - pinned Lean, Mathlib, Formal Conjectures, Comparator, and helper-tool versions. `Solution.lean` is fixed. It fails to build if the submission changes the @@ -107,9 +116,24 @@ Use `--out` to choose the parent directory. The generator refuses to overwrite an existing workspace. It writes into a temporary directory and renames the complete workspace into place. -The generator also stops when the selected source differs from the pinned -upstream revision. This prevents a workspace from combining a working-tree -statement with an older imported context. +The importer stops when the selected source differs from the pinned upstream +revision. This prevents a workspace from combining a working-tree statement +with an older imported context. + +`--verify` elaborates the marked-up module against this checkout's Mathlib +before anything is written, so a copying defect fails here rather than in +LeanEval CI. + +### Emit only what this repository owns + +```bash +python3 scripts/make_comparator_workspace.py erdos_1038.parts.i \ + --emit-import .comparator-import +``` + +This writes `Problem.lean` and `manifest.json` and generates no workspace. It +is the pair the importer contributes to a LeanEval problem pull request once +the shared generator is a pinned dependency there. ### Supported prototype inputs @@ -120,10 +144,15 @@ statement with an older imported context. Plain-statement disproofs remain out of scope until Comparator provides an upstream interface for them. -## Prototype problem manifests +## Problem files + +`problems/*.toml` is an input, not the LeanEval manifest: it records the +choices this repository's Lean source cannot make for itself, and the importer +reads it. The manifest the generator receives, and writes into the workspace as +`manifest.json`, is derived. -Most declarations need no prototype manifest. Add one TOML file under -`problems/` only when the source cannot select the declaration by itself. +Most declarations need no problem file. Add one TOML file under `problems/` +only when the source cannot select the declaration by itself. | Field | Meaning | |---|---| @@ -134,7 +163,7 @@ Most declarations need no prototype manifest. Add one TOML file under | `source` | Optional source link for the generated README. | | `notes` | Optional reviewer note for the generated README. | -Run the manifest check after moving or renaming a declaration: +Run the problem-file check after moving or renaming a declaration: ```bash python3 scripts/make_comparator_workspace.py --validate diff --git a/scripts/comparator_facts.lean b/scripts/comparator_facts.lean index 2287b6fe65..0dc6501b07 100644 --- a/scripts/comparator_facts.lean +++ b/scripts/comparator_facts.lean @@ -18,8 +18,8 @@ import FormalConjecturesUtil.Answer import FormalConjecturesUtil.Attributes.Basic /-! -The elaborator-side facts `make_comparator_workspace.py` currently gets by -reading Lean with regular expressions. +The elaborator-side facts `scripts/fc_leaneval_importer.py` would otherwise +get by reading Lean with regular expressions. Given a module and a declaration name, this prints JSON with what the elaborated environment knows exactly and the text layer can only guess: @@ -27,14 +27,15 @@ elaborated environment knows exactly and the text layer can only guess: - the declaration's source range, for slicing its original text; - its binders, with names and explicitness, for the Solution adapter; - the type of each `sorry` inside the *statement*, which is the type of an - `answer(sorry)` slot. The generator's manifest `answer_type` field exists - only because surface syntax does not carry this; the environment does. + `answer(sorry)` slot. The `answer_type` field in a `comparator/problems` + file exists only because surface syntax does not carry this; the + environment does. Usage: lake exe comparator_facts The declaration may be given in full or by any whole suffix, the same rule -the Python generator uses. +the Python importer uses. -/ open Lean Meta @@ -172,7 +173,7 @@ where -- `Finset.greedySidon.aux._proof_1` and `.match_1` in the closure. They -- have no source range because they have no source: copying the parent -- declaration's text regenerates them. Emit them separately so the - -- generator can check each one has an ancestor that is being copied, + -- importer can check each one has an ancestor that is being copied, -- rather than dropping them silently. let mut deps := #[] let mut generated := #[] diff --git a/scripts/fc_leaneval_importer.py b/scripts/fc_leaneval_importer.py new file mode 100644 index 0000000000..847f428572 --- /dev/null +++ b/scripts/fc_leaneval_importer.py @@ -0,0 +1,843 @@ +#!/usr/bin/env python3 +"""Map one Formal Conjectures declaration to a LeanEval module and manifest. + +This is the Formal Conjectures side of the ownership split in +`leanprover/lean-eval#536`, and it is the part this repository owns +permanently. It resolves a declaration against an exact Formal Conjectures +commit, asks Lean what the elaborated environment knows about it, copies the +declarations it depends on, types each `answer(sorry)` slot, and records where +all of that came from. + +What it produces is the pair defined in `scripts/leaneval_interface.py`: one +marked-up Mathlib-only Lean module, and one manifest carrying the FC source +commit and declaration id. Turning that pair into a Challenge / Solution / +Submission workspace is the generator's job, not this module's; see +`scripts/leaneval_generator.py`, which is the part that goes away when +`leanprover/lean-eval-generator` is extracted. + +Nothing here writes a workspace file, names a workspace layout, or decides +which generated module imports which. If a change to this file would do one of +those, it belongs on the other side of the seam. + +Two things the Lean source cannot settle live in `comparator/problems/.toml`, +one file per problem: an answer type Lean reports ambiguously, and which file +is meant when two declare the same name. See that directory's README. +""" + +import json +import pathlib +import re +import subprocess +import sys +import tempfile +import tomllib + +from leaneval_interface import ( + DefinitionHole, + MarkedUpModule, + ProblemManifest, + SourceRecord, +) + +ROOT = pathlib.Path(__file__).resolve().parent.parent +SOURCE_DIRS = [ROOT / "FormalConjectures"] +COMPARATOR_DIR = ROOT / "comparator" +MANIFEST_DIR = COMPARATOR_DIR / "problems" + +SOURCE_REPOSITORY = "https://github.com/google-deepmind/formal-conjectures" + +PERMITTED_AXIOMS = ("propext", "Quot.sound", "Classical.choice") + +DECL_START = re.compile( + # `local notation` and `scoped notation` carry the modifier before the + # keyword. Without them here, Erdos 125's `local notation "A" => ...` typed + # as nothing and was dropped, and its statements lost the sets they name. + r"^(?:noncomputable\s+|private\s+|protected\s+|local\s+|scoped\s+)*" + r"(theorem|lemma|def|abbrev|structure|inductive|instance|notation)\s", +) +KEEP_LOOSE = re.compile( + r"^(open|variable|universe|section|namespace|end|attribute|set_option)\b" +) + + +def tool_pins(): + """The locked external tool revisions; comparator/tools.toml is the one + machine-readable source, and this module refuses to restate it.""" + with (COMPARATOR_DIR / "tools.toml").open("rb") as handle: + return tomllib.load(handle)["tools"] + + +def elaborator_facts(module, declaration): + """What the elaborated environment knows about a declaration. + + Runs `lake exe comparator_facts`, which imports the module and reports the + declaration's source range, its binders with real explicitness, and the + inferred type of each `answer(sorry)` slot. Every one of these used to be + reconstructed from text, and each reconstruction had failure modes the + elaborator does not. + """ + proc = subprocess.run( + ["lake", "exe", "comparator_facts", module, declaration], + capture_output=True, + text=True, + cwd=ROOT, + ) + if proc.returncode != 0: + raise SystemExit( + f"comparator_facts {declaration}: " + f"{proc.stderr.strip() or proc.stdout.strip()}" + ) + out = proc.stdout + if "{" not in out: + raise SystemExit(f"comparator_facts {declaration}: no JSON in output") + return json.loads(out[out.index("{") :]) + + +def file_scoped_preamble(lines, start_line): + """Directives in force at `start_line`, and the namespace stack there. + + Lean scopes `open`, `variable`, `universe`, `set_option` and notation to + the file, so the marked-up module has to restate them; nothing in the + olean records them. A directive counts only if it precedes the statement + and its scope still encloses it. + """ + stack, preamble, depth = [], [], 0 + for line in lines[: start_line - 1]: + if depth == 0 and KEEP_LOOSE.match(line) and not line.rstrip().endswith(" in"): + kind = line.split()[0] + parts = line.split(None, 1) + name = parts[1].strip() if len(parts) > 1 else None + if kind in ("namespace", "section"): + stack.append((kind, name)) + elif kind == "end": + if stack and ( + stack[-1][1] == name or (name is None and stack[-1][0] == "section") + ): + stack.pop() + else: + preamble.append((line, list(stack))) + depth += len(re.findall(r"/-", line)) - len(re.findall(r"-/", line)) + depth = max(depth, 0) + scope = list(stack) + in_force = [text for text, s in preamble if s == scope[: len(s)]] + return in_force, [n for k, n in scope if k == "namespace" and n] + + +def load_manifest(problem_id): + """Read explicit choices that Lean source cannot select by itself. + + An FC problem file selects the module when names collide. It may also + override an answer-slot type when Lean reports several types that cannot + be matched to source positions. The importer refuses both cases without an + explicit choice. + + `leanprover/lean-eval` keeps one TOML per problem, and the reason is worth + copying: two pull requests adding different problems never touch the same + file. + + id the filename stem, and the workspace directory name + declaration the Lean name, which need not be unique across the repository + module the file declaring it, relative to the repository root + answer_type the type of a non-`Prop` answer slot + notes free text for a reviewer + source a citation or URL + """ + path = MANIFEST_DIR / f"{problem_id}.toml" + if not path.exists(): + return {} + with path.open("rb") as handle: + data = tomllib.load(handle) + if data.get("id") != problem_id: + raise SystemExit( + f"{path} declares id {data.get('id')!r}, but its filename says " + f"{problem_id!r}; the two must agree" + ) + if "declaration" not in data: + raise SystemExit(f"{path} has no `declaration` field") + return data + + +def manifest_ids(): + return sorted(p.stem for p in MANIFEST_DIR.glob("*.toml")) + + +def module_name(rel_path): + """The Lean module name for a path under `FormalConjectures/`. + + Most problem files are named for a number, which is not an identifier, so + the component is written in guillemets: + `FormalConjectures.ErdosProblems.«940»`. + """ + parts = [ + c if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", c) else f"«{c}»" + for c in str(rel_path)[: -len(".lean")].split("/") + ] + return ".".join(parts) + + +def find_declaration(basename, module=None): + """Locate the file declaring `basename`. Returns (path, imports, doc, body). + + `module` names the file when more than one declares the name, and comes + from the problem's FC problem file. + """ + if module is not None: + named = ROOT / module + if not named.exists(): + raise SystemExit(f"manifest names {module}, which does not exist") + return _read_source(named) + hits = [] + for src in SOURCE_DIRS: + for path in sorted(src.rglob("*.lean")): + text = path.read_text(encoding="utf-8") + if re.search( + rf"(?:theorem|lemma)\s+(?:[\w.«»]*\.)?{re.escape(basename)}[\s:]", text + ): + hits.append(path) + if not hits: + raise SystemExit( + f"no declaration named {basename!r} found under FormalConjectures/" + ) + if len(hits) > 1: + raise SystemExit( + f"{basename!r} is ambiguous: " + + ", ".join(str(h.relative_to(ROOT)) for h in hits) + + "; pass --module to choose one, or record the choice in " + "comparator/problems/.toml" + ) + return _read_source(hits[0]) + + +def _read_source(path): + text = path.read_text(encoding="utf-8") + # Drop the license header; keep the module docstring; the rest is the body. + text = re.sub(r"\A/-.*?-/\s*", "", text, flags=re.DOTALL) + doc = "" + m = re.match(r"\s*(/-!.*?-/)\s*", text, flags=re.DOTALL) + if m: + doc = m.group(1) + text = text[m.end() :] + # Imports precede the docstring in source order; recover them from the original. + imports = re.findall( + r"^import\s+(\S+)", path.read_text(encoding="utf-8"), re.MULTILINE + ) + return path, imports, doc, text + + +def strip_decorations(block_text): + """Remove the docstring, line comments and attributes from a declaration. + + These interleave. Erdos 918 puts a `--` formalisation note between its + docstring and its `@[category ...]` line, and one anchored pass each left + the attribute in place. `@[category research open, AMS 5]` then reached the + marked-up module, where the workspace has no such attribute, and Lean + parsed as far as the `open` inside it before giving up. + """ + # `open X in` binds to the declaration and has to survive, but it sits + # above the docstring, so stripping anchored at the start would stop dead + # on it. + prefix = "" + m = re.match(r"\A\s*(open\b[^\n]*\bin)\n", block_text) + if m: + prefix = m.group(1) + "\n" + block_text = block_text[m.end() :] + while True: + stripped = re.sub(r"\A\s*/--.*?-/\s*", "", block_text, flags=re.DOTALL) + stripped = re.sub(r"\A\s*--[^\n]*\n", "", stripped) + stripped = re.sub(r"\A\s*@\[[^\]]*\]\s*", "", stripped, flags=re.DOTALL) + if stripped == block_text: + return prefix + stripped + block_text = stripped + + +# Attributes this repository defines. A generated workspace requires Mathlib +# and nothing else, so these have to go; everything else has to stay. +FC_ATTRIBUTES = ("category", "AMS", "formal_proof") + + +def strip_fc_attributes(block_text): + """Remove this repository's own attributes from a copied declaration. + + Unlike `strip_decorations`, which clears every attribute off the target + statement, this keeps the rest. A dependency is copied to be elaborated, + not restated, and dropping `simp`, `reducible` or `instance` attributes + changes how the declarations after it in the same closure elaborate. + """ + + def replace(match): + inner = match.group(1) + # Nested brackets mean an argument this simple split would cut in + # half, so leave the whole attribute alone rather than mangle it. + if "[" in inner: + return match.group(0) + kept = [ + part.strip() + for part in inner.split(",") + if part.strip() and part.strip().split()[0] not in FC_ATTRIBUTES + ] + return f"@[{', '.join(kept)}]" if kept else "" + + text = re.sub(r"@\[([^\]]*)\]", replace, block_text) + # An attribute line that emptied out leaves a blank line behind. + return re.sub(r"^[ \t]*\n", "", text, flags=re.MULTILINE) + + +def module_source_path(module): + """The file declaring a dotted Lean module name, undoing guillemets.""" + parts = [ + component[1:-1] if component.startswith("«") else component + for component in module.split(".") + ] + path = ROOT.joinpath(*parts).with_suffix(".lean") + if not path.is_file(): + raise SystemExit(f"{module}: no source file at {path}") + return path + + +def slice_range(lines, source_range): + """The source text a declaration range covers, and the line it starts on. + + `open X in` binds to the declaration below it but sits above what the + range covers in some toolchains, so it is pulled in when present. + """ + lo, hi = source_range["startLine"], source_range["endLine"] + end_column = source_range.get("endColumn") + while ( + lo > 1 + and lines[lo - 2].rstrip().endswith(" in") + and KEEP_LOOSE.match(lines[lo - 2]) + ): + lo -= 1 + sliced = lines[lo - 1 : hi] + if end_column is not None and sliced: + sliced = sliced[:-1] + [sliced[-1][:end_column]] + return "\n".join(sliced), lo + + +def closure_region(dependencies, generated, declaration, opened_namespaces=()): + """A declaration's FC-local closure, copied, needing Mathlib and nothing else. + + lean-eval vendors problems, so a generated Challenge cannot fetch this + repository at evaluation time and has to stand on Mathlib alone. That + rules out importing the problem's own module, and brings back the failure + modes an import does not have: file-scoped `open` and `variable` lost, + `local notation` unrecognised, a namespace swallowing what follows. + + So each declaration is emitted inside its own `section`, carrying the + preamble in force where it was written and reopening the namespace it was + written in. That is a construction, not a proof, and the only check that + covers every one of those failure modes at once is elaborating the + marked-up module, which `--verify` does. + """ + copied = [dep["name"] for dep in dependencies] + orphans = [ + name + for name in generated + if not any(name.startswith(parent + ".") for parent in copied) + ] + if orphans: + raise SystemExit( + f"{declaration}: {len(orphans)} elaborator-generated constant(s) " + "have no copied ancestor, so copying the closure would not " + f"reproduce them: {', '.join(orphans[:5])}" + ) + + # A constructor, a `where` auxiliary and a `_sparseCasesOn` all carry a + # source range inside the declaration that produces them, so copying them + # in their own right either duplicates a declaration or slices a fragment + # of one. `MonochromaticQuantumGraph.EdgeN.mk` covers line 88 of a + # structure spanning 83 to 93; `pmSumListAux._sparseCasesOn_1` has exactly + # its parent's range. Copying the outer declaration reproduces both. + def covered_by_another(dep): + inner = dep["range"] + for other in dependencies: + if other is dep or other["module"] != dep["module"]: + continue + outer = other["range"] + if outer is None or inner is None: + continue + if not ( + outer["startLine"] <= inner["startLine"] + and outer["endLine"] >= inner["endLine"] + ): + continue + same_span = ( + outer["startLine"] == inner["startLine"] + and outer["endLine"] == inner["endLine"] + ) + # A tie on the span is broken by name: the parent is the prefix. + if not same_span or len(other["name"]) < len(dep["name"]): + return True + return False + + subsumed = [dep["name"] for dep in dependencies if covered_by_another(dep)] + dependencies = [dep for dep in dependencies if dep["name"] not in subsumed] + + blocks, provenance = [], [] + for dep in dependencies: + if dep["range"] is None: + raise SystemExit(f"{declaration}: {dep['name']} has no source range") + path = module_source_path(dep["module"]) + lines = path.read_text(encoding="utf-8").split("\n") + text, start = slice_range(lines, dep["range"]) + preamble, namespaces = file_scoped_preamble(lines, start) + body = strip_fc_attributes(text).strip("\n") + if not body: + raise SystemExit(f"{declaration}: {dep['name']} sliced to nothing") + namespace = ".".join(namespaces) + chunk = [f"-- {dep['name']}, from {path.relative_to(ROOT)}", "section"] + chunk += preamble + if namespace: + chunk.append(f"namespace {namespace}") + chunk += ["", body, ""] + if namespace: + chunk.append(f"end {namespace}") + chunk.append("end") + blocks.append("\n".join(chunk)) + provenance.append(dep["name"]) + + # The statement reopens the namespace stack the target sat in, so it can + # name siblings short. `open` on a namespace nothing has declared is an + # error, and with the problem's module no longer imported only the copied + # declarations can declare one. An empty namespace block is enough to make + # the name exist. + declared_namespaces = {name.rsplit(".", 1)[0] for name in provenance if "." in name} + for depth in range(len(opened_namespaces)): + prefix = ".".join(opened_namespaces[: depth + 1]) + if not any( + ns == prefix or ns.startswith(prefix + ".") for ns in declared_namespaces + ): + blocks.append(f"namespace {prefix}\nend {prefix}") + + listing = "\n".join(f"* `{name}`" for name in provenance) + return ( + "/-!\n" + f"The Formal Conjectures declarations `{declaration}` needs, copied so\n" + "that the statement requires Mathlib and nothing else. Dependencies\n" + "come before the declarations that use them:\n\n" + f"{listing}\n" + "-/\n\n" + "\n\n".join(blocks) + "\n" + ), provenance + + +def replace_proof_with_sorry(text): + """Cut the proof body after `:=`, keeping the statement. + + A tactic proof is found by `:= by`, which a statement cannot contain, + `by` being a keyword. A term proof leaves only a bare `:=` to cut at, and + a statement can contain one of those: a structure literal `{ a := b }` + inside the statement would be cut in half. With more than one candidate + the importer refuses, as everywhere else it cannot decide. + """ + m = re.search(r":=\s*by\b", text) + if m: + return text[: m.start()].rstrip() + " := by\n sorry" + if text.count(":=") > 1: + raise SystemExit( + "the declaration has a term-mode proof and more than one `:=`, so " + "the start of the proof cannot be read off the text" + ) + m = re.search(r":=", text) + if m: + return text[: m.start()].rstrip() + " := by\n sorry" + return text.rstrip() + " := by\n sorry" + + +def answer_spans(text): + """Return the source spans of syntactic `answer(...)` calls. + + This small lexer skips strings and nested line/block comments and balances + parentheses, so an answer term may itself contain parentheses. It is not a + Lean parser; malformed or unterminated syntax is refused. + """ + spans = [] + i = 0 + block_depth = 0 + in_string = False + escaped = False + while i < len(text): + pair = text[i : i + 2] + if block_depth: + if pair == "/-": + block_depth += 1 + i += 2 + elif pair == "-/": + block_depth -= 1 + i += 2 + else: + i += 1 + continue + if in_string: + if escaped: + escaped = False + elif text[i] == "\\": + escaped = True + elif text[i] == '"': + in_string = False + i += 1 + continue + if pair == "/-": + block_depth = 1 + i += 2 + continue + if pair == "--": + newline = text.find("\n", i + 2) + i = len(text) if newline < 0 else newline + 1 + continue + if text[i] == '"': + in_string = True + i += 1 + continue + if text.startswith("answer", i) and ( + i == 0 or not (text[i - 1].isalnum() or text[i - 1] in "_.'") + ): + j = i + len("answer") + while j < len(text) and text[j].isspace(): + j += 1 + if j < len(text) and text[j] == "(": + depth = 1 + k = j + 1 + nested_string = False + nested_escaped = False + nested_comment = 0 + while k < len(text) and depth: + nested_pair = text[k : k + 2] + if nested_comment: + if nested_pair == "/-": + nested_comment += 1 + k += 2 + elif nested_pair == "-/": + nested_comment -= 1 + k += 2 + else: + k += 1 + continue + if nested_string: + if nested_escaped: + nested_escaped = False + elif text[k] == "\\": + nested_escaped = True + elif text[k] == '"': + nested_string = False + k += 1 + continue + if nested_pair == "/-": + nested_comment = 1 + k += 2 + elif nested_pair == "--": + newline = text.find("\n", k + 2) + k = len(text) if newline < 0 else newline + 1 + elif text[k] == '"': + nested_string = True + k += 1 + else: + if text[k] == "(": + depth += 1 + elif text[k] == ")": + depth -= 1 + k += 1 + if depth: + raise SystemExit("unterminated answer(...) term") + spans.append((i, k, text[j + 1 : k - 1])) + i = k + continue + i += 1 + if block_depth or in_string: + raise SystemExit("unterminated comment or string while reading answers") + return spans + + +def unwrap_answers(statement): + """Replace any surviving `answer(t)` with `(t)`. + + `answer` is this repository's own elaborator, so a Mathlib-only workspace + cannot parse it. `hoist_answers` removes the `answer(sorry)` slots by + turning them into definition holes; a slot that already carries its answer, + which is how a `research solved` statement is written, is left behind and + used to reach the marked-up module as literal text that does not parse. + + Unwrapping is faithful. In the default `postpone` mode the elaborator + elaborates the term and attaches an annotation + (`FormalConjecturesUtil/Answer.lean`), so `answer(t)` and `t` denote the + same term and only the annotation is lost. The annotation is what marks + which part of the statement was the question, and the manifest records + that instead. + """ + for start, end, argument in reversed(answer_spans(statement)): + statement = statement[:start] + f"({argument.strip()})" + statement[end:] + return statement + + +def hoist_answers(statement, basename, slot_types, override=None): + """Replace each `answer(sorry)` with a named definition hole. + + The slot types come from the elaborated environment, where the `answer` + elaborator ran with the expected type in hand; the old surface-syntax + guess (an `↔` beside the slot means `Prop`) and the FC problem file's + hand-kept `answer_type` both survive only as overrides. Slots of different + types in one statement are refused: the environment reports the types as a + set, and matching them to positions would be a guess. + """ + holes = [] + calls = answer_spans(statement) + selected = [call for call in calls if call[2].strip() == "sorry"] + count = len(selected) + if count == 0: + return statement, holes + # Under the default `alwaysTrue` setting, the `answer` elaborator erases a + # slot to `True` if and only if its expected type is `Prop` + # (FormalConjecturesUtil/Answer.lean). So a slot the environment carries + # no annotation for is a `Prop` slot by the elaborator's own rule, not by + # guesswork, and no postpone build is needed. + missing = count - len(slot_types) + if override: + types = [override] * count + elif missing == count: + types = ["Prop"] * count + elif missing == 0 and len(set(slot_types)) == 1: + types = [slot_types[0]] * count + elif missing == 0: + raise SystemExit( + f"{basename} has {count} answer slots of differing types " + f"{slot_types}; pass --answer-type" + ) + else: + # Some slots are Prop and some are not: which positions are which + # cannot be read off an unordered set, so refuse rather than assign. + raise SystemExit( + f"{basename}: {missing} Prop slot(s) and {len(slot_types)} typed " + f"slot(s) {slot_types} cannot be matched to positions; pass " + "--answer-type" + ) + replacements = [] + for i, (start, end, _argument) in enumerate(selected): + name = f"{basename}_answer" if count == 1 else f"{basename}_answer_{i + 1}" + holes.append(DefinitionHole(name=name, type=types[i])) + replacements.append((start, end, name)) + for start, end, name in reversed(replacements): + statement = statement[:start] + name + statement[end:] + return statement, holes + + +def pins(source_path=None): + """Revisions the workspace's own build can actually fetch. + + The FC pin must be reachable from the upstream repository the lakefile + names, so it is the merge-base with `origin/main`, not HEAD: a local + branch commit would generate a workspace whose build fails at fetch time. + The importer stops if the selected source differs from that revision. + Otherwise it could combine a working-tree statement with an older imported + context. + """ + manifest = json.loads((ROOT / "lake-manifest.json").read_text()) + mathlib_rev = next(p["rev"] for p in manifest["packages"] if p["name"] == "mathlib") + merge_base = subprocess.run( + ["git", "-C", str(ROOT), "merge-base", "HEAD", "origin/main"], + capture_output=True, + text=True, + ) + if merge_base.returncode != 0 or not merge_base.stdout.strip(): + raise SystemExit("cannot resolve the Formal Conjectures source revision") + fc_rev = merge_base.stdout.strip() + if source_path is not None: + comparison = subprocess.run( + ["git", "-C", str(ROOT), "diff", "--quiet", fc_rev, "--", str(source_path)] + ) + if comparison.returncode not in (0, 1): + raise SystemExit(f"cannot compare {source_path} with {fc_rev[:12]}") + if comparison.returncode == 1: + raise SystemExit( + f"{source_path} differs from pinned revision {fc_rev[:12]}; " + "land the source on upstream main before generating" + ) + return mathlib_rev, fc_rev + + +def source_record(declaration, module, source_path, fc_rev, dependencies, original): + """Where the copied statement and its dependencies came from. + + lean-eval#536 requires the manifest to record the FC source commit and + declaration id, and it is the FC side that has to supply them: the + generator sees a Lean module, not a repository. They are also what makes + the importer's regeneration duty possible — when Formal Conjectures fixes + a misformalisation upstream, this record says which problem to redo. + """ + blob = subprocess.run( + ["git", "-C", str(ROOT), "rev-parse", f"{fc_rev}:{source_path}"], + capture_output=True, + text=True, + check=False, + ) + return SourceRecord( + repository=SOURCE_REPOSITORY, + commit=fc_rev, + path=str(source_path), + blob_sha=blob.stdout.strip() or "", + module=module, + declaration=declaration, + copied_dependencies=tuple(dependencies), + original_declaration=original, + ) + + +def import_problem(problem, answer_type=None, module=None): + """Map one declaration to a marked-up module and a manifest. + + Importing a closure out of a repository full of `sorry` is safe because + Comparator checks axioms. A solution closing the goal with a copied + statement reports `sorryAx`, which `permitted_axioms` does not allow. + """ + problem_file = load_manifest(problem) + declaration = problem_file.get("declaration", problem) + # An argument given on the command line is explicit, so it wins over the + # problem file; the file is the durable record of the same choice. + answer_type = answer_type or problem_file.get("answer_type") + module = module or problem_file.get("module") + path, _imports, _module_doc, _body = find_declaration(declaration, module) + fc_module = module_name(path.relative_to(ROOT)) + facts = elaborator_facts(fc_module, declaration) + if facts["range"] is None: + raise SystemExit(f"{declaration}: no source range recorded") + + source_lines = path.read_text(encoding="utf-8").split("\n") + original, lo = slice_range(source_lines, facts["range"]) + statement = original + + preamble, namespaces_at_target = file_scoped_preamble(source_lines, lo) + dependencies, copied = closure_region( + facts.get("dependencies", []), + facts.get("generatedDependencies", []), + declaration, + namespaces_at_target, + ) + + statement = strip_decorations(statement) + statement = replace_proof_with_sorry(statement) + declared = None + for line in statement.split("\n"): + dm = DECL_START.match(line) + if dm: + declared = re.match(r"\s*([\w.«»]+)", line[dm.end() :]).group(1) + break + if declared is None: + raise SystemExit(f"{declaration}: no declaration line in the slice") + statement, holes = hoist_answers( + statement, declared, facts.get("answerTypes", []), answer_type + ) + # A `research solved` statement carries its answer rather than a `sorry` + # slot, so nothing above removed it and `answer(` would reach a workspace + # that cannot parse it. + statement = unwrap_answers(statement) + + args = [b["name"] for b in facts["binders"] if b["explicit"]] + bad = [a for a in args if "✝" in a or "._" in a] + if bad: + raise SystemExit( + f"{declared} has inaccessible explicit binders {bad}; the " + "Solution adapter cannot apply them by name" + ) + + # `open A`, then `open A.B`: opening the inner namespace does not open the + # outer one, and a statement may name siblings from either. + opens = [ + f"open {'.'.join(namespaces_at_target[: i + 1])}" + for i in range(len(namespaces_at_target)) + ] + + mathlib_rev, fc_rev = pins(path.relative_to(ROOT)) + marked_up = MarkedUpModule( + dependencies=dependencies, + scope="\n".join(opens + preamble), + holes="\n\n".join(hole.declaration() for hole in holes), + statement=statement, + ) + manifest = ProblemManifest( + id=problem_file.get("id", declared), + theorem=declared, + qualified_theorem=".".join(namespaces_at_target + [declared]), + apply_arguments=tuple(args), + holes=tuple(holes), + permitted_axioms=PERMITTED_AXIOMS, + lean_toolchain=(ROOT / "lean-toolchain").read_text(encoding="utf-8").strip(), + mathlib_revision=mathlib_rev, + source=source_record( + declared, + fc_module, + path.relative_to(ROOT), + fc_rev, + [dep["name"] for dep in facts.get("dependencies", [])], + original, + ), + tools=tool_pins(), + source_url=str(problem_file.get("source", "")), + notes=str(problem_file.get("notes", "")), + ) + return marked_up, manifest + + +def elaborate(marked_up): + """Elaborate the marked-up module against this checkout's Mathlib. + + Copying a closure is a construction, and its failure modes are the ones + Lean sees and a reader does not: a lost `open`, an unrecognised + `local notation`, a namespace that no longer exists because nothing + declares it any more. Each of those is a clean build away from being + caught and a long review away from being spotted. + + The check runs here rather than on a generated workspace because the + module is what this repository hands over: an FC-side defect should fail + on the FC side, not in lean-eval's CI. It is offline, and it uses the + Mathlib revision the manifest pins because that is this checkout's. It + checks elaboration, not a lakefile; a Comparator run exercises the build. + """ + with tempfile.NamedTemporaryFile( + "w", suffix=".lean", delete=False, encoding="utf-8" + ) as handle: + handle.write(marked_up.render()) + combined = handle.name + try: + proc = subprocess.run( + ["lake", "env", "lean", combined], + capture_output=True, + text=True, + cwd=ROOT, + check=False, + ) + finally: + pathlib.Path(combined).unlink(missing_ok=True) + output = (proc.stdout + proc.stderr).replace(combined, "Problem") + # Only errors fail the check. The target statement's proof is `sorry` by + # construction and each `answer(sorry)` hole is one the solver fills, so + # those warnings are the importer working. Linter warnings such as + # `unused variable` come from the copied source and say nothing about + # whether the copy is faithful. + errors = [line for line in output.splitlines() if "error:" in line] + if proc.returncode != 0 or errors: + raise SystemExit( + "the marked-up module does not elaborate:\n" + + "\n".join(errors or output.splitlines()[-10:]) + ) + return 0 + + +def validate(): + """Check every FC problem file resolves to exactly one declaration. + + Run this rather than discovering a stale `module` field when someone + imports the problem months later. + """ + bad = 0 + for problem_id in manifest_ids(): + try: + problem_file = load_manifest(problem_id) + declaration = problem_file["declaration"] + path, _i, _d, _b = find_declaration(declaration, problem_file.get("module")) + elaborator_facts(module_name(path.relative_to(ROOT)), declaration) + except SystemExit as exc: + print(f"{problem_id}: {exc}", file=sys.stderr) + bad += 1 + continue + print(f"{problem_id}: {declaration} in {path.relative_to(ROOT)}") + if bad: + print(f"{bad} problem file(s) do not resolve", file=sys.stderr) + return 1 if bad else 0 diff --git a/scripts/leaneval_generator.py b/scripts/leaneval_generator.py new file mode 100644 index 0000000000..d2a1132d61 --- /dev/null +++ b/scripts/leaneval_generator.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Turn a marked-up module and a manifest into a Challenge/Solution workspace. + +**This file is the placeholder for a dependency, and it is meant to be +deleted.** `leanprover/lean-eval#536` extracts lean-eval's generator core into +`leanprover/lean-eval-generator`, consumed as a pinned dependency by lean-eval +and by this importer, and says in as many words that the Formal Conjectures +importer does not fork the generation logic. Until that repository exists there +is nothing to pin, so this module stands in for it, deliberately holding +everything that is not Formal Conjectures' to own: + +- the workspace layout and every file emitted into it; +- which generated module imports which, and where the scope directives are + restated so the same statement text elaborates in all three files; +- the lakefile, the toolchain file and the Mathlib requirement; +- the fixed Solution adapter that pins the statement; +- the Comparator `config.json` shape. + +It reads nothing from this repository except the marked-up module, the +manifest, and the workspace test template it copies. It never resolves a +declaration, reads Lean source, or runs Lean. When `lean-eval-generator` lands, +this file is deleted, `generate` becomes a call into the pinned package, and +`scripts/fc_leaneval_importer.py` does not change. + +See `comparator/OWNERSHIP.md` for the line counts either side of that deletion. +""" + +import json +import pathlib + +from leaneval_interface import slug + +ROOT = pathlib.Path(__file__).resolve().parent.parent +# The workspace test template lean-eval's generator supplies for its own +# workspaces; it is vendored here only while this module stands in for it. +TEMPLATE_DIR = ROOT / "comparator" / "templates" + +PROOF_SUFFIX = ":= by\n sorry" + + +def lakefile(package, mathlib_rev): + """Mathlib and nothing else. + + The workspace used to require Formal Conjectures too, so that the + Challenge could import the problem's module. lean-eval vendors its + problems and cannot fetch that repository at evaluation time, so the + closure travels in `ChallengeDeps.lean` instead and the require is gone. + The commit the copy came from is recorded in `manifest.json`, which is + where a reader should look for it. + """ + return f"""name = "{package}" +testDriver = "workspace_test" +defaultTargets = ["ChallengeDeps", "Challenge", "Solution", "Submission"] + +[leanOptions] +autoImplicit = false + +[[require]] +name = "mathlib" +git = "https://github.com/leanprover-community/mathlib4.git" +rev = "{mathlib_rev}" + +[[lean_lib]] +name = "ChallengeDeps" + +[[lean_lib]] +name = "Challenge" + +[[lean_lib]] +name = "Solution" + +[[lean_lib]] +name = "Submission" + +[[lean_exe]] +name = "workspace_test" +root = "WorkspaceTest" +""" + + +def _readme(package, manifest): + holes_line = ( + "\nFill each definition hole in `Submission.lean` too. Hole answers " + "also get a\nhuman check, because a hole can be gamed in ways the " + "comparator cannot see.\nChecking holes needs a comparator built at " + f"commit `{manifest.tools['comparator'][:8]}`, which\nadded definition " + "support.\n" + if manifest.holes + else "" + ) + fields = "".join( + f"- {label}: {' '.join(str(value).split())}\n" + for label, value in (("Source", manifest.source_url), ("Notes", manifest.notes)) + if value + ) + return ( + f"# {package}\n\n" + f"A comparator challenge for `{manifest.theorem}`, generated from\n" + f"`{manifest.source.path}` in google-deepmind/formal-conjectures.\n\n" + + fields + + "\nProve the statement in `Submission.lean`, keeping it as it stands; " + "put helper\nmodules under `Submission/` if you need them. Do not " + "modify `Challenge.lean` or\n`Solution.lean`: the trusted statement " + "lives there, and `Solution.lean` closes it\nwith your `Submission` " + "theorem, so it fails to compile if the submission proves\nanything " + "else.\n" + "\nComparator accepts the workspace only if the statement is proved " + "under the\naxioms in `config.json`. `sorry` adds `sorryAx`, which is " + "not permitted, and\nclosing the goal with the imported original " + "fails the same way, since that is\n`sorry` too. `lake test` runs " + "comparator, from `PATH` or `COMPARATOR_BIN`.\n" + "\nIf comparator fails with `incompatible header` on an `.olean`, the " + "mismatch is\nbetween this workspace's toolchain and the one " + "`lean4export` was built with,\nnever a problem with the proof: copy " + "this workspace's `lean-toolchain` into\nyour `lean4export` checkout, " + "rebuild it, and clear `.lake/build` here.\n" + + holes_line + + "\nFetch the Mathlib cache before the first build; a cold build takes " + "the best\npart of an hour without it:\n\n" + " lake exe cache get\n" + " lake build\n" + ) + + +HELPERS = ( + "import Mathlib\n\n" + "/-! Helper lemmas for the submission go here, or in further modules\n" + "under `Submission/`, each imported from `Submission.lean`. -/\n\n" + "namespace Submission\n\nend Submission\n" +) + + +def generate(marked_up, manifest): + """The workspace files for one problem, as a path-to-content mapping. + + Pure: it writes nothing, and it reads nothing but its two arguments and + the workspace test template. Putting the result on disk is the caller's. + + The three Lean files carry the same statement text, so what that text + needs to elaborate has to be restated in each: that is what the module's + `scope` region is for, and placing it is this side's job. `ChallengeDeps` + takes the module's dependency region and the `import Mathlib` that makes + the workspace stand on Mathlib alone; the other three import it. + """ + package = slug(manifest.id) + header = marked_up.scope.strip("\n") + header = header + "\n\n" if header else "" + holes = marked_up.holes.strip("\n") + holes = holes + "\n\n" if holes else "" + statement = marked_up.statement.strip("\n") + + signature = statement.rstrip() + if signature.endswith(PROOF_SUFFIX): + signature = signature[: -len(PROOF_SUFFIX)].rstrip() + + challenge = "import ChallengeDeps\n\n" + header + holes + statement + "\n" + + # The participant's file. The statement sits inside `namespace Submission` + # so nothing here can collide with, or stand in for, the trusted names. + submission = ( + "import ChallengeDeps\nimport Submission.Helpers\n\n" + + header + + "namespace Submission\n\n" + + holes + + statement + + "\n\nend Submission\n" + ) + + # The fixed adapter, lean-eval's shape: it restates the trusted statement + # and closes it with the Submission theorem, so it fails to compile the + # moment the submission proves anything else. The participant never edits + # it, which is what keeps the statement pinned. + delegated = "".join( + f"noncomputable def {hole.name} : {hole.type} := Submission.{hole.name}\n\n" + for hole in manifest.holes + ) + solution = ( + "import ChallengeDeps\nimport Submission\n\n" + + header + + delegated + + signature + + " :=\n Submission." + + manifest.theorem + + "".join(" " + argument for argument in manifest.apply_arguments) + + "\n" + ) + + config = { + "challenge_module": "Challenge", + "solution_module": "Solution", + "theorem_names": [manifest.theorem], + "permitted_axioms": list(manifest.permitted_axioms), + "enable_nanoda": False, + } + if manifest.holes: + # Comparator's documented no-hole config carries no such field. + config["definition_names"] = manifest.hole_names() + + return { + "lakefile.toml": lakefile(package, manifest.mathlib_revision), + "lean-toolchain": manifest.lean_toolchain + "\n", + "README.md": _readme(package, manifest), + "ChallengeDeps.lean": "import Mathlib\n\n" + + marked_up.dependencies.strip("\n") + + "\n", + "Challenge.lean": challenge, + "Solution.lean": solution, + "Submission.lean": submission, + "Submission/Helpers.lean": HELPERS, + "WorkspaceTest.lean": (TEMPLATE_DIR / "WorkspaceTest.lean").read_text( + encoding="utf-8" + ), + "config.json": json.dumps(config, indent=2) + "\n", + # The manifest crosses into the workspace unaltered. lean-eval#536 + # requires it to record the FC source commit and declaration id, and + # this side neither supplies nor edits those. + "manifest.json": manifest.to_json(), + } diff --git a/scripts/leaneval_interface.py b/scripts/leaneval_interface.py new file mode 100644 index 0000000000..06bd37c369 --- /dev/null +++ b/scripts/leaneval_interface.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +"""The one interface between the Formal Conjectures importer and the generator. + +`leanprover/lean-eval#536` splits this work in two. The generator core inside +lean-eval's `EvalTools` — the part that turns a marked-up Lean module plus a +manifest into a Challenge / Solution / Submission workspace — is being +extracted into `leanprover/lean-eval-generator` and consumed as a pinned +dependency. The Formal Conjectures side owns an importer that maps FC +declarations and metadata to LeanEval modules and manifests. The FC importer +does not fork the generation logic. + +This module is that seam, and nothing else. It holds the two values the +importer hands the generator and no code that produces or consumes them: + + MarkedUpModule one Mathlib-only Lean module, in labelled regions + ProblemManifest the facts about the problem that the module's text does + not carry, including the FC source commit and the FC + declaration id + +`scripts/fc_leaneval_importer.py` produces both. `scripts/leaneval_generator.py` +consumes both and returns a workspace. When `lean-eval-generator` lands, the +generator module goes and this file becomes an import from the pinned package; +the importer keeps building the same two values and does not change. + +## Why a marked-up module rather than a bag of strings + +The generator's job includes the import and scope fidelity work from +lean-eval#531: deciding which generated file imports which, and where the +file-scoped `open`, `variable` and notation have to be restated so that the +same statement text elaborates in Challenge, Submission and Solution alike. +That decision belongs to the generator, so the importer must not pre-split the +source into those files. It emits one module that elaborates on its own +against Mathlib, with the four parts labelled, and the generator slices it. + +Emitting one module also gives the importer a check it could not otherwise +have: the module it is about to hand over is exactly the text it can elaborate +locally (`--verify`), so a copied closure that has lost an `open` fails on the +FC side rather than in lean-eval's CI. + +The regions, in the order they must appear: + + dependencies the FC-local closure of the statement, copied, Mathlib-only + scope the `open` and file-scoped directives the statement needs + holes one `noncomputable def : := sorry` per + `answer(sorry)` slot the importer hoisted + statement the target statement, its proof replaced by `sorry` +""" + +import dataclasses +import json +import re + +REGION_MARKER = "-- @region " +REGIONS = ("dependencies", "scope", "holes", "statement") + +MODULE_PREAMBLE = "import Mathlib\n" + +MANIFEST_SCHEMA_VERSION = 1 + + +def slug(name): + """A Lake package name and directory name for a problem id. + + A Lake package name is an identifier, so the dots in a qualified + declaration cannot go into one verbatim. + """ + return re.sub(r"[^0-9A-Za-z_]", "_", name) + + +@dataclasses.dataclass(frozen=True) +class DefinitionHole: + """One `answer(sorry)` slot, hoisted into a definition the solver fills. + + `name` is the unqualified definition name as it appears in the module's + `holes` region; `type` is the type the elaborated environment reported for + the slot, which surface syntax does not carry. + """ + + name: str + type: str + + def declaration(self): + return f"noncomputable def {self.name} : {self.type} := sorry" + + +@dataclasses.dataclass(frozen=True) +class SourceRecord: + """Where the marked-up module's text came from. + + lean-eval#536 requires that each manifest record the FC source commit and + declaration id. Neither is recoverable from the Lean text, and neither is + something the generator can supply: the generator sees a module, not a + repository. So they cross the seam here, and the generator's only duty is + to carry them into the workspace unaltered. + """ + + repository: str + commit: str + path: str + blob_sha: str + module: str + declaration: str + copied_dependencies: tuple + original_declaration: str + + +@dataclasses.dataclass(frozen=True) +class ProblemManifest: + """What the marked-up module's text does not say. + + `theorem` is the statement's own unqualified name, which the generator + needs for the Solution adapter, and `qualified_theorem` is that name under + the namespace the scope region reopens, which is what Comparator checks. + `apply_arguments` are the statement's explicit declaration parameters, in + order: the Solution adapter applies them by name, and `∀` binders in the + conclusion are not among them. + """ + + id: str + theorem: str + qualified_theorem: str + apply_arguments: tuple + holes: tuple + permitted_axioms: tuple + lean_toolchain: str + mathlib_revision: str + source: SourceRecord + tools: dict + source_url: str = "" + notes: str = "" + + def __post_init__(self): + for field in ("id", "theorem", "qualified_theorem", "lean_toolchain"): + if not getattr(self, field): + raise SystemExit(f"manifest has no {field}") + # lean-eval#536 names these two explicitly, and a manifest without + # them cannot be traced back to a revision of this repository or + # regenerated when FC fixes a misformalisation upstream. + if not self.source.commit: + raise SystemExit(f"manifest {self.id} records no FC source commit") + if not self.source.declaration: + raise SystemExit(f"manifest {self.id} records no FC declaration id") + + def hole_names(self): + return [hole.name for hole in self.holes] + + def to_json_object(self): + payload = { + "schema_version": MANIFEST_SCHEMA_VERSION, + "id": self.id, + "theorem": self.theorem, + "qualified_theorem": self.qualified_theorem, + "apply_arguments": list(self.apply_arguments), + "holes": [dataclasses.asdict(hole) for hole in self.holes], + "permitted_axioms": list(self.permitted_axioms), + "lean_toolchain": self.lean_toolchain, + "mathlib_revision": self.mathlib_revision, + "source": { + **dataclasses.asdict(self.source), + "copied_dependencies": list(self.source.copied_dependencies), + }, + "tools": dict(self.tools), + } + if self.source_url: + payload["source_url"] = self.source_url + if self.notes: + payload["notes"] = self.notes + return payload + + @classmethod + def from_json_object(cls, payload): + version = payload.get("schema_version") + if version != MANIFEST_SCHEMA_VERSION: + raise SystemExit( + f"manifest schema version {version!r} is not " + f"{MANIFEST_SCHEMA_VERSION}" + ) + source = dict(payload["source"]) + source["copied_dependencies"] = tuple(source["copied_dependencies"]) + return cls( + id=payload["id"], + theorem=payload["theorem"], + qualified_theorem=payload["qualified_theorem"], + apply_arguments=tuple(payload["apply_arguments"]), + holes=tuple(DefinitionHole(**hole) for hole in payload["holes"]), + permitted_axioms=tuple(payload["permitted_axioms"]), + lean_toolchain=payload["lean_toolchain"], + mathlib_revision=payload["mathlib_revision"], + source=SourceRecord(**source), + tools=dict(payload["tools"]), + source_url=payload.get("source_url", ""), + notes=payload.get("notes", ""), + ) + + def to_json(self): + return json.dumps(self.to_json_object(), indent=2, ensure_ascii=False) + "\n" + + @classmethod + def from_json(cls, text): + return cls.from_json_object(json.loads(text)) + + +@dataclasses.dataclass(frozen=True) +class MarkedUpModule: + """One Mathlib-only Lean module, in the four labelled regions. + + Rendering and parsing are inverse on the region bodies, so the artifact + the importer emits for review is the artifact the generator reads. + """ + + dependencies: str + scope: str + holes: str + statement: str + + def __post_init__(self): + # Rendering separates the regions itself, so leading and trailing + # blank lines are not part of a region's content. Normalising them + # here is what makes rendering and parsing inverse. + for name in REGIONS: + object.__setattr__(self, name, getattr(self, name).strip("\n")) + + def regions(self): + return {name: getattr(self, name) for name in REGIONS} + + def render(self): + parts = [MODULE_PREAMBLE] + for name, body in self.regions().items(): + body = body.strip("\n") + # A copied declaration carrying a line that reads as a marker + # would split the module somewhere the importer did not choose, + # and the generator would never know. Refuse instead. + for line in body.split("\n"): + if line.startswith(REGION_MARKER): + raise SystemExit( + f"the {name} region contains a region marker: {line!r}" + ) + parts.append(f"\n{REGION_MARKER}{name}\n" + (body + "\n" if body else "")) + return "".join(parts) + + @classmethod + def parse(cls, text): + """Read a rendered module back, refusing anything the shape forbids.""" + bodies, current = {}, None + for line in text.split("\n"): + if line.startswith(REGION_MARKER): + current = line[len(REGION_MARKER) :].strip() + if current not in REGIONS: + raise SystemExit(f"unknown region {current!r} in marked-up module") + if current in bodies: + raise SystemExit(f"region {current!r} appears twice") + bodies[current] = [] + continue + if current is not None: + bodies[current].append(line) + missing = [name for name in REGIONS if name not in bodies] + if missing: + raise SystemExit( + "marked-up module has no " + ", ".join(f"`{m}`" for m in missing) + + " region" + ) + if list(bodies) != list(REGIONS): + raise SystemExit( + "marked-up module regions are out of order: " + + ", ".join(bodies) + + f"; expected {', '.join(REGIONS)}" + ) + return cls( + **{name: "\n".join(lines) for name, lines in bodies.items()} + ) diff --git a/scripts/make_comparator_workspace.py b/scripts/make_comparator_workspace.py index 42a93102dc..e581ebab7c 100644 --- a/scripts/make_comparator_workspace.py +++ b/scripts/make_comparator_workspace.py @@ -1,23 +1,32 @@ #!/usr/bin/env python3 -"""Generate a comparator workspace for one problem statement. +"""Import one Formal Conjectures declaration and generate its workspace. `leanprover/lean-eval` verifies a submission by building it against a Challenge module whose statement the maintainers trust, under a config that pins the -permitted axioms. This script generates that shape for one Formal Conjectures -declaration. - -The generated workspace requires Mathlib and nothing else. lean-eval vendors -its problems, so a Challenge cannot fetch this repository at evaluation time, -which rules out importing the problem's own module. This repository's -statements are not authored self-contained, so the declarations a statement -needs are copied into `ChallengeDeps.lean` instead, dependencies first, each -carrying the `open`, `variable`, `universe`, `set_option` and `local notation` -in force where it was written. +permitted axioms. This command produces that shape for one Formal Conjectures +declaration, in the two steps `leanprover/lean-eval#536` separates: + + fc_leaneval_importer FC declaration -> marked-up module + manifest + leaneval_generator marked-up module + manifest -> workspace + +The first half is Formal Conjectures'. The second half is lean-eval's, and is +to be replaced by a pinned dependency on `leanprover/lean-eval-generator`; the +module standing in for it here is the code that gets deleted when that lands. +`comparator/OWNERSHIP.md` says exactly what goes and what stays. This file is +the wiring between them and belongs to neither. + +The marked-up module requires Mathlib and nothing else. lean-eval vendors its +problems, so a Challenge cannot fetch this repository at evaluation time, which +rules out importing the problem's own module. This repository's statements are +not authored self-contained, so the declarations a statement needs are copied +into the module's dependency region, dependencies first, each carrying the +`open`, `variable`, `universe`, `set_option` and `local notation` in force +where it was written. Copying is a construction and it can be wrong in ways only Lean sees, so -`--verify` builds the generated workspace before you trust it. +`--verify` elaborates the marked-up module before you trust it. -Layout produced: +Layout produced by the generator: // lakefile.toml pins: this checkout's Mathlib rev @@ -34,706 +43,45 @@ WorkspaceTest.lean `lake test` runs comparator on config.json README.md what the solver needs to know, cache fetch included config.json theorem and definition names, permitted axioms - holes.json the extracted blocks, for tooling and for review - -Lean reports the type of each `answer(sorry)` slot. The generator refuses a -case when it cannot match the reported types to their source positions. + manifest.json the manifest the importer handed the generator: the FC + source commit and declaration id, the copied closure, + the hole types, and the pins -Two things the source cannot settle live in `comparator/problems/.toml`, -one file per problem: that answer type, and which file is meant when two -declare the same name. See that directory's README. +Lean reports the type of each `answer(sorry)` slot. The importer refuses a case +when it cannot match the reported types to their source positions. Usage: python make_comparator_workspace.py (ID | DECLARATION) [--out DIR] - [--answer-type T] [--module FILE] + [--answer-type T] [--module FILE] [--verify] + python make_comparator_workspace.py ID --emit-import DIR python make_comparator_workspace.py --validate The workspace's own build needs a network fetch of its pinned dependencies, so -this script does not attempt it; generation is offline and the build belongs to -the comparator run. +this command does not attempt it; generation is offline and the build belongs +to the comparator run. """ import argparse -import json import pathlib -import re import shutil -import subprocess import sys import tempfile -import tomllib - -ROOT = pathlib.Path(__file__).resolve().parent.parent -SOURCE_DIRS = [ROOT / "FormalConjectures"] -COMPARATOR_DIR = ROOT / "comparator" -MANIFEST_DIR = COMPARATOR_DIR / "problems" - - -def tool_pins(): - """The locked external tool revisions; comparator/tools.toml is the one - machine-readable source, and this module refuses to restate it.""" - with (COMPARATOR_DIR / "tools.toml").open("rb") as handle: - return tomllib.load(handle)["tools"] - - -PERMITTED_AXIOMS = ["propext", "Quot.sound", "Classical.choice"] - -DECL_START = re.compile( - # `local notation` and `scoped notation` carry the modifier before the - # keyword. Without them here, Erdos 125's `local notation "A" => ...` typed - # as nothing and was dropped, and its statements lost the sets they name. - r"^(?:noncomputable\s+|private\s+|protected\s+|local\s+|scoped\s+)*" - r"(theorem|lemma|def|abbrev|structure|inductive|instance|notation)\s", -) -KEEP_LOOSE = re.compile( - r"^(open|variable|universe|section|namespace|end|attribute|set_option)\b" -) - - -def elaborator_facts(module, declaration): - """What the elaborated environment knows about a declaration. - - Runs `lake exe comparator_facts`, which imports the module and reports the - declaration's source range, its binders with real explicitness, and the - inferred type of each `answer(sorry)` slot. Every one of these used to be - reconstructed from text, and each reconstruction had failure modes the - elaborator does not. - """ - proc = subprocess.run( - ["lake", "exe", "comparator_facts", module, declaration], - capture_output=True, - text=True, - cwd=ROOT, - ) - if proc.returncode != 0: - raise SystemExit( - f"comparator_facts {declaration}: " - f"{proc.stderr.strip() or proc.stdout.strip()}" - ) - out = proc.stdout - if "{" not in out: - raise SystemExit(f"comparator_facts {declaration}: no JSON in output") - return json.loads(out[out.index("{") :]) - - -def file_scoped_preamble(lines, start_line): - """Directives in force at `start_line`, and the namespace stack there. - - Lean scopes `open`, `variable`, `universe`, `set_option` and notation to - the file, so Challenge.lean has to restate them; nothing in the olean - records them. A directive counts only if it precedes the statement and - its scope still encloses it. - """ - stack, preamble, depth = [], [], 0 - for line in lines[: start_line - 1]: - if depth == 0 and KEEP_LOOSE.match(line) and not line.rstrip().endswith(" in"): - kind = line.split()[0] - parts = line.split(None, 1) - name = parts[1].strip() if len(parts) > 1 else None - if kind in ("namespace", "section"): - stack.append((kind, name)) - elif kind == "end": - if stack and ( - stack[-1][1] == name or (name is None and stack[-1][0] == "section") - ): - stack.pop() - else: - preamble.append((line, list(stack))) - depth += len(re.findall(r"/-", line)) - len(re.findall(r"-/", line)) - depth = max(depth, 0) - scope = list(stack) - in_force = [text for text, s in preamble if s == scope[: len(s)]] - return in_force, [n for k, n in scope if k == "namespace" and n] - - -def slug(name): - """A Lake package name and directory name for a declaration. - - A Lake package name is an identifier, so the dots in a qualified - declaration cannot go into one verbatim. - """ - return re.sub(r"[^0-9A-Za-z_]", "_", name) - - -def load_manifest(problem_id): - """Read explicit choices that Lean source cannot select by itself. - - A manifest selects the module when names collide. It may also override an - answer-slot type when Lean reports several types that cannot be matched to - source positions. The generator refuses both cases without an explicit - choice. - - `leanprover/lean-eval` keeps one TOML per problem, and the reason is worth - copying: two pull requests adding different problems never touch the same - file. - - id the filename stem, and the workspace directory name - declaration the Lean name, which need not be unique across the repository - module the file declaring it, relative to the repository root - answer_type the type of a non-`Prop` answer slot - notes free text for a reviewer - source a citation or URL - """ - path = MANIFEST_DIR / f"{problem_id}.toml" - if not path.exists(): - return {} - with path.open("rb") as handle: - data = tomllib.load(handle) - if data.get("id") != problem_id: - raise SystemExit( - f"{path} declares id {data.get('id')!r}, but its filename says " - f"{problem_id!r}; the two must agree" - ) - if "declaration" not in data: - raise SystemExit(f"{path} has no `declaration` field") - return data - - -def manifest_ids(): - return sorted(p.stem for p in MANIFEST_DIR.glob("*.toml")) - - -def module_name(rel_path): - """The Lean module name for a path under `FormalConjectures/`. - - Most problem files are named for a number, which is not an identifier, so - the component is written in guillemets: - `FormalConjectures.ErdosProblems.«940»`. - """ - parts = [ - c if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", c) else f"«{c}»" - for c in str(rel_path)[: -len(".lean")].split("/") - ] - return ".".join(parts) - - -def find_declaration(basename, module=None): - """Locate the file declaring `basename`. Returns (path, module_docstring, body). - - `module` names the file when more than one declares the name, and comes - from the problem's manifest. - """ - if module is not None: - named = ROOT / module - if not named.exists(): - raise SystemExit(f"manifest names {module}, which does not exist") - return _read_source(named) - hits = [] - for src in SOURCE_DIRS: - for path in sorted(src.rglob("*.lean")): - text = path.read_text(encoding="utf-8") - if re.search( - rf"(?:theorem|lemma)\s+(?:[\w.«»]*\.)?{re.escape(basename)}[\s:]", text - ): - hits.append(path) - if not hits: - raise SystemExit( - f"no declaration named {basename!r} found under FormalConjectures/" - ) - if len(hits) > 1: - raise SystemExit( - f"{basename!r} is ambiguous: " - + ", ".join(str(h.relative_to(ROOT)) for h in hits) - + "; pass --module to choose one, or record the choice in " - "comparator/problems/.toml" - ) - return _read_source(hits[0]) - - -def _read_source(path): - text = path.read_text(encoding="utf-8") - # Drop the license header; keep the module docstring; the rest is the body. - text = re.sub(r"\A/-.*?-/\s*", "", text, flags=re.DOTALL) - doc = "" - m = re.match(r"\s*(/-!.*?-/)\s*", text, flags=re.DOTALL) - if m: - doc = m.group(1) - text = text[m.end() :] - # Imports precede the docstring in source order; recover them from the original. - imports = re.findall( - r"^import\s+(\S+)", path.read_text(encoding="utf-8"), re.MULTILINE - ) - return path, imports, doc, text - - -def strip_decorations(block_text): - """Remove the docstring, line comments and attributes from a declaration. - - These interleave. Erdos 918 puts a `--` formalisation note between its - docstring and its `@[category ...]` line, and one anchored pass each left - the attribute in place. `@[category research open, AMS 5]` then reached - Challenge.lean, where the workspace has no such attribute, and Lean parsed - as far as the `open` inside it before giving up. - """ - # `open X in` binds to the declaration and has to survive, but it sits - # above the docstring, so stripping anchored at the start would stop dead - # on it. - prefix = "" - m = re.match(r"\A\s*(open\b[^\n]*\bin)\n", block_text) - if m: - prefix = m.group(1) + "\n" - block_text = block_text[m.end() :] - while True: - stripped = re.sub(r"\A\s*/--.*?-/\s*", "", block_text, flags=re.DOTALL) - stripped = re.sub(r"\A\s*--[^\n]*\n", "", stripped) - stripped = re.sub(r"\A\s*@\[[^\]]*\]\s*", "", stripped, flags=re.DOTALL) - if stripped == block_text: - return prefix + stripped - block_text = stripped - - -# Attributes this repository defines. A generated workspace requires Mathlib -# and nothing else, so these have to go; everything else has to stay. -FC_ATTRIBUTES = ("category", "AMS", "formal_proof") - - -def strip_fc_attributes(block_text): - """Remove this repository's own attributes from a copied declaration. - - Unlike `strip_decorations`, which clears every attribute off the target - statement, this keeps the rest. A dependency is copied to be elaborated, - not restated, and dropping `simp`, `reducible` or `instance` attributes - changes how the declarations after it in the same closure elaborate. - """ - - def replace(match): - inner = match.group(1) - # Nested brackets mean an argument this simple split would cut in - # half, so leave the whole attribute alone rather than mangle it. - if "[" in inner: - return match.group(0) - kept = [ - part.strip() - for part in inner.split(",") - if part.strip() and part.strip().split()[0] not in FC_ATTRIBUTES - ] - return f"@[{', '.join(kept)}]" if kept else "" - - text = re.sub(r"@\[([^\]]*)\]", replace, block_text) - # An attribute line that emptied out leaves a blank line behind. - return re.sub(r"^[ \t]*\n", "", text, flags=re.MULTILINE) - - -def module_source_path(module): - """The file declaring a dotted Lean module name, undoing guillemets.""" - parts = [ - component[1:-1] if component.startswith("«") else component - for component in module.split(".") - ] - path = ROOT.joinpath(*parts).with_suffix(".lean") - if not path.is_file(): - raise SystemExit(f"{module}: no source file at {path}") - return path - - -def slice_range(lines, source_range): - """The source text a declaration range covers, and the line it starts on. - - `open X in` binds to the declaration below it but sits above what the - range covers in some toolchains, so it is pulled in when present. - """ - lo, hi = source_range["startLine"], source_range["endLine"] - end_column = source_range.get("endColumn") - while ( - lo > 1 - and lines[lo - 2].rstrip().endswith(" in") - and KEEP_LOOSE.match(lines[lo - 2]) - ): - lo -= 1 - sliced = lines[lo - 1 : hi] - if end_column is not None and sliced: - sliced = sliced[:-1] + [sliced[-1][:end_column]] - return "\n".join(sliced), lo - - -def challenge_deps(dependencies, generated, declaration, opened_namespaces=()): - """One Mathlib-only module carrying a declaration's FC-local closure. - - lean-eval vendors problems, so a generated Challenge cannot fetch this - repository at evaluation time and has to stand on Mathlib alone. That - rules out importing the problem's own module, and brings back the failure - modes an import does not have: file-scoped `open` and `variable` lost, - `local notation` unrecognised, a namespace swallowing what follows. - - So each declaration is emitted inside its own `section`, carrying the - preamble in force where it was written and reopening the namespace it was - written in. That is a construction, not a proof, and the only check that - covers every one of those failure modes at once is building the generated - workspace, which `--verify` does. - """ - copied = [dep["name"] for dep in dependencies] - orphans = [ - name - for name in generated - if not any(name.startswith(parent + ".") for parent in copied) - ] - if orphans: - raise SystemExit( - f"{declaration}: {len(orphans)} elaborator-generated constant(s) " - "have no copied ancestor, so copying the closure would not " - f"reproduce them: {', '.join(orphans[:5])}" - ) - - # A constructor, a `where` auxiliary and a `_sparseCasesOn` all carry a - # source range inside the declaration that produces them, so copying them - # in their own right either duplicates a declaration or slices a fragment - # of one. `MonochromaticQuantumGraph.EdgeN.mk` covers line 88 of a - # structure spanning 83 to 93; `pmSumListAux._sparseCasesOn_1` has exactly - # its parent's range. Copying the outer declaration reproduces both. - def covered_by_another(dep): - inner = dep["range"] - for other in dependencies: - if other is dep or other["module"] != dep["module"]: - continue - outer = other["range"] - if outer is None or inner is None: - continue - if not ( - outer["startLine"] <= inner["startLine"] - and outer["endLine"] >= inner["endLine"] - ): - continue - same_span = ( - outer["startLine"] == inner["startLine"] - and outer["endLine"] == inner["endLine"] - ) - # A tie on the span is broken by name: the parent is the prefix. - if not same_span or len(other["name"]) < len(dep["name"]): - return True - return False - - subsumed = [dep["name"] for dep in dependencies if covered_by_another(dep)] - dependencies = [dep for dep in dependencies if dep["name"] not in subsumed] - - blocks, provenance = [], [] - for dep in dependencies: - if dep["range"] is None: - raise SystemExit(f"{declaration}: {dep['name']} has no source range") - path = module_source_path(dep["module"]) - lines = path.read_text(encoding="utf-8").split("\n") - text, start = slice_range(lines, dep["range"]) - preamble, namespaces = file_scoped_preamble(lines, start) - body = strip_fc_attributes(text).strip("\n") - if not body: - raise SystemExit(f"{declaration}: {dep['name']} sliced to nothing") - namespace = ".".join(namespaces) - chunk = [f"-- {dep['name']}, from {path.relative_to(ROOT)}", "section"] - chunk += preamble - if namespace: - chunk.append(f"namespace {namespace}") - chunk += ["", body, ""] - if namespace: - chunk.append(f"end {namespace}") - chunk.append("end") - blocks.append("\n".join(chunk)) - provenance.append(dep["name"]) - - # Challenge.lean reopens the namespace stack the target sat in, so its - # statement can name siblings short. `open` on a namespace nothing has - # declared is an error, and with the problem's module no longer imported - # only the copied declarations can declare one. An empty namespace block - # is enough to make the name exist. - declared_namespaces = { - name.rsplit(".", 1)[0] for name in provenance if "." in name - } - for depth in range(len(opened_namespaces)): - prefix = ".".join(opened_namespaces[: depth + 1]) - if not any( - ns == prefix or ns.startswith(prefix + ".") for ns in declared_namespaces - ): - blocks.append(f"namespace {prefix}\nend {prefix}") - - listing = "\n".join(f"* `{name}`" for name in provenance) - return ( - "import Mathlib\n\n" - "/-!\n" - f"The Formal Conjectures declarations `{declaration}` needs, copied so\n" - "that `Challenge.lean` requires Mathlib and nothing else. Dependencies\n" - "come before the declarations that use them:\n\n" - f"{listing}\n" - "-/\n\n" + "\n\n".join(blocks) + "\n" - ) - - -def replace_proof_with_sorry(text): - """Cut the proof body after `:=`, keeping the statement. - - A tactic proof is found by `:= by`, which a statement cannot contain, - `by` being a keyword. A term proof leaves only a bare `:=` to cut at, and - a statement can contain one of those: a structure literal `{ a := b }` - inside the statement would be cut in half. With more than one candidate - the script refuses, as everywhere else it cannot decide. - """ - m = re.search(r":=\s*by\b", text) - if m: - return text[: m.start()].rstrip() + " := by\n sorry" - if text.count(":=") > 1: - raise SystemExit( - "the declaration has a term-mode proof and more than one `:=`, so " - "the start of the proof cannot be read off the text" - ) - m = re.search(r":=", text) - if m: - return text[: m.start()].rstrip() + " := by\n sorry" - return text.rstrip() + " := by\n sorry" +import fc_leaneval_importer as importer +import leaneval_generator as generator +from leaneval_interface import slug -def answer_spans(text): - """Return the source spans of syntactic `answer(...)` calls. - - This small lexer skips strings and nested line/block comments and balances - parentheses, so an answer term may itself contain parentheses. It is not a - Lean parser; malformed or unterminated syntax is refused. - """ - spans = [] - i = 0 - block_depth = 0 - in_string = False - escaped = False - while i < len(text): - pair = text[i : i + 2] - if block_depth: - if pair == "/-": - block_depth += 1 - i += 2 - elif pair == "-/": - block_depth -= 1 - i += 2 - else: - i += 1 - continue - if in_string: - if escaped: - escaped = False - elif text[i] == "\\": - escaped = True - elif text[i] == '"': - in_string = False - i += 1 - continue - if pair == "/-": - block_depth = 1 - i += 2 - continue - if pair == "--": - newline = text.find("\n", i + 2) - i = len(text) if newline < 0 else newline + 1 - continue - if text[i] == '"': - in_string = True - i += 1 - continue - if text.startswith("answer", i) and ( - i == 0 or not (text[i - 1].isalnum() or text[i - 1] in "_.'") - ): - j = i + len("answer") - while j < len(text) and text[j].isspace(): - j += 1 - if j < len(text) and text[j] == "(": - depth = 1 - k = j + 1 - nested_string = False - nested_escaped = False - nested_comment = 0 - while k < len(text) and depth: - nested_pair = text[k : k + 2] - if nested_comment: - if nested_pair == "/-": - nested_comment += 1 - k += 2 - elif nested_pair == "-/": - nested_comment -= 1 - k += 2 - else: - k += 1 - continue - if nested_string: - if nested_escaped: - nested_escaped = False - elif text[k] == "\\": - nested_escaped = True - elif text[k] == '"': - nested_string = False - k += 1 - continue - if nested_pair == "/-": - nested_comment = 1 - k += 2 - elif nested_pair == "--": - newline = text.find("\n", k + 2) - k = len(text) if newline < 0 else newline + 1 - elif text[k] == '"': - nested_string = True - k += 1 - else: - if text[k] == "(": - depth += 1 - elif text[k] == ")": - depth -= 1 - k += 1 - if depth: - raise SystemExit("unterminated answer(...) term") - spans.append((i, k, text[j + 1 : k - 1])) - i = k - continue - i += 1 - if block_depth or in_string: - raise SystemExit("unterminated comment or string while reading answers") - return spans - - -def unwrap_answers(statement): - """Replace any surviving `answer(t)` with `(t)`. - - `answer` is this repository's own elaborator, so a Mathlib-only workspace - cannot parse it. `hoist_answers` removes the `answer(sorry)` slots by - turning them into definition holes; a slot that already carries its answer, - which is how a `research solved` statement is written, is left behind and - used to reach Challenge.lean as literal text that does not parse. - - Unwrapping is faithful. In the default `postpone` mode the elaborator - elaborates the term and attaches an annotation - (`FormalConjecturesUtil/Answer.lean`), so `answer(t)` and `t` denote the - same term and only the annotation is lost. The annotation is what marks - which part of the statement was the question, and a generated Challenge - records that in `holes.json` instead. - """ - for start, end, argument in reversed(answer_spans(statement)): - statement = statement[:start] + f"({argument.strip()})" + statement[end:] - return statement +ROOT = importer.ROOT -def hoist_answers(statement, basename, slot_types, override=None): - """Replace each `answer(sorry)` with a named definition hole. +def write_tree(target, files): + """Write a complete directory without overwriting or leaving a partial one. - The slot types come from the elaborated environment, where the `answer` - elaborator ran with the expected type in hand; the old surface-syntax - guess (an `↔` beside the slot means `Prop`) and the manifest's hand-kept - `answer_type` both survive only as overrides. Slots of different types in - one statement are refused: the environment reports the types as a set, - and matching them to positions would be a guess. + Plumbing, and on neither side of the seam: the generator returns a + path-to-content mapping and never touches the filesystem, so putting one + on disk is the command's job whether the mapping is a workspace or the + pair this repository hands over. """ - holes = [] - calls = answer_spans(statement) - selected = [call for call in calls if call[2].strip() == "sorry"] - count = len(selected) - if count == 0: - return statement, holes - # Under the default `alwaysTrue` setting, the `answer` elaborator erases a - # slot to `True` if and only if its expected type is `Prop` - # (FormalConjecturesUtil/Answer.lean). So a slot the environment carries - # no annotation for is a `Prop` slot by the elaborator's own rule, not by - # guesswork, and no postpone build is needed. - missing = count - len(slot_types) - if override: - types = [override] * count - elif missing == count: - types = ["Prop"] * count - elif missing == 0 and len(set(slot_types)) == 1: - types = [slot_types[0]] * count - elif missing == 0: - raise SystemExit( - f"{basename} has {count} answer slots of differing types " - f"{slot_types}; pass --answer-type" - ) - else: - # Some slots are Prop and some are not: which positions are which - # cannot be read off an unordered set, so refuse rather than assign. - raise SystemExit( - f"{basename}: {missing} Prop slot(s) and {len(slot_types)} typed " - f"slot(s) {slot_types} cannot be matched to positions; pass " - "--answer-type" - ) - replacements = [] - for i, (start, end, _argument) in enumerate(selected): - hole = f"{basename}_answer" if count == 1 else f"{basename}_answer_{i + 1}" - holes.append(f"noncomputable def {hole} : {types[i]} := sorry") - replacements.append((start, end, hole)) - for start, end, hole in reversed(replacements): - statement = statement[:start] + hole + statement[end:] - return statement, holes - - -def pins(source_path=None): - """Revisions the workspace's own build can actually fetch. - - The FC pin must be reachable from the upstream repository the lakefile - names, so it is the merge-base with `origin/main`, not HEAD: a local - branch commit would generate a workspace whose build fails at fetch time. - The generator stops if the selected source differs from that revision. - Otherwise it could combine a working-tree statement with an older imported - context. - """ - manifest = json.loads((ROOT / "lake-manifest.json").read_text()) - mathlib_rev = next(p["rev"] for p in manifest["packages"] if p["name"] == "mathlib") - merge_base = subprocess.run( - ["git", "-C", str(ROOT), "merge-base", "HEAD", "origin/main"], - capture_output=True, - text=True, - ) - if merge_base.returncode != 0 or not merge_base.stdout.strip(): - raise SystemExit("cannot resolve the Formal Conjectures source revision") - fc_rev = merge_base.stdout.strip() - if source_path is not None: - comparison = subprocess.run( - ["git", "-C", str(ROOT), "diff", "--quiet", fc_rev, "--", str(source_path)] - ) - if comparison.returncode not in (0, 1): - raise SystemExit(f"cannot compare {source_path} with {fc_rev[:12]}") - if comparison.returncode == 1: - raise SystemExit( - f"{source_path} differs from pinned revision {fc_rev[:12]}; " - "land the source on upstream main before generating" - ) - return mathlib_rev, fc_rev - - -def lakefile(workspace_id, mathlib_rev): - """Mathlib and nothing else. - - The workspace used to require this repository too, so that Challenge.lean - could import the problem's module. lean-eval vendors its problems and - cannot fetch Formal Conjectures at evaluation time, so the closure travels - in `ChallengeDeps.lean` instead and the require is gone. The Formal - Conjectures commit the copy came from is recorded in `provenance.json`, - which is where a reader should look for it. - """ - return f"""name = "{workspace_id}" -testDriver = "workspace_test" -defaultTargets = ["ChallengeDeps", "Challenge", "Solution", "Submission"] - -[leanOptions] -autoImplicit = false - -[[require]] -name = "mathlib" -git = "https://github.com/leanprover-community/mathlib4.git" -rev = "{mathlib_rev}" - -[[lean_lib]] -name = "ChallengeDeps" - -[[lean_lib]] -name = "Challenge" - -[[lean_lib]] -name = "Solution" - -[[lean_lib]] -name = "Submission" - -[[lean_exe]] -name = "workspace_test" -root = "WorkspaceTest" -""" - - -def hole_names_of(holes): - return [h.split()[2] for h in holes] - - -def write_workspace(target, files): - """Write a complete workspace without overwriting or leaving a partial one.""" target = pathlib.Path(target) if target.exists(): raise SystemExit(f"refusing to overwrite existing workspace: {target}") @@ -750,413 +98,76 @@ def write_workspace(target, files): except BaseException: shutil.rmtree(staging, ignore_errors=True) raise + return target -def generate(basename, out_dir, answer_type=None, module=None): - """Write a comparator workspace for one declaration. +def emit_import(marked_up, manifest, out_dir): + """Write only the pair this repository owns: the module and the manifest. - Challenge.lean imports `ChallengeDeps`, which carries the statement's - Formal Conjectures closure and requires Mathlib alone. Importing the - problem's own module would be safer to construct and was what this script - did first, but lean-eval vendors its problems and cannot fetch this - repository at evaluation time, so the closure has to travel with the - workspace. - - That brings back the failure modes an import does not have. Reconstructing - definitions by hand cost six defects that only Lean could find: file-scoped - `open` and `variable` lost, `local notation` unrecognised, a `namespace` - swallowing the declaration below it, `section` lines left unclosed. The - answer is not to construct more carefully but to check: `--verify` builds - the workspace, which catches all six at once. - - Copying a closure out of a repository full of `sorry` is safe because - comparator checks axioms. A solution closing the goal with a copied - statement reports `sorryAx`, which `permitted_axioms` does not allow. + This is the artifact the FC importer contributes to a lean-eval problem + pull request once the generator is a pinned dependency there. Emitting it + on its own keeps the seam checkable today: the bytes here are the bytes + the generator gets, and nothing in this directory is workspace layout. """ - manifest = load_manifest(basename) - declaration = manifest.get("declaration", basename) - # An argument given on the command line is explicit, so it wins over the - # manifest; the manifest is the durable record of the same choice. - answer_type = answer_type or manifest.get("answer_type") - module = module or manifest.get("module") - path, _imports, _module_doc, body = find_declaration(declaration, module) - fc_module = module_name(path.relative_to(ROOT)) - facts = elaborator_facts(fc_module, declaration) - if facts["range"] is None: - raise SystemExit(f"{declaration}: no source range recorded") - - - source_lines = path.read_text(encoding="utf-8").split("\n") - lo, hi = facts["range"]["startLine"], facts["range"]["endLine"] - end_col = facts["range"].get("endColumn") - # `open X in` is part of the command but sits above what the range covers - # in some toolchains; pull it in when the line above ends with ` in`. - while ( - lo > 1 - and source_lines[lo - 2].rstrip().endswith(" in") - and KEEP_LOOSE.match(source_lines[lo - 2]) - ): - lo -= 1 - sliced = source_lines[lo - 1 : hi] - if end_col is not None and sliced: - sliced = sliced[:-1] + [sliced[-1][:end_col]] - original = "\n".join(sliced) - statement = original - - preamble, namespaces_at_target = file_scoped_preamble(source_lines, lo) - deps_module = challenge_deps( - facts.get("dependencies", []), - facts.get("generatedDependencies", []), - declaration, - namespaces_at_target, + return write_tree( + pathlib.Path(out_dir) / slug(manifest.id), + {"Problem.lean": marked_up.render(), "manifest.json": manifest.to_json()}, ) - statement = strip_decorations(statement) - statement = replace_proof_with_sorry(statement) - declared = None - for line in statement.split("\n"): - dm = DECL_START.match(line) - if dm: - declared = re.match(r"\s*([\w.«»]+)", line[dm.end() :]).group(1) - break - if declared is None: - raise SystemExit(f"{declaration}: no declaration line in the slice") - statement, holes = hoist_answers( - statement, declared, facts.get("answerTypes", []), answer_type - ) - # A `research solved` statement carries its answer rather than a `sorry` - # slot, so nothing above removed it and `answer(` would reach a workspace - # that cannot parse it. - statement = unwrap_answers(statement) - - args = [b["name"] for b in facts["binders"] if b["explicit"]] - bad = [a for a in args if "✝" in a or "._" in a] - if bad: - raise SystemExit( - f"{declared} has inaccessible explicit binders {bad}; the " - "Solution adapter cannot apply them by name" - ) - - # `open A`, then `open A.B`: opening the inner namespace does not open the - # outer one, and a statement may name siblings from either. - opens = [ - f"open {'.'.join(namespaces_at_target[:i + 1])}" - for i in range(len(namespaces_at_target)) - ] - - # One header shared by all three Lean files: the statement's text is - # identical in each, so what it needs to elaborate must be too. - header = ( - ("\n".join(opens) + "\n" if opens else "") - + ("\n".join(preamble) + "\n" if preamble else "") - + ("\n" if opens or preamble else "") - ) - suffix = ":= by\n sorry" - signature = statement.rstrip() - if signature.endswith(suffix): - signature = signature[: -len(suffix)].rstrip() - - challenge = ( - "import ChallengeDeps\n\n" - + header - + "\n\n".join(holes) - + ("\n\n" if holes else "") - + statement - + "\n" - ) - - # The participant's file. The statement sits inside `namespace Submission` - # so nothing here can collide with, or stand in for, the trusted names. - submission = ( - "import ChallengeDeps\nimport Submission.Helpers\n\n" - + header - + "namespace Submission\n\n" - + "\n\n".join(holes) - + ("\n\n" if holes else "") - + statement - + "\n\n" - + "end Submission\n" - ) - - # The fixed adapter, lean-eval's shape: it restates the trusted statement - # and closes it with the Submission theorem, so it fails to compile the - # moment the submission proves anything else. The participant never edits - # it, which is what keeps the statement pinned. - delegated = [ - h.rsplit(":= sorry", 1)[0] + ":= Submission." + hn - for h, hn in zip(holes, hole_names_of(holes)) - ] - solution = ( - "import ChallengeDeps\nimport Submission\n\n" - + header - + "\n\n".join(delegated) - + ("\n\n" if delegated else "") - + signature - + " :=\n Submission." - + declared - + ("".join(" " + a for a in args)) - + "\n" - ) - - mathlib_rev, fc_rev = pins(path.relative_to(ROOT)) - full_name = ".".join(namespaces_at_target + [declared]) - hole_names = hole_names_of(holes) - - workspace_id = slug(manifest.get("id", declared)) - ws = pathlib.Path(out_dir) / workspace_id - holes_line = ( - "\nFill each definition hole in `Submission.lean` too. Hole answers " - "also get a\nhuman check, because a hole can be gamed in ways the " - "comparator cannot see.\nChecking holes needs a comparator built at " - f"commit `{tool_pins()['comparator'][:8]}`, which\nadded definition " - "support.\n" - if holes - else "" - ) - manifest_lines = "".join( - f"- {field.capitalize()}: {' '.join(str(manifest[field]).split())}\n" - for field in ("source", "notes") - if manifest.get(field) - ) - workspace_readme = ( - f"# {workspace_id}\n\n" - f"A comparator challenge for `{declared}`, generated from\n" - f"`{path.relative_to(ROOT)}` in google-deepmind/formal-conjectures.\n\n" - + manifest_lines - + "\nProve the statement in `Submission.lean`, keeping it as it stands; " - "put helper\nmodules under `Submission/` if you need them. Do not " - "modify `Challenge.lean` or\n`Solution.lean`: the trusted statement " - "lives there, and `Solution.lean` closes it\nwith your `Submission` " - "theorem, so it fails to compile if the submission proves\nanything " - "else.\n" - "\nComparator accepts the workspace only if the statement is proved " - "under the\naxioms in `config.json`. `sorry` adds `sorryAx`, which is " - "not permitted, and\nclosing the goal with the imported original " - "fails the same way, since that is\n`sorry` too. `lake test` runs " - "comparator, from `PATH` or `COMPARATOR_BIN`.\n" - "\nIf comparator fails with `incompatible header` on an `.olean`, the " - "mismatch is\nbetween this workspace's toolchain and the one " - "`lean4export` was built with,\nnever a problem with the proof: copy " - "this workspace's `lean-toolchain` into\nyour `lean4export` checkout, " - "rebuild it, and clear `.lake/build` here.\n" - + holes_line - + "\nFetch the Mathlib cache before the first build; a cold build takes " - "the best\npart of an hour without it:\n\n" - " lake exe cache get\n" - " lake build\n" - ) - helper = ( - "import Mathlib\n\n" - "/-! Helper lemmas for the submission go here, or in further modules\n" - "under `Submission/`, each imported from `Submission.lean`. -/\n\n" - "namespace Submission\n\nend Submission\n" - ) - config = { - "challenge_module": "Challenge", - "solution_module": "Solution", - "theorem_names": [declared], - "permitted_axioms": PERMITTED_AXIOMS, - "enable_nanoda": False, - } - if hole_names: - # Comparator's documented no-hole config carries no such field. - config["definition_names"] = hole_names - holes_payload = { - "id": manifest.get("id", declared), - "module": str(path.relative_to(ROOT)), - "holes": [ - { - "name": ".".join(namespaces_at_target + [hn]), - "basename": hn, - "kind": "def", - "body": body_, - } - for hn, body_ in zip(hole_names, holes) - ] - + [ - { - "name": full_name, - "basename": declared, - "kind": "theorem", - "body": original, - } - ], - } - write_workspace( - ws, - { - "lakefile.toml": lakefile(workspace_id, mathlib_rev), - "lean-toolchain": (ROOT / "lean-toolchain").read_text(encoding="utf-8"), - "README.md": workspace_readme, - "ChallengeDeps.lean": deps_module, - "Challenge.lean": challenge, - "Solution.lean": solution, - "Submission.lean": submission, - "Submission/Helpers.lean": helper, - "WorkspaceTest.lean": ( - COMPARATOR_DIR / "templates" / "WorkspaceTest.lean" - ).read_text(encoding="utf-8"), - "provenance.json": json.dumps( - provenance( - declared, - fc_module, - path.relative_to(ROOT), - fc_rev, - [dep["name"] for dep in facts.get("dependencies", [])], - ), - indent=2, - ) - + "\n", - "config.json": json.dumps(config, indent=2) + "\n", - "holes.json": json.dumps(holes_payload, indent=2, ensure_ascii=False) - + "\n", - }, - ) - return ws - - -def validate(): - """Check every manifest resolves to exactly one declaration. - - Run this rather than discovering a stale `module` field when someone - generates the workspace months later. - """ - bad = 0 - for problem_id in manifest_ids(): - try: - manifest = load_manifest(problem_id) - declaration = manifest["declaration"] - path, _i, _d, _b = find_declaration(declaration, manifest.get("module")) - elaborator_facts(module_name(path.relative_to(ROOT)), declaration) - except SystemExit as exc: - print(f"{problem_id}: {exc}", file=sys.stderr) - bad += 1 - continue - print(f"{problem_id}: {declaration} in {path.relative_to(ROOT)}") - if bad: - print(f"{bad} manifest(s) do not resolve", file=sys.stderr) - return 1 if bad else 0 - - -def provenance(declaration, module, source_path, fc_rev, dependencies): - """Where the copied statement and its dependencies came from. - - Until this workspace carried its own copy of the closure, the lakefile's - `formal_conjectures` requirement named the commit and that was the record. - Nothing else did, so removing the requirement would have left a workspace - whose statement cannot be traced back to a revision of this repository. - """ - blob = subprocess.run( - ["git", "-C", str(ROOT), "rev-parse", f"{fc_rev}:{source_path}"], - capture_output=True, - text=True, - check=False, - ) - return { - "source_repository": "https://github.com/google-deepmind/formal-conjectures", - "source_commit": fc_rev, - "source_path": str(source_path), - "source_blob_sha": blob.stdout.strip() or None, - "declaration": declaration, - "module": module, - "copied_dependencies": dependencies, - "toolchain": (ROOT / "lean-toolchain").read_text(encoding="utf-8").strip(), - "tools": tool_pins(), - } - - -def verify(workspace): - """Elaborate the generated Challenge against its copied dependencies. - - Copying a closure is a construction, and its failure modes are the ones - Lean sees and a reader does not: a lost `open`, an unrecognised - `local notation`, a namespace that no longer exists because nothing - declares it any more. Each of those is a clean build away from being - caught and a long review away from being spotted. - - This concatenates `ChallengeDeps.lean` and `Challenge.lean` and elaborates - them with this checkout's Mathlib, which is the revision the workspace - pins, so the check is offline and does not fetch a second Mathlib. It - checks elaboration and not the lakefile; a comparator run is what - exercises the build. - """ - deps = (workspace / "ChallengeDeps.lean").read_text(encoding="utf-8") - challenge = (workspace / "Challenge.lean").read_text(encoding="utf-8") - challenge = "\n".join( - line - for line in challenge.split("\n") - if line.strip() != "import ChallengeDeps" - ) - with tempfile.NamedTemporaryFile( - "w", suffix=".lean", delete=False, encoding="utf-8" - ) as handle: - handle.write(deps + "\n" + challenge) - combined = handle.name - try: - proc = subprocess.run( - ["lake", "env", "lean", combined], - capture_output=True, - text=True, - cwd=ROOT, - check=False, - ) - finally: - pathlib.Path(combined).unlink(missing_ok=True) - output = (proc.stdout + proc.stderr).replace(combined, "Challenge") - # Only errors fail the check. The target statement's proof is `sorry` by - # construction and each `answer(sorry)` hole is one the solver fills, so - # those warnings are the generator working. Linter warnings such as - # `unused variable` come from the copied source and say nothing about - # whether the copy is faithful. - errors = [line for line in output.splitlines() if "error:" in line] - if proc.returncode != 0 or errors: - raise SystemExit( - f"{workspace.name}: the generated workspace does not elaborate:\n" - + "\n".join(errors or output.splitlines()[-10:]) - ) - return 0 - def main(argv): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument( "declaration", nargs="?", - help="a manifest id, or a declaration name such as erdos_940", + help="a problem id, or a declaration name such as erdos_940", ) ap.add_argument("--out", default=str(ROOT / ".comparator")) ap.add_argument( "--answer-type", default=None, help="type of a non-Prop answer(sorry) slot; " - "the manifest's `answer_type` is used when absent", + "the problem file's `answer_type` is used when absent", ) ap.add_argument( "--module", default=None, help="the file declaring it, when more than one does; " - "overrides the manifest's `module`", + "overrides the problem file's `module`", ) ap.add_argument( "--verify", action="store_true", - help="elaborate the generated workspace against this checkout's " - "Mathlib before accepting it", + help="elaborate the marked-up module against this checkout's Mathlib " + "before accepting it", + ) + ap.add_argument( + "--emit-import", + default=None, + metavar="DIR", + help="write only the marked-up module and its manifest, the pair this " + "repository hands the generator, and generate no workspace", ) ap.add_argument( "--validate", action="store_true", - help="check every manifest resolves, and generate nothing", + help="check every problem file resolves, and import nothing", ) args = ap.parse_args(argv) if args.validate: - return validate() + return importer.validate() if not args.declaration: ap.error("give a declaration, or --validate") - ws = generate(args.declaration, args.out, args.answer_type, args.module) + marked_up, manifest = importer.import_problem( + args.declaration, args.answer_type, args.module + ) if args.verify: - verify(pathlib.Path(ws)) - print(ws) + importer.elaborate(marked_up) + if args.emit_import: + print(emit_import(marked_up, manifest, args.emit_import)) + return 0 + files = generator.generate(marked_up, manifest) + print(write_tree(pathlib.Path(args.out) / slug(manifest.id), files)) return 0 diff --git a/scripts/test_fc_leaneval_importer.py b/scripts/test_fc_leaneval_importer.py new file mode 100644 index 0000000000..a720242f50 --- /dev/null +++ b/scripts/test_fc_leaneval_importer.py @@ -0,0 +1,400 @@ +# Copyright 2026 The Formal Conjectures Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline tests for the Formal Conjectures side of the LeanEval importer. + +Every case here pins a failure the first real workspace build produced, or a +rule whose violation would produce a marked-up module that elaborates but poses +the wrong problem. The build itself is the comparator's job, not these tests'. +""" + +import contextlib +import json +import pathlib +import subprocess +import tempfile +import unittest +from unittest import mock + +import fc_leaneval_importer as importer +from fc_leaneval_importer import ( + answer_spans, + closure_region, + file_scoped_preamble, + hoist_answers, + load_manifest, + pins, + replace_proof_with_sorry, + strip_decorations, + strip_fc_attributes, + unwrap_answers, +) + + +class HoistTest(unittest.TestCase): + """Slot types come from the elaborated environment.""" + + def test_slot_takes_the_environment_type(self): + stmt, holes = hoist_answers( + "theorem t : answer(sorry) ↔ ∀ n, n ≤ n := by\n sorry", "t", ["Prop"] + ) + self.assertIn("t_answer", stmt) + self.assertEqual( + holes[0].declaration(), "noncomputable def t_answer : Prop := sorry" + ) + + def test_erased_slot_is_prop_by_the_elaborators_rule(self): + # The default `alwaysTrue` setting erases a slot iff its expected + # type is Prop, so a missing annotation names the type exactly. + _, holes = hoist_answers("theorem t : answer(sorry) ↔ P := by\n sorry", "t", []) + self.assertEqual( + holes[0].declaration(), "noncomputable def t_answer : Prop := sorry" + ) + + def test_mixed_prop_and_typed_slots_are_refused(self): + with self.assertRaises(SystemExit): + hoist_answers( + "theorem t : answer(sorry) ∧ (answer(sorry) = 3) := by\n sorry", + "t", + ["Nat"], + ) + + def test_non_prop_type_is_read_not_guessed(self): + _, holes = hoist_answers( + "theorem t : sSup S = answer(sorry) := by\n sorry", "t", ["ENNReal"] + ) + self.assertEqual(holes[0].type, "ENNReal") + + def test_override_wins(self): + _, holes = hoist_answers( + "theorem t : sSup S = answer(sorry) := by\n sorry", "t", ["ENNReal"], "ℝ" + ) + self.assertEqual(holes[0].type, "ℝ") + + def test_differing_slot_types_are_refused(self): + # Matching types to positions would be a guess. + with self.assertRaises(SystemExit): + hoist_answers( + "theorem t : answer(sorry) = answer(sorry) := by\n sorry", + "t", + ["Nat", "Int"], + ) + + def test_no_slot_is_left_alone(self): + stmt, holes = hoist_answers("theorem t : True := by\n sorry", "t", []) + self.assertEqual(holes, []) + + def test_fixed_answer_is_not_turned_into_a_hole(self): + original = "theorem t : IsGLB S answer(2) := by\n sorry" + unchanged, holes = hoist_answers(original, "t", ["ENNReal"]) + self.assertEqual(unchanged, original) + self.assertEqual(holes, []) + + def test_nested_answer_term_is_one_balanced_slot(self): + calls = answer_spans("theorem t : f answer((fun x => x) (g 2)) := by\n sorry") + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0][2], "(fun x => x) (g 2)") + + def test_answer_text_in_comments_and_strings_is_ignored(self): + calls = answer_spans( + '-- answer(1)\ntheorem t : p "answer(2)" answer(3) := by sorry' + ) + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0][2], "3") + + +class PreambleTest(unittest.TestCase): + """Only directives in force at the statement are carried.""" + + def test_variable_in_a_closed_section_is_dropped(self): + lines = [ + "section S", + "variable {n : Nat}", + "end S", + "", + "open Nat", + "", + "theorem t : True := trivial", + ] + pre, ns = file_scoped_preamble(lines, 7) + self.assertEqual(pre, ["open Nat"]) + self.assertEqual(ns, []) + + def test_namespace_stack_is_reported(self): + lines = ["namespace A", "open Nat", "theorem t : True := trivial"] + pre, ns = file_scoped_preamble(lines, 3) + self.assertEqual(pre, ["open Nat"]) + self.assertEqual(ns, ["A"]) + + def test_directive_inside_a_comment_is_not_a_directive(self): + lines = ["/--", "open the door", "-/", "theorem t : True := trivial"] + pre, _ = file_scoped_preamble(lines, 4) + self.assertEqual(pre, []) + + +class StatementTest(unittest.TestCase): + def test_proof_is_replaced_but_statement_kept(self): + out = replace_proof_with_sorry( + "theorem t : True := by\n have h := trivial\n exact h" + ) + self.assertIn("theorem t : True", out) + self.assertNotIn("have h", out) + self.assertTrue(out.rstrip().endswith("sorry")) + + def test_term_mode_proof_is_replaced_too(self): + out = replace_proof_with_sorry("theorem t : True := trivial") + self.assertNotIn("trivial", out) + self.assertTrue(out.rstrip().endswith("sorry")) + + def test_term_proof_with_a_structure_literal_is_refused(self): + # The statement's own `:=` cannot be told from the proof's, and + # cutting at the wrong one truncates the statement. + with self.assertRaises(SystemExit): + replace_proof_with_sorry("theorem t : F { a := 1 } := ⟨rfl⟩") + + def test_a_line_comment_between_docstring_and_attribute_is_stripped(self): + # Erdos 918 writes a `--` formalisation note there. One anchored pass + # each left `@[category research open]` on the statement, and Lean + # parsed as far as the `open` inside it. + out = strip_decorations( + "/-- doc -/\n-- note\n@[category research open, AMS 5]\n" + "theorem t : True := by\n sorry" + ) + self.assertTrue(out.startswith("theorem")) + + def test_open_in_survives_stripping(self): + # It binds to the declaration, and it sits above the docstring. + out = strip_decorations( + "open scoped Classical in\n/-- doc -/\n@[category research open]\n" + "theorem t : True := by\n sorry" + ) + self.assertTrue(out.startswith("open scoped Classical in\ntheorem")) + + def test_decorations_are_stripped_from_the_target(self): + out = strip_decorations( + "/-- doc -/\n@[category research open]\ntheorem t : True := by\n sorry" + ) + self.assertTrue(out.startswith("theorem")) + + +class ProblemFileTest(unittest.TestCase): + """An FC problem file supplies what the Lean source cannot.""" + + def setUp(self): + self._dir = tempfile.TemporaryDirectory() + self._saved = importer.MANIFEST_DIR + importer.MANIFEST_DIR = pathlib.Path(self._dir.name) + + def tearDown(self): + importer.MANIFEST_DIR = self._saved + self._dir.cleanup() + + def write(self, name, body): + (importer.MANIFEST_DIR / name).write_text(body) + + def test_absent_problem_file_is_not_an_error(self): + # Most statements need none, and the importer works without one. + self.assertEqual(load_manifest("no_such_problem"), {}) + + def test_fields_are_read(self): + self.write("p.toml", 'id = "p"\ndeclaration = "d"\nanswer_type = "ENNReal"\n') + self.assertEqual(load_manifest("p")["answer_type"], "ENNReal") + + def test_id_must_match_the_filename(self): + # The filename is what the importer looks up, so a disagreeing `id` + # would silently name a workspace directory nobody asked for. + self.write("p.toml", 'id = "other"\ndeclaration = "d"\n') + with self.assertRaises(SystemExit): + load_manifest("p") + + def test_declaration_is_required(self): + self.write("p.toml", 'id = "p"\n') + with self.assertRaises(SystemExit): + load_manifest("p") + + +class PinTest(unittest.TestCase): + def test_changed_source_is_refused(self): + saved_root = importer.ROOT + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + importer.ROOT = root + (root / "lake-manifest.json").write_text( + json.dumps( + { + "packages": [{"name": "mathlib", "rev": "b" * 40}], + } + ), + encoding="utf-8", + ) + results = [ + subprocess.CompletedProcess([], 0, stdout="a" * 40 + "\n"), + subprocess.CompletedProcess([], 1), + ] + try: + with mock.patch.object(importer.subprocess, "run", side_effect=results): + with self.assertRaisesRegex(SystemExit, "differs from pinned"): + pins(pathlib.Path("FormalConjectures/Example.lean")) + finally: + importer.ROOT = saved_root + + +@contextlib.contextmanager +def _root_at(directory): + """Point the module's ROOT at a fixture tree. + + `closure_region` records each copied declaration's path relative to ROOT, + so a fixture written outside it cannot be described. + """ + saved = importer.ROOT + importer.ROOT = pathlib.Path(directory) + try: + yield + finally: + importer.ROOT = saved + + +class MathlibOnlyClosureTest(unittest.TestCase): + """The closure travels with the module, so copying has to be right. + + Each case here is a defect a generated workspace actually had, found by + elaborating it rather than by reading it. + """ + + def test_answer_with_a_value_is_unwrapped(self): + # `answer` is this repository's elaborator. `hoist_answers` removes the + # `answer(sorry)` slots; `conjecture327` is `research solved` and + # carries `answer(False)`, which reached the module verbatim and + # failed to parse against Mathlib alone. + self.assertEqual( + unwrap_answers("theorem t : answer(False) ↔ P := by\n sorry"), + "theorem t : (False) ↔ P := by\n sorry", + ) + + def test_unwrapping_keeps_a_parenthesised_argument_whole(self): + self.assertEqual(unwrap_answers("answer(f (n + 1))"), "(f (n + 1))") + + def test_only_this_repository_s_attributes_are_dropped(self): + # `strip_decorations` clears every attribute off the target statement. + # A copied dependency keeps the rest: dropping `simp` or `reducible` + # changes how the declarations after it in the closure elaborate. + self.assertEqual( + strip_fc_attributes("@[simp, category API, AMS 11]\ntheorem t : P"), + "@[simp]\ntheorem t : P", + ) + self.assertEqual( + strip_fc_attributes("@[category API]\ntheorem t : P"), "theorem t : P" + ) + self.assertEqual( + strip_fc_attributes("@[simp]\ndef f := 1"), "@[simp]\ndef f := 1" + ) + + def test_a_generated_constant_with_no_copied_ancestor_is_refused(self): + with self.assertRaisesRegex(SystemExit, "no copied ancestor"): + closure_region([], ["Foo.bar._proof_1"], "t") + + def test_a_generated_constant_under_a_copied_parent_is_accepted(self): + # `_proof_1` and `.match_1` have no source: copying the parent + # declaration regenerates them, so they are not an error. + deps = [ + { + "name": "Foo.bar", + "module": "FormalConjectures.Example", + "range": {"startLine": 1, "endLine": 1, "endColumn": None}, + } + ] + with ( + mock.patch.object(importer, "module_source_path") as resolve, + tempfile.TemporaryDirectory() as tmp, + _root_at(tmp), + ): + source = pathlib.Path(tmp) / "Example.lean" + source.write_text("def Foo.bar := 1\n", encoding="utf-8") + resolve.return_value = source + out, copied = closure_region(deps, ["Foo.bar._proof_1"], "t") + self.assertIn("def Foo.bar := 1", out) + self.assertEqual(copied, ["Foo.bar"]) + + def test_a_declaration_inside_another_s_range_is_not_copied_twice(self): + # `EdgeN.mk` covers line 88 of a structure spanning 83 to 93, and + # `pmSumListAux._sparseCasesOn_1` has exactly its parent's range. + # Copying either in its own right duplicated a declaration or sliced a + # fragment of one. + def span(name, lo, hi): + return { + "name": name, + "module": "FormalConjectures.Example", + "range": {"startLine": lo, "endLine": hi, "endColumn": None}, + } + + deps = [ + span("Foo.EdgeN.mk", 2, 2), + span("Foo.EdgeN", 1, 3), + span("Foo.aux._sparseCasesOn_1", 5, 5), + span("Foo.aux", 5, 5), + ] + with ( + mock.patch.object(importer, "module_source_path") as resolve, + tempfile.TemporaryDirectory() as tmp, + _root_at(tmp), + ): + source = pathlib.Path(tmp) / "Example.lean" + source.write_text( + "structure EdgeN where\n u : Nat\n deriving DecidableEq\n" + "\ndef aux := 1\n", + encoding="utf-8", + ) + resolve.return_value = source + out, _copied = closure_region(deps, [], "t") + self.assertIn("Foo.EdgeN`", out) + self.assertNotIn("Foo.EdgeN.mk`", out) + self.assertIn("Foo.aux`", out) + self.assertNotIn("_sparseCasesOn_1`", out) + + def test_an_opened_namespace_no_dependency_declares_is_created(self): + # The statement reopens the namespace stack its target sat in. With + # the problem's module no longer imported, `open Grimm` is an error + # unless something declares that namespace. + out, _copied = closure_region([], [], "grimm_conjecture", ["Grimm"]) + self.assertIn("namespace Grimm\nend Grimm", out) + + def test_a_namespace_a_dependency_declares_is_not_restated(self): + deps = [ + { + "name": "Grimm.helper", + "module": "FormalConjectures.Example", + "range": {"startLine": 1, "endLine": 1, "endColumn": None}, + } + ] + with ( + mock.patch.object(importer, "module_source_path") as resolve, + tempfile.TemporaryDirectory() as tmp, + _root_at(tmp), + ): + source = pathlib.Path(tmp) / "Example.lean" + source.write_text("def Grimm.helper := 1\n", encoding="utf-8") + resolve.return_value = source + out, _copied = closure_region(deps, [], "t", ["Grimm"]) + self.assertNotIn("namespace Grimm\nend Grimm", out) + + def test_the_closure_region_does_not_carry_the_import(self): + # `import Mathlib` belongs to the module as a whole, and the generator + # is what decides which emitted file carries it. A region that + # restated it would put an import in the middle of a Lean file. + out, _copied = closure_region([], [], "t") + self.assertNotIn("import", out) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_leaneval_generator.py b/scripts/test_leaneval_generator.py new file mode 100644 index 0000000000..c449b2ebd5 --- /dev/null +++ b/scripts/test_leaneval_generator.py @@ -0,0 +1,153 @@ +# Copyright 2026 The Formal Conjectures Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the side of the seam that becomes a pinned dependency. + +These cases describe what a Challenge/Solution/Submission workspace must look +like given a marked-up module and a manifest. They are written against nothing +but those two values on purpose: when `leanprover/lean-eval-generator` replaces +`scripts/leaneval_generator.py`, this file is what says whether the pinned +generator still produces what Formal Conjectures' import expects. +""" + +import ast +import json +import pathlib +import unittest + +import leaneval_generator as generator +from leaneval_generator import generate +from test_leaneval_interface import A_MODULE, a_manifest + +# The modules that live beside the generator in `scripts/`. Anything the +# generator imports from here has to move with it into the pinned package. +LOCAL = {path.name for path in pathlib.Path(generator.__file__).parent.glob("*.py")} + + +def a_workspace(**overrides): + return generate(A_MODULE, a_manifest(**overrides)) + + +class SplitTest(unittest.TestCase): + """One module in, four Lean files out, and the imports have to line up.""" + + def test_the_closure_is_the_only_file_importing_mathlib(self): + files = a_workspace() + self.assertTrue(files["ChallengeDeps.lean"].startswith("import Mathlib\n")) + self.assertIn("def Foo.bar := 1", files["ChallengeDeps.lean"]) + for name in ("Challenge.lean", "Solution.lean", "Submission.lean"): + self.assertTrue(files[name].startswith("import ChallengeDeps"), name) + + def test_the_statement_text_is_identical_in_all_three_files(self): + # The Solution adapter only pins the statement if the statement it + # restates is the one the Challenge poses. + files = a_workspace() + statement = A_MODULE.statement + self.assertIn(statement, files["Challenge.lean"]) + self.assertIn(statement, files["Submission.lean"]) + self.assertIn(statement.split(":= by")[0].rstrip(), files["Solution.lean"]) + + def test_the_scope_is_restated_in_every_file_that_carries_the_statement(self): + # `open Erdos` is file-scoped, so an import cannot carry it. + files = a_workspace() + for name in ("Challenge.lean", "Solution.lean", "Submission.lean"): + self.assertIn("open Erdos", files[name], name) + + def test_the_submission_is_namespaced_away_from_the_trusted_names(self): + submission = a_workspace()["Submission.lean"] + self.assertIn("namespace Submission", submission) + self.assertIn("end Submission", submission) + + def test_the_solution_delegates_the_hole_and_applies_the_arguments(self): + solution = a_workspace()["Solution.lean"] + self.assertIn( + "noncomputable def erdos_940_answer : ENNReal := " + "Submission.erdos_940_answer", + solution, + ) + self.assertTrue(solution.rstrip().endswith("Submission.erdos_940 n")) + + def test_a_statement_with_no_arguments_is_not_applied_to_anything(self): + # A `∀` binder in the conclusion is not a declaration parameter, and + # applying one would fail to elaborate. + solution = a_workspace(apply_arguments=())["Solution.lean"] + self.assertTrue(solution.rstrip().endswith("Submission.erdos_940")) + + def test_a_workspace_with_no_hole_declares_no_definition_names(self): + config = json.loads(a_workspace(holes=())["config.json"]) + self.assertNotIn("definition_names", config) + + def test_the_config_names_the_theorem_the_holes_and_the_axioms(self): + config = json.loads(a_workspace()["config.json"]) + self.assertEqual(config["theorem_names"], ["erdos_940"]) + self.assertEqual(config["definition_names"], ["erdos_940_answer"]) + self.assertIn("propext", config["permitted_axioms"]) + self.assertNotIn("sorryAx", config["permitted_axioms"]) + + def test_the_lakefile_pins_mathlib_and_requires_nothing_else(self): + lakefile = a_workspace()["lakefile.toml"] + self.assertIn('rev = "' + "c" * 40 + '"', lakefile) + self.assertEqual(lakefile.count("[[require]]"), 1) + self.assertNotIn("formal-conjectures", lakefile) + + def test_the_package_name_is_an_identifier(self): + files = generate(A_MODULE, a_manifest(id="erdos_940.variants.large_integers")) + self.assertIn( + 'name = "erdos_940_variants_large_integers"', files["lakefile.toml"] + ) + + def test_the_toolchain_file_is_the_one_the_manifest_pins(self): + self.assertEqual(a_workspace()["lean-toolchain"], "leanprover/lean4:v4.27.0\n") + + +class ManifestPassThroughTest(unittest.TestCase): + def test_the_workspace_carries_the_fc_commit_and_declaration(self): + # lean-eval#536: each manifest records the FC source commit and + # declaration id. This side supplies neither and edits neither. + payload = json.loads(a_workspace()["manifest.json"]) + self.assertEqual(payload["source"]["commit"], "a" * 40) + self.assertEqual(payload["source"]["declaration"], "erdos_940") + + def test_the_manifest_is_passed_through_unaltered(self): + manifest = a_manifest() + files = generate(A_MODULE, manifest) + self.assertEqual(files["manifest.json"], manifest.to_json()) + + +class SeamTest(unittest.TestCase): + def test_the_generator_depends_on_the_interface_and_nothing_else_local(self): + # The direction of the dependency is the whole point: a generator that + # imported the importer could not be swapped for a pinned package. + tree = ast.parse(pathlib.Path(generator.__file__).read_text(encoding="utf-8")) + imported = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module) + local = {name for name in imported if pathlib.Path(f"{name}.py").name in LOCAL} + self.assertEqual(local, {"leaneval_interface"}) + + +class TemplateTest(unittest.TestCase): + def test_workspace_test_template_exists_and_is_the_runner(self): + # The generator copies this file into every workspace; a missing or + # gutted template would only surface at `lake test` time, elsewhere. + text = (generator.TEMPLATE_DIR / "WorkspaceTest.lean").read_text() + self.assertIn("def main", text) + self.assertIn("COMPARATOR_BIN", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_leaneval_interface.py b/scripts/test_leaneval_interface.py new file mode 100644 index 0000000000..b68f82ea1e --- /dev/null +++ b/scripts/test_leaneval_interface.py @@ -0,0 +1,175 @@ +# Copyright 2026 The Formal Conjectures Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the importer-to-generator interface. + +The two values here are the whole contract between the half of this work +Formal Conjectures owns and the half `leanprover/lean-eval-generator` will +own. A value that does not survive being written out and read back is not a +contract, and a manifest that has lost the source commit cannot be regenerated +when Formal Conjectures corrects the statement upstream. +""" + +import unittest + +from leaneval_interface import ( + DefinitionHole, + MarkedUpModule, + ProblemManifest, + SourceRecord, + slug, +) + + +def a_source(**overrides): + fields = { + "repository": "https://github.com/google-deepmind/formal-conjectures", + "commit": "a" * 40, + "path": "FormalConjectures/Example.lean", + "blob_sha": "b" * 40, + "module": "FormalConjectures.Example", + "declaration": "erdos_940", + "copied_dependencies": ("Foo.bar",), + "original_declaration": "theorem erdos_940 : True := by\n sorry", + } + fields.update(overrides) + return SourceRecord(**fields) + + +def a_manifest(**overrides): + fields = { + "id": "erdos_940", + "theorem": "erdos_940", + "qualified_theorem": "Erdos.erdos_940", + "apply_arguments": ("n",), + "holes": (DefinitionHole(name="erdos_940_answer", type="ENNReal"),), + "permitted_axioms": ("propext", "Quot.sound", "Classical.choice"), + "lean_toolchain": "leanprover/lean4:v4.27.0", + "mathlib_revision": "c" * 40, + "source": a_source(), + "tools": {"comparator": "d" * 40}, + "source_url": "https://www.erdosproblems.com/940", + "notes": "a reviewer note", + } + fields.update(overrides) + return ProblemManifest(**fields) + + +A_MODULE = MarkedUpModule( + dependencies="def Foo.bar := 1", + scope="open Erdos", + holes="noncomputable def erdos_940_answer : ENNReal := sorry", + statement="theorem erdos_940 : erdos_940_answer = 0 := by\n sorry", +) + + +class ManifestTest(unittest.TestCase): + def test_source_commit_and_declaration_are_required(self): + # lean-eval#536 names both, and neither is something the generator can + # supply: it sees a Lean module, not a repository. + with self.assertRaisesRegex(SystemExit, "no FC source commit"): + a_manifest(source=a_source(commit="")) + with self.assertRaisesRegex(SystemExit, "no FC declaration id"): + a_manifest(source=a_source(declaration="")) + + def test_the_manifest_survives_a_round_trip(self): + manifest = a_manifest() + self.assertEqual(ProblemManifest.from_json(manifest.to_json()), manifest) + + def test_the_serialised_manifest_carries_the_commit_and_declaration(self): + # A reviewer of a generated workspace reads this file, so the two + # fields have to be in it under their own names. + payload = a_manifest().to_json_object() + self.assertEqual(payload["source"]["commit"], "a" * 40) + self.assertEqual(payload["source"]["declaration"], "erdos_940") + + def test_a_manifest_from_another_schema_version_is_refused(self): + payload = a_manifest().to_json_object() + payload["schema_version"] = 99 + with self.assertRaises(SystemExit): + ProblemManifest.from_json_object(payload) + + def test_hole_declaration_is_the_text_the_module_carries(self): + hole = DefinitionHole(name="t_answer", type="Prop") + self.assertEqual( + hole.declaration(), "noncomputable def t_answer : Prop := sorry" + ) + + +class MarkedUpModuleTest(unittest.TestCase): + def test_the_module_stands_on_mathlib_alone(self): + self.assertTrue(A_MODULE.render().startswith("import Mathlib\n")) + + def test_the_module_survives_a_round_trip(self): + self.assertEqual(MarkedUpModule.parse(A_MODULE.render()), A_MODULE) + + def test_an_empty_region_still_round_trips(self): + # Most statements have no answer slot, so the holes region is empty + # and the generator must still find it. + module = MarkedUpModule( + dependencies="def f := 1", scope="", holes="", statement="theorem t : True" + ) + self.assertEqual(MarkedUpModule.parse(module.render()), module) + + def test_a_missing_region_is_refused(self): + text = A_MODULE.render().replace("-- @region holes\n", "") + with self.assertRaisesRegex(SystemExit, "`holes` region"): + MarkedUpModule.parse(text) + + def test_an_unknown_region_is_refused(self): + with self.assertRaisesRegex(SystemExit, "unknown region"): + MarkedUpModule.parse("import Mathlib\n\n-- @region proof\n") + + def test_a_repeated_region_is_refused(self): + with self.assertRaisesRegex(SystemExit, "appears twice"): + MarkedUpModule.parse(A_MODULE.render() + "\n-- @region scope\n") + + def test_a_copied_declaration_that_looks_like_a_marker_is_refused(self): + # It would split the module somewhere the importer did not choose, + # and the generator would have no way to notice. + module = MarkedUpModule( + dependencies="-- @region statement\ndef f := 1", + scope="", + holes="", + statement="theorem t : True", + ) + with self.assertRaisesRegex(SystemExit, "contains a region marker"): + module.render() + + def test_regions_out_of_order_are_refused(self): + # The order is what makes the module elaborate: a hole is used by the + # statement below it, and both need the scope above them. + module = MarkedUpModule.parse(A_MODULE.render()) + reordered = ( + "import Mathlib\n" + f"\n-- @region scope\n{module.scope}\n" + f"\n-- @region dependencies\n{module.dependencies}\n" + f"\n-- @region holes\n{module.holes}\n" + f"\n-- @region statement\n{module.statement}\n" + ) + with self.assertRaisesRegex(SystemExit, "out of order"): + MarkedUpModule.parse(reordered) + + +class SlugTest(unittest.TestCase): + def test_a_qualified_declaration_becomes_an_identifier(self): + # A Lake package name is an identifier, so the dots cannot survive. + self.assertEqual( + slug("erdos_940.variants.large_integers"), + "erdos_940_variants_large_integers", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_make_comparator_workspace.py b/scripts/test_make_comparator_workspace.py index 9687b3b7d2..11303ad55c 100644 --- a/scripts/test_make_comparator_workspace.py +++ b/scripts/test_make_comparator_workspace.py @@ -12,238 +12,77 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Offline tests for `make_comparator_workspace.py`. +"""Tests for the command that runs the importer and then the generator. -Every case here pins a failure the first real workspace build produced, or a -rule whose violation would generate a workspace that builds but poses the -wrong problem. The build itself is the comparator's job, not these tests'. +The case that matters here is the seam itself: what this repository hands over +has to be enough. If a workspace cannot be rebuilt from the emitted module and +manifest alone, then some of the interface is still travelling inside the +process, and a pinned `lean-eval-generator` could not be dropped in. """ -import contextlib -import json import pathlib -import subprocess import tempfile import unittest from unittest import mock -import make_comparator_workspace as mcw -from make_comparator_workspace import ( - answer_spans, - challenge_deps, - file_scoped_preamble, - hoist_answers, - load_manifest, - pins, - replace_proof_with_sorry, - strip_decorations, - strip_fc_attributes, - unwrap_answers, - write_workspace, -) +import leaneval_generator as generator +from leaneval_interface import MarkedUpModule, ProblemManifest +from make_comparator_workspace import emit_import, write_tree +from test_leaneval_interface import A_MODULE, a_manifest -class HoistTest(unittest.TestCase): - """Slot types come from the elaborated environment.""" - - def test_slot_takes_the_environment_type(self): - stmt, holes = hoist_answers( - "theorem t : answer(sorry) ↔ ∀ n, n ≤ n := by\n sorry", "t", ["Prop"] - ) - self.assertIn("t_answer", stmt) - self.assertIn("noncomputable def t_answer : Prop := sorry", holes) - - def test_erased_slot_is_prop_by_the_elaborators_rule(self): - # The default `alwaysTrue` setting erases a slot iff its expected - # type is Prop, so a missing annotation names the type exactly. - _, holes = hoist_answers( - "theorem t : answer(sorry) ↔ P := by\n sorry", "t", [] - ) - self.assertIn("noncomputable def t_answer : Prop := sorry", holes) - - def test_mixed_prop_and_typed_slots_are_refused(self): - with self.assertRaises(SystemExit): - hoist_answers( - "theorem t : answer(sorry) ∧ (answer(sorry) = 3) := by\n sorry", - "t", - ["Nat"], +class EmitImportTest(unittest.TestCase): + def test_only_the_module_and_the_manifest_are_emitted(self): + with tempfile.TemporaryDirectory() as tmp: + out = emit_import(A_MODULE, a_manifest(), tmp) + self.assertEqual( + sorted(p.name for p in out.iterdir()), + ["Problem.lean", "manifest.json"], ) - def test_non_prop_type_is_read_not_guessed(self): - _, holes = hoist_answers( - "theorem t : sSup S = answer(sorry) := by\n sorry", "t", ["ENNReal"] - ) - self.assertIn("t_answer : ENNReal", holes[0]) - - def test_override_wins(self): - _, holes = hoist_answers( - "theorem t : sSup S = answer(sorry) := by\n sorry", "t", ["ENNReal"], "ℝ" - ) - self.assertIn("t_answer : ℝ", holes[0]) - - def test_differing_slot_types_are_refused(self): - # Matching types to positions would be a guess. - with self.assertRaises(SystemExit): - hoist_answers( - "theorem t : answer(sorry) = answer(sorry) := by\n sorry", - "t", - ["Nat", "Int"], + def test_the_emitted_pair_rebuilds_the_workspace_exactly(self): + manifest = a_manifest() + with tempfile.TemporaryDirectory() as tmp: + out = emit_import(A_MODULE, manifest, tmp) + module = MarkedUpModule.parse( + (out / "Problem.lean").read_text(encoding="utf-8") ) - - def test_no_slot_is_left_alone(self): - stmt, holes = hoist_answers("theorem t : True := by\n sorry", "t", []) - self.assertEqual(holes, []) - - def test_fixed_answer_is_not_turned_into_a_hole(self): - original = "theorem t : IsGLB S answer(2) := by\n sorry" - unchanged, holes = hoist_answers(original, "t", ["ENNReal"]) - self.assertEqual(unchanged, original) - self.assertEqual(holes, []) - - def test_nested_answer_term_is_one_balanced_slot(self): - calls = answer_spans("theorem t : f answer((fun x => x) (g 2)) := by\n sorry") - self.assertEqual(len(calls), 1) - self.assertEqual(calls[0][2], "(fun x => x) (g 2)") - - def test_answer_text_in_comments_and_strings_is_ignored(self): - calls = answer_spans( - '-- answer(1)\ntheorem t : p "answer(2)" answer(3) := by sorry' - ) - self.assertEqual(len(calls), 1) - self.assertEqual(calls[0][2], "3") - - -class PreambleTest(unittest.TestCase): - """Only directives in force at the statement are carried.""" - - def test_variable_in_a_closed_section_is_dropped(self): - lines = [ - "section S", - "variable {n : Nat}", - "end S", - "", - "open Nat", - "", - "theorem t : True := trivial", - ] - pre, ns = file_scoped_preamble(lines, 7) - self.assertEqual(pre, ["open Nat"]) - self.assertEqual(ns, []) - - def test_namespace_stack_is_reported(self): - lines = ["namespace A", "open Nat", "theorem t : True := trivial"] - pre, ns = file_scoped_preamble(lines, 3) - self.assertEqual(pre, ["open Nat"]) - self.assertEqual(ns, ["A"]) - - def test_directive_inside_a_comment_is_not_a_directive(self): - lines = ["/--", "open the door", "-/", "theorem t : True := trivial"] - pre, _ = file_scoped_preamble(lines, 4) - self.assertEqual(pre, []) - - -class StatementTest(unittest.TestCase): - def test_proof_is_replaced_but_statement_kept(self): - out = replace_proof_with_sorry( - "theorem t : True := by\n have h := trivial\n exact h" - ) - self.assertIn("theorem t : True", out) - self.assertNotIn("have h", out) - self.assertTrue(out.rstrip().endswith("sorry")) - - def test_term_mode_proof_is_replaced_too(self): - out = replace_proof_with_sorry("theorem t : True := trivial") - self.assertNotIn("trivial", out) - self.assertTrue(out.rstrip().endswith("sorry")) - - def test_term_proof_with_a_structure_literal_is_refused(self): - # The statement's own `:=` cannot be told from the proof's, and - # cutting at the wrong one truncates the statement. - with self.assertRaises(SystemExit): - replace_proof_with_sorry("theorem t : F { a := 1 } := ⟨rfl⟩") - - def test_a_line_comment_between_docstring_and_attribute_is_stripped(self): - # Erdos 918 writes a `--` formalisation note there. One anchored pass - # each left `@[category research open]` on the statement, and Lean - # parsed as far as the `open` inside it. - out = strip_decorations( - "/-- doc -/\n-- note\n@[category research open, AMS 5]\n" - "theorem t : True := by\n sorry" - ) - self.assertTrue(out.startswith("theorem")) - - def test_open_in_survives_stripping(self): - # It binds to the declaration, and it sits above the docstring. - out = strip_decorations( - "open scoped Classical in\n/-- doc -/\n@[category research open]\n" - "theorem t : True := by\n sorry" - ) - self.assertTrue(out.startswith("open scoped Classical in\ntheorem")) - - def test_decorations_are_stripped_from_the_target(self): - out = strip_decorations( - "/-- doc -/\n@[category research open]\ntheorem t : True := by\n sorry" + read_back = ProblemManifest.from_json( + (out / "manifest.json").read_text(encoding="utf-8") + ) + self.assertEqual( + generator.generate(module, read_back), + generator.generate(A_MODULE, manifest), ) - self.assertTrue(out.startswith("theorem")) - - -class TemplateTest(unittest.TestCase): - def test_workspace_test_template_exists_and_is_the_runner(self): - # The generator copies this file into every workspace; a missing or - # gutted template would only surface at `lake test` time, elsewhere. - text = (mcw.COMPARATOR_DIR / "templates" / "WorkspaceTest.lean").read_text() - self.assertIn("def main", text) - self.assertIn("COMPARATOR_BIN", text) - - -class ManifestTest(unittest.TestCase): - """A manifest supplies what the Lean source cannot.""" - def setUp(self): - self._dir = tempfile.TemporaryDirectory() - self._saved = mcw.MANIFEST_DIR - mcw.MANIFEST_DIR = pathlib.Path(self._dir.name) - - def tearDown(self): - mcw.MANIFEST_DIR = self._saved - self._dir.cleanup() - - def write(self, name, body): - (mcw.MANIFEST_DIR / name).write_text(body) - - def test_absent_manifest_is_not_an_error(self): - # Most statements need none, and the generator works without one. - self.assertEqual(load_manifest("no_such_problem"), {}) - - def test_fields_are_read(self): - self.write("p.toml", 'id = "p"\ndeclaration = "d"\nanswer_type = "ENNReal"\n') - self.assertEqual(load_manifest("p")["answer_type"], "ENNReal") - - def test_id_must_match_the_filename(self): - # The filename is what the generator looks up, so a disagreeing `id` - # would silently name a workspace directory nobody asked for. - self.write("p.toml", 'id = "other"\ndeclaration = "d"\n') - with self.assertRaises(SystemExit): - load_manifest("p") + def test_an_existing_directory_is_not_overwritten(self): + with tempfile.TemporaryDirectory() as tmp: + emit_import(A_MODULE, a_manifest(), tmp) + with self.assertRaisesRegex(SystemExit, "refusing to overwrite"): + emit_import(A_MODULE, a_manifest(), tmp) - def test_declaration_is_required(self): - self.write("p.toml", 'id = "p"\n') - with self.assertRaises(SystemExit): - load_manifest("p") + def test_the_emitted_directory_is_named_by_the_problem_id(self): + with tempfile.TemporaryDirectory() as tmp: + out = emit_import( + A_MODULE, a_manifest(id="erdos_940.variants.large_integers"), tmp + ) + self.assertEqual( + out, pathlib.Path(tmp) / "erdos_940_variants_large_integers" + ) -class OutputTest(unittest.TestCase): - def test_existing_workspace_is_not_overwritten(self): +class WriteTreeTest(unittest.TestCase): + def test_existing_directory_is_not_overwritten(self): with tempfile.TemporaryDirectory() as tmp: target = pathlib.Path(tmp) / "workspace" target.mkdir() sentinel = target / "keep.txt" sentinel.write_text("keep", encoding="utf-8") with self.assertRaisesRegex(SystemExit, "refusing to overwrite"): - write_workspace(target, {"Challenge.lean": "theorem t : True"}) + write_tree(target, {"Challenge.lean": "theorem t : True"}) self.assertEqual(sentinel.read_text(encoding="utf-8"), "keep") - def test_failed_write_leaves_no_partial_workspace(self): + def test_failed_write_leaves_no_partial_directory(self): with tempfile.TemporaryDirectory() as tmp: root = pathlib.Path(tmp) target = root / "workspace" @@ -251,173 +90,10 @@ def test_failed_write_leaves_no_partial_workspace(self): pathlib.Path, "write_text", side_effect=OSError("disk error") ): with self.assertRaisesRegex(OSError, "disk error"): - write_workspace(target, {"Challenge.lean": "theorem t : True"}) + write_tree(target, {"Challenge.lean": "theorem t : True"}) self.assertFalse(target.exists()) self.assertEqual(list(root.iterdir()), []) -class PinTest(unittest.TestCase): - def test_changed_source_is_refused(self): - saved_root = mcw.ROOT - with tempfile.TemporaryDirectory() as tmp: - root = pathlib.Path(tmp) - mcw.ROOT = root - (root / "lake-manifest.json").write_text( - json.dumps( - { - "packages": [{"name": "mathlib", "rev": "b" * 40}], - } - ), - encoding="utf-8", - ) - results = [ - subprocess.CompletedProcess([], 0, stdout="a" * 40 + "\n"), - subprocess.CompletedProcess([], 1), - ] - try: - with mock.patch.object(mcw.subprocess, "run", side_effect=results): - with self.assertRaisesRegex(SystemExit, "differs from pinned"): - pins(pathlib.Path("FormalConjectures/Example.lean")) - finally: - mcw.ROOT = saved_root - - -@contextlib.contextmanager -def _root_at(directory): - """Point the module's ROOT at a fixture tree. - - `challenge_deps` records each copied declaration's path relative to ROOT, - so a fixture written outside it cannot be described. - """ - saved = mcw.ROOT - mcw.ROOT = pathlib.Path(directory) - try: - yield - finally: - mcw.ROOT = saved - - -class MathlibOnlyChallengeTest(unittest.TestCase): - """The closure travels with the workspace, so copying has to be right. - - Each case here is a defect a generated workspace actually had, found by - elaborating it rather than by reading it. - """ - - def test_answer_with_a_value_is_unwrapped(self): - # `answer` is this repository's elaborator. `hoist_answers` removes the - # `answer(sorry)` slots; `conjecture327` is `research solved` and - # carries `answer(False)`, which reached Challenge.lean verbatim and - # failed to parse against Mathlib alone. - self.assertEqual( - unwrap_answers("theorem t : answer(False) ↔ P := by\n sorry"), - "theorem t : (False) ↔ P := by\n sorry", - ) - - def test_unwrapping_keeps_a_parenthesised_argument_whole(self): - self.assertEqual(unwrap_answers("answer(f (n + 1))"), "(f (n + 1))") - - def test_only_this_repository_s_attributes_are_dropped(self): - # `strip_decorations` clears every attribute off the target statement. - # A copied dependency keeps the rest: dropping `simp` or `reducible` - # changes how the declarations after it in the closure elaborate. - self.assertEqual( - strip_fc_attributes("@[simp, category API, AMS 11]\ntheorem t : P"), - "@[simp]\ntheorem t : P", - ) - self.assertEqual( - strip_fc_attributes("@[category API]\ntheorem t : P"), "theorem t : P" - ) - self.assertEqual(strip_fc_attributes("@[simp]\ndef f := 1"), "@[simp]\ndef f := 1") - - def test_a_generated_constant_with_no_copied_ancestor_is_refused(self): - with self.assertRaisesRegex(SystemExit, "no copied ancestor"): - challenge_deps([], ["Foo.bar._proof_1"], "t") - - def test_a_generated_constant_under_a_copied_parent_is_accepted(self): - # `_proof_1` and `.match_1` have no source: copying the parent - # declaration regenerates them, so they are not an error. - deps = [ - { - "name": "Foo.bar", - "module": "FormalConjectures.Example", - "range": {"startLine": 1, "endLine": 1, "endColumn": None}, - } - ] - with ( - mock.patch.object(mcw, "module_source_path") as resolve, - tempfile.TemporaryDirectory() as tmp, - _root_at(tmp), - ): - source = pathlib.Path(tmp) / "Example.lean" - source.write_text("def Foo.bar := 1\n", encoding="utf-8") - resolve.return_value = source - out = challenge_deps(deps, ["Foo.bar._proof_1"], "t") - self.assertIn("import Mathlib", out) - self.assertIn("def Foo.bar := 1", out) - - def test_a_declaration_inside_another_s_range_is_not_copied_twice(self): - # `EdgeN.mk` covers line 88 of a structure spanning 83 to 93, and - # `pmSumListAux._sparseCasesOn_1` has exactly its parent's range. - # Copying either in its own right duplicated a declaration or sliced a - # fragment of one. - def span(name, lo, hi): - return { - "name": name, - "module": "FormalConjectures.Example", - "range": {"startLine": lo, "endLine": hi, "endColumn": None}, - } - - deps = [ - span("Foo.EdgeN.mk", 2, 2), - span("Foo.EdgeN", 1, 3), - span("Foo.aux._sparseCasesOn_1", 5, 5), - span("Foo.aux", 5, 5), - ] - with ( - mock.patch.object(mcw, "module_source_path") as resolve, - tempfile.TemporaryDirectory() as tmp, - _root_at(tmp), - ): - source = pathlib.Path(tmp) / "Example.lean" - source.write_text( - "structure EdgeN where\n u : Nat\n deriving DecidableEq\n" - "\ndef aux := 1\n", - encoding="utf-8", - ) - resolve.return_value = source - out = challenge_deps(deps, [], "t") - self.assertIn("Foo.EdgeN`", out) - self.assertNotIn("Foo.EdgeN.mk`", out) - self.assertIn("Foo.aux`", out) - self.assertNotIn("_sparseCasesOn_1`", out) - - def test_an_opened_namespace_no_dependency_declares_is_created(self): - # Challenge.lean reopens the namespace stack its target sat in. With - # the problem's module no longer imported, `open Grimm` is an error - # unless something declares that namespace. - out = challenge_deps([], [], "grimm_conjecture", ["Grimm"]) - self.assertIn("namespace Grimm\nend Grimm", out) - - def test_a_namespace_a_dependency_declares_is_not_restated(self): - deps = [ - { - "name": "Grimm.helper", - "module": "FormalConjectures.Example", - "range": {"startLine": 1, "endLine": 1, "endColumn": None}, - } - ] - with ( - mock.patch.object(mcw, "module_source_path") as resolve, - tempfile.TemporaryDirectory() as tmp, - _root_at(tmp), - ): - source = pathlib.Path(tmp) / "Example.lean" - source.write_text("def Grimm.helper := 1\n", encoding="utf-8") - resolve.return_value = source - out = challenge_deps(deps, [], "t", ["Grimm"]) - self.assertNotIn("namespace Grimm\nend Grimm", out) - - if __name__ == "__main__": unittest.main() From be2f92f01fa9119fcb770e5803f7bbc348563c4f Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:04:56 -0400 Subject: [PATCH 12/70] Generate the Lean 4.33 evidence instead of vendoring it comparator/pilots/fc_sum_of_three_cubes/ was eleven hand-written files: a copy of generator output, checked in. A copy drifts from the generator, and it says nothing about the importer because a human wrote it. Its workflow was the only thing that referenced it, and the only end-to-end evidence that the Comparator path works at LeanEval's toolchain. So redirect the workflow rather than delete the evidence. It now imports two declarations from FormalConjectures/Wikipedia/SumOfThreeCubes.lean, generates their workspaces, builds them at LeanEval's Lean 4.33 and Mathlib, and takes three Comparator verdicts: the workspace as generated must be rejected, because its Submission is sorry; the same workspace with the statement proved must be accepted; and the answer(sorry) workspace with its hole filled by the proposition on the other side of the iff and the bridge closed by Iff.rfl is accepted, which is the demonstration that a definition hole needs a human reading the answer. That is importer -> generator -> Comparator, end to end, with nothing vendored. For the workspace to build where it is going, it has to be pinned there. comparator/tools.toml gains a [target] table holding LeanEval's toolchain, Mathlib, Comparator and lean4export pins, preserved from the deleted pilot's provenance record. The generator writes lean-toolchain and the lakefile Mathlib revision from it, and every manifest now records both pin sets: source.lean_toolchain, where the answer-slot types were read, and target.lean_toolchain, where they will be used. That gap was open question 4 in OWNERSHIP.md as an assertion; the new job observes it. The Solution adapter marks its delegated hole @[reducible], matching the one workspace known to have built at 4.33. comparator/README.md drops what OWNERSHIP.md already says about ownership and the seam, and three claims that the target pins made false: that generated workspaces carry this repository's toolchain, that they pin Mathlib from lake-manifest.json, and that target pins are not yet recorded. Prose is 268 lines against 313. Python tests: 94. --- .github/workflows/comparator-lean-4-33.yml | 149 +++++++++++---- comparator/OWNERSHIP.md | 41 ++-- comparator/README.md | 175 +++++++----------- .../fc_sum_of_three_cubes/Challenge.lean | 41 ---- .../fc_sum_of_three_cubes/ChallengeDeps.lean | 30 --- .../pilots/fc_sum_of_three_cubes/README.md | 69 ------- .../fc_sum_of_three_cubes/Solution.lean | 36 ---- .../fc_sum_of_three_cubes/Submission.lean | 48 ----- .../Submission/Helpers.lean | 23 --- .../fc_sum_of_three_cubes/WorkspaceTest.lean | 36 ---- .../pilots/fc_sum_of_three_cubes/config.json | 17 -- .../fc_sum_of_three_cubes/lakefile.toml | 27 --- .../fc_sum_of_three_cubes/lean-toolchain | 1 - .../fc_sum_of_three_cubes/provenance.json | 37 ---- comparator/tools.toml | 18 +- scripts/fc_leaneval_importer.py | 49 +++-- scripts/leaneval_generator.py | 20 +- scripts/leaneval_interface.py | 47 +++-- scripts/make_comparator_workspace.py | 22 +-- scripts/test_leaneval_generator.py | 24 ++- scripts/test_leaneval_interface.py | 32 +++- 21 files changed, 363 insertions(+), 579 deletions(-) delete mode 100644 comparator/pilots/fc_sum_of_three_cubes/Challenge.lean delete mode 100644 comparator/pilots/fc_sum_of_three_cubes/ChallengeDeps.lean delete mode 100644 comparator/pilots/fc_sum_of_three_cubes/README.md delete mode 100644 comparator/pilots/fc_sum_of_three_cubes/Solution.lean delete mode 100644 comparator/pilots/fc_sum_of_three_cubes/Submission.lean delete mode 100644 comparator/pilots/fc_sum_of_three_cubes/Submission/Helpers.lean delete mode 100644 comparator/pilots/fc_sum_of_three_cubes/WorkspaceTest.lean delete mode 100644 comparator/pilots/fc_sum_of_three_cubes/config.json delete mode 100644 comparator/pilots/fc_sum_of_three_cubes/lakefile.toml delete mode 100644 comparator/pilots/fc_sum_of_three_cubes/lean-toolchain delete mode 100644 comparator/pilots/fc_sum_of_three_cubes/provenance.json diff --git a/.github/workflows/comparator-lean-4-33.yml b/.github/workflows/comparator-lean-4-33.yml index 6627abaf93..1cf1108c2d 100644 --- a/.github/workflows/comparator-lean-4-33.yml +++ b/.github/workflows/comparator-lean-4-33.yml @@ -1,15 +1,29 @@ -name: Comparator Lean 4.33 pilot +name: Generated workspace at LeanEval pins + +# End to end: import a declaration from this repository's source, generate a +# workspace, build it at LeanEval's Lean 4.33 and Mathlib, and run Comparator +# on it. Nothing here is checked in. A workspace that lives in the tree is a +# copy of generator output that drifts from the generator and says nothing +# about the importer, because a human wrote it; this job says the importer, the +# generator and Comparator work together, which is what +# `leanprover/lean-eval#536` needs before FC opens problem pull requests. +# +# The two toolchains in this job are the point, not an accident. The importer +# reads the declaration's source range, binders and `answer(sorry)` slot types +# from an environment elaborated at this repository's Lean 4.27; the workspace +# is built and checked at LeanEval's Lean 4.33. If those two disagree about a +# statement or a slot type, this job is where it shows. on: - push: - branches: - - comparator-workspaces - paths: - - 'comparator/pilots/fc_sum_of_three_cubes/**' - - '.github/workflows/comparator-lean-4-33.yml' pull_request: paths: - - 'comparator/pilots/fc_sum_of_three_cubes/**' + - 'scripts/fc_leaneval_importer.py' + - 'scripts/leaneval_generator.py' + - 'scripts/leaneval_interface.py' + - 'scripts/make_comparator_workspace.py' + - 'scripts/comparator_facts.lean' + - 'comparator/**' + - 'FormalConjectures/Wikipedia/SumOfThreeCubes.lean' - '.github/workflows/comparator-lean-4-33.yml' workflow_dispatch: @@ -17,12 +31,16 @@ permissions: contents: read jobs: - build-and-compare: + generate-build-and-compare: runs-on: ubuntu-latest - name: Build and run Comparator + name: Import, generate, build and run Comparator steps: - name: Checkout Formal Conjectures uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + # The importer pins each workspace to the merge base with upstream + # main, so that ref has to be present. + fetch-depth: 0 - name: Install elan run: | @@ -31,43 +49,108 @@ jobs: ./elan-init -y --default-toolchain none echo "$HOME/.elan/bin" >> "$GITHUB_PATH" - - name: Verify pins and trusted-source fingerprint + - name: Read the target pins + id: target run: | - python3 - <<'PY' - import hashlib - import json - from pathlib import Path - - root = Path('comparator/pilots/fc_sum_of_three_cubes') - provenance = json.loads((root / 'provenance.json').read_text()) - assert (root / 'lean-toolchain').read_text().strip() == provenance['target']['lean_toolchain'] - lakefile = (root / 'lakefile.toml').read_text() - assert provenance['target']['mathlib_revision'] in lakefile - trusted = (root / 'ChallengeDeps.lean').read_text() + '\n' + (root / 'Challenge.lean').read_text() - actual = hashlib.sha256(trusted.encode()).hexdigest() - assert actual == provenance['trusted_files_sha256'], (actual, provenance['trusted_files_sha256']) - print(actual) + python3 - <<'PY' >> "$GITHUB_OUTPUT" + import tomllib + + with open("comparator/tools.toml", "rb") as handle: + target = tomllib.load(handle)["target"] + for key in ("comparator", "lean4export", "lean_toolchain"): + print(f"{key}={target[key]}") PY - - name: Build the Lean 4.33 workspace - working-directory: comparator/pilots/fc_sum_of_three_cubes + # At this repository's toolchain: the declaration's facts come from an + # elaborated environment, so the module it lives in has to be built. + - name: Build the source module and the extractor run: | - lake update lake exe cache get - lake build + lake build comparator_facts FormalConjectures.Wikipedia.SumOfThreeCubes + + # `--verify` elaborates the marked-up module here, at 4.27. It is not a + # substitute for the 4.33 build below; it is what keeps an FC-side + # copying defect from being reported as a LeanEval build failure. + - name: Import and generate two workspaces + run: | + for d in isSumOfThreeCubes_2 isSumOfThreeCubes_iff_mod_9; do + python3 scripts/make_comparator_workspace.py "$d" \ + --out .comparator --verify + done + # One plain theorem and one `answer(sorry)` slot typed at 4.27. + grep -q "isSumOfThreeCubes_iff_mod_9_answer : Prop" \ + .comparator/isSumOfThreeCubes_iff_mod_9/Challenge.lean + # Generated for LeanEval, not for here. + grep -q "${{ steps.target.outputs.lean_toolchain }}" \ + .comparator/isSumOfThreeCubes_2/lean-toolchain + + - name: Build both workspaces at Lean 4.33 + run: | + for ws in .comparator/isSumOfThreeCubes_2 \ + .comparator/isSumOfThreeCubes_iff_mod_9; do + (cd "$ws" && lake update && lake exe cache get && lake build) + done - name: Build the pinned Lean 4.33 verifier stack env: - COMPARATOR_REV: c0c5a52d2aff92b457c3e5ed4a68c1ebc5795809 + COMPARATOR_REV: ${{ steps.target.outputs.comparator }} run: | git clone https://github.com/leanprover/comparator.git "$RUNNER_TEMP/comparator" git -C "$RUNNER_TEMP/comparator" checkout "$COMPARATOR_REV" (cd "$RUNNER_TEMP/comparator" && lake build comparator lean4export) - - name: Run Comparator smoke test - working-directory: comparator/pilots/fc_sum_of_three_cubes + # Three Comparator verdicts, each of which has to come out the stated way. + # A run that only ever accepts proves nothing about the gate. + - name: Run Comparator on the generated workspaces run: | export COMPARATOR_BIN="$RUNNER_TEMP/comparator/.lake/build/bin/comparator" export COMPARATOR_LANDRUN="$RUNNER_TEMP/comparator/scripts/fake-landrun.sh" export COMPARATOR_LEAN4EXPORT="$RUNNER_TEMP/comparator/.lake/packages/lean4export/.lake/build/bin/lean4export" - lake test + + # 1. The workspace as generated. Its Submission is `sorry`, which + # adds `sorryAx`, which `permitted_axioms` does not allow. A + # generated workspace that passed before anyone proved anything + # would be worthless. + if (cd .comparator/isSumOfThreeCubes_2 && lake test); then + echo "::error::Comparator accepted an unproved generated workspace" + exit 1 + fi + + # 2. The same workspace with the statement actually proved. The + # witness is the one this repository's own source gives. + python3 - <<'PY' + import pathlib + + submission = pathlib.Path(".comparator/isSumOfThreeCubes_2/Submission.lean") + text = submission.read_text(encoding="utf-8") + filled = text.replace(":= by\n sorry", ":= by\n exact ⟨1, 1, 0, by norm_num⟩") + assert filled != text, "nothing to fill in the generated submission" + submission.write_text(filled, encoding="utf-8") + PY + (cd .comparator/isSumOfThreeCubes_2 && lake build && lake test) + + # 3. The `answer(sorry)` workspace, with the hole filled by the + # proposition on the other side of the iff and the bridge closed + # by `Iff.rfl`. Comparator accepts this, and it resolves nothing: + # the definition hole is a place where a machine check cannot + # stand in for a human reading the answer. The generated README + # tells the solver so; this asserts it. + python3 - <<'PY' + import pathlib + + root = pathlib.Path(".comparator/isSumOfThreeCubes_iff_mod_9") + submission = root / "Submission.lean" + text = submission.read_text(encoding="utf-8") + answer = ( + "noncomputable def isSumOfThreeCubes_iff_mod_9_answer : Prop :=\n" + " ∀ n : ℤ, IsSumOfThreeCubes n ↔ ¬(n ≡ 4 [ZMOD 9] ∨ n ≡ 5 [ZMOD 9])" + ) + text = text.replace( + "noncomputable def isSumOfThreeCubes_iff_mod_9_answer : Prop := sorry", + answer, + ) + text = text.replace(":= by\n sorry", ":=\n Iff.rfl") + submission.write_text(text, encoding="utf-8") + PY + (cd .comparator/isSumOfThreeCubes_iff_mod_9 && lake build && lake test) + echo "Comparator accepted a gamed definition hole; hole values need a human." diff --git a/comparator/OWNERSHIP.md b/comparator/OWNERSHIP.md index 175d654233..f072660b8f 100644 --- a/comparator/OWNERSHIP.md +++ b/comparator/OWNERSHIP.md @@ -46,8 +46,9 @@ here and not in lean-eval's CI. `ProblemManifest` carries what the Lean text does not say: the theorem's name and its explicit parameters, the hole types Lean reported, the permitted -axioms, the toolchain and Mathlib pins, and a `source` record with the FC -repository, commit, blob, module and declaration id. lean-eval#536 requires the +axioms, a `source` record with the FC repository, commit, blob, module, +declaration id and this repository's Lean and Mathlib pins, and a `target` +record with LeanEval's pins, which are the ones the workspace is built at. lean-eval#536 requires the commit and the declaration id by name, and they are FC-side by necessity: the generator sees a Lean module, not a repository. They are also what makes regeneration possible when Formal Conjectures corrects a misformalisation @@ -58,12 +59,12 @@ upstream. The generator writes the manifest into the workspace unaltered, as | File | Lines | Then | |---|---|---| -| `scripts/leaneval_generator.py` | 218 | deleted; `generate` becomes a call into the pinned package | -| `scripts/test_leaneval_generator.py` | 153 | deleted, less whatever remains useful as a contract test against the pinned generator | +| `scripts/leaneval_generator.py` | 228 | deleted; `generate` becomes a call into the pinned package | +| `scripts/test_leaneval_generator.py` | 163 | deleted, less whatever remains useful as a contract test against the pinned generator | | `comparator/templates/WorkspaceTest.lean` | 37 | deleted; the generator supplies its own workspace test | -| `scripts/leaneval_interface.py` | 270 | replaced by an import from the pinned package, to the extent its types match | +| `scripts/leaneval_interface.py` | 293 | replaced by an import from the pinned package, to the extent its types match | -That is 408 lines deleted outright and 270 more replaced. Nothing in +That is 428 lines deleted outright and 293 more replaced. Nothing in `scripts/fc_leaneval_importer.py` changes, and `make_comparator_workspace.py` changes by one import. @@ -71,12 +72,13 @@ changes by one import. | File | Lines | Why it cannot move | |---|---|---| -| `scripts/fc_leaneval_importer.py` | 843 | resolves a declaration against an exact FC commit, reads the elaborated environment, copies the FC-local closure, types each `answer(sorry)` slot, and records the provenance | +| `scripts/fc_leaneval_importer.py` | 870 | resolves a declaration against an exact FC commit, reads the elaborated environment, copies the FC-local closure, types each `answer(sorry)` slot, and records the provenance | | `scripts/comparator_facts.lean` | 205 | the Lean extractor: source ranges, binder explicitness, and answer-slot types, all of which only this repository's elaborated environment knows | | `scripts/test_fc_leaneval_importer.py` | 400 | every case pins a real extraction defect | -| `scripts/make_comparator_workspace.py` | 175 | the command, and the directory write that belongs to neither side | +| `scripts/make_comparator_workspace.py` | 157 | the command, and the directory write that belongs to neither side | | `scripts/test_make_comparator_workspace.py` | 99 | asserts the emitted pair rebuilds the workspace exactly | | `comparator/problems/*.toml` | — | the choices FC source cannot make for itself: which module, and an answer type Lean reports ambiguously | +| `comparator/tools.toml` | — | the pins, in one machine-readable place: this repository's under `[tools]`, LeanEval's under `[target]` | Nothing in the importer names a workspace file, a workspace layout, or an import graph. If a change to it would, the change belongs on the other side. @@ -94,6 +96,11 @@ would have to say how they group. lean-eval#536 asks for this to be scoped against the actual FC100 statements rather than in the abstract, so it is not built here. +**A vendored workspace.** A workspace checked into this repository is a copy +of generator output, so it drifts from the generator, and it says nothing about +the importer because a human wrote it. The Lean 4.33 evidence comes from +generating one in CI instead. + **Lifecycle.** Result records, resubmission, and revision tracking are LeanEval's, per lean-eval#536. This repository regenerates and opens a pull request; it keeps no state about what happened to one. @@ -119,10 +126,20 @@ would change bytes at the seam. `answer(sorry)` hole is only checkable against that build. 4. **Answer-slot types are read under this repository's toolchain.** The importer asks Formal Conjectures' elaborated environment, at FC's Lean and - Mathlib pins, for the type of each slot; LeanEval builds at Lean 4.33 and - its own Mathlib. A type whose name or elaboration differs between the two - revisions would be wrong in a way `--verify` cannot see, because `--verify` - also runs at FC's pins. Only a build on the LeanEval side closes this. + Mathlib pins, for the type of each slot; the workspace is built at + LeanEval's Lean 4.33 and its own Mathlib. A type whose name or elaboration + differs between the two revisions would be wrong in a way `--verify` cannot + see, because `--verify` also runs at FC's pins. + + `.github/workflows/comparator-lean-4-33.yml` now does both halves in one + job: it generates at 4.27 and builds and Comparator-checks at 4.33, on one + plain theorem and one `Prop`-valued `answer(sorry)` slot. So the gap is + observed rather than asserted, and every manifest states it — + `source.lean_toolchain` against `target.lean_toolchain`. What is still open + is the general case: two declarations passing says nothing about a slot + whose type name changed between the two Mathlib revisions. A frozen-set + import needs that job over the whole set, and the decision about which side + owns the answer when they disagree is lean-eval's. 5. **Who triggers regeneration is unassigned.** The plan gives the importer the duty to regenerate and re-PR when Formal Conjectures fixes a misformalisation upstream, and gives lifecycle to LeanEval. Nothing yet says diff --git a/comparator/README.md b/comparator/README.md index ad153545ed..7e67228b4b 100644 --- a/comparator/README.md +++ b/comparator/README.md @@ -2,127 +2,59 @@ This directory contains the Formal Conjectures side of the integration with [`leanprover/lean-eval`](https://github.com/leanprover/lean-eval) and -[`leanprover/comparator`](https://github.com/leanprover/comparator). - -## Status and version boundary - -The code in this draft is a **conformance prototype**, not a second permanent -workspace generator. - -- Formal Conjectures currently elaborates its source under its own pinned - toolchain. -- LeanEval is the benchmark host and target environment. Imported problems must - compile under LeanEval's pinned **Lean 4.33** toolchain and matching Mathlib - revision. -- Formal Conjectures does not need a repository-wide toolchain upgrade merely - to support the integration. -- LeanEval owns the shared Challenge/Solution/Submission generator and - Comparator execution path. -- Formal Conjectures owns an importer that resolves declarations, preserves - provenance, maps `answer(sorry)` semantics, and emits reviewable LeanEval - source and manifests. - -The code is already arranged along that line. `scripts/fc_leaneval_importer.py` -is the FC side and stays; `scripts/leaneval_generator.py` stands in for the -shared generator and is written to be deleted, not rewritten, once -`leanprover/lean-eval-generator` exists. [`OWNERSHIP.md`](OWNERSHIP.md) gives -the interface between them and the line counts either side of that deletion. - -This follows the ownership split proposed in +[`leanprover/comparator`](https://github.com/leanprover/comparator), following +the ownership split proposed in [`lean-eval#536`](https://github.com/leanprover/lean-eval/pull/536), with coordination tracked in [`lean-eval#533`](https://github.com/leanprover/lean-eval/issues/533) and [`formal-conjectures#4930`](https://github.com/google-deepmind/formal-conjectures/issues/4930). -## Final integration flow - -1. The FC importer resolves a declaration against an exact Formal Conjectures - commit and obtains its source range, binders, namespace, dependencies, and - `answer(sorry)` slot types from Lean. -2. It emits one marked-up LeanEval module and one problem manifest. The - manifest records at least the FC repository, commit, source path, blob, and - fully qualified declaration name, so that a workspace can be traced back to - a revision of this repository and regenerated when that revision is - corrected. -3. LeanEval builds the vendored source under Lean 4.33 and its matching Mathlib - pin. -4. The shared LeanEval generator creates `Challenge`, `ChallengeDeps`, - `Submission`, `Solution`, and Comparator configuration. -5. LeanEval CI builds the generated workspace and runs Comparator with - `sorryAx` rejected. -6. A deterministic trusted-statement fingerprint links the imported source, - generated challenge, result record, and later upstream corrections. - -The importer must fail closed on ambiguous declarations, source drift, -inaccessible binders, unsupported dependencies, answer-slot types that cannot -be matched safely, and existing output. - -## Conformance suite before a public import +**[`OWNERSHIP.md`](OWNERSHIP.md) is the map**: which code is Formal +Conjectures' permanently, which code is standing in for +`leanprover/lean-eval-generator` and is deleted when that lands, what crosses +between them, and what the interface still needs from lean-eval. Read it first. +This file is the operator's page: the commands, their inputs, and the pins. -The adapter should cover these boundary cases before importing a frozen set: - -- a plain theorem proof; -- a `Prop`-valued `answer(sorry)` slot; -- a non-`Prop` answer slot; -- explicit declaration parameters versus `∀` binders in the conclusion; -- trusted helper dependencies requiring `ChallengeDeps` or multiple trusted - files. - -The smoke cases in this draft exercise those distinctions. They validate -extraction and adapter behavior, not mathematical correctness or maintainer -acceptance. - -The first public open-conjectures import also needs a corrected source set. -`FC100OpenSet1` currently verifies itself as 92 `research open` entries and 8 -`research solved` entries, so it must not be imported wholesale as one hundred -open conjectures. - -## Current prototype - -`scripts/comparator_facts.lean` asks Lean for the selected declaration's source -range, binders, and `answer(sorry)` slot types. - -`scripts/fc_leaneval_importer.py` maps that declaration to the pair the -generator consumes: one marked-up Mathlib-only Lean module, and one manifest. -`scripts/leaneval_generator.py` turns the pair into a pinned standalone -workspace as a conformance harness, and -`scripts/make_comparator_workspace.py` runs the two in order. The generated -workspace uses the Formal Conjectures toolchain and dependency pins. It is -**not** the final LeanEval 4.33 artifact. - -The prototype workspace contains: +## Two toolchains -- `ChallengeDeps.lean`, with the statement's copied Formal Conjectures closure; -- `Challenge.lean`, with the trusted statement and proof hole; -- `Submission.lean` and `Submission/`, where a solver works; -- `Solution.lean`, which connects the submission to the trusted statement; -- `config.json`, with theorem targets, definition targets, and permitted axioms; -- `manifest.json`, the manifest the importer handed the generator: the Formal - Conjectures source commit and declaration id, the copied closure, the exact - original declaration text, the hole types, and the pins; -- pinned Lean, Mathlib, Formal Conjectures, Comparator, and helper-tool versions. +Formal Conjectures elaborates its own source under its own pinned toolchain. +LeanEval is the benchmark host: an imported problem is built and checked under +LeanEval's pinned Lean 4.33 and matching Mathlib. Supporting the integration +does not require a repository-wide toolchain upgrade here. -`Solution.lean` is fixed. It fails to build if the submission changes the -statement. Comparator also rejects `sorryAx` because it is not in the permitted -axiom list. +So the importer reads a declaration's source range, binders, dependencies and +`answer(sorry)` slot types from an environment elaborated at *this* +repository's toolchain, and the workspace it produces is pinned to *LeanEval's* +toolchain and Mathlib. `manifest.json` records both pin sets, under `source` +and `target`. `.github/workflows/comparator-lean-4-33.yml` generates a +workspace here and builds and Comparator-checks it there, in one job, which is +what turns the gap between them into something observed rather than assumed. -### Generate one prototype workspace +## Generate one workspace ```bash python3 scripts/make_comparator_workspace.py erdos_940.variants.large_integers ``` -Use `--out` to choose the parent directory. The generator refuses to overwrite -an existing workspace. It writes into a temporary directory and renames the +Use `--out` to choose the parent directory. Generation refuses to overwrite an +existing workspace: it writes into a temporary directory and renames the complete workspace into place. The importer stops when the selected source differs from the pinned upstream revision. This prevents a workspace from combining a working-tree statement with an older imported context. -`--verify` elaborates the marked-up module against this checkout's Mathlib -before anything is written, so a copying defect fails here rather than in -LeanEval CI. +`--verify` elaborates the marked-up module before anything is written, so an +FC-side copying defect fails here rather than in LeanEval CI. It runs at this +repository's Lean and Mathlib, so it is not evidence about the 4.33 build. + +The workspace contains `ChallengeDeps.lean` with the statement's copied Formal +Conjectures closure, `Challenge.lean` with the trusted statement and its proof +hole, `Submission.lean` and `Submission/` where a solver works, `Solution.lean` +connecting the two, `config.json` with the theorem targets, definition targets +and permitted axioms, and `manifest.json`. `Solution.lean` is fixed: it fails +to build if the submission changes the statement. Comparator rejects `sorryAx`, +because it is not in the permitted axiom list. ### Emit only what this repository owns @@ -135,7 +67,7 @@ This writes `Problem.lean` and `manifest.json` and generates no workspace. It is the pair the importer contributes to a LeanEval problem pull request once the shared generator is a pinned dependency there. -### Supported prototype inputs +### Supported inputs - theorem proofs; - definition answers represented by `answer(sorry)`; @@ -144,6 +76,10 @@ the shared generator is a pinned dependency there. Plain-statement disproofs remain out of scope until Comparator provides an upstream interface for them. +The importer fails closed on ambiguous declarations, source drift, inaccessible +binders, unsupported dependencies, answer-slot types that cannot be matched +safely, and existing output. + ## Problem files `problems/*.toml` is an input, not the LeanEval manifest: it records the @@ -171,12 +107,31 @@ python3 scripts/make_comparator_workspace.py --validate ## Tool pins -`tools.toml` records the external tool revisions used by the prototype. -Generated prototype workspaces pin Mathlib from `lake-manifest.json` and Formal -Conjectures to the current upstream revision. Workspace generation itself does -not run Comparator. +`tools.toml` is the one machine-readable source. `[tools]` are the revisions a +local run uses under this repository's toolchain. `[target]` are LeanEval's: +the Lean toolchain and Mathlib revision every generated workspace is pinned to, +and the Comparator and `lean4export` commits that check it. Every manifest +records `[target]` beside the source pins. Generation itself does not run +Comparator. + +## Conformance before a public import + +The adapter should cover these boundary cases before importing a frozen set: -The final importer instead targets LeanEval's Lean 4.33 toolchain, Mathlib pin, -manifest schema, shared generator revision, and CI policy. Those target pins -belong on the LeanEval side and must be recorded in every generated import or -its provenance record. +- a plain theorem proof; +- a `Prop`-valued `answer(sorry)` slot; +- a non-`Prop` answer slot; +- explicit declaration parameters versus `∀` binders in the conclusion; +- trusted helper dependencies requiring `ChallengeDeps` or multiple trusted + files. + +The two CI jobs exercise those distinctions: `build-and-docs.yml` generates +five declarations covering each case and checks the importer-to-generator seam, +and `comparator-lean-4-33.yml` builds two of them at LeanEval's pins and runs +Comparator on them. They validate extraction and adapter behaviour, not +mathematical correctness or maintainer acceptance. + +The first public open-conjectures import also needs a corrected source set. +`FC100OpenSet1` currently verifies itself as 92 `research open` entries and 8 +`research solved` entries, so it must not be imported wholesale as one hundred +open conjectures. diff --git a/comparator/pilots/fc_sum_of_three_cubes/Challenge.lean b/comparator/pilots/fc_sum_of_three_cubes/Challenge.lean deleted file mode 100644 index 7aa5487475..0000000000 --- a/comparator/pilots/fc_sum_of_three_cubes/Challenge.lean +++ /dev/null @@ -1,41 +0,0 @@ -/- -Copyright 2026 The Formal Conjectures Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. --/ - -import ChallengeDeps - -/-! -Lean 4.33 conformance pilot for the Formal Conjectures to LeanEval importer. - -The first target is a solved smoke theorem. The second preserves the -`answer(sorry)` shape of an open conjecture as a Comparator definition hole. --/ - -namespace SumOfThreeCubes - -def isSumOfThreeCubes_iff_mod_9_answer : Prop := sorry - -theorem isSumOfThreeCubes_2 : - IsSumOfThreeCubes (2 : ℤ) := by - sorry - -theorem isSumOfThreeCubes_iff_mod_9 : - isSumOfThreeCubes_iff_mod_9_answer ↔ - ∀ n : ℤ, - IsSumOfThreeCubes n ↔ - ¬(n ≡ 4 [ZMOD 9] ∨ n ≡ 5 [ZMOD 9]) := by - sorry - -end SumOfThreeCubes diff --git a/comparator/pilots/fc_sum_of_three_cubes/ChallengeDeps.lean b/comparator/pilots/fc_sum_of_three_cubes/ChallengeDeps.lean deleted file mode 100644 index 2e0c388af8..0000000000 --- a/comparator/pilots/fc_sum_of_three_cubes/ChallengeDeps.lean +++ /dev/null @@ -1,30 +0,0 @@ -/- -Copyright 2026 The Formal Conjectures Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. --/ - -import Mathlib - -/-! -Trusted dependency copied from -`FormalConjectures/Wikipedia/SumOfThreeCubes.lean`. --/ - -namespace SumOfThreeCubes - -/-- The predicate that `n : R` is a sum of three cubes. -/ -def IsSumOfThreeCubes {R : Type*} [Ring R] (n : R) : Prop := - ∃ x y z : R, n = x ^ 3 + y ^ 3 + z ^ 3 - -end SumOfThreeCubes diff --git a/comparator/pilots/fc_sum_of_three_cubes/README.md b/comparator/pilots/fc_sum_of_three_cubes/README.md deleted file mode 100644 index af19a78df0..0000000000 --- a/comparator/pilots/fc_sum_of_three_cubes/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# FC Sum of Three Cubes: Lean 4.33 Comparator pilot - -This is a vendored, reviewable conformance pilot for the Formal Conjectures to -LeanEval boundary. It does not upgrade the Formal Conjectures repository and it -does not claim to solve the open sum-of-three-cubes conjecture. - -## Pins - -- Formal Conjectures source commit: `9f5ee773841921f460b4a26a3552f5eca4accaa0` -- LeanEval reference commit: `7699436464052268e6c04b41554bfbc2c6908ec5` -- Lean: `leanprover/lean4:v4.33.0` -- Mathlib: `6f1ef4e5dd604a435bddba4747b13970cd65d2a1` -- Comparator, final Lean 4.33 commit: `c0c5a52d2aff92b457c3e5ed4a68c1ebc5795809` -- Lean 4.33 exporter: `15f6055e299ad5b89345e533cc2192f4cc00f659` - -`provenance.json` records the source declarations, transformation log, target -pins, and a SHA-256 fingerprint over the exact trusted `ChallengeDeps.lean` and -`Challenge.lean` files. - -## What the workspace checks - -The workspace follows LeanEval's generated structure: - -- `ChallengeDeps.lean` contains the trusted predicate copied from Formal - Conjectures. -- `Challenge.lean` contains a solved smoke theorem and the open conjecture's - `answer(sorry)` slot, hoisted into a `Prop`-valued definition hole. -- `Submission.lean` supplies an actual proof of the solved smoke theorem. -- `Solution.lean` is fixed and delegates the challenge names to the submission. -- `config.json` asks Comparator to check both theorem names and the definition - hole under LeanEval's permitted-axiom policy. - -The open-conjecture smoke submission intentionally defines the answer hole to -be the proposition itself and proves the bridge by `Iff.rfl`. Comparator should -accept that declaration-level shape. It is not a mathematical resolution and a -human semantic reviewer must reject it. Keeping this case explicit tests the -reason definition answers require a separate review stage. - -## Build - -```bash -lake exe cache get -lake build -``` - -## Comparator smoke test - -Build the pinned Comparator commit recorded above. Its manifest pins the matching -Lean 4.33 exporter, then run: - -```bash -COMPARATOR_BIN=/absolute/path/to/comparator \ -COMPARATOR_LANDRUN=/absolute/path/to/comparator/scripts/fake-landrun.sh \ -COMPARATOR_LEAN4EXPORT=/absolute/path/to/lean4export \ -lake test -``` - -The fake landrun is acceptable only for this trusted development smoke test. A -real submission service must use the production sandbox and preserve -Comparator's clean-build assumptions. - -## What this does not establish - -A passing run establishes that one vendored FC problem shape builds under -LeanEval's Lean 4.33 and Mathlib pins and crosses Comparator's theorem and -definition-hole interfaces. It does not establish that the general importer is -complete, that arbitrary FC dependencies port automatically, that the open -conjecture is solved, or that maintainers should accept the problem into a -frozen release. diff --git a/comparator/pilots/fc_sum_of_three_cubes/Solution.lean b/comparator/pilots/fc_sum_of_three_cubes/Solution.lean deleted file mode 100644 index e67b9d07fb..0000000000 --- a/comparator/pilots/fc_sum_of_three_cubes/Solution.lean +++ /dev/null @@ -1,36 +0,0 @@ -/- -Copyright 2026 The Formal Conjectures Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. --/ - -import ChallengeDeps -import Submission - -namespace SumOfThreeCubes - -@[reducible] noncomputable def isSumOfThreeCubes_iff_mod_9_answer : Prop := - Submission.SumOfThreeCubes.isSumOfThreeCubes_iff_mod_9_answer - -theorem isSumOfThreeCubes_2 : - IsSumOfThreeCubes (2 : ℤ) := - Submission.SumOfThreeCubes.isSumOfThreeCubes_2 - -theorem isSumOfThreeCubes_iff_mod_9 : - isSumOfThreeCubes_iff_mod_9_answer ↔ - ∀ n : ℤ, - IsSumOfThreeCubes n ↔ - ¬(n ≡ 4 [ZMOD 9] ∨ n ≡ 5 [ZMOD 9]) := - Submission.SumOfThreeCubes.isSumOfThreeCubes_iff_mod_9 - -end SumOfThreeCubes diff --git a/comparator/pilots/fc_sum_of_three_cubes/Submission.lean b/comparator/pilots/fc_sum_of_three_cubes/Submission.lean deleted file mode 100644 index c037dfb478..0000000000 --- a/comparator/pilots/fc_sum_of_three_cubes/Submission.lean +++ /dev/null @@ -1,48 +0,0 @@ -/- -Copyright 2026 The Formal Conjectures Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. --/ - -import ChallengeDeps -import Submission.Helpers - -/-! -Trusted smoke submission for the integration harness. - -The open-conjecture target deliberately uses the proposition itself as the -definition-hole value and proves the bridge by reflexivity. Comparator should -accept this shape, while a human semantic reviewer must reject it as a -mathematical resolution. That is the definition-hole threat model this pilot -is meant to preserve. --/ - -namespace Submission.SumOfThreeCubes - -def isSumOfThreeCubes_iff_mod_9_answer : Prop := - ∀ n : ℤ, - _root_.SumOfThreeCubes.IsSumOfThreeCubes n ↔ - ¬(n ≡ 4 [ZMOD 9] ∨ n ≡ 5 [ZMOD 9]) - -theorem isSumOfThreeCubes_2 : - _root_.SumOfThreeCubes.IsSumOfThreeCubes (2 : ℤ) := by - exact ⟨1, 1, 0, by norm_num⟩ - -theorem isSumOfThreeCubes_iff_mod_9 : - isSumOfThreeCubes_iff_mod_9_answer ↔ - ∀ n : ℤ, - _root_.SumOfThreeCubes.IsSumOfThreeCubes n ↔ - ¬(n ≡ 4 [ZMOD 9] ∨ n ≡ 5 [ZMOD 9]) := - Iff.rfl - -end Submission.SumOfThreeCubes diff --git a/comparator/pilots/fc_sum_of_three_cubes/Submission/Helpers.lean b/comparator/pilots/fc_sum_of_three_cubes/Submission/Helpers.lean deleted file mode 100644 index 4d2e18725c..0000000000 --- a/comparator/pilots/fc_sum_of_three_cubes/Submission/Helpers.lean +++ /dev/null @@ -1,23 +0,0 @@ -/- -Copyright 2026 The Formal Conjectures Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. --/ - -import Mathlib - -/-! Helper lemmas for the trusted smoke submission. -/ - -namespace Submission - -end Submission diff --git a/comparator/pilots/fc_sum_of_three_cubes/WorkspaceTest.lean b/comparator/pilots/fc_sum_of_three_cubes/WorkspaceTest.lean deleted file mode 100644 index 7ce5489c7a..0000000000 --- a/comparator/pilots/fc_sum_of_three_cubes/WorkspaceTest.lean +++ /dev/null @@ -1,36 +0,0 @@ -/- -Copyright 2026 The Formal Conjectures Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. --/ - -import Lean - -open Lean - -/-- Run Comparator on this workspace's `config.json`, so that `lake test` -is the integration check. The binary comes from `PATH`, or from -`COMPARATOR_BIN`. -/ -def main : IO UInt32 := do - let comparatorBin := (← IO.getEnv "COMPARATOR_BIN").getD "comparator" - try - let child ← IO.Process.spawn { - cmd := "lake" - args := #["env", comparatorBin, "config.json"] - } - child.wait - catch err => - IO.eprintln s!"Failed to run Comparator via `{comparatorBin}`." - IO.eprintln "Install Comparator with a Lean 4.33-compatible lean4export, or set COMPARATOR_BIN." - IO.eprintln s!"Original error: {err}" - pure 1 diff --git a/comparator/pilots/fc_sum_of_three_cubes/config.json b/comparator/pilots/fc_sum_of_three_cubes/config.json deleted file mode 100644 index cb99776f93..0000000000 --- a/comparator/pilots/fc_sum_of_three_cubes/config.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "challenge_module": "Challenge", - "solution_module": "Solution", - "theorem_names": [ - "SumOfThreeCubes.isSumOfThreeCubes_2", - "SumOfThreeCubes.isSumOfThreeCubes_iff_mod_9" - ], - "definition_names": [ - "SumOfThreeCubes.isSumOfThreeCubes_iff_mod_9_answer" - ], - "permitted_axioms": [ - "propext", - "Quot.sound", - "Classical.choice" - ], - "enable_nanoda": false -} diff --git a/comparator/pilots/fc_sum_of_three_cubes/lakefile.toml b/comparator/pilots/fc_sum_of_three_cubes/lakefile.toml deleted file mode 100644 index 18ef8135ed..0000000000 --- a/comparator/pilots/fc_sum_of_three_cubes/lakefile.toml +++ /dev/null @@ -1,27 +0,0 @@ -name = "fc_sum_of_three_cubes_lean433" -testDriver = "workspace_test" -defaultTargets = ["ChallengeDeps", "Challenge", "Solution", "Submission"] - -[leanOptions] -autoImplicit = false - -[[require]] -name = "mathlib" -git = "https://github.com/leanprover-community/mathlib4.git" -rev = "6f1ef4e5dd604a435bddba4747b13970cd65d2a1" - -[[lean_lib]] -name = "ChallengeDeps" - -[[lean_lib]] -name = "Challenge" - -[[lean_lib]] -name = "Solution" - -[[lean_lib]] -name = "Submission" - -[[lean_exe]] -name = "workspace_test" -root = "WorkspaceTest" diff --git a/comparator/pilots/fc_sum_of_three_cubes/lean-toolchain b/comparator/pilots/fc_sum_of_three_cubes/lean-toolchain deleted file mode 100644 index 025e59548e..0000000000 --- a/comparator/pilots/fc_sum_of_three_cubes/lean-toolchain +++ /dev/null @@ -1 +0,0 @@ -leanprover/lean4:v4.33.0 diff --git a/comparator/pilots/fc_sum_of_three_cubes/provenance.json b/comparator/pilots/fc_sum_of_three_cubes/provenance.json deleted file mode 100644 index 763cf0f2ef..0000000000 --- a/comparator/pilots/fc_sum_of_three_cubes/provenance.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "schema_version": 1, - "source": { - "repository": "google-deepmind/formal-conjectures", - "commit": "9f5ee773841921f460b4a26a3552f5eca4accaa0", - "path": "FormalConjectures/Wikipedia/SumOfThreeCubes.lean", - "blob_sha": "f36a362c53f254a14ed0e0fb9239abefb4b762f1", - "declarations": [ - "SumOfThreeCubes.IsSumOfThreeCubes", - "SumOfThreeCubes.isSumOfThreeCubes_2", - "SumOfThreeCubes.isSumOfThreeCubes_iff_mod_9" - ], - "subset": "Subsets.FC100OpenSet1.problems", - "category": "research open" - }, - "target": { - "repository": "leanprover/lean-eval", - "commit": "7699436464052268e6c04b41554bfbc2c6908ec5", - "lean_toolchain": "leanprover/lean4:v4.33.0", - "mathlib_revision": "6f1ef4e5dd604a435bddba4747b13970cd65d2a1", - "comparator_commit": "c0c5a52d2aff92b457c3e5ed4a68c1ebc5795809", - "lean4export_commit": "15f6055e299ad5b89345e533cc2192f4cc00f659", - "workspace_shape": "ChallengeDeps/Challenge/Submission/Solution" - }, - "transformations": [ - "removed Formal Conjectures category attributes", - "copied the trusted IsSumOfThreeCubes dependency", - "hoisted answer(sorry) into a Prop-valued definition hole", - "added a fixed Solution adapter delegating to Submission" - ], - "trusted_files_sha256": "ad2ddda464477fd13661730acda71e4e13eaf2e3f90c97a3821290ae9b82b4b3", - "review": { - "mathematical_status": "open", - "smoke_submission_is_not_a_resolution": true, - "human_definition_value_review_required": true - } -} diff --git a/comparator/tools.toml b/comparator/tools.toml index a04bb304e9..838e7add4b 100644 --- a/comparator/tools.toml +++ b/comparator/tools.toml @@ -1,7 +1,23 @@ # The pinned external tools, one machine-readable source of truth. "At or # after" prose is not a lock; CI, local setup and documentation read this. + +# What a local run of the importer uses, under this repository's toolchain. [tools] comparator = "71b52ec29e06d4b7d882726553b1ceb99a2499e0" landrun = "5ed4a3db3a4ad930d577215c6b9abaa19df7f99f" -# lean4export: the tag matching the workspace's lean-toolchain, v4.27.0 today. +# lean4export: the tag matching this repository's lean-toolchain, v4.27.0 today. lean4export = "v4.27.0" + +# Where a generated workspace is built and checked. These are LeanEval's pins, +# not this repository's: a workspace is vendored into lean-eval and built +# there, so it has to be buildable where it is going. The generator writes +# `lean_toolchain` and `mathlib_revision` into every workspace, and every +# manifest records the pair beside Formal Conjectures' own, which is what makes +# the gap between the two readable rather than assumed. +[target] +repository = "leanprover/lean-eval" +commit = "7699436464052268e6c04b41554bfbc2c6908ec5" +lean_toolchain = "leanprover/lean4:v4.33.0" +mathlib_revision = "6f1ef4e5dd604a435bddba4747b13970cd65d2a1" +comparator = "c0c5a52d2aff92b457c3e5ed4a68c1ebc5795809" +lean4export = "15f6055e299ad5b89345e533cc2192f4cc00f659" diff --git a/scripts/fc_leaneval_importer.py b/scripts/fc_leaneval_importer.py index 847f428572..6f8f6a4258 100644 --- a/scripts/fc_leaneval_importer.py +++ b/scripts/fc_leaneval_importer.py @@ -37,6 +37,7 @@ MarkedUpModule, ProblemManifest, SourceRecord, + TargetRecord, ) ROOT = pathlib.Path(__file__).resolve().parent.parent @@ -60,11 +61,31 @@ ) -def tool_pins(): - """The locked external tool revisions; comparator/tools.toml is the one - machine-readable source, and this module refuses to restate it.""" +def _tools_file(): + """comparator/tools.toml is the one machine-readable source of pins, and + this module refuses to restate it.""" with (COMPARATOR_DIR / "tools.toml").open("rb") as handle: - return tomllib.load(handle)["tools"] + return tomllib.load(handle) + + +def target_pins(): + """LeanEval's pins, where a generated workspace is built and checked. + + They are not this repository's, and the importer neither chooses them nor + elaborates against them. It records them because a workspace that is + vendored into lean-eval has to be buildable there, and because a manifest + that carries both pin sets makes the gap between where the hole types were + read and where they will be used a readable fact rather than an assumption. + """ + target = _tools_file()["target"] + return TargetRecord( + repository=target["repository"], + commit=target["commit"], + lean_toolchain=target["lean_toolchain"], + mathlib_revision=target["mathlib_revision"], + comparator=target["comparator"], + lean4export=target["lean4export"], + ) def elaborator_facts(module, declaration): @@ -653,7 +674,9 @@ def pins(source_path=None): return mathlib_rev, fc_rev -def source_record(declaration, module, source_path, fc_rev, dependencies, original): +def source_record( + declaration, module, source_path, fc_rev, dependencies, original, mathlib_rev +): """Where the copied statement and its dependencies came from. lean-eval#536 requires the manifest to record the FC source commit and @@ -677,6 +700,8 @@ def source_record(declaration, module, source_path, fc_rev, dependencies, origin declaration=declaration, copied_dependencies=tuple(dependencies), original_declaration=original, + lean_toolchain=(ROOT / "lean-toolchain").read_text(encoding="utf-8").strip(), + mathlib_revision=mathlib_rev, ) @@ -758,8 +783,6 @@ def import_problem(problem, answer_type=None, module=None): apply_arguments=tuple(args), holes=tuple(holes), permitted_axioms=PERMITTED_AXIOMS, - lean_toolchain=(ROOT / "lean-toolchain").read_text(encoding="utf-8").strip(), - mathlib_revision=mathlib_rev, source=source_record( declared, fc_module, @@ -767,8 +790,9 @@ def import_problem(problem, answer_type=None, module=None): fc_rev, [dep["name"] for dep in facts.get("dependencies", [])], original, + mathlib_rev, ), - tools=tool_pins(), + target=target_pins(), source_url=str(problem_file.get("source", "")), notes=str(problem_file.get("notes", "")), ) @@ -786,9 +810,12 @@ def elaborate(marked_up): The check runs here rather than on a generated workspace because the module is what this repository hands over: an FC-side defect should fail - on the FC side, not in lean-eval's CI. It is offline, and it uses the - Mathlib revision the manifest pins because that is this checkout's. It - checks elaboration, not a lakefile; a Comparator run exercises the build. + on the FC side, not in lean-eval's CI. It is offline, and it runs at this + repository's Lean and Mathlib, which are the manifest's `source` pins and + not its `target` pins: a module that elaborates here is not thereby known + to elaborate at LeanEval's toolchain, and only a build there settles that. + It checks elaboration, not a lakefile; a Comparator run exercises the + build. """ with tempfile.NamedTemporaryFile( "w", suffix=".lean", delete=False, encoding="utf-8" diff --git a/scripts/leaneval_generator.py b/scripts/leaneval_generator.py index d2a1132d61..22ae29a988 100644 --- a/scripts/leaneval_generator.py +++ b/scripts/leaneval_generator.py @@ -39,7 +39,7 @@ def lakefile(package, mathlib_rev): - """Mathlib and nothing else. + """Mathlib at the target revision, and nothing else. The workspace used to require Formal Conjectures too, so that the Challenge could import the problem's module. lean-eval vendors its @@ -83,7 +83,7 @@ def _readme(package, manifest): "\nFill each definition hole in `Submission.lean` too. Hole answers " "also get a\nhuman check, because a hole can be gamed in ways the " "comparator cannot see.\nChecking holes needs a comparator built at " - f"commit `{manifest.tools['comparator'][:8]}`, which\nadded definition " + f"commit `{manifest.target.comparator[:8]}`, which\nadded definition " "support.\n" if manifest.holes else "" @@ -170,8 +170,15 @@ def generate(marked_up, manifest): # and closes it with the Submission theorem, so it fails to compile the # moment the submission proves anything else. The participant never edits # it, which is what keeps the statement pinned. + # `@[reducible]`: the Challenge's statement mentions the Solution's copy of + # the hole and the Submission theorem's type mentions the participant's, so + # the adapter only typechecks if the unifier unfolds one into the other. + # Default transparency does, but the one workspace known to have built at + # LeanEval's toolchain marks it reducible, and there is no reason to be the + # first to find out whether that mattered. delegated = "".join( - f"noncomputable def {hole.name} : {hole.type} := Submission.{hole.name}\n\n" + f"@[reducible] noncomputable def {hole.name} : {hole.type} :=" + f"\n Submission.{hole.name}\n\n" for hole in manifest.holes ) solution = ( @@ -197,8 +204,11 @@ def generate(marked_up, manifest): config["definition_names"] = manifest.hole_names() return { - "lakefile.toml": lakefile(package, manifest.mathlib_revision), - "lean-toolchain": manifest.lean_toolchain + "\n", + # The workspace is built where it is going, not where it was made: + # these are LeanEval's pins, and the manifest carries this repository's + # beside them. + "lakefile.toml": lakefile(package, manifest.target.mathlib_revision), + "lean-toolchain": manifest.target.lean_toolchain + "\n", "README.md": _readme(package, manifest), "ChallengeDeps.lean": "import Mathlib\n\n" + marked_up.dependencies.strip("\n") diff --git a/scripts/leaneval_interface.py b/scripts/leaneval_interface.py index 06bd37c369..4aa4d1c1d2 100644 --- a/scripts/leaneval_interface.py +++ b/scripts/leaneval_interface.py @@ -14,8 +14,8 @@ MarkedUpModule one Mathlib-only Lean module, in labelled regions ProblemManifest the facts about the problem that the module's text does - not carry, including the FC source commit and the FC - declaration id + not carry, including the FC source commit, the FC + declaration id, and the pins the workspace is built with `scripts/fc_leaneval_importer.py` produces both. `scripts/leaneval_generator.py` consumes both and returns a workspace. When `lean-eval-generator` lands, the @@ -92,6 +92,12 @@ class SourceRecord: something the generator can supply: the generator sees a module, not a repository. So they cross the seam here, and the generator's only duty is to carry them into the workspace unaltered. + + `lean_toolchain` and `mathlib_revision` are Formal Conjectures' own, and + are not the pins the workspace is built with. They are here because the + hole types in the manifest were read from an environment elaborated at + them, so a reader comparing them with `TargetRecord` can see whether the + types were read where they will be used. """ repository: str @@ -102,6 +108,27 @@ class SourceRecord: declaration: str copied_dependencies: tuple original_declaration: str + lean_toolchain: str + mathlib_revision: str + + +@dataclasses.dataclass(frozen=True) +class TargetRecord: + """The pins the generated workspace is built and checked with. + + These are LeanEval's, not this repository's: a workspace is vendored into + lean-eval and built there. Formal Conjectures records them so that a + generated workspace is buildable where it is going rather than only where + it was made, and `comparator/tools.toml` is the one place they are + written down. + """ + + repository: str + commit: str + lean_toolchain: str + mathlib_revision: str + comparator: str + lean4export: str @dataclasses.dataclass(frozen=True) @@ -122,17 +149,17 @@ class ProblemManifest: apply_arguments: tuple holes: tuple permitted_axioms: tuple - lean_toolchain: str - mathlib_revision: str source: SourceRecord - tools: dict + target: TargetRecord source_url: str = "" notes: str = "" def __post_init__(self): - for field in ("id", "theorem", "qualified_theorem", "lean_toolchain"): + for field in ("id", "theorem", "qualified_theorem"): if not getattr(self, field): raise SystemExit(f"manifest has no {field}") + if not self.target.lean_toolchain or not self.target.mathlib_revision: + raise SystemExit(f"manifest {self.id} records no target pins") # lean-eval#536 names these two explicitly, and a manifest without # them cannot be traced back to a revision of this repository or # regenerated when FC fixes a misformalisation upstream. @@ -153,13 +180,11 @@ def to_json_object(self): "apply_arguments": list(self.apply_arguments), "holes": [dataclasses.asdict(hole) for hole in self.holes], "permitted_axioms": list(self.permitted_axioms), - "lean_toolchain": self.lean_toolchain, - "mathlib_revision": self.mathlib_revision, "source": { **dataclasses.asdict(self.source), "copied_dependencies": list(self.source.copied_dependencies), }, - "tools": dict(self.tools), + "target": dataclasses.asdict(self.target), } if self.source_url: payload["source_url"] = self.source_url @@ -184,10 +209,8 @@ def from_json_object(cls, payload): apply_arguments=tuple(payload["apply_arguments"]), holes=tuple(DefinitionHole(**hole) for hole in payload["holes"]), permitted_axioms=tuple(payload["permitted_axioms"]), - lean_toolchain=payload["lean_toolchain"], - mathlib_revision=payload["mathlib_revision"], source=SourceRecord(**source), - tools=dict(payload["tools"]), + target=TargetRecord(**payload["target"]), source_url=payload.get("source_url", ""), notes=payload.get("notes", ""), ) diff --git a/scripts/make_comparator_workspace.py b/scripts/make_comparator_workspace.py index e581ebab7c..23ca6225a6 100644 --- a/scripts/make_comparator_workspace.py +++ b/scripts/make_comparator_workspace.py @@ -26,26 +26,8 @@ Copying is a construction and it can be wrong in ways only Lean sees, so `--verify` elaborates the marked-up module before you trust it. -Layout produced by the generator: - - // - lakefile.toml pins: this checkout's Mathlib rev - ChallengeDeps.lean the statement's Formal Conjectures closure, copied, - importing Mathlib alone - Challenge.lean the import, the file-scoped preamble, the target - statement with attributes stripped and its proof - replaced by `sorry`, and each `answer(sorry)` hoisted - into a definition hole the solver must fill - Submission.lean where the solver works; helper modules go under - Submission/ - Solution.lean fixed: restates the statement and closes it with the - Submission theorem, so the statement cannot drift - WorkspaceTest.lean `lake test` runs comparator on config.json - README.md what the solver needs to know, cache fetch included - config.json theorem and definition names, permitted axioms - manifest.json the manifest the importer handed the generator: the FC - source commit and declaration id, the copied closure, - the hole types, and the pins +`comparator/README.md` describes the workspace this produces and the pins it +carries; this file does not restate them. Lean reports the type of each `answer(sorry)` slot. The importer refuses a case when it cannot match the reported types to their source positions. diff --git a/scripts/test_leaneval_generator.py b/scripts/test_leaneval_generator.py index c449b2ebd5..af23e6a401 100644 --- a/scripts/test_leaneval_generator.py +++ b/scripts/test_leaneval_generator.py @@ -28,7 +28,7 @@ import leaneval_generator as generator from leaneval_generator import generate -from test_leaneval_interface import A_MODULE, a_manifest +from test_leaneval_interface import A_MODULE, a_manifest, a_target # The modules that live beside the generator in `scripts/`. Anything the # generator imports from here has to move with it into the pinned package. @@ -71,9 +71,11 @@ def test_the_submission_is_namespaced_away_from_the_trusted_names(self): def test_the_solution_delegates_the_hole_and_applies_the_arguments(self): solution = a_workspace()["Solution.lean"] + # Reducible, so the unifier is certain to unfold the Solution's copy of + # the hole into the Submission's when it checks the adapter. self.assertIn( - "noncomputable def erdos_940_answer : ENNReal := " - "Submission.erdos_940_answer", + "@[reducible] noncomputable def erdos_940_answer : ENNReal :=\n" + " Submission.erdos_940_answer", solution, ) self.assertTrue(solution.rstrip().endswith("Submission.erdos_940 n")) @@ -95,9 +97,10 @@ def test_the_config_names_the_theorem_the_holes_and_the_axioms(self): self.assertIn("propext", config["permitted_axioms"]) self.assertNotIn("sorryAx", config["permitted_axioms"]) - def test_the_lakefile_pins_mathlib_and_requires_nothing_else(self): + def test_the_lakefile_pins_the_target_mathlib(self): + # Not this repository's Mathlib: the workspace is built in lean-eval. lakefile = a_workspace()["lakefile.toml"] - self.assertIn('rev = "' + "c" * 40 + '"', lakefile) + self.assertIn('rev = "' + "f" * 40 + '"', lakefile) self.assertEqual(lakefile.count("[[require]]"), 1) self.assertNotIn("formal-conjectures", lakefile) @@ -107,8 +110,15 @@ def test_the_package_name_is_an_identifier(self): 'name = "erdos_940_variants_large_integers"', files["lakefile.toml"] ) - def test_the_toolchain_file_is_the_one_the_manifest_pins(self): - self.assertEqual(a_workspace()["lean-toolchain"], "leanprover/lean4:v4.27.0\n") + def test_the_toolchain_file_is_the_target_toolchain(self): + self.assertEqual(a_workspace()["lean-toolchain"], "leanprover/lean4:v4.33.0\n") + + def test_nothing_in_the_workspace_carries_this_repositorys_toolchain(self): + # A workspace built at FC's toolchain is not the artifact lean-eval + # vendors, and shipping one would hide the pin gap the manifest states. + files = a_workspace() + self.assertNotIn("v4.27.0", files["lean-toolchain"]) + self.assertNotIn("v4.27.0", files["lakefile.toml"]) class ManifestPassThroughTest(unittest.TestCase): diff --git a/scripts/test_leaneval_interface.py b/scripts/test_leaneval_interface.py index b68f82ea1e..13847bce76 100644 --- a/scripts/test_leaneval_interface.py +++ b/scripts/test_leaneval_interface.py @@ -28,6 +28,7 @@ MarkedUpModule, ProblemManifest, SourceRecord, + TargetRecord, slug, ) @@ -42,11 +43,26 @@ def a_source(**overrides): "declaration": "erdos_940", "copied_dependencies": ("Foo.bar",), "original_declaration": "theorem erdos_940 : True := by\n sorry", + "lean_toolchain": "leanprover/lean4:v4.27.0", + "mathlib_revision": "c" * 40, } fields.update(overrides) return SourceRecord(**fields) +def a_target(**overrides): + fields = { + "repository": "leanprover/lean-eval", + "commit": "e" * 40, + "lean_toolchain": "leanprover/lean4:v4.33.0", + "mathlib_revision": "f" * 40, + "comparator": "d" * 40, + "lean4export": "0" * 40, + } + fields.update(overrides) + return TargetRecord(**fields) + + def a_manifest(**overrides): fields = { "id": "erdos_940", @@ -55,10 +71,8 @@ def a_manifest(**overrides): "apply_arguments": ("n",), "holes": (DefinitionHole(name="erdos_940_answer", type="ENNReal"),), "permitted_axioms": ("propext", "Quot.sound", "Classical.choice"), - "lean_toolchain": "leanprover/lean4:v4.27.0", - "mathlib_revision": "c" * 40, "source": a_source(), - "tools": {"comparator": "d" * 40}, + "target": a_target(), "source_url": "https://www.erdosproblems.com/940", "notes": "a reviewer note", } @@ -83,6 +97,18 @@ def test_source_commit_and_declaration_are_required(self): with self.assertRaisesRegex(SystemExit, "no FC declaration id"): a_manifest(source=a_source(declaration="")) + def test_target_pins_are_required(self): + # A workspace without them cannot be built where it is going. + with self.assertRaisesRegex(SystemExit, "no target pins"): + a_manifest(target=a_target(lean_toolchain="")) + + def test_both_pin_sets_are_recorded(self): + # The hole types were read at the source pins and will be used at the + # target pins. A manifest that carried only one could not say so. + payload = a_manifest().to_json_object() + self.assertEqual(payload["source"]["lean_toolchain"], "leanprover/lean4:v4.27.0") + self.assertEqual(payload["target"]["lean_toolchain"], "leanprover/lean4:v4.33.0") + def test_the_manifest_survives_a_round_trip(self): manifest = a_manifest() self.assertEqual(ProblemManifest.from_json(manifest.to_json()), manifest) From 84386db6f99266427fb38ab15159120636ea6d2b Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:16:45 -0400 Subject: [PATCH 13/70] Record the formalization.yaml overlap in the manifest question --- comparator/OWNERSHIP.md | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/comparator/OWNERSHIP.md b/comparator/OWNERSHIP.md index f072660b8f..b58cbdbecd 100644 --- a/comparator/OWNERSHIP.md +++ b/comparator/OWNERSHIP.md @@ -114,12 +114,35 @@ would change bytes at the seam. 1. **The markup convention is invented here.** `-- @region ` and the four region names are local. The generator core is the natural owner of the convention, since it is the reader. -2. **The manifest schema is invented here.** `schema_version = 1` and the field - names are this repository's. lean-eval#536 says the importer emits PRs that - lean-eval CI validates like any other problem PR, which needs a published - schema to validate against. The two fields the plan does name — the FC - source commit and the declaration id — are present under - `source.commit` and `source.declaration`. +2. **The manifest schema is invented here, and part of it need not be.** + `schema_version = 1` and the field names are this repository's. lean-eval#536 + says the importer emits PRs that lean-eval CI validates like any other + problem PR, which needs something published to validate against. The two + fields the plan does name — the FC source commit and the declaration id — + are present under `source.commit` and `source.declaration`. + + `mathlib-initiative/formalization.yaml` already standardises much of this: + `repository.substantive_formalization` carries a source repository and + revision, `status.main_results[]` carries a declaration, its file, its + permitted axioms and its Comparator config. Formal Conjectures does not + currently carry that file, but `Paul-Lez/hadamard-668-comparator` uses it to + describe a wrapper around FC at revision `1721605c`. + + It is not a drop-in replacement, for two reasons worth stating rather than + glossing. Its required `project`, `sources`, `automation` and `review` + sections describe who formalised something, from what, with what help, and + who reviewed it — an importer cannot fill those truthfully for an arbitrary + FC statement, because they belong to the FC contributor rather than to the + import. And it deliberately omits pins, on the stated grounds that the file + sits alongside the formalization it describes and the tree already encodes + them; a generated workspace has two pin sets and sits alongside neither. + + So the open question is not "publish a schema" but which object is which: a + manifest that drives generation and crosses the seam, and possibly a + `formalization.yaml` describing the generated workspace as a thin wrapper + once it exists somewhere durable. Nothing here implements the second, since + filling its required sections without a real answer would be worse than + omitting it. 3. **The `definition_names` config field is undocumented.** Comparator's published no-hole config does not carry it, and hole support depends on the comparator commit pinned in `tools.toml`. A generated workspace with an From fc450fa18ec5df6f422c06488d22179e9918d4bf Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:30:47 -0400 Subject: [PATCH 14/70] Read the source citation from Formal Conjectures instead of copying it --- comparator/OWNERSHIP.md | 2 +- comparator/README.md | 14 +++-- comparator/problems/erdos_1038_part_i.toml | 10 ---- scripts/fc_leaneval_importer.py | 69 +++++++++++++--------- scripts/leaneval_generator.py | 2 +- scripts/leaneval_interface.py | 4 -- scripts/test_fc_leaneval_importer.py | 33 +++++++++++ scripts/test_leaneval_interface.py | 1 - 8 files changed, 87 insertions(+), 48 deletions(-) delete mode 100644 comparator/problems/erdos_1038_part_i.toml diff --git a/comparator/OWNERSHIP.md b/comparator/OWNERSHIP.md index b58cbdbecd..6ceabdd9a3 100644 --- a/comparator/OWNERSHIP.md +++ b/comparator/OWNERSHIP.md @@ -77,7 +77,7 @@ changes by one import. | `scripts/test_fc_leaneval_importer.py` | 400 | every case pins a real extraction defect | | `scripts/make_comparator_workspace.py` | 157 | the command, and the directory write that belongs to neither side | | `scripts/test_make_comparator_workspace.py` | 99 | asserts the emitted pair rebuilds the workspace exactly | -| `comparator/problems/*.toml` | — | the choices FC source cannot make for itself: which module, and an answer type Lean reports ambiguously | +| `comparator/problems/*.toml` | — | the one choice FC source cannot make for itself: which module, when two declare the same name | | `comparator/tools.toml` | — | the pins, in one machine-readable place: this repository's under `[tools]`, LeanEval's under `[target]` | Nothing in the importer names a workspace file, a workspace layout, or an diff --git a/comparator/README.md b/comparator/README.md index 7e67228b4b..9e143db972 100644 --- a/comparator/README.md +++ b/comparator/README.md @@ -88,16 +88,22 @@ reads it. The manifest the generator receives, and writes into the workspace as `manifest.json`, is derived. Most declarations need no problem file. Add one TOML file under `problems/` -only when the source cannot select the declaration by itself. +only when two files declare the same name, which is the one thing the Lean +environment cannot resolve. | Field | Meaning | |---|---| | `id` | Workspace name. It must match the TOML filename. | | `declaration` | Lean declaration name. | | `module` | Source file when the declaration name is ambiguous. | -| `answer_type` | Explicit override when slot types cannot be matched safely. | -| `source` | Optional source link for the generated README. | -| `notes` | Optional reviewer note for the generated README. | + +There is deliberately nothing else. The source citation comes from the module +docstring's `*Reference:*` line, because a copy kept here drifts from the one +the repository maintains — the copy this directory used to hold for +`Margulis.lean` had already lost the `v3` the docstring pins. An answer-slot +type Lean reports ambiguously is a `--answer-type` argument rather than a +field, since no problem currently needs one and a field nothing sets is a +format nobody checks. Run the problem-file check after moving or renaming a declaration: diff --git a/comparator/problems/erdos_1038_part_i.toml b/comparator/problems/erdos_1038_part_i.toml deleted file mode 100644 index 57d95bef01..0000000000 --- a/comparator/problems/erdos_1038_part_i.toml +++ /dev/null @@ -1,10 +0,0 @@ -# The answer type is inferred from the elaborated statement; this manifest -# remains as the worked example of `id` naming a workspace, and its `notes`. -id = "erdos_1038_part_i" -declaration = "erdos_1038.parts.i" -module = "FormalConjectures/ErdosProblems/1038.lean" -source = "https://www.erdosproblems.com/1038" -notes = """ -Asks for the infimum of `|{x : |f x| < 1}|` over nonconstant monic real -polynomials whose roots all lie in `[-1,1]`. -""" diff --git a/scripts/fc_leaneval_importer.py b/scripts/fc_leaneval_importer.py index 6f8f6a4258..144b798e7a 100644 --- a/scripts/fc_leaneval_importer.py +++ b/scripts/fc_leaneval_importer.py @@ -19,9 +19,8 @@ which generated module imports which. If a change to this file would do one of those, it belongs on the other side of the seam. -Two things the Lean source cannot settle live in `comparator/problems/.toml`, -one file per problem: an answer type Lean reports ambiguously, and which file -is meant when two declare the same name. See that directory's README. +One thing the Lean source cannot settle lives in `comparator/problems/.toml`, +one file per problem: which file is meant when two declare the same name. """ import json @@ -145,12 +144,11 @@ def file_scoped_preamble(lines, start_line): def load_manifest(problem_id): - """Read explicit choices that Lean source cannot select by itself. + """Read the one choice Lean source cannot select by itself. - An FC problem file selects the module when names collide. It may also - override an answer-slot type when Lean reports several types that cannot - be matched to source positions. The importer refuses both cases without an - explicit choice. + When two files declare the same name, nothing in the Lean environment says + which one was meant, so the importer refuses until a module is named. That + is the whole contract. `leanprover/lean-eval` keeps one TOML per problem, and the reason is worth copying: two pull requests adding different problems never touch the same @@ -159,9 +157,12 @@ def load_manifest(problem_id): id the filename stem, and the workspace directory name declaration the Lean name, which need not be unique across the repository module the file declaring it, relative to the repository root - answer_type the type of a non-`Prop` answer slot - notes free text for a reviewer - source a citation or URL + + Anything Formal Conjectures already states stays where it is stated. The + source citation is read from the module docstring rather than copied here, + because a copy can drift from the docstring the repository maintains. An + ambiguous answer-slot type is a `--answer-type` argument: it is rare, and + a field no problem uses is a format nobody can check. """ path = MANIFEST_DIR / f"{problem_id}.toml" if not path.exists(): @@ -178,6 +179,22 @@ def load_manifest(problem_id): return data +def docstring_reference(module_doc): + """The source citation Formal Conjectures already writes in the module. + + Module docstrings carry a `*Reference:*` line naming where the problem + comes from, sometimes with several links under it. The first is the + problem's own; later ones are commentary and proof notes. + """ + if not module_doc: + return "" + after = module_doc.split("*Reference:*", 1) + if len(after) != 2: + return "" + link = re.search(r"\]\((https?://[^)\s]+)\)", after[1]) + return link.group(1) if link else "" + + def manifest_ids(): return sorted(p.stem for p in MANIFEST_DIR.glob("*.toml")) @@ -230,18 +247,18 @@ def find_declaration(basename, module=None): def _read_source(path): - text = path.read_text(encoding="utf-8") - # Drop the license header; keep the module docstring; the rest is the body. - text = re.sub(r"\A/-.*?-/\s*", "", text, flags=re.DOTALL) - doc = "" - m = re.match(r"\s*(/-!.*?-/)\s*", text, flags=re.DOTALL) - if m: - doc = m.group(1) - text = text[m.end() :] - # Imports precede the docstring in source order; recover them from the original. - imports = re.findall( - r"^import\s+(\S+)", path.read_text(encoding="utf-8"), re.MULTILINE - ) + """Return (path, imports, module docstring, body after the licence header). + + The docstring is read but not removed. It sits below the imports rather + than at the top, so it is found by searching; the body deliberately still + contains it, because `strip_decorations` removes docstrings per + declaration and the generated files are compared byte for byte. + """ + original = path.read_text(encoding="utf-8") + text = re.sub(r"\A/-.*?-/\s*", "", original, flags=re.DOTALL) + found = re.search(r"/-!.*?-/", text, flags=re.DOTALL) + doc = found.group(0) if found else "" + imports = re.findall(r"^import\s+(\S+)", original, re.MULTILINE) return path, imports, doc, text @@ -716,9 +733,8 @@ def import_problem(problem, answer_type=None, module=None): declaration = problem_file.get("declaration", problem) # An argument given on the command line is explicit, so it wins over the # problem file; the file is the durable record of the same choice. - answer_type = answer_type or problem_file.get("answer_type") module = module or problem_file.get("module") - path, _imports, _module_doc, _body = find_declaration(declaration, module) + path, _imports, module_doc, _body = find_declaration(declaration, module) fc_module = module_name(path.relative_to(ROOT)) facts = elaborator_facts(fc_module, declaration) if facts["range"] is None: @@ -793,8 +809,7 @@ def import_problem(problem, answer_type=None, module=None): mathlib_rev, ), target=target_pins(), - source_url=str(problem_file.get("source", "")), - notes=str(problem_file.get("notes", "")), + source_url=docstring_reference(module_doc), ) return marked_up, manifest diff --git a/scripts/leaneval_generator.py b/scripts/leaneval_generator.py index 22ae29a988..ff3a49d1a0 100644 --- a/scripts/leaneval_generator.py +++ b/scripts/leaneval_generator.py @@ -90,7 +90,7 @@ def _readme(package, manifest): ) fields = "".join( f"- {label}: {' '.join(str(value).split())}\n" - for label, value in (("Source", manifest.source_url), ("Notes", manifest.notes)) + for label, value in (("Source", manifest.source_url),) if value ) return ( diff --git a/scripts/leaneval_interface.py b/scripts/leaneval_interface.py index 4aa4d1c1d2..8c74ed0117 100644 --- a/scripts/leaneval_interface.py +++ b/scripts/leaneval_interface.py @@ -152,7 +152,6 @@ class ProblemManifest: source: SourceRecord target: TargetRecord source_url: str = "" - notes: str = "" def __post_init__(self): for field in ("id", "theorem", "qualified_theorem"): @@ -188,8 +187,6 @@ def to_json_object(self): } if self.source_url: payload["source_url"] = self.source_url - if self.notes: - payload["notes"] = self.notes return payload @classmethod @@ -212,7 +209,6 @@ def from_json_object(cls, payload): source=SourceRecord(**source), target=TargetRecord(**payload["target"]), source_url=payload.get("source_url", ""), - notes=payload.get("notes", ""), ) def to_json(self): diff --git a/scripts/test_fc_leaneval_importer.py b/scripts/test_fc_leaneval_importer.py index a720242f50..10d5e9f020 100644 --- a/scripts/test_fc_leaneval_importer.py +++ b/scripts/test_fc_leaneval_importer.py @@ -398,3 +398,36 @@ def test_the_closure_region_does_not_carry_the_import(self): if __name__ == "__main__": unittest.main() + + +class DocstringReferenceTest(unittest.TestCase): + """The source citation is read from Formal Conjectures, not copied. + + A hand-kept copy drifts. The Margulis module's docstring pins + `arxiv/2504.17644v3`; the problem file that used to carry the same + citation had the unversioned URL, so the copy was already less exact + than the docstring it was copied from. + """ + + def test_the_first_reference_link_is_the_citation(self): + doc = ( + "/-!\n# Erdős Problem 1038\n\n*Reference:*\n" + " - [erdosproblems.com/1038](https://www.erdosproblems.com/1038)\n" + " - [Tao25] a blog post (https://example.com/other)\n-/" + ) + self.assertEqual( + importer.docstring_reference(doc), "https://www.erdosproblems.com/1038" + ) + + def test_an_arxiv_version_suffix_is_preserved(self): + doc = "/-!\n*Reference:* [arxiv/2504.17644v3](https://arxiv.org/abs/2504.17644v3)\n-/" + self.assertEqual( + importer.docstring_reference(doc), "https://arxiv.org/abs/2504.17644v3" + ) + + def test_links_above_the_reference_line_are_not_the_citation(self): + doc = "/-!\n# A problem\n\nSee [Mathlib](https://leanprover-community.github.io).\n-/" + self.assertEqual(importer.docstring_reference(doc), "") + + def test_a_module_without_a_docstring_has_no_citation(self): + self.assertEqual(importer.docstring_reference(""), "") diff --git a/scripts/test_leaneval_interface.py b/scripts/test_leaneval_interface.py index 13847bce76..b46e697434 100644 --- a/scripts/test_leaneval_interface.py +++ b/scripts/test_leaneval_interface.py @@ -74,7 +74,6 @@ def a_manifest(**overrides): "source": a_source(), "target": a_target(), "source_url": "https://www.erdosproblems.com/940", - "notes": "a reviewer note", } fields.update(overrides) return ProblemManifest(**fields) From 6406bea30808400c19c054c85ba3b21eea164a27 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:34:47 -0400 Subject: [PATCH 15/70] Let the consumer supply its own pins instead of asserting them here --- .github/workflows/build-and-docs.yml | 4 +++- comparator/OWNERSHIP.md | 2 +- scripts/fc_leaneval_importer.py | 1 - scripts/leaneval_generator.py | 12 +++++------ scripts/leaneval_interface.py | 25 ++++++++++++----------- scripts/make_comparator_workspace.py | 4 +++- scripts/test_leaneval_generator.py | 6 +++--- scripts/test_leaneval_interface.py | 16 +++++++-------- scripts/test_make_comparator_workspace.py | 6 +++--- 9 files changed, 39 insertions(+), 37 deletions(-) diff --git a/.github/workflows/build-and-docs.yml b/.github/workflows/build-and-docs.yml index f47c412e7c..8d49da7635 100644 --- a/.github/workflows/build-and-docs.yml +++ b/.github/workflows/build-and-docs.yml @@ -189,6 +189,7 @@ jobs: import sys sys.path.insert(0, "scripts") + import fc_leaneval_importer as importer import leaneval_generator as generator from leaneval_interface import MarkedUpModule, ProblemManifest @@ -203,7 +204,8 @@ jobs: assert manifest.source.declaration, "no FC declaration id" workspace = pathlib.Path(".comparator/erdos_1038_parts_i") - regenerated = generator.generate(module, manifest) + # The consumer's pins are its own; locally that is comparator/tools.toml. + regenerated = generator.generate(module, manifest, importer.target_pins()) for name, content in regenerated.items(): expected = (workspace / name).read_text(encoding="utf-8") assert content == expected, name diff --git a/comparator/OWNERSHIP.md b/comparator/OWNERSHIP.md index 6ceabdd9a3..2492e6ccd3 100644 --- a/comparator/OWNERSHIP.md +++ b/comparator/OWNERSHIP.md @@ -17,7 +17,7 @@ than a rewrite. This file says exactly what goes. scripts/fc_leaneval_importer.py FC declaration -> (module, manifest) scripts/leaneval_interface.py the two values, and nothing else - scripts/leaneval_generator.py (module, manifest) -> workspace files + scripts/leaneval_generator.py (module, manifest, pins) -> workspace files `scripts/make_comparator_workspace.py` is the command that runs one after the other. The arrow points one way: the generator imports the interface and never diff --git a/scripts/fc_leaneval_importer.py b/scripts/fc_leaneval_importer.py index 144b798e7a..a6e372473e 100644 --- a/scripts/fc_leaneval_importer.py +++ b/scripts/fc_leaneval_importer.py @@ -808,7 +808,6 @@ def import_problem(problem, answer_type=None, module=None): original, mathlib_rev, ), - target=target_pins(), source_url=docstring_reference(module_doc), ) return marked_up, manifest diff --git a/scripts/leaneval_generator.py b/scripts/leaneval_generator.py index ff3a49d1a0..94b0fa50f3 100644 --- a/scripts/leaneval_generator.py +++ b/scripts/leaneval_generator.py @@ -78,12 +78,12 @@ def lakefile(package, mathlib_rev): """ -def _readme(package, manifest): +def _readme(package, manifest, target): holes_line = ( "\nFill each definition hole in `Submission.lean` too. Hole answers " "also get a\nhuman check, because a hole can be gamed in ways the " "comparator cannot see.\nChecking holes needs a comparator built at " - f"commit `{manifest.target.comparator[:8]}`, which\nadded definition " + f"commit `{target.comparator[:8]}`, which\nadded definition " "support.\n" if manifest.holes else "" @@ -130,7 +130,7 @@ def _readme(package, manifest): ) -def generate(marked_up, manifest): +def generate(marked_up, manifest, target): """The workspace files for one problem, as a path-to-content mapping. Pure: it writes nothing, and it reads nothing but its two arguments and @@ -207,9 +207,9 @@ def generate(marked_up, manifest): # The workspace is built where it is going, not where it was made: # these are LeanEval's pins, and the manifest carries this repository's # beside them. - "lakefile.toml": lakefile(package, manifest.target.mathlib_revision), - "lean-toolchain": manifest.target.lean_toolchain + "\n", - "README.md": _readme(package, manifest), + "lakefile.toml": lakefile(package, target.mathlib_revision), + "lean-toolchain": target.lean_toolchain + "\n", + "README.md": _readme(package, manifest, target), "ChallengeDeps.lean": "import Mathlib\n\n" + marked_up.dependencies.strip("\n") + "\n", diff --git a/scripts/leaneval_interface.py b/scripts/leaneval_interface.py index 8c74ed0117..6332af254c 100644 --- a/scripts/leaneval_interface.py +++ b/scripts/leaneval_interface.py @@ -114,13 +114,19 @@ class SourceRecord: @dataclasses.dataclass(frozen=True) class TargetRecord: - """The pins the generated workspace is built and checked with. - - These are LeanEval's, not this repository's: a workspace is vendored into - lean-eval and built there. Formal Conjectures records them so that a - generated workspace is buildable where it is going rather than only where - it was made, and `comparator/tools.toml` is the one place they are - written down. + """The pins a generated workspace is built and checked with. + + These belong to whoever consumes the generator: lean-eval#536 says + LeanEval "remains the trusted statement repository and supplies the pin + regime and CI". So they are an argument to `generate`, not a field of the + manifest — a manifest carrying them would be Formal Conjectures asserting + another repository's regime, and would go stale the moment that repository + bumped anything, with nothing here to notice. + + Formal Conjectures keeps a copy under `[target]` in `comparator/tools.toml` + for one purpose: the CI job that generates at this repository's toolchain + and builds at LeanEval's, which is how the gap between the two is observed + rather than assumed. """ repository: str @@ -150,15 +156,12 @@ class ProblemManifest: holes: tuple permitted_axioms: tuple source: SourceRecord - target: TargetRecord source_url: str = "" def __post_init__(self): for field in ("id", "theorem", "qualified_theorem"): if not getattr(self, field): raise SystemExit(f"manifest has no {field}") - if not self.target.lean_toolchain or not self.target.mathlib_revision: - raise SystemExit(f"manifest {self.id} records no target pins") # lean-eval#536 names these two explicitly, and a manifest without # them cannot be traced back to a revision of this repository or # regenerated when FC fixes a misformalisation upstream. @@ -183,7 +186,6 @@ def to_json_object(self): **dataclasses.asdict(self.source), "copied_dependencies": list(self.source.copied_dependencies), }, - "target": dataclasses.asdict(self.target), } if self.source_url: payload["source_url"] = self.source_url @@ -207,7 +209,6 @@ def from_json_object(cls, payload): holes=tuple(DefinitionHole(**hole) for hole in payload["holes"]), permitted_axioms=tuple(payload["permitted_axioms"]), source=SourceRecord(**source), - target=TargetRecord(**payload["target"]), source_url=payload.get("source_url", ""), ) diff --git a/scripts/make_comparator_workspace.py b/scripts/make_comparator_workspace.py index 23ca6225a6..8780fd2010 100644 --- a/scripts/make_comparator_workspace.py +++ b/scripts/make_comparator_workspace.py @@ -148,7 +148,9 @@ def main(argv): if args.emit_import: print(emit_import(marked_up, manifest, args.emit_import)) return 0 - files = generator.generate(marked_up, manifest) + # The consumer supplies its own pins; see TargetRecord. Locally that is + # `[target]` in comparator/tools.toml, standing in for lean-eval's. + files = generator.generate(marked_up, manifest, importer.target_pins()) print(write_tree(pathlib.Path(args.out) / slug(manifest.id), files)) return 0 diff --git a/scripts/test_leaneval_generator.py b/scripts/test_leaneval_generator.py index af23e6a401..e86dc4b1ea 100644 --- a/scripts/test_leaneval_generator.py +++ b/scripts/test_leaneval_generator.py @@ -36,7 +36,7 @@ def a_workspace(**overrides): - return generate(A_MODULE, a_manifest(**overrides)) + return generate(A_MODULE, a_manifest(**overrides), a_target()) class SplitTest(unittest.TestCase): @@ -105,7 +105,7 @@ def test_the_lakefile_pins_the_target_mathlib(self): self.assertNotIn("formal-conjectures", lakefile) def test_the_package_name_is_an_identifier(self): - files = generate(A_MODULE, a_manifest(id="erdos_940.variants.large_integers")) + files = generate(A_MODULE, a_manifest(id="erdos_940.variants.large_integers"), a_target()) self.assertIn( 'name = "erdos_940_variants_large_integers"', files["lakefile.toml"] ) @@ -131,7 +131,7 @@ def test_the_workspace_carries_the_fc_commit_and_declaration(self): def test_the_manifest_is_passed_through_unaltered(self): manifest = a_manifest() - files = generate(A_MODULE, manifest) + files = generate(A_MODULE, manifest, a_target()) self.assertEqual(files["manifest.json"], manifest.to_json()) diff --git a/scripts/test_leaneval_interface.py b/scripts/test_leaneval_interface.py index b46e697434..896dd66b60 100644 --- a/scripts/test_leaneval_interface.py +++ b/scripts/test_leaneval_interface.py @@ -72,7 +72,6 @@ def a_manifest(**overrides): "holes": (DefinitionHole(name="erdos_940_answer", type="ENNReal"),), "permitted_axioms": ("propext", "Quot.sound", "Classical.choice"), "source": a_source(), - "target": a_target(), "source_url": "https://www.erdosproblems.com/940", } fields.update(overrides) @@ -96,17 +95,16 @@ def test_source_commit_and_declaration_are_required(self): with self.assertRaisesRegex(SystemExit, "no FC declaration id"): a_manifest(source=a_source(declaration="")) - def test_target_pins_are_required(self): - # A workspace without them cannot be built where it is going. - with self.assertRaisesRegex(SystemExit, "no target pins"): - a_manifest(target=a_target(lean_toolchain="")) + def test_the_manifest_does_not_carry_the_consumers_pins(self): + # lean-eval#536 gives LeanEval the pin regime. A manifest asserting it + # would go stale when LeanEval bumped, with nothing here to notice. + self.assertNotIn("target", a_manifest().to_json_object()) - def test_both_pin_sets_are_recorded(self): - # The hole types were read at the source pins and will be used at the - # target pins. A manifest that carried only one could not say so. + def test_the_pins_the_hole_types_were_read_at_are_recorded(self): + # The consumer supplies its own pins, but it cannot know where these + # hole types were read unless the manifest says so. payload = a_manifest().to_json_object() self.assertEqual(payload["source"]["lean_toolchain"], "leanprover/lean4:v4.27.0") - self.assertEqual(payload["target"]["lean_toolchain"], "leanprover/lean4:v4.33.0") def test_the_manifest_survives_a_round_trip(self): manifest = a_manifest() diff --git a/scripts/test_make_comparator_workspace.py b/scripts/test_make_comparator_workspace.py index 11303ad55c..077e3de9bd 100644 --- a/scripts/test_make_comparator_workspace.py +++ b/scripts/test_make_comparator_workspace.py @@ -28,7 +28,7 @@ import leaneval_generator as generator from leaneval_interface import MarkedUpModule, ProblemManifest from make_comparator_workspace import emit_import, write_tree -from test_leaneval_interface import A_MODULE, a_manifest +from test_leaneval_interface import a_target, A_MODULE, a_manifest class EmitImportTest(unittest.TestCase): @@ -51,8 +51,8 @@ def test_the_emitted_pair_rebuilds_the_workspace_exactly(self): (out / "manifest.json").read_text(encoding="utf-8") ) self.assertEqual( - generator.generate(module, read_back), - generator.generate(A_MODULE, manifest), + generator.generate(module, read_back, a_target()), + generator.generate(A_MODULE, manifest, a_target()), ) def test_an_existing_directory_is_not_overwritten(self): From 824f86b3f952ab03e9aa608c93c137ba8ab3fe8d Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Wed, 19 Aug 2026 22:14:18 -0400 Subject: [PATCH 16/70] Drop the checkout credentials the Comparator job never uses --- .github/workflows/comparator-lean-4-33.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/comparator-lean-4-33.yml b/.github/workflows/comparator-lean-4-33.yml index 1cf1108c2d..552140359c 100644 --- a/.github/workflows/comparator-lean-4-33.yml +++ b/.github/workflows/comparator-lean-4-33.yml @@ -41,6 +41,10 @@ jobs: # The importer pins each workspace to the merge base with upstream # main, so that ref has to be present. fetch-depth: 0 + # This job only reads the tree and builds; nothing here pushes, so + # the checkout token has no reason to stay in .git/config where a + # later step or a build script could reach it. + persist-credentials: false - name: Install elan run: | From a101a9c72d2abec5f62f4fcd93b2c0b65c6cec5e Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Wed, 19 Aug 2026 22:17:38 -0400 Subject: [PATCH 17/70] Keep the generated-workspace job clean under the Actions scan --- .github/workflows/comparator-lean-4-33.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/comparator-lean-4-33.yml b/.github/workflows/comparator-lean-4-33.yml index 552140359c..6c24ad9122 100644 --- a/.github/workflows/comparator-lean-4-33.yml +++ b/.github/workflows/comparator-lean-4-33.yml @@ -14,6 +14,12 @@ name: Generated workspace at LeanEval pins # is built and checked at LeanEval's Lean 4.33. If those two disagree about a # statement or a slot type, this job is where it shows. +# Two Mathlib builds make this the most expensive job in the repository, so a +# new push supersedes the run it replaces rather than queueing behind it. +concurrency: + group: comparator-lean-4-33-${{ github.ref }} + cancel-in-progress: true + on: pull_request: paths: @@ -76,6 +82,11 @@ jobs: # substitute for the 4.33 build below; it is what keeps an FC-side # copying defect from being reported as a LeanEval build failure. - name: Import and generate two workspaces + # The target toolchain arrives as an environment variable rather than a + # `${{ }}` expansion inside the script, so the shell never has workflow + # syntax substituted into it before it runs. + env: + TARGET_TOOLCHAIN: ${{ steps.target.outputs.lean_toolchain }} run: | for d in isSumOfThreeCubes_2 isSumOfThreeCubes_iff_mod_9; do python3 scripts/make_comparator_workspace.py "$d" \ @@ -85,7 +96,7 @@ jobs: grep -q "isSumOfThreeCubes_iff_mod_9_answer : Prop" \ .comparator/isSumOfThreeCubes_iff_mod_9/Challenge.lean # Generated for LeanEval, not for here. - grep -q "${{ steps.target.outputs.lean_toolchain }}" \ + grep -q "$TARGET_TOOLCHAIN" \ .comparator/isSumOfThreeCubes_2/lean-toolchain - name: Build both workspaces at Lean 4.33 From acab94d9c871dfb64efa4788c380d319068cf817 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:15:37 -0400 Subject: [PATCH 18/70] Decode guillemet module components without splitting their dots --- scripts/fc_leaneval_importer.py | 24 +++++++++++++++---- scripts/test_fc_leaneval_importer.py | 35 ++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/scripts/fc_leaneval_importer.py b/scripts/fc_leaneval_importer.py index a6e372473e..19eeab9cb7 100644 --- a/scripts/fc_leaneval_importer.py +++ b/scripts/fc_leaneval_importer.py @@ -320,13 +320,27 @@ def replace(match): return re.sub(r"^[ \t]*\n", "", text, flags=re.MULTILINE) +def split_module(module): + """The components of a dotted module name, respecting guillemet quoting. + + A guillemet-quoted component may itself contain dots — + `FormalConjectures.Arxiv.«0912.2382».CurlingNumberConjecture` names the + directory `0912.2382` — so splitting on every dot decodes a path that + does not exist. This is the one place a module name is taken apart; + `module_name` is its inverse and a test holds the pair to that. + """ + parts = re.findall(r"«[^»]*»|[^.«»]+", module) + if ".".join(parts) != module: + raise SystemExit(f"{module!r} is not a well-formed module name") + return [p[1:-1] if p.startswith("«") else p for p in parts] + + def module_source_path(module): """The file declaring a dotted Lean module name, undoing guillemets.""" - parts = [ - component[1:-1] if component.startswith("«") else component - for component in module.split(".") - ] - path = ROOT.joinpath(*parts).with_suffix(".lean") + parts = split_module(module) + # Not `with_suffix`: a final component containing a dot would lose its + # tail to the suffix replacement. + path = ROOT.joinpath(*parts[:-1], parts[-1] + ".lean") if not path.is_file(): raise SystemExit(f"{module}: no source file at {path}") return path diff --git a/scripts/test_fc_leaneval_importer.py b/scripts/test_fc_leaneval_importer.py index 10d5e9f020..e95fc922d5 100644 --- a/scripts/test_fc_leaneval_importer.py +++ b/scripts/test_fc_leaneval_importer.py @@ -431,3 +431,38 @@ def test_links_above_the_reference_line_are_not_the_citation(self): def test_a_module_without_a_docstring_has_no_citation(self): self.assertEqual(importer.docstring_reference(""), "") + + +class ModuleNameCodecTest(unittest.TestCase): + """`module_name` and `module_source_path` are inverse on real modules.""" + + def test_a_guillemet_component_keeps_its_dots(self): + self.assertEqual( + importer.split_module( + "FormalConjectures.Arxiv.«0912.2382».CurlingNumberConjecture" + ), + ["FormalConjectures", "Arxiv", "0912.2382", "CurlingNumberConjecture"], + ) + + def test_a_dotted_final_component_keeps_its_tail(self): + # `with_suffix` would have turned `«2501.03234»` into `«2501.lean`. + path = importer.module_source_path( + "FormalConjectures.Arxiv.«2501.03234».ArithmeticSumS" + ) + self.assertEqual(path.name, "ArithmeticSumS.lean") + self.assertEqual(path.parent.name, "2501.03234") + + def test_a_malformed_name_is_refused(self): + with self.assertRaises(SystemExit): + importer.split_module("FormalConjectures.«unterminated") + + def test_every_real_module_round_trips(self): + # The property that keeps the codec from drifting again: for every + # file the importer can name, decoding the name reaches the file. + for src in importer.SOURCE_DIRS: + for path in src.rglob("*.lean"): + rel = path.relative_to(importer.ROOT) + with self.subTest(module=str(rel)): + self.assertEqual( + importer.module_source_path(importer.module_name(rel)), path + ) From 17dba4472787ad04bfd87b9e66b11a5f2b6b2368 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:16:55 -0400 Subject: [PATCH 19/70] Default workspace ids to the qualified name and resolve qualified requests --- scripts/fc_leaneval_importer.py | 67 +++++++++++++++++++++++----- scripts/test_fc_leaneval_importer.py | 33 ++++++++++++++ 2 files changed, 90 insertions(+), 10 deletions(-) diff --git a/scripts/fc_leaneval_importer.py b/scripts/fc_leaneval_importer.py index 19eeab9cb7..3e276687eb 100644 --- a/scripts/fc_leaneval_importer.py +++ b/scripts/fc_leaneval_importer.py @@ -213,9 +213,42 @@ def module_name(rel_path): return ".".join(parts) +def _declaring_files(name): + """The files whose text declares `name` as a theorem or lemma.""" + pattern = re.compile( + rf"(?:theorem|lemma)\s+(?:[\w.«»]*\.)?{re.escape(name)}[\s:]" + ) + hits = [] + for src in SOURCE_DIRS: + for path in sorted(src.rglob("*.lean")): + if pattern.search(path.read_text(encoding="utf-8")): + hits.append(path) + return hits + + +def _declares_namespaces(text, components): + """True if the file opens namespaces spelling out `components` in order. + + A single `namespace A.B` line declares both at once, so the check is on + the concatenated stack, not line by line. Text-level and approximate on + purpose — the elaborated environment settles the truth later; this only + ranks candidate files. + """ + stack = [] + for line in text.split("\n"): + m = re.match(r"\s*namespace\s+([\w.«»]+)", line) + if m: + stack.extend(m.group(1).split(".")) + return any( + stack[i : i + len(components)] == list(components) + for i in range(len(stack) - len(components) + 1) + ) + + def find_declaration(basename, module=None): """Locate the file declaring `basename`. Returns (path, imports, doc, body). + A fully qualified name resolves through the enclosing `namespace` stack; `module` names the file when more than one declares the name, and comes from the problem's FC problem file. """ @@ -224,14 +257,25 @@ def find_declaration(basename, module=None): if not named.exists(): raise SystemExit(f"manifest names {module}, which does not exist") return _read_source(named) - hits = [] - for src in SOURCE_DIRS: - for path in sorted(src.rglob("*.lean")): - text = path.read_text(encoding="utf-8") - if re.search( - rf"(?:theorem|lemma)\s+(?:[\w.«»]*\.)?{re.escape(basename)}[\s:]", text - ): - hits.append(path) + hits = _declaring_files(basename) + if not hits and "." in basename: + # A fully qualified request such as `OeisA303656.conjecture` names a + # declaration whose file spells only `conjecture`, the prefix coming + # from an enclosing `namespace`. Try each split of the request into + # (namespace prefix, declared suffix), keeping files that declare the + # suffix inside that namespace. Splits are tried longest-suffix first, + # because a declared name may itself contain dots + # (`erdos_125.variants.positive_unequal_density`). + parts = basename.split(".") + for cut in range(1, len(parts)): + prefix, suffix = parts[:cut], ".".join(parts[cut:]) + hits = [ + path + for path in _declaring_files(suffix) + if _declares_namespaces(path.read_text(encoding="utf-8"), prefix) + ] + if hits: + break if not hits: raise SystemExit( f"no declaration named {basename!r} found under FormalConjectures/" @@ -806,10 +850,13 @@ def import_problem(problem, answer_type=None, module=None): holes="\n\n".join(hole.declaration() for hole in holes), statement=statement, ) + qualified = ".".join(namespaces_at_target + [declared]) manifest = ProblemManifest( - id=problem_file.get("id", declared), + # The default id is the qualified name: two modules declaring + # `conjecture` in different namespaces must not share a workspace. + id=problem_file.get("id", qualified), theorem=declared, - qualified_theorem=".".join(namespaces_at_target + [declared]), + qualified_theorem=qualified, apply_arguments=tuple(args), holes=tuple(holes), permitted_axioms=PERMITTED_AXIOMS, diff --git a/scripts/test_fc_leaneval_importer.py b/scripts/test_fc_leaneval_importer.py index e95fc922d5..fb0171f225 100644 --- a/scripts/test_fc_leaneval_importer.py +++ b/scripts/test_fc_leaneval_importer.py @@ -466,3 +466,36 @@ def test_every_real_module_round_trips(self): self.assertEqual( importer.module_source_path(importer.module_name(rel)), path ) + + +class QualifiedResolutionTest(unittest.TestCase): + """Qualified requests resolve through the namespace stack.""" + + def test_the_bare_colliding_name_is_ambiguous(self): + with self.assertRaises(SystemExit) as ctx: + importer.find_declaration("conjecture") + self.assertIn("ambiguous", str(ctx.exception)) + + def test_each_qualified_name_reaches_its_own_file(self): + for qualified, filename in ( + ("OeisA303656.conjecture", "303656.lean"), + ("OeisA308734.conjecture", "308734.lean"), + ): + with self.subTest(qualified=qualified): + path, _, _, _ = importer.find_declaration(qualified) + self.assertEqual(path.name, filename) + + def test_a_declared_name_with_dots_still_resolves(self): + # The declared name itself contains dots; no namespace split applies. + path, _, _, _ = importer.find_declaration( + "erdos_125.variants.positive_unequal_density" + ) + self.assertEqual(path.name, "125.lean") + + def test_longest_declared_suffix_wins(self): + # `Erdos125.erdos_125.variants.positive_unequal_density`: the first + # component is the namespace, the rest is the declared name. + path, _, _, _ = importer.find_declaration( + "Erdos125.erdos_125.variants.positive_unequal_density" + ) + self.assertEqual(path.name, "125.lean") From 85556c607e5bb31fee8d166185b1c461354a162d Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:19:42 -0400 Subject: [PATCH 20/70] Read the category tag from the environment and map it to a lean-eval group --- scripts/comparator_facts.lean | 15 ++++++++++++++ scripts/fc_leaneval_importer.py | 26 ++++++++++++++++++++++++ scripts/leaneval_interface.py | 8 ++++++++ scripts/test_fc_leaneval_importer.py | 30 ++++++++++++++++++++++++++++ scripts/test_leaneval_interface.py | 1 + 5 files changed, 80 insertions(+) diff --git a/scripts/comparator_facts.lean b/scripts/comparator_facts.lean index 0dc6501b07..10811218bb 100644 --- a/scripts/comparator_facts.lean +++ b/scripts/comparator_facts.lean @@ -185,9 +185,24 @@ where ("module", toJson (moduleOf env d)), ("range", rangeToJson (some r))] | none => generated := generated.push (toJson d.toString) + -- The `@[category ...]` tag, spelled the way the attribute is written. + -- lean-eval displays open conjectures apart from its evaluation set, so + -- the importer needs to know which of the two a declaration is; the tag + -- lives in the environment extension, not in anything the text layer + -- could read reliably. + let category := match (ProblemAttributes.categoryExt.getState env).toList.find? + (·.declName == name) with + | some tag => Json.str <| match tag.category with + | .research .open => "research open" + | .research .solved => "research solved" + | .textbook => "textbook" + | .test => "test" + | .API => "API" + | none => Json.null let payload := Json.mkObj [ ("declaration", toJson decl), ("name", toJson name.toString), + ("category", category), ("range", rangeJson), ("binders", toJson binders.toList), ("answerTypes", toJson answerTypes.toList), diff --git a/scripts/fc_leaneval_importer.py b/scripts/fc_leaneval_importer.py index 3e276687eb..61a3c8eefa 100644 --- a/scripts/fc_leaneval_importer.py +++ b/scripts/fc_leaneval_importer.py @@ -870,10 +870,36 @@ def import_problem(problem, answer_type=None, module=None): mathlib_rev, ), source_url=docstring_reference(module_doc), + category=facts.get("category") or "", ) return marked_up, manifest +# The `@[category ...]` tags that name a problem, and the lean-eval group +# each belongs to. `research open` is the point of the FC import and goes to +# the open-conjectures display; everything already settled — solved research, +# textbook and test statements — is evaluation material. `API` declarations +# and untagged ones are not problems and are refused. +CATEGORY_GROUPS = { + "research open": "open-conjectures", + "research solved": "formalization-evaluation", + "textbook": "formalization-evaluation", + "test": "formalization-evaluation", +} + + +def problem_group(manifest): + """The lean-eval problem group for an imported declaration's category.""" + group = CATEGORY_GROUPS.get(manifest.category) + if group is None: + raise SystemExit( + f"{manifest.id}: category {manifest.category!r} is not a " + "problem category; expected one of " + + ", ".join(sorted(CATEGORY_GROUPS)) + ) + return group + + def elaborate(marked_up): """Elaborate the marked-up module against this checkout's Mathlib. diff --git a/scripts/leaneval_interface.py b/scripts/leaneval_interface.py index 6332af254c..3c682d18f9 100644 --- a/scripts/leaneval_interface.py +++ b/scripts/leaneval_interface.py @@ -157,6 +157,12 @@ class ProblemManifest: permitted_axioms: tuple source: SourceRecord source_url: str = "" + # The `@[category ...]` tag as the source spells it: `research open`, + # `research solved`, `textbook` or `test`. lean-eval keeps open + # conjectures out of its evaluation set, so which group a problem joins + # is decided by this and nothing else; recording the raw tag rather than + # the mapped group keeps the mapping in one place, beside the request. + category: str = "" def __post_init__(self): for field in ("id", "theorem", "qualified_theorem"): @@ -179,6 +185,7 @@ def to_json_object(self): "id": self.id, "theorem": self.theorem, "qualified_theorem": self.qualified_theorem, + "category": self.category, "apply_arguments": list(self.apply_arguments), "holes": [dataclasses.asdict(hole) for hole in self.holes], "permitted_axioms": list(self.permitted_axioms), @@ -210,6 +217,7 @@ def from_json_object(cls, payload): permitted_axioms=tuple(payload["permitted_axioms"]), source=SourceRecord(**source), source_url=payload.get("source_url", ""), + category=payload.get("category", ""), ) def to_json(self): diff --git a/scripts/test_fc_leaneval_importer.py b/scripts/test_fc_leaneval_importer.py index fb0171f225..50b2b14f1e 100644 --- a/scripts/test_fc_leaneval_importer.py +++ b/scripts/test_fc_leaneval_importer.py @@ -499,3 +499,33 @@ def test_longest_declared_suffix_wins(self): "Erdos125.erdos_125.variants.positive_unequal_density" ) self.assertEqual(path.name, "125.lean") + + +class ProblemGroupTest(unittest.TestCase): + """Categories map to lean-eval groups; non-problems are refused.""" + + def _manifest(self, category): + manifest = mock.Mock() + manifest.category = category + manifest.id = "some_problem" + return manifest + + def test_open_research_is_an_open_conjecture(self): + self.assertEqual( + importer.problem_group(self._manifest("research open")), + "open-conjectures", + ) + + def test_settled_statements_are_evaluation_material(self): + for category in ("research solved", "textbook", "test"): + with self.subTest(category=category): + self.assertEqual( + importer.problem_group(self._manifest(category)), + "formalization-evaluation", + ) + + def test_api_and_untagged_declarations_are_refused(self): + for category in ("API", ""): + with self.subTest(category=category): + with self.assertRaises(SystemExit): + importer.problem_group(self._manifest(category)) diff --git a/scripts/test_leaneval_interface.py b/scripts/test_leaneval_interface.py index 896dd66b60..935cc23bd1 100644 --- a/scripts/test_leaneval_interface.py +++ b/scripts/test_leaneval_interface.py @@ -73,6 +73,7 @@ def a_manifest(**overrides): "permitted_axioms": ("propext", "Quot.sound", "Classical.choice"), "source": a_source(), "source_url": "https://www.erdosproblems.com/940", + "category": "research open", } fields.update(overrides) return ProblemManifest(**fields) From 906e30ebb59cd98c55ee04f2890da7557ad2a35c Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:30:34 -0400 Subject: [PATCH 21/70] Consume the pinned lean-eval-generator through its v1 JSON contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The seam is no longer a Python import waiting to be replaced: it is the versioned CLI contract leanprover/lean-eval-generator froze at the revision comparator/tools.toml now pins. The interface builds the v1 request — module content, resolved hole ranges, target pins, template — and checks every digest in the response; the placeholder generator and its tests are deleted rather than extracted. Provenance travels beside the request as fc-provenance.json, because the v1 wire format has no field for it. --- comparator/tools.toml | 8 + scripts/fc_leaneval_importer.py | 35 +-- scripts/leaneval_generator.py | 228 -------------- scripts/leaneval_generator_cli.py | 95 ++++++ scripts/leaneval_interface.py | 357 ++++++++++++++++++---- scripts/make_comparator_workspace.py | 111 +++++-- scripts/test_fc_leaneval_importer.py | 2 +- scripts/test_leaneval_generator.py | 163 ---------- scripts/test_leaneval_interface.py | 187 +++++++++--- scripts/test_make_comparator_workspace.py | 188 ++++++++---- 10 files changed, 763 insertions(+), 611 deletions(-) delete mode 100644 scripts/leaneval_generator.py create mode 100644 scripts/leaneval_generator_cli.py delete mode 100644 scripts/test_leaneval_generator.py diff --git a/comparator/tools.toml b/comparator/tools.toml index 838e7add4b..cb8184aba5 100644 --- a/comparator/tools.toml +++ b/comparator/tools.toml @@ -21,3 +21,11 @@ lean_toolchain = "leanprover/lean4:v4.33.0" mathlib_revision = "6f1ef4e5dd604a435bddba4747b13970cd65d2a1" comparator = "c0c5a52d2aff92b457c3e5ed4a68c1ebc5795809" lean4export = "15f6055e299ad5b89345e533cc2192f4cc00f659" + +# The extracted generator this repository's requests are written against. +# `schemas/request-v1.schema.json` and `response-v1.schema.json` at this +# revision are the normative wire format; a bump here is a contract change +# and has to survive the seam round-trip test. +[generator] +repository = "https://github.com/leanprover/lean-eval-generator" +rev = "a726789593eeac5c32ad82760061cd5bf6cae662" diff --git a/scripts/fc_leaneval_importer.py b/scripts/fc_leaneval_importer.py index 61a3c8eefa..4ad10864d2 100644 --- a/scripts/fc_leaneval_importer.py +++ b/scripts/fc_leaneval_importer.py @@ -37,6 +37,7 @@ ProblemManifest, SourceRecord, TargetRecord, + problem_group, ) ROOT = pathlib.Path(__file__).resolve().parent.parent @@ -490,14 +491,16 @@ def covered_by_another(dep): chunk.append(f"end {namespace}") chunk.append("end") blocks.append("\n".join(chunk)) - provenance.append(dep["name"]) + provenance.append((dep["name"], body)) # The statement reopens the namespace stack the target sat in, so it can # name siblings short. `open` on a namespace nothing has declared is an # error, and with the problem's module no longer imported only the copied # declarations can declare one. An empty namespace block is enough to make # the name exist. - declared_namespaces = {name.rsplit(".", 1)[0] for name in provenance if "." in name} + declared_namespaces = { + name.rsplit(".", 1)[0] for name, _ in provenance if "." in name + } for depth in range(len(opened_namespaces)): prefix = ".".join(opened_namespaces[: depth + 1]) if not any( @@ -505,7 +508,7 @@ def covered_by_another(dep): ): blocks.append(f"namespace {prefix}\nend {prefix}") - listing = "\n".join(f"* `{name}`" for name in provenance) + listing = "\n".join(f"* `{name}`" for name, _ in provenance) return ( "/-!\n" f"The Formal Conjectures declarations `{declaration}` needs, copied so\n" @@ -849,6 +852,7 @@ def import_problem(problem, answer_type=None, module=None): scope="\n".join(opens + preamble), holes="\n\n".join(hole.declaration() for hole in holes), statement=statement, + dependency_declarations=tuple(copied), ) qualified = ".".join(namespaces_at_target + [declared]) manifest = ProblemManifest( @@ -875,31 +879,6 @@ def import_problem(problem, answer_type=None, module=None): return marked_up, manifest -# The `@[category ...]` tags that name a problem, and the lean-eval group -# each belongs to. `research open` is the point of the FC import and goes to -# the open-conjectures display; everything already settled — solved research, -# textbook and test statements — is evaluation material. `API` declarations -# and untagged ones are not problems and are refused. -CATEGORY_GROUPS = { - "research open": "open-conjectures", - "research solved": "formalization-evaluation", - "textbook": "formalization-evaluation", - "test": "formalization-evaluation", -} - - -def problem_group(manifest): - """The lean-eval problem group for an imported declaration's category.""" - group = CATEGORY_GROUPS.get(manifest.category) - if group is None: - raise SystemExit( - f"{manifest.id}: category {manifest.category!r} is not a " - "problem category; expected one of " - + ", ".join(sorted(CATEGORY_GROUPS)) - ) - return group - - def elaborate(marked_up): """Elaborate the marked-up module against this checkout's Mathlib. diff --git a/scripts/leaneval_generator.py b/scripts/leaneval_generator.py deleted file mode 100644 index 94b0fa50f3..0000000000 --- a/scripts/leaneval_generator.py +++ /dev/null @@ -1,228 +0,0 @@ -#!/usr/bin/env python3 -"""Turn a marked-up module and a manifest into a Challenge/Solution workspace. - -**This file is the placeholder for a dependency, and it is meant to be -deleted.** `leanprover/lean-eval#536` extracts lean-eval's generator core into -`leanprover/lean-eval-generator`, consumed as a pinned dependency by lean-eval -and by this importer, and says in as many words that the Formal Conjectures -importer does not fork the generation logic. Until that repository exists there -is nothing to pin, so this module stands in for it, deliberately holding -everything that is not Formal Conjectures' to own: - -- the workspace layout and every file emitted into it; -- which generated module imports which, and where the scope directives are - restated so the same statement text elaborates in all three files; -- the lakefile, the toolchain file and the Mathlib requirement; -- the fixed Solution adapter that pins the statement; -- the Comparator `config.json` shape. - -It reads nothing from this repository except the marked-up module, the -manifest, and the workspace test template it copies. It never resolves a -declaration, reads Lean source, or runs Lean. When `lean-eval-generator` lands, -this file is deleted, `generate` becomes a call into the pinned package, and -`scripts/fc_leaneval_importer.py` does not change. - -See `comparator/OWNERSHIP.md` for the line counts either side of that deletion. -""" - -import json -import pathlib - -from leaneval_interface import slug - -ROOT = pathlib.Path(__file__).resolve().parent.parent -# The workspace test template lean-eval's generator supplies for its own -# workspaces; it is vendored here only while this module stands in for it. -TEMPLATE_DIR = ROOT / "comparator" / "templates" - -PROOF_SUFFIX = ":= by\n sorry" - - -def lakefile(package, mathlib_rev): - """Mathlib at the target revision, and nothing else. - - The workspace used to require Formal Conjectures too, so that the - Challenge could import the problem's module. lean-eval vendors its - problems and cannot fetch that repository at evaluation time, so the - closure travels in `ChallengeDeps.lean` instead and the require is gone. - The commit the copy came from is recorded in `manifest.json`, which is - where a reader should look for it. - """ - return f"""name = "{package}" -testDriver = "workspace_test" -defaultTargets = ["ChallengeDeps", "Challenge", "Solution", "Submission"] - -[leanOptions] -autoImplicit = false - -[[require]] -name = "mathlib" -git = "https://github.com/leanprover-community/mathlib4.git" -rev = "{mathlib_rev}" - -[[lean_lib]] -name = "ChallengeDeps" - -[[lean_lib]] -name = "Challenge" - -[[lean_lib]] -name = "Solution" - -[[lean_lib]] -name = "Submission" - -[[lean_exe]] -name = "workspace_test" -root = "WorkspaceTest" -""" - - -def _readme(package, manifest, target): - holes_line = ( - "\nFill each definition hole in `Submission.lean` too. Hole answers " - "also get a\nhuman check, because a hole can be gamed in ways the " - "comparator cannot see.\nChecking holes needs a comparator built at " - f"commit `{target.comparator[:8]}`, which\nadded definition " - "support.\n" - if manifest.holes - else "" - ) - fields = "".join( - f"- {label}: {' '.join(str(value).split())}\n" - for label, value in (("Source", manifest.source_url),) - if value - ) - return ( - f"# {package}\n\n" - f"A comparator challenge for `{manifest.theorem}`, generated from\n" - f"`{manifest.source.path}` in google-deepmind/formal-conjectures.\n\n" - + fields - + "\nProve the statement in `Submission.lean`, keeping it as it stands; " - "put helper\nmodules under `Submission/` if you need them. Do not " - "modify `Challenge.lean` or\n`Solution.lean`: the trusted statement " - "lives there, and `Solution.lean` closes it\nwith your `Submission` " - "theorem, so it fails to compile if the submission proves\nanything " - "else.\n" - "\nComparator accepts the workspace only if the statement is proved " - "under the\naxioms in `config.json`. `sorry` adds `sorryAx`, which is " - "not permitted, and\nclosing the goal with the imported original " - "fails the same way, since that is\n`sorry` too. `lake test` runs " - "comparator, from `PATH` or `COMPARATOR_BIN`.\n" - "\nIf comparator fails with `incompatible header` on an `.olean`, the " - "mismatch is\nbetween this workspace's toolchain and the one " - "`lean4export` was built with,\nnever a problem with the proof: copy " - "this workspace's `lean-toolchain` into\nyour `lean4export` checkout, " - "rebuild it, and clear `.lake/build` here.\n" - + holes_line - + "\nFetch the Mathlib cache before the first build; a cold build takes " - "the best\npart of an hour without it:\n\n" - " lake exe cache get\n" - " lake build\n" - ) - - -HELPERS = ( - "import Mathlib\n\n" - "/-! Helper lemmas for the submission go here, or in further modules\n" - "under `Submission/`, each imported from `Submission.lean`. -/\n\n" - "namespace Submission\n\nend Submission\n" -) - - -def generate(marked_up, manifest, target): - """The workspace files for one problem, as a path-to-content mapping. - - Pure: it writes nothing, and it reads nothing but its two arguments and - the workspace test template. Putting the result on disk is the caller's. - - The three Lean files carry the same statement text, so what that text - needs to elaborate has to be restated in each: that is what the module's - `scope` region is for, and placing it is this side's job. `ChallengeDeps` - takes the module's dependency region and the `import Mathlib` that makes - the workspace stand on Mathlib alone; the other three import it. - """ - package = slug(manifest.id) - header = marked_up.scope.strip("\n") - header = header + "\n\n" if header else "" - holes = marked_up.holes.strip("\n") - holes = holes + "\n\n" if holes else "" - statement = marked_up.statement.strip("\n") - - signature = statement.rstrip() - if signature.endswith(PROOF_SUFFIX): - signature = signature[: -len(PROOF_SUFFIX)].rstrip() - - challenge = "import ChallengeDeps\n\n" + header + holes + statement + "\n" - - # The participant's file. The statement sits inside `namespace Submission` - # so nothing here can collide with, or stand in for, the trusted names. - submission = ( - "import ChallengeDeps\nimport Submission.Helpers\n\n" - + header - + "namespace Submission\n\n" - + holes - + statement - + "\n\nend Submission\n" - ) - - # The fixed adapter, lean-eval's shape: it restates the trusted statement - # and closes it with the Submission theorem, so it fails to compile the - # moment the submission proves anything else. The participant never edits - # it, which is what keeps the statement pinned. - # `@[reducible]`: the Challenge's statement mentions the Solution's copy of - # the hole and the Submission theorem's type mentions the participant's, so - # the adapter only typechecks if the unifier unfolds one into the other. - # Default transparency does, but the one workspace known to have built at - # LeanEval's toolchain marks it reducible, and there is no reason to be the - # first to find out whether that mattered. - delegated = "".join( - f"@[reducible] noncomputable def {hole.name} : {hole.type} :=" - f"\n Submission.{hole.name}\n\n" - for hole in manifest.holes - ) - solution = ( - "import ChallengeDeps\nimport Submission\n\n" - + header - + delegated - + signature - + " :=\n Submission." - + manifest.theorem - + "".join(" " + argument for argument in manifest.apply_arguments) - + "\n" - ) - - config = { - "challenge_module": "Challenge", - "solution_module": "Solution", - "theorem_names": [manifest.theorem], - "permitted_axioms": list(manifest.permitted_axioms), - "enable_nanoda": False, - } - if manifest.holes: - # Comparator's documented no-hole config carries no such field. - config["definition_names"] = manifest.hole_names() - - return { - # The workspace is built where it is going, not where it was made: - # these are LeanEval's pins, and the manifest carries this repository's - # beside them. - "lakefile.toml": lakefile(package, target.mathlib_revision), - "lean-toolchain": target.lean_toolchain + "\n", - "README.md": _readme(package, manifest, target), - "ChallengeDeps.lean": "import Mathlib\n\n" - + marked_up.dependencies.strip("\n") - + "\n", - "Challenge.lean": challenge, - "Solution.lean": solution, - "Submission.lean": submission, - "Submission/Helpers.lean": HELPERS, - "WorkspaceTest.lean": (TEMPLATE_DIR / "WorkspaceTest.lean").read_text( - encoding="utf-8" - ), - "config.json": json.dumps(config, indent=2) + "\n", - # The manifest crosses into the workspace unaltered. lean-eval#536 - # requires it to record the FC source commit and declaration id, and - # this side neither supplies nor edits those. - "manifest.json": manifest.to_json(), - } diff --git a/scripts/leaneval_generator_cli.py b/scripts/leaneval_generator_cli.py new file mode 100644 index 0000000000..b5910c8536 --- /dev/null +++ b/scripts/leaneval_generator_cli.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Run the pinned `lean-eval-generator` binary on a v1 request. + +`leanprover/lean-eval#536` extracts lean-eval's generator core into +`leanprover/lean-eval-generator`, a deterministic Lean CLI: one JSON request +on stdin, one JSON response on stdout, diagnostics on stderr. This module is +the plumbing that runs it and the context directory it expects; everything +the request and response mean lives in `scripts/leaneval_interface.py`, and +the pinned revision lives in `comparator/tools.toml` under `[generator]`. + +The binary is found through `LEAN_EVAL_GENERATOR_BIN` or `PATH`. Building it +is cheap — the package depends on nothing — so CI clones the pinned revision +and runs `lake build`; `comparator/README.md` shows the same for a local run. + +The context root exists because the v1 contract still resolves two things +from a benchmark checkout rather than from the request: the module source +(which must byte-match the request's `moduleContent`) and each declaration's +span from compiled `.ilean` metadata. This consumer is not a benchmark +checkout, so it materialises a minimal one: the rendered module at +`/.lean`, and a synthesised `.ilean` carrying exactly the spans +`build_problem` computed. It can do that honestly because it rendered the +module; nothing is guessed. +""" + +import json +import os +import pathlib +import shutil +import subprocess + +BINARY_ENV = "LEAN_EVAL_GENERATOR_BIN" +BINARY_NAME = "lean-eval-generator" + + +def binary(): + """The pinned generator executable, from the environment or PATH.""" + named = os.environ.get(BINARY_ENV) + if named: + path = pathlib.Path(named) + if not path.is_file(): + raise SystemExit(f"{BINARY_ENV}={named}: no such file") + return str(path) + found = shutil.which(BINARY_NAME) + if found: + return found + raise SystemExit( + f"no `{BINARY_NAME}` on PATH and {BINARY_ENV} is not set; build the " + "revision pinned under [generator] in comparator/tools.toml and " + "point either at it" + ) + + +def write_context_root(root, problems): + """A minimal benchmark checkout for the request's problems. + + `problems` are `(problem, ilean_decls)` pairs from `build_problem`. Each + module lands at `/.lean` — the byte-match the generator + enforces against `moduleContent` — and its spans at the `.ilean` path the + generator reads, `.lake/build/lib/lean/.ilean`. + """ + root = pathlib.Path(root) + ilean_dir = root / ".lake" / "build" / "lib" / "lean" + ilean_dir.mkdir(parents=True, exist_ok=True) + for problem, ilean in problems: + module = problem["moduleName"] + (root / f"{module}.lean").write_text( + problem["moduleContent"], encoding="utf-8" + ) + (ilean_dir / f"{module}.ilean").write_text( + json.dumps({"version": 1, "module": module, "decls": ilean}) + "\n", + encoding="utf-8", + ) + return root + + +def generate(request): + """The generator's verified file maps for one request. + + Returns `{problem_id: {path: content}}`. Import here rather than at + module top keeps the arrow pointing one way: the interface never imports + this plumbing. + """ + from leaneval_interface import parse_response + + proc = subprocess.run( + [binary()], + input=json.dumps(request), + capture_output=True, + text=True, + ) + if proc.returncode != 0: + raise SystemExit( + f"lean-eval-generator failed:\n{proc.stderr.strip() or proc.stdout.strip()}" + ) + return parse_response(proc.stdout) diff --git a/scripts/leaneval_interface.py b/scripts/leaneval_interface.py index 3c682d18f9..2765fa7df3 100644 --- a/scripts/leaneval_interface.py +++ b/scripts/leaneval_interface.py @@ -1,62 +1,75 @@ #!/usr/bin/env python3 """The one interface between the Formal Conjectures importer and the generator. -`leanprover/lean-eval#536` splits this work in two. The generator core inside -lean-eval's `EvalTools` — the part that turns a marked-up Lean module plus a -manifest into a Challenge / Solution / Submission workspace — is being -extracted into `leanprover/lean-eval-generator` and consumed as a pinned -dependency. The Formal Conjectures side owns an importer that maps FC -declarations and metadata to LeanEval modules and manifests. The FC importer -does not fork the generation logic. - -This module is that seam, and nothing else. It holds the two values the -importer hands the generator and no code that produces or consumes them: - - MarkedUpModule one Mathlib-only Lean module, in labelled regions +`leanprover/lean-eval#536` splits this work in two, and the generator half now +exists: `leanprover/lean-eval-generator` is a deterministic Lean CLI with a +versioned JSON contract, pinned in `comparator/tools.toml`. The consumer sends +one request on stdin — target pins, templates, and per problem a Lean module +with resolved hole ranges — and receives the complete workspace file map with +a SHA-256 digest per file. The Formal Conjectures side owns an importer that +maps FC declarations and metadata to that request. The FC importer does not +fork the generation logic. + +This module is that seam, and nothing else. It holds the values that cross it +and the code that turns them into contract JSON: + + MarkedUpModule one Mathlib-only Lean module, in four internal regions ProblemManifest the facts about the problem that the module's text does - not carry, including the FC source commit, the FC - declaration id, and the pins the workspace is built with + not carry, including the FC source commit and the FC + declaration id; written beside the generated workspace + as `fc-provenance.json`, because the v1 contract has no + provenance fields of its own + build_request (module, manifest) pairs -> the v1 request object + parse_response response text -> file maps, digests checked -`scripts/fc_leaneval_importer.py` produces both. `scripts/leaneval_generator.py` -consumes both and returns a workspace. When `lean-eval-generator` lands, the -generator module goes and this file becomes an import from the pinned package; -the importer keeps building the same two values and does not change. +`scripts/fc_leaneval_importer.py` produces the pairs. +`scripts/leaneval_generator_cli.py` runs the pinned binary. Nothing on the FC +side decides a workspace file's contents any more. -## Why a marked-up module rather than a bag of strings +## Why one module rather than a bag of strings The generator's job includes the import and scope fidelity work from lean-eval#531: deciding which generated file imports which, and where the file-scoped `open`, `variable` and notation have to be restated so that the same statement text elaborates in Challenge, Submission and Solution alike. That decision belongs to the generator, so the importer must not pre-split the -source into those files. It emits one module that elaborates on its own -against Mathlib, with the four parts labelled, and the generator slices it. +source. It emits one module that elaborates on its own against Mathlib, and +the generator slices it by the hole ranges the request carries. Emitting one module also gives the importer a check it could not otherwise have: the module it is about to hand over is exactly the text it can elaborate locally (`--verify`), so a copied closure that has lost an `open` fails on the -FC side rather than in lean-eval's CI. +FC side rather than in lean-eval's CI. For the same reason the module carries +no `@[eval_problem]` markers: that attribute does not exist outside lean-eval, +and the ranges in the request already say where the holes are. -The regions, in the order they must appear: +The regions, in the order they are rendered: dependencies the FC-local closure of the statement, copied, Mathlib-only scope the `open` and file-scoped directives the statement needs holes one `noncomputable def : := sorry` per `answer(sorry)` slot the importer hoisted statement the target statement, its proof replaced by `sorry` + +The regions are an internal structure. What crosses the seam is the rendered +module inside the request, byte for byte. """ import dataclasses +import hashlib import json import re -REGION_MARKER = "-- @region " REGIONS = ("dependencies", "scope", "holes", "statement") MODULE_PREAMBLE = "import Mathlib\n" MANIFEST_SCHEMA_VERSION = 1 +# The generator's frozen wire format; `schemas/request-v1.schema.json` and +# `response-v1.schema.json` in the pinned revision are normative. +CONTRACT_VERSION = 1 + def slug(name): """A Lake package name and directory name for a problem id. @@ -232,19 +245,21 @@ def from_json(cls, text): class MarkedUpModule: """One Mathlib-only Lean module, in the four labelled regions. - Rendering and parsing are inverse on the region bodies, so the artifact - the importer emits for review is the artifact the generator reads. + `dependency_declarations` names each copied declaration and the exact + text the dependencies region carries for it, in order. The contract wants + a source span per declaration, and only the renderer knows which bytes + belong to which copied name. """ dependencies: str scope: str holes: str statement: str + dependency_declarations: tuple = () def __post_init__(self): # Rendering separates the regions itself, so leading and trailing - # blank lines are not part of a region's content. Normalising them - # here is what makes rendering and parsing inverse. + # blank lines are not part of a region's content. for name in REGIONS: object.__setattr__(self, name, getattr(self, name).strip("\n")) @@ -252,47 +267,261 @@ def regions(self): return {name: getattr(self, name) for name in REGIONS} def render(self): + """The module as handed over: plain Lean, no markers of any kind.""" parts = [MODULE_PREAMBLE] - for name, body in self.regions().items(): + for body in self.regions().values(): body = body.strip("\n") - # A copied declaration carrying a line that reads as a marker - # would split the module somewhere the importer did not choose, - # and the generator would never know. Refuse instead. - for line in body.split("\n"): - if line.startswith(REGION_MARKER): - raise SystemExit( - f"the {name} region contains a region marker: {line!r}" - ) - parts.append(f"\n{REGION_MARKER}{name}\n" + (body + "\n" if body else "")) + if body: + parts.append("\n" + body + "\n") return "".join(parts) - @classmethod - def parse(cls, text): - """Read a rendered module back, refusing anything the shape forbids.""" - bodies, current = {}, None - for line in text.split("\n"): - if line.startswith(REGION_MARKER): - current = line[len(REGION_MARKER) :].strip() - if current not in REGIONS: - raise SystemExit(f"unknown region {current!r} in marked-up module") - if current in bodies: - raise SystemExit(f"region {current!r} appears twice") - bodies[current] = [] - continue - if current is not None: - bodies[current].append(line) - missing = [name for name in REGIONS if name not in bodies] - if missing: + +# The `@[category ...]` tags that name a problem, and the lean-eval group +# each belongs to. `research open` is the point of the FC import and goes to +# the open-conjectures display; everything already settled — solved research, +# textbook and test statements — is evaluation material. `API` declarations +# and untagged ones are not problems and are refused. +CATEGORY_GROUPS = { + "research open": "open-conjectures", + "research solved": "formalization-evaluation", + "textbook": "formalization-evaluation", + "test": "formalization-evaluation", +} + + +def problem_group(manifest): + """The lean-eval problem group for an imported declaration's category.""" + group = CATEGORY_GROUPS.get(manifest.category) + if group is None: + raise SystemExit( + f"{manifest.id}: category {manifest.category!r} is not a " + "problem category; expected one of " + + ", ".join(sorted(CATEGORY_GROUPS)) + ) + return group + + +MATHLIB_GIT = "https://github.com/leanprover-community/mathlib4.git" + +SUBMITTER = "formal-conjectures-importer" + + +def module_declarations(marked_up, manifest): + """Every declaration in the rendered module, in order. + + Returns `(name, body, kind, explicit_parameters)` tuples: the copied + dependencies first, then the hoisted answer holes, then the statement. + The generator needs a span for each — holes to slice, dependencies to + keep or drop per generated file — and the bodies are what the spans are + computed from. + """ + declarations = [ + (name, body, "helper", None) + for name, body in marked_up.dependency_declarations + ] + declarations += [ + (hole.name, hole.declaration(), "def", None) for hole in manifest.holes + ] + declarations.append( + ( + manifest.theorem, + marked_up.statement, + "theorem", + list(manifest.apply_arguments), + ) + ) + return declarations + + +def _positions(text): + """Codepoint offset of the start of each 1-indexed line.""" + starts = [0] + for line in text.split("\n")[:-1]: + starts.append(starts[-1] + len(line) + 1) + return starts + + +def _utf16_column(line_text, column): + """The UTF-16 code-unit column for a codepoint column. + + `.ilean` files store LSP ranges, and LSP counts UTF-16 code units; a + supplementary-plane character (𝔽, 𝕜) earlier in the line makes the two + disagree. + """ + return sum(2 if ord(c) > 0xFFFF else 1 for c in line_text[:column]) + + +def declaration_spans(module_text, declarations): + """A source span for each declaration, located by its exact text. + + The renderer wrote every declaration into the module verbatim, so each + body appears in the text; a body appearing more than once would make the + span a guess, and is refused. Lines are 1-indexed. `codepoint` columns + are what the contract's `resolvedHoles` carry; `utf16` columns are what + an `.ilean` carries. + """ + line_starts = _positions(module_text) + lines = module_text.split("\n") + + def line_of(offset): + low, high = 0, len(line_starts) - 1 + while low < high: + mid = (low + high + 1) // 2 + if line_starts[mid] <= offset: + low = mid + else: + high = mid - 1 + return low + + spans = [] + for name, body, kind, explicit in declarations: + body = body.strip("\n") + first = module_text.find(body) + if first < 0: + raise SystemExit(f"{name}: declaration text not found in the module") + if module_text.find(body, first + 1) >= 0: + raise SystemExit( + f"{name}: declaration text appears more than once in the module" + ) + end = first + len(body) + start_line, end_line = line_of(first), line_of(end) + start_col = first - line_starts[start_line] + end_col = end - line_starts[end_line] + spans.append( + { + "name": name, + "kind": kind, + "explicitParameters": explicit, + "startLine": start_line + 1, + "startColumn": start_col, + "endLine": end_line + 1, + "endColumn": end_col, + "utf16StartColumn": _utf16_column(lines[start_line], start_col), + "utf16EndColumn": _utf16_column(lines[end_line], end_col), + } + ) + return spans + + +def build_problem(marked_up, manifest, module_name=None): + """One problem entry of the v1 request, and its `.ilean` declaration map. + + The module name is a single identifier on purpose: the generator resolves + module names to paths by splitting on every dot, guillemets included, so + a dotted or quoted name would trip the same decoder defect this + repository fixed on its own side. + + Returns `(problem, ilean_decls)`. The `.ilean` payload exists because the + generator reads helper-declaration spans from compiled metadata it + expects to find under the context root; this consumer synthesises that + metadata from the spans it computed, which it can do exactly because it + rendered the module. + """ + module_name = module_name or slug(manifest.id) + text = marked_up.render() + spans = declaration_spans(text, module_declarations(marked_up, manifest)) + resolved, ilean = [], {} + for span in spans: + # `.ilean` lines are 0-indexed; `loadIleanDeclRanges` adds one back. + ilean[span["name"]] = [ + span["startLine"] - 1, + span["utf16StartColumn"], + span["endLine"] - 1, + span["utf16EndColumn"], + ] + if span["kind"] == "helper": + continue + resolved.append( + { + "declarationName": span["name"], + "module": module_name, + "startLine": span["startLine"], + "startColumn": span["startColumn"], + "endLine": span["endLine"], + "endColumn": span["endColumn"], + "explicitParameters": span["explicitParameters"], + "sameModuleDependencies": ( + [name for name, _ in marked_up.dependency_declarations] + if span["kind"] == "theorem" + else [] + ), + "holeDependentDependencies": [], + "kind": span["kind"], + } + ) + problem = { + "id": slug(manifest.id), + "title": manifest.qualified_theorem, + "group": problem_group(manifest), + "status": "draft", + "visible": True, + "statementRevision": 1, + "tags": ["formal-conjectures"], + "moduleName": module_name, + "holes": [entry["declarationName"] for entry in resolved], + "submitter": SUBMITTER, + "notes": None, + "source": manifest.source_url or None, + "informalSolution": None, + "moduleContent": text, + "resolvedHoles": resolved, + } + return problem, ilean + + +def build_request(problems, target, workspace_test, context_root): + """The complete v1 request for a batch of `(problem, ilean)` pairs. + + `problems` are the entries `build_problem` returned; the ilean halves go + to whoever writes the context root, not into the request. Ids must be + unique across the batch — two problems generating into one directory is + the collision the qualified default id exists to prevent. + """ + seen = set() + for problem in problems: + if problem["id"] in seen: + raise SystemExit(f"duplicate workspace id {problem['id']!r}") + seen.add(problem["id"]) + return { + "schemaVersion": CONTRACT_VERSION, + "contextRoot": str(context_root), + "leanToolchain": target.lean_toolchain, + "mathlib": { + "name": "mathlib", + "git": MATHLIB_GIT, + "rev": target.mathlib_revision, + }, + "templates": {"workspaceTest": workspace_test}, + "problems": problems, + } + + +def parse_response(text): + """The generator's file maps, with every digest checked. + + Returns `{problem_id: {path: content}}`. A digest mismatch means the + bytes were damaged in transit or the pinned generator is not the one + this code was written against; either way the workspace cannot be + trusted, so refuse. + """ + payload = json.loads(text) + version = payload.get("schemaVersion") + if version != CONTRACT_VERSION: + raise SystemExit( + f"generator response schema version {version!r} is not {CONTRACT_VERSION}" + ) + workspaces = {} + for entry in payload["files"]: + digest = hashlib.sha256(entry["content"].encode("utf-8")).hexdigest() + if digest != entry["sha256"]: raise SystemExit( - "marked-up module has no " + ", ".join(f"`{m}`" for m in missing) - + " region" + f"{entry['problemId']}/{entry['path']}: content does not match " + "its digest" ) - if list(bodies) != list(REGIONS): + files = workspaces.setdefault(entry["problemId"], {}) + if entry["path"] in files: raise SystemExit( - "marked-up module regions are out of order: " - + ", ".join(bodies) - + f"; expected {', '.join(REGIONS)}" + f"{entry['problemId']}/{entry['path']}: appears twice in response" ) - return cls( - **{name: "\n".join(lines) for name, lines in bodies.items()} - ) + files[entry["path"]] = entry["content"] + return workspaces diff --git a/scripts/make_comparator_workspace.py b/scripts/make_comparator_workspace.py index 8780fd2010..1f9ab1778f 100644 --- a/scripts/make_comparator_workspace.py +++ b/scripts/make_comparator_workspace.py @@ -6,14 +6,16 @@ permitted axioms. This command produces that shape for one Formal Conjectures declaration, in the two steps `leanprover/lean-eval#536` separates: - fc_leaneval_importer FC declaration -> marked-up module + manifest - leaneval_generator marked-up module + manifest -> workspace + fc_leaneval_importer FC declaration -> marked-up module + manifest + lean-eval-generator v1 request -> workspace file map -The first half is Formal Conjectures'. The second half is lean-eval's, and is -to be replaced by a pinned dependency on `leanprover/lean-eval-generator`; the -module standing in for it here is the code that gets deleted when that lands. -`comparator/OWNERSHIP.md` says exactly what goes and what stays. This file is -the wiring between them and belongs to neither. +The first half is Formal Conjectures'. The second is the pinned +`leanprover/lean-eval-generator` binary — a deterministic Lean CLI with a +versioned JSON contract — run by `scripts/leaneval_generator_cli.py` at the +revision `comparator/tools.toml` pins. `scripts/leaneval_interface.py` builds +the request and checks the response. `comparator/OWNERSHIP.md` says exactly +what belongs to which side. This file is the wiring between them and belongs +to neither. The marked-up module requires Mathlib and nothing else. lean-eval vendors its problems, so a Challenge cannot fetch this repository at evaluation time, which @@ -38,23 +40,35 @@ python make_comparator_workspace.py ID --emit-import DIR python make_comparator_workspace.py --validate +`--emit-import` writes the exact bytes that cross the seam — the v1 request, +with its context directory — and generates no workspace; running the pinned +binary on that request from inside the emitted directory yields the same file +map this command would have written. + The workspace's own build needs a network fetch of its pinned dependencies, so -this command does not attempt it; generation is offline and the build belongs -to the comparator run. +this command does not attempt it; generation is offline apart from the +generator binary, and the build belongs to the comparator run. """ import argparse +import json import pathlib import shutil import sys import tempfile import fc_leaneval_importer as importer -import leaneval_generator as generator -from leaneval_interface import slug +import leaneval_generator_cli as generator_cli +from leaneval_interface import build_problem, build_request, slug ROOT = importer.ROOT +PROVENANCE_FILE = "fc-provenance.json" + +# The request's context directory, relative to the request file, so an +# emitted seam artifact is self-contained and reproducible from any path. +CONTEXT_DIR = "context" + def write_tree(target, files): """Write a complete directory without overwriting or leaving a partial one. @@ -62,7 +76,7 @@ def write_tree(target, files): Plumbing, and on neither side of the seam: the generator returns a path-to-content mapping and never touches the filesystem, so putting one on disk is the command's job whether the mapping is a workspace or the - pair this repository hands over. + request this repository hands over. """ target = pathlib.Path(target) if target.exists(): @@ -83,18 +97,60 @@ def write_tree(target, files): return target -def emit_import(marked_up, manifest, out_dir): - """Write only the pair this repository owns: the module and the manifest. +def seam_files(pairs): + """The request and context for `(marked_up, manifest)` pairs, as files. - This is the artifact the FC importer contributes to a lean-eval problem - pull request once the generator is a pinned dependency there. Emitting it - on its own keeps the seam checkable today: the bytes here are the bytes - the generator gets, and nothing in this directory is workspace layout. + This is the artifact the FC importer contributes once lean-eval consumes + the shared generator: the request bytes, the context directory the v1 + contract still reads, and one provenance record per problem — the FC + source commit and declaration id §10 requires, which the v1 wire format + has no field for, so they travel beside it rather than through it. """ - return write_tree( - pathlib.Path(out_dir) / slug(manifest.id), - {"Problem.lean": marked_up.render(), "manifest.json": manifest.to_json()}, + problems = [build_problem(marked_up, manifest) for marked_up, manifest in pairs] + target = importer.target_pins() + template = ( + importer.COMPARATOR_DIR / "templates" / "WorkspaceTest.lean" + ).read_text(encoding="utf-8") + request = build_request( + [problem for problem, _ in problems], target, template, CONTEXT_DIR ) + files = {"request.json": json.dumps(request, indent=2, ensure_ascii=False) + "\n"} + for (problem, ilean), (_, manifest) in zip(problems, pairs): + module = problem["moduleName"] + files[f"{CONTEXT_DIR}/{module}.lean"] = problem["moduleContent"] + files[f"{CONTEXT_DIR}/.lake/build/lib/lean/{module}.ilean"] = ( + json.dumps({"version": 1, "module": module, "decls": ilean}) + "\n" + ) + files[f"{PROVENANCE_FILE.removesuffix('.json')}-{problem['id']}.json"] = ( + manifest.to_json() + ) + return request, files + + +def generate_workspaces(pairs, out_dir): + """Generate one workspace per pair under `out_dir`, via the pinned binary.""" + request, files = seam_files(pairs) + staging = pathlib.Path(tempfile.mkdtemp(prefix=".fc-seam.")) + try: + for relative, content in files.items(): + destination = staging / relative + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(content, encoding="utf-8") + request["contextRoot"] = str(staging / CONTEXT_DIR) + workspaces = generator_cli.generate(request) + finally: + shutil.rmtree(staging, ignore_errors=True) + written = [] + for _, manifest in pairs: + problem_id = slug(manifest.id) + if problem_id not in workspaces: + raise SystemExit(f"the generator returned no files for {problem_id}") + workspace = dict(workspaces[problem_id]) + # The provenance sidecar rides in the workspace directory, not in the + # generator's file map: the generator neither knows nor checks it. + workspace[PROVENANCE_FILE] = manifest.to_json() + written.append(write_tree(pathlib.Path(out_dir) / problem_id, workspace)) + return written def main(argv): @@ -127,8 +183,8 @@ def main(argv): "--emit-import", default=None, metavar="DIR", - help="write only the marked-up module and its manifest, the pair this " - "repository hands the generator, and generate no workspace", + help="write only the v1 request and its context, the bytes this " + "repository hands the pinned generator, and generate no workspace", ) ap.add_argument( "--validate", @@ -146,12 +202,11 @@ def main(argv): if args.verify: importer.elaborate(marked_up) if args.emit_import: - print(emit_import(marked_up, manifest, args.emit_import)) + _, files = seam_files([(marked_up, manifest)]) + print(write_tree(pathlib.Path(args.emit_import) / slug(manifest.id), files)) return 0 - # The consumer supplies its own pins; see TargetRecord. Locally that is - # `[target]` in comparator/tools.toml, standing in for lean-eval's. - files = generator.generate(marked_up, manifest, importer.target_pins()) - print(write_tree(pathlib.Path(args.out) / slug(manifest.id), files)) + for path in generate_workspaces([(marked_up, manifest)], args.out): + print(path) return 0 diff --git a/scripts/test_fc_leaneval_importer.py b/scripts/test_fc_leaneval_importer.py index 50b2b14f1e..e26b1a627e 100644 --- a/scripts/test_fc_leaneval_importer.py +++ b/scripts/test_fc_leaneval_importer.py @@ -324,7 +324,7 @@ def test_a_generated_constant_under_a_copied_parent_is_accepted(self): resolve.return_value = source out, copied = closure_region(deps, ["Foo.bar._proof_1"], "t") self.assertIn("def Foo.bar := 1", out) - self.assertEqual(copied, ["Foo.bar"]) + self.assertEqual(copied, [("Foo.bar", "def Foo.bar := 1")]) def test_a_declaration_inside_another_s_range_is_not_copied_twice(self): # `EdgeN.mk` covers line 88 of a structure spanning 83 to 93, and diff --git a/scripts/test_leaneval_generator.py b/scripts/test_leaneval_generator.py deleted file mode 100644 index e86dc4b1ea..0000000000 --- a/scripts/test_leaneval_generator.py +++ /dev/null @@ -1,163 +0,0 @@ -# Copyright 2026 The Formal Conjectures Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tests for the side of the seam that becomes a pinned dependency. - -These cases describe what a Challenge/Solution/Submission workspace must look -like given a marked-up module and a manifest. They are written against nothing -but those two values on purpose: when `leanprover/lean-eval-generator` replaces -`scripts/leaneval_generator.py`, this file is what says whether the pinned -generator still produces what Formal Conjectures' import expects. -""" - -import ast -import json -import pathlib -import unittest - -import leaneval_generator as generator -from leaneval_generator import generate -from test_leaneval_interface import A_MODULE, a_manifest, a_target - -# The modules that live beside the generator in `scripts/`. Anything the -# generator imports from here has to move with it into the pinned package. -LOCAL = {path.name for path in pathlib.Path(generator.__file__).parent.glob("*.py")} - - -def a_workspace(**overrides): - return generate(A_MODULE, a_manifest(**overrides), a_target()) - - -class SplitTest(unittest.TestCase): - """One module in, four Lean files out, and the imports have to line up.""" - - def test_the_closure_is_the_only_file_importing_mathlib(self): - files = a_workspace() - self.assertTrue(files["ChallengeDeps.lean"].startswith("import Mathlib\n")) - self.assertIn("def Foo.bar := 1", files["ChallengeDeps.lean"]) - for name in ("Challenge.lean", "Solution.lean", "Submission.lean"): - self.assertTrue(files[name].startswith("import ChallengeDeps"), name) - - def test_the_statement_text_is_identical_in_all_three_files(self): - # The Solution adapter only pins the statement if the statement it - # restates is the one the Challenge poses. - files = a_workspace() - statement = A_MODULE.statement - self.assertIn(statement, files["Challenge.lean"]) - self.assertIn(statement, files["Submission.lean"]) - self.assertIn(statement.split(":= by")[0].rstrip(), files["Solution.lean"]) - - def test_the_scope_is_restated_in_every_file_that_carries_the_statement(self): - # `open Erdos` is file-scoped, so an import cannot carry it. - files = a_workspace() - for name in ("Challenge.lean", "Solution.lean", "Submission.lean"): - self.assertIn("open Erdos", files[name], name) - - def test_the_submission_is_namespaced_away_from_the_trusted_names(self): - submission = a_workspace()["Submission.lean"] - self.assertIn("namespace Submission", submission) - self.assertIn("end Submission", submission) - - def test_the_solution_delegates_the_hole_and_applies_the_arguments(self): - solution = a_workspace()["Solution.lean"] - # Reducible, so the unifier is certain to unfold the Solution's copy of - # the hole into the Submission's when it checks the adapter. - self.assertIn( - "@[reducible] noncomputable def erdos_940_answer : ENNReal :=\n" - " Submission.erdos_940_answer", - solution, - ) - self.assertTrue(solution.rstrip().endswith("Submission.erdos_940 n")) - - def test_a_statement_with_no_arguments_is_not_applied_to_anything(self): - # A `∀` binder in the conclusion is not a declaration parameter, and - # applying one would fail to elaborate. - solution = a_workspace(apply_arguments=())["Solution.lean"] - self.assertTrue(solution.rstrip().endswith("Submission.erdos_940")) - - def test_a_workspace_with_no_hole_declares_no_definition_names(self): - config = json.loads(a_workspace(holes=())["config.json"]) - self.assertNotIn("definition_names", config) - - def test_the_config_names_the_theorem_the_holes_and_the_axioms(self): - config = json.loads(a_workspace()["config.json"]) - self.assertEqual(config["theorem_names"], ["erdos_940"]) - self.assertEqual(config["definition_names"], ["erdos_940_answer"]) - self.assertIn("propext", config["permitted_axioms"]) - self.assertNotIn("sorryAx", config["permitted_axioms"]) - - def test_the_lakefile_pins_the_target_mathlib(self): - # Not this repository's Mathlib: the workspace is built in lean-eval. - lakefile = a_workspace()["lakefile.toml"] - self.assertIn('rev = "' + "f" * 40 + '"', lakefile) - self.assertEqual(lakefile.count("[[require]]"), 1) - self.assertNotIn("formal-conjectures", lakefile) - - def test_the_package_name_is_an_identifier(self): - files = generate(A_MODULE, a_manifest(id="erdos_940.variants.large_integers"), a_target()) - self.assertIn( - 'name = "erdos_940_variants_large_integers"', files["lakefile.toml"] - ) - - def test_the_toolchain_file_is_the_target_toolchain(self): - self.assertEqual(a_workspace()["lean-toolchain"], "leanprover/lean4:v4.33.0\n") - - def test_nothing_in_the_workspace_carries_this_repositorys_toolchain(self): - # A workspace built at FC's toolchain is not the artifact lean-eval - # vendors, and shipping one would hide the pin gap the manifest states. - files = a_workspace() - self.assertNotIn("v4.27.0", files["lean-toolchain"]) - self.assertNotIn("v4.27.0", files["lakefile.toml"]) - - -class ManifestPassThroughTest(unittest.TestCase): - def test_the_workspace_carries_the_fc_commit_and_declaration(self): - # lean-eval#536: each manifest records the FC source commit and - # declaration id. This side supplies neither and edits neither. - payload = json.loads(a_workspace()["manifest.json"]) - self.assertEqual(payload["source"]["commit"], "a" * 40) - self.assertEqual(payload["source"]["declaration"], "erdos_940") - - def test_the_manifest_is_passed_through_unaltered(self): - manifest = a_manifest() - files = generate(A_MODULE, manifest, a_target()) - self.assertEqual(files["manifest.json"], manifest.to_json()) - - -class SeamTest(unittest.TestCase): - def test_the_generator_depends_on_the_interface_and_nothing_else_local(self): - # The direction of the dependency is the whole point: a generator that - # imported the importer could not be swapped for a pinned package. - tree = ast.parse(pathlib.Path(generator.__file__).read_text(encoding="utf-8")) - imported = set() - for node in ast.walk(tree): - if isinstance(node, ast.Import): - imported.update(alias.name for alias in node.names) - elif isinstance(node, ast.ImportFrom) and node.module: - imported.add(node.module) - local = {name for name in imported if pathlib.Path(f"{name}.py").name in LOCAL} - self.assertEqual(local, {"leaneval_interface"}) - - -class TemplateTest(unittest.TestCase): - def test_workspace_test_template_exists_and_is_the_runner(self): - # The generator copies this file into every workspace; a missing or - # gutted template would only surface at `lake test` time, elsewhere. - text = (generator.TEMPLATE_DIR / "WorkspaceTest.lean").read_text() - self.assertIn("def main", text) - self.assertIn("COMPARATOR_BIN", text) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/test_leaneval_interface.py b/scripts/test_leaneval_interface.py index 935cc23bd1..f27e52a257 100644 --- a/scripts/test_leaneval_interface.py +++ b/scripts/test_leaneval_interface.py @@ -29,6 +29,12 @@ ProblemManifest, SourceRecord, TargetRecord, + _utf16_column, + build_problem, + build_request, + declaration_spans, + module_declarations, + parse_response, slug, ) @@ -68,7 +74,7 @@ def a_manifest(**overrides): "id": "erdos_940", "theorem": "erdos_940", "qualified_theorem": "Erdos.erdos_940", - "apply_arguments": ("n",), + "apply_arguments": (), "holes": (DefinitionHole(name="erdos_940_answer", type="ENNReal"),), "permitted_axioms": ("propext", "Quot.sound", "Classical.choice"), "source": a_source(), @@ -84,6 +90,7 @@ def a_manifest(**overrides): scope="open Erdos", holes="noncomputable def erdos_940_answer : ENNReal := sorry", statement="theorem erdos_940 : erdos_940_answer = 0 := by\n sorry", + dependency_declarations=(("Foo.bar", "def Foo.bar := 1"),), ) @@ -135,55 +142,32 @@ class MarkedUpModuleTest(unittest.TestCase): def test_the_module_stands_on_mathlib_alone(self): self.assertTrue(A_MODULE.render().startswith("import Mathlib\n")) - def test_the_module_survives_a_round_trip(self): - self.assertEqual(MarkedUpModule.parse(A_MODULE.render()), A_MODULE) - - def test_an_empty_region_still_round_trips(self): - # Most statements have no answer slot, so the holes region is empty - # and the generator must still find it. + def test_the_module_carries_no_markers(self): + # The handed-over module is plain Lean: `@[eval_problem]` does not + # exist outside lean-eval, and the request's resolved holes already + # say where the declarations are. + rendered = A_MODULE.render() + self.assertNotIn("@[eval_problem]", rendered) + self.assertNotIn("-- @region", rendered) + + def test_regions_render_in_declaration_order(self): + # A hole is used by the statement below it, and both need the scope + # above them. + rendered = A_MODULE.render() + positions = [ + rendered.index(getattr(A_MODULE, region)) + for region in ("dependencies", "scope", "holes", "statement") + ] + self.assertEqual(positions, sorted(positions)) + + def test_an_empty_region_leaves_no_blank_gap(self): module = MarkedUpModule( dependencies="def f := 1", scope="", holes="", statement="theorem t : True" ) - self.assertEqual(MarkedUpModule.parse(module.render()), module) - - def test_a_missing_region_is_refused(self): - text = A_MODULE.render().replace("-- @region holes\n", "") - with self.assertRaisesRegex(SystemExit, "`holes` region"): - MarkedUpModule.parse(text) - - def test_an_unknown_region_is_refused(self): - with self.assertRaisesRegex(SystemExit, "unknown region"): - MarkedUpModule.parse("import Mathlib\n\n-- @region proof\n") - - def test_a_repeated_region_is_refused(self): - with self.assertRaisesRegex(SystemExit, "appears twice"): - MarkedUpModule.parse(A_MODULE.render() + "\n-- @region scope\n") - - def test_a_copied_declaration_that_looks_like_a_marker_is_refused(self): - # It would split the module somewhere the importer did not choose, - # and the generator would have no way to notice. - module = MarkedUpModule( - dependencies="-- @region statement\ndef f := 1", - scope="", - holes="", - statement="theorem t : True", - ) - with self.assertRaisesRegex(SystemExit, "contains a region marker"): - module.render() - - def test_regions_out_of_order_are_refused(self): - # The order is what makes the module elaborate: a hole is used by the - # statement below it, and both need the scope above them. - module = MarkedUpModule.parse(A_MODULE.render()) - reordered = ( - "import Mathlib\n" - f"\n-- @region scope\n{module.scope}\n" - f"\n-- @region dependencies\n{module.dependencies}\n" - f"\n-- @region holes\n{module.holes}\n" - f"\n-- @region statement\n{module.statement}\n" + self.assertEqual( + module.render(), + "import Mathlib\n\ndef f := 1\n\ntheorem t : True\n", ) - with self.assertRaisesRegex(SystemExit, "out of order"): - MarkedUpModule.parse(reordered) class SlugTest(unittest.TestCase): @@ -197,3 +181,114 @@ def test_a_qualified_declaration_becomes_an_identifier(self): if __name__ == "__main__": unittest.main() + + +class DeclarationSpanTest(unittest.TestCase): + """Spans are computed from the rendered text, exactly.""" + + def test_every_declaration_gets_the_span_of_its_own_text(self): + text = A_MODULE.render() + spans = declaration_spans(text, module_declarations(A_MODULE, a_manifest())) + lines = text.split("\n") + for span in spans: + with self.subTest(name=span["name"]): + sliced = "\n".join( + lines[span["startLine"] - 1 : span["endLine"]] + )[span["startColumn"] :] + self.assertTrue(sliced.startswith(("def", "noncomputable", "theorem"))) + + def test_a_body_appearing_twice_is_refused(self): + with self.assertRaisesRegex(SystemExit, "more than once"): + declaration_spans( + "def f := 1\ndef f := 1\n", [("f", "def f := 1", "def", None)] + ) + + def test_a_missing_body_is_refused(self): + with self.assertRaisesRegex(SystemExit, "not found"): + declaration_spans("def g := 1\n", [("f", "def f := 1", "def", None)]) + + def test_utf16_columns_count_supplementary_plane_pairs(self): + # 𝕜 is beyond the BMP: one codepoint, two UTF-16 units. An `.ilean` + # column after it disagrees with the codepoint column by one. + text = "def 𝕜x := 1\ntheorem t : True := trivial\n" + (span,) = declaration_spans( + text, [("t", "theorem t : True := trivial", "theorem", None)] + ) + self.assertEqual(span["startColumn"], span["utf16StartColumn"]) + line = "abc𝕜 def f := 1" + self.assertEqual(_utf16_column(line, 5), 6) + + +class BuildProblemTest(unittest.TestCase): + def test_the_problem_satisfies_the_contract_shape(self): + problem, ilean = build_problem(A_MODULE, a_manifest()) + self.assertEqual(problem["id"], "erdos_940") + self.assertEqual(problem["group"], "open-conjectures") + self.assertEqual(problem["moduleName"], "erdos_940") + self.assertEqual( + problem["holes"], ["erdos_940_answer", "erdos_940"] + ) + self.assertEqual(problem["moduleContent"], A_MODULE.render()) + kinds = [hole["kind"] for hole in problem["resolvedHoles"]] + self.assertEqual(kinds, ["def", "theorem"]) + # Helpers are `.ilean` material, not holes. + self.assertIn("Foo.bar", ilean) + self.assertNotIn( + "Foo.bar", [hole["declarationName"] for hole in problem["resolvedHoles"]] + ) + + def test_the_theorem_hole_carries_the_copied_dependencies(self): + problem, _ = build_problem(A_MODULE, a_manifest()) + theorem = problem["resolvedHoles"][-1] + self.assertEqual(theorem["sameModuleDependencies"], ["Foo.bar"]) + self.assertEqual(problem["resolvedHoles"][0]["sameModuleDependencies"], []) + + def test_a_non_problem_category_is_refused(self): + with self.assertRaises(SystemExit): + build_problem(A_MODULE, a_manifest(category="API")) + + +class BuildRequestTest(unittest.TestCase): + def test_the_request_carries_the_targets_pins(self): + problem, _ = build_problem(A_MODULE, a_manifest()) + request = build_request([problem], a_target(), "-- test", "context") + self.assertEqual(request["schemaVersion"], 1) + self.assertEqual(request["leanToolchain"], "leanprover/lean4:v4.33.0") + self.assertEqual(request["mathlib"]["rev"], "f" * 40) + self.assertEqual(request["templates"]["workspaceTest"], "-- test") + + def test_duplicate_ids_are_refused(self): + problem, _ = build_problem(A_MODULE, a_manifest()) + with self.assertRaisesRegex(SystemExit, "duplicate workspace id"): + build_request([problem, problem], a_target(), "", "context") + + +class ParseResponseTest(unittest.TestCase): + def _response(self, content="hello"): + import hashlib + import json + + return json.dumps( + { + "schemaVersion": 1, + "files": [ + { + "problemId": "p", + "path": "a.txt", + "sha256": hashlib.sha256(content.encode()).hexdigest(), + "content": "hello", + } + ], + } + ) + + def test_a_good_response_yields_the_file_map(self): + self.assertEqual(parse_response(self._response()), {"p": {"a.txt": "hello"}}) + + def test_a_damaged_digest_is_refused(self): + with self.assertRaisesRegex(SystemExit, "does not match"): + parse_response(self._response(content="tampered")) + + def test_an_unknown_schema_version_is_refused(self): + with self.assertRaisesRegex(SystemExit, "schema version"): + parse_response('{"schemaVersion": 2, "files": []}') diff --git a/scripts/test_make_comparator_workspace.py b/scripts/test_make_comparator_workspace.py index 077e3de9bd..7d95e40b4c 100644 --- a/scripts/test_make_comparator_workspace.py +++ b/scripts/test_make_comparator_workspace.py @@ -15,84 +15,166 @@ """Tests for the command that runs the importer and then the generator. The case that matters here is the seam itself: what this repository hands over -has to be enough. If a workspace cannot be rebuilt from the emitted module and -manifest alone, then some of the interface is still travelling inside the -process, and a pinned `lean-eval-generator` could not be dropped in. +has to be enough. If a workspace cannot be generated from the emitted request +and context directory alone, then some of the interface is still travelling +inside the process, and the pinned `lean-eval-generator` binary could not +reproduce this command's output. """ +import json +import os import pathlib +import shutil import tempfile import unittest from unittest import mock -import leaneval_generator as generator -from leaneval_interface import MarkedUpModule, ProblemManifest -from make_comparator_workspace import emit_import, write_tree -from test_leaneval_interface import a_target, A_MODULE, a_manifest +import leaneval_generator_cli as generator_cli +from make_comparator_workspace import ( + CONTEXT_DIR, + generate_workspaces, + seam_files, + write_tree, +) +from test_leaneval_interface import A_MODULE, a_manifest -class EmitImportTest(unittest.TestCase): - def test_only_the_module_and_the_manifest_are_emitted(self): - with tempfile.TemporaryDirectory() as tmp: - out = emit_import(A_MODULE, a_manifest(), tmp) - self.assertEqual( - sorted(p.name for p in out.iterdir()), - ["Problem.lean", "manifest.json"], - ) +class SeamFilesTest(unittest.TestCase): + def test_the_emitted_files_are_the_request_its_context_and_provenance(self): + _, files = seam_files([(A_MODULE, a_manifest())]) + self.assertEqual( + sorted(files), + [ + f"{CONTEXT_DIR}/.lake/build/lib/lean/erdos_940.ilean", + f"{CONTEXT_DIR}/erdos_940.lean", + "fc-provenance-erdos_940.json", + "request.json", + ], + ) - def test_the_emitted_pair_rebuilds_the_workspace_exactly(self): - manifest = a_manifest() - with tempfile.TemporaryDirectory() as tmp: - out = emit_import(A_MODULE, manifest, tmp) - module = MarkedUpModule.parse( - (out / "Problem.lean").read_text(encoding="utf-8") - ) - read_back = ProblemManifest.from_json( - (out / "manifest.json").read_text(encoding="utf-8") - ) + def test_the_request_names_a_relative_context_root(self): + # The emitted artifact must be reproducible from any path, so the + # request cannot bake in where this machine staged it. + request, _ = seam_files([(A_MODULE, a_manifest())]) + self.assertEqual(request["contextRoot"], CONTEXT_DIR) + + def test_the_context_module_is_the_request_module_byte_for_byte(self): + # The generator refuses a request whose `moduleContent` differs from + # the file at the context root; emitting both from one value is what + # makes that check pass by construction. + request, files = seam_files([(A_MODULE, a_manifest())]) self.assertEqual( - generator.generate(module, read_back, a_target()), - generator.generate(A_MODULE, manifest, a_target()), + files[f"{CONTEXT_DIR}/erdos_940.lean"], + request["problems"][0]["moduleContent"], ) - def test_an_existing_directory_is_not_overwritten(self): - with tempfile.TemporaryDirectory() as tmp: - emit_import(A_MODULE, a_manifest(), tmp) - with self.assertRaisesRegex(SystemExit, "refusing to overwrite"): - emit_import(A_MODULE, a_manifest(), tmp) + def test_the_ilean_carries_every_declaration(self): + _, files = seam_files([(A_MODULE, a_manifest())]) + decls = json.loads( + files[f"{CONTEXT_DIR}/.lake/build/lib/lean/erdos_940.ilean"] + )["decls"] + self.assertEqual( + sorted(decls), + ["Foo.bar", "erdos_940", "erdos_940_answer"], + ) - def test_the_emitted_directory_is_named_by_the_problem_id(self): - with tempfile.TemporaryDirectory() as tmp: - out = emit_import( - A_MODULE, a_manifest(id="erdos_940.variants.large_integers"), tmp - ) - self.assertEqual( - out, pathlib.Path(tmp) / "erdos_940_variants_large_integers" - ) + def test_the_provenance_sidecar_is_the_manifest(self): + # The v1 wire format has no provenance fields, so the FC source + # commit and declaration id §10 requires travel beside the request. + _, files = seam_files([(A_MODULE, a_manifest())]) + payload = json.loads(files["fc-provenance-erdos_940.json"]) + self.assertEqual(payload["source"]["commit"], "a" * 40) + self.assertEqual(payload["source"]["declaration"], "erdos_940") + + def test_two_problems_with_one_id_are_refused(self): + with self.assertRaisesRegex(SystemExit, "duplicate workspace id"): + seam_files([(A_MODULE, a_manifest()), (A_MODULE, a_manifest())]) class WriteTreeTest(unittest.TestCase): def test_existing_directory_is_not_overwritten(self): with tempfile.TemporaryDirectory() as tmp: - target = pathlib.Path(tmp) / "workspace" + target = pathlib.Path(tmp) / "ws" target.mkdir() - sentinel = target / "keep.txt" - sentinel.write_text("keep", encoding="utf-8") with self.assertRaisesRegex(SystemExit, "refusing to overwrite"): - write_tree(target, {"Challenge.lean": "theorem t : True"}) - self.assertEqual(sentinel.read_text(encoding="utf-8"), "keep") + write_tree(target, {"a.txt": "a"}) def test_failed_write_leaves_no_partial_directory(self): with tempfile.TemporaryDirectory() as tmp: - root = pathlib.Path(tmp) - target = root / "workspace" - with mock.patch.object( - pathlib.Path, "write_text", side_effect=OSError("disk error") - ): - with self.assertRaisesRegex(OSError, "disk error"): - write_tree(target, {"Challenge.lean": "theorem t : True"}) + target = pathlib.Path(tmp) / "ws" + files = {"a.txt": "a", "b.txt": None} # None: write_text raises + with self.assertRaises(Exception): + write_tree(target, files) self.assertFalse(target.exists()) - self.assertEqual(list(root.iterdir()), []) + self.assertEqual(list(pathlib.Path(tmp).iterdir()), []) + + +@unittest.skipUnless( + os.environ.get(generator_cli.BINARY_ENV) or shutil.which(generator_cli.BINARY_NAME), + "pinned lean-eval-generator binary not available", +) +class PinnedGeneratorTest(unittest.TestCase): + """End to end against the real pinned binary, when one is built. + + CI builds the revision `comparator/tools.toml` pins and exports + `LEAN_EVAL_GENERATOR_BIN`; locally the test is skipped unless you have + done the same. + """ + + EXPECTED_FILES = [ + "Challenge.lean", + "ChallengeDeps.lean", + "README.md", + "Solution.lean", + "Submission.lean", + "Submission/Helpers.lean", + "WorkspaceTest.lean", + "config.json", + "fc-provenance.json", + "holes.json", + "lakefile.toml", + "lean-toolchain", + ] + + def test_a_workspace_generates_and_the_seam_reproduces_it(self): + with tempfile.TemporaryDirectory() as tmp: + out = pathlib.Path(tmp) / "out" + (workspace,) = generate_workspaces([(A_MODULE, a_manifest())], out) + written = sorted( + str(p.relative_to(workspace)) + for p in workspace.rglob("*") + if p.is_file() + ) + self.assertEqual(written, self.EXPECTED_FILES) + # The definition hole reaches the config, and the delegation is + # reducible in the Solution. + config = json.loads((workspace / "config.json").read_text()) + self.assertEqual(config["definition_names"], ["erdos_940_answer"]) + self.assertIn( + "@[reducible] noncomputable def erdos_940_answer", + (workspace / "Solution.lean").read_text(), + ) + + def test_the_emitted_request_yields_identical_digests(self): + # The seam is real only if the emitted bytes regenerate the same + # workspace: run the binary on the emitted request, from inside the + # emitted directory, and compare content digests per file. + request, files = seam_files([(A_MODULE, a_manifest())]) + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + for relative, content in files.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + cwd = os.getcwd() + try: + os.chdir(root) + first = generator_cli.generate(request) + second = generator_cli.generate(request) + finally: + os.chdir(cwd) + self.assertEqual(first, second) + self.assertEqual(sorted(first), ["erdos_940"]) if __name__ == "__main__": From 2fc99fbb3d1620a74c025f695df9431ec25886cb Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:31:59 -0400 Subject: [PATCH 22/70] Add whole-set batch import with a per-declaration report and known-failures gate --- scripts/make_comparator_workspace.py | 157 +++++++++++++++++++++- scripts/test_make_comparator_workspace.py | 46 +++++++ 2 files changed, 202 insertions(+), 1 deletion(-) diff --git a/scripts/make_comparator_workspace.py b/scripts/make_comparator_workspace.py index 1f9ab1778f..ecea9454ed 100644 --- a/scripts/make_comparator_workspace.py +++ b/scripts/make_comparator_workspace.py @@ -38,8 +38,16 @@ python make_comparator_workspace.py (ID | DECLARATION) [--out DIR] [--answer-type T] [--module FILE] [--verify] python make_comparator_workspace.py ID --emit-import DIR + python make_comparator_workspace.py --set NAME [--out DIR] [--verify] + [--report FILE] [--known-failures FILE] python make_comparator_workspace.py --validate +`--set` imports every declaration of a `FormalConjectures/Subsets` list, +builds one request for all of them, and writes a per-declaration report. +With `--known-failures`, the run fails unless the failures are exactly the +recorded ones: an unexpected failure and a silently fixed one both count, +because a gate that only ever passes proves nothing. + `--emit-import` writes the exact bytes that cross the seam — the v1 request, with its context directory — and generates no workspace; running the pinned binary on that request from inside the emitted directory yields the same file @@ -53,9 +61,11 @@ import argparse import json import pathlib +import re import shutil import sys import tempfile +import tomllib import fc_leaneval_importer as importer import leaneval_generator_cli as generator_cli @@ -153,6 +163,117 @@ def generate_workspaces(pairs, out_dir): return written +def subset_declarations(set_name): + """The declaration list of a `FormalConjectures/Subsets` module. + + The subset files hold one `decl_name% ` per line; the + `decl_name%` elaborator is what guarantees each name resolves, so the + text layer can read the list without re-proving that. + """ + path = ROOT / "FormalConjectures" / "Subsets" / f"{set_name}.lean" + if not path.is_file(): + raise SystemExit(f"no subset module at {path}") + names = re.findall( + r"decl_name%\s+([\w.«»]+)", path.read_text(encoding="utf-8") + ) + if not names: + raise SystemExit(f"{path} lists no decl_name% entries") + return names + + +def load_known_failures(path): + """The recorded failures, `{declaration: {stage, reason}}`.""" + with open(path, "rb") as handle: + data = tomllib.load(handle) + failures = {} + for entry in data.get("failure", []): + for field in ("declaration", "stage", "reason"): + if field not in entry: + raise SystemExit(f"{path}: a failure entry has no `{field}`") + if entry["stage"] not in ("source", "target"): + raise SystemExit( + f"{path}: {entry['declaration']} has stage {entry['stage']!r}; " + "expected source or target" + ) + failures[entry["declaration"]] = entry + return failures + + +def import_set(set_name, out_dir, verify=False, known_failures=None): + """Import a whole subset, generate what imports, and report the rest. + + Returns the report object. Source-side failures — the importer refusing, + or `--verify` elaboration failing — are recorded per declaration rather + than aborting the run, because the whole-set result is the artifact: + lean-eval#536 gates the FC import on this audit being reproducible. + """ + declarations = subset_declarations(set_name) + pairs, results = [], [] + for declaration in declarations: + try: + marked_up, manifest = importer.import_problem(declaration) + if verify: + importer.elaborate(marked_up) + except SystemExit as failure: + results.append( + { + "declaration": declaration, + "status": "source-failed", + "reason": str(failure), + } + ) + continue + pairs.append((marked_up, manifest)) + results.append( + { + "declaration": declaration, + "id": slug(manifest.id), + "category": manifest.category, + "status": "imported", + } + ) + written = generate_workspaces(pairs, out_dir) if pairs else [] + categories = {} + for entry in results: + if entry["status"] == "imported": + categories[entry["category"]] = categories.get(entry["category"], 0) + 1 + report = { + "set": set_name, + "total": len(declarations), + "imported": len(pairs), + "source_failed": len(declarations) - len(pairs), + "categories": dict(sorted(categories.items())), + "workspaces": [str(path) for path in written], + "declarations": results, + } + if known_failures is not None: + expected = { + name + for name, entry in known_failures.items() + if entry["stage"] == "source" + } + actual = { + entry["declaration"] + for entry in results + if entry["status"] == "source-failed" + } + unexpected = sorted(actual - expected) + fixed = sorted(expected - actual) + if unexpected or fixed: + for name in unexpected: + print(f"unexpected source failure: {name}", file=sys.stderr) + for name in fixed: + print( + f"{name} is recorded as a known source failure but " + "imported; remove it from the record", + file=sys.stderr, + ) + report["known_failures_match"] = False + else: + report["known_failures_match"] = True + return report + + def main(argv): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument( @@ -191,11 +312,45 @@ def main(argv): action="store_true", help="check every problem file resolves, and import nothing", ) + ap.add_argument( + "--set", + default=None, + metavar="NAME", + help="import every declaration of FormalConjectures/Subsets/NAME.lean", + ) + ap.add_argument( + "--report", + default=None, + metavar="FILE", + help="with --set: write the per-declaration report here as JSON", + ) + ap.add_argument( + "--known-failures", + default=None, + metavar="FILE", + help="with --set: fail unless the failures are exactly the recorded ones", + ) args = ap.parse_args(argv) if args.validate: return importer.validate() + if args.set: + known = ( + load_known_failures(args.known_failures) + if args.known_failures + else None + ) + report = import_set( + args.set, args.out, verify=args.verify, known_failures=known + ) + text = json.dumps(report, indent=2, ensure_ascii=False) + "\n" + if args.report: + pathlib.Path(args.report).write_text(text, encoding="utf-8") + print(text, end="") + if known is not None and not report.get("known_failures_match", True): + return 1 + return 0 if not args.declaration: - ap.error("give a declaration, or --validate") + ap.error("give a declaration, --set, or --validate") marked_up, manifest = importer.import_problem( args.declaration, args.answer_type, args.module ) diff --git a/scripts/test_make_comparator_workspace.py b/scripts/test_make_comparator_workspace.py index 7d95e40b4c..481e922400 100644 --- a/scripts/test_make_comparator_workspace.py +++ b/scripts/test_make_comparator_workspace.py @@ -177,5 +177,51 @@ def test_the_emitted_request_yields_identical_digests(self): self.assertEqual(sorted(first), ["erdos_940"]) + +class SubsetTest(unittest.TestCase): + def test_the_open_set_lists_one_hundred_declarations(self): + from make_comparator_workspace import subset_declarations + + names = subset_declarations("FC100OpenSet1") + self.assertEqual(len(names), 100) + self.assertIn("OeisA308734.conjecture", names) + self.assertIn("Erdos125.erdos_125.variants.positive_unequal_density", names) + + def test_a_missing_subset_is_refused(self): + from make_comparator_workspace import subset_declarations + + with self.assertRaises(SystemExit): + subset_declarations("NoSuchSet") + + +class KnownFailuresTest(unittest.TestCase): + def _load(self, text): + from make_comparator_workspace import load_known_failures + + with tempfile.NamedTemporaryFile("w", suffix=".toml", delete=False) as f: + f.write(text) + name = f.name + try: + return load_known_failures(name) + finally: + os.unlink(name) + + def test_entries_are_keyed_by_declaration(self): + failures = self._load( + '[[failure]]\ndeclaration = "A.b"\nstage = "source"\nreason = "x"\n' + ) + self.assertEqual(failures["A.b"]["stage"], "source") + + def test_an_unknown_stage_is_refused(self): + with self.assertRaises(SystemExit): + self._load( + '[[failure]]\ndeclaration = "A.b"\nstage = "later"\nreason = "x"\n' + ) + + def test_a_missing_field_is_refused(self): + with self.assertRaises(SystemExit): + self._load('[[failure]]\ndeclaration = "A.b"\nstage = "source"\n') + + if __name__ == "__main__": unittest.main() From 67ee3615f5325aeda8a17c1f3d28dcc0581fa3df Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:33:58 -0400 Subject: [PATCH 23/70] Rewrite the seam documentation around the pinned v1 contract --- comparator/OWNERSHIP.md | 233 +++++++++++++++++----------------------- comparator/README.md | 64 ++++++++--- 2 files changed, 150 insertions(+), 147 deletions(-) diff --git a/comparator/OWNERSHIP.md b/comparator/OWNERSHIP.md index 2492e6ccd3..e94cdeab94 100644 --- a/comparator/OWNERSHIP.md +++ b/comparator/OWNERSHIP.md @@ -1,84 +1,86 @@ # What this repository owns, and what it hands over [`lean-eval#536`](https://github.com/leanprover/lean-eval/pull/536) §10 divides -this integration in two. lean-eval's generator core — the part that turns a -marked-up Lean module plus a manifest into a Challenge / Solution / Submission -workspace, with the import and scope fidelity work from -[`lean-eval#531`](https://github.com/leanprover/lean-eval/pull/531) — is being -extracted into `leanprover/lean-eval-generator` and consumed as a pinned -dependency. **The Formal Conjectures importer does not fork the generation -logic.** It maps FC declarations and metadata to LeanEval modules and manifests, -and each manifest records the FC source commit and declaration id. - -The code here is arranged along that line so the handover is a deletion rather -than a rewrite. This file says exactly what goes. +this integration in two, and the other half now exists: +[`leanprover/lean-eval-generator`](https://github.com/leanprover/lean-eval-generator) +is the extracted generator core — the part that turns a Lean module plus hole +metadata into a Challenge / Solution / Submission workspace, with the import +and scope fidelity work from +[`lean-eval#531`](https://github.com/leanprover/lean-eval#531). It is a +deterministic Lean CLI with a frozen, versioned JSON contract, consumed at the +exact revision `comparator/tools.toml` pins under `[generator]`. **The Formal +Conjectures importer does not fork the generation logic.** It maps FC +declarations and metadata to a v1 request, and records the FC source commit +and declaration id for every problem. ## The seam scripts/fc_leaneval_importer.py FC declaration -> (module, manifest) - scripts/leaneval_interface.py the two values, and nothing else - scripts/leaneval_generator.py (module, manifest, pins) -> workspace files + scripts/leaneval_interface.py the request built from them, the + response checked against its digests + scripts/leaneval_generator_cli.py runs the pinned binary, nothing else `scripts/make_comparator_workspace.py` is the command that runs one after the -other. The arrow points one way: the generator imports the interface and never -the importer, and a test asserts that. +other. The arrow points one way: the CLI plumbing imports the interface and +never the importer, and a test asserts that. ### What crosses it -`MarkedUpModule` is one Lean module that requires Mathlib and nothing else, -divided into four labelled regions: +One **v1 request** (`schemas/request-v1.schema.json` at the pinned generator +revision is normative). Per problem it carries: -| Region | Contents | +| Field | Comes from | |---|---| -| `dependencies` | the statement's FC-local closure, copied, each declaration carrying the `open`, `variable`, `universe`, `set_option` and `local notation` in force where it was written | -| `scope` | the directives the statement itself needs, and the namespaces it is stated in | -| `holes` | one `noncomputable def : := sorry` for each `answer(sorry)` slot | -| `statement` | the target statement, decorations stripped, proof replaced by `sorry` | - -The module is not pre-split into Challenge and ChallengeDeps, because deciding -which generated file imports which, and where the scope has to be restated so -that the same statement text elaborates in all three, is the generator's work. -It is one module rather than four strings because the importer can then +| `moduleContent` | the rendered marked-up module: the statement's copied FC-local closure, the scope directives in force where it was written, one `noncomputable def : := sorry` per `answer(sorry)` slot, and the statement with its proof replaced by `sorry` — in that order, requiring Mathlib and nothing else | +| `resolvedHoles` | a source span, kind, and explicit parameters for each hole, computed from the rendered text — exactly, because this side rendered it | +| `holes`, `id`, `moduleName` | the qualified declaration name, slugged; two modules declaring `conjecture` in different namespaces must not share a workspace | +| `group` | the declaration's `@[category ...]` tag: `research open` is an open conjecture, settled statements are evaluation material, anything else is refused | +| `leanToolchain`, `mathlib` | LeanEval's pins, from `[target]` in `tools.toml` — the consumer's, never this repository's | +| `templates.workspaceTest` | `comparator/templates/WorkspaceTest.lean`, which stays FC-supplied: the contract requires the consumer to provide it | +| `contextRoot` | a directory this side materialises: the module file the generator byte-checks against `moduleContent`, and a synthesised `.ilean` carrying the spans above, because v1 still resolves declaration spans from compiled metadata | + +The module carries no markers of any kind. `@[eval_problem]` does not exist +outside lean-eval, so a module carrying it could not elaborate under +`--verify`; the ranges in the request already say where the holes are. + +The module is one file rather than four strings because the importer can then elaborate exactly what it is about to hand over: `--verify` runs the module through this checkout's Mathlib, so an FC-side defect — a lost `open`, an unrecognised `local notation`, a namespace nothing declares any more — fails here and not in lean-eval's CI. -`ProblemManifest` carries what the Lean text does not say: the theorem's name -and its explicit parameters, the hole types Lean reported, the permitted -axioms, a `source` record with the FC repository, commit, blob, module, -declaration id and this repository's Lean and Mathlib pins, and a `target` -record with LeanEval's pins, which are the ones the workspace is built at. lean-eval#536 requires the -commit and the declaration id by name, and they are FC-side by necessity: the -generator sees a Lean module, not a repository. They are also what makes -regeneration possible when Formal Conjectures corrects a misformalisation -upstream. The generator writes the manifest into the workspace unaltered, as -`manifest.json`. - -## What is deleted when `lean-eval-generator` lands - -| File | Lines | Then | -|---|---|---| -| `scripts/leaneval_generator.py` | 228 | deleted; `generate` becomes a call into the pinned package | -| `scripts/test_leaneval_generator.py` | 163 | deleted, less whatever remains useful as a contract test against the pinned generator | -| `comparator/templates/WorkspaceTest.lean` | 37 | deleted; the generator supplies its own workspace test | -| `scripts/leaneval_interface.py` | 293 | replaced by an import from the pinned package, to the extent its types match | - -That is 428 lines deleted outright and 293 more replaced. Nothing in -`scripts/fc_leaneval_importer.py` changes, and `make_comparator_workspace.py` -changes by one import. +The response is the complete workspace file map with a SHA-256 digest per +file, and every digest is checked before a byte lands on disk. + +### Provenance rides beside the request, not in it + +lean-eval#536 requires each imported problem to record the FC source commit +and declaration id. The v1 wire format has no field for either — its optional +`source` is one free-text line — so the manifest this repository always built +(`ProblemManifest`: commit, path, blob, module, declaration, copied +dependencies, the pins the hole types were read at) is written **beside** the +generated workspace as `fc-provenance.json`, and beside the emitted request as +`fc-provenance-.json`. It is also what makes regeneration possible when +Formal Conjectures corrects a misformalisation upstream. Whether v2 of the +contract should carry these fields itself is an open question for lean-eval; +see below. ## What stays Formal Conjectures' permanently -| File | Lines | Why it cannot move | -|---|---|---| -| `scripts/fc_leaneval_importer.py` | 870 | resolves a declaration against an exact FC commit, reads the elaborated environment, copies the FC-local closure, types each `answer(sorry)` slot, and records the provenance | -| `scripts/comparator_facts.lean` | 205 | the Lean extractor: source ranges, binder explicitness, and answer-slot types, all of which only this repository's elaborated environment knows | -| `scripts/test_fc_leaneval_importer.py` | 400 | every case pins a real extraction defect | -| `scripts/make_comparator_workspace.py` | 157 | the command, and the directory write that belongs to neither side | -| `scripts/test_make_comparator_workspace.py` | 99 | asserts the emitted pair rebuilds the workspace exactly | -| `comparator/problems/*.toml` | — | the one choice FC source cannot make for itself: which module, when two declare the same name | -| `comparator/tools.toml` | — | the pins, in one machine-readable place: this repository's under `[tools]`, LeanEval's under `[target]` | +| File | Why it cannot move | +|---|---| +| `scripts/fc_leaneval_importer.py` | resolves a declaration against an exact FC commit, reads the elaborated environment, copies the FC-local closure, types each `answer(sorry)` slot, and records the provenance | +| `scripts/comparator_facts.lean` | the Lean extractor: source ranges, binder explicitness, answer-slot types, and the `@[category ...]` tag, all of which only this repository's elaborated environment knows | +| `scripts/leaneval_interface.py` | the request builder and response checker — the FC side of the wire format, permanently, since the consumer owns hole resolution under the v1 contract | +| `scripts/leaneval_generator_cli.py` | plumbing for the pinned binary | +| `scripts/make_comparator_workspace.py` | the command, the emitted seam artifact, and the whole-set batch run | +| `comparator/templates/WorkspaceTest.lean` | the workspace test template the contract requires the consumer to supply | +| `comparator/problems/*.toml` | the one choice FC source cannot make for itself: which module, when two declare the same name | +| `comparator/tools.toml` | the pins, in one machine-readable place: this repository's under `[tools]`, LeanEval's under `[target]`, the generator revision under `[generator]` | + +The tests beside each file pin real defects: the importer suite covers +extraction, the interface suite covers the wire shapes, and the command suite +runs the real pinned binary end to end when one is built (CI always does). Nothing in the importer names a workspace file, a workspace layout, or an import graph. If a change to it would, the change belongs on the other side. @@ -86,20 +88,18 @@ import graph. If a change to it would, the change belongs on the other side. ## Not built, on purpose **Disproof support.** Blocked upstream: Comparator has no interface for a -plain-statement disproof. Nothing here anticipates one. +plain-statement disproof, and the overhaul plan defers it to the +open-conjectures phase. Nothing here anticipates one. -**Multi-file Challenge support.** The generator already carries a statement's -whole closure in `ChallengeDeps`, which is one file. Splitting that closure -across several trusted files is a generator-side change: the importer would -hand over the same declarations, and only the `dependencies` region's shape -would have to say how they group. lean-eval#536 asks for this to be scoped -against the actual FC100 statements rather than in the abstract, so it is not -built here. +**Multi-file Challenge support.** The generator carries a statement's whole +closure in `ChallengeDeps`, which is one file. Measured over `FC100OpenSet1`, +no statement needs another FC problem module, so this does not block the +first import. **A vendored workspace.** A workspace checked into this repository is a copy -of generator output, so it drifts from the generator, and it says nothing about -the importer because a human wrote it. The Lean 4.33 evidence comes from -generating one in CI instead. +of generator output, so it drifts from the generator, and it says nothing +about the importer because a human wrote it. The Lean 4.33 evidence comes +from generating one in CI instead. **Lifecycle.** Result records, resubmission, and revision tracking are LeanEval's, per lean-eval#536. This repository regenerates and opens a pull @@ -107,65 +107,34 @@ request; it keeps no state about what happened to one. ## What this side cannot settle alone -Each of these is a place where the interface above is a guess that lean-eval -has to confirm or replace. None of them is blocking the FC work; all of them -would change bytes at the seam. - -1. **The markup convention is invented here.** `-- @region ` and the four - region names are local. The generator core is the natural owner of the - convention, since it is the reader. -2. **The manifest schema is invented here, and part of it need not be.** - `schema_version = 1` and the field names are this repository's. lean-eval#536 - says the importer emits PRs that lean-eval CI validates like any other - problem PR, which needs something published to validate against. The two - fields the plan does name — the FC source commit and the declaration id — - are present under `source.commit` and `source.declaration`. - - `mathlib-initiative/formalization.yaml` already standardises much of this: - `repository.substantive_formalization` carries a source repository and - revision, `status.main_results[]` carries a declaration, its file, its - permitted axioms and its Comparator config. Formal Conjectures does not - currently carry that file, but `Paul-Lez/hadamard-668-comparator` uses it to - describe a wrapper around FC at revision `1721605c`. - - It is not a drop-in replacement, for two reasons worth stating rather than - glossing. Its required `project`, `sources`, `automation` and `review` - sections describe who formalised something, from what, with what help, and - who reviewed it — an importer cannot fill those truthfully for an arbitrary - FC statement, because they belong to the FC contributor rather than to the - import. And it deliberately omits pins, on the stated grounds that the file - sits alongside the formalization it describes and the tree already encodes - them; a generated workspace has two pin sets and sits alongside neither. - - So the open question is not "publish a schema" but which object is which: a - manifest that drives generation and crosses the seam, and possibly a - `formalization.yaml` describing the generated workspace as a thin wrapper - once it exists somewhere durable. Nothing here implements the second, since - filling its required sections without a real answer would be worse than - omitting it. -3. **The `definition_names` config field is undocumented.** Comparator's - published no-hole config does not carry it, and hole support depends on the - comparator commit pinned in `tools.toml`. A generated workspace with an - `answer(sorry)` hole is only checkable against that build. -4. **Answer-slot types are read under this repository's toolchain.** The - importer asks Formal Conjectures' elaborated environment, at FC's Lean and - Mathlib pins, for the type of each slot; the workspace is built at - LeanEval's Lean 4.33 and its own Mathlib. A type whose name or elaboration - differs between the two revisions would be wrong in a way `--verify` cannot - see, because `--verify` also runs at FC's pins. - - `.github/workflows/comparator-lean-4-33.yml` now does both halves in one - job: it generates at 4.27 and builds and Comparator-checks at 4.33, on one - plain theorem and one `Prop`-valued `answer(sorry)` slot. So the gap is - observed rather than asserted, and every manifest states it — - `source.lean_toolchain` against `target.lean_toolchain`. What is still open - is the general case: two declarations passing says nothing about a slot - whose type name changed between the two Mathlib revisions. A frozen-set - import needs that job over the whole set, and the decision about which side - owns the answer when they disagree is lean-eval's. -5. **Who triggers regeneration is unassigned.** The plan gives the importer the - duty to regenerate and re-PR when Formal Conjectures fixes a - misformalisation upstream, and gives lifecycle to LeanEval. Nothing yet says - which side watches FC commits for a change to an imported declaration. The - manifest records what is needed to answer the question — commit, path, blob - and declaration — but nobody is asking it. +1. **Provenance fields in the contract.** v1 has no home for the FC source + commit and declaration id that §10 requires by name, so they travel as a + sidecar. A v2 passthrough or provenance field would let a generated + workspace carry its own origin. Related: + `mathlib-initiative/formalization.yaml` already standardises a source + repository, revision, declaration and Comparator config — + `Paul-Lez/hadamard-668-comparator` uses it to describe a wrapper around FC + at `1721605c` — but its required `project`, `sources`, `automation` and + `review` sections belong to whoever formalised the statement rather than + to an import, and it deliberately omits pins. So the question is which + object is which, not whether to publish a schema. +2. **The `definition_names` config field is undocumented.** Comparator's + published no-hole config does not carry it, and hole support depends on + the comparator commit pinned in `tools.toml`. A generated workspace with + an `answer(sorry)` hole is only checkable against that build. +3. **Answer-slot types are read under this repository's toolchain.** The + importer asks Formal Conjectures' elaborated environment, at FC's pins, + for the type of each slot; the workspace is built at LeanEval's. The + overhaul plan assigns the re-resolution to LeanEval — the consumer + re-resolves hole metadata under its own target environment — and the + whole-set audit run is what observes the gap meanwhile: it generates at + FC's pins and compiles every generated workspace at LeanEval's, with the + failures recorded by name in `comparator/known_failures.toml` and + asserted exactly. Which side owns the answer when the two environments + disagree about a type is lean-eval's call. +4. **Who triggers regeneration is unassigned.** The plan gives the importer + the duty to regenerate and re-PR when Formal Conjectures fixes a + misformalisation upstream, and gives lifecycle to LeanEval. Nothing yet + says which side watches FC commits for a change to an imported + declaration. The provenance sidecar records what is needed to answer the + question — commit, path, blob and declaration — but nobody is asking it. diff --git a/comparator/README.md b/comparator/README.md index 9e143db972..6f695ebc2d 100644 --- a/comparator/README.md +++ b/comparator/README.md @@ -10,10 +10,26 @@ coordination tracked in [`formal-conjectures#4930`](https://github.com/google-deepmind/formal-conjectures/issues/4930). **[`OWNERSHIP.md`](OWNERSHIP.md) is the map**: which code is Formal -Conjectures' permanently, which code is standing in for -`leanprover/lean-eval-generator` and is deleted when that lands, what crosses -between them, and what the interface still needs from lean-eval. Read it first. -This file is the operator's page: the commands, their inputs, and the pins. +Conjectures' permanently, what crosses the seam to the pinned +[`leanprover/lean-eval-generator`](https://github.com/leanprover/lean-eval-generator), +and what the interface still needs from lean-eval. Read it first. This file is +the operator's page: the commands, their inputs, and the pins. + +## The generator binary + +Workspace generation runs the extracted generator at the revision `tools.toml` +pins under `[generator]`. Build it once — the package depends on nothing, so +this is quick — and point the importer at it: + +```bash +git clone https://github.com/leanprover/lean-eval-generator /tmp/lean-eval-generator +git -C /tmp/lean-eval-generator checkout "$(python3 -c ' +import tomllib; print(tomllib.load(open("comparator/tools.toml","rb"))["generator"]["rev"])')" +(cd /tmp/lean-eval-generator && lake build) +export LEAN_EVAL_GENERATOR_BIN=/tmp/lean-eval-generator/.lake/build/bin/lean-eval-generator +``` + +Import and `--verify` are offline as before; only generation needs the binary. ## Two toolchains @@ -25,8 +41,9 @@ does not require a repository-wide toolchain upgrade here. So the importer reads a declaration's source range, binders, dependencies and `answer(sorry)` slot types from an environment elaborated at *this* repository's toolchain, and the workspace it produces is pinned to *LeanEval's* -toolchain and Mathlib. `manifest.json` records both pin sets, under `source` -and `target`. `.github/workflows/comparator-lean-4-33.yml` generates a +toolchain and Mathlib. The request carries LeanEval's pins; the provenance +sidecar `fc-provenance.json` records the pins the hole types were read at. +`.github/workflows/comparator-lean-4-33.yml` generates a workspace here and builds and Comparator-checks it there, in one job, which is what turns the gap between them into something observed rather than assumed. @@ -52,7 +69,8 @@ The workspace contains `ChallengeDeps.lean` with the statement's copied Formal Conjectures closure, `Challenge.lean` with the trusted statement and its proof hole, `Submission.lean` and `Submission/` where a solver works, `Solution.lean` connecting the two, `config.json` with the theorem targets, definition targets -and permitted axioms, and `manifest.json`. `Solution.lean` is fixed: it fails +and permitted axioms, `holes.json`, and the `fc-provenance.json` sidecar this +side adds beside the generator's files. `Solution.lean` is fixed: it fails to build if the submission changes the statement. Comparator rejects `sorryAx`, because it is not in the permitted axiom list. @@ -63,9 +81,24 @@ python3 scripts/make_comparator_workspace.py erdos_1038.parts.i \ --emit-import .comparator-import ``` -This writes `Problem.lean` and `manifest.json` and generates no workspace. It -is the pair the importer contributes to a LeanEval problem pull request once -the shared generator is a pinned dependency there. +This writes the exact bytes that cross the seam — `request.json`, the +`context/` directory the v1 contract reads, and the provenance sidecar — and +generates no workspace. Running the pinned binary on that request from inside +the emitted directory yields the same file map generation would have written, +which is what makes the seam checkable rather than asserted. + +### Import a whole set + +```bash +python3 scripts/make_comparator_workspace.py --set FC100OpenSet1 \ + --verify --report fc100-report.json \ + --known-failures comparator/known_failures.toml +``` + +One request carries every declaration that imports; failures are recorded per +declaration in the report instead of aborting the run. With +`--known-failures`, the run fails unless the failures are exactly the recorded +ones — an unexpected failure and a silently fixed one both count. ### Supported inputs @@ -84,8 +117,8 @@ safely, and existing output. `problems/*.toml` is an input, not the LeanEval manifest: it records the choices this repository's Lean source cannot make for itself, and the importer -reads it. The manifest the generator receives, and writes into the workspace as -`manifest.json`, is derived. +reads it. The request the generator receives, and the provenance sidecar, are +derived. Most declarations need no problem file. Add one TOML file under `problems/` only when two files declare the same name, which is the one thing the Lean @@ -116,9 +149,10 @@ python3 scripts/make_comparator_workspace.py --validate `tools.toml` is the one machine-readable source. `[tools]` are the revisions a local run uses under this repository's toolchain. `[target]` are LeanEval's: the Lean toolchain and Mathlib revision every generated workspace is pinned to, -and the Comparator and `lean4export` commits that check it. Every manifest -records `[target]` beside the source pins. Generation itself does not run -Comparator. +and the Comparator and `lean4export` commits that check it. `[generator]` is +the extracted generator revision every request is written against; bumping it +is a contract change and has to survive the seam round-trip test. Generation +itself does not run Comparator. ## Conformance before a public import From 7d894df383eddeefbb7bd85c354780f9e0023e78 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:39:22 -0400 Subject: [PATCH 24/70] Flatten dotted declaration names for the generator's single-identifier convention The generator anchors every text operation on the declaration's last name component, because lean-eval sources always declare a plain identifier inside a namespace. A dotted FC name like erdos_1038.parts.i would come out as 'theorem i'; restating it under its slug keeps the name meaningful and anchorable, and the provenance sidecar records the FC name. --- scripts/fc_leaneval_importer.py | 40 ++++++++++++++++++++++++++-- scripts/test_fc_leaneval_importer.py | 27 +++++++++++++++++++ 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/scripts/fc_leaneval_importer.py b/scripts/fc_leaneval_importer.py index 4ad10864d2..9b94a193cb 100644 --- a/scripts/fc_leaneval_importer.py +++ b/scripts/fc_leaneval_importer.py @@ -519,6 +519,30 @@ def covered_by_another(dep): ), provenance +def flatten_declared_name(declared, statement): + """Restate a dotted declaration name as its slug, in the statement text. + + Returns `(new_name, new_statement)`. Only the declaring occurrence is + rewritten — a statement does not reference its own name — and the + rewrite is refused rather than guessed if the name cannot be found where + the declaration keyword put it. + """ + from leaneval_interface import slug + + flattened = slug(declared) + lines = statement.split("\n") + for index, line in enumerate(lines): + match = DECL_START.match(line) + if not match: + continue + name = re.match(r"\s*([\w.«»]+)", line[match.end() :]) + if name and name.group(1) == declared: + start = match.end() + name.start(1) + lines[index] = line[:start] + flattened + line[start + len(declared) :] + return flattened, "\n".join(lines) + raise SystemExit(f"{declared}: cannot find the declaring occurrence to rename") + + def replace_proof_with_sorry(text): """Cut the proof body after `:=`, keeping the statement. @@ -823,6 +847,15 @@ def import_problem(problem, answer_type=None, module=None): break if declared is None: raise SystemExit(f"{declaration}: no declaration line in the slice") + original_declared = declared + if "." in declared: + # The generator anchors on the declaration's last name component — + # its own sources always declare a plain identifier inside a + # namespace — so a dotted name like `erdos_100.variants.strong` + # would come out as `theorem strong`, and `parts.i` as `theorem i`. + # Restate the declaration under its slug instead: single identifier, + # still meaningful, and the provenance sidecar records the FC name. + declared, statement = flatten_declared_name(declared, statement) statement, holes = hoist_answers( statement, declared, facts.get("answerTypes", []), answer_type ) @@ -854,7 +887,10 @@ def import_problem(problem, answer_type=None, module=None): statement=statement, dependency_declarations=tuple(copied), ) - qualified = ".".join(namespaces_at_target + [declared]) + # The FC name, under the namespaces the source declared it in; the + # workspace statement may carry the flattened `declared` instead, and + # this is what ties the two together. + qualified = ".".join(namespaces_at_target + [original_declared]) manifest = ProblemManifest( # The default id is the qualified name: two modules declaring # `conjecture` in different namespaces must not share a workspace. @@ -865,7 +901,7 @@ def import_problem(problem, answer_type=None, module=None): holes=tuple(holes), permitted_axioms=PERMITTED_AXIOMS, source=source_record( - declared, + qualified, fc_module, path.relative_to(ROOT), fc_rev, diff --git a/scripts/test_fc_leaneval_importer.py b/scripts/test_fc_leaneval_importer.py index e26b1a627e..e030e7a2a6 100644 --- a/scripts/test_fc_leaneval_importer.py +++ b/scripts/test_fc_leaneval_importer.py @@ -529,3 +529,30 @@ def test_api_and_untagged_declarations_are_refused(self): with self.subTest(category=category): with self.assertRaises(SystemExit): importer.problem_group(self._manifest(category)) + + +class FlattenDeclaredNameTest(unittest.TestCase): + """Dotted declaration names are restated as slugs for the generator.""" + + def test_the_declaring_occurrence_is_renamed(self): + name, statement = importer.flatten_declared_name( + "erdos_100.variants.strong", + "theorem erdos_100.variants.strong : True := by\n sorry", + ) + self.assertEqual(name, "erdos_100_variants_strong") + self.assertEqual( + statement, "theorem erdos_100_variants_strong : True := by\n sorry" + ) + + def test_a_prefix_line_does_not_confuse_the_rename(self): + # `open X in` binds to the declaration below and travels with the + # slice; the declaring line is not the first line. + name, statement = importer.flatten_declared_name( + "a.b", "open Nat in\ntheorem a.b : True := by\n sorry" + ) + self.assertEqual(name, "a_b") + self.assertIn("theorem a_b :", statement) + + def test_an_absent_declaration_is_refused(self): + with self.assertRaises(SystemExit): + importer.flatten_declared_name("a.b", "theorem c.d : True := sorry") From 1c035d50a57c15c83a64a6265e9874551436f61e Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:57:33 -0400 Subject: [PATCH 25/70] Fix the eleven whole-set source-extraction failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The audit on the PR found 11 of the 100 set declarations failing before target generation. Each root cause, and its fix here: - File-scoped notation and macros were dropped: KEEP_LOOSE now carries 'local notation', 'local macro' and friends with their indented bodies, which fixes Irrational (local notation e), Poincare (the 𝕊ⁿ macro), Ramsey (the R(k,l) notation) and Erdos125 (local notation A/B). - 'noncomputable section' was not recognised as a section or a mode: restating it fixes OpenQuantumProblem23's copied definitions. - FC-defined notation from other modules is text, not a constant, so the elaborated closure never reports it: notation commands from the shared trees are now copied when their token appears in the module — scoped ones only where the module opens their namespace, which is what keeps a modal-logic ⊆ overload out of every statement about sets. Fixes Erdos92 and Poincare (scoped ℝ², ℝ^) and Green9 (the global ≪). - A copied preamble may open a namespace before anything declares it: empty namespace blocks are now created up front, generally, fixing EllipticCurveRank and Koethe. - A statement's own match auxiliaries have the statement as ancestor and are regenerated when it re-elaborates: Erdos324 no longer fails closed. - '(answer(sorry) : T)' loses its annotation during elaboration, so the ascription is now read at its own position, fixing Erdos332's slot that the erasure rule would have called Prop. Erdos890 fell out of the preamble fixes. The remaining failure from the audit, Erdos1092 at target pins, is Mathlib instance drift recorded in comparator/known_failures.toml until the repository's 4.33 bump. --- scripts/fc_leaneval_importer.py | 362 ++++++++++++++++++++++----- scripts/test_fc_leaneval_importer.py | 112 ++++++++- 2 files changed, 416 insertions(+), 58 deletions(-) diff --git a/scripts/fc_leaneval_importer.py b/scripts/fc_leaneval_importer.py index 9b94a193cb..77ca26d740 100644 --- a/scripts/fc_leaneval_importer.py +++ b/scripts/fc_leaneval_importer.py @@ -11,9 +11,9 @@ What it produces is the pair defined in `scripts/leaneval_interface.py`: one marked-up Mathlib-only Lean module, and one manifest carrying the FC source commit and declaration id. Turning that pair into a Challenge / Solution / -Submission workspace is the generator's job, not this module's; see -`scripts/leaneval_generator.py`, which is the part that goes away when -`leanprover/lean-eval-generator` is extracted. +Submission workspace is the pinned `leanprover/lean-eval-generator` binary's +job, not this module's; `leaneval_interface.build_request` is where the pair +becomes that binary's input. Nothing here writes a workspace file, names a workspace layout, or decides which generated module imports which. If a change to this file would do one of @@ -57,7 +57,14 @@ r"(theorem|lemma|def|abbrev|structure|inductive|instance|notation)\s", ) KEEP_LOOSE = re.compile( - r"^(open|variable|universe|section|namespace|end|attribute|set_option)\b" + # `local notation`, `local macro` and friends scope to the file exactly + # like `open` does, and a statement that names what they define does not + # parse without them. `noncomputable section` is a section for the scope + # stack and a compilation mode for everything inside it. + r"^(?:(?:local|scoped)\s+)?" + r"(open|variable|universe|section|namespace|end|attribute|set_option" + r"|notation|postfix|prefix|infixl|infixr|infix|macro|syntax|macro_rules)\b" + r"|^noncomputable section\b" ) @@ -123,25 +130,48 @@ def file_scoped_preamble(lines, start_line): and its scope still encloses it. """ stack, preamble, depth = [], [], 0 - for line in lines[: start_line - 1]: + lines = lines[: start_line - 1] + index = 0 + while index < len(lines): + line = lines[index] if depth == 0 and KEEP_LOOSE.match(line) and not line.rstrip().endswith(" in"): kind = line.split()[0] - parts = line.split(None, 1) - name = parts[1].strip() if len(parts) > 1 else None + if kind == "noncomputable": + # `noncomputable section [name]` opens a section. + kind = "section" + parts = line.split(None, 2) + name = parts[2].strip() if len(parts) > 2 else None + else: + parts = line.split(None, 1) + name = parts[1].strip() if len(parts) > 1 else None if kind in ("namespace", "section"): - stack.append((kind, name)) + stack.append((kind, name, line)) elif kind == "end": if stack and ( stack[-1][1] == name or (name is None and stack[-1][0] == "section") ): stack.pop() else: - preamble.append((line, list(stack))) + # A `macro` or `notation` body may continue on indented + # lines; a single kept line would be broken syntax. + text = [line] + while index + 1 < len(lines) and ( + lines[index + 1][:1].isspace() and lines[index + 1].strip() + ): + index += 1 + text.append(lines[index]) + preamble.append(("\n".join(text), list(stack))) depth += len(re.findall(r"/-", line)) - len(re.findall(r"-/", line)) depth = max(depth, 0) + index += 1 scope = list(stack) in_force = [text for text, s in preamble if s == scope[: len(s)]] - return in_force, [n for k, n in scope if k == "namespace" and n] + # A statement inside `noncomputable section` restates the mode, since the + # copy has left the section behind and a noncomputable definition in the + # statement's closure would otherwise fail to compile. + if any(k == "section" and line.startswith("noncomputable") for k, _, line in scope): + in_force.append("noncomputable section") + return in_force, [n for k, n, _ in scope if k == "namespace" and n] def load_manifest(problem_id): @@ -411,7 +441,9 @@ def slice_range(lines, source_range): return "\n".join(sliced), lo -def closure_region(dependencies, generated, declaration, opened_namespaces=()): +def closure_region( + dependencies, generated, declaration, opened_namespaces=(), target_name=None +): """A declaration's FC-local closure, copied, needing Mathlib and nothing else. lean-eval vendors problems, so a generated Challenge cannot fetch this @@ -427,10 +459,15 @@ def closure_region(dependencies, generated, declaration, opened_namespaces=()): marked-up module, which `--verify` does. """ copied = [dep["name"] for dep in dependencies] + # The statement's own `match` and `proof` auxiliaries have the statement + # as their ancestor, and the statement is restated in the workspace, so + # re-elaborating it regenerates them; only an auxiliary of something not + # being copied at all is unreachable. + ancestors = copied + ([target_name] if target_name else []) orphans = [ name for name in generated - if not any(name.startswith(parent + ".") for parent in copied) + if not any(name.startswith(parent + ".") for parent in ancestors) ] if orphans: raise SystemExit( @@ -471,6 +508,46 @@ def covered_by_another(dep): dependencies = [dep for dep in dependencies if dep["name"] not in subsumed] blocks, provenance = [], [] + # `open X` on a namespace nothing has declared yet is an error, and a + # copied preamble may open a namespace whose declaring block comes later + # in the copy, or never: with the problem's module no longer imported, + # only the copy itself can make a name exist. An empty namespace block + # up front is enough, and creating one that a later declaration fills is + # harmless. This covers the statement's own namespace stack and every + # namespace a copied preamble opens. + created = [] + for dep in dependencies: + if dep["range"] is None: + continue + dep_path = module_source_path(dep["module"]) + dep_lines = dep_path.read_text(encoding="utf-8").split("\n") + dep_preamble, dep_namespaces = file_scoped_preamble( + dep_lines, slice_range(dep_lines, dep["range"])[1] + ) + for entry in dep_preamble: + words = entry.split("\n")[0].split() + if not words or words[0] != "open": + continue + for word in words[1:]: + if word == "scoped": + continue + if not re.fullmatch(r"[\w.«»]+", word): + break + created.append(word) + created.extend( + ".".join(dep_namespaces[: depth + 1]) + for depth in range(len(dep_namespaces)) + ) + created.extend( + ".".join(opened_namespaces[: depth + 1]) + for depth in range(len(opened_namespaces)) + ) + seen_namespaces = set() + for namespace in created: + if namespace in seen_namespaces: + continue + seen_namespaces.add(namespace) + blocks.append(f"namespace {namespace}\nend {namespace}") for dep in dependencies: if dep["range"] is None: raise SystemExit(f"{declaration}: {dep['name']} has no source range") @@ -493,21 +570,6 @@ def covered_by_another(dep): blocks.append("\n".join(chunk)) provenance.append((dep["name"], body)) - # The statement reopens the namespace stack the target sat in, so it can - # name siblings short. `open` on a namespace nothing has declared is an - # error, and with the problem's module no longer imported only the copied - # declarations can declare one. An empty namespace block is enough to make - # the name exist. - declared_namespaces = { - name.rsplit(".", 1)[0] for name, _ in provenance if "." in name - } - for depth in range(len(opened_namespaces)): - prefix = ".".join(opened_namespaces[: depth + 1]) - if not any( - ns == prefix or ns.startswith(prefix + ".") for ns in declared_namespaces - ): - blocks.append(f"namespace {prefix}\nend {prefix}") - listing = "\n".join(f"* `{name}`" for name, _ in provenance) return ( "/-!\n" @@ -519,6 +581,107 @@ def covered_by_another(dep): ), provenance +NOTATION_COMMAND = re.compile( + r"^(?:@\[[^\]]*\]\s*)?(?:scoped\[[\w.«»]+\]\s+)?(?:scoped\s+)?" + r"(?:notation[0-9]*|postfix|prefix|infixl|infixr|infix)[:\s]" +) + +_NOTATION_CACHE = None + + +def fc_notation_commands(): + """Every exportable notation command an FC module defines, with its token. + + A notation is not a constant, so the elaborated closure never reports it: + a statement written as `ℝ²` names `EuclideanSpace ℝ (Fin 2)` in the + environment and `ℝ²` only in its text. The copy carries the text, so the + commands that make such tokens parse have to be found at the text layer. + `local` notations are file-scoped at their origin and cannot be in force + in a problem file, so they are not candidates. + + Returns `[(tokens, command, namespaces)]`, where `tokens` are the + command's string literals that contain a non-ASCII character — the + distinctive ones worth matching on — and `namespaces` is the stack a + plain `scoped` command needs restated around it. + """ + global _NOTATION_CACHE + if _NOTATION_CACHE is not None: + return _NOTATION_CACHE + commands = [] + roots = [ROOT / "FormalConjecturesForMathlib", ROOT / "FormalConjecturesUtil"] + for src in roots + SOURCE_DIRS: + for path in sorted(src.rglob("*.lean")): + lines = path.read_text(encoding="utf-8").split("\n") + for index, line in enumerate(lines): + if not NOTATION_COMMAND.match(line): + continue + text = [line] + follow = index + 1 + while follow < len(lines) and ( + lines[follow][:1].isspace() and lines[follow].strip() + ): + text.append(lines[follow]) + follow += 1 + command = "\n".join(text) + tokens = [ + token + for token in re.findall(r'"([^"]+)"', command) + if any(ord(c) > 127 for c in token) + ] + if not tokens: + continue + bracket = re.match(r"^(?:@\[[^\]]*\]\s*)?scoped\[([\w.«»]+)\]", line) + if bracket: + scope = bracket.group(1) + elif re.match(r"^scoped\s", line): + _, namespaces = file_scoped_preamble(lines, index + 1) + scope = ".".join(namespaces) + else: + scope = None + # A global notation in FormalConjecturesForMathlib or + # FormalConjecturesUtil is in force in every problem file, + # which imports both; one in a problem module is not, since + # problem files do not import each other, and the problem + # file's own notations travel with the preamble. + shared = src.name != "FormalConjectures" + commands.append((tokens, command, scope, shared)) + _NOTATION_CACHE = commands + return commands + + +def notation_blocks(module_texts, opened): + """The FC notation commands the module's text uses, as copyable blocks. + + A token match alone over-copies: `⊆` from a modal-logic module matched + every statement about sets. A scoped notation can only have been in + force in the source file if its namespace is among the file's opens, so + `opened` — the namespaces the module's scope and copied preambles open — + gates every scoped command. A global `notation` in a module nothing + imports was never in force either, but the corpus keeps global notation + in the problem file itself, which the preamble already carries, so + unscoped commands from other files are not candidates at all. + """ + combined = "\n".join(module_texts) + blocks, seen = [], set() + for tokens, command, scope, shared in fc_notation_commands(): + if scope: + if scope not in opened: + continue + elif not shared: + continue + if command in seen or command in combined: + continue + if not any(token in combined for token in tokens): + continue + seen.add(command) + # A plain `scoped` command needs its namespace restated around it; + # the bracket form and a global command carry their own scope. + if scope and not command.startswith("scoped["): + command = f"namespace {scope}\n{command}\nend {scope}" + blocks.append(command) + return blocks + + def flatten_declared_name(declared, statement): """Restate a dotted declaration name as its slug, in the statement text. @@ -691,15 +854,49 @@ def unwrap_answers(statement): return statement +def _ascribed_type(statement, start, end): + """The `T` of `(answer(sorry) : T)`, when the slot is written that way. + + A type ascription is the one place the surface syntax states a slot's + type at its position, and it matters because the elaborated environment + can lose the annotation for exactly this shape: the ascribed term is + applied or rewritten during elaboration and the metadata does not + survive into the stored statement type. + """ + before = statement[:start].rstrip() + if not before.endswith("("): + return None + index = end + while index < len(statement) and statement[index].isspace(): + index += 1 + if index >= len(statement) or statement[index] != ":": + return None + index += 1 + depth, cursor = 1, index + while cursor < len(statement): + char = statement[cursor] + if char == "(": + depth += 1 + elif char == ")": + depth -= 1 + if depth == 0: + ascribed = statement[index:cursor].strip() + return ascribed or None + cursor += 1 + return None + + def hoist_answers(statement, basename, slot_types, override=None): """Replace each `answer(sorry)` with a named definition hole. - The slot types come from the elaborated environment, where the `answer` - elaborator ran with the expected type in hand; the old surface-syntax - guess (an `↔` beside the slot means `Prop`) and the FC problem file's - hand-kept `answer_type` both survive only as overrides. Slots of different - types in one statement are refused: the environment reports the types as a - set, and matching them to positions would be a guess. + A slot written `(answer(sorry) : T)` states its own type at its own + position, and that reading wins. For the rest, the types come from the + elaborated environment, where the `answer` elaborator ran with the + expected type in hand; the old surface-syntax guess (an `↔` beside the + slot means `Prop`) and the FC problem file's hand-kept `answer_type` + both survive only as overrides. Unascribed slots of differing types are + refused: the environment reports the types as a set, and matching them + to positions would be a guess. """ holes = [] calls = answer_spans(statement) @@ -707,31 +904,48 @@ def hoist_answers(statement, basename, slot_types, override=None): count = len(selected) if count == 0: return statement, holes - # Under the default `alwaysTrue` setting, the `answer` elaborator erases a - # slot to `True` if and only if its expected type is `Prop` - # (FormalConjecturesUtil/Answer.lean). So a slot the environment carries - # no annotation for is a `Prop` slot by the elaborator's own rule, not by - # guesswork, and no postpone build is needed. - missing = count - len(slot_types) + types = [None] * count if override: types = [override] * count - elif missing == count: - types = ["Prop"] * count - elif missing == 0 and len(set(slot_types)) == 1: - types = [slot_types[0]] * count - elif missing == 0: - raise SystemExit( - f"{basename} has {count} answer slots of differing types " - f"{slot_types}; pass --answer-type" - ) else: - # Some slots are Prop and some are not: which positions are which - # cannot be read off an unordered set, so refuse rather than assign. - raise SystemExit( - f"{basename}: {missing} Prop slot(s) and {len(slot_types)} typed " - f"slot(s) {slot_types} cannot be matched to positions; pass " - "--answer-type" - ) + remaining_env = list(slot_types) + for i, (start, end, _argument) in enumerate(selected): + ascribed = _ascribed_type(statement, start, end) + if ascribed is not None: + types[i] = ascribed + # The environment may have reported this slot too; retire one + # matching entry so the counting below stays honest. + if ascribed in remaining_env: + remaining_env.remove(ascribed) + remaining = [i for i in range(count) if types[i] is None] + # Under the default `alwaysTrue` setting, the `answer` elaborator + # erases a slot to `True` if and only if its expected type is `Prop` + # (FormalConjecturesUtil/Answer.lean). So a slot the environment + # carries no annotation for is a `Prop` slot by the elaborator's own + # rule, not by guesswork, and no postpone build is needed. + missing = len(remaining) - len(remaining_env) + if missing == len(remaining): + for i in remaining: + types[i] = "Prop" + elif missing == 0 and remaining and len(set(remaining_env)) == 1: + for i in remaining: + types[i] = remaining_env[0] + elif missing == 0 and not remaining: + pass + elif missing == 0: + raise SystemExit( + f"{basename} has {len(remaining)} answer slots of differing " + f"types {remaining_env}; pass --answer-type" + ) + else: + # Some slots are Prop and some are not: which positions are which + # cannot be read off an unordered set, so refuse rather than + # assign. + raise SystemExit( + f"{basename}: {missing} Prop slot(s) and {len(remaining_env)} " + f"typed slot(s) {remaining_env} cannot be matched to " + "positions; pass --answer-type" + ) replacements = [] for i, (start, end, _argument) in enumerate(selected): name = f"{basename}_answer" if count == 1 else f"{basename}_answer_{i + 1}" @@ -835,6 +1049,7 @@ def import_problem(problem, answer_type=None, module=None): facts.get("generatedDependencies", []), declaration, namespaces_at_target, + target_name=facts.get("name"), ) statement = strip_decorations(statement) @@ -880,9 +1095,44 @@ def import_problem(problem, answer_type=None, module=None): ] mathlib_rev, fc_rev = pins(path.relative_to(ROOT)) + scope_text = "\n".join(opens + preamble) + # Notation is text, not a constant: a statement or copied declaration + # spelled with an FC-defined token needs the defining command copied too, + # and the elaborated closure cannot say so. + # Namespaces the module opens, at its scope and inside every copied + # block: a scoped notation can only have been in force where one of + # these opens it. + opened_for_notation = set() + for line in (dependencies + "\n" + scope_text).split("\n"): + words = line.split() + if words[:1] == ["open"]: + opened_for_notation.update(w for w in words[1:] if w != "scoped") + notations = notation_blocks( + [dependencies, scope_text, statement], opened_for_notation + ) + if notations: + # A notation whose right-hand side names a copied declaration must + # come after the block declaring it; every other notation comes + # first, because copied declarations may use its token textually. A + # single notation needing both would need interleaving; none does, + # and `--verify` is what says so. + copied_last_components = {name.rsplit(".", 1)[-1] for name, _ in copied} + before, after = [], [] + for block in notations: + rhs = block.split("=>", 1)[-1] + names = set(re.findall(r"[\w«»'.]+", rhs)) + names |= {name.rsplit(".", 1)[-1] for name in names} + if names & copied_last_components: + after.append(block) + else: + before.append(block) + if before: + dependencies = "\n\n".join(before) + "\n\n" + dependencies + if after: + dependencies = dependencies + "\n\n" + "\n\n".join(after) marked_up = MarkedUpModule( dependencies=dependencies, - scope="\n".join(opens + preamble), + scope=scope_text, holes="\n\n".join(hole.declaration() for hole in holes), statement=statement, dependency_declarations=tuple(copied), diff --git a/scripts/test_fc_leaneval_importer.py b/scripts/test_fc_leaneval_importer.py index e030e7a2a6..7f69521893 100644 --- a/scripts/test_fc_leaneval_importer.py +++ b/scripts/test_fc_leaneval_importer.py @@ -369,7 +369,7 @@ def test_an_opened_namespace_no_dependency_declares_is_created(self): out, _copied = closure_region([], [], "grimm_conjecture", ["Grimm"]) self.assertIn("namespace Grimm\nend Grimm", out) - def test_a_namespace_a_dependency_declares_is_not_restated(self): + def test_namespaces_exist_before_any_copied_block_opens_them(self): deps = [ { "name": "Grimm.helper", @@ -386,7 +386,12 @@ def test_a_namespace_a_dependency_declares_is_not_restated(self): source.write_text("def Grimm.helper := 1\n", encoding="utf-8") resolve.return_value = source out, _copied = closure_region(deps, [], "t", ["Grimm"]) - self.assertNotIn("namespace Grimm\nend Grimm", out) + # The empty block that makes the namespace exist comes before any + # copied block: a copied preamble may `open` it before anything + # declares it. Redundant creation is harmless. + self.assertLess( + out.index("namespace Grimm\nend Grimm"), out.index("def Grimm.helper") + ) def test_the_closure_region_does_not_carry_the_import(self): # `import Mathlib` belongs to the module as a whole, and the generator @@ -556,3 +561,106 @@ def test_a_prefix_line_does_not_confuse_the_rename(self): def test_an_absent_declaration_is_refused(self): with self.assertRaises(SystemExit): importer.flatten_declared_name("a.b", "theorem c.d : True := sorry") + + +class PreambleNotationTest(unittest.TestCase): + """File-scoped notation and macros travel with the preamble.""" + + def test_local_notation_is_kept(self): + # Irrational.lean: dropping `local notation "e" => exp 1` left `e` + # to auto-bind as an implicit at FC pins and fail at LeanEval's. + lines = ['local notation "e" => exp 1', "theorem t : True := trivial"] + pre, _ = file_scoped_preamble(lines, 2) + self.assertEqual(pre, ['local notation "e" => exp 1']) + + def test_a_macro_keeps_its_indented_body(self): + # Poincare.lean: the 𝕊ⁿ macro's body is on the next line; one kept + # line would be broken syntax. + lines = [ + 'local macro:max "𝕊" noWs n:superscript(term) : term =>', + " `(Metric.sphere 0 1)", + "theorem t : True := trivial", + ] + pre, _ = file_scoped_preamble(lines, 3) + self.assertEqual(len(pre), 1) + self.assertIn("`(Metric.sphere 0 1)", pre[0]) + + def test_noncomputable_section_is_restated(self): + # OpenQuantumProblems/23: a copied def that was total inside + # `noncomputable section` fails to compile outside it. + lines = ["noncomputable section", "theorem t : True := trivial"] + pre, _ = file_scoped_preamble(lines, 2) + self.assertIn("noncomputable section", pre) + + def test_a_closed_noncomputable_section_is_not_restated(self): + lines = ["noncomputable section", "end", "theorem t : True := trivial"] + pre, _ = file_scoped_preamble(lines, 3) + self.assertNotIn("noncomputable section", pre) + + +class AscribedSlotTest(unittest.TestCase): + """`(answer(sorry) : T)` states its own type at its own position.""" + + def test_the_ascription_wins_over_the_erasure_rule(self): + # Erdos332: the annotation for an ascribed-and-applied slot does not + # survive elaboration, so the environment reports nothing and the + # erasure rule would call it Prop. + statement = ( + "theorem erdos_332 (A : Set ℕ) : " + "(answer(sorry) : Set ℕ → Prop) A → True := by\n sorry" + ) + _, holes = hoist_answers(statement, "erdos_332", []) + self.assertEqual(holes[0].type, "Set ℕ → Prop") + + def test_a_nested_paren_type_stays_whole(self): + statement = "theorem t : (answer(sorry) : (ℕ → ℕ) → Prop) f := by\n sorry" + _, holes = hoist_answers(statement, "t", []) + self.assertEqual(holes[0].type, "(ℕ → ℕ) → Prop") + + def test_an_unascribed_slot_still_follows_the_erasure_rule(self): + statement = "theorem t : answer(sorry) ↔ True := by\n sorry" + _, holes = hoist_answers(statement, "t", []) + self.assertEqual(holes[0].type, "Prop") + + +class NotationBlocksTest(unittest.TestCase): + """FC-defined notation is copied only where it was in force.""" + + def _with_commands(self, commands): + return mock.patch.object( + importer, "fc_notation_commands", return_value=commands + ) + + def test_a_scoped_notation_needs_its_namespace_opened(self): + commands = [ + (["ℝ²"], 'scoped[EuclideanGeometry] notation "ℝ²" => E', "EuclideanGeometry", True), + ] + with self._with_commands(commands): + self.assertEqual( + importer.notation_blocks(["def f : ℝ² := sorry"], {"EuclideanGeometry"}), + ['scoped[EuclideanGeometry] notation "ℝ²" => E'], + ) + # Green9's `⊆` false positive: same token, namespace never opened. + self.assertEqual( + importer.notation_blocks(["def f : ℝ² := sorry"], set()), [] + ) + + def test_a_shared_global_notation_matches_without_opens(self): + commands = [(["≪"], 'notation g " ≪ " f => IsBigO g f', None, True)] + with self._with_commands(commands): + self.assertEqual( + importer.notation_blocks(["theorem t : a ≪ b := sorry"], set()), + ['notation g " ≪ " f => IsBigO g f'], + ) + + def test_a_problem_module_global_notation_is_never_copied(self): + commands = [(["≪"], 'notation g " ≪ " f => X g f', None, False)] + with self._with_commands(commands): + self.assertEqual( + importer.notation_blocks(["theorem t : a ≪ b := sorry"], set()), [] + ) + + def test_an_unused_token_is_not_copied(self): + commands = [(["ℝ²"], 'notation "ℝ²" => E', None, True)] + with self._with_commands(commands): + self.assertEqual(importer.notation_blocks(["theorem t : True"], set()), []) From bc5a2116ef57bc00225b9780703fc0e3af01627b Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:57:40 -0400 Subject: [PATCH 26/70] Run the whole-set audit in CI and gate it on the recorded failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fc100-audit.yml is the run lean-eval#536 gates the import on, as a CI artifact: import and verify all of FC100OpenSet1 at this repository's pins, generate every workspace through the pinned binary, and compile every generated Challenge at LeanEval's pins in one shared Lake project so Mathlib builds once for the set. Failures must match comparator/known_failures.toml exactly, in both stages. Two Mathlib builds keep it off the pull-request path: it runs on demand and weekly, and the per-PR jobs cover a representative declaration for each fixed defect class instead — including the guillemet module and the qualified id collision pair the audit surfaced. --- .github/workflows/build-and-docs.yml | 85 ++++++++---- .github/workflows/fc100-audit.yml | 135 +++++++++++++++++++ comparator/known_failures.toml | 21 +++ scripts/compile_fc100_target.py | 195 +++++++++++++++++++++++++++ 4 files changed, 410 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/fc100-audit.yml create mode 100644 comparator/known_failures.toml create mode 100644 scripts/compile_fc100_target.py diff --git a/.github/workflows/build-and-docs.yml b/.github/workflows/build-and-docs.yml index 8d49da7635..23d3a3a54d 100644 --- a/.github/workflows/build-and-docs.yml +++ b/.github/workflows/build-and-docs.yml @@ -155,61 +155,94 @@ jobs: lake --wfail build rm -f FormalConjectures/All.lean + # Workspace generation runs the extracted generator at the pinned + # revision. The package depends on nothing, so this is a small Lean + # build, not a Mathlib one. + - name: Build the pinned lean-eval-generator + if: steps.mode.outputs.website_only != 'true' + run: | + read -r GEN_REPO GEN_REV <<< "$(python3 - <<'PY' + import tomllib + + with open("comparator/tools.toml", "rb") as handle: + generator = tomllib.load(handle)["generator"] + print(generator["repository"], generator["rev"]) + PY + )" + git clone "$GEN_REPO" "$RUNNER_TEMP/lean-eval-generator" + git -C "$RUNNER_TEMP/lean-eval-generator" checkout "$GEN_REV" + (cd "$RUNNER_TEMP/lean-eval-generator" && lake build) + echo "LEAN_EVAL_GENERATOR_BIN=$RUNNER_TEMP/lean-eval-generator/.lake/build/bin/lean-eval-generator" >> "$GITHUB_ENV" + # The elaborator-to-importer boundary, exercised on the oleans the # build above produced: one plain theorem, a Prop answer slot, a # non-Prop answer slot, a forall-conclusion (whose binder must not be - # applied), and explicit parameters (which must be). The comparator run - # itself needs landrun and stays in a separate Linux job. + # applied), explicit parameters (which must be), and the qualified-id + # collision pair. The comparator run itself needs landrun and stays in + # a separate Linux job. - name: Comparator generation smoke test if: steps.mode.outputs.website_only != 'true' run: | lake build comparator_facts for d in exists_hadamard_zero erdos_940.variants.large_integers \ erdos_1038.parts.i erdos_100.variants.strong \ - KotherConjecture.variants.le_KotherRadical; do + KotherConjecture.variants.le_KotherRadical \ + OeisA303656.conjecture OeisA308734.conjecture \ + curling_number_conjecture; do python3 scripts/make_comparator_workspace.py "$d" --out .comparator done - grep -q "large_integers_answer : Prop" .comparator/erdos_940_variants_large_integers/Challenge.lean - grep -q "i_answer : ENNReal" .comparator/erdos_1038_parts_i/Challenge.lean - grep -q "Submission.erdos_100.variants.strong$" .comparator/erdos_100_variants_strong/Solution.lean - grep -q "le_KotherRadical hI" .comparator/KotherConjecture_variants_le_KotherRadical/Solution.lean - - # The importer-to-generator seam, on a real declaration. `--emit-import` - # writes only what this repository owns, and feeding those bytes back - # through the generator has to reproduce the workspace exactly; if it - # does not, the pair is not the whole interface and a pinned - # `lean-eval-generator` could not be dropped in. See comparator/OWNERSHIP.md. + grep -q "large_integers_answer : Prop" .comparator/Erdos940_erdos_940_variants_large_integers/Challenge.lean + grep -q "i_answer : ENNReal" .comparator/Erdos1038_erdos_1038_parts_i/Challenge.lean + grep -q "Submission.erdos_100_variants_strong$" .comparator/Erdos100_erdos_100_variants_strong/Solution.lean + grep -q "le_KotherRadical hI" .comparator/Koethe_KotherConjecture_variants_le_KotherRadical/Solution.lean + # Two modules declare `conjecture`; qualified default ids keep the + # workspaces apart, and a guillemet module path decodes correctly. + test -d .comparator/OeisA303656_conjecture + test -d .comparator/OeisA308734_conjecture + test -d .comparator/Arxiv__0912_2382__curling_number_conjecture + + # The importer-to-generator seam, on a real declaration. + # `--emit-import` writes the exact bytes that cross it — the v1 + # request and its context directory — and running the pinned binary on + # those bytes from inside the emitted directory has to reproduce the + # workspace exactly; if it does not, the emitted artifact is not the + # whole interface. See comparator/OWNERSHIP.md. - name: Importer to generator seam if: steps.mode.outputs.website_only != 'true' run: | python3 scripts/make_comparator_workspace.py erdos_1038.parts.i \ --emit-import .comparator-import python3 - <<'PY' + import json + import os import pathlib import sys sys.path.insert(0, "scripts") - import fc_leaneval_importer as importer - import leaneval_generator as generator - from leaneval_interface import MarkedUpModule, ProblemManifest + import leaneval_generator_cli as generator_cli + from leaneval_interface import ProblemManifest - handed_over = pathlib.Path(".comparator-import/erdos_1038_parts_i") - module = MarkedUpModule.parse( - (handed_over / "Problem.lean").read_text(encoding="utf-8") - ) + handed_over = pathlib.Path( + ".comparator-import/Erdos1038_erdos_1038_parts_i" + ).resolve() manifest = ProblemManifest.from_json( - (handed_over / "manifest.json").read_text(encoding="utf-8") + (handed_over / "fc-provenance-Erdos1038_erdos_1038_parts_i.json") + .read_text(encoding="utf-8") ) assert len(manifest.source.commit) == 40, manifest.source.commit assert manifest.source.declaration, "no FC declaration id" - workspace = pathlib.Path(".comparator/erdos_1038_parts_i") - # The consumer's pins are its own; locally that is comparator/tools.toml. - regenerated = generator.generate(module, manifest, importer.target_pins()) - for name, content in regenerated.items(): + request = json.loads( + (handed_over / "request.json").read_text(encoding="utf-8") + ) + workspace = pathlib.Path(".comparator/Erdos1038_erdos_1038_parts_i").resolve() + os.chdir(handed_over) + regenerated = generator_cli.generate(request) + files = regenerated["Erdos1038_erdos_1038_parts_i"] + for name, content in files.items(): expected = (workspace / name).read_text(encoding="utf-8") assert content == expected, name - print(f"{len(regenerated)} files reproduced from the module and manifest") + print(f"{len(files)} files reproduced from the emitted request") PY - name: Build literate source pages diff --git a/.github/workflows/fc100-audit.yml b/.github/workflows/fc100-audit.yml new file mode 100644 index 0000000000..c5d528b371 --- /dev/null +++ b/.github/workflows/fc100-audit.yml @@ -0,0 +1,135 @@ +# Copyright 2026 The Formal Conjectures Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# https://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: FC100 whole-set audit + +# The whole-set run lean-eval#536 gates the FC import on, as a CI artifact: +# import every declaration of the frozen set, elaborate each marked-up module +# at this repository's pins, generate every workspace through the pinned +# generator, and compile every generated Challenge at LeanEval's pins in one +# shared project. Failures must match comparator/known_failures.toml exactly — +# an unexpected failure and a silently fixed one both fail the job, because a +# gate that only ever passes proves nothing. +# +# Two full Mathlib builds make this far too heavy for every pull request, so +# it runs on demand and weekly; the per-PR jobs cover representative +# declarations for each defect class instead. + +concurrency: + group: fc100-audit-${{ github.ref }} + cancel-in-progress: true + +on: + workflow_dispatch: + schedule: + # Weekly, early Monday UTC. + - cron: '17 4 * * 1' + +permissions: + contents: read + +jobs: + audit: + runs-on: ubuntu-latest + name: Import, verify, generate and compile FC100 + timeout-minutes: 300 + steps: + - name: Checkout Formal Conjectures + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + # The importer pins each workspace to the merge base with upstream + # main, so that ref has to be present. + fetch-depth: 0 + persist-credentials: false + + - name: Install elan + run: | + set -o pipefail + curl -sSfL https://github.com/leanprover/elan/releases/download/v1.4.2/elan-x86_64-unknown-linux-gnu.tar.gz | tar xz + ./elan-init -y --default-toolchain none + echo "$HOME/.elan/bin" >> "$GITHUB_PATH" + + # The category tags, source ranges and answer-slot types come from the + # elaborated environment, so every module in the set has to be built. + - name: Build the source modules and the extractor + run: | + lake exe cache get + lake build FormalConjectures.Subsets.FC100OpenSet1 comparator_facts + + - name: Build the pinned lean-eval-generator + run: | + read -r GEN_REPO GEN_REV <<< "$(python3 - <<'PY' + import tomllib + + with open("comparator/tools.toml", "rb") as handle: + generator = tomllib.load(handle)["generator"] + print(generator["repository"], generator["rev"]) + PY + )" + git clone "$GEN_REPO" "$RUNNER_TEMP/lean-eval-generator" + git -C "$RUNNER_TEMP/lean-eval-generator" checkout "$GEN_REV" + (cd "$RUNNER_TEMP/lean-eval-generator" && lake build) + echo "LEAN_EVAL_GENERATOR_BIN=$RUNNER_TEMP/lean-eval-generator/.lake/build/bin/lean-eval-generator" >> "$GITHUB_ENV" + + # `--verify` elaborates every marked-up module at this repository's + # pins; generation runs each import through the pinned binary. Failures + # are recorded per declaration, and the run fails unless they are + # exactly the recorded ones. + - name: Import and verify the whole set + run: | + python3 scripts/make_comparator_workspace.py --set FC100OpenSet1 \ + --verify --out .fc100 --report fc100-report.json \ + --known-failures comparator/known_failures.toml + + # The open/solved split is a fact about the set the report must state: + # lean-eval displays open conjectures apart from its evaluation set, so + # a silent reclassification upstream must fail loudly here. The counts + # are the audited 92 open + 8 solved, less whatever failed before its + # category could be read; a change means the frozen set itself changed + # and the import plan needs re-approval, not a quiet pass. + - name: Check the set classification + run: | + python3 - <<'PY' + import json + + with open("fc100-report.json") as handle: + report = json.load(handle) + categories = report["categories"] + print("classification:", categories) + assert sum(categories.values()) == report["imported"], categories + solved = categories.get("research solved", 0) + open_count = categories.get("research open", 0) + assert open_count + solved + report["source_failed"] == 100, categories + assert solved <= 8, f"more solved entries than the audited 8: {solved}" + PY + + # Kim's shared-project arrangement: every generated ChallengeDeps and + # Challenge compiles in one Lake project at LeanEval's pins, so Mathlib + # is built once for the whole set rather than once per workspace. + - name: Compile every generated Challenge at LeanEval pins + run: | + python3 scripts/compile_fc100_target.py .fc100 \ + --project "$RUNNER_TEMP/fc100-target" \ + --report fc100-target-report.json \ + --known-failures comparator/known_failures.toml + + - name: Upload the audit report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: fc100-audit-report + path: | + fc100-report.json + fc100-target-report.json + if-no-files-found: warn diff --git a/comparator/known_failures.toml b/comparator/known_failures.toml new file mode 100644 index 0000000000..b319881619 --- /dev/null +++ b/comparator/known_failures.toml @@ -0,0 +1,21 @@ +# Declarations the whole-set audit is expected to fail on, each with the +# reason and the stage it fails at. `--known-failures` asserts this list +# exactly: an unexpected failure and a silently fixed one both fail the run, +# because a gate that only ever passes proves nothing. +# +# declaration the qualified FC declaration name +# workspace the workspace id, for the target-stage compile report +# stage "source" (import or --verify at FC pins) or +# "target" (compile at LeanEval pins) +# reason what fails and why it is not fixed here + +[[failure]] +declaration = "Erdos1092.f_asymptotic_general" +workspace = "Erdos1092_f_asymptotic_general" +stage = "target" +reason = """The copied `f` definition synthesizes `Fintype ↑H.verts` and +`Fintype ↑H.coe.edgeSet` at this repository's Mathlib but not at LeanEval's +(observed at target pin 6f1ef4e5): instance drift between the two revisions, +not a copying defect. Formal Conjectures' own bump to Lean 4.33 rewrites the +source against the target-side Mathlib and retires the gap; re-run the audit +after the bump and remove this entry.""" diff --git a/scripts/compile_fc100_target.py b/scripts/compile_fc100_target.py new file mode 100644 index 0000000000..a1bf552b92 --- /dev/null +++ b/scripts/compile_fc100_target.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Compile every generated Challenge in one shared project at LeanEval pins. + +Each generated workspace pins the same Lean toolchain and Mathlib revision, +so building them separately would build Mathlib once per workspace. This +arranges them as modules of one Lake project instead — Mathlib is fetched and +built once for the whole set — which is what makes a hundred-workspace target +audit affordable, and is the arrangement the audit on +`google-deepmind/formal-conjectures#4951` used. + +Per workspace it copies `ChallengeDeps.lean` and `Challenge.lean` in as +`Deps_.lean` and `Chal_.lean`, rewriting the one import between them, +and builds each `Chal_` target separately so a failure is attributed to +its workspace rather than to the batch. `sorry` warnings are the workspaces +working; only errors fail a target. + +Usage: + python3 compile_fc100_target.py WORKSPACES_DIR --project DIR + [--report FILE] [--known-failures FILE] + +With `--known-failures`, exit non-zero unless the failing workspaces are +exactly the recorded `target`-stage ones. +""" + +import argparse +import json +import pathlib +import re +import subprocess +import sys +import tomllib + + +def arrange_project(workspaces_dir, project_dir): + """Lay out the shared project; returns `{workspace_id: module_name}`. + + Workspace ids are already identifiers (the importer slugs them), so the + module names need no further encoding. + """ + workspaces_dir = pathlib.Path(workspaces_dir) + project_dir = pathlib.Path(project_dir) + project_dir.mkdir(parents=True, exist_ok=True) + workspaces = sorted( + entry for entry in workspaces_dir.iterdir() if (entry / "Challenge.lean").is_file() + ) + if not workspaces: + raise SystemExit(f"no generated workspaces under {workspaces_dir}") + + toolchain = (workspaces[0] / "lean-toolchain").read_text(encoding="utf-8") + mathlib = None + modules = {} + libs = [] + for workspace in workspaces: + this_toolchain = (workspace / "lean-toolchain").read_text(encoding="utf-8") + if this_toolchain != toolchain: + raise SystemExit( + f"{workspace.name} pins {this_toolchain.strip()}, but the set " + f"started with {toolchain.strip()}; one project needs one pin" + ) + lakefile = tomllib.loads( + (workspace / "lakefile.toml").read_text(encoding="utf-8") + ) + this_mathlib = next( + requirement for requirement in lakefile["require"] + if requirement["name"] == "mathlib" + ) + if mathlib is None: + mathlib = this_mathlib + elif this_mathlib != mathlib: + raise SystemExit(f"{workspace.name} pins a different Mathlib") + + challenge = (workspace / "Challenge.lean").read_text(encoding="utf-8") + deps_path = workspace / "ChallengeDeps.lean" + deps_module = f"Deps_{workspace.name}" + challenge_module = f"Chal_{workspace.name}" + if deps_path.is_file(): + (project_dir / f"{deps_module}.lean").write_text( + deps_path.read_text(encoding="utf-8"), encoding="utf-8" + ) + challenge = re.sub( + r"^import ChallengeDeps$", + f"import {deps_module}", + challenge, + flags=re.MULTILINE, + ) + libs.append(deps_module) + (project_dir / f"{challenge_module}.lean").write_text( + challenge, encoding="utf-8" + ) + libs.append(challenge_module) + modules[workspace.name] = challenge_module + + (project_dir / "lean-toolchain").write_text(toolchain, encoding="utf-8") + lakefile = ['name = "fc100_target"', "", "[leanOptions]", "autoImplicit = false"] + lakefile += [ + "", + "[[require]]", + 'name = "mathlib"', + f'git = "{mathlib["git"]}"', + f'rev = "{mathlib["rev"]}"', + ] + for lib in libs: + lakefile += ["", "[[lean_lib]]", f'name = "{lib}"'] + (project_dir / "lakefile.toml").write_text( + "\n".join(lakefile) + "\n", encoding="utf-8" + ) + return modules + + +def build(project_dir, modules): + """Build each Challenge target, attributing failures per workspace.""" + project_dir = pathlib.Path(project_dir) + for command in (["lake", "update"], ["lake", "exe", "cache", "get"]): + completed = subprocess.run(command, cwd=project_dir) + if completed.returncode != 0: + raise SystemExit(f"{' '.join(command)} failed in {project_dir}") + results = [] + for workspace_id, module in sorted(modules.items()): + completed = subprocess.run( + ["lake", "build", module], + cwd=project_dir, + capture_output=True, + text=True, + ) + errors = [ + line + for line in (completed.stdout + completed.stderr).splitlines() + if "error:" in line + ] + ok = completed.returncode == 0 and not errors + results.append( + { + "workspace": workspace_id, + "status": "ok" if ok else "target-failed", + **({} if ok else {"reason": "\n".join(errors[:10]) or "build failed"}), + } + ) + print(f"{workspace_id}: {'ok' if ok else 'FAILED'}", flush=True) + return results + + +def main(argv): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("workspaces", help="directory of generated workspaces") + ap.add_argument("--project", required=True, help="shared project directory") + ap.add_argument("--report", default=None, help="write the JSON report here") + ap.add_argument( + "--known-failures", + default=None, + help="fail unless failing workspaces are exactly the recorded target ones", + ) + args = ap.parse_args(argv) + + modules = arrange_project(args.workspaces, args.project) + results = build(args.project, modules) + failed = {entry["workspace"] for entry in results if entry["status"] != "ok"} + report = { + "total": len(results), + "ok": len(results) - len(failed), + "failed": sorted(failed), + "results": results, + } + if args.report: + pathlib.Path(args.report).write_text( + json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" + ) + print(f"{report['ok']}/{report['total']} Challenges compile at target pins") + + if args.known_failures: + with open(args.known_failures, "rb") as handle: + recorded = tomllib.load(handle) + # Known failures are recorded by declaration; workspaces are named by + # the slugged id, which the `workspace` field of each entry supplies. + expected = { + entry["workspace"] + for entry in recorded.get("failure", []) + if entry.get("stage") == "target" and "workspace" in entry + } + unexpected = sorted(failed - expected) + fixed = sorted(expected - failed) + for name in unexpected: + print(f"unexpected target failure: {name}", file=sys.stderr) + for name in fixed: + print( + f"{name} is recorded as a known target failure but compiled; " + "remove it from the record", + file=sys.stderr, + ) + if unexpected or fixed: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From 8f8a0b1e78e9dec40d974b943f4f7661b924ebb2 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Thu, 20 Aug 2026 12:22:18 -0400 Subject: [PATCH 27/70] Let the whole-set audit run on the pull request that configures it Dispatch and schedule only reach a workflow on the default branch, so the audit could never produce its artifact before merging. Triggering on the audit's own configuration paths runs it on this pull request, which is where the evidence is needed. --- .github/workflows/fc100-audit.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/fc100-audit.yml b/.github/workflows/fc100-audit.yml index c5d528b371..8f8068cbf7 100644 --- a/.github/workflows/fc100-audit.yml +++ b/.github/workflows/fc100-audit.yml @@ -35,6 +35,14 @@ on: schedule: # Weekly, early Monday UTC. - cron: '17 4 * * 1' + # Dispatch and schedule only reach a workflow on the default branch, so a + # pull request introducing or reconfiguring the audit could never show its + # run. Trigger on the audit's own configuration instead: these paths change + # when the audit changes, and the artifact is the review evidence. + pull_request: + paths: + - '.github/workflows/fc100-audit.yml' + - 'comparator/known_failures.toml' permissions: contents: read From 65796e6db78eecde54b68b2ebc8087f66b956f78 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Thu, 20 Aug 2026 12:31:09 -0400 Subject: [PATCH 28/70] Give the Comparator job the pinned generator and the qualified workspace ids The job still generated through the deleted placeholder path: it needs the pinned binary on PATH like the smoke job, and the workspace directories are named by the qualified declaration ids now. --- .github/workflows/comparator-lean-4-33.yml | 38 ++++++++++++++++------ 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/.github/workflows/comparator-lean-4-33.yml b/.github/workflows/comparator-lean-4-33.yml index 6c24ad9122..6ca83a90a7 100644 --- a/.github/workflows/comparator-lean-4-33.yml +++ b/.github/workflows/comparator-lean-4-33.yml @@ -24,7 +24,7 @@ on: pull_request: paths: - 'scripts/fc_leaneval_importer.py' - - 'scripts/leaneval_generator.py' + - 'scripts/leaneval_generator_cli.py' - 'scripts/leaneval_interface.py' - 'scripts/make_comparator_workspace.py' - 'scripts/comparator_facts.lean' @@ -78,6 +78,24 @@ jobs: lake exe cache get lake build comparator_facts FormalConjectures.Wikipedia.SumOfThreeCubes + # Workspace generation runs the extracted generator at the pinned + # revision. The package depends on nothing, so this is a small Lean + # build, not a Mathlib one. + - name: Build the pinned lean-eval-generator + run: | + read -r GEN_REPO GEN_REV <<< "$(python3 - <<'PY' + import tomllib + + with open("comparator/tools.toml", "rb") as handle: + generator = tomllib.load(handle)["generator"] + print(generator["repository"], generator["rev"]) + PY + )" + git clone "$GEN_REPO" "$RUNNER_TEMP/lean-eval-generator" + git -C "$RUNNER_TEMP/lean-eval-generator" checkout "$GEN_REV" + (cd "$RUNNER_TEMP/lean-eval-generator" && lake build) + echo "LEAN_EVAL_GENERATOR_BIN=$RUNNER_TEMP/lean-eval-generator/.lake/build/bin/lean-eval-generator" >> "$GITHUB_ENV" + # `--verify` elaborates the marked-up module here, at 4.27. It is not a # substitute for the 4.33 build below; it is what keeps an FC-side # copying defect from being reported as a LeanEval build failure. @@ -94,15 +112,15 @@ jobs: done # One plain theorem and one `answer(sorry)` slot typed at 4.27. grep -q "isSumOfThreeCubes_iff_mod_9_answer : Prop" \ - .comparator/isSumOfThreeCubes_iff_mod_9/Challenge.lean + .comparator/SumOfThreeCubes_isSumOfThreeCubes_iff_mod_9/Challenge.lean # Generated for LeanEval, not for here. grep -q "$TARGET_TOOLCHAIN" \ - .comparator/isSumOfThreeCubes_2/lean-toolchain + .comparator/SumOfThreeCubes_isSumOfThreeCubes_2/lean-toolchain - name: Build both workspaces at Lean 4.33 run: | - for ws in .comparator/isSumOfThreeCubes_2 \ - .comparator/isSumOfThreeCubes_iff_mod_9; do + for ws in .comparator/SumOfThreeCubes_isSumOfThreeCubes_2 \ + .comparator/SumOfThreeCubes_isSumOfThreeCubes_iff_mod_9; do (cd "$ws" && lake update && lake exe cache get && lake build) done @@ -126,7 +144,7 @@ jobs: # adds `sorryAx`, which `permitted_axioms` does not allow. A # generated workspace that passed before anyone proved anything # would be worthless. - if (cd .comparator/isSumOfThreeCubes_2 && lake test); then + if (cd .comparator/SumOfThreeCubes_isSumOfThreeCubes_2 && lake test); then echo "::error::Comparator accepted an unproved generated workspace" exit 1 fi @@ -136,13 +154,13 @@ jobs: python3 - <<'PY' import pathlib - submission = pathlib.Path(".comparator/isSumOfThreeCubes_2/Submission.lean") + submission = pathlib.Path(".comparator/SumOfThreeCubes_isSumOfThreeCubes_2/Submission.lean") text = submission.read_text(encoding="utf-8") filled = text.replace(":= by\n sorry", ":= by\n exact ⟨1, 1, 0, by norm_num⟩") assert filled != text, "nothing to fill in the generated submission" submission.write_text(filled, encoding="utf-8") PY - (cd .comparator/isSumOfThreeCubes_2 && lake build && lake test) + (cd .comparator/SumOfThreeCubes_isSumOfThreeCubes_2 && lake build && lake test) # 3. The `answer(sorry)` workspace, with the hole filled by the # proposition on the other side of the iff and the bridge closed @@ -153,7 +171,7 @@ jobs: python3 - <<'PY' import pathlib - root = pathlib.Path(".comparator/isSumOfThreeCubes_iff_mod_9") + root = pathlib.Path(".comparator/SumOfThreeCubes_isSumOfThreeCubes_iff_mod_9") submission = root / "Submission.lean" text = submission.read_text(encoding="utf-8") answer = ( @@ -167,5 +185,5 @@ jobs: text = text.replace(":= by\n sorry", ":=\n Iff.rfl") submission.write_text(text, encoding="utf-8") PY - (cd .comparator/isSumOfThreeCubes_iff_mod_9 && lake build && lake test) + (cd .comparator/SumOfThreeCubes_isSumOfThreeCubes_iff_mod_9 && lake build && lake test) echo "Comparator accepted a gamed definition hole; hole values need a human." From ab4c49dcc385e66246314274aa84302dba634109 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Thu, 20 Aug 2026 12:44:40 -0400 Subject: [PATCH 29/70] Ignore whole-set audit output directories --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 628dd087f7..d464550c62 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,6 @@ FormalConjectures/All.lean __pycache__/ .comparator/ .comparator-import/ +# Whole-set audit output (make_comparator_workspace.py --set) +.fc100/ +.fc100v/ From bd9b2141d6d83d029c6200364d6e478e69759e21 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Thu, 20 Aug 2026 13:07:51 -0400 Subject: [PATCH 30/70] Let the frozen set decide the display group, with the category as a tag FC100OpenSet1 is immutable while its members keep getting solved: that is designed lifecycle, not a defect (#5075). So a frozen-set import keeps every member in the open-conjectures group and carries the category as a tag, a single import still maps by category, and the audit's classification check becomes structural rather than pinning a solved count that time is guaranteed to move. --- .github/workflows/fc100-audit.yml | 17 +++++++---------- comparator/OWNERSHIP.md | 2 +- scripts/leaneval_interface.py | 13 ++++++++++--- scripts/make_comparator_workspace.py | 20 +++++++++++++++----- scripts/test_leaneval_interface.py | 21 +++++++++++++++++++++ 5 files changed, 54 insertions(+), 19 deletions(-) diff --git a/.github/workflows/fc100-audit.yml b/.github/workflows/fc100-audit.yml index 8f8068cbf7..2d46292be9 100644 --- a/.github/workflows/fc100-audit.yml +++ b/.github/workflows/fc100-audit.yml @@ -100,12 +100,11 @@ jobs: --verify --out .fc100 --report fc100-report.json \ --known-failures comparator/known_failures.toml - # The open/solved split is a fact about the set the report must state: - # lean-eval displays open conjectures apart from its evaluation set, so - # a silent reclassification upstream must fail loudly here. The counts - # are the audited 92 open + 8 solved, less whatever failed before its - # category could be read; a change means the frozen set itself changed - # and the import plan needs re-approval, not a quiet pass. + # The open/solved split is a fact about the set the report must state. + # The list is frozen but its members keep getting solved — that is + # designed lifecycle, not an anomaly (formal-conjectures#5075) — so + # the check here is structural: every member accounted for, every + # category a research one. The report carries the current split. - name: Check the set classification run: | python3 - <<'PY' @@ -116,10 +115,8 @@ jobs: categories = report["categories"] print("classification:", categories) assert sum(categories.values()) == report["imported"], categories - solved = categories.get("research solved", 0) - open_count = categories.get("research open", 0) - assert open_count + solved + report["source_failed"] == 100, categories - assert solved <= 8, f"more solved entries than the audited 8: {solved}" + assert set(categories) <= {"research open", "research solved"}, categories + assert sum(categories.values()) + report["source_failed"] == 100, categories PY # Kim's shared-project arrangement: every generated ChallengeDeps and diff --git a/comparator/OWNERSHIP.md b/comparator/OWNERSHIP.md index e94cdeab94..a3a2984794 100644 --- a/comparator/OWNERSHIP.md +++ b/comparator/OWNERSHIP.md @@ -34,7 +34,7 @@ revision is normative). Per problem it carries: | `moduleContent` | the rendered marked-up module: the statement's copied FC-local closure, the scope directives in force where it was written, one `noncomputable def : := sorry` per `answer(sorry)` slot, and the statement with its proof replaced by `sorry` — in that order, requiring Mathlib and nothing else | | `resolvedHoles` | a source span, kind, and explicit parameters for each hole, computed from the rendered text — exactly, because this side rendered it | | `holes`, `id`, `moduleName` | the qualified declaration name, slugged; two modules declaring `conjecture` in different namespaces must not share a workspace | -| `group` | the declaration's `@[category ...]` tag: `research open` is an open conjecture, settled statements are evaluation material, anything else is refused | +| `group` | for a frozen-set import, the set itself: the list is immutable while its members keep getting solved, so every member stays in the open-conjectures display and the category rides along as a tag. For a single import, the declaration's `@[category ...]` tag decides; a declaration that is not a problem is refused either way | | `leanToolchain`, `mathlib` | LeanEval's pins, from `[target]` in `tools.toml` — the consumer's, never this repository's | | `templates.workspaceTest` | `comparator/templates/WorkspaceTest.lean`, which stays FC-supplied: the contract requires the consumer to provide it | | `contextRoot` | a directory this side materialises: the module file the generator byte-checks against `moduleContent`, and a synthesised `.ilean` carrying the spans above, because v1 still resolves declaration spans from compiled metadata | diff --git a/scripts/leaneval_interface.py b/scripts/leaneval_interface.py index 2765fa7df3..6cceb097c5 100644 --- a/scripts/leaneval_interface.py +++ b/scripts/leaneval_interface.py @@ -403,7 +403,7 @@ def line_of(offset): return spans -def build_problem(marked_up, manifest, module_name=None): +def build_problem(marked_up, manifest, module_name=None, group=None): """One problem entry of the v1 request, and its `.ilean` declaration map. The module name is a single identifier on purpose: the generator resolves @@ -411,6 +411,12 @@ def build_problem(marked_up, manifest, module_name=None): a dotted or quoted name would trip the same decoder defect this repository fixed on its own side. + `group` overrides the category-derived group for members of a frozen + set: the set decides the display tab, because the list is immutable + while its members keep getting solved, and the category rides along as + a tag. The category is still validated either way — a declaration that + is not a problem has no business in any group. + Returns `(problem, ilean_decls)`. The `.ilean` payload exists because the generator reads helper-declaration spans from compiled metadata it expects to find under the context root; this consumer synthesises that @@ -449,14 +455,15 @@ def build_problem(marked_up, manifest, module_name=None): "kind": span["kind"], } ) + category_group = problem_group(manifest) problem = { "id": slug(manifest.id), "title": manifest.qualified_theorem, - "group": problem_group(manifest), + "group": group or category_group, "status": "draft", "visible": True, "statementRevision": 1, - "tags": ["formal-conjectures"], + "tags": ["formal-conjectures", manifest.category.replace(" ", "-")], "moduleName": module_name, "holes": [entry["declarationName"] for entry in resolved], "submitter": SUBMITTER, diff --git a/scripts/make_comparator_workspace.py b/scripts/make_comparator_workspace.py index ecea9454ed..76d7e7079e 100644 --- a/scripts/make_comparator_workspace.py +++ b/scripts/make_comparator_workspace.py @@ -107,7 +107,7 @@ def write_tree(target, files): return target -def seam_files(pairs): +def seam_files(pairs, group=None): """The request and context for `(marked_up, manifest)` pairs, as files. This is the artifact the FC importer contributes once lean-eval consumes @@ -116,7 +116,10 @@ def seam_files(pairs): source commit and declaration id §10 requires, which the v1 wire format has no field for, so they travel beside it rather than through it. """ - problems = [build_problem(marked_up, manifest) for marked_up, manifest in pairs] + problems = [ + build_problem(marked_up, manifest, group=group) + for marked_up, manifest in pairs + ] target = importer.target_pins() template = ( importer.COMPARATOR_DIR / "templates" / "WorkspaceTest.lean" @@ -137,9 +140,9 @@ def seam_files(pairs): return request, files -def generate_workspaces(pairs, out_dir): +def generate_workspaces(pairs, out_dir, group=None): """Generate one workspace per pair under `out_dir`, via the pinned binary.""" - request, files = seam_files(pairs) + request, files = seam_files(pairs, group=group) staging = pathlib.Path(tempfile.mkdtemp(prefix=".fc-seam.")) try: for relative, content in files.items(): @@ -232,7 +235,14 @@ def import_set(set_name, out_dir, verify=False, known_failures=None): "status": "imported", } ) - written = generate_workspaces(pairs, out_dir) if pairs else [] + # The set decides the tab: a frozen list stays advertised whole, with + # solved members marked by their category tag, so every member goes to + # the open-conjectures group (google-deepmind/formal-conjectures#5075). + written = ( + generate_workspaces(pairs, out_dir, group="open-conjectures") + if pairs + else [] + ) categories = {} for entry in results: if entry["status"] == "imported": diff --git a/scripts/test_leaneval_interface.py b/scripts/test_leaneval_interface.py index f27e52a257..830d907266 100644 --- a/scripts/test_leaneval_interface.py +++ b/scripts/test_leaneval_interface.py @@ -247,6 +247,27 @@ def test_a_non_problem_category_is_refused(self): with self.assertRaises(SystemExit): build_problem(A_MODULE, a_manifest(category="API")) + def test_the_category_rides_along_as_a_tag(self): + problem, _ = build_problem(A_MODULE, a_manifest(category="research solved")) + self.assertIn("research-solved", problem["tags"]) + + def test_a_set_override_keeps_a_solved_member_in_its_set(self): + # The frozen list is immutable while its members keep getting + # solved, so the set decides the tab and the tag says which are + # solved (formal-conjectures#5075). + problem, _ = build_problem( + A_MODULE, + a_manifest(category="research solved"), + group="open-conjectures", + ) + self.assertEqual(problem["group"], "open-conjectures") + + def test_a_set_override_does_not_admit_a_non_problem(self): + with self.assertRaises(SystemExit): + build_problem( + A_MODULE, a_manifest(category="API"), group="open-conjectures" + ) + class BuildRequestTest(unittest.TestCase): def test_the_request_carries_the_targets_pins(self): From 18bd52903e76ea98159fb103191f9cf8f2492664 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Thu, 20 Aug 2026 13:31:19 -0400 Subject: [PATCH 31/70] Trim what the migration left behind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The v1 request consumes two of TargetRecord's six fields, so the record now carries two and comparator/tools.toml stays the full pin ledger. The CLI plumbing imports the interface at module top — there was never a cycle to avoid — and workspace generation stages only the context the binary reads, not the request and sidecars it does not. The pinned generator build, written out three times across workflows, is one composite action. Two dead imports go. --- .../build-lean-eval-generator/action.yml | 40 +++++++++++++++++++ .github/workflows/build-and-docs.yml | 17 +------- .github/workflows/comparator-lean-4-33.yml | 17 +------- .github/workflows/fc100-audit.yml | 14 +------ scripts/fc_leaneval_importer.py | 5 --- scripts/leaneval_generator_cli.py | 8 ++-- scripts/leaneval_interface.py | 12 ++---- scripts/make_comparator_workspace.py | 11 +++-- scripts/test_fc_leaneval_importer.py | 7 ++-- scripts/test_leaneval_interface.py | 4 -- scripts/test_make_comparator_workspace.py | 1 - 11 files changed, 61 insertions(+), 75 deletions(-) create mode 100644 .github/actions/build-lean-eval-generator/action.yml diff --git a/.github/actions/build-lean-eval-generator/action.yml b/.github/actions/build-lean-eval-generator/action.yml new file mode 100644 index 0000000000..e023027333 --- /dev/null +++ b/.github/actions/build-lean-eval-generator/action.yml @@ -0,0 +1,40 @@ +# Copyright 2026 The Formal Conjectures Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# https://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Build the pinned lean-eval-generator +description: >- + Clone and build leanprover/lean-eval-generator at the revision + comparator/tools.toml pins under [generator], and export + LEAN_EVAL_GENERATOR_BIN. The package depends on nothing, so this is a + small Lean build, not a Mathlib one. Requires elan on PATH and a + checked-out Formal Conjectures tree. + +runs: + using: composite + steps: + - name: Clone and build the pinned revision + shell: bash + run: | + read -r GEN_REPO GEN_REV <<< "$(python3 - <<'PY' + import tomllib + + with open("comparator/tools.toml", "rb") as handle: + generator = tomllib.load(handle)["generator"] + print(generator["repository"], generator["rev"]) + PY + )" + git clone "$GEN_REPO" "$RUNNER_TEMP/lean-eval-generator" + git -C "$RUNNER_TEMP/lean-eval-generator" checkout "$GEN_REV" + (cd "$RUNNER_TEMP/lean-eval-generator" && lake build) + echo "LEAN_EVAL_GENERATOR_BIN=$RUNNER_TEMP/lean-eval-generator/.lake/build/bin/lean-eval-generator" >> "$GITHUB_ENV" diff --git a/.github/workflows/build-and-docs.yml b/.github/workflows/build-and-docs.yml index 23d3a3a54d..19678a4665 100644 --- a/.github/workflows/build-and-docs.yml +++ b/.github/workflows/build-and-docs.yml @@ -155,24 +155,9 @@ jobs: lake --wfail build rm -f FormalConjectures/All.lean - # Workspace generation runs the extracted generator at the pinned - # revision. The package depends on nothing, so this is a small Lean - # build, not a Mathlib one. - name: Build the pinned lean-eval-generator if: steps.mode.outputs.website_only != 'true' - run: | - read -r GEN_REPO GEN_REV <<< "$(python3 - <<'PY' - import tomllib - - with open("comparator/tools.toml", "rb") as handle: - generator = tomllib.load(handle)["generator"] - print(generator["repository"], generator["rev"]) - PY - )" - git clone "$GEN_REPO" "$RUNNER_TEMP/lean-eval-generator" - git -C "$RUNNER_TEMP/lean-eval-generator" checkout "$GEN_REV" - (cd "$RUNNER_TEMP/lean-eval-generator" && lake build) - echo "LEAN_EVAL_GENERATOR_BIN=$RUNNER_TEMP/lean-eval-generator/.lake/build/bin/lean-eval-generator" >> "$GITHUB_ENV" + uses: ./.github/actions/build-lean-eval-generator # The elaborator-to-importer boundary, exercised on the oleans the # build above produced: one plain theorem, a Prop answer slot, a diff --git a/.github/workflows/comparator-lean-4-33.yml b/.github/workflows/comparator-lean-4-33.yml index 6ca83a90a7..137278c353 100644 --- a/.github/workflows/comparator-lean-4-33.yml +++ b/.github/workflows/comparator-lean-4-33.yml @@ -78,23 +78,8 @@ jobs: lake exe cache get lake build comparator_facts FormalConjectures.Wikipedia.SumOfThreeCubes - # Workspace generation runs the extracted generator at the pinned - # revision. The package depends on nothing, so this is a small Lean - # build, not a Mathlib one. - name: Build the pinned lean-eval-generator - run: | - read -r GEN_REPO GEN_REV <<< "$(python3 - <<'PY' - import tomllib - - with open("comparator/tools.toml", "rb") as handle: - generator = tomllib.load(handle)["generator"] - print(generator["repository"], generator["rev"]) - PY - )" - git clone "$GEN_REPO" "$RUNNER_TEMP/lean-eval-generator" - git -C "$RUNNER_TEMP/lean-eval-generator" checkout "$GEN_REV" - (cd "$RUNNER_TEMP/lean-eval-generator" && lake build) - echo "LEAN_EVAL_GENERATOR_BIN=$RUNNER_TEMP/lean-eval-generator/.lake/build/bin/lean-eval-generator" >> "$GITHUB_ENV" + uses: ./.github/actions/build-lean-eval-generator # `--verify` elaborates the marked-up module here, at 4.27. It is not a # substitute for the 4.33 build below; it is what keeps an FC-side diff --git a/.github/workflows/fc100-audit.yml b/.github/workflows/fc100-audit.yml index 2d46292be9..ae6a1530f1 100644 --- a/.github/workflows/fc100-audit.yml +++ b/.github/workflows/fc100-audit.yml @@ -76,19 +76,7 @@ jobs: lake build FormalConjectures.Subsets.FC100OpenSet1 comparator_facts - name: Build the pinned lean-eval-generator - run: | - read -r GEN_REPO GEN_REV <<< "$(python3 - <<'PY' - import tomllib - - with open("comparator/tools.toml", "rb") as handle: - generator = tomllib.load(handle)["generator"] - print(generator["repository"], generator["rev"]) - PY - )" - git clone "$GEN_REPO" "$RUNNER_TEMP/lean-eval-generator" - git -C "$RUNNER_TEMP/lean-eval-generator" checkout "$GEN_REV" - (cd "$RUNNER_TEMP/lean-eval-generator" && lake build) - echo "LEAN_EVAL_GENERATOR_BIN=$RUNNER_TEMP/lean-eval-generator/.lake/build/bin/lean-eval-generator" >> "$GITHUB_ENV" + uses: ./.github/actions/build-lean-eval-generator # `--verify` elaborates every marked-up module at this repository's # pins; generation runs each import through the pinned binary. Failures diff --git a/scripts/fc_leaneval_importer.py b/scripts/fc_leaneval_importer.py index 77ca26d740..c759d426ed 100644 --- a/scripts/fc_leaneval_importer.py +++ b/scripts/fc_leaneval_importer.py @@ -37,7 +37,6 @@ ProblemManifest, SourceRecord, TargetRecord, - problem_group, ) ROOT = pathlib.Path(__file__).resolve().parent.parent @@ -86,12 +85,8 @@ def target_pins(): """ target = _tools_file()["target"] return TargetRecord( - repository=target["repository"], - commit=target["commit"], lean_toolchain=target["lean_toolchain"], mathlib_revision=target["mathlib_revision"], - comparator=target["comparator"], - lean4export=target["lean4export"], ) diff --git a/scripts/leaneval_generator_cli.py b/scripts/leaneval_generator_cli.py index b5910c8536..165c655d6e 100644 --- a/scripts/leaneval_generator_cli.py +++ b/scripts/leaneval_generator_cli.py @@ -28,6 +28,8 @@ import shutil import subprocess +from leaneval_interface import parse_response + BINARY_ENV = "LEAN_EVAL_GENERATOR_BIN" BINARY_NAME = "lean-eval-generator" @@ -76,12 +78,8 @@ def write_context_root(root, problems): def generate(request): """The generator's verified file maps for one request. - Returns `{problem_id: {path: content}}`. Import here rather than at - module top keeps the arrow pointing one way: the interface never imports - this plumbing. + Returns `{problem_id: {path: content}}`. """ - from leaneval_interface import parse_response - proc = subprocess.run( [binary()], input=json.dumps(request), diff --git a/scripts/leaneval_interface.py b/scripts/leaneval_interface.py index 6cceb097c5..38a0b8cb52 100644 --- a/scripts/leaneval_interface.py +++ b/scripts/leaneval_interface.py @@ -136,18 +136,14 @@ class TargetRecord: another repository's regime, and would go stale the moment that repository bumped anything, with nothing here to notice. - Formal Conjectures keeps a copy under `[target]` in `comparator/tools.toml` - for one purpose: the CI job that generates at this repository's toolchain - and builds at LeanEval's, which is how the gap between the two is observed - rather than assumed. + Formal Conjectures keeps the full pin set under `[target]` in + `comparator/tools.toml`; this record carries only the two fields the v1 + request consumes. The comparator and lean4export pins are read from the + TOML directly by the CI job that runs them. """ - repository: str - commit: str lean_toolchain: str mathlib_revision: str - comparator: str - lean4export: str @dataclasses.dataclass(frozen=True) diff --git a/scripts/make_comparator_workspace.py b/scripts/make_comparator_workspace.py index 76d7e7079e..f8d256cd54 100644 --- a/scripts/make_comparator_workspace.py +++ b/scripts/make_comparator_workspace.py @@ -73,7 +73,8 @@ ROOT = importer.ROOT -PROVENANCE_FILE = "fc-provenance.json" +PROVENANCE_STEM = "fc-provenance" +PROVENANCE_FILE = f"{PROVENANCE_STEM}.json" # The request's context directory, relative to the request file, so an # emitted seam artifact is self-contained and reproducible from any path. @@ -134,9 +135,7 @@ def seam_files(pairs, group=None): files[f"{CONTEXT_DIR}/.lake/build/lib/lean/{module}.ilean"] = ( json.dumps({"version": 1, "module": module, "decls": ilean}) + "\n" ) - files[f"{PROVENANCE_FILE.removesuffix('.json')}-{problem['id']}.json"] = ( - manifest.to_json() - ) + files[f"{PROVENANCE_STEM}-{problem['id']}.json"] = manifest.to_json() return request, files @@ -145,7 +144,11 @@ def generate_workspaces(pairs, out_dir, group=None): request, files = seam_files(pairs, group=group) staging = pathlib.Path(tempfile.mkdtemp(prefix=".fc-seam.")) try: + # Only the context crosses to the binary; the request goes on stdin + # and the provenance sidecars are for the written workspaces. for relative, content in files.items(): + if not relative.startswith(f"{CONTEXT_DIR}/"): + continue destination = staging / relative destination.parent.mkdir(parents=True, exist_ok=True) destination.write_text(content, encoding="utf-8") diff --git a/scripts/test_fc_leaneval_importer.py b/scripts/test_fc_leaneval_importer.py index 7f69521893..9a86bd5550 100644 --- a/scripts/test_fc_leaneval_importer.py +++ b/scripts/test_fc_leaneval_importer.py @@ -28,6 +28,7 @@ from unittest import mock import fc_leaneval_importer as importer +from leaneval_interface import problem_group from fc_leaneval_importer import ( answer_spans, closure_region, @@ -517,7 +518,7 @@ def _manifest(self, category): def test_open_research_is_an_open_conjecture(self): self.assertEqual( - importer.problem_group(self._manifest("research open")), + problem_group(self._manifest("research open")), "open-conjectures", ) @@ -525,7 +526,7 @@ def test_settled_statements_are_evaluation_material(self): for category in ("research solved", "textbook", "test"): with self.subTest(category=category): self.assertEqual( - importer.problem_group(self._manifest(category)), + problem_group(self._manifest(category)), "formalization-evaluation", ) @@ -533,7 +534,7 @@ def test_api_and_untagged_declarations_are_refused(self): for category in ("API", ""): with self.subTest(category=category): with self.assertRaises(SystemExit): - importer.problem_group(self._manifest(category)) + problem_group(self._manifest(category)) class FlattenDeclaredNameTest(unittest.TestCase): diff --git a/scripts/test_leaneval_interface.py b/scripts/test_leaneval_interface.py index 830d907266..a1b2caf080 100644 --- a/scripts/test_leaneval_interface.py +++ b/scripts/test_leaneval_interface.py @@ -58,12 +58,8 @@ def a_source(**overrides): def a_target(**overrides): fields = { - "repository": "leanprover/lean-eval", - "commit": "e" * 40, "lean_toolchain": "leanprover/lean4:v4.33.0", "mathlib_revision": "f" * 40, - "comparator": "d" * 40, - "lean4export": "0" * 40, } fields.update(overrides) return TargetRecord(**fields) diff --git a/scripts/test_make_comparator_workspace.py b/scripts/test_make_comparator_workspace.py index 481e922400..2e97aa581c 100644 --- a/scripts/test_make_comparator_workspace.py +++ b/scripts/test_make_comparator_workspace.py @@ -27,7 +27,6 @@ import shutil import tempfile import unittest -from unittest import mock import leaneval_generator_cli as generator_cli from make_comparator_workspace import ( From b66328cd0542c11d09a8251986becd1aff17ebfc Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:13:30 -0400 Subject: [PATCH 32/70] Keep generated context single-copy and complete across reconstruction The first whole-set target compile surfaced how the pinned generator rebuilds each workspace file's context by re-extraction, which the emitted module has to survive: - A namespace open with nothing copied to declare it is an orphan in a file with no ChallengeDeps to import; with an empty closure there are no siblings to name, so the opens go. - A global notation ends up declared in ChallengeDeps and re-extracted into the file importing it, and every use becomes ambiguous; all emitted notation is now local, so each generated file carries exactly its own copy. - An "open scoped ... in" prefix hidden inside the statement's span was dropped from the reconstruction; the span now starts at the declaration keyword and the generator re-attaches the prefix itself. Seven of the nine unrecorded failures fall to these; the two that remain - the Mordell-Weil notation drift in EllipticCurveRank and the set_option the generator's Challenge reconstruction cannot carry for Erdos125 - are recorded in known_failures.toml with their reasons, beside Erdos1092. Whole-set result at these pins: 100/100 verify at source, 97/100 compile at target, failures matching the record exactly. --- comparator/known_failures.toml | 22 ++++++++++++ scripts/fc_leaneval_importer.py | 52 ++++++++++++++++++++++++---- scripts/leaneval_interface.py | 12 ++++++- scripts/test_fc_leaneval_importer.py | 51 +++++++++++++++++++++++++-- 4 files changed, 127 insertions(+), 10 deletions(-) diff --git a/comparator/known_failures.toml b/comparator/known_failures.toml index b319881619..9525f5ffa5 100644 --- a/comparator/known_failures.toml +++ b/comparator/known_failures.toml @@ -19,3 +19,25 @@ reason = """The copied `f` definition synthesizes `Fintype ↑H.verts` and not a copying defect. Formal Conjectures' own bump to Lean 4.33 rewrites the source against the target-side Mathlib and retires the gap; re-run the audit after the bump and remove this entry.""" + +[[failure]] +declaration = "EllipticCurveRank.RatEllipticCurve.twentyone_le_rank_height_count_asymptotic" +workspace = "EllipticCurveRank_RatEllipticCurve_twentyone_le_rank_height_count_asymptotic" +stage = "target" +reason = """The copied `toWeierstrass⟮ℚ⟯` Mordell-Weil notation elaborates at +this repository's Mathlib but not at LeanEval's (observed at target pin +6f1ef4e5): the notation's shape changed between the two revisions, in a +dependency copied faithfully from source. Retired by the repository's 4.33 +bump; re-run the audit after it and remove this entry.""" + +[[failure]] +declaration = "Erdos125.erdos_125.variants.positive_unequal_density" +workspace = "Erdos125_erdos_125_variants_positive_unequal_density" +stage = "target" +reason = """The statement's `local notation` uses set-builder syntax and needs +`set_option quotPrecheck false`, which the source states file-scoped and the +emitted module carries attached to the notation command. The generator's +Challenge reconstruction re-extracts the notation line without its +set_option, and `Challenge.lean` is generator-owned text this side cannot +amend. Reported upstream as a context-reconstruction gap; remove this entry +when the pinned generator carries set_option context.""" diff --git a/scripts/fc_leaneval_importer.py b/scripts/fc_leaneval_importer.py index c759d426ed..75475c9820 100644 --- a/scripts/fc_leaneval_importer.py +++ b/scripts/fc_leaneval_importer.py @@ -644,6 +644,34 @@ def fc_notation_commands(): return commands +NOTATION_FAMILY = re.compile(r"^(?:notation[0-9]*|postfix|prefix|infixl|infixr|infix)[:\s]") + + +def localise_notation(preamble): + """File-scope the preamble's notation commands, with their set_options. + + The generator reconstructs each workspace file's context by re-extracting + these commands from the module, so a *global* notation ends up declared + both in `ChallengeDeps` and in the file importing it — two identical + notations, and every use becomes ambiguous. `local` keeps each copy to + its own file. A standalone `set_option quotPrecheck false` does not + survive that reconstruction, so a notation that needs it gets it + attached as part of its own command. + """ + precheck_off = any( + entry.split("\n")[0].strip() == "set_option quotPrecheck false" + for entry in preamble + ) + out = [] + for entry in preamble: + if NOTATION_FAMILY.match(entry): + entry = "local " + entry + if precheck_off and re.match(r"^(?:local\s+)?(?:notation|postfix|prefix|infix)", entry): + entry = "set_option quotPrecheck false in\n" + entry + out.append(entry) + return out + + def notation_blocks(module_texts, opened): """The FC notation commands the module's text uses, as copyable blocks. @@ -670,9 +698,13 @@ def notation_blocks(module_texts, opened): continue seen.add(command) # A plain `scoped` command needs its namespace restated around it; - # the bracket form and a global command carry their own scope. + # the bracket form carries its own scope. A global command becomes + # `local`: the generator re-extracts it into every file that needs + # it, and a module-crossing global would be declared twice. if scope and not command.startswith("scoped["): command = f"namespace {scope}\n{command}\nend {scope}" + elif not scope: + command = "local " + command blocks.append(command) return blocks @@ -1083,13 +1115,21 @@ def import_problem(problem, answer_type=None, module=None): ) # `open A`, then `open A.B`: opening the inner namespace does not open the - # outer one, and a statement may name siblings from either. - opens = [ - f"open {'.'.join(namespaces_at_target[: i + 1])}" - for i in range(len(namespaces_at_target)) - ] + # outer one, and a statement may name siblings from either. With nothing + # copied there are no siblings to name and nothing declares the + # namespace, so an open would be an unresolvable orphan in a generated + # file that has no ChallengeDeps to import. + opens = ( + [ + f"open {'.'.join(namespaces_at_target[: i + 1])}" + for i in range(len(namespaces_at_target)) + ] + if copied + else [] + ) mathlib_rev, fc_rev = pins(path.relative_to(ROOT)) + preamble = localise_notation(preamble) scope_text = "\n".join(opens + preamble) # Notation is text, not a constant: a statement or copied declaration # spelled with an FC-defined token needs the defining command copied too, diff --git a/scripts/leaneval_interface.py b/scripts/leaneval_interface.py index 38a0b8cb52..1e3c8ebebb 100644 --- a/scripts/leaneval_interface.py +++ b/scripts/leaneval_interface.py @@ -318,10 +318,20 @@ def module_declarations(marked_up, manifest): declarations += [ (hole.name, hole.declaration(), "def", None) for hole in manifest.holes ] + # `open X in` prefix lines travel inside the statement slice, but the + # span the generator receives must start at the declaration keyword: + # the generator re-attaches whatever sits between the previous span and + # this one as the declaration's prefix, and a prefix hidden inside the + # span would be dropped from the reconstructed files. + statement = marked_up.statement + lines = statement.split("\n") + start = 0 + while start < len(lines) - 1 and lines[start].rstrip().endswith(" in"): + start += 1 declarations.append( ( manifest.theorem, - marked_up.statement, + "\n".join(lines[start:]), "theorem", list(manifest.apply_arguments), ) diff --git a/scripts/test_fc_leaneval_importer.py b/scripts/test_fc_leaneval_importer.py index 9a86bd5550..7da99a9c21 100644 --- a/scripts/test_fc_leaneval_importer.py +++ b/scripts/test_fc_leaneval_importer.py @@ -28,7 +28,8 @@ from unittest import mock import fc_leaneval_importer as importer -from leaneval_interface import problem_group +from leaneval_interface import MarkedUpModule, problem_group +from test_leaneval_interface import a_manifest from fc_leaneval_importer import ( answer_spans, closure_region, @@ -646,12 +647,14 @@ def test_a_scoped_notation_needs_its_namespace_opened(self): importer.notation_blocks(["def f : ℝ² := sorry"], set()), [] ) - def test_a_shared_global_notation_matches_without_opens(self): + def test_a_shared_global_notation_is_copied_as_local(self): + # Global would be declared in ChallengeDeps and re-extracted into + # the importing file too; `local` keeps each copy to its own file. commands = [(["≪"], 'notation g " ≪ " f => IsBigO g f', None, True)] with self._with_commands(commands): self.assertEqual( importer.notation_blocks(["theorem t : a ≪ b := sorry"], set()), - ['notation g " ≪ " f => IsBigO g f'], + ['local notation g " ≪ " f => IsBigO g f'], ) def test_a_problem_module_global_notation_is_never_copied(self): @@ -665,3 +668,45 @@ def test_an_unused_token_is_not_copied(self): commands = [(["ℝ²"], 'notation "ℝ²" => E', None, True)] with self._with_commands(commands): self.assertEqual(importer.notation_blocks(["theorem t : True"], set()), []) + + +class LocaliseNotationTest(unittest.TestCase): + def test_a_global_notation_becomes_local(self): + self.assertEqual( + importer.localise_notation(['notation "R(" k ")" => f k']), + ['local notation "R(" k ")" => f k'], + ) + + def test_quot_precheck_travels_with_the_notation(self): + out = importer.localise_notation( + ["set_option quotPrecheck false", 'local notation "A" => s'] + ) + self.assertEqual( + out[1], + 'set_option quotPrecheck false in\nlocal notation "A" => s', + ) + + def test_other_preamble_lines_pass_through(self): + self.assertEqual( + importer.localise_notation(["open Nat", "variable (n : Nat)"]), + ["open Nat", "variable (n : Nat)"], + ) + + +class StatementPrefixSpanTest(unittest.TestCase): + def test_the_span_starts_at_the_declaration_keyword(self): + from leaneval_interface import module_declarations + + module = MarkedUpModule( + dependencies="def Foo.bar := 1", + scope="", + holes="", + statement="open scoped Classical in\ntheorem t : True := by\n sorry", + dependency_declarations=(("Foo.bar", "def Foo.bar := 1"),), + ) + manifest = a_manifest( + theorem="t", qualified_theorem="t", holes=(), apply_arguments=() + ) + *_, statement_entry = module_declarations(module, manifest) + self.assertTrue(statement_entry[1].startswith("theorem t")) + self.assertNotIn("Classical in", statement_entry[1]) From e6c45e97bae77e4d677b48f3c93cae0fb000b42b Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:16:34 -0400 Subject: [PATCH 33/70] State the context-directory layout once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit write_context_root was dead — generation goes through seam_files — and its existence left the context layout encoded twice. context_files is now the one statement of it, and seam_files places the result under the emitted artifact's context directory. --- scripts/leaneval_generator_cli.py | 26 +++++++++++--------------- scripts/make_comparator_workspace.py | 9 +++------ 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/scripts/leaneval_generator_cli.py b/scripts/leaneval_generator_cli.py index 165c655d6e..35a581bbc3 100644 --- a/scripts/leaneval_generator_cli.py +++ b/scripts/leaneval_generator_cli.py @@ -52,27 +52,23 @@ def binary(): ) -def write_context_root(root, problems): - """A minimal benchmark checkout for the request's problems. +def context_files(problems): + """The minimal benchmark checkout for a request, as `{path: content}`. `problems` are `(problem, ilean_decls)` pairs from `build_problem`. Each - module lands at `/.lean` — the byte-match the generator - enforces against `moduleContent` — and its spans at the `.ilean` path the - generator reads, `.lake/build/lib/lean/.ilean`. + module lands at `.lean` — the byte-match the generator enforces + against `moduleContent` — and its spans at the `.ilean` path the + generator reads. This is the one statement of that layout; whoever puts + the files on disk decides where the root lives. """ - root = pathlib.Path(root) - ilean_dir = root / ".lake" / "build" / "lib" / "lean" - ilean_dir.mkdir(parents=True, exist_ok=True) + files = {} for problem, ilean in problems: module = problem["moduleName"] - (root / f"{module}.lean").write_text( - problem["moduleContent"], encoding="utf-8" + files[f"{module}.lean"] = problem["moduleContent"] + files[f".lake/build/lib/lean/{module}.ilean"] = ( + json.dumps({"version": 1, "module": module, "decls": ilean}) + "\n" ) - (ilean_dir / f"{module}.ilean").write_text( - json.dumps({"version": 1, "module": module, "decls": ilean}) + "\n", - encoding="utf-8", - ) - return root + return files def generate(request): diff --git a/scripts/make_comparator_workspace.py b/scripts/make_comparator_workspace.py index f8d256cd54..043d1cbffd 100644 --- a/scripts/make_comparator_workspace.py +++ b/scripts/make_comparator_workspace.py @@ -129,12 +129,9 @@ def seam_files(pairs, group=None): [problem for problem, _ in problems], target, template, CONTEXT_DIR ) files = {"request.json": json.dumps(request, indent=2, ensure_ascii=False) + "\n"} - for (problem, ilean), (_, manifest) in zip(problems, pairs): - module = problem["moduleName"] - files[f"{CONTEXT_DIR}/{module}.lean"] = problem["moduleContent"] - files[f"{CONTEXT_DIR}/.lake/build/lib/lean/{module}.ilean"] = ( - json.dumps({"version": 1, "module": module, "decls": ilean}) + "\n" - ) + for path, content in generator_cli.context_files(problems).items(): + files[f"{CONTEXT_DIR}/{path}"] = content + for (problem, _), (_, manifest) in zip(problems, pairs): files[f"{PROVENANCE_STEM}-{problem['id']}.json"] = manifest.to_json() return request, files From 615a0ad175714100a4230767e22c08bd70fbf094 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:30:56 -0400 Subject: [PATCH 34/70] Give the LeanEval adapter one home under comparator/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nine of the files under scripts/ were one subsystem held together by naming prefixes and a map in OWNERSHIP.md. comparator/ already owned the adapter's configuration, templates and documentation, so the code and its tests now live beside them and the directory is the whole program: importer, interface, CLI plumbing, batch audit, extractor, pins, problem files, known failures, docs. Pure moves — git follows the history — plus the path updates in the lakefile, workflows and docs that point at them. --- .github/workflows/build-and-docs.yml | 10 ++++++---- .github/workflows/comparator-lean-4-33.yml | 12 ++++++------ .github/workflows/fc100-audit.yml | 4 ++-- comparator/OWNERSHIP.md | 18 +++++++++--------- comparator/README.md | 8 ++++---- {scripts => comparator}/comparator_facts.lean | 2 +- .../compile_fc100_target.py | 0 .../fc_leaneval_importer.py | 2 +- .../leaneval_generator_cli.py | 2 +- {scripts => comparator}/leaneval_interface.py | 4 ++-- .../make_comparator_workspace.py | 4 ++-- .../test_fc_leaneval_importer.py | 0 .../test_leaneval_interface.py | 0 .../test_make_comparator_workspace.py | 0 lakefile.toml | 2 +- 15 files changed, 35 insertions(+), 33 deletions(-) rename {scripts => comparator}/comparator_facts.lean (99%) rename {scripts => comparator}/compile_fc100_target.py (100%) rename {scripts => comparator}/fc_leaneval_importer.py (99%) rename {scripts => comparator}/leaneval_generator_cli.py (97%) rename {scripts => comparator}/leaneval_interface.py (99%) rename {scripts => comparator}/make_comparator_workspace.py (98%) rename {scripts => comparator}/test_fc_leaneval_importer.py (100%) rename {scripts => comparator}/test_leaneval_interface.py (100%) rename {scripts => comparator}/test_make_comparator_workspace.py (100%) diff --git a/.github/workflows/build-and-docs.yml b/.github/workflows/build-and-docs.yml index 19678a4665..f3224648fe 100644 --- a/.github/workflows/build-and-docs.yml +++ b/.github/workflows/build-and-docs.yml @@ -49,7 +49,9 @@ jobs: python-version: '3.12.9' - name: Run script tests - run: python3 -m unittest discover -s scripts -p 'test_*.py' -v + run: | + python3 -m unittest discover -s scripts -p 'test_*.py' -v + python3 -m unittest discover -s comparator -p 'test_*.py' -v build: runs-on: ubuntu-latest @@ -174,7 +176,7 @@ jobs: KotherConjecture.variants.le_KotherRadical \ OeisA303656.conjecture OeisA308734.conjecture \ curling_number_conjecture; do - python3 scripts/make_comparator_workspace.py "$d" --out .comparator + python3 comparator/make_comparator_workspace.py "$d" --out .comparator done grep -q "large_integers_answer : Prop" .comparator/Erdos940_erdos_940_variants_large_integers/Challenge.lean grep -q "i_answer : ENNReal" .comparator/Erdos1038_erdos_1038_parts_i/Challenge.lean @@ -195,7 +197,7 @@ jobs: - name: Importer to generator seam if: steps.mode.outputs.website_only != 'true' run: | - python3 scripts/make_comparator_workspace.py erdos_1038.parts.i \ + python3 comparator/make_comparator_workspace.py erdos_1038.parts.i \ --emit-import .comparator-import python3 - <<'PY' import json @@ -203,7 +205,7 @@ jobs: import pathlib import sys - sys.path.insert(0, "scripts") + sys.path.insert(0, "comparator") import leaneval_generator_cli as generator_cli from leaneval_interface import ProblemManifest diff --git a/.github/workflows/comparator-lean-4-33.yml b/.github/workflows/comparator-lean-4-33.yml index 137278c353..07b9dd34d1 100644 --- a/.github/workflows/comparator-lean-4-33.yml +++ b/.github/workflows/comparator-lean-4-33.yml @@ -23,11 +23,11 @@ concurrency: on: pull_request: paths: - - 'scripts/fc_leaneval_importer.py' - - 'scripts/leaneval_generator_cli.py' - - 'scripts/leaneval_interface.py' - - 'scripts/make_comparator_workspace.py' - - 'scripts/comparator_facts.lean' + - 'comparator/fc_leaneval_importer.py' + - 'comparator/leaneval_generator_cli.py' + - 'comparator/leaneval_interface.py' + - 'comparator/make_comparator_workspace.py' + - 'comparator/comparator_facts.lean' - 'comparator/**' - 'FormalConjectures/Wikipedia/SumOfThreeCubes.lean' - '.github/workflows/comparator-lean-4-33.yml' @@ -92,7 +92,7 @@ jobs: TARGET_TOOLCHAIN: ${{ steps.target.outputs.lean_toolchain }} run: | for d in isSumOfThreeCubes_2 isSumOfThreeCubes_iff_mod_9; do - python3 scripts/make_comparator_workspace.py "$d" \ + python3 comparator/make_comparator_workspace.py "$d" \ --out .comparator --verify done # One plain theorem and one `answer(sorry)` slot typed at 4.27. diff --git a/.github/workflows/fc100-audit.yml b/.github/workflows/fc100-audit.yml index ae6a1530f1..a2c8f06406 100644 --- a/.github/workflows/fc100-audit.yml +++ b/.github/workflows/fc100-audit.yml @@ -84,7 +84,7 @@ jobs: # exactly the recorded ones. - name: Import and verify the whole set run: | - python3 scripts/make_comparator_workspace.py --set FC100OpenSet1 \ + python3 comparator/make_comparator_workspace.py --set FC100OpenSet1 \ --verify --out .fc100 --report fc100-report.json \ --known-failures comparator/known_failures.toml @@ -112,7 +112,7 @@ jobs: # is built once for the whole set rather than once per workspace. - name: Compile every generated Challenge at LeanEval pins run: | - python3 scripts/compile_fc100_target.py .fc100 \ + python3 comparator/compile_fc100_target.py .fc100 \ --project "$RUNNER_TEMP/fc100-target" \ --report fc100-target-report.json \ --known-failures comparator/known_failures.toml diff --git a/comparator/OWNERSHIP.md b/comparator/OWNERSHIP.md index a3a2984794..17348c5873 100644 --- a/comparator/OWNERSHIP.md +++ b/comparator/OWNERSHIP.md @@ -15,12 +15,12 @@ and declaration id for every problem. ## The seam - scripts/fc_leaneval_importer.py FC declaration -> (module, manifest) - scripts/leaneval_interface.py the request built from them, the + comparator/fc_leaneval_importer.py FC declaration -> (module, manifest) + comparator/leaneval_interface.py the request built from them, the response checked against its digests - scripts/leaneval_generator_cli.py runs the pinned binary, nothing else + comparator/leaneval_generator_cli.py runs the pinned binary, nothing else -`scripts/make_comparator_workspace.py` is the command that runs one after the +`comparator/make_comparator_workspace.py` is the command that runs one after the other. The arrow points one way: the CLI plumbing imports the interface and never the importer, and a test asserts that. @@ -69,11 +69,11 @@ see below. | File | Why it cannot move | |---|---| -| `scripts/fc_leaneval_importer.py` | resolves a declaration against an exact FC commit, reads the elaborated environment, copies the FC-local closure, types each `answer(sorry)` slot, and records the provenance | -| `scripts/comparator_facts.lean` | the Lean extractor: source ranges, binder explicitness, answer-slot types, and the `@[category ...]` tag, all of which only this repository's elaborated environment knows | -| `scripts/leaneval_interface.py` | the request builder and response checker — the FC side of the wire format, permanently, since the consumer owns hole resolution under the v1 contract | -| `scripts/leaneval_generator_cli.py` | plumbing for the pinned binary | -| `scripts/make_comparator_workspace.py` | the command, the emitted seam artifact, and the whole-set batch run | +| `comparator/fc_leaneval_importer.py` | resolves a declaration against an exact FC commit, reads the elaborated environment, copies the FC-local closure, types each `answer(sorry)` slot, and records the provenance | +| `comparator/comparator_facts.lean` | the Lean extractor: source ranges, binder explicitness, answer-slot types, and the `@[category ...]` tag, all of which only this repository's elaborated environment knows | +| `comparator/leaneval_interface.py` | the request builder and response checker — the FC side of the wire format, permanently, since the consumer owns hole resolution under the v1 contract | +| `comparator/leaneval_generator_cli.py` | plumbing for the pinned binary | +| `comparator/make_comparator_workspace.py` | the command, the emitted seam artifact, and the whole-set batch run | | `comparator/templates/WorkspaceTest.lean` | the workspace test template the contract requires the consumer to supply | | `comparator/problems/*.toml` | the one choice FC source cannot make for itself: which module, when two declare the same name | | `comparator/tools.toml` | the pins, in one machine-readable place: this repository's under `[tools]`, LeanEval's under `[target]`, the generator revision under `[generator]` | diff --git a/comparator/README.md b/comparator/README.md index 6f695ebc2d..8eecf5777b 100644 --- a/comparator/README.md +++ b/comparator/README.md @@ -50,7 +50,7 @@ what turns the gap between them into something observed rather than assumed. ## Generate one workspace ```bash -python3 scripts/make_comparator_workspace.py erdos_940.variants.large_integers +python3 comparator/make_comparator_workspace.py erdos_940.variants.large_integers ``` Use `--out` to choose the parent directory. Generation refuses to overwrite an @@ -77,7 +77,7 @@ because it is not in the permitted axiom list. ### Emit only what this repository owns ```bash -python3 scripts/make_comparator_workspace.py erdos_1038.parts.i \ +python3 comparator/make_comparator_workspace.py erdos_1038.parts.i \ --emit-import .comparator-import ``` @@ -90,7 +90,7 @@ which is what makes the seam checkable rather than asserted. ### Import a whole set ```bash -python3 scripts/make_comparator_workspace.py --set FC100OpenSet1 \ +python3 comparator/make_comparator_workspace.py --set FC100OpenSet1 \ --verify --report fc100-report.json \ --known-failures comparator/known_failures.toml ``` @@ -141,7 +141,7 @@ format nobody checks. Run the problem-file check after moving or renaming a declaration: ```bash -python3 scripts/make_comparator_workspace.py --validate +python3 comparator/make_comparator_workspace.py --validate ``` ## Tool pins diff --git a/scripts/comparator_facts.lean b/comparator/comparator_facts.lean similarity index 99% rename from scripts/comparator_facts.lean rename to comparator/comparator_facts.lean index 10811218bb..f73b6a3966 100644 --- a/scripts/comparator_facts.lean +++ b/comparator/comparator_facts.lean @@ -18,7 +18,7 @@ import FormalConjecturesUtil.Answer import FormalConjecturesUtil.Attributes.Basic /-! -The elaborator-side facts `scripts/fc_leaneval_importer.py` would otherwise +The elaborator-side facts `comparator/fc_leaneval_importer.py` would otherwise get by reading Lean with regular expressions. Given a module and a declaration name, this prints JSON with what the diff --git a/scripts/compile_fc100_target.py b/comparator/compile_fc100_target.py similarity index 100% rename from scripts/compile_fc100_target.py rename to comparator/compile_fc100_target.py diff --git a/scripts/fc_leaneval_importer.py b/comparator/fc_leaneval_importer.py similarity index 99% rename from scripts/fc_leaneval_importer.py rename to comparator/fc_leaneval_importer.py index 75475c9820..e8ac03fbe2 100644 --- a/scripts/fc_leaneval_importer.py +++ b/comparator/fc_leaneval_importer.py @@ -8,7 +8,7 @@ declarations it depends on, types each `answer(sorry)` slot, and records where all of that came from. -What it produces is the pair defined in `scripts/leaneval_interface.py`: one +What it produces is the pair defined in `comparator/leaneval_interface.py`: one marked-up Mathlib-only Lean module, and one manifest carrying the FC source commit and declaration id. Turning that pair into a Challenge / Solution / Submission workspace is the pinned `leanprover/lean-eval-generator` binary's diff --git a/scripts/leaneval_generator_cli.py b/comparator/leaneval_generator_cli.py similarity index 97% rename from scripts/leaneval_generator_cli.py rename to comparator/leaneval_generator_cli.py index 35a581bbc3..0f4ab4491e 100644 --- a/scripts/leaneval_generator_cli.py +++ b/comparator/leaneval_generator_cli.py @@ -5,7 +5,7 @@ `leanprover/lean-eval-generator`, a deterministic Lean CLI: one JSON request on stdin, one JSON response on stdout, diagnostics on stderr. This module is the plumbing that runs it and the context directory it expects; everything -the request and response mean lives in `scripts/leaneval_interface.py`, and +the request and response mean lives in `comparator/leaneval_interface.py`, and the pinned revision lives in `comparator/tools.toml` under `[generator]`. The binary is found through `LEAN_EVAL_GENERATOR_BIN` or `PATH`. Building it diff --git a/scripts/leaneval_interface.py b/comparator/leaneval_interface.py similarity index 99% rename from scripts/leaneval_interface.py rename to comparator/leaneval_interface.py index 1e3c8ebebb..317a831fd9 100644 --- a/scripts/leaneval_interface.py +++ b/comparator/leaneval_interface.py @@ -22,8 +22,8 @@ build_request (module, manifest) pairs -> the v1 request object parse_response response text -> file maps, digests checked -`scripts/fc_leaneval_importer.py` produces the pairs. -`scripts/leaneval_generator_cli.py` runs the pinned binary. Nothing on the FC +`comparator/fc_leaneval_importer.py` produces the pairs. +`comparator/leaneval_generator_cli.py` runs the pinned binary. Nothing on the FC side decides a workspace file's contents any more. ## Why one module rather than a bag of strings diff --git a/scripts/make_comparator_workspace.py b/comparator/make_comparator_workspace.py similarity index 98% rename from scripts/make_comparator_workspace.py rename to comparator/make_comparator_workspace.py index 043d1cbffd..8ff83907b0 100644 --- a/scripts/make_comparator_workspace.py +++ b/comparator/make_comparator_workspace.py @@ -11,8 +11,8 @@ The first half is Formal Conjectures'. The second is the pinned `leanprover/lean-eval-generator` binary — a deterministic Lean CLI with a -versioned JSON contract — run by `scripts/leaneval_generator_cli.py` at the -revision `comparator/tools.toml` pins. `scripts/leaneval_interface.py` builds +versioned JSON contract — run by `comparator/leaneval_generator_cli.py` at the +revision `comparator/tools.toml` pins. `comparator/leaneval_interface.py` builds the request and checks the response. `comparator/OWNERSHIP.md` says exactly what belongs to which side. This file is the wiring between them and belongs to neither. diff --git a/scripts/test_fc_leaneval_importer.py b/comparator/test_fc_leaneval_importer.py similarity index 100% rename from scripts/test_fc_leaneval_importer.py rename to comparator/test_fc_leaneval_importer.py diff --git a/scripts/test_leaneval_interface.py b/comparator/test_leaneval_interface.py similarity index 100% rename from scripts/test_leaneval_interface.py rename to comparator/test_leaneval_interface.py diff --git a/scripts/test_make_comparator_workspace.py b/comparator/test_make_comparator_workspace.py similarity index 100% rename from scripts/test_make_comparator_workspace.py rename to comparator/test_make_comparator_workspace.py diff --git a/lakefile.toml b/lakefile.toml index 246104783f..6f505e027f 100644 --- a/lakefile.toml +++ b/lakefile.toml @@ -79,7 +79,7 @@ weak.google.answer = "postpone" [[lean_exe]] name = "comparator_facts" -srcDir = "scripts" +srcDir = "comparator" root = "comparator_facts" exeName = "comparator_facts" supportInterpreter = true From 54abfe518d4484b74bba5470589841bb37e927c6 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:04:01 -0400 Subject: [PATCH 35/70] Separate the adapter's code from its configuration comparator/ now reads at a glance: documentation and the two config ledgers at the root, problem files and templates in their directories, and the program itself - modules, extractor, tests - under adapter/, which is what the upstream plan calls this code. --- .github/workflows/build-and-docs.yml | 8 ++++---- .github/workflows/comparator-lean-4-33.yml | 12 ++++++------ .github/workflows/fc100-audit.yml | 4 ++-- comparator/OWNERSHIP.md | 18 +++++++++--------- comparator/README.md | 8 ++++---- comparator/{ => adapter}/comparator_facts.lean | 2 +- .../{ => adapter}/compile_fc100_target.py | 0 .../{ => adapter}/fc_leaneval_importer.py | 4 ++-- .../{ => adapter}/leaneval_generator_cli.py | 2 +- comparator/{ => adapter}/leaneval_interface.py | 4 ++-- .../{ => adapter}/make_comparator_workspace.py | 4 ++-- .../{ => adapter}/test_fc_leaneval_importer.py | 0 .../{ => adapter}/test_leaneval_interface.py | 0 .../test_make_comparator_workspace.py | 0 lakefile.toml | 2 +- 15 files changed, 34 insertions(+), 34 deletions(-) rename comparator/{ => adapter}/comparator_facts.lean (99%) rename comparator/{ => adapter}/compile_fc100_target.py (100%) rename comparator/{ => adapter}/fc_leaneval_importer.py (99%) rename comparator/{ => adapter}/leaneval_generator_cli.py (97%) rename comparator/{ => adapter}/leaneval_interface.py (99%) rename comparator/{ => adapter}/make_comparator_workspace.py (98%) rename comparator/{ => adapter}/test_fc_leaneval_importer.py (100%) rename comparator/{ => adapter}/test_leaneval_interface.py (100%) rename comparator/{ => adapter}/test_make_comparator_workspace.py (100%) diff --git a/.github/workflows/build-and-docs.yml b/.github/workflows/build-and-docs.yml index f3224648fe..10a5a67646 100644 --- a/.github/workflows/build-and-docs.yml +++ b/.github/workflows/build-and-docs.yml @@ -51,7 +51,7 @@ jobs: - name: Run script tests run: | python3 -m unittest discover -s scripts -p 'test_*.py' -v - python3 -m unittest discover -s comparator -p 'test_*.py' -v + python3 -m unittest discover -s comparator/adapter -p 'test_*.py' -v build: runs-on: ubuntu-latest @@ -176,7 +176,7 @@ jobs: KotherConjecture.variants.le_KotherRadical \ OeisA303656.conjecture OeisA308734.conjecture \ curling_number_conjecture; do - python3 comparator/make_comparator_workspace.py "$d" --out .comparator + python3 comparator/adapter/make_comparator_workspace.py "$d" --out .comparator done grep -q "large_integers_answer : Prop" .comparator/Erdos940_erdos_940_variants_large_integers/Challenge.lean grep -q "i_answer : ENNReal" .comparator/Erdos1038_erdos_1038_parts_i/Challenge.lean @@ -197,7 +197,7 @@ jobs: - name: Importer to generator seam if: steps.mode.outputs.website_only != 'true' run: | - python3 comparator/make_comparator_workspace.py erdos_1038.parts.i \ + python3 comparator/adapter/make_comparator_workspace.py erdos_1038.parts.i \ --emit-import .comparator-import python3 - <<'PY' import json @@ -205,7 +205,7 @@ jobs: import pathlib import sys - sys.path.insert(0, "comparator") + sys.path.insert(0, "comparator/adapter") import leaneval_generator_cli as generator_cli from leaneval_interface import ProblemManifest diff --git a/.github/workflows/comparator-lean-4-33.yml b/.github/workflows/comparator-lean-4-33.yml index 07b9dd34d1..e06b401d6b 100644 --- a/.github/workflows/comparator-lean-4-33.yml +++ b/.github/workflows/comparator-lean-4-33.yml @@ -23,11 +23,11 @@ concurrency: on: pull_request: paths: - - 'comparator/fc_leaneval_importer.py' - - 'comparator/leaneval_generator_cli.py' - - 'comparator/leaneval_interface.py' - - 'comparator/make_comparator_workspace.py' - - 'comparator/comparator_facts.lean' + - 'comparator/adapter/fc_leaneval_importer.py' + - 'comparator/adapter/leaneval_generator_cli.py' + - 'comparator/adapter/leaneval_interface.py' + - 'comparator/adapter/make_comparator_workspace.py' + - 'comparator/adapter/comparator_facts.lean' - 'comparator/**' - 'FormalConjectures/Wikipedia/SumOfThreeCubes.lean' - '.github/workflows/comparator-lean-4-33.yml' @@ -92,7 +92,7 @@ jobs: TARGET_TOOLCHAIN: ${{ steps.target.outputs.lean_toolchain }} run: | for d in isSumOfThreeCubes_2 isSumOfThreeCubes_iff_mod_9; do - python3 comparator/make_comparator_workspace.py "$d" \ + python3 comparator/adapter/make_comparator_workspace.py "$d" \ --out .comparator --verify done # One plain theorem and one `answer(sorry)` slot typed at 4.27. diff --git a/.github/workflows/fc100-audit.yml b/.github/workflows/fc100-audit.yml index a2c8f06406..c0afb798fc 100644 --- a/.github/workflows/fc100-audit.yml +++ b/.github/workflows/fc100-audit.yml @@ -84,7 +84,7 @@ jobs: # exactly the recorded ones. - name: Import and verify the whole set run: | - python3 comparator/make_comparator_workspace.py --set FC100OpenSet1 \ + python3 comparator/adapter/make_comparator_workspace.py --set FC100OpenSet1 \ --verify --out .fc100 --report fc100-report.json \ --known-failures comparator/known_failures.toml @@ -112,7 +112,7 @@ jobs: # is built once for the whole set rather than once per workspace. - name: Compile every generated Challenge at LeanEval pins run: | - python3 comparator/compile_fc100_target.py .fc100 \ + python3 comparator/adapter/compile_fc100_target.py .fc100 \ --project "$RUNNER_TEMP/fc100-target" \ --report fc100-target-report.json \ --known-failures comparator/known_failures.toml diff --git a/comparator/OWNERSHIP.md b/comparator/OWNERSHIP.md index 17348c5873..470ecd7975 100644 --- a/comparator/OWNERSHIP.md +++ b/comparator/OWNERSHIP.md @@ -15,12 +15,12 @@ and declaration id for every problem. ## The seam - comparator/fc_leaneval_importer.py FC declaration -> (module, manifest) - comparator/leaneval_interface.py the request built from them, the + comparator/adapter/fc_leaneval_importer.py FC declaration -> (module, manifest) + comparator/adapter/leaneval_interface.py the request built from them, the response checked against its digests - comparator/leaneval_generator_cli.py runs the pinned binary, nothing else + comparator/adapter/leaneval_generator_cli.py runs the pinned binary, nothing else -`comparator/make_comparator_workspace.py` is the command that runs one after the +`comparator/adapter/make_comparator_workspace.py` is the command that runs one after the other. The arrow points one way: the CLI plumbing imports the interface and never the importer, and a test asserts that. @@ -69,11 +69,11 @@ see below. | File | Why it cannot move | |---|---| -| `comparator/fc_leaneval_importer.py` | resolves a declaration against an exact FC commit, reads the elaborated environment, copies the FC-local closure, types each `answer(sorry)` slot, and records the provenance | -| `comparator/comparator_facts.lean` | the Lean extractor: source ranges, binder explicitness, answer-slot types, and the `@[category ...]` tag, all of which only this repository's elaborated environment knows | -| `comparator/leaneval_interface.py` | the request builder and response checker — the FC side of the wire format, permanently, since the consumer owns hole resolution under the v1 contract | -| `comparator/leaneval_generator_cli.py` | plumbing for the pinned binary | -| `comparator/make_comparator_workspace.py` | the command, the emitted seam artifact, and the whole-set batch run | +| `comparator/adapter/fc_leaneval_importer.py` | resolves a declaration against an exact FC commit, reads the elaborated environment, copies the FC-local closure, types each `answer(sorry)` slot, and records the provenance | +| `comparator/adapter/comparator_facts.lean` | the Lean extractor: source ranges, binder explicitness, answer-slot types, and the `@[category ...]` tag, all of which only this repository's elaborated environment knows | +| `comparator/adapter/leaneval_interface.py` | the request builder and response checker — the FC side of the wire format, permanently, since the consumer owns hole resolution under the v1 contract | +| `comparator/adapter/leaneval_generator_cli.py` | plumbing for the pinned binary | +| `comparator/adapter/make_comparator_workspace.py` | the command, the emitted seam artifact, and the whole-set batch run | | `comparator/templates/WorkspaceTest.lean` | the workspace test template the contract requires the consumer to supply | | `comparator/problems/*.toml` | the one choice FC source cannot make for itself: which module, when two declare the same name | | `comparator/tools.toml` | the pins, in one machine-readable place: this repository's under `[tools]`, LeanEval's under `[target]`, the generator revision under `[generator]` | diff --git a/comparator/README.md b/comparator/README.md index 8eecf5777b..c24971951a 100644 --- a/comparator/README.md +++ b/comparator/README.md @@ -50,7 +50,7 @@ what turns the gap between them into something observed rather than assumed. ## Generate one workspace ```bash -python3 comparator/make_comparator_workspace.py erdos_940.variants.large_integers +python3 comparator/adapter/make_comparator_workspace.py erdos_940.variants.large_integers ``` Use `--out` to choose the parent directory. Generation refuses to overwrite an @@ -77,7 +77,7 @@ because it is not in the permitted axiom list. ### Emit only what this repository owns ```bash -python3 comparator/make_comparator_workspace.py erdos_1038.parts.i \ +python3 comparator/adapter/make_comparator_workspace.py erdos_1038.parts.i \ --emit-import .comparator-import ``` @@ -90,7 +90,7 @@ which is what makes the seam checkable rather than asserted. ### Import a whole set ```bash -python3 comparator/make_comparator_workspace.py --set FC100OpenSet1 \ +python3 comparator/adapter/make_comparator_workspace.py --set FC100OpenSet1 \ --verify --report fc100-report.json \ --known-failures comparator/known_failures.toml ``` @@ -141,7 +141,7 @@ format nobody checks. Run the problem-file check after moving or renaming a declaration: ```bash -python3 comparator/make_comparator_workspace.py --validate +python3 comparator/adapter/make_comparator_workspace.py --validate ``` ## Tool pins diff --git a/comparator/comparator_facts.lean b/comparator/adapter/comparator_facts.lean similarity index 99% rename from comparator/comparator_facts.lean rename to comparator/adapter/comparator_facts.lean index f73b6a3966..a9cf3d9ec3 100644 --- a/comparator/comparator_facts.lean +++ b/comparator/adapter/comparator_facts.lean @@ -18,7 +18,7 @@ import FormalConjecturesUtil.Answer import FormalConjecturesUtil.Attributes.Basic /-! -The elaborator-side facts `comparator/fc_leaneval_importer.py` would otherwise +The elaborator-side facts `comparator/adapter/fc_leaneval_importer.py` would otherwise get by reading Lean with regular expressions. Given a module and a declaration name, this prints JSON with what the diff --git a/comparator/compile_fc100_target.py b/comparator/adapter/compile_fc100_target.py similarity index 100% rename from comparator/compile_fc100_target.py rename to comparator/adapter/compile_fc100_target.py diff --git a/comparator/fc_leaneval_importer.py b/comparator/adapter/fc_leaneval_importer.py similarity index 99% rename from comparator/fc_leaneval_importer.py rename to comparator/adapter/fc_leaneval_importer.py index e8ac03fbe2..4309206514 100644 --- a/comparator/fc_leaneval_importer.py +++ b/comparator/adapter/fc_leaneval_importer.py @@ -8,7 +8,7 @@ declarations it depends on, types each `answer(sorry)` slot, and records where all of that came from. -What it produces is the pair defined in `comparator/leaneval_interface.py`: one +What it produces is the pair defined in `comparator/adapter/leaneval_interface.py`: one marked-up Mathlib-only Lean module, and one manifest carrying the FC source commit and declaration id. Turning that pair into a Challenge / Solution / Submission workspace is the pinned `leanprover/lean-eval-generator` binary's @@ -39,7 +39,7 @@ TargetRecord, ) -ROOT = pathlib.Path(__file__).resolve().parent.parent +ROOT = pathlib.Path(__file__).resolve().parent.parent.parent SOURCE_DIRS = [ROOT / "FormalConjectures"] COMPARATOR_DIR = ROOT / "comparator" MANIFEST_DIR = COMPARATOR_DIR / "problems" diff --git a/comparator/leaneval_generator_cli.py b/comparator/adapter/leaneval_generator_cli.py similarity index 97% rename from comparator/leaneval_generator_cli.py rename to comparator/adapter/leaneval_generator_cli.py index 0f4ab4491e..5238d24b4c 100644 --- a/comparator/leaneval_generator_cli.py +++ b/comparator/adapter/leaneval_generator_cli.py @@ -5,7 +5,7 @@ `leanprover/lean-eval-generator`, a deterministic Lean CLI: one JSON request on stdin, one JSON response on stdout, diagnostics on stderr. This module is the plumbing that runs it and the context directory it expects; everything -the request and response mean lives in `comparator/leaneval_interface.py`, and +the request and response mean lives in `comparator/adapter/leaneval_interface.py`, and the pinned revision lives in `comparator/tools.toml` under `[generator]`. The binary is found through `LEAN_EVAL_GENERATOR_BIN` or `PATH`. Building it diff --git a/comparator/leaneval_interface.py b/comparator/adapter/leaneval_interface.py similarity index 99% rename from comparator/leaneval_interface.py rename to comparator/adapter/leaneval_interface.py index 317a831fd9..a6513394b7 100644 --- a/comparator/leaneval_interface.py +++ b/comparator/adapter/leaneval_interface.py @@ -22,8 +22,8 @@ build_request (module, manifest) pairs -> the v1 request object parse_response response text -> file maps, digests checked -`comparator/fc_leaneval_importer.py` produces the pairs. -`comparator/leaneval_generator_cli.py` runs the pinned binary. Nothing on the FC +`comparator/adapter/fc_leaneval_importer.py` produces the pairs. +`comparator/adapter/leaneval_generator_cli.py` runs the pinned binary. Nothing on the FC side decides a workspace file's contents any more. ## Why one module rather than a bag of strings diff --git a/comparator/make_comparator_workspace.py b/comparator/adapter/make_comparator_workspace.py similarity index 98% rename from comparator/make_comparator_workspace.py rename to comparator/adapter/make_comparator_workspace.py index 8ff83907b0..87ffedb96d 100644 --- a/comparator/make_comparator_workspace.py +++ b/comparator/adapter/make_comparator_workspace.py @@ -11,8 +11,8 @@ The first half is Formal Conjectures'. The second is the pinned `leanprover/lean-eval-generator` binary — a deterministic Lean CLI with a -versioned JSON contract — run by `comparator/leaneval_generator_cli.py` at the -revision `comparator/tools.toml` pins. `comparator/leaneval_interface.py` builds +versioned JSON contract — run by `comparator/adapter/leaneval_generator_cli.py` at the +revision `comparator/tools.toml` pins. `comparator/adapter/leaneval_interface.py` builds the request and checks the response. `comparator/OWNERSHIP.md` says exactly what belongs to which side. This file is the wiring between them and belongs to neither. diff --git a/comparator/test_fc_leaneval_importer.py b/comparator/adapter/test_fc_leaneval_importer.py similarity index 100% rename from comparator/test_fc_leaneval_importer.py rename to comparator/adapter/test_fc_leaneval_importer.py diff --git a/comparator/test_leaneval_interface.py b/comparator/adapter/test_leaneval_interface.py similarity index 100% rename from comparator/test_leaneval_interface.py rename to comparator/adapter/test_leaneval_interface.py diff --git a/comparator/test_make_comparator_workspace.py b/comparator/adapter/test_make_comparator_workspace.py similarity index 100% rename from comparator/test_make_comparator_workspace.py rename to comparator/adapter/test_make_comparator_workspace.py diff --git a/lakefile.toml b/lakefile.toml index 6f505e027f..16bd4bdc2b 100644 --- a/lakefile.toml +++ b/lakefile.toml @@ -79,7 +79,7 @@ weak.google.answer = "postpone" [[lean_exe]] name = "comparator_facts" -srcDir = "comparator" +srcDir = "comparator/adapter" root = "comparator_facts" exeName = "comparator_facts" supportInterpreter = true From cef02049664622e48b62a247ebaa8a4afe5c04bb Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Fri, 21 Aug 2026 13:58:51 -0400 Subject: [PATCH 36/70] Make the provenance sidecar strict, deterministic and digested lean-eval keeps the v1 generator wire format frozen, so fc-provenance.json is the v1 provenance boundary by design rather than a stopgap. A record with a key the schema does not name is refused on load; serialisation is key-sorted, so the same record is always the same bytes; and the record now carries the SHA-256 of the exact moduleContent that crossed the seam and of every generated file the response returned, so a workspace holds its own chain from FC commit to module bytes to generated bytes. --- comparator/OWNERSHIP.md | 26 ++++++--- comparator/adapter/leaneval_interface.py | 54 ++++++++++++++++++- .../adapter/make_comparator_workspace.py | 17 ++++-- comparator/adapter/test_leaneval_interface.py | 27 ++++++++++ 4 files changed, 111 insertions(+), 13 deletions(-) diff --git a/comparator/OWNERSHIP.md b/comparator/OWNERSHIP.md index 470ecd7975..94d2dc2969 100644 --- a/comparator/OWNERSHIP.md +++ b/comparator/OWNERSHIP.md @@ -56,14 +56,24 @@ file, and every digest is checked before a byte lands on disk. lean-eval#536 requires each imported problem to record the FC source commit and declaration id. The v1 wire format has no field for either — its optional -`source` is one free-text line — so the manifest this repository always built -(`ProblemManifest`: commit, path, blob, module, declaration, copied -dependencies, the pins the hole types were read at) is written **beside** the -generated workspace as `fc-provenance.json`, and beside the emitted request as -`fc-provenance-.json`. It is also what makes regeneration possible when -Formal Conjectures corrects a misformalisation upstream. Whether v2 of the -contract should carry these fields itself is an open question for lean-eval; -see below. +`source` is one free-text line — and lean-eval keeps that format frozen on +purpose, so the sidecar is the v1 provenance boundary **by design**, not a +stopgap (kim-em on #4951, 2026-08-21); a typed provenance object is a v2 +matter and does not gate the FC100 import. The manifest this repository +builds (`ProblemManifest`: commit, path, blob, module, declaration, copied +dependencies, the pins the hole types were read at) is written beside the +generated workspace as `fc-provenance.json`, and beside the emitted request +as `fc-provenance-.json`. + +Three properties make it fit to be that boundary. It is **strict**: a record +with a key the schema does not name is refused on load. It is +**deterministic**: serialisation is key-sorted, so the same record is the +same bytes. And it is **digested**: it carries the SHA-256 of the exact +`moduleContent` that crossed the seam and of every generated file the +response returned, so a workspace holds its own chain — FC commit → module +bytes → generated bytes — and each link can be checked without this +repository. It is also what makes regeneration possible when Formal +Conjectures corrects a misformalisation upstream. ## What stays Formal Conjectures' permanently diff --git a/comparator/adapter/leaneval_interface.py b/comparator/adapter/leaneval_interface.py index a6513394b7..07267097fe 100644 --- a/comparator/adapter/leaneval_interface.py +++ b/comparator/adapter/leaneval_interface.py @@ -66,6 +66,11 @@ MANIFEST_SCHEMA_VERSION = 1 + +def sha256_text(text): + """The digest the generator response uses: SHA-256 of the UTF-8 bytes.""" + return hashlib.sha256(text.encode("utf-8")).hexdigest() + # The generator's frozen wire format; `schemas/request-v1.schema.json` and # `response-v1.schema.json` in the pinned revision are normative. CONTRACT_VERSION = 1 @@ -172,6 +177,24 @@ class ProblemManifest: # is decided by this and nothing else; recording the raw tag rather than # the mapped group keeps the mapping in one place, beside the request. category: str = "" + # Digests bind the record to bytes: the exact `moduleContent` that crossed + # the seam, and every generated file the response returned for it. With + # them a workspace carries its own chain — FC commit → module bytes → + # generated bytes — and a reader can check each link without this + # repository. The sidecar is the v1 provenance boundary by design + # (lean-eval-generator keeps its wire format frozen), so it has to be + # strict and deterministic as well: unknown keys are refused on load and + # serialisation is key-sorted. + module_sha256: str = "" + file_sha256: tuple = () + + def with_digests(self, module_sha256, files): + """The same manifest, bound to the module bytes and generated files.""" + return dataclasses.replace( + self, + module_sha256=module_sha256, + file_sha256=tuple(sorted((path, digest) for path, digest in files.items())), + ) def __post_init__(self): for field in ("id", "theorem", "qualified_theorem"): @@ -205,8 +228,21 @@ def to_json_object(self): } if self.source_url: payload["source_url"] = self.source_url + if self.module_sha256: + payload["digests"] = { + "module": self.module_sha256, + "files": dict(self.file_sha256), + } return payload + KNOWN_KEYS = frozenset( + { + "schema_version", "id", "theorem", "qualified_theorem", "category", + "apply_arguments", "holes", "permitted_axioms", "source", + "source_url", "digests", + } + ) + @classmethod def from_json_object(cls, payload): version = payload.get("schema_version") @@ -215,8 +251,18 @@ def from_json_object(cls, payload): f"manifest schema version {version!r} is not " f"{MANIFEST_SCHEMA_VERSION}" ) + unknown = sorted(set(payload) - cls.KNOWN_KEYS) + if unknown: + raise SystemExit(f"provenance record has unknown keys: {', '.join(unknown)}") source = dict(payload["source"]) source["copied_dependencies"] = tuple(source["copied_dependencies"]) + unknown = sorted(set(source) - {f.name for f in dataclasses.fields(SourceRecord)}) + if unknown: + raise SystemExit(f"provenance source has unknown keys: {', '.join(unknown)}") + digests = dict(payload.get("digests", {})) + unknown = sorted(set(digests) - {"module", "files"}) + if unknown: + raise SystemExit(f"provenance digests have unknown keys: {', '.join(unknown)}") return cls( id=payload["id"], theorem=payload["theorem"], @@ -227,10 +273,16 @@ def from_json_object(cls, payload): source=SourceRecord(**source), source_url=payload.get("source_url", ""), category=payload.get("category", ""), + module_sha256=digests.get("module", ""), + file_sha256=tuple(sorted(dict(digests.get("files", {})).items())), ) def to_json(self): - return json.dumps(self.to_json_object(), indent=2, ensure_ascii=False) + "\n" + # Key-sorted: the same record always serialises to the same bytes. + return ( + json.dumps(self.to_json_object(), indent=2, ensure_ascii=False, sort_keys=True) + + "\n" + ) @classmethod def from_json(cls, text): diff --git a/comparator/adapter/make_comparator_workspace.py b/comparator/adapter/make_comparator_workspace.py index 87ffedb96d..ff943c2daf 100644 --- a/comparator/adapter/make_comparator_workspace.py +++ b/comparator/adapter/make_comparator_workspace.py @@ -69,7 +69,7 @@ import fc_leaneval_importer as importer import leaneval_generator_cli as generator_cli -from leaneval_interface import build_problem, build_request, slug +from leaneval_interface import build_problem, build_request, slug, sha256_text ROOT = importer.ROOT @@ -132,7 +132,10 @@ def seam_files(pairs, group=None): for path, content in generator_cli.context_files(problems).items(): files[f"{CONTEXT_DIR}/{path}"] = content for (problem, _), (_, manifest) in zip(problems, pairs): - files[f"{PROVENANCE_STEM}-{problem['id']}.json"] = manifest.to_json() + # Before generation the record binds the module bytes only; the + # workspace's copy adds the generated files. + bound = manifest.with_digests(sha256_text(problem["moduleContent"]), {}) + files[f"{PROVENANCE_STEM}-{problem['id']}.json"] = bound.to_json() return request, files @@ -153,6 +156,7 @@ def generate_workspaces(pairs, out_dir, group=None): workspaces = generator_cli.generate(request) finally: shutil.rmtree(staging, ignore_errors=True) + module_content = {p["id"]: p["moduleContent"] for p in request["problems"]} written = [] for _, manifest in pairs: problem_id = slug(manifest.id) @@ -160,8 +164,13 @@ def generate_workspaces(pairs, out_dir, group=None): raise SystemExit(f"the generator returned no files for {problem_id}") workspace = dict(workspaces[problem_id]) # The provenance sidecar rides in the workspace directory, not in the - # generator's file map: the generator neither knows nor checks it. - workspace[PROVENANCE_FILE] = manifest.to_json() + # generator's file map: the generator neither knows nor checks it. It + # binds the exact module bytes sent and every file received. + bound = manifest.with_digests( + sha256_text(module_content[problem_id]), + {path: sha256_text(content) for path, content in workspace.items()}, + ) + workspace[PROVENANCE_FILE] = bound.to_json() written.append(write_tree(pathlib.Path(out_dir) / problem_id, workspace)) return written diff --git a/comparator/adapter/test_leaneval_interface.py b/comparator/adapter/test_leaneval_interface.py index a1b2caf080..67326281d4 100644 --- a/comparator/adapter/test_leaneval_interface.py +++ b/comparator/adapter/test_leaneval_interface.py @@ -309,3 +309,30 @@ def test_a_damaged_digest_is_refused(self): def test_an_unknown_schema_version_is_refused(self): with self.assertRaisesRegex(SystemExit, "schema version"): parse_response('{"schemaVersion": 2, "files": []}') + + +class ProvenanceSidecarTest(unittest.TestCase): + """The sidecar is the v1 provenance boundary: strict, deterministic, digested.""" + + def test_digests_round_trip(self): + bound = a_manifest().with_digests("a" * 64, {"Challenge.lean": "b" * 64, "A.lean": "c" * 64}) + again = ProblemManifest.from_json(bound.to_json()) + self.assertEqual(again.module_sha256, "a" * 64) + self.assertEqual(dict(again.file_sha256), {"A.lean": "c" * 64, "Challenge.lean": "b" * 64}) + + def test_serialisation_is_key_sorted(self): + text = a_manifest().with_digests("a" * 64, {"b": "1" * 64, "a": "2" * 64}).to_json() + keys = [line.split('"')[1] for line in text.splitlines() if line.startswith(' "')] + self.assertEqual(keys, sorted(keys)) + + def test_unknown_keys_are_refused(self): + payload = a_manifest().to_json_object() + payload["notes"] = "anything" + with self.assertRaises(SystemExit): + ProblemManifest.from_json_object(payload) + + def test_unknown_digest_keys_are_refused(self): + payload = a_manifest().with_digests("a" * 64, {}).to_json_object() + payload["digests"]["request"] = "d" * 64 + with self.assertRaises(SystemExit): + ProblemManifest.from_json_object(payload) From 0bfb9f7a18a3f3bc97078ff25f51704245801008 Mon Sep 17 00:00:00 2001 From: Kim Morrison Date: Fri, 21 Aug 2026 23:12:47 +0000 Subject: [PATCH 37/70] fix(comparator): preserve declaration binder boundaries --- .github/workflows/build-and-docs.yml | 3 +- .github/workflows/comparator-lean-4-33.yml | 1 + .github/workflows/fc100-audit.yml | 1 + comparator/OWNERSHIP.md | 4 +- comparator/adapter/comparator_facts.lean | 466 +++++++++++++++++- comparator/adapter/fc_leaneval_importer.py | 88 +++- .../adapter/test_fc_leaneval_importer.py | 62 +++ ...rdos_340.variants.co_density_zero_sub.toml | 18 + 8 files changed, 607 insertions(+), 36 deletions(-) create mode 100644 comparator/problems/Erdos340.erdos_340.variants.co_density_zero_sub.toml diff --git a/.github/workflows/build-and-docs.yml b/.github/workflows/build-and-docs.yml index 10a5a67646..3c22bd68f4 100644 --- a/.github/workflows/build-and-docs.yml +++ b/.github/workflows/build-and-docs.yml @@ -114,7 +114,7 @@ jobs: set -o pipefail curl -sSfL https://github.com/leanprover/elan/releases/download/v1.4.2/elan-x86_64-unknown-linux-gnu.tar.gz | tar xz ./elan-init -y --default-toolchain none - echo "$HOME/.elan/bin" >> $GITHUB_PATH + echo "$HOME/.elan/bin" >> "$GITHUB_PATH" - name: Restore ~/.cache/mathlib if: steps.mode.outputs.website_only != 'true' @@ -171,6 +171,7 @@ jobs: if: steps.mode.outputs.website_only != 'true' run: | lake build comparator_facts + lake exe comparator_facts --self-test for d in exists_hadamard_zero erdos_940.variants.large_integers \ erdos_1038.parts.i erdos_100.variants.strong \ KotherConjecture.variants.le_KotherRadical \ diff --git a/.github/workflows/comparator-lean-4-33.yml b/.github/workflows/comparator-lean-4-33.yml index e06b401d6b..9d51b7fd1a 100644 --- a/.github/workflows/comparator-lean-4-33.yml +++ b/.github/workflows/comparator-lean-4-33.yml @@ -77,6 +77,7 @@ jobs: run: | lake exe cache get lake build comparator_facts FormalConjectures.Wikipedia.SumOfThreeCubes + lake exe comparator_facts --self-test - name: Build the pinned lean-eval-generator uses: ./.github/actions/build-lean-eval-generator diff --git a/.github/workflows/fc100-audit.yml b/.github/workflows/fc100-audit.yml index c0afb798fc..6f6c0b0b77 100644 --- a/.github/workflows/fc100-audit.yml +++ b/.github/workflows/fc100-audit.yml @@ -74,6 +74,7 @@ jobs: run: | lake exe cache get lake build FormalConjectures.Subsets.FC100OpenSet1 comparator_facts + lake exe comparator_facts --self-test - name: Build the pinned lean-eval-generator uses: ./.github/actions/build-lean-eval-generator diff --git a/comparator/OWNERSHIP.md b/comparator/OWNERSHIP.md index 94d2dc2969..91197d1822 100644 --- a/comparator/OWNERSHIP.md +++ b/comparator/OWNERSHIP.md @@ -80,12 +80,12 @@ Conjectures corrects a misformalisation upstream. | File | Why it cannot move | |---|---| | `comparator/adapter/fc_leaneval_importer.py` | resolves a declaration against an exact FC commit, reads the elaborated environment, copies the FC-local closure, types each `answer(sorry)` slot, and records the provenance | -| `comparator/adapter/comparator_facts.lean` | the Lean extractor: source ranges, binder explicitness, answer-slot types, and the `@[category ...]` tag, all of which only this repository's elaborated environment knows | +| `comparator/adapter/comparator_facts.lean` | the Lean extractor: source ranges, declaration-header binder boundaries, elaborated binder names/explicitness, answer-slot types, and the `@[category ...]` tag. The parsed source distinguishes header parameters from `∀` binders in the conclusion; every emitted binder fact still comes from the elaborated environment. | | `comparator/adapter/leaneval_interface.py` | the request builder and response checker — the FC side of the wire format, permanently, since the consumer owns hole resolution under the v1 contract | | `comparator/adapter/leaneval_generator_cli.py` | plumbing for the pinned binary | | `comparator/adapter/make_comparator_workspace.py` | the command, the emitted seam artifact, and the whole-set batch run | | `comparator/templates/WorkspaceTest.lean` | the workspace test template the contract requires the consumer to supply | -| `comparator/problems/*.toml` | the one choice FC source cannot make for itself: which module, when two declare the same name | +| `comparator/problems/*.toml` | the rare source-boundary facts the compiled environment cannot recover: which module when two declare the same name, and an explicit copied proof dependency when opaque theorem-value erasure removes it from the compiled dependency graph | | `comparator/tools.toml` | the pins, in one machine-readable place: this repository's under `[tools]`, LeanEval's under `[target]`, the generator revision under `[generator]` | The tests beside each file pin real defects: the importer suite covers diff --git a/comparator/adapter/comparator_facts.lean b/comparator/adapter/comparator_facts.lean index a9cf3d9ec3..2ed5d4d5d6 100644 --- a/comparator/adapter/comparator_facts.lean +++ b/comparator/adapter/comparator_facts.lean @@ -25,7 +25,8 @@ Given a module and a declaration name, this prints JSON with what the elaborated environment knows exactly and the text layer can only guess: - the declaration's source range, for slicing its original text; -- its binders, with names and explicitness, for the Solution adapter; +- its declaration-header binders, with names and explicitness, for the + Solution adapter; - the type of each `sorry` inside the *statement*, which is the type of an `answer(sorry)` slot. The `answer_type` field in a `comparator/problems` file exists only because surface syntax does not carry this; the @@ -45,17 +46,406 @@ def declares (declared : Name) (requested : String) : Bool := let s := declared.toString s == requested || s.endsWith ("." ++ requested) -/-- Declaration parameters, as opposed to `∀` binders in the conclusion. +/-- Collect every node of one syntax kind. A declaration command must contain +exactly one `declSig`; finding zero or more than one means we did not parse the +source range we thought we did, and the importer fails closed. -/ +partial def collectKind (kind : Name) (stx : Syntax) (found : Array Syntax := #[]) : + Array Syntax := + let found := if stx.isOfKind kind then found.push stx else found + stx.getArgs.foldl (fun acc child => collectKind kind child acc) found -`theorem foo (n : Nat) : P n` lambda-abstracts `n` in its proof value; -`theorem foo : ∀ n : Nat, P n` does not. Only the former are applied by the -generated Solution adapter, and `forallTelescope` alone cannot tell them -apart: the lambda arity of the (sorry) value can. lean-eval's extractor -draws the same line for the same reason. -/ -partial def lambdaArity : Expr → Nat - | .lam _ _ b _ => lambdaArity b + 1 - | .mdata _ b => lambdaArity b - | _ => 0 +structure SourceBinder where + name? : Option Name + info : BinderInfo + deriving Repr + +def sourceBinderName (stx : Syntax) : Except String (Option Name) := + if stx.isIdent then .ok (some stx.getId) + else if stx.isOfKind ``Lean.Parser.Term.hole then .ok none + else .error s!"unsupported declaration binder name {stx.getKind}: {stx}" + +/-- The names and binder kinds introduced by one declaration-header group. -/ +def declarationBinderGroup (binder : Syntax) : Except String (Array SourceBinder) := do + if binder.isIdent || binder.isOfKind ``Lean.Parser.Term.hole then + return #[{ name? := ← sourceBinderName binder, info := .default }] + if binder.isOfKind ``Lean.Parser.Term.instBinder then + let some optionalName := binder[1]? + | throw s!"malformed instance binder: {binder}" + let name? ← match optionalName.getArgs[0]? with + | some name => sourceBinderName name + | none => pure none + return #[{ name?, info := .instImplicit }] + let info? := + if binder.isOfKind ``Lean.Parser.Term.explicitBinder then some BinderInfo.default + else if binder.isOfKind ``Lean.Parser.Term.implicitBinder then some .implicit + else if binder.isOfKind ``Lean.Parser.Term.strictImplicitBinder then some .strictImplicit + else none + let some info := info? + | throw s!"unsupported declaration binder syntax {binder.getKind}: {binder}" + let some names := binder[1]? + | throw s!"malformed declaration binder: {binder}" + if names.getArgs.isEmpty then + throw s!"declaration binder has no names: {binder}" + names.getArgs.mapM fun name => do + return { name? := ← sourceBinderName name, info } + +def declarationSignature (command : Syntax) : Except String Syntax := do + let signatures := collectKind ``Lean.Parser.Command.declSig command + let [signature] := signatures.toList + | throw s!"expected exactly one declaration signature, found {signatures.size}" + return signature + +def declarationBinders (command : Syntax) : Except String (Array SourceBinder) := do + let signature ← declarationSignature command + let some header := signature[0]? + | throw "declaration signature has no binder header" + header.getArgs.foldlM (fun binders binder => do + return binders ++ (← declarationBinderGroup binder)) #[] + +/-- Number of parameters written before the colon in a declaration header. + +This is deliberately a syntax boundary, not a proof-value heuristic: +`theorem foo (n : Nat) : P n` has one declaration parameter, while +`theorem foo : ∀ n : Nat, P n` has none. Lean 4.32 may store a theorem proved +by `sorry` as a `sorryAx` at the full forall type, with no lambda wrappers, so +the old proof-value lambda arity silently reported zero for the first form. +The syntax supplies only the boundary; names, types, and explicitness still +come from the elaborated telescope below. -/ +def declarationBinderCount (command : Syntax) : Except String Nat := do + return (← declarationBinders command).size + +def sourceBinderMatches (source : SourceBinder) (elaborated : LocalDecl) : Bool := + source.info == elaborated.binderInfo && match source.name? with + | some name => name == elaborated.userName + | none => true + +def sourceBindersMatchAt (source : Array SourceBinder) (elaborated : Array LocalDecl) + (start : Nat) : Bool := + source.zipIdx.all fun (binder, offset) => + match elaborated[start + offset]? with + | some candidate => sourceBinderMatches binder candidate + | none => false + +/-- Locate the end of the declaration parameters in the elaborated telescope. + +Lean inserts used outer `variable`s before the binders written in the +declaration header. We therefore locate the exact typed/named header sequence +inside the telescope and retain everything through its end. This handles +Köthe's `{R} [Ring R]` outer parameters as well as ordinary self-contained +headers. Multiple matches are rejected rather than guessed. -/ +def declarationParameterBoundary (command : Syntax) (conclusion : Array SourceBinder) + (elaborated : Array LocalDecl) : Except String Nat := do + let source ← declarationBinders command + if source.isEmpty then + if conclusion.size > elaborated.size then + throw s!"source conclusion has {conclusion.size} binders, but the elaborated type has only {elaborated.size}" + let boundary := elaborated.size - conclusion.size + unless sourceBindersMatchAt conclusion elaborated boundary do + throw s!"source conclusion binders {repr conclusion} do not match the elaborated telescope suffix" + return boundary + if source.size > elaborated.size then + throw s!"source header has {source.size} binders, but the elaborated type has only {elaborated.size}" + let starts := (Array.range (elaborated.size - source.size + 1)).filter + (sourceBindersMatchAt source elaborated) + let [start] := starts.toList + | throw s!"expected one match for {source.size} source-header binders in the elaborated telescope, found {starts.size}" + return start + source.size + +def parseDeclarationBinderCount (env : Environment) (source : String) : Except String Nat := do + let command ← Parser.runParserCategory env `command source + declarationBinderCount command + +inductive ScanToken where + | signatureColon + | bodyMarker + | comma + | arrow + | iff + | openParen + | openBrace + | openBracket + | openStrict + | closeParen + | closeBrace + | closeBracket + | closeStrict + +def scanTokenAt (wanted : ScanToken) (chars : Array Char) (i : Nat) : Option Nat := + match wanted with + | .signatureColon => + if chars[i]? == some ':' && chars[i + 1]? != some '=' then some 1 else none + | .bodyMarker => + if chars[i]? == some ':' && chars[i + 1]? == some '=' then some 2 else none + | .comma => if chars[i]? == some ',' then some 1 else none + | .arrow => + if chars[i]? == some '→' then some 1 + else if chars[i]? == some '-' && chars[i + 1]? == some '>' then some 2 + else none + | .iff => if chars[i]? == some '↔' then some 1 else none + | .openParen => if chars[i]? == some '(' then some 1 else none + | .openBrace => if chars[i]? == some '{' then some 1 else none + | .openBracket => if chars[i]? == some '[' then some 1 else none + | .openStrict => if chars[i]? == some '⦃' then some 1 else none + | .closeParen => if chars[i]? == some ')' then some 1 else none + | .closeBrace => if chars[i]? == some '}' then some 1 else none + | .closeBracket => if chars[i]? == some ']' then some 1 else none + | .closeStrict => if chars[i]? == some '⦄' then some 1 else none + +/-- Find syntax punctuation at delimiter depth zero, ignoring comments and +strings. This scanner does not interpret terms; it only lets us replace the +result type with `True` before asking Lean's real command parser to read the +declaration header. Scoped notation in the result therefore cannot make an +otherwise ordinary header unparseable. -/ +partial def findTopLevelToken (wanted : ScanToken) (text : String) : Option (Nat × Nat) := + let chars := text.toList.toArray + let rec loop (i paren brace bracket strict blockComment : Nat) + (lineComment inString escaped : Bool) : Option (Nat × Nat) := + if i >= chars.size then none else + let current := chars[i]! + let next := chars[i + 1]? + if lineComment then + loop (i + 1) paren brace bracket strict blockComment (current != '\n') inString false + else if blockComment > 0 then + if current == '/' && next == some '-' then + loop (i + 2) paren brace bracket strict (blockComment + 1) false inString false + else if current == '-' && next == some '/' then + loop (i + 2) paren brace bracket strict (blockComment - 1) false inString false + else + loop (i + 1) paren brace bracket strict blockComment false inString false + else if inString then + if escaped then loop (i + 1) paren brace bracket strict 0 false true false + else if current == '\\' then loop (i + 1) paren brace bracket strict 0 false true true + else loop (i + 1) paren brace bracket strict 0 false (current != '"') false + else if current == '-' && next == some '-' then + loop (i + 2) paren brace bracket strict 0 true false false + else if current == '/' && next == some '-' then + loop (i + 2) paren brace bracket strict 1 false false false + else if current == '"' then + loop (i + 1) paren brace bracket strict 0 false true false + else if paren == 0 && brace == 0 && bracket == 0 && strict == 0 then + match scanTokenAt wanted chars i with + | some width => some (i, width) + | none => match current with + | '(' => loop (i + 1) 1 brace bracket strict 0 false false false + | '{' => loop (i + 1) paren 1 bracket strict 0 false false false + | '[' => loop (i + 1) paren brace 1 strict 0 false false false + | '⦃' => loop (i + 1) paren brace bracket 1 0 false false false + | _ => loop (i + 1) paren brace bracket strict 0 false false false + else match current with + | '(' => loop (i + 1) (paren + 1) brace bracket strict 0 false false false + | ')' => loop (i + 1) (paren - 1) brace bracket strict 0 false false false + | '{' => loop (i + 1) paren (brace + 1) bracket strict 0 false false false + | '}' => loop (i + 1) paren (brace - 1) bracket strict 0 false false false + | '[' => loop (i + 1) paren brace (bracket + 1) strict 0 false false false + | ']' => loop (i + 1) paren brace (bracket - 1) strict 0 false false false + | '⦃' => loop (i + 1) paren brace bracket (strict + 1) 0 false false false + | '⦄' => loop (i + 1) paren brace bracket (strict - 1) 0 false false false + | _ => loop (i + 1) paren brace bracket strict 0 false false false + loop 0 0 0 0 0 0 false false false + +def sliceChars (text : String) (start stop : Nat) : String := + String.ofList (text.toList.toArray.extract start stop).toList + +def forallBinders (term : Syntax) : Except String (Array SourceBinder) := do + let some marker := term[0]? + | throw s!"malformed forall syntax: {term}" + unless marker.isAtom && (marker.getAtomVal == "∀" || marker.getAtomVal == "forall") do + throw s!"expected forall syntax, got {term.getKind}: {term}" + let some binderSlot := term[1]? + | throw s!"forall syntax has no binder: {term}" + let mut binders := #[] + if binderSlot.isOfKind `null then + for binder in binderSlot.getArgs do + binders := binders ++ (← declarationBinderGroup binder) + else if binderSlot.isIdent then + binders := binders.push { name? := some binderSlot.getId, info := .default } + else + let some name := binderSlot.getArgs[0]? + | throw s!"unsupported forall binder syntax {binderSlot.getKind}: {binderSlot}" + binders := binders.push { name? := ← sourceBinderName name, info := .default } + let some predicate := term[2]? + | throw s!"forall syntax has no predicate slot: {term}" + let isBareTypeSpec := predicate.getArgs[0]?.any + (·.isOfKind ``Lean.Parser.Term.typeSpec) + if !predicate.isNone && !isBareTypeSpec then + binders := binders.push { name? := none, info := .default } + return binders + +/-- Parse only the leading Pi structure of a result type. The remainder is +replaced with `True` before parsing, so scoped term notation later in the +statement is irrelevant. -/ +partial def conclusionBinders (env : Environment) (text : String) : Except String (Array SourceBinder) := do + let text := text.trimAsciiStart.toString + let chars := text.toList.toArray + if chars[0]? == some '(' then + let afterOpen := sliceChars text 1 text.length + match findTopLevelToken .closeParen afterOpen with + | some (close, width) => + let trailing := sliceChars afterOpen (close + width) afterOpen.length + if trailing.trimAscii.isEmpty then + return ← conclusionBinders env (sliceChars afterOpen 0 close) + | none => pure () + let unicodeForall := chars[0]? == some '∀' && chars[1]?.any fun c => + c.isWhitespace || c == '(' || c == '{' || c == '[' || c == '⦃' + let asciiForall := text.startsWith "forall" && chars[6]?.any fun c => + c.isWhitespace || c == '(' || c == '{' || c == '[' || c == '⦃' + if unicodeForall || asciiForall then + let some (comma, width) := findTopLevelToken .comma text + | throw "leading forall has no top-level comma" + let forallPrefix := sliceChars text 0 (comma + width) + let command ← Parser.runParserCategory env `command + ("theorem _boundary : " ++ forallPrefix ++ " True := by trivial") + let signature ← declarationSignature command + let some typeSpec := signature[1]? + | throw "synthetic forall signature has no result type" + let some term := typeSpec[1]? + | throw "synthetic forall result type is malformed" + let here ← forallBinders term + let rest := sliceChars text (comma + width) text.length + return here ++ (← conclusionBinders env rest) + let unicodeExists := chars[0]? == some '∃' && chars[1]?.any fun c => + c.isWhitespace || c == '(' || c == '{' || c == '[' || c == '⦃' + let asciiExists := text.startsWith "exists" && chars[6]?.any fun c => + c.isWhitespace || c == '(' || c == '{' || c == '[' || c == '⦃' + if unicodeExists || asciiExists then + return #[] + match findTopLevelToken .arrow text, findTopLevelToken .iff text with + | some (arrow, width), none => + let rest := sliceChars text (arrow + width) text.length + return #[{ name? := none, info := .default }] ++ (← conclusionBinders env rest) + | _, _ => return #[] + +structure DeclarationText where + beforeNameEnd : String + afterNameEnd : String + +def firstHeaderGroup (text : String) : Option (Nat × ScanToken × ScanToken × Char × Char) := + let candidates := #[ + (.openParen, .closeParen, '(', ')'), + (.openBrace, .closeBrace, '{', '}'), + (.openBracket, .closeBracket, '[', ']'), + (.openStrict, .closeStrict, '⦃', '⦄')] + candidates.foldl (init := none) fun best (openToken, closeToken, opener, closer) => + match findTopLevelToken openToken text with + | none => best + | some (position, _) => match best with + | none => some (position, openToken, closeToken, opener, closer) + | some current => + if position < current.1 then some (position, openToken, closeToken, opener, closer) + else best + +/-- Erase binder *types* while preserving binder names and kinds. The +elaborated telescope supplies the types; this parser pass needs only the +surface boundary. Erasing types prevents scoped notation inside a binder type +from making the header impossible to parse out of its original file context. -/ +partial def sanitizeHeaderBinders (text : String) : Except String String := do + let some (start, _, closeToken, opener, closer) := firstHeaderGroup text + | return text + let before := sliceChars text 0 start + let afterOpen := sliceChars text (start + 1) text.length + let some (close, width) := findTopLevelToken closeToken afterOpen + | throw s!"unclosed declaration binder beginning with {opener}" + let inner := sliceChars afterOpen 0 close + let rest := sliceChars afterOpen (close + width) afterOpen.length + let universeGroup := opener == '{' && before.trimAsciiEnd.toString.endsWith "." + let rewritten := if universeGroup then + String.singleton opener ++ inner ++ String.singleton closer + else match findTopLevelToken .signatureColon inner with + | some (colon, _) => + String.singleton opener ++ sliceChars inner 0 colon ++ " : True" ++ + String.singleton closer + | none => + if opener == '[' then "[True]" + else String.singleton opener ++ inner ++ String.singleton closer + return before ++ rewritten ++ (← sanitizeHeaderBinders rest) + +/-- Turn an exact declaration slice into a parser-safe header command and the +original result type. -/ +def DeclarationText.headerAndResult (text : DeclarationText) : Except String (String × String) := do + let some (colon, width) := findTopLevelToken .signatureColon text.afterNameEnd + | throw "declaration header has no top-level result colon" + let rawHeader := sliceChars text.afterNameEnd 0 colon + let header := text.beforeNameEnd ++ (← sanitizeHeaderBinders rawHeader) ++ + " : True := by trivial" + let afterColon := sliceChars text.afterNameEnd (colon + width) text.afterNameEnd.length + let result := match findTopLevelToken .bodyMarker afterColon with + | some (body, _) => sliceChars afterColon 0 body + | none => afterColon + return (header, result) + +/-- Read the exact declaration range from the source module. The elaborated +environment remains authoritative for the range and telescope; parsing the +slice is only how we recover where the source header ended. -/ +def declarationSource (modName : Name) (ranges : DeclarationRanges) : IO (Except String DeclarationText) := do + let some path ← (← getSrcSearchPath).findModuleWithExt "lean" modName + | return .error s!"source file for {modName} was not found" + let source ← IO.FS.readFile path + let fileMap := FileMap.ofString source + let start := fileMap.ofPosition ranges.range.pos + let stop := fileMap.ofPosition ranges.range.endPos + let nameStop := fileMap.ofPosition ranges.selectionRange.endPos + if start > nameStop || nameStop > stop || stop > source.rawEndPos then + return .error s!"invalid declaration range for {modName}: {repr ranges.range}" + return .ok { + beforeNameEnd := source.toRawSubstring.extract start nameStop |>.toString + afterNameEnd := source.toRawSubstring.extract nameStop stop |>.toString } + +def binderBoundarySelfTest (env : Environment) : IO UInt32 := do + let cases : Array (String × Nat) := #[ + ("theorem t (n : Nat) (hn : 1 < n) : True := by trivial", 2), + ("theorem t : ∀ n : Nat, 1 < n → True := by intro; trivial", 0), + ("theorem t (x y : Nat) {α : Type} {{β : Type}} [i : Inhabited α] z : True := by trivial", 6) + ] + for (source, expected) in cases do + match parseDeclarationBinderCount env source with + | .ok actual => + if actual != expected then + IO.eprintln s!"binder-boundary self-test expected {expected}, got {actual}: {source}" + return 1 + | .error message => + IO.eprintln s!"binder-boundary self-test failed: {message}: {source}" + return 1 + let mkLocal (index : Nat) (name : Name) (info : BinderInfo) : LocalDecl := + .cdecl index { name := `_selfTest |>.appendIndexAfter index } name (.sort .zero) info .default + let outerAndHeader := #[ + mkLocal 0 `R .implicit, + mkLocal 1 `instR .instImplicit, + mkLocal 2 `I .implicit, + mkLocal 3 `hI .default, + mkLocal 4 `n .default, + mkLocal 5 `instN .instImplicit] + let alignmentCases : Array (String × String × Array LocalDecl × Nat) := #[ + ("theorem t {I : Type} (hI : True) (n : Type*) [Fintype n] : True := by trivial", + "True", outerAndHeader, 6), + ("theorem t (n : Nat) (hn : 1 < n) : True := by trivial", + "True", #[mkLocal 0 `n .default, mkLocal 1 `hn .default], 2), + ("theorem t : ∀ n : Nat, True := by intro; trivial", + "∀ n : Nat, True", #[mkLocal 0 `R .default, mkLocal 1 `n .default], 1), + ("theorem t : ∀ n : Nat, 1 < n → True := by intro; trivial", + "∀ n : Nat, 1 < n → True", #[mkLocal 0 `n .default, mkLocal 1 `h .default], 0), + ("theorem t : True ↔ ∃ n : Nat, 1 < n → True := by simp", + "True ↔ ∃ n : Nat, 1 < n → True", #[], 0), + ("theorem t : ∃ f : Nat → Nat, ∀ n, True → f n = f n := by simp", + "∃ f : Nat → Nat, ∀ n, True → f n = f n", #[], 0), + ("theorem t : True → (∀ n : Nat, 1 < n → True) := by simp", + "True → (∀ n : Nat, 1 < n → True)", + #[mkLocal 0 `h₁ .default, mkLocal 1 `n .default, mkLocal 2 `h₂ .default], 0) + ] + for (source, resultType, elaborated, expected) in alignmentCases do + let result := do + let command ← Parser.runParserCategory env `command source + let conclusion ← conclusionBinders env resultType + declarationParameterBoundary command conclusion elaborated + match result with + | .ok actual => + if actual != expected then + IO.eprintln s!"parameter-alignment self-test expected {expected}, got {actual}: {source}" + return 1 + | .error message => + IO.eprintln s!"parameter-alignment self-test failed: {message}: {source}" + return 1 + IO.println "binder-boundary self-test passed" + return 0 def binderJson (name : Name) (bi : BinderInfo) : Json := Json.mkObj [("name", toJson name.toString), ("explicit", toJson bi.isExplicit)] @@ -137,17 +527,23 @@ def resolveIn (env : Environment) (modName : Name) (declName : String) : | _ => .error s!"{declName} is ambiguous: {matches_}" unsafe def main (args : List String) : IO UInt32 := do - let [modName, declName] := args - | IO.eprintln "usage: comparator_facts "; return 1 - runWithImports #[modName.toName] do - let env ← getEnv - match resolveIn env modName.toName declName with - | .error msg => IO.eprintln msg; return 1 - | .ok n => emit env n declName + match args with + | ["--self-test"] => + runWithImports #[`Mathlib] do binderBoundarySelfTest (← getEnv) + | [modName, declName] => + runWithImports #[modName.toName] do + let env ← getEnv + match resolveIn env modName.toName declName with + | .error msg => IO.eprintln msg; return 1 + | .ok n => emit env modName.toName n declName + | _ => + IO.eprintln "usage: comparator_facts | --self-test" + return 1 where - emit (env : Environment) (name : Name) (decl : String) : MetaM UInt32 := do + emit (env : Environment) (modName name : Name) (decl : String) : MetaM UInt32 := do let some info := env.find? name | IO.eprintln "vanished"; return 1 - let ranges ← findDeclarationRanges? name + let some ranges ← findDeclarationRanges? name + | IO.eprintln s!"{name} has no source range"; return 1 -- The statement's sorries are `answer(sorry)` slots; a proof's sorry is -- not in the *type*, so everything found here is a slot. -- `findAnswerExprs` is the repository's own detection: it reads the @@ -156,14 +552,30 @@ where let answerTypes ← forallTelescope info.type fun _ body => do let found := Google.findAnswerExprs body found.mapM fun a => do pure (toString (← ppExpr (← inferType a))) - let arity := match info.value? with - | some v => lambdaArity v - | none => 0 - let binders ← forallTelescope info.type fun xs _ => - (xs.extract 0 arity).mapM fun x => do - let d ← x.fvarId!.getDecl + let sourceResult ← declarationSource modName ranges + let declarationText ← match sourceResult with + | .ok source => pure source + | .error message => throwError message + let (header, resultType) ← match declarationText.headerAndResult with + | .ok pieces => pure pieces + | .error message => throwError message + let command ← match Parser.runParserCategory env `command header with + | .ok command => pure command + | .error message => + throwError "could not recover declaration parameters for {name}: {message}" + let conclusion ← match conclusionBinders env resultType with + | .ok binders => pure binders + | .error message => + throwError "could not recover conclusion parameters for {name}: {message}" + let binders ← forallTelescope info.type fun xs _ => do + let declarations ← xs.mapM fun x => x.fvarId!.getDecl + let arity ← match declarationParameterBoundary command conclusion declarations with + | .ok arity => pure arity + | .error message => + throwError "could not align declaration parameters for {name}: {message}" + (declarations.extract 0 arity).mapM fun d => pure (binderJson d.userName d.binderInfo) - let rangeJson := rangeToJson ranges + let rangeJson := rangeToJson (some ranges) -- Only the statement's dependencies: the proof is replaced by `sorry` in -- the generated Challenge, so nothing the value names has to be carried. let direct := info.type.getUsedConstants.filter (isFCLocal env) diff --git a/comparator/adapter/fc_leaneval_importer.py b/comparator/adapter/fc_leaneval_importer.py index 4309206514..9851579962 100644 --- a/comparator/adapter/fc_leaneval_importer.py +++ b/comparator/adapter/fc_leaneval_importer.py @@ -19,8 +19,10 @@ which generated module imports which. If a change to this file would do one of those, it belongs on the other side of the seam. -One thing the Lean source cannot settle lives in `comparator/problems/.toml`, -one file per problem: which file is meant when two declare the same name. +Two source-boundary facts may live in `comparator/problems/.toml`, one +file per problem: which file is meant when two declare the same name, and an +explicit source-only proof dependency when Lean's opaque-value erasure makes +that dependency unrecoverable from the compiled environment. """ import json @@ -116,6 +118,66 @@ def elaborator_facts(module, declaration): return json.loads(out[out.index("{") :]) +def explicit_copy_dependencies(problem_file): + """Source-only dependencies the compiled environment cannot retain. + + Lean erases the values of opaque theorem constants. If the source body + of a copied definition invokes such a theorem only to construct a proof + argument, the compiled definition refers to a generated `_proof_*` + constant and no longer records which source theorem produced it. The + marked-up module still copies source text, so that theorem must be named + explicitly and audibly in the problem manifest. + """ + records, generated = [], [] + for entry in problem_file.get("copy_dependencies", []): + if set(entry) != {"declaration", "module"}: + raise SystemExit( + "each `copy_dependencies` entry must contain exactly " + "`declaration` and `module`" + ) + relative = pathlib.Path(entry["module"]) + if ( + relative.is_absolute() + or ".." in relative.parts + or not relative.parts + or relative.parts[0] + not in {"FormalConjectures", "FormalConjecturesForMathlib"} + ): + raise SystemExit( + f"copy dependency module must stay under a source tree: {relative}" + ) + path = ROOT / relative + if not path.is_file() or relative.suffix != ".lean": + raise SystemExit(f"copy dependency module does not exist: {relative}") + module = module_name(relative) + facts = elaborator_facts(module, entry["declaration"]) + records.extend(facts.get("dependencies", [])) + records.append( + { + "name": facts["name"], + "module": module, + "range": facts["range"], + } + ) + generated.extend(facts.get("generatedDependencies", [])) + return records, generated + + +def merge_dependency_records(*groups): + """Deduplicate topologically ordered dependency records, fail closed.""" + merged, seen = [], {} + for group in groups: + for record in group: + name = record["name"] + if name in seen: + if seen[name] != record: + raise SystemExit(f"conflicting dependency records for {name}") + continue + seen[name] = record + merged.append(record) + return merged + + def file_scoped_preamble(lines, start_line): """Directives in force at `start_line`, and the namespace stack there. @@ -170,11 +232,13 @@ def file_scoped_preamble(lines, start_line): def load_manifest(problem_id): - """Read the one choice Lean source cannot select by itself. + """Read the rare source-boundary facts Lean cannot select by itself. When two files declare the same name, nothing in the Lean environment says - which one was meant, so the importer refuses until a module is named. That - is the whole contract. + which one was meant, so the importer refuses until a module is named. + `copy_dependencies` is the other exceptional field: it names a theorem + used only in copied source proof text when opaque-value erasure removes + the reference from Lean's compiled dependency graph. `leanprover/lean-eval` keeps one TOML per problem, and the reason is worth copying: two pull requests adding different problems never touch the same @@ -183,6 +247,8 @@ def load_manifest(problem_id): id the filename stem, and the workspace directory name declaration the Lean name, which need not be unique across the repository module the file declaring it, relative to the repository root + copy_dependencies exact declaration/module pairs to copy before the + environment-derived closure Anything Formal Conjectures already states stays where it is stated. The source citation is read from the module docstring rather than copied here, @@ -554,7 +620,10 @@ def covered_by_another(dep): if not body: raise SystemExit(f"{declaration}: {dep['name']} sliced to nothing") namespace = ".".join(namespaces) - chunk = [f"-- {dep['name']}, from {path.relative_to(ROOT)}", "section"] + chunk = [ + f"-- {dep['name']}, from {path.relative_to(ROOT)}", + "noncomputable section", + ] chunk += preamble if namespace: chunk.append(f"namespace {namespace}") @@ -1065,6 +1134,13 @@ def import_problem(problem, answer_type=None, module=None): facts = elaborator_facts(fc_module, declaration) if facts["range"] is None: raise SystemExit(f"{declaration}: no source range recorded") + explicit_dependencies, explicit_generated = explicit_copy_dependencies(problem_file) + facts["dependencies"] = merge_dependency_records( + explicit_dependencies, facts.get("dependencies", []) + ) + facts["generatedDependencies"] = list( + dict.fromkeys(explicit_generated + facts.get("generatedDependencies", [])) + ) source_lines = path.read_text(encoding="utf-8").split("\n") original, lo = slice_range(source_lines, facts["range"]) diff --git a/comparator/adapter/test_fc_leaneval_importer.py b/comparator/adapter/test_fc_leaneval_importer.py index 7da99a9c21..37f8730f9e 100644 --- a/comparator/adapter/test_fc_leaneval_importer.py +++ b/comparator/adapter/test_fc_leaneval_importer.py @@ -326,8 +326,70 @@ def test_a_generated_constant_under_a_copied_parent_is_accepted(self): resolve.return_value = source out, copied = closure_region(deps, ["Foo.bar._proof_1"], "t") self.assertIn("def Foo.bar := 1", out) + self.assertIn("noncomputable section", out) self.assertEqual(copied, [("Foo.bar", "def Foo.bar := 1")]) + def test_an_explicit_source_only_dependency_carries_its_closure(self): + facts = { + "name": "Foo.opaqueLemma", + "range": {"startLine": 2, "endLine": 2, "endColumn": None}, + "dependencies": [ + { + "name": "Foo.Predicate", + "module": "FormalConjectures.Example", + "range": {"startLine": 1, "endLine": 1, "endColumn": None}, + } + ], + "generatedDependencies": ["Foo.opaqueLemma._proof_1"], + } + with tempfile.TemporaryDirectory() as tmp, _root_at(tmp): + module = pathlib.Path(tmp) / "FormalConjectures" / "Example.lean" + module.parent.mkdir(parents=True) + module.write_text("def Predicate := True\ntheorem opaqueLemma : Predicate := by trivial\n") + with mock.patch.object(importer, "elaborator_facts", return_value=facts): + records, generated = importer.explicit_copy_dependencies( + { + "copy_dependencies": [ + { + "declaration": "Foo.opaqueLemma", + "module": "FormalConjectures/Example.lean", + } + ] + } + ) + self.assertEqual( + [record["name"] for record in records], + ["Foo.Predicate", "Foo.opaqueLemma"], + ) + self.assertEqual(generated, ["Foo.opaqueLemma._proof_1"]) + + def test_explicit_source_only_dependency_rejects_extra_fields(self): + with self.assertRaisesRegex(SystemExit, "must contain exactly"): + importer.explicit_copy_dependencies( + { + "copy_dependencies": [ + { + "declaration": "Foo.opaqueLemma", + "module": "FormalConjectures/Example.lean", + "guess": True, + } + ] + } + ) + + def test_explicit_source_only_dependency_stays_in_a_source_tree(self): + with self.assertRaisesRegex(SystemExit, "must stay under a source tree"): + importer.explicit_copy_dependencies( + { + "copy_dependencies": [ + { + "declaration": "Foo.opaqueLemma", + "module": "../Elsewhere/Example.lean", + } + ] + } + ) + def test_a_declaration_inside_another_s_range_is_not_copied_twice(self): # `EdgeN.mk` covers line 88 of a structure spanning 83 to 93, and # `pmSumListAux._sparseCasesOn_1` has exactly its parent's range. diff --git a/comparator/problems/Erdos340.erdos_340.variants.co_density_zero_sub.toml b/comparator/problems/Erdos340.erdos_340.variants.co_density_zero_sub.toml new file mode 100644 index 0000000000..ceecc4f228 --- /dev/null +++ b/comparator/problems/Erdos340.erdos_340.variants.co_density_zero_sub.toml @@ -0,0 +1,18 @@ +# Lean erases the value of the opaque theorem used inside +# `Finset.greedySidon.go`'s generated proof constant. The compiled dependency +# graph therefore cannot recover this source-only reference, but the copied +# definition body still names it and needs its statement to elaborate. +id = "Erdos340.erdos_340.variants.co_density_zero_sub" +declaration = "Erdos340.erdos_340.variants.co_density_zero_sub" + +[[copy_dependencies]] +declaration = "IsSidon.insert" +module = "FormalConjecturesForMathlib/Combinatorics/Basic.lean" + +[[copy_dependencies]] +declaration = "Finset.IsSidon.insert_ge_max'" +module = "FormalConjecturesForMathlib/Combinatorics/Basic.lean" + +[[copy_dependencies]] +declaration = "Finset.IsSidon.exists_insert_ge" +module = "FormalConjecturesForMathlib/Combinatorics/Basic.lean" From 7c24bef729422e49f084d16e4097df554ee1986d Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:21:32 -0400 Subject: [PATCH 38/70] Record how each recorded failure retires The ledger now names the verified path for each entry: the two drift cases retire with the toolchain bump (kim-em compiled both generated Challenges at the target pins on a merge with #4428), and the Erdos125 entry retires when the generator pin advances past leanprover/lean-eval-generator#1. --- comparator/known_failures.toml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/comparator/known_failures.toml b/comparator/known_failures.toml index 9525f5ffa5..7084ebf409 100644 --- a/comparator/known_failures.toml +++ b/comparator/known_failures.toml @@ -16,9 +16,10 @@ stage = "target" reason = """The copied `f` definition synthesizes `Fintype ↑H.verts` and `Fintype ↑H.coe.edgeSet` at this repository's Mathlib but not at LeanEval's (observed at target pin 6f1ef4e5): instance drift between the two revisions, -not a copying defect. Formal Conjectures' own bump to Lean 4.33 rewrites the -source against the target-side Mathlib and retires the gap; re-run the audit -after the bump and remove this entry.""" +not a copying defect. Formal Conjectures' planned toolchain bump (to Lean 4.33; +#4428 stages v4.32.0) adapts the source and retires the gap — kim-em verified on +2026-08-21 that the generated Challenge compiles at LeanEval's target pins on a +merge with #4428. Re-run the audit after the bump merges and remove this entry.""" [[failure]] declaration = "EllipticCurveRank.RatEllipticCurve.twentyone_le_rank_height_count_asymptotic" @@ -27,8 +28,10 @@ stage = "target" reason = """The copied `toWeierstrass⟮ℚ⟯` Mordell-Weil notation elaborates at this repository's Mathlib but not at LeanEval's (observed at target pin 6f1ef4e5): the notation's shape changed between the two revisions, in a -dependency copied faithfully from source. Retired by the repository's 4.33 -bump; re-run the audit after it and remove this entry.""" +dependency copied faithfully from source. Retired by the repository's planned +toolchain bump (to Lean 4.33; #4428 stages v4.32.0) — kim-em verified on +2026-08-21 at LeanEval's target pins on a merge with #4428. Re-run the audit +after the bump merges and remove this entry.""" [[failure]] declaration = "Erdos125.erdos_125.variants.positive_unequal_density" @@ -39,5 +42,6 @@ reason = """The statement's `local notation` uses set-builder syntax and needs emitted module carries attached to the notation command. The generator's Challenge reconstruction re-extracts the notation line without its set_option, and `Challenge.lean` is generator-owned text this side cannot -amend. Reported upstream as a context-reconstruction gap; remove this entry -when the pinned generator carries set_option context.""" +amend. Fixed upstream in leanprover/lean-eval-generator#1 (verified by kim-em at +this PR's head); remove this entry when `[generator]` in tools.toml advances to a +revision containing that fix.""" From 7a101bfe777fe3a24ffaff50f1352cbebaa54dbc Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:40:22 -0400 Subject: [PATCH 39/70] Say each shared thing once The two Comparator jobs prepared the extractor with the same elan install, cache restore, build and self-test; that is one composite action now, taking the modules a job needs elaborated. The response parser computed a digest the interface already had a function for, and four artifacts spelled out the same JSON serialisation; one helper each. The extractor's two programs - the source-syntax binder-boundary recovery and the environment extraction - are marked as sections, ahead of moving them into a library after merge. --- .github/actions/prepare-extractor/action.yml | 49 +++++++++++++++++++ .github/workflows/comparator-lean-4-33.yml | 16 ++---- .github/workflows/fc100-audit.yml | 16 ++---- comparator/adapter/comparator_facts.lean | 10 ++++ comparator/adapter/compile_fc100_target.py | 5 +- comparator/adapter/leaneval_interface.py | 13 ++--- .../adapter/make_comparator_workspace.py | 7 ++- 7 files changed, 80 insertions(+), 36 deletions(-) create mode 100644 .github/actions/prepare-extractor/action.yml diff --git a/.github/actions/prepare-extractor/action.yml b/.github/actions/prepare-extractor/action.yml new file mode 100644 index 0000000000..d99038d93d --- /dev/null +++ b/.github/actions/prepare-extractor/action.yml @@ -0,0 +1,49 @@ +# Copyright 2026 The Formal Conjectures Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# https://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Prepare the comparator extractor +description: >- + Install elan at this repository's toolchain, restore the Mathlib cache, + build the comparator_facts extractor together with the source modules a + job needs elaborated, and run the extractor's self-test. The facts the + importer reads (ranges, binder boundaries, answer-slot types, category + tags) come from an elaborated environment, so every module a job imports + from has to be built first. + +inputs: + modules: + description: Space-separated Lake targets to build beside the extractor. + required: false + default: "" + +runs: + using: composite + steps: + - name: Install elan + shell: bash + run: | + set -o pipefail + curl -sSfL https://github.com/leanprover/elan/releases/download/v1.4.2/elan-x86_64-unknown-linux-gnu.tar.gz | tar xz + ./elan-init -y --default-toolchain none + echo "$HOME/.elan/bin" >> "$GITHUB_PATH" + + - name: Build the extractor and the source modules + shell: bash + env: + MODULES: ${{ inputs.modules }} + run: | + lake exe cache get + # shellcheck disable=SC2086 + lake build comparator_facts $MODULES + lake exe comparator_facts --self-test diff --git a/.github/workflows/comparator-lean-4-33.yml b/.github/workflows/comparator-lean-4-33.yml index 9d51b7fd1a..2312a753ff 100644 --- a/.github/workflows/comparator-lean-4-33.yml +++ b/.github/workflows/comparator-lean-4-33.yml @@ -52,13 +52,6 @@ jobs: # later step or a build script could reach it. persist-credentials: false - - name: Install elan - run: | - set -o pipefail - curl -sSfL https://github.com/leanprover/elan/releases/download/v1.4.2/elan-x86_64-unknown-linux-gnu.tar.gz | tar xz - ./elan-init -y --default-toolchain none - echo "$HOME/.elan/bin" >> "$GITHUB_PATH" - - name: Read the target pins id: target run: | @@ -73,11 +66,10 @@ jobs: # At this repository's toolchain: the declaration's facts come from an # elaborated environment, so the module it lives in has to be built. - - name: Build the source module and the extractor - run: | - lake exe cache get - lake build comparator_facts FormalConjectures.Wikipedia.SumOfThreeCubes - lake exe comparator_facts --self-test + - name: Prepare the extractor and the source module + uses: ./.github/actions/prepare-extractor + with: + modules: FormalConjectures.Wikipedia.SumOfThreeCubes - name: Build the pinned lean-eval-generator uses: ./.github/actions/build-lean-eval-generator diff --git a/.github/workflows/fc100-audit.yml b/.github/workflows/fc100-audit.yml index 6f6c0b0b77..e04b2c7c91 100644 --- a/.github/workflows/fc100-audit.yml +++ b/.github/workflows/fc100-audit.yml @@ -61,20 +61,12 @@ jobs: fetch-depth: 0 persist-credentials: false - - name: Install elan - run: | - set -o pipefail - curl -sSfL https://github.com/leanprover/elan/releases/download/v1.4.2/elan-x86_64-unknown-linux-gnu.tar.gz | tar xz - ./elan-init -y --default-toolchain none - echo "$HOME/.elan/bin" >> "$GITHUB_PATH" - # The category tags, source ranges and answer-slot types come from the # elaborated environment, so every module in the set has to be built. - - name: Build the source modules and the extractor - run: | - lake exe cache get - lake build FormalConjectures.Subsets.FC100OpenSet1 comparator_facts - lake exe comparator_facts --self-test + - name: Prepare the extractor and the whole set + uses: ./.github/actions/prepare-extractor + with: + modules: FormalConjectures.Subsets.FC100OpenSet1 - name: Build the pinned lean-eval-generator uses: ./.github/actions/build-lean-eval-generator diff --git a/comparator/adapter/comparator_facts.lean b/comparator/adapter/comparator_facts.lean index 2ed5d4d5d6..a83e1ec1a0 100644 --- a/comparator/adapter/comparator_facts.lean +++ b/comparator/adapter/comparator_facts.lean @@ -46,6 +46,14 @@ def declares (declared : Name) (requested : String) : Bool := let s := declared.toString s == requested || s.endsWith ("." ++ requested) +/-! ## Declaration binder boundaries, from source syntax + +A theorem proved by bare `sorry` can be stored as a full-type `sorryAx`, so the +proof value's lambda arity does not reliably separate header parameters from +binders in the conclusion. This subsystem recovers that boundary from the +declaration's source text and matches it against the elaborated telescope; +every binder fact emitted still comes from the environment. -/ + /-- Collect every node of one syntax kind. A declaration command must contain exactly one `declSig`; finding zero or more than one means we did not parse the source range we thought we did, and the importer fails closed. -/ @@ -447,6 +455,8 @@ def binderBoundarySelfTest (env : Environment) : IO UInt32 := do IO.println "binder-boundary self-test passed" return 0 +/-! ## Environment extraction -/ + def binderJson (name : Name) (bi : BinderInfo) : Json := Json.mkObj [("name", toJson name.toString), ("explicit", toJson bi.isExplicit)] diff --git a/comparator/adapter/compile_fc100_target.py b/comparator/adapter/compile_fc100_target.py index a1bf552b92..95847cbb0a 100644 --- a/comparator/adapter/compile_fc100_target.py +++ b/comparator/adapter/compile_fc100_target.py @@ -23,13 +23,14 @@ """ import argparse -import json import pathlib import re import subprocess import sys import tomllib +from leaneval_interface import dump_json + def arrange_project(workspaces_dir, project_dir): """Lay out the shared project; returns `{workspace_id: module_name}`. @@ -162,7 +163,7 @@ def main(argv): } if args.report: pathlib.Path(args.report).write_text( - json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" + dump_json(report), encoding="utf-8" ) print(f"{report['ok']}/{report['total']} Challenges compile at target pins") diff --git a/comparator/adapter/leaneval_interface.py b/comparator/adapter/leaneval_interface.py index 07267097fe..db373a8cce 100644 --- a/comparator/adapter/leaneval_interface.py +++ b/comparator/adapter/leaneval_interface.py @@ -71,6 +71,11 @@ def sha256_text(text): """The digest the generator response uses: SHA-256 of the UTF-8 bytes.""" return hashlib.sha256(text.encode("utf-8")).hexdigest() + +def dump_json(obj, sort_keys=False): + """The one JSON serialisation every artifact here uses: readable, UTF-8, newline-terminated.""" + return json.dumps(obj, indent=2, ensure_ascii=False, sort_keys=sort_keys) + "\n" + # The generator's frozen wire format; `schemas/request-v1.schema.json` and # `response-v1.schema.json` in the pinned revision are normative. CONTRACT_VERSION = 1 @@ -279,10 +284,7 @@ def from_json_object(cls, payload): def to_json(self): # Key-sorted: the same record always serialises to the same bytes. - return ( - json.dumps(self.to_json_object(), indent=2, ensure_ascii=False, sort_keys=True) - + "\n" - ) + return dump_json(self.to_json_object(), sort_keys=True) @classmethod def from_json(cls, text): @@ -577,8 +579,7 @@ def parse_response(text): ) workspaces = {} for entry in payload["files"]: - digest = hashlib.sha256(entry["content"].encode("utf-8")).hexdigest() - if digest != entry["sha256"]: + if sha256_text(entry["content"]) != entry["sha256"]: raise SystemExit( f"{entry['problemId']}/{entry['path']}: content does not match " "its digest" diff --git a/comparator/adapter/make_comparator_workspace.py b/comparator/adapter/make_comparator_workspace.py index ff943c2daf..e5a40b4628 100644 --- a/comparator/adapter/make_comparator_workspace.py +++ b/comparator/adapter/make_comparator_workspace.py @@ -59,7 +59,6 @@ """ import argparse -import json import pathlib import re import shutil @@ -69,7 +68,7 @@ import fc_leaneval_importer as importer import leaneval_generator_cli as generator_cli -from leaneval_interface import build_problem, build_request, slug, sha256_text +from leaneval_interface import build_problem, build_request, dump_json, sha256_text, slug ROOT = importer.ROOT @@ -128,7 +127,7 @@ def seam_files(pairs, group=None): request = build_request( [problem for problem, _ in problems], target, template, CONTEXT_DIR ) - files = {"request.json": json.dumps(request, indent=2, ensure_ascii=False) + "\n"} + files = {"request.json": dump_json(request)} for path, content in generator_cli.context_files(problems).items(): files[f"{CONTEXT_DIR}/{path}"] = content for (problem, _), (_, manifest) in zip(problems, pairs): @@ -361,7 +360,7 @@ def main(argv): report = import_set( args.set, args.out, verify=args.verify, known_failures=known ) - text = json.dumps(report, indent=2, ensure_ascii=False) + "\n" + text = dump_json(report) if args.report: pathlib.Path(args.report).write_text(text, encoding="utf-8") print(text, end="") From 66dcc617070a72e234e8e00f81a72fd40ed2eb6f Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:46:56 -0400 Subject: [PATCH 40/70] Separate reading Formal Conjectures from assembling a workspace Two pure moves. fc_source.py now holds everything that answers questions about this repository's own Lean - where a declaration is, the file-scoped directives in force where it was written, the FC-defined notation it uses, its answer slots and their elaborated types, the pins the text was read at - and fc_leaneval_importer.py assembles the marked-up module and the provenance record from those answers. The extractor becomes a small library, ComparatorFacts.Binders (declaration- header binder boundaries from source syntax) and ComparatorFacts.Extract (the elaborated environment), under a thin executable. No function changed; the tests that reach into the source layer now import it by its name. --- comparator/OWNERSHIP.md | 16 +- comparator/adapter/ComparatorFacts.lean | 18 + .../adapter/ComparatorFacts/Binders.lean | 429 ++++++++++ .../adapter/ComparatorFacts/Extract.lean | 111 +++ comparator/adapter/comparator_facts.lean | 505 +---------- comparator/adapter/fc_leaneval_importer.py | 790 +----------------- comparator/adapter/fc_source.py | 764 +++++++++++++++++ .../adapter/test_fc_leaneval_importer.py | 56 +- lakefile.toml | 4 + 9 files changed, 1393 insertions(+), 1300 deletions(-) create mode 100644 comparator/adapter/ComparatorFacts.lean create mode 100644 comparator/adapter/ComparatorFacts/Binders.lean create mode 100644 comparator/adapter/ComparatorFacts/Extract.lean create mode 100644 comparator/adapter/fc_source.py diff --git a/comparator/OWNERSHIP.md b/comparator/OWNERSHIP.md index 91197d1822..721f4aeec4 100644 --- a/comparator/OWNERSHIP.md +++ b/comparator/OWNERSHIP.md @@ -15,10 +15,13 @@ and declaration id for every problem. ## The seam - comparator/adapter/fc_leaneval_importer.py FC declaration -> (module, manifest) - comparator/adapter/leaneval_interface.py the request built from them, the - response checked against its digests - comparator/adapter/leaneval_generator_cli.py runs the pinned binary, nothing else + comparator/adapter/fc_source.py reading FC source: where a declaration + is, its preamble, notation, answer slots + comparator/adapter/fc_leaneval_importer.py FC declaration -> (module, manifest), + assembled from those answers + comparator/adapter/leaneval_interface.py the request built from them, the + response checked against its digests + comparator/adapter/leaneval_generator_cli.py runs the pinned binary, nothing else `comparator/adapter/make_comparator_workspace.py` is the command that runs one after the other. The arrow points one way: the CLI plumbing imports the interface and @@ -79,8 +82,9 @@ Conjectures corrects a misformalisation upstream. | File | Why it cannot move | |---|---| -| `comparator/adapter/fc_leaneval_importer.py` | resolves a declaration against an exact FC commit, reads the elaborated environment, copies the FC-local closure, types each `answer(sorry)` slot, and records the provenance | -| `comparator/adapter/comparator_facts.lean` | the Lean extractor: source ranges, declaration-header binder boundaries, elaborated binder names/explicitness, answer-slot types, and the `@[category ...]` tag. The parsed source distinguishes header parameters from `∀` binders in the conclusion; every emitted binder fact still comes from the elaborated environment. | +| `comparator/adapter/fc_source.py` | reads this repository's own Lean: where a declaration is, the file-scoped directives in force where it was written, the FC-defined notation it uses, its `answer(sorry)` slots and their elaborated types, and the pins the text was read at | +| `comparator/adapter/fc_leaneval_importer.py` | assembles the marked-up module and the provenance record from those answers: resolves the declaration against an exact FC commit, copies the FC-local closure, hoists each slot, and records the provenance | +| `comparator/adapter/ComparatorFacts/` and `comparator_facts.lean` | the Lean extractor (a small library — `Binders.lean` recovers declaration-header binder boundaries from source syntax, `Extract.lean` reads the elaborated environment — and a thin executable): source ranges, declaration-header binder boundaries, elaborated binder names/explicitness, answer-slot types, and the `@[category ...]` tag. The parsed source distinguishes header parameters from `∀` binders in the conclusion; every emitted binder fact still comes from the elaborated environment. | | `comparator/adapter/leaneval_interface.py` | the request builder and response checker — the FC side of the wire format, permanently, since the consumer owns hole resolution under the v1 contract | | `comparator/adapter/leaneval_generator_cli.py` | plumbing for the pinned binary | | `comparator/adapter/make_comparator_workspace.py` | the command, the emitted seam artifact, and the whole-set batch run | diff --git a/comparator/adapter/ComparatorFacts.lean b/comparator/adapter/ComparatorFacts.lean new file mode 100644 index 0000000000..a988409ea2 --- /dev/null +++ b/comparator/adapter/ComparatorFacts.lean @@ -0,0 +1,18 @@ +/- +Copyright 2026 The Formal Conjectures Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +-/ + +import ComparatorFacts.Binders +import ComparatorFacts.Extract diff --git a/comparator/adapter/ComparatorFacts/Binders.lean b/comparator/adapter/ComparatorFacts/Binders.lean new file mode 100644 index 0000000000..017759f179 --- /dev/null +++ b/comparator/adapter/ComparatorFacts/Binders.lean @@ -0,0 +1,429 @@ +/- +Copyright 2026 The Formal Conjectures Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +-/ + +import Lean + +open Lean Meta + +/-! +# Declaration binder boundaries, from source syntax + +A theorem proved by bare `sorry` can be stored as a full-type `sorryAx`, so the +proof value's lambda arity does not reliably separate header parameters from +binders in the conclusion. This subsystem recovers that boundary from the +declaration's source text and matches it against the elaborated telescope; +every binder fact emitted still comes from the environment. -/ + +/-- Collect every node of one syntax kind. A declaration command must contain +exactly one `declSig`; finding zero or more than one means we did not parse the +source range we thought we did, and the importer fails closed. -/ +partial def collectKind (kind : Name) (stx : Syntax) (found : Array Syntax := #[]) : + Array Syntax := + let found := if stx.isOfKind kind then found.push stx else found + stx.getArgs.foldl (fun acc child => collectKind kind child acc) found + +structure SourceBinder where + name? : Option Name + info : BinderInfo + deriving Repr + +def sourceBinderName (stx : Syntax) : Except String (Option Name) := + if stx.isIdent then .ok (some stx.getId) + else if stx.isOfKind ``Lean.Parser.Term.hole then .ok none + else .error s!"unsupported declaration binder name {stx.getKind}: {stx}" + +/-- The names and binder kinds introduced by one declaration-header group. -/ +def declarationBinderGroup (binder : Syntax) : Except String (Array SourceBinder) := do + if binder.isIdent || binder.isOfKind ``Lean.Parser.Term.hole then + return #[{ name? := ← sourceBinderName binder, info := .default }] + if binder.isOfKind ``Lean.Parser.Term.instBinder then + let some optionalName := binder[1]? + | throw s!"malformed instance binder: {binder}" + let name? ← match optionalName.getArgs[0]? with + | some name => sourceBinderName name + | none => pure none + return #[{ name?, info := .instImplicit }] + let info? := + if binder.isOfKind ``Lean.Parser.Term.explicitBinder then some BinderInfo.default + else if binder.isOfKind ``Lean.Parser.Term.implicitBinder then some .implicit + else if binder.isOfKind ``Lean.Parser.Term.strictImplicitBinder then some .strictImplicit + else none + let some info := info? + | throw s!"unsupported declaration binder syntax {binder.getKind}: {binder}" + let some names := binder[1]? + | throw s!"malformed declaration binder: {binder}" + if names.getArgs.isEmpty then + throw s!"declaration binder has no names: {binder}" + names.getArgs.mapM fun name => do + return { name? := ← sourceBinderName name, info } + +def declarationSignature (command : Syntax) : Except String Syntax := do + let signatures := collectKind ``Lean.Parser.Command.declSig command + let [signature] := signatures.toList + | throw s!"expected exactly one declaration signature, found {signatures.size}" + return signature + +def declarationBinders (command : Syntax) : Except String (Array SourceBinder) := do + let signature ← declarationSignature command + let some header := signature[0]? + | throw "declaration signature has no binder header" + header.getArgs.foldlM (fun binders binder => do + return binders ++ (← declarationBinderGroup binder)) #[] + +/-- Number of parameters written before the colon in a declaration header. + +This is deliberately a syntax boundary, not a proof-value heuristic: +`theorem foo (n : Nat) : P n` has one declaration parameter, while +`theorem foo : ∀ n : Nat, P n` has none. Lean 4.32 may store a theorem proved +by `sorry` as a `sorryAx` at the full forall type, with no lambda wrappers, so +the old proof-value lambda arity silently reported zero for the first form. +The syntax supplies only the boundary; names, types, and explicitness still +come from the elaborated telescope below. -/ +def declarationBinderCount (command : Syntax) : Except String Nat := do + return (← declarationBinders command).size + +def sourceBinderMatches (source : SourceBinder) (elaborated : LocalDecl) : Bool := + source.info == elaborated.binderInfo && match source.name? with + | some name => name == elaborated.userName + | none => true + +def sourceBindersMatchAt (source : Array SourceBinder) (elaborated : Array LocalDecl) + (start : Nat) : Bool := + source.zipIdx.all fun (binder, offset) => + match elaborated[start + offset]? with + | some candidate => sourceBinderMatches binder candidate + | none => false + +/-- Locate the end of the declaration parameters in the elaborated telescope. + +Lean inserts used outer `variable`s before the binders written in the +declaration header. We therefore locate the exact typed/named header sequence +inside the telescope and retain everything through its end. This handles +Köthe's `{R} [Ring R]` outer parameters as well as ordinary self-contained +headers. Multiple matches are rejected rather than guessed. -/ +def declarationParameterBoundary (command : Syntax) (conclusion : Array SourceBinder) + (elaborated : Array LocalDecl) : Except String Nat := do + let source ← declarationBinders command + if source.isEmpty then + if conclusion.size > elaborated.size then + throw s!"source conclusion has {conclusion.size} binders, but the elaborated type has only {elaborated.size}" + let boundary := elaborated.size - conclusion.size + unless sourceBindersMatchAt conclusion elaborated boundary do + throw s!"source conclusion binders {repr conclusion} do not match the elaborated telescope suffix" + return boundary + if source.size > elaborated.size then + throw s!"source header has {source.size} binders, but the elaborated type has only {elaborated.size}" + let starts := (Array.range (elaborated.size - source.size + 1)).filter + (sourceBindersMatchAt source elaborated) + let [start] := starts.toList + | throw s!"expected one match for {source.size} source-header binders in the elaborated telescope, found {starts.size}" + return start + source.size + +def parseDeclarationBinderCount (env : Environment) (source : String) : Except String Nat := do + let command ← Parser.runParserCategory env `command source + declarationBinderCount command + +inductive ScanToken where + | signatureColon + | bodyMarker + | comma + | arrow + | iff + | openParen + | openBrace + | openBracket + | openStrict + | closeParen + | closeBrace + | closeBracket + | closeStrict + +def scanTokenAt (wanted : ScanToken) (chars : Array Char) (i : Nat) : Option Nat := + match wanted with + | .signatureColon => + if chars[i]? == some ':' && chars[i + 1]? != some '=' then some 1 else none + | .bodyMarker => + if chars[i]? == some ':' && chars[i + 1]? == some '=' then some 2 else none + | .comma => if chars[i]? == some ',' then some 1 else none + | .arrow => + if chars[i]? == some '→' then some 1 + else if chars[i]? == some '-' && chars[i + 1]? == some '>' then some 2 + else none + | .iff => if chars[i]? == some '↔' then some 1 else none + | .openParen => if chars[i]? == some '(' then some 1 else none + | .openBrace => if chars[i]? == some '{' then some 1 else none + | .openBracket => if chars[i]? == some '[' then some 1 else none + | .openStrict => if chars[i]? == some '⦃' then some 1 else none + | .closeParen => if chars[i]? == some ')' then some 1 else none + | .closeBrace => if chars[i]? == some '}' then some 1 else none + | .closeBracket => if chars[i]? == some ']' then some 1 else none + | .closeStrict => if chars[i]? == some '⦄' then some 1 else none + +/-- Find syntax punctuation at delimiter depth zero, ignoring comments and +strings. This scanner does not interpret terms; it only lets us replace the +result type with `True` before asking Lean's real command parser to read the +declaration header. Scoped notation in the result therefore cannot make an +otherwise ordinary header unparseable. -/ +partial def findTopLevelToken (wanted : ScanToken) (text : String) : Option (Nat × Nat) := + let chars := text.toList.toArray + let rec loop (i paren brace bracket strict blockComment : Nat) + (lineComment inString escaped : Bool) : Option (Nat × Nat) := + if i >= chars.size then none else + let current := chars[i]! + let next := chars[i + 1]? + if lineComment then + loop (i + 1) paren brace bracket strict blockComment (current != '\n') inString false + else if blockComment > 0 then + if current == '/' && next == some '-' then + loop (i + 2) paren brace bracket strict (blockComment + 1) false inString false + else if current == '-' && next == some '/' then + loop (i + 2) paren brace bracket strict (blockComment - 1) false inString false + else + loop (i + 1) paren brace bracket strict blockComment false inString false + else if inString then + if escaped then loop (i + 1) paren brace bracket strict 0 false true false + else if current == '\\' then loop (i + 1) paren brace bracket strict 0 false true true + else loop (i + 1) paren brace bracket strict 0 false (current != '"') false + else if current == '-' && next == some '-' then + loop (i + 2) paren brace bracket strict 0 true false false + else if current == '/' && next == some '-' then + loop (i + 2) paren brace bracket strict 1 false false false + else if current == '"' then + loop (i + 1) paren brace bracket strict 0 false true false + else if paren == 0 && brace == 0 && bracket == 0 && strict == 0 then + match scanTokenAt wanted chars i with + | some width => some (i, width) + | none => match current with + | '(' => loop (i + 1) 1 brace bracket strict 0 false false false + | '{' => loop (i + 1) paren 1 bracket strict 0 false false false + | '[' => loop (i + 1) paren brace 1 strict 0 false false false + | '⦃' => loop (i + 1) paren brace bracket 1 0 false false false + | _ => loop (i + 1) paren brace bracket strict 0 false false false + else match current with + | '(' => loop (i + 1) (paren + 1) brace bracket strict 0 false false false + | ')' => loop (i + 1) (paren - 1) brace bracket strict 0 false false false + | '{' => loop (i + 1) paren (brace + 1) bracket strict 0 false false false + | '}' => loop (i + 1) paren (brace - 1) bracket strict 0 false false false + | '[' => loop (i + 1) paren brace (bracket + 1) strict 0 false false false + | ']' => loop (i + 1) paren brace (bracket - 1) strict 0 false false false + | '⦃' => loop (i + 1) paren brace bracket (strict + 1) 0 false false false + | '⦄' => loop (i + 1) paren brace bracket (strict - 1) 0 false false false + | _ => loop (i + 1) paren brace bracket strict 0 false false false + loop 0 0 0 0 0 0 false false false + +def sliceChars (text : String) (start stop : Nat) : String := + String.ofList (text.toList.toArray.extract start stop).toList + +def forallBinders (term : Syntax) : Except String (Array SourceBinder) := do + let some marker := term[0]? + | throw s!"malformed forall syntax: {term}" + unless marker.isAtom && (marker.getAtomVal == "∀" || marker.getAtomVal == "forall") do + throw s!"expected forall syntax, got {term.getKind}: {term}" + let some binderSlot := term[1]? + | throw s!"forall syntax has no binder: {term}" + let mut binders := #[] + if binderSlot.isOfKind `null then + for binder in binderSlot.getArgs do + binders := binders ++ (← declarationBinderGroup binder) + else if binderSlot.isIdent then + binders := binders.push { name? := some binderSlot.getId, info := .default } + else + let some name := binderSlot.getArgs[0]? + | throw s!"unsupported forall binder syntax {binderSlot.getKind}: {binderSlot}" + binders := binders.push { name? := ← sourceBinderName name, info := .default } + let some predicate := term[2]? + | throw s!"forall syntax has no predicate slot: {term}" + let isBareTypeSpec := predicate.getArgs[0]?.any + (·.isOfKind ``Lean.Parser.Term.typeSpec) + if !predicate.isNone && !isBareTypeSpec then + binders := binders.push { name? := none, info := .default } + return binders + +/-- Parse only the leading Pi structure of a result type. The remainder is +replaced with `True` before parsing, so scoped term notation later in the +statement is irrelevant. -/ +partial def conclusionBinders (env : Environment) (text : String) : Except String (Array SourceBinder) := do + let text := text.trimAsciiStart.toString + let chars := text.toList.toArray + if chars[0]? == some '(' then + let afterOpen := sliceChars text 1 text.length + match findTopLevelToken .closeParen afterOpen with + | some (close, width) => + let trailing := sliceChars afterOpen (close + width) afterOpen.length + if trailing.trimAscii.isEmpty then + return ← conclusionBinders env (sliceChars afterOpen 0 close) + | none => pure () + let unicodeForall := chars[0]? == some '∀' && chars[1]?.any fun c => + c.isWhitespace || c == '(' || c == '{' || c == '[' || c == '⦃' + let asciiForall := text.startsWith "forall" && chars[6]?.any fun c => + c.isWhitespace || c == '(' || c == '{' || c == '[' || c == '⦃' + if unicodeForall || asciiForall then + let some (comma, width) := findTopLevelToken .comma text + | throw "leading forall has no top-level comma" + let forallPrefix := sliceChars text 0 (comma + width) + let command ← Parser.runParserCategory env `command + ("theorem _boundary : " ++ forallPrefix ++ " True := by trivial") + let signature ← declarationSignature command + let some typeSpec := signature[1]? + | throw "synthetic forall signature has no result type" + let some term := typeSpec[1]? + | throw "synthetic forall result type is malformed" + let here ← forallBinders term + let rest := sliceChars text (comma + width) text.length + return here ++ (← conclusionBinders env rest) + let unicodeExists := chars[0]? == some '∃' && chars[1]?.any fun c => + c.isWhitespace || c == '(' || c == '{' || c == '[' || c == '⦃' + let asciiExists := text.startsWith "exists" && chars[6]?.any fun c => + c.isWhitespace || c == '(' || c == '{' || c == '[' || c == '⦃' + if unicodeExists || asciiExists then + return #[] + match findTopLevelToken .arrow text, findTopLevelToken .iff text with + | some (arrow, width), none => + let rest := sliceChars text (arrow + width) text.length + return #[{ name? := none, info := .default }] ++ (← conclusionBinders env rest) + | _, _ => return #[] + +structure DeclarationText where + beforeNameEnd : String + afterNameEnd : String + +def firstHeaderGroup (text : String) : Option (Nat × ScanToken × ScanToken × Char × Char) := + let candidates := #[ + (.openParen, .closeParen, '(', ')'), + (.openBrace, .closeBrace, '{', '}'), + (.openBracket, .closeBracket, '[', ']'), + (.openStrict, .closeStrict, '⦃', '⦄')] + candidates.foldl (init := none) fun best (openToken, closeToken, opener, closer) => + match findTopLevelToken openToken text with + | none => best + | some (position, _) => match best with + | none => some (position, openToken, closeToken, opener, closer) + | some current => + if position < current.1 then some (position, openToken, closeToken, opener, closer) + else best + +/-- Erase binder *types* while preserving binder names and kinds. The +elaborated telescope supplies the types; this parser pass needs only the +surface boundary. Erasing types prevents scoped notation inside a binder type +from making the header impossible to parse out of its original file context. -/ +partial def sanitizeHeaderBinders (text : String) : Except String String := do + let some (start, _, closeToken, opener, closer) := firstHeaderGroup text + | return text + let before := sliceChars text 0 start + let afterOpen := sliceChars text (start + 1) text.length + let some (close, width) := findTopLevelToken closeToken afterOpen + | throw s!"unclosed declaration binder beginning with {opener}" + let inner := sliceChars afterOpen 0 close + let rest := sliceChars afterOpen (close + width) afterOpen.length + let universeGroup := opener == '{' && before.trimAsciiEnd.toString.endsWith "." + let rewritten := if universeGroup then + String.singleton opener ++ inner ++ String.singleton closer + else match findTopLevelToken .signatureColon inner with + | some (colon, _) => + String.singleton opener ++ sliceChars inner 0 colon ++ " : True" ++ + String.singleton closer + | none => + if opener == '[' then "[True]" + else String.singleton opener ++ inner ++ String.singleton closer + return before ++ rewritten ++ (← sanitizeHeaderBinders rest) + +/-- Turn an exact declaration slice into a parser-safe header command and the +original result type. -/ +def DeclarationText.headerAndResult (text : DeclarationText) : Except String (String × String) := do + let some (colon, width) := findTopLevelToken .signatureColon text.afterNameEnd + | throw "declaration header has no top-level result colon" + let rawHeader := sliceChars text.afterNameEnd 0 colon + let header := text.beforeNameEnd ++ (← sanitizeHeaderBinders rawHeader) ++ + " : True := by trivial" + let afterColon := sliceChars text.afterNameEnd (colon + width) text.afterNameEnd.length + let result := match findTopLevelToken .bodyMarker afterColon with + | some (body, _) => sliceChars afterColon 0 body + | none => afterColon + return (header, result) + +/-- Read the exact declaration range from the source module. The elaborated +environment remains authoritative for the range and telescope; parsing the +slice is only how we recover where the source header ended. -/ +def declarationSource (modName : Name) (ranges : DeclarationRanges) : IO (Except String DeclarationText) := do + let some path ← (← getSrcSearchPath).findModuleWithExt "lean" modName + | return .error s!"source file for {modName} was not found" + let source ← IO.FS.readFile path + let fileMap := FileMap.ofString source + let start := fileMap.ofPosition ranges.range.pos + let stop := fileMap.ofPosition ranges.range.endPos + let nameStop := fileMap.ofPosition ranges.selectionRange.endPos + if start > nameStop || nameStop > stop || stop > source.rawEndPos then + return .error s!"invalid declaration range for {modName}: {repr ranges.range}" + return .ok { + beforeNameEnd := source.toRawSubstring.extract start nameStop |>.toString + afterNameEnd := source.toRawSubstring.extract nameStop stop |>.toString } + +def binderBoundarySelfTest (env : Environment) : IO UInt32 := do + let cases : Array (String × Nat) := #[ + ("theorem t (n : Nat) (hn : 1 < n) : True := by trivial", 2), + ("theorem t : ∀ n : Nat, 1 < n → True := by intro; trivial", 0), + ("theorem t (x y : Nat) {α : Type} {{β : Type}} [i : Inhabited α] z : True := by trivial", 6) + ] + for (source, expected) in cases do + match parseDeclarationBinderCount env source with + | .ok actual => + if actual != expected then + IO.eprintln s!"binder-boundary self-test expected {expected}, got {actual}: {source}" + return 1 + | .error message => + IO.eprintln s!"binder-boundary self-test failed: {message}: {source}" + return 1 + let mkLocal (index : Nat) (name : Name) (info : BinderInfo) : LocalDecl := + .cdecl index { name := `_selfTest |>.appendIndexAfter index } name (.sort .zero) info .default + let outerAndHeader := #[ + mkLocal 0 `R .implicit, + mkLocal 1 `instR .instImplicit, + mkLocal 2 `I .implicit, + mkLocal 3 `hI .default, + mkLocal 4 `n .default, + mkLocal 5 `instN .instImplicit] + let alignmentCases : Array (String × String × Array LocalDecl × Nat) := #[ + ("theorem t {I : Type} (hI : True) (n : Type*) [Fintype n] : True := by trivial", + "True", outerAndHeader, 6), + ("theorem t (n : Nat) (hn : 1 < n) : True := by trivial", + "True", #[mkLocal 0 `n .default, mkLocal 1 `hn .default], 2), + ("theorem t : ∀ n : Nat, True := by intro; trivial", + "∀ n : Nat, True", #[mkLocal 0 `R .default, mkLocal 1 `n .default], 1), + ("theorem t : ∀ n : Nat, 1 < n → True := by intro; trivial", + "∀ n : Nat, 1 < n → True", #[mkLocal 0 `n .default, mkLocal 1 `h .default], 0), + ("theorem t : True ↔ ∃ n : Nat, 1 < n → True := by simp", + "True ↔ ∃ n : Nat, 1 < n → True", #[], 0), + ("theorem t : ∃ f : Nat → Nat, ∀ n, True → f n = f n := by simp", + "∃ f : Nat → Nat, ∀ n, True → f n = f n", #[], 0), + ("theorem t : True → (∀ n : Nat, 1 < n → True) := by simp", + "True → (∀ n : Nat, 1 < n → True)", + #[mkLocal 0 `h₁ .default, mkLocal 1 `n .default, mkLocal 2 `h₂ .default], 0) + ] + for (source, resultType, elaborated, expected) in alignmentCases do + let result := do + let command ← Parser.runParserCategory env `command source + let conclusion ← conclusionBinders env resultType + declarationParameterBoundary command conclusion elaborated + match result with + | .ok actual => + if actual != expected then + IO.eprintln s!"parameter-alignment self-test expected {expected}, got {actual}: {source}" + return 1 + | .error message => + IO.eprintln s!"parameter-alignment self-test failed: {message}: {source}" + return 1 + IO.println "binder-boundary self-test passed" + return 0 diff --git a/comparator/adapter/ComparatorFacts/Extract.lean b/comparator/adapter/ComparatorFacts/Extract.lean new file mode 100644 index 0000000000..33f5718a65 --- /dev/null +++ b/comparator/adapter/ComparatorFacts/Extract.lean @@ -0,0 +1,111 @@ +/- +Copyright 2026 The Formal Conjectures Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +-/ + +import Lean + +/-! +# Environment extraction + +What the elaborated environment knows about a declaration: which FC-local +constants it needs and in what order, how a requested name resolves within a +module, and the binder facts the importer emits. +-/ + +open Lean Meta + +/-- A request matches a name in full, or by dropping any whole prefix. -/ +def declares (declared : Name) (requested : String) : Bool := + let s := declared.toString + s == requested || s.endsWith ("." ++ requested) + +def binderJson (name : Name) (bi : BinderInfo) : Json := + Json.mkObj [("name", toJson name.toString), ("explicit", toJson bi.isExplicit)] + +def moduleOf (env : Environment) (n : Name) : String := + match env.getModuleIdxFor? n with + | some idx => (env.header.moduleNames[idx.toNat]?.getD Name.anonymous).toString + | none => "" + +/-- Declared by this repository, as opposed to arriving with `import Mathlib`. -/ +def isFCLocal (env : Environment) (n : Name) : Bool := + (moduleOf env n).startsWith "FormalConjectures" + +/-- The FC-local constants a declaration needs, dependencies before dependents. + +Post-order over the dependency graph, expanding through both the type and the +value of each FC-local constant: a definition's body names constants its type +does not, and `ChallengeDeps` has to carry them or the copy will not elaborate. +Mathlib and core constants are not expanded, since they arrive with +`import Mathlib`. -/ +partial def fcOrder (env : Environment) (n : Name) + (seen : Std.HashSet Name) (acc : Array Name) : Std.HashSet Name × Array Name := + if seen.contains n then (seen, acc) else + let seen := seen.insert n + match env.find? n with + | none => (seen, acc) + | some info => + let fromValue := match info.value? with + | some v => v.getUsedConstants + | none => #[] + -- An inductive has no value, and its fields live in the constructor + -- rather than in its own type: `structure EdgeN (N D : Nat) where u : V N` + -- has type `Nat → Nat → Type`, which never mentions `V`. Without the + -- constructors here the closure still contains `V`, reached some other + -- way, but orders it after `EdgeN`, and the copy does not elaborate. + let fromCtors := match info with + | .inductInfo val => val.ctors.toArray + | _ => #[] + let children := (info.type.getUsedConstants ++ fromValue ++ fromCtors).filter + fun c => isFCLocal env c && c != n + let (seen, acc) := children.foldl (fun p c => fcOrder env c p.1 p.2) (seen, acc) + (seen, acc.push n) + +unsafe def runWithImports {α : Type} (moduleNames : Array Name) + (actionToRun : MetaM α) : IO α := do + initSearchPath (← getBuildDir) + let imports := moduleNames.map fun n => { module := n } + Lean.enableInitializersExecution + let env ← Lean.importModules imports {} (trustLevel := 1024) (loadExts := true) + -- Twice the default budget, in the context's raw units, which are a + -- thousand times the `maxHeartbeats` option's: 800000 here meant "800" and + -- killed the first query. Finite, so a pathological statement errors and is + -- caught rather than grinding forever, which maxHeartbeats := 0 did. + let ctx := { fileName := "", fileMap := default, maxHeartbeats := 400000000 } + let (result, _) ← Core.CoreM.toIO (actionToRun.run' {} {}) ctx { env := env } + return result + +/-- Resolve within one module. Names declared elsewhere are not candidates, +which is what lets one environment holding every module still disambiguate +`conjecture_1_1` the way a per-module import does. -/ +def resolveIn (env : Environment) (modName : Name) (declName : String) : + Except String Name := + let inModule (n : Name) : Bool := + match env.getModuleIdxFor? n with + | some idx => env.header.moduleNames[idx.toNat]? == some modName + | none => false + -- No `isInternal` filter: `erdos_340.variants._33_mem_sub` has a component + -- starting with an underscore, which that heuristic calls internal. The + -- whole-suffix rule in `declares` already keeps auxiliary declarations out, + -- since `foo.proof_1` is not a suffix match for `foo`. + let matches_ := env.constants.toList.filterMap fun (n, _) => + if declares n declName && inModule n then some n else none + match matches_ with + | [] => .error s!"{declName} not found in {modName}" + | [n] => .ok n + | _ => + match matches_.filter (·.toString == declName) with + | [n] => .ok n + | _ => .error s!"{declName} is ambiguous: {matches_}" diff --git a/comparator/adapter/comparator_facts.lean b/comparator/adapter/comparator_facts.lean index a83e1ec1a0..f10da35e0c 100644 --- a/comparator/adapter/comparator_facts.lean +++ b/comparator/adapter/comparator_facts.lean @@ -13,12 +13,19 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -/ + +import Lean +import FormalConjecturesUtil.Answer +import FormalConjecturesUtil.Attributes.Basic +import ComparatorFacts.Binders +import ComparatorFacts.Extract + import Lean import FormalConjecturesUtil.Answer import FormalConjecturesUtil.Attributes.Basic /-! -The elaborator-side facts `comparator/adapter/fc_leaneval_importer.py` would otherwise +The executable over `ComparatorFacts/`: the elaborator-side facts `comparator/adapter/fc_leaneval_importer.py` would otherwise get by reading Lean with regular expressions. Given a module and a declaration name, this prints JSON with what the @@ -39,502 +46,8 @@ The declaration may be given in full or by any whole suffix, the same rule the Python importer uses. -/ -open Lean Meta - -/-- A request matches a name in full, or by dropping any whole prefix. -/ -def declares (declared : Name) (requested : String) : Bool := - let s := declared.toString - s == requested || s.endsWith ("." ++ requested) - -/-! ## Declaration binder boundaries, from source syntax - -A theorem proved by bare `sorry` can be stored as a full-type `sorryAx`, so the -proof value's lambda arity does not reliably separate header parameters from -binders in the conclusion. This subsystem recovers that boundary from the -declaration's source text and matches it against the elaborated telescope; -every binder fact emitted still comes from the environment. -/ - -/-- Collect every node of one syntax kind. A declaration command must contain -exactly one `declSig`; finding zero or more than one means we did not parse the -source range we thought we did, and the importer fails closed. -/ -partial def collectKind (kind : Name) (stx : Syntax) (found : Array Syntax := #[]) : - Array Syntax := - let found := if stx.isOfKind kind then found.push stx else found - stx.getArgs.foldl (fun acc child => collectKind kind child acc) found - -structure SourceBinder where - name? : Option Name - info : BinderInfo - deriving Repr - -def sourceBinderName (stx : Syntax) : Except String (Option Name) := - if stx.isIdent then .ok (some stx.getId) - else if stx.isOfKind ``Lean.Parser.Term.hole then .ok none - else .error s!"unsupported declaration binder name {stx.getKind}: {stx}" - -/-- The names and binder kinds introduced by one declaration-header group. -/ -def declarationBinderGroup (binder : Syntax) : Except String (Array SourceBinder) := do - if binder.isIdent || binder.isOfKind ``Lean.Parser.Term.hole then - return #[{ name? := ← sourceBinderName binder, info := .default }] - if binder.isOfKind ``Lean.Parser.Term.instBinder then - let some optionalName := binder[1]? - | throw s!"malformed instance binder: {binder}" - let name? ← match optionalName.getArgs[0]? with - | some name => sourceBinderName name - | none => pure none - return #[{ name?, info := .instImplicit }] - let info? := - if binder.isOfKind ``Lean.Parser.Term.explicitBinder then some BinderInfo.default - else if binder.isOfKind ``Lean.Parser.Term.implicitBinder then some .implicit - else if binder.isOfKind ``Lean.Parser.Term.strictImplicitBinder then some .strictImplicit - else none - let some info := info? - | throw s!"unsupported declaration binder syntax {binder.getKind}: {binder}" - let some names := binder[1]? - | throw s!"malformed declaration binder: {binder}" - if names.getArgs.isEmpty then - throw s!"declaration binder has no names: {binder}" - names.getArgs.mapM fun name => do - return { name? := ← sourceBinderName name, info } - -def declarationSignature (command : Syntax) : Except String Syntax := do - let signatures := collectKind ``Lean.Parser.Command.declSig command - let [signature] := signatures.toList - | throw s!"expected exactly one declaration signature, found {signatures.size}" - return signature - -def declarationBinders (command : Syntax) : Except String (Array SourceBinder) := do - let signature ← declarationSignature command - let some header := signature[0]? - | throw "declaration signature has no binder header" - header.getArgs.foldlM (fun binders binder => do - return binders ++ (← declarationBinderGroup binder)) #[] - -/-- Number of parameters written before the colon in a declaration header. - -This is deliberately a syntax boundary, not a proof-value heuristic: -`theorem foo (n : Nat) : P n` has one declaration parameter, while -`theorem foo : ∀ n : Nat, P n` has none. Lean 4.32 may store a theorem proved -by `sorry` as a `sorryAx` at the full forall type, with no lambda wrappers, so -the old proof-value lambda arity silently reported zero for the first form. -The syntax supplies only the boundary; names, types, and explicitness still -come from the elaborated telescope below. -/ -def declarationBinderCount (command : Syntax) : Except String Nat := do - return (← declarationBinders command).size - -def sourceBinderMatches (source : SourceBinder) (elaborated : LocalDecl) : Bool := - source.info == elaborated.binderInfo && match source.name? with - | some name => name == elaborated.userName - | none => true - -def sourceBindersMatchAt (source : Array SourceBinder) (elaborated : Array LocalDecl) - (start : Nat) : Bool := - source.zipIdx.all fun (binder, offset) => - match elaborated[start + offset]? with - | some candidate => sourceBinderMatches binder candidate - | none => false - -/-- Locate the end of the declaration parameters in the elaborated telescope. - -Lean inserts used outer `variable`s before the binders written in the -declaration header. We therefore locate the exact typed/named header sequence -inside the telescope and retain everything through its end. This handles -Köthe's `{R} [Ring R]` outer parameters as well as ordinary self-contained -headers. Multiple matches are rejected rather than guessed. -/ -def declarationParameterBoundary (command : Syntax) (conclusion : Array SourceBinder) - (elaborated : Array LocalDecl) : Except String Nat := do - let source ← declarationBinders command - if source.isEmpty then - if conclusion.size > elaborated.size then - throw s!"source conclusion has {conclusion.size} binders, but the elaborated type has only {elaborated.size}" - let boundary := elaborated.size - conclusion.size - unless sourceBindersMatchAt conclusion elaborated boundary do - throw s!"source conclusion binders {repr conclusion} do not match the elaborated telescope suffix" - return boundary - if source.size > elaborated.size then - throw s!"source header has {source.size} binders, but the elaborated type has only {elaborated.size}" - let starts := (Array.range (elaborated.size - source.size + 1)).filter - (sourceBindersMatchAt source elaborated) - let [start] := starts.toList - | throw s!"expected one match for {source.size} source-header binders in the elaborated telescope, found {starts.size}" - return start + source.size - -def parseDeclarationBinderCount (env : Environment) (source : String) : Except String Nat := do - let command ← Parser.runParserCategory env `command source - declarationBinderCount command -inductive ScanToken where - | signatureColon - | bodyMarker - | comma - | arrow - | iff - | openParen - | openBrace - | openBracket - | openStrict - | closeParen - | closeBrace - | closeBracket - | closeStrict - -def scanTokenAt (wanted : ScanToken) (chars : Array Char) (i : Nat) : Option Nat := - match wanted with - | .signatureColon => - if chars[i]? == some ':' && chars[i + 1]? != some '=' then some 1 else none - | .bodyMarker => - if chars[i]? == some ':' && chars[i + 1]? == some '=' then some 2 else none - | .comma => if chars[i]? == some ',' then some 1 else none - | .arrow => - if chars[i]? == some '→' then some 1 - else if chars[i]? == some '-' && chars[i + 1]? == some '>' then some 2 - else none - | .iff => if chars[i]? == some '↔' then some 1 else none - | .openParen => if chars[i]? == some '(' then some 1 else none - | .openBrace => if chars[i]? == some '{' then some 1 else none - | .openBracket => if chars[i]? == some '[' then some 1 else none - | .openStrict => if chars[i]? == some '⦃' then some 1 else none - | .closeParen => if chars[i]? == some ')' then some 1 else none - | .closeBrace => if chars[i]? == some '}' then some 1 else none - | .closeBracket => if chars[i]? == some ']' then some 1 else none - | .closeStrict => if chars[i]? == some '⦄' then some 1 else none - -/-- Find syntax punctuation at delimiter depth zero, ignoring comments and -strings. This scanner does not interpret terms; it only lets us replace the -result type with `True` before asking Lean's real command parser to read the -declaration header. Scoped notation in the result therefore cannot make an -otherwise ordinary header unparseable. -/ -partial def findTopLevelToken (wanted : ScanToken) (text : String) : Option (Nat × Nat) := - let chars := text.toList.toArray - let rec loop (i paren brace bracket strict blockComment : Nat) - (lineComment inString escaped : Bool) : Option (Nat × Nat) := - if i >= chars.size then none else - let current := chars[i]! - let next := chars[i + 1]? - if lineComment then - loop (i + 1) paren brace bracket strict blockComment (current != '\n') inString false - else if blockComment > 0 then - if current == '/' && next == some '-' then - loop (i + 2) paren brace bracket strict (blockComment + 1) false inString false - else if current == '-' && next == some '/' then - loop (i + 2) paren brace bracket strict (blockComment - 1) false inString false - else - loop (i + 1) paren brace bracket strict blockComment false inString false - else if inString then - if escaped then loop (i + 1) paren brace bracket strict 0 false true false - else if current == '\\' then loop (i + 1) paren brace bracket strict 0 false true true - else loop (i + 1) paren brace bracket strict 0 false (current != '"') false - else if current == '-' && next == some '-' then - loop (i + 2) paren brace bracket strict 0 true false false - else if current == '/' && next == some '-' then - loop (i + 2) paren brace bracket strict 1 false false false - else if current == '"' then - loop (i + 1) paren brace bracket strict 0 false true false - else if paren == 0 && brace == 0 && bracket == 0 && strict == 0 then - match scanTokenAt wanted chars i with - | some width => some (i, width) - | none => match current with - | '(' => loop (i + 1) 1 brace bracket strict 0 false false false - | '{' => loop (i + 1) paren 1 bracket strict 0 false false false - | '[' => loop (i + 1) paren brace 1 strict 0 false false false - | '⦃' => loop (i + 1) paren brace bracket 1 0 false false false - | _ => loop (i + 1) paren brace bracket strict 0 false false false - else match current with - | '(' => loop (i + 1) (paren + 1) brace bracket strict 0 false false false - | ')' => loop (i + 1) (paren - 1) brace bracket strict 0 false false false - | '{' => loop (i + 1) paren (brace + 1) bracket strict 0 false false false - | '}' => loop (i + 1) paren (brace - 1) bracket strict 0 false false false - | '[' => loop (i + 1) paren brace (bracket + 1) strict 0 false false false - | ']' => loop (i + 1) paren brace (bracket - 1) strict 0 false false false - | '⦃' => loop (i + 1) paren brace bracket (strict + 1) 0 false false false - | '⦄' => loop (i + 1) paren brace bracket (strict - 1) 0 false false false - | _ => loop (i + 1) paren brace bracket strict 0 false false false - loop 0 0 0 0 0 0 false false false - -def sliceChars (text : String) (start stop : Nat) : String := - String.ofList (text.toList.toArray.extract start stop).toList - -def forallBinders (term : Syntax) : Except String (Array SourceBinder) := do - let some marker := term[0]? - | throw s!"malformed forall syntax: {term}" - unless marker.isAtom && (marker.getAtomVal == "∀" || marker.getAtomVal == "forall") do - throw s!"expected forall syntax, got {term.getKind}: {term}" - let some binderSlot := term[1]? - | throw s!"forall syntax has no binder: {term}" - let mut binders := #[] - if binderSlot.isOfKind `null then - for binder in binderSlot.getArgs do - binders := binders ++ (← declarationBinderGroup binder) - else if binderSlot.isIdent then - binders := binders.push { name? := some binderSlot.getId, info := .default } - else - let some name := binderSlot.getArgs[0]? - | throw s!"unsupported forall binder syntax {binderSlot.getKind}: {binderSlot}" - binders := binders.push { name? := ← sourceBinderName name, info := .default } - let some predicate := term[2]? - | throw s!"forall syntax has no predicate slot: {term}" - let isBareTypeSpec := predicate.getArgs[0]?.any - (·.isOfKind ``Lean.Parser.Term.typeSpec) - if !predicate.isNone && !isBareTypeSpec then - binders := binders.push { name? := none, info := .default } - return binders - -/-- Parse only the leading Pi structure of a result type. The remainder is -replaced with `True` before parsing, so scoped term notation later in the -statement is irrelevant. -/ -partial def conclusionBinders (env : Environment) (text : String) : Except String (Array SourceBinder) := do - let text := text.trimAsciiStart.toString - let chars := text.toList.toArray - if chars[0]? == some '(' then - let afterOpen := sliceChars text 1 text.length - match findTopLevelToken .closeParen afterOpen with - | some (close, width) => - let trailing := sliceChars afterOpen (close + width) afterOpen.length - if trailing.trimAscii.isEmpty then - return ← conclusionBinders env (sliceChars afterOpen 0 close) - | none => pure () - let unicodeForall := chars[0]? == some '∀' && chars[1]?.any fun c => - c.isWhitespace || c == '(' || c == '{' || c == '[' || c == '⦃' - let asciiForall := text.startsWith "forall" && chars[6]?.any fun c => - c.isWhitespace || c == '(' || c == '{' || c == '[' || c == '⦃' - if unicodeForall || asciiForall then - let some (comma, width) := findTopLevelToken .comma text - | throw "leading forall has no top-level comma" - let forallPrefix := sliceChars text 0 (comma + width) - let command ← Parser.runParserCategory env `command - ("theorem _boundary : " ++ forallPrefix ++ " True := by trivial") - let signature ← declarationSignature command - let some typeSpec := signature[1]? - | throw "synthetic forall signature has no result type" - let some term := typeSpec[1]? - | throw "synthetic forall result type is malformed" - let here ← forallBinders term - let rest := sliceChars text (comma + width) text.length - return here ++ (← conclusionBinders env rest) - let unicodeExists := chars[0]? == some '∃' && chars[1]?.any fun c => - c.isWhitespace || c == '(' || c == '{' || c == '[' || c == '⦃' - let asciiExists := text.startsWith "exists" && chars[6]?.any fun c => - c.isWhitespace || c == '(' || c == '{' || c == '[' || c == '⦃' - if unicodeExists || asciiExists then - return #[] - match findTopLevelToken .arrow text, findTopLevelToken .iff text with - | some (arrow, width), none => - let rest := sliceChars text (arrow + width) text.length - return #[{ name? := none, info := .default }] ++ (← conclusionBinders env rest) - | _, _ => return #[] - -structure DeclarationText where - beforeNameEnd : String - afterNameEnd : String - -def firstHeaderGroup (text : String) : Option (Nat × ScanToken × ScanToken × Char × Char) := - let candidates := #[ - (.openParen, .closeParen, '(', ')'), - (.openBrace, .closeBrace, '{', '}'), - (.openBracket, .closeBracket, '[', ']'), - (.openStrict, .closeStrict, '⦃', '⦄')] - candidates.foldl (init := none) fun best (openToken, closeToken, opener, closer) => - match findTopLevelToken openToken text with - | none => best - | some (position, _) => match best with - | none => some (position, openToken, closeToken, opener, closer) - | some current => - if position < current.1 then some (position, openToken, closeToken, opener, closer) - else best - -/-- Erase binder *types* while preserving binder names and kinds. The -elaborated telescope supplies the types; this parser pass needs only the -surface boundary. Erasing types prevents scoped notation inside a binder type -from making the header impossible to parse out of its original file context. -/ -partial def sanitizeHeaderBinders (text : String) : Except String String := do - let some (start, _, closeToken, opener, closer) := firstHeaderGroup text - | return text - let before := sliceChars text 0 start - let afterOpen := sliceChars text (start + 1) text.length - let some (close, width) := findTopLevelToken closeToken afterOpen - | throw s!"unclosed declaration binder beginning with {opener}" - let inner := sliceChars afterOpen 0 close - let rest := sliceChars afterOpen (close + width) afterOpen.length - let universeGroup := opener == '{' && before.trimAsciiEnd.toString.endsWith "." - let rewritten := if universeGroup then - String.singleton opener ++ inner ++ String.singleton closer - else match findTopLevelToken .signatureColon inner with - | some (colon, _) => - String.singleton opener ++ sliceChars inner 0 colon ++ " : True" ++ - String.singleton closer - | none => - if opener == '[' then "[True]" - else String.singleton opener ++ inner ++ String.singleton closer - return before ++ rewritten ++ (← sanitizeHeaderBinders rest) - -/-- Turn an exact declaration slice into a parser-safe header command and the -original result type. -/ -def DeclarationText.headerAndResult (text : DeclarationText) : Except String (String × String) := do - let some (colon, width) := findTopLevelToken .signatureColon text.afterNameEnd - | throw "declaration header has no top-level result colon" - let rawHeader := sliceChars text.afterNameEnd 0 colon - let header := text.beforeNameEnd ++ (← sanitizeHeaderBinders rawHeader) ++ - " : True := by trivial" - let afterColon := sliceChars text.afterNameEnd (colon + width) text.afterNameEnd.length - let result := match findTopLevelToken .bodyMarker afterColon with - | some (body, _) => sliceChars afterColon 0 body - | none => afterColon - return (header, result) - -/-- Read the exact declaration range from the source module. The elaborated -environment remains authoritative for the range and telescope; parsing the -slice is only how we recover where the source header ended. -/ -def declarationSource (modName : Name) (ranges : DeclarationRanges) : IO (Except String DeclarationText) := do - let some path ← (← getSrcSearchPath).findModuleWithExt "lean" modName - | return .error s!"source file for {modName} was not found" - let source ← IO.FS.readFile path - let fileMap := FileMap.ofString source - let start := fileMap.ofPosition ranges.range.pos - let stop := fileMap.ofPosition ranges.range.endPos - let nameStop := fileMap.ofPosition ranges.selectionRange.endPos - if start > nameStop || nameStop > stop || stop > source.rawEndPos then - return .error s!"invalid declaration range for {modName}: {repr ranges.range}" - return .ok { - beforeNameEnd := source.toRawSubstring.extract start nameStop |>.toString - afterNameEnd := source.toRawSubstring.extract nameStop stop |>.toString } - -def binderBoundarySelfTest (env : Environment) : IO UInt32 := do - let cases : Array (String × Nat) := #[ - ("theorem t (n : Nat) (hn : 1 < n) : True := by trivial", 2), - ("theorem t : ∀ n : Nat, 1 < n → True := by intro; trivial", 0), - ("theorem t (x y : Nat) {α : Type} {{β : Type}} [i : Inhabited α] z : True := by trivial", 6) - ] - for (source, expected) in cases do - match parseDeclarationBinderCount env source with - | .ok actual => - if actual != expected then - IO.eprintln s!"binder-boundary self-test expected {expected}, got {actual}: {source}" - return 1 - | .error message => - IO.eprintln s!"binder-boundary self-test failed: {message}: {source}" - return 1 - let mkLocal (index : Nat) (name : Name) (info : BinderInfo) : LocalDecl := - .cdecl index { name := `_selfTest |>.appendIndexAfter index } name (.sort .zero) info .default - let outerAndHeader := #[ - mkLocal 0 `R .implicit, - mkLocal 1 `instR .instImplicit, - mkLocal 2 `I .implicit, - mkLocal 3 `hI .default, - mkLocal 4 `n .default, - mkLocal 5 `instN .instImplicit] - let alignmentCases : Array (String × String × Array LocalDecl × Nat) := #[ - ("theorem t {I : Type} (hI : True) (n : Type*) [Fintype n] : True := by trivial", - "True", outerAndHeader, 6), - ("theorem t (n : Nat) (hn : 1 < n) : True := by trivial", - "True", #[mkLocal 0 `n .default, mkLocal 1 `hn .default], 2), - ("theorem t : ∀ n : Nat, True := by intro; trivial", - "∀ n : Nat, True", #[mkLocal 0 `R .default, mkLocal 1 `n .default], 1), - ("theorem t : ∀ n : Nat, 1 < n → True := by intro; trivial", - "∀ n : Nat, 1 < n → True", #[mkLocal 0 `n .default, mkLocal 1 `h .default], 0), - ("theorem t : True ↔ ∃ n : Nat, 1 < n → True := by simp", - "True ↔ ∃ n : Nat, 1 < n → True", #[], 0), - ("theorem t : ∃ f : Nat → Nat, ∀ n, True → f n = f n := by simp", - "∃ f : Nat → Nat, ∀ n, True → f n = f n", #[], 0), - ("theorem t : True → (∀ n : Nat, 1 < n → True) := by simp", - "True → (∀ n : Nat, 1 < n → True)", - #[mkLocal 0 `h₁ .default, mkLocal 1 `n .default, mkLocal 2 `h₂ .default], 0) - ] - for (source, resultType, elaborated, expected) in alignmentCases do - let result := do - let command ← Parser.runParserCategory env `command source - let conclusion ← conclusionBinders env resultType - declarationParameterBoundary command conclusion elaborated - match result with - | .ok actual => - if actual != expected then - IO.eprintln s!"parameter-alignment self-test expected {expected}, got {actual}: {source}" - return 1 - | .error message => - IO.eprintln s!"parameter-alignment self-test failed: {message}: {source}" - return 1 - IO.println "binder-boundary self-test passed" - return 0 - -/-! ## Environment extraction -/ - -def binderJson (name : Name) (bi : BinderInfo) : Json := - Json.mkObj [("name", toJson name.toString), ("explicit", toJson bi.isExplicit)] - -def moduleOf (env : Environment) (n : Name) : String := - match env.getModuleIdxFor? n with - | some idx => (env.header.moduleNames[idx.toNat]?.getD Name.anonymous).toString - | none => "" - -/-- Declared by this repository, as opposed to arriving with `import Mathlib`. -/ -def isFCLocal (env : Environment) (n : Name) : Bool := - (moduleOf env n).startsWith "FormalConjectures" - -/-- The FC-local constants a declaration needs, dependencies before dependents. - -Post-order over the dependency graph, expanding through both the type and the -value of each FC-local constant: a definition's body names constants its type -does not, and `ChallengeDeps` has to carry them or the copy will not elaborate. -Mathlib and core constants are not expanded, since they arrive with -`import Mathlib`. -/ -partial def fcOrder (env : Environment) (n : Name) - (seen : Std.HashSet Name) (acc : Array Name) : Std.HashSet Name × Array Name := - if seen.contains n then (seen, acc) else - let seen := seen.insert n - match env.find? n with - | none => (seen, acc) - | some info => - let fromValue := match info.value? with - | some v => v.getUsedConstants - | none => #[] - -- An inductive has no value, and its fields live in the constructor - -- rather than in its own type: `structure EdgeN (N D : Nat) where u : V N` - -- has type `Nat → Nat → Type`, which never mentions `V`. Without the - -- constructors here the closure still contains `V`, reached some other - -- way, but orders it after `EdgeN`, and the copy does not elaborate. - let fromCtors := match info with - | .inductInfo val => val.ctors.toArray - | _ => #[] - let children := (info.type.getUsedConstants ++ fromValue ++ fromCtors).filter - fun c => isFCLocal env c && c != n - let (seen, acc) := children.foldl (fun p c => fcOrder env c p.1 p.2) (seen, acc) - (seen, acc.push n) - -unsafe def runWithImports {α : Type} (moduleNames : Array Name) - (actionToRun : MetaM α) : IO α := do - initSearchPath (← getBuildDir) - let imports := moduleNames.map fun n => { module := n } - Lean.enableInitializersExecution - let env ← Lean.importModules imports {} (trustLevel := 1024) (loadExts := true) - -- Twice the default budget, in the context's raw units, which are a - -- thousand times the `maxHeartbeats` option's: 800000 here meant "800" and - -- killed the first query. Finite, so a pathological statement errors and is - -- caught rather than grinding forever, which maxHeartbeats := 0 did. - let ctx := { fileName := "", fileMap := default, maxHeartbeats := 400000000 } - let (result, _) ← Core.CoreM.toIO (actionToRun.run' {} {}) ctx { env := env } - return result - -/-- Resolve within one module. Names declared elsewhere are not candidates, -which is what lets one environment holding every module still disambiguate -`conjecture_1_1` the way a per-module import does. -/ -def resolveIn (env : Environment) (modName : Name) (declName : String) : - Except String Name := - let inModule (n : Name) : Bool := - match env.getModuleIdxFor? n with - | some idx => env.header.moduleNames[idx.toNat]? == some modName - | none => false - -- No `isInternal` filter: `erdos_340.variants._33_mem_sub` has a component - -- starting with an underscore, which that heuristic calls internal. The - -- whole-suffix rule in `declares` already keeps auxiliary declarations out, - -- since `foo.proof_1` is not a suffix match for `foo`. - let matches_ := env.constants.toList.filterMap fun (n, _) => - if declares n declName && inModule n then some n else none - match matches_ with - | [] => .error s!"{declName} not found in {modName}" - | [n] => .ok n - | _ => - match matches_.filter (·.toString == declName) with - | [n] => .ok n - | _ => .error s!"{declName} is ambiguous: {matches_}" +open Lean Meta unsafe def main (args : List String) : IO UInt32 := do match args with diff --git a/comparator/adapter/fc_leaneval_importer.py b/comparator/adapter/fc_leaneval_importer.py index 9851579962..abe63ee112 100644 --- a/comparator/adapter/fc_leaneval_importer.py +++ b/comparator/adapter/fc_leaneval_importer.py @@ -25,7 +25,6 @@ that dependency unrecoverable from the compiled environment. """ -import json import pathlib import re import subprocess @@ -34,15 +33,31 @@ import tomllib from leaneval_interface import ( - DefinitionHole, MarkedUpModule, ProblemManifest, SourceRecord, TargetRecord, ) - -ROOT = pathlib.Path(__file__).resolve().parent.parent.parent -SOURCE_DIRS = [ROOT / "FormalConjectures"] +from fc_source import ( + DECL_START, + docstring_reference, + elaborator_facts, + file_scoped_preamble, + find_declaration, + flatten_declared_name, + hoist_answers, + localise_notation, + module_name, + module_source_path, + notation_blocks, + pins, + replace_proof_with_sorry, + ROOT, + slice_range, + strip_decorations, + strip_fc_attributes, + unwrap_answers, +) COMPARATOR_DIR = ROOT / "comparator" MANIFEST_DIR = COMPARATOR_DIR / "problems" @@ -50,24 +65,6 @@ PERMITTED_AXIOMS = ("propext", "Quot.sound", "Classical.choice") -DECL_START = re.compile( - # `local notation` and `scoped notation` carry the modifier before the - # keyword. Without them here, Erdos 125's `local notation "A" => ...` typed - # as nothing and was dropped, and its statements lost the sets they name. - r"^(?:noncomputable\s+|private\s+|protected\s+|local\s+|scoped\s+)*" - r"(theorem|lemma|def|abbrev|structure|inductive|instance|notation)\s", -) -KEEP_LOOSE = re.compile( - # `local notation`, `local macro` and friends scope to the file exactly - # like `open` does, and a statement that names what they define does not - # parse without them. `noncomputable section` is a section for the scope - # stack and a compilation mode for everything inside it. - r"^(?:(?:local|scoped)\s+)?" - r"(open|variable|universe|section|namespace|end|attribute|set_option" - r"|notation|postfix|prefix|infixl|infixr|infix|macro|syntax|macro_rules)\b" - r"|^noncomputable section\b" -) - def _tools_file(): """comparator/tools.toml is the one machine-readable source of pins, and @@ -92,32 +89,6 @@ def target_pins(): ) -def elaborator_facts(module, declaration): - """What the elaborated environment knows about a declaration. - - Runs `lake exe comparator_facts`, which imports the module and reports the - declaration's source range, its binders with real explicitness, and the - inferred type of each `answer(sorry)` slot. Every one of these used to be - reconstructed from text, and each reconstruction had failure modes the - elaborator does not. - """ - proc = subprocess.run( - ["lake", "exe", "comparator_facts", module, declaration], - capture_output=True, - text=True, - cwd=ROOT, - ) - if proc.returncode != 0: - raise SystemExit( - f"comparator_facts {declaration}: " - f"{proc.stderr.strip() or proc.stdout.strip()}" - ) - out = proc.stdout - if "{" not in out: - raise SystemExit(f"comparator_facts {declaration}: no JSON in output") - return json.loads(out[out.index("{") :]) - - def explicit_copy_dependencies(problem_file): """Source-only dependencies the compiled environment cannot retain. @@ -178,59 +149,6 @@ def merge_dependency_records(*groups): return merged -def file_scoped_preamble(lines, start_line): - """Directives in force at `start_line`, and the namespace stack there. - - Lean scopes `open`, `variable`, `universe`, `set_option` and notation to - the file, so the marked-up module has to restate them; nothing in the - olean records them. A directive counts only if it precedes the statement - and its scope still encloses it. - """ - stack, preamble, depth = [], [], 0 - lines = lines[: start_line - 1] - index = 0 - while index < len(lines): - line = lines[index] - if depth == 0 and KEEP_LOOSE.match(line) and not line.rstrip().endswith(" in"): - kind = line.split()[0] - if kind == "noncomputable": - # `noncomputable section [name]` opens a section. - kind = "section" - parts = line.split(None, 2) - name = parts[2].strip() if len(parts) > 2 else None - else: - parts = line.split(None, 1) - name = parts[1].strip() if len(parts) > 1 else None - if kind in ("namespace", "section"): - stack.append((kind, name, line)) - elif kind == "end": - if stack and ( - stack[-1][1] == name or (name is None and stack[-1][0] == "section") - ): - stack.pop() - else: - # A `macro` or `notation` body may continue on indented - # lines; a single kept line would be broken syntax. - text = [line] - while index + 1 < len(lines) and ( - lines[index + 1][:1].isspace() and lines[index + 1].strip() - ): - index += 1 - text.append(lines[index]) - preamble.append(("\n".join(text), list(stack))) - depth += len(re.findall(r"/-", line)) - len(re.findall(r"-/", line)) - depth = max(depth, 0) - index += 1 - scope = list(stack) - in_force = [text for text, s in preamble if s == scope[: len(s)]] - # A statement inside `noncomputable section` restates the mode, since the - # copy has left the section behind and a noncomputable definition in the - # statement's closure would otherwise fail to compile. - if any(k == "section" and line.startswith("noncomputable") for k, _, line in scope): - in_force.append("noncomputable section") - return in_force, [n for k, n, _ in scope if k == "namespace" and n] - - def load_manifest(problem_id): """Read the rare source-boundary facts Lean cannot select by itself. @@ -271,237 +189,10 @@ def load_manifest(problem_id): return data -def docstring_reference(module_doc): - """The source citation Formal Conjectures already writes in the module. - - Module docstrings carry a `*Reference:*` line naming where the problem - comes from, sometimes with several links under it. The first is the - problem's own; later ones are commentary and proof notes. - """ - if not module_doc: - return "" - after = module_doc.split("*Reference:*", 1) - if len(after) != 2: - return "" - link = re.search(r"\]\((https?://[^)\s]+)\)", after[1]) - return link.group(1) if link else "" - - def manifest_ids(): return sorted(p.stem for p in MANIFEST_DIR.glob("*.toml")) -def module_name(rel_path): - """The Lean module name for a path under `FormalConjectures/`. - - Most problem files are named for a number, which is not an identifier, so - the component is written in guillemets: - `FormalConjectures.ErdosProblems.«940»`. - """ - parts = [ - c if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", c) else f"«{c}»" - for c in str(rel_path)[: -len(".lean")].split("/") - ] - return ".".join(parts) - - -def _declaring_files(name): - """The files whose text declares `name` as a theorem or lemma.""" - pattern = re.compile( - rf"(?:theorem|lemma)\s+(?:[\w.«»]*\.)?{re.escape(name)}[\s:]" - ) - hits = [] - for src in SOURCE_DIRS: - for path in sorted(src.rglob("*.lean")): - if pattern.search(path.read_text(encoding="utf-8")): - hits.append(path) - return hits - - -def _declares_namespaces(text, components): - """True if the file opens namespaces spelling out `components` in order. - - A single `namespace A.B` line declares both at once, so the check is on - the concatenated stack, not line by line. Text-level and approximate on - purpose — the elaborated environment settles the truth later; this only - ranks candidate files. - """ - stack = [] - for line in text.split("\n"): - m = re.match(r"\s*namespace\s+([\w.«»]+)", line) - if m: - stack.extend(m.group(1).split(".")) - return any( - stack[i : i + len(components)] == list(components) - for i in range(len(stack) - len(components) + 1) - ) - - -def find_declaration(basename, module=None): - """Locate the file declaring `basename`. Returns (path, imports, doc, body). - - A fully qualified name resolves through the enclosing `namespace` stack; - `module` names the file when more than one declares the name, and comes - from the problem's FC problem file. - """ - if module is not None: - named = ROOT / module - if not named.exists(): - raise SystemExit(f"manifest names {module}, which does not exist") - return _read_source(named) - hits = _declaring_files(basename) - if not hits and "." in basename: - # A fully qualified request such as `OeisA303656.conjecture` names a - # declaration whose file spells only `conjecture`, the prefix coming - # from an enclosing `namespace`. Try each split of the request into - # (namespace prefix, declared suffix), keeping files that declare the - # suffix inside that namespace. Splits are tried longest-suffix first, - # because a declared name may itself contain dots - # (`erdos_125.variants.positive_unequal_density`). - parts = basename.split(".") - for cut in range(1, len(parts)): - prefix, suffix = parts[:cut], ".".join(parts[cut:]) - hits = [ - path - for path in _declaring_files(suffix) - if _declares_namespaces(path.read_text(encoding="utf-8"), prefix) - ] - if hits: - break - if not hits: - raise SystemExit( - f"no declaration named {basename!r} found under FormalConjectures/" - ) - if len(hits) > 1: - raise SystemExit( - f"{basename!r} is ambiguous: " - + ", ".join(str(h.relative_to(ROOT)) for h in hits) - + "; pass --module to choose one, or record the choice in " - "comparator/problems/.toml" - ) - return _read_source(hits[0]) - - -def _read_source(path): - """Return (path, imports, module docstring, body after the licence header). - - The docstring is read but not removed. It sits below the imports rather - than at the top, so it is found by searching; the body deliberately still - contains it, because `strip_decorations` removes docstrings per - declaration and the generated files are compared byte for byte. - """ - original = path.read_text(encoding="utf-8") - text = re.sub(r"\A/-.*?-/\s*", "", original, flags=re.DOTALL) - found = re.search(r"/-!.*?-/", text, flags=re.DOTALL) - doc = found.group(0) if found else "" - imports = re.findall(r"^import\s+(\S+)", original, re.MULTILINE) - return path, imports, doc, text - - -def strip_decorations(block_text): - """Remove the docstring, line comments and attributes from a declaration. - - These interleave. Erdos 918 puts a `--` formalisation note between its - docstring and its `@[category ...]` line, and one anchored pass each left - the attribute in place. `@[category research open, AMS 5]` then reached the - marked-up module, where the workspace has no such attribute, and Lean - parsed as far as the `open` inside it before giving up. - """ - # `open X in` binds to the declaration and has to survive, but it sits - # above the docstring, so stripping anchored at the start would stop dead - # on it. - prefix = "" - m = re.match(r"\A\s*(open\b[^\n]*\bin)\n", block_text) - if m: - prefix = m.group(1) + "\n" - block_text = block_text[m.end() :] - while True: - stripped = re.sub(r"\A\s*/--.*?-/\s*", "", block_text, flags=re.DOTALL) - stripped = re.sub(r"\A\s*--[^\n]*\n", "", stripped) - stripped = re.sub(r"\A\s*@\[[^\]]*\]\s*", "", stripped, flags=re.DOTALL) - if stripped == block_text: - return prefix + stripped - block_text = stripped - - -# Attributes this repository defines. A generated workspace requires Mathlib -# and nothing else, so these have to go; everything else has to stay. -FC_ATTRIBUTES = ("category", "AMS", "formal_proof") - - -def strip_fc_attributes(block_text): - """Remove this repository's own attributes from a copied declaration. - - Unlike `strip_decorations`, which clears every attribute off the target - statement, this keeps the rest. A dependency is copied to be elaborated, - not restated, and dropping `simp`, `reducible` or `instance` attributes - changes how the declarations after it in the same closure elaborate. - """ - - def replace(match): - inner = match.group(1) - # Nested brackets mean an argument this simple split would cut in - # half, so leave the whole attribute alone rather than mangle it. - if "[" in inner: - return match.group(0) - kept = [ - part.strip() - for part in inner.split(",") - if part.strip() and part.strip().split()[0] not in FC_ATTRIBUTES - ] - return f"@[{', '.join(kept)}]" if kept else "" - - text = re.sub(r"@\[([^\]]*)\]", replace, block_text) - # An attribute line that emptied out leaves a blank line behind. - return re.sub(r"^[ \t]*\n", "", text, flags=re.MULTILINE) - - -def split_module(module): - """The components of a dotted module name, respecting guillemet quoting. - - A guillemet-quoted component may itself contain dots — - `FormalConjectures.Arxiv.«0912.2382».CurlingNumberConjecture` names the - directory `0912.2382` — so splitting on every dot decodes a path that - does not exist. This is the one place a module name is taken apart; - `module_name` is its inverse and a test holds the pair to that. - """ - parts = re.findall(r"«[^»]*»|[^.«»]+", module) - if ".".join(parts) != module: - raise SystemExit(f"{module!r} is not a well-formed module name") - return [p[1:-1] if p.startswith("«") else p for p in parts] - - -def module_source_path(module): - """The file declaring a dotted Lean module name, undoing guillemets.""" - parts = split_module(module) - # Not `with_suffix`: a final component containing a dot would lose its - # tail to the suffix replacement. - path = ROOT.joinpath(*parts[:-1], parts[-1] + ".lean") - if not path.is_file(): - raise SystemExit(f"{module}: no source file at {path}") - return path - - -def slice_range(lines, source_range): - """The source text a declaration range covers, and the line it starts on. - - `open X in` binds to the declaration below it but sits above what the - range covers in some toolchains, so it is pulled in when present. - """ - lo, hi = source_range["startLine"], source_range["endLine"] - end_column = source_range.get("endColumn") - while ( - lo > 1 - and lines[lo - 2].rstrip().endswith(" in") - and KEEP_LOOSE.match(lines[lo - 2]) - ): - lo -= 1 - sliced = lines[lo - 1 : hi] - if end_column is not None and sliced: - sliced = sliced[:-1] + [sliced[-1][:end_column]] - return "\n".join(sliced), lo - - def closure_region( dependencies, generated, declaration, opened_namespaces=(), target_name=None ): @@ -645,447 +336,6 @@ def covered_by_another(dep): ), provenance -NOTATION_COMMAND = re.compile( - r"^(?:@\[[^\]]*\]\s*)?(?:scoped\[[\w.«»]+\]\s+)?(?:scoped\s+)?" - r"(?:notation[0-9]*|postfix|prefix|infixl|infixr|infix)[:\s]" -) - -_NOTATION_CACHE = None - - -def fc_notation_commands(): - """Every exportable notation command an FC module defines, with its token. - - A notation is not a constant, so the elaborated closure never reports it: - a statement written as `ℝ²` names `EuclideanSpace ℝ (Fin 2)` in the - environment and `ℝ²` only in its text. The copy carries the text, so the - commands that make such tokens parse have to be found at the text layer. - `local` notations are file-scoped at their origin and cannot be in force - in a problem file, so they are not candidates. - - Returns `[(tokens, command, namespaces)]`, where `tokens` are the - command's string literals that contain a non-ASCII character — the - distinctive ones worth matching on — and `namespaces` is the stack a - plain `scoped` command needs restated around it. - """ - global _NOTATION_CACHE - if _NOTATION_CACHE is not None: - return _NOTATION_CACHE - commands = [] - roots = [ROOT / "FormalConjecturesForMathlib", ROOT / "FormalConjecturesUtil"] - for src in roots + SOURCE_DIRS: - for path in sorted(src.rglob("*.lean")): - lines = path.read_text(encoding="utf-8").split("\n") - for index, line in enumerate(lines): - if not NOTATION_COMMAND.match(line): - continue - text = [line] - follow = index + 1 - while follow < len(lines) and ( - lines[follow][:1].isspace() and lines[follow].strip() - ): - text.append(lines[follow]) - follow += 1 - command = "\n".join(text) - tokens = [ - token - for token in re.findall(r'"([^"]+)"', command) - if any(ord(c) > 127 for c in token) - ] - if not tokens: - continue - bracket = re.match(r"^(?:@\[[^\]]*\]\s*)?scoped\[([\w.«»]+)\]", line) - if bracket: - scope = bracket.group(1) - elif re.match(r"^scoped\s", line): - _, namespaces = file_scoped_preamble(lines, index + 1) - scope = ".".join(namespaces) - else: - scope = None - # A global notation in FormalConjecturesForMathlib or - # FormalConjecturesUtil is in force in every problem file, - # which imports both; one in a problem module is not, since - # problem files do not import each other, and the problem - # file's own notations travel with the preamble. - shared = src.name != "FormalConjectures" - commands.append((tokens, command, scope, shared)) - _NOTATION_CACHE = commands - return commands - - -NOTATION_FAMILY = re.compile(r"^(?:notation[0-9]*|postfix|prefix|infixl|infixr|infix)[:\s]") - - -def localise_notation(preamble): - """File-scope the preamble's notation commands, with their set_options. - - The generator reconstructs each workspace file's context by re-extracting - these commands from the module, so a *global* notation ends up declared - both in `ChallengeDeps` and in the file importing it — two identical - notations, and every use becomes ambiguous. `local` keeps each copy to - its own file. A standalone `set_option quotPrecheck false` does not - survive that reconstruction, so a notation that needs it gets it - attached as part of its own command. - """ - precheck_off = any( - entry.split("\n")[0].strip() == "set_option quotPrecheck false" - for entry in preamble - ) - out = [] - for entry in preamble: - if NOTATION_FAMILY.match(entry): - entry = "local " + entry - if precheck_off and re.match(r"^(?:local\s+)?(?:notation|postfix|prefix|infix)", entry): - entry = "set_option quotPrecheck false in\n" + entry - out.append(entry) - return out - - -def notation_blocks(module_texts, opened): - """The FC notation commands the module's text uses, as copyable blocks. - - A token match alone over-copies: `⊆` from a modal-logic module matched - every statement about sets. A scoped notation can only have been in - force in the source file if its namespace is among the file's opens, so - `opened` — the namespaces the module's scope and copied preambles open — - gates every scoped command. A global `notation` in a module nothing - imports was never in force either, but the corpus keeps global notation - in the problem file itself, which the preamble already carries, so - unscoped commands from other files are not candidates at all. - """ - combined = "\n".join(module_texts) - blocks, seen = [], set() - for tokens, command, scope, shared in fc_notation_commands(): - if scope: - if scope not in opened: - continue - elif not shared: - continue - if command in seen or command in combined: - continue - if not any(token in combined for token in tokens): - continue - seen.add(command) - # A plain `scoped` command needs its namespace restated around it; - # the bracket form carries its own scope. A global command becomes - # `local`: the generator re-extracts it into every file that needs - # it, and a module-crossing global would be declared twice. - if scope and not command.startswith("scoped["): - command = f"namespace {scope}\n{command}\nend {scope}" - elif not scope: - command = "local " + command - blocks.append(command) - return blocks - - -def flatten_declared_name(declared, statement): - """Restate a dotted declaration name as its slug, in the statement text. - - Returns `(new_name, new_statement)`. Only the declaring occurrence is - rewritten — a statement does not reference its own name — and the - rewrite is refused rather than guessed if the name cannot be found where - the declaration keyword put it. - """ - from leaneval_interface import slug - - flattened = slug(declared) - lines = statement.split("\n") - for index, line in enumerate(lines): - match = DECL_START.match(line) - if not match: - continue - name = re.match(r"\s*([\w.«»]+)", line[match.end() :]) - if name and name.group(1) == declared: - start = match.end() + name.start(1) - lines[index] = line[:start] + flattened + line[start + len(declared) :] - return flattened, "\n".join(lines) - raise SystemExit(f"{declared}: cannot find the declaring occurrence to rename") - - -def replace_proof_with_sorry(text): - """Cut the proof body after `:=`, keeping the statement. - - A tactic proof is found by `:= by`, which a statement cannot contain, - `by` being a keyword. A term proof leaves only a bare `:=` to cut at, and - a statement can contain one of those: a structure literal `{ a := b }` - inside the statement would be cut in half. With more than one candidate - the importer refuses, as everywhere else it cannot decide. - """ - m = re.search(r":=\s*by\b", text) - if m: - return text[: m.start()].rstrip() + " := by\n sorry" - if text.count(":=") > 1: - raise SystemExit( - "the declaration has a term-mode proof and more than one `:=`, so " - "the start of the proof cannot be read off the text" - ) - m = re.search(r":=", text) - if m: - return text[: m.start()].rstrip() + " := by\n sorry" - return text.rstrip() + " := by\n sorry" - - -def answer_spans(text): - """Return the source spans of syntactic `answer(...)` calls. - - This small lexer skips strings and nested line/block comments and balances - parentheses, so an answer term may itself contain parentheses. It is not a - Lean parser; malformed or unterminated syntax is refused. - """ - spans = [] - i = 0 - block_depth = 0 - in_string = False - escaped = False - while i < len(text): - pair = text[i : i + 2] - if block_depth: - if pair == "/-": - block_depth += 1 - i += 2 - elif pair == "-/": - block_depth -= 1 - i += 2 - else: - i += 1 - continue - if in_string: - if escaped: - escaped = False - elif text[i] == "\\": - escaped = True - elif text[i] == '"': - in_string = False - i += 1 - continue - if pair == "/-": - block_depth = 1 - i += 2 - continue - if pair == "--": - newline = text.find("\n", i + 2) - i = len(text) if newline < 0 else newline + 1 - continue - if text[i] == '"': - in_string = True - i += 1 - continue - if text.startswith("answer", i) and ( - i == 0 or not (text[i - 1].isalnum() or text[i - 1] in "_.'") - ): - j = i + len("answer") - while j < len(text) and text[j].isspace(): - j += 1 - if j < len(text) and text[j] == "(": - depth = 1 - k = j + 1 - nested_string = False - nested_escaped = False - nested_comment = 0 - while k < len(text) and depth: - nested_pair = text[k : k + 2] - if nested_comment: - if nested_pair == "/-": - nested_comment += 1 - k += 2 - elif nested_pair == "-/": - nested_comment -= 1 - k += 2 - else: - k += 1 - continue - if nested_string: - if nested_escaped: - nested_escaped = False - elif text[k] == "\\": - nested_escaped = True - elif text[k] == '"': - nested_string = False - k += 1 - continue - if nested_pair == "/-": - nested_comment = 1 - k += 2 - elif nested_pair == "--": - newline = text.find("\n", k + 2) - k = len(text) if newline < 0 else newline + 1 - elif text[k] == '"': - nested_string = True - k += 1 - else: - if text[k] == "(": - depth += 1 - elif text[k] == ")": - depth -= 1 - k += 1 - if depth: - raise SystemExit("unterminated answer(...) term") - spans.append((i, k, text[j + 1 : k - 1])) - i = k - continue - i += 1 - if block_depth or in_string: - raise SystemExit("unterminated comment or string while reading answers") - return spans - - -def unwrap_answers(statement): - """Replace any surviving `answer(t)` with `(t)`. - - `answer` is this repository's own elaborator, so a Mathlib-only workspace - cannot parse it. `hoist_answers` removes the `answer(sorry)` slots by - turning them into definition holes; a slot that already carries its answer, - which is how a `research solved` statement is written, is left behind and - used to reach the marked-up module as literal text that does not parse. - - Unwrapping is faithful. In the default `postpone` mode the elaborator - elaborates the term and attaches an annotation - (`FormalConjecturesUtil/Answer.lean`), so `answer(t)` and `t` denote the - same term and only the annotation is lost. The annotation is what marks - which part of the statement was the question, and the manifest records - that instead. - """ - for start, end, argument in reversed(answer_spans(statement)): - statement = statement[:start] + f"({argument.strip()})" + statement[end:] - return statement - - -def _ascribed_type(statement, start, end): - """The `T` of `(answer(sorry) : T)`, when the slot is written that way. - - A type ascription is the one place the surface syntax states a slot's - type at its position, and it matters because the elaborated environment - can lose the annotation for exactly this shape: the ascribed term is - applied or rewritten during elaboration and the metadata does not - survive into the stored statement type. - """ - before = statement[:start].rstrip() - if not before.endswith("("): - return None - index = end - while index < len(statement) and statement[index].isspace(): - index += 1 - if index >= len(statement) or statement[index] != ":": - return None - index += 1 - depth, cursor = 1, index - while cursor < len(statement): - char = statement[cursor] - if char == "(": - depth += 1 - elif char == ")": - depth -= 1 - if depth == 0: - ascribed = statement[index:cursor].strip() - return ascribed or None - cursor += 1 - return None - - -def hoist_answers(statement, basename, slot_types, override=None): - """Replace each `answer(sorry)` with a named definition hole. - - A slot written `(answer(sorry) : T)` states its own type at its own - position, and that reading wins. For the rest, the types come from the - elaborated environment, where the `answer` elaborator ran with the - expected type in hand; the old surface-syntax guess (an `↔` beside the - slot means `Prop`) and the FC problem file's hand-kept `answer_type` - both survive only as overrides. Unascribed slots of differing types are - refused: the environment reports the types as a set, and matching them - to positions would be a guess. - """ - holes = [] - calls = answer_spans(statement) - selected = [call for call in calls if call[2].strip() == "sorry"] - count = len(selected) - if count == 0: - return statement, holes - types = [None] * count - if override: - types = [override] * count - else: - remaining_env = list(slot_types) - for i, (start, end, _argument) in enumerate(selected): - ascribed = _ascribed_type(statement, start, end) - if ascribed is not None: - types[i] = ascribed - # The environment may have reported this slot too; retire one - # matching entry so the counting below stays honest. - if ascribed in remaining_env: - remaining_env.remove(ascribed) - remaining = [i for i in range(count) if types[i] is None] - # Under the default `alwaysTrue` setting, the `answer` elaborator - # erases a slot to `True` if and only if its expected type is `Prop` - # (FormalConjecturesUtil/Answer.lean). So a slot the environment - # carries no annotation for is a `Prop` slot by the elaborator's own - # rule, not by guesswork, and no postpone build is needed. - missing = len(remaining) - len(remaining_env) - if missing == len(remaining): - for i in remaining: - types[i] = "Prop" - elif missing == 0 and remaining and len(set(remaining_env)) == 1: - for i in remaining: - types[i] = remaining_env[0] - elif missing == 0 and not remaining: - pass - elif missing == 0: - raise SystemExit( - f"{basename} has {len(remaining)} answer slots of differing " - f"types {remaining_env}; pass --answer-type" - ) - else: - # Some slots are Prop and some are not: which positions are which - # cannot be read off an unordered set, so refuse rather than - # assign. - raise SystemExit( - f"{basename}: {missing} Prop slot(s) and {len(remaining_env)} " - f"typed slot(s) {remaining_env} cannot be matched to " - "positions; pass --answer-type" - ) - replacements = [] - for i, (start, end, _argument) in enumerate(selected): - name = f"{basename}_answer" if count == 1 else f"{basename}_answer_{i + 1}" - holes.append(DefinitionHole(name=name, type=types[i])) - replacements.append((start, end, name)) - for start, end, name in reversed(replacements): - statement = statement[:start] + name + statement[end:] - return statement, holes - - -def pins(source_path=None): - """Revisions the workspace's own build can actually fetch. - - The FC pin must be reachable from the upstream repository the lakefile - names, so it is the merge-base with `origin/main`, not HEAD: a local - branch commit would generate a workspace whose build fails at fetch time. - The importer stops if the selected source differs from that revision. - Otherwise it could combine a working-tree statement with an older imported - context. - """ - manifest = json.loads((ROOT / "lake-manifest.json").read_text()) - mathlib_rev = next(p["rev"] for p in manifest["packages"] if p["name"] == "mathlib") - merge_base = subprocess.run( - ["git", "-C", str(ROOT), "merge-base", "HEAD", "origin/main"], - capture_output=True, - text=True, - ) - if merge_base.returncode != 0 or not merge_base.stdout.strip(): - raise SystemExit("cannot resolve the Formal Conjectures source revision") - fc_rev = merge_base.stdout.strip() - if source_path is not None: - comparison = subprocess.run( - ["git", "-C", str(ROOT), "diff", "--quiet", fc_rev, "--", str(source_path)] - ) - if comparison.returncode not in (0, 1): - raise SystemExit(f"cannot compare {source_path} with {fc_rev[:12]}") - if comparison.returncode == 1: - raise SystemExit( - f"{source_path} differs from pinned revision {fc_rev[:12]}; " - "land the source on upstream main before generating" - ) - return mathlib_rev, fc_rev - - def source_record( declaration, module, source_path, fc_rev, dependencies, original, mathlib_rev ): diff --git a/comparator/adapter/fc_source.py b/comparator/adapter/fc_source.py new file mode 100644 index 0000000000..59b3f8266d --- /dev/null +++ b/comparator/adapter/fc_source.py @@ -0,0 +1,764 @@ +#!/usr/bin/env python3 +"""Reading Formal Conjectures source. + +Everything here answers questions about this repository's own Lean files: +where a declaration is, which file-scoped directives were in force where it +was written, which FC-defined notation it uses, where its `answer(sorry)` +slots are and what types the elaborated environment gives them, and which +pins the text was read at. Nothing here knows what a workspace, a request or +a manifest is; `fc_leaneval_importer.py` assembles those from these answers. +""" + + +import json +import pathlib +import re +import subprocess + +from leaneval_interface import ( + DefinitionHole, +) + +ROOT = pathlib.Path(__file__).resolve().parent.parent.parent + +SOURCE_DIRS = [ROOT / "FormalConjectures"] + +DECL_START = re.compile( + # `local notation` and `scoped notation` carry the modifier before the + # keyword. Without them here, Erdos 125's `local notation "A" => ...` typed + # as nothing and was dropped, and its statements lost the sets they name. + r"^(?:noncomputable\s+|private\s+|protected\s+|local\s+|scoped\s+)*" + r"(theorem|lemma|def|abbrev|structure|inductive|instance|notation)\s", +) + +KEEP_LOOSE = re.compile( + # `local notation`, `local macro` and friends scope to the file exactly + # like `open` does, and a statement that names what they define does not + # parse without them. `noncomputable section` is a section for the scope + # stack and a compilation mode for everything inside it. + r"^(?:(?:local|scoped)\s+)?" + r"(open|variable|universe|section|namespace|end|attribute|set_option" + r"|notation|postfix|prefix|infixl|infixr|infix|macro|syntax|macro_rules)\b" + r"|^noncomputable section\b" +) + +def elaborator_facts(module, declaration): + """What the elaborated environment knows about a declaration. + + Runs `lake exe comparator_facts`, which imports the module and reports the + declaration's source range, its binders with real explicitness, and the + inferred type of each `answer(sorry)` slot. Every one of these used to be + reconstructed from text, and each reconstruction had failure modes the + elaborator does not. + """ + proc = subprocess.run( + ["lake", "exe", "comparator_facts", module, declaration], + capture_output=True, + text=True, + cwd=ROOT, + ) + if proc.returncode != 0: + raise SystemExit( + f"comparator_facts {declaration}: " + f"{proc.stderr.strip() or proc.stdout.strip()}" + ) + out = proc.stdout + if "{" not in out: + raise SystemExit(f"comparator_facts {declaration}: no JSON in output") + return json.loads(out[out.index("{") :]) + +def file_scoped_preamble(lines, start_line): + """Directives in force at `start_line`, and the namespace stack there. + + Lean scopes `open`, `variable`, `universe`, `set_option` and notation to + the file, so the marked-up module has to restate them; nothing in the + olean records them. A directive counts only if it precedes the statement + and its scope still encloses it. + """ + stack, preamble, depth = [], [], 0 + lines = lines[: start_line - 1] + index = 0 + while index < len(lines): + line = lines[index] + if depth == 0 and KEEP_LOOSE.match(line) and not line.rstrip().endswith(" in"): + kind = line.split()[0] + if kind == "noncomputable": + # `noncomputable section [name]` opens a section. + kind = "section" + parts = line.split(None, 2) + name = parts[2].strip() if len(parts) > 2 else None + else: + parts = line.split(None, 1) + name = parts[1].strip() if len(parts) > 1 else None + if kind in ("namespace", "section"): + stack.append((kind, name, line)) + elif kind == "end": + if stack and ( + stack[-1][1] == name or (name is None and stack[-1][0] == "section") + ): + stack.pop() + else: + # A `macro` or `notation` body may continue on indented + # lines; a single kept line would be broken syntax. + text = [line] + while index + 1 < len(lines) and ( + lines[index + 1][:1].isspace() and lines[index + 1].strip() + ): + index += 1 + text.append(lines[index]) + preamble.append(("\n".join(text), list(stack))) + depth += len(re.findall(r"/-", line)) - len(re.findall(r"-/", line)) + depth = max(depth, 0) + index += 1 + scope = list(stack) + in_force = [text for text, s in preamble if s == scope[: len(s)]] + # A statement inside `noncomputable section` restates the mode, since the + # copy has left the section behind and a noncomputable definition in the + # statement's closure would otherwise fail to compile. + if any(k == "section" and line.startswith("noncomputable") for k, _, line in scope): + in_force.append("noncomputable section") + return in_force, [n for k, n, _ in scope if k == "namespace" and n] + +def docstring_reference(module_doc): + """The source citation Formal Conjectures already writes in the module. + + Module docstrings carry a `*Reference:*` line naming where the problem + comes from, sometimes with several links under it. The first is the + problem's own; later ones are commentary and proof notes. + """ + if not module_doc: + return "" + after = module_doc.split("*Reference:*", 1) + if len(after) != 2: + return "" + link = re.search(r"\]\((https?://[^)\s]+)\)", after[1]) + return link.group(1) if link else "" + +def module_name(rel_path): + """The Lean module name for a path under `FormalConjectures/`. + + Most problem files are named for a number, which is not an identifier, so + the component is written in guillemets: + `FormalConjectures.ErdosProblems.«940»`. + """ + parts = [ + c if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", c) else f"«{c}»" + for c in str(rel_path)[: -len(".lean")].split("/") + ] + return ".".join(parts) + +def _declaring_files(name): + """The files whose text declares `name` as a theorem or lemma.""" + pattern = re.compile( + rf"(?:theorem|lemma)\s+(?:[\w.«»]*\.)?{re.escape(name)}[\s:]" + ) + hits = [] + for src in SOURCE_DIRS: + for path in sorted(src.rglob("*.lean")): + if pattern.search(path.read_text(encoding="utf-8")): + hits.append(path) + return hits + +def _declares_namespaces(text, components): + """True if the file opens namespaces spelling out `components` in order. + + A single `namespace A.B` line declares both at once, so the check is on + the concatenated stack, not line by line. Text-level and approximate on + purpose — the elaborated environment settles the truth later; this only + ranks candidate files. + """ + stack = [] + for line in text.split("\n"): + m = re.match(r"\s*namespace\s+([\w.«»]+)", line) + if m: + stack.extend(m.group(1).split(".")) + return any( + stack[i : i + len(components)] == list(components) + for i in range(len(stack) - len(components) + 1) + ) + +def find_declaration(basename, module=None): + """Locate the file declaring `basename`. Returns (path, imports, doc, body). + + A fully qualified name resolves through the enclosing `namespace` stack; + `module` names the file when more than one declares the name, and comes + from the problem's FC problem file. + """ + if module is not None: + named = ROOT / module + if not named.exists(): + raise SystemExit(f"manifest names {module}, which does not exist") + return _read_source(named) + hits = _declaring_files(basename) + if not hits and "." in basename: + # A fully qualified request such as `OeisA303656.conjecture` names a + # declaration whose file spells only `conjecture`, the prefix coming + # from an enclosing `namespace`. Try each split of the request into + # (namespace prefix, declared suffix), keeping files that declare the + # suffix inside that namespace. Splits are tried longest-suffix first, + # because a declared name may itself contain dots + # (`erdos_125.variants.positive_unequal_density`). + parts = basename.split(".") + for cut in range(1, len(parts)): + prefix, suffix = parts[:cut], ".".join(parts[cut:]) + hits = [ + path + for path in _declaring_files(suffix) + if _declares_namespaces(path.read_text(encoding="utf-8"), prefix) + ] + if hits: + break + if not hits: + raise SystemExit( + f"no declaration named {basename!r} found under FormalConjectures/" + ) + if len(hits) > 1: + raise SystemExit( + f"{basename!r} is ambiguous: " + + ", ".join(str(h.relative_to(ROOT)) for h in hits) + + "; pass --module to choose one, or record the choice in " + "comparator/problems/.toml" + ) + return _read_source(hits[0]) + +def _read_source(path): + """Return (path, imports, module docstring, body after the licence header). + + The docstring is read but not removed. It sits below the imports rather + than at the top, so it is found by searching; the body deliberately still + contains it, because `strip_decorations` removes docstrings per + declaration and the generated files are compared byte for byte. + """ + original = path.read_text(encoding="utf-8") + text = re.sub(r"\A/-.*?-/\s*", "", original, flags=re.DOTALL) + found = re.search(r"/-!.*?-/", text, flags=re.DOTALL) + doc = found.group(0) if found else "" + imports = re.findall(r"^import\s+(\S+)", original, re.MULTILINE) + return path, imports, doc, text + +def strip_decorations(block_text): + """Remove the docstring, line comments and attributes from a declaration. + + These interleave. Erdos 918 puts a `--` formalisation note between its + docstring and its `@[category ...]` line, and one anchored pass each left + the attribute in place. `@[category research open, AMS 5]` then reached the + marked-up module, where the workspace has no such attribute, and Lean + parsed as far as the `open` inside it before giving up. + """ + # `open X in` binds to the declaration and has to survive, but it sits + # above the docstring, so stripping anchored at the start would stop dead + # on it. + prefix = "" + m = re.match(r"\A\s*(open\b[^\n]*\bin)\n", block_text) + if m: + prefix = m.group(1) + "\n" + block_text = block_text[m.end() :] + while True: + stripped = re.sub(r"\A\s*/--.*?-/\s*", "", block_text, flags=re.DOTALL) + stripped = re.sub(r"\A\s*--[^\n]*\n", "", stripped) + stripped = re.sub(r"\A\s*@\[[^\]]*\]\s*", "", stripped, flags=re.DOTALL) + if stripped == block_text: + return prefix + stripped + block_text = stripped + +# Attributes this repository defines. A generated workspace requires Mathlib +# and nothing else, so these have to go; everything else has to stay. +FC_ATTRIBUTES = ("category", "AMS", "formal_proof") + +def strip_fc_attributes(block_text): + """Remove this repository's own attributes from a copied declaration. + + Unlike `strip_decorations`, which clears every attribute off the target + statement, this keeps the rest. A dependency is copied to be elaborated, + not restated, and dropping `simp`, `reducible` or `instance` attributes + changes how the declarations after it in the same closure elaborate. + """ + + def replace(match): + inner = match.group(1) + # Nested brackets mean an argument this simple split would cut in + # half, so leave the whole attribute alone rather than mangle it. + if "[" in inner: + return match.group(0) + kept = [ + part.strip() + for part in inner.split(",") + if part.strip() and part.strip().split()[0] not in FC_ATTRIBUTES + ] + return f"@[{', '.join(kept)}]" if kept else "" + + text = re.sub(r"@\[([^\]]*)\]", replace, block_text) + # An attribute line that emptied out leaves a blank line behind. + return re.sub(r"^[ \t]*\n", "", text, flags=re.MULTILINE) + +def split_module(module): + """The components of a dotted module name, respecting guillemet quoting. + + A guillemet-quoted component may itself contain dots — + `FormalConjectures.Arxiv.«0912.2382».CurlingNumberConjecture` names the + directory `0912.2382` — so splitting on every dot decodes a path that + does not exist. This is the one place a module name is taken apart; + `module_name` is its inverse and a test holds the pair to that. + """ + parts = re.findall(r"«[^»]*»|[^.«»]+", module) + if ".".join(parts) != module: + raise SystemExit(f"{module!r} is not a well-formed module name") + return [p[1:-1] if p.startswith("«") else p for p in parts] + +def module_source_path(module): + """The file declaring a dotted Lean module name, undoing guillemets.""" + parts = split_module(module) + # Not `with_suffix`: a final component containing a dot would lose its + # tail to the suffix replacement. + path = ROOT.joinpath(*parts[:-1], parts[-1] + ".lean") + if not path.is_file(): + raise SystemExit(f"{module}: no source file at {path}") + return path + +def slice_range(lines, source_range): + """The source text a declaration range covers, and the line it starts on. + + `open X in` binds to the declaration below it but sits above what the + range covers in some toolchains, so it is pulled in when present. + """ + lo, hi = source_range["startLine"], source_range["endLine"] + end_column = source_range.get("endColumn") + while ( + lo > 1 + and lines[lo - 2].rstrip().endswith(" in") + and KEEP_LOOSE.match(lines[lo - 2]) + ): + lo -= 1 + sliced = lines[lo - 1 : hi] + if end_column is not None and sliced: + sliced = sliced[:-1] + [sliced[-1][:end_column]] + return "\n".join(sliced), lo + +NOTATION_COMMAND = re.compile( + r"^(?:@\[[^\]]*\]\s*)?(?:scoped\[[\w.«»]+\]\s+)?(?:scoped\s+)?" + r"(?:notation[0-9]*|postfix|prefix|infixl|infixr|infix)[:\s]" +) + +_NOTATION_CACHE = None + +def fc_notation_commands(): + """Every exportable notation command an FC module defines, with its token. + + A notation is not a constant, so the elaborated closure never reports it: + a statement written as `ℝ²` names `EuclideanSpace ℝ (Fin 2)` in the + environment and `ℝ²` only in its text. The copy carries the text, so the + commands that make such tokens parse have to be found at the text layer. + `local` notations are file-scoped at their origin and cannot be in force + in a problem file, so they are not candidates. + + Returns `[(tokens, command, namespaces)]`, where `tokens` are the + command's string literals that contain a non-ASCII character — the + distinctive ones worth matching on — and `namespaces` is the stack a + plain `scoped` command needs restated around it. + """ + global _NOTATION_CACHE + if _NOTATION_CACHE is not None: + return _NOTATION_CACHE + commands = [] + roots = [ROOT / "FormalConjecturesForMathlib", ROOT / "FormalConjecturesUtil"] + for src in roots + SOURCE_DIRS: + for path in sorted(src.rglob("*.lean")): + lines = path.read_text(encoding="utf-8").split("\n") + for index, line in enumerate(lines): + if not NOTATION_COMMAND.match(line): + continue + text = [line] + follow = index + 1 + while follow < len(lines) and ( + lines[follow][:1].isspace() and lines[follow].strip() + ): + text.append(lines[follow]) + follow += 1 + command = "\n".join(text) + tokens = [ + token + for token in re.findall(r'"([^"]+)"', command) + if any(ord(c) > 127 for c in token) + ] + if not tokens: + continue + bracket = re.match(r"^(?:@\[[^\]]*\]\s*)?scoped\[([\w.«»]+)\]", line) + if bracket: + scope = bracket.group(1) + elif re.match(r"^scoped\s", line): + _, namespaces = file_scoped_preamble(lines, index + 1) + scope = ".".join(namespaces) + else: + scope = None + # A global notation in FormalConjecturesForMathlib or + # FormalConjecturesUtil is in force in every problem file, + # which imports both; one in a problem module is not, since + # problem files do not import each other, and the problem + # file's own notations travel with the preamble. + shared = src.name != "FormalConjectures" + commands.append((tokens, command, scope, shared)) + _NOTATION_CACHE = commands + return commands + +NOTATION_FAMILY = re.compile(r"^(?:notation[0-9]*|postfix|prefix|infixl|infixr|infix)[:\s]") + +def localise_notation(preamble): + """File-scope the preamble's notation commands, with their set_options. + + The generator reconstructs each workspace file's context by re-extracting + these commands from the module, so a *global* notation ends up declared + both in `ChallengeDeps` and in the file importing it — two identical + notations, and every use becomes ambiguous. `local` keeps each copy to + its own file. A standalone `set_option quotPrecheck false` does not + survive that reconstruction, so a notation that needs it gets it + attached as part of its own command. + """ + precheck_off = any( + entry.split("\n")[0].strip() == "set_option quotPrecheck false" + for entry in preamble + ) + out = [] + for entry in preamble: + if NOTATION_FAMILY.match(entry): + entry = "local " + entry + if precheck_off and re.match(r"^(?:local\s+)?(?:notation|postfix|prefix|infix)", entry): + entry = "set_option quotPrecheck false in\n" + entry + out.append(entry) + return out + +def notation_blocks(module_texts, opened): + """The FC notation commands the module's text uses, as copyable blocks. + + A token match alone over-copies: `⊆` from a modal-logic module matched + every statement about sets. A scoped notation can only have been in + force in the source file if its namespace is among the file's opens, so + `opened` — the namespaces the module's scope and copied preambles open — + gates every scoped command. A global `notation` in a module nothing + imports was never in force either, but the corpus keeps global notation + in the problem file itself, which the preamble already carries, so + unscoped commands from other files are not candidates at all. + """ + combined = "\n".join(module_texts) + blocks, seen = [], set() + for tokens, command, scope, shared in fc_notation_commands(): + if scope: + if scope not in opened: + continue + elif not shared: + continue + if command in seen or command in combined: + continue + if not any(token in combined for token in tokens): + continue + seen.add(command) + # A plain `scoped` command needs its namespace restated around it; + # the bracket form carries its own scope. A global command becomes + # `local`: the generator re-extracts it into every file that needs + # it, and a module-crossing global would be declared twice. + if scope and not command.startswith("scoped["): + command = f"namespace {scope}\n{command}\nend {scope}" + elif not scope: + command = "local " + command + blocks.append(command) + return blocks + +def flatten_declared_name(declared, statement): + """Restate a dotted declaration name as its slug, in the statement text. + + Returns `(new_name, new_statement)`. Only the declaring occurrence is + rewritten — a statement does not reference its own name — and the + rewrite is refused rather than guessed if the name cannot be found where + the declaration keyword put it. + """ + from leaneval_interface import slug + + flattened = slug(declared) + lines = statement.split("\n") + for index, line in enumerate(lines): + match = DECL_START.match(line) + if not match: + continue + name = re.match(r"\s*([\w.«»]+)", line[match.end() :]) + if name and name.group(1) == declared: + start = match.end() + name.start(1) + lines[index] = line[:start] + flattened + line[start + len(declared) :] + return flattened, "\n".join(lines) + raise SystemExit(f"{declared}: cannot find the declaring occurrence to rename") + +def replace_proof_with_sorry(text): + """Cut the proof body after `:=`, keeping the statement. + + A tactic proof is found by `:= by`, which a statement cannot contain, + `by` being a keyword. A term proof leaves only a bare `:=` to cut at, and + a statement can contain one of those: a structure literal `{ a := b }` + inside the statement would be cut in half. With more than one candidate + the importer refuses, as everywhere else it cannot decide. + """ + m = re.search(r":=\s*by\b", text) + if m: + return text[: m.start()].rstrip() + " := by\n sorry" + if text.count(":=") > 1: + raise SystemExit( + "the declaration has a term-mode proof and more than one `:=`, so " + "the start of the proof cannot be read off the text" + ) + m = re.search(r":=", text) + if m: + return text[: m.start()].rstrip() + " := by\n sorry" + return text.rstrip() + " := by\n sorry" + +def answer_spans(text): + """Return the source spans of syntactic `answer(...)` calls. + + This small lexer skips strings and nested line/block comments and balances + parentheses, so an answer term may itself contain parentheses. It is not a + Lean parser; malformed or unterminated syntax is refused. + """ + spans = [] + i = 0 + block_depth = 0 + in_string = False + escaped = False + while i < len(text): + pair = text[i : i + 2] + if block_depth: + if pair == "/-": + block_depth += 1 + i += 2 + elif pair == "-/": + block_depth -= 1 + i += 2 + else: + i += 1 + continue + if in_string: + if escaped: + escaped = False + elif text[i] == "\\": + escaped = True + elif text[i] == '"': + in_string = False + i += 1 + continue + if pair == "/-": + block_depth = 1 + i += 2 + continue + if pair == "--": + newline = text.find("\n", i + 2) + i = len(text) if newline < 0 else newline + 1 + continue + if text[i] == '"': + in_string = True + i += 1 + continue + if text.startswith("answer", i) and ( + i == 0 or not (text[i - 1].isalnum() or text[i - 1] in "_.'") + ): + j = i + len("answer") + while j < len(text) and text[j].isspace(): + j += 1 + if j < len(text) and text[j] == "(": + depth = 1 + k = j + 1 + nested_string = False + nested_escaped = False + nested_comment = 0 + while k < len(text) and depth: + nested_pair = text[k : k + 2] + if nested_comment: + if nested_pair == "/-": + nested_comment += 1 + k += 2 + elif nested_pair == "-/": + nested_comment -= 1 + k += 2 + else: + k += 1 + continue + if nested_string: + if nested_escaped: + nested_escaped = False + elif text[k] == "\\": + nested_escaped = True + elif text[k] == '"': + nested_string = False + k += 1 + continue + if nested_pair == "/-": + nested_comment = 1 + k += 2 + elif nested_pair == "--": + newline = text.find("\n", k + 2) + k = len(text) if newline < 0 else newline + 1 + elif text[k] == '"': + nested_string = True + k += 1 + else: + if text[k] == "(": + depth += 1 + elif text[k] == ")": + depth -= 1 + k += 1 + if depth: + raise SystemExit("unterminated answer(...) term") + spans.append((i, k, text[j + 1 : k - 1])) + i = k + continue + i += 1 + if block_depth or in_string: + raise SystemExit("unterminated comment or string while reading answers") + return spans + +def unwrap_answers(statement): + """Replace any surviving `answer(t)` with `(t)`. + + `answer` is this repository's own elaborator, so a Mathlib-only workspace + cannot parse it. `hoist_answers` removes the `answer(sorry)` slots by + turning them into definition holes; a slot that already carries its answer, + which is how a `research solved` statement is written, is left behind and + used to reach the marked-up module as literal text that does not parse. + + Unwrapping is faithful. In the default `postpone` mode the elaborator + elaborates the term and attaches an annotation + (`FormalConjecturesUtil/Answer.lean`), so `answer(t)` and `t` denote the + same term and only the annotation is lost. The annotation is what marks + which part of the statement was the question, and the manifest records + that instead. + """ + for start, end, argument in reversed(answer_spans(statement)): + statement = statement[:start] + f"({argument.strip()})" + statement[end:] + return statement + +def _ascribed_type(statement, start, end): + """The `T` of `(answer(sorry) : T)`, when the slot is written that way. + + A type ascription is the one place the surface syntax states a slot's + type at its position, and it matters because the elaborated environment + can lose the annotation for exactly this shape: the ascribed term is + applied or rewritten during elaboration and the metadata does not + survive into the stored statement type. + """ + before = statement[:start].rstrip() + if not before.endswith("("): + return None + index = end + while index < len(statement) and statement[index].isspace(): + index += 1 + if index >= len(statement) or statement[index] != ":": + return None + index += 1 + depth, cursor = 1, index + while cursor < len(statement): + char = statement[cursor] + if char == "(": + depth += 1 + elif char == ")": + depth -= 1 + if depth == 0: + ascribed = statement[index:cursor].strip() + return ascribed or None + cursor += 1 + return None + +def hoist_answers(statement, basename, slot_types, override=None): + """Replace each `answer(sorry)` with a named definition hole. + + A slot written `(answer(sorry) : T)` states its own type at its own + position, and that reading wins. For the rest, the types come from the + elaborated environment, where the `answer` elaborator ran with the + expected type in hand; the old surface-syntax guess (an `↔` beside the + slot means `Prop`) and the FC problem file's hand-kept `answer_type` + both survive only as overrides. Unascribed slots of differing types are + refused: the environment reports the types as a set, and matching them + to positions would be a guess. + """ + holes = [] + calls = answer_spans(statement) + selected = [call for call in calls if call[2].strip() == "sorry"] + count = len(selected) + if count == 0: + return statement, holes + types = [None] * count + if override: + types = [override] * count + else: + remaining_env = list(slot_types) + for i, (start, end, _argument) in enumerate(selected): + ascribed = _ascribed_type(statement, start, end) + if ascribed is not None: + types[i] = ascribed + # The environment may have reported this slot too; retire one + # matching entry so the counting below stays honest. + if ascribed in remaining_env: + remaining_env.remove(ascribed) + remaining = [i for i in range(count) if types[i] is None] + # Under the default `alwaysTrue` setting, the `answer` elaborator + # erases a slot to `True` if and only if its expected type is `Prop` + # (FormalConjecturesUtil/Answer.lean). So a slot the environment + # carries no annotation for is a `Prop` slot by the elaborator's own + # rule, not by guesswork, and no postpone build is needed. + missing = len(remaining) - len(remaining_env) + if missing == len(remaining): + for i in remaining: + types[i] = "Prop" + elif missing == 0 and remaining and len(set(remaining_env)) == 1: + for i in remaining: + types[i] = remaining_env[0] + elif missing == 0 and not remaining: + pass + elif missing == 0: + raise SystemExit( + f"{basename} has {len(remaining)} answer slots of differing " + f"types {remaining_env}; pass --answer-type" + ) + else: + # Some slots are Prop and some are not: which positions are which + # cannot be read off an unordered set, so refuse rather than + # assign. + raise SystemExit( + f"{basename}: {missing} Prop slot(s) and {len(remaining_env)} " + f"typed slot(s) {remaining_env} cannot be matched to " + "positions; pass --answer-type" + ) + replacements = [] + for i, (start, end, _argument) in enumerate(selected): + name = f"{basename}_answer" if count == 1 else f"{basename}_answer_{i + 1}" + holes.append(DefinitionHole(name=name, type=types[i])) + replacements.append((start, end, name)) + for start, end, name in reversed(replacements): + statement = statement[:start] + name + statement[end:] + return statement, holes + +def pins(source_path=None): + """Revisions the workspace's own build can actually fetch. + + The FC pin must be reachable from the upstream repository the lakefile + names, so it is the merge-base with `origin/main`, not HEAD: a local + branch commit would generate a workspace whose build fails at fetch time. + The importer stops if the selected source differs from that revision. + Otherwise it could combine a working-tree statement with an older imported + context. + """ + manifest = json.loads((ROOT / "lake-manifest.json").read_text()) + mathlib_rev = next(p["rev"] for p in manifest["packages"] if p["name"] == "mathlib") + merge_base = subprocess.run( + ["git", "-C", str(ROOT), "merge-base", "HEAD", "origin/main"], + capture_output=True, + text=True, + ) + if merge_base.returncode != 0 or not merge_base.stdout.strip(): + raise SystemExit("cannot resolve the Formal Conjectures source revision") + fc_rev = merge_base.stdout.strip() + if source_path is not None: + comparison = subprocess.run( + ["git", "-C", str(ROOT), "diff", "--quiet", fc_rev, "--", str(source_path)] + ) + if comparison.returncode not in (0, 1): + raise SystemExit(f"cannot compare {source_path} with {fc_rev[:12]}") + if comparison.returncode == 1: + raise SystemExit( + f"{source_path} differs from pinned revision {fc_rev[:12]}; " + "land the source on upstream main before generating" + ) + return mathlib_rev, fc_rev diff --git a/comparator/adapter/test_fc_leaneval_importer.py b/comparator/adapter/test_fc_leaneval_importer.py index 37f8730f9e..f709387fda 100644 --- a/comparator/adapter/test_fc_leaneval_importer.py +++ b/comparator/adapter/test_fc_leaneval_importer.py @@ -28,14 +28,14 @@ from unittest import mock import fc_leaneval_importer as importer +import fc_source from leaneval_interface import MarkedUpModule, problem_group from test_leaneval_interface import a_manifest -from fc_leaneval_importer import ( +from fc_leaneval_importer import closure_region, load_manifest +from fc_source import ( answer_spans, - closure_region, file_scoped_preamble, hoist_answers, - load_manifest, pins, replace_proof_with_sorry, strip_decorations, @@ -485,21 +485,21 @@ def test_the_first_reference_link_is_the_citation(self): " - [Tao25] a blog post (https://example.com/other)\n-/" ) self.assertEqual( - importer.docstring_reference(doc), "https://www.erdosproblems.com/1038" + fc_source.docstring_reference(doc), "https://www.erdosproblems.com/1038" ) def test_an_arxiv_version_suffix_is_preserved(self): doc = "/-!\n*Reference:* [arxiv/2504.17644v3](https://arxiv.org/abs/2504.17644v3)\n-/" self.assertEqual( - importer.docstring_reference(doc), "https://arxiv.org/abs/2504.17644v3" + fc_source.docstring_reference(doc), "https://arxiv.org/abs/2504.17644v3" ) def test_links_above_the_reference_line_are_not_the_citation(self): doc = "/-!\n# A problem\n\nSee [Mathlib](https://leanprover-community.github.io).\n-/" - self.assertEqual(importer.docstring_reference(doc), "") + self.assertEqual(fc_source.docstring_reference(doc), "") def test_a_module_without_a_docstring_has_no_citation(self): - self.assertEqual(importer.docstring_reference(""), "") + self.assertEqual(fc_source.docstring_reference(""), "") class ModuleNameCodecTest(unittest.TestCase): @@ -507,7 +507,7 @@ class ModuleNameCodecTest(unittest.TestCase): def test_a_guillemet_component_keeps_its_dots(self): self.assertEqual( - importer.split_module( + fc_source.split_module( "FormalConjectures.Arxiv.«0912.2382».CurlingNumberConjecture" ), ["FormalConjectures", "Arxiv", "0912.2382", "CurlingNumberConjecture"], @@ -515,7 +515,7 @@ def test_a_guillemet_component_keeps_its_dots(self): def test_a_dotted_final_component_keeps_its_tail(self): # `with_suffix` would have turned `«2501.03234»` into `«2501.lean`. - path = importer.module_source_path( + path = fc_source.module_source_path( "FormalConjectures.Arxiv.«2501.03234».ArithmeticSumS" ) self.assertEqual(path.name, "ArithmeticSumS.lean") @@ -523,17 +523,17 @@ def test_a_dotted_final_component_keeps_its_tail(self): def test_a_malformed_name_is_refused(self): with self.assertRaises(SystemExit): - importer.split_module("FormalConjectures.«unterminated") + fc_source.split_module("FormalConjectures.«unterminated") def test_every_real_module_round_trips(self): # The property that keeps the codec from drifting again: for every # file the importer can name, decoding the name reaches the file. - for src in importer.SOURCE_DIRS: + for src in fc_source.SOURCE_DIRS: for path in src.rglob("*.lean"): rel = path.relative_to(importer.ROOT) with self.subTest(module=str(rel)): self.assertEqual( - importer.module_source_path(importer.module_name(rel)), path + fc_source.module_source_path(fc_source.module_name(rel)), path ) @@ -542,7 +542,7 @@ class QualifiedResolutionTest(unittest.TestCase): def test_the_bare_colliding_name_is_ambiguous(self): with self.assertRaises(SystemExit) as ctx: - importer.find_declaration("conjecture") + fc_source.find_declaration("conjecture") self.assertIn("ambiguous", str(ctx.exception)) def test_each_qualified_name_reaches_its_own_file(self): @@ -551,12 +551,12 @@ def test_each_qualified_name_reaches_its_own_file(self): ("OeisA308734.conjecture", "308734.lean"), ): with self.subTest(qualified=qualified): - path, _, _, _ = importer.find_declaration(qualified) + path, _, _, _ = fc_source.find_declaration(qualified) self.assertEqual(path.name, filename) def test_a_declared_name_with_dots_still_resolves(self): # The declared name itself contains dots; no namespace split applies. - path, _, _, _ = importer.find_declaration( + path, _, _, _ = fc_source.find_declaration( "erdos_125.variants.positive_unequal_density" ) self.assertEqual(path.name, "125.lean") @@ -564,7 +564,7 @@ def test_a_declared_name_with_dots_still_resolves(self): def test_longest_declared_suffix_wins(self): # `Erdos125.erdos_125.variants.positive_unequal_density`: the first # component is the namespace, the rest is the declared name. - path, _, _, _ = importer.find_declaration( + path, _, _, _ = fc_source.find_declaration( "Erdos125.erdos_125.variants.positive_unequal_density" ) self.assertEqual(path.name, "125.lean") @@ -604,7 +604,7 @@ class FlattenDeclaredNameTest(unittest.TestCase): """Dotted declaration names are restated as slugs for the generator.""" def test_the_declaring_occurrence_is_renamed(self): - name, statement = importer.flatten_declared_name( + name, statement = fc_source.flatten_declared_name( "erdos_100.variants.strong", "theorem erdos_100.variants.strong : True := by\n sorry", ) @@ -616,7 +616,7 @@ def test_the_declaring_occurrence_is_renamed(self): def test_a_prefix_line_does_not_confuse_the_rename(self): # `open X in` binds to the declaration below and travels with the # slice; the declaring line is not the first line. - name, statement = importer.flatten_declared_name( + name, statement = fc_source.flatten_declared_name( "a.b", "open Nat in\ntheorem a.b : True := by\n sorry" ) self.assertEqual(name, "a_b") @@ -624,7 +624,7 @@ def test_a_prefix_line_does_not_confuse_the_rename(self): def test_an_absent_declaration_is_refused(self): with self.assertRaises(SystemExit): - importer.flatten_declared_name("a.b", "theorem c.d : True := sorry") + fc_source.flatten_declared_name("a.b", "theorem c.d : True := sorry") class PreambleNotationTest(unittest.TestCase): @@ -692,7 +692,7 @@ class NotationBlocksTest(unittest.TestCase): def _with_commands(self, commands): return mock.patch.object( - importer, "fc_notation_commands", return_value=commands + fc_source, "fc_notation_commands", return_value=commands ) def test_a_scoped_notation_needs_its_namespace_opened(self): @@ -701,12 +701,12 @@ def test_a_scoped_notation_needs_its_namespace_opened(self): ] with self._with_commands(commands): self.assertEqual( - importer.notation_blocks(["def f : ℝ² := sorry"], {"EuclideanGeometry"}), + fc_source.notation_blocks(["def f : ℝ² := sorry"], {"EuclideanGeometry"}), ['scoped[EuclideanGeometry] notation "ℝ²" => E'], ) # Green9's `⊆` false positive: same token, namespace never opened. self.assertEqual( - importer.notation_blocks(["def f : ℝ² := sorry"], set()), [] + fc_source.notation_blocks(["def f : ℝ² := sorry"], set()), [] ) def test_a_shared_global_notation_is_copied_as_local(self): @@ -715,7 +715,7 @@ def test_a_shared_global_notation_is_copied_as_local(self): commands = [(["≪"], 'notation g " ≪ " f => IsBigO g f', None, True)] with self._with_commands(commands): self.assertEqual( - importer.notation_blocks(["theorem t : a ≪ b := sorry"], set()), + fc_source.notation_blocks(["theorem t : a ≪ b := sorry"], set()), ['local notation g " ≪ " f => IsBigO g f'], ) @@ -723,24 +723,24 @@ def test_a_problem_module_global_notation_is_never_copied(self): commands = [(["≪"], 'notation g " ≪ " f => X g f', None, False)] with self._with_commands(commands): self.assertEqual( - importer.notation_blocks(["theorem t : a ≪ b := sorry"], set()), [] + fc_source.notation_blocks(["theorem t : a ≪ b := sorry"], set()), [] ) def test_an_unused_token_is_not_copied(self): commands = [(["ℝ²"], 'notation "ℝ²" => E', None, True)] with self._with_commands(commands): - self.assertEqual(importer.notation_blocks(["theorem t : True"], set()), []) + self.assertEqual(fc_source.notation_blocks(["theorem t : True"], set()), []) class LocaliseNotationTest(unittest.TestCase): def test_a_global_notation_becomes_local(self): self.assertEqual( - importer.localise_notation(['notation "R(" k ")" => f k']), + fc_source.localise_notation(['notation "R(" k ")" => f k']), ['local notation "R(" k ")" => f k'], ) def test_quot_precheck_travels_with_the_notation(self): - out = importer.localise_notation( + out = fc_source.localise_notation( ["set_option quotPrecheck false", 'local notation "A" => s'] ) self.assertEqual( @@ -750,7 +750,7 @@ def test_quot_precheck_travels_with_the_notation(self): def test_other_preamble_lines_pass_through(self): self.assertEqual( - importer.localise_notation(["open Nat", "variable (n : Nat)"]), + fc_source.localise_notation(["open Nat", "variable (n : Nat)"]), ["open Nat", "variable (n : Nat)"], ) diff --git a/lakefile.toml b/lakefile.toml index 16bd4bdc2b..0d40865783 100644 --- a/lakefile.toml +++ b/lakefile.toml @@ -77,6 +77,10 @@ weak.linter.style.imports = true weak.google.answer = "postpone" +[[lean_lib]] +name = "ComparatorFacts" +srcDir = "comparator/adapter" + [[lean_exe]] name = "comparator_facts" srcDir = "comparator/adapter" From 58c3851463a9a95a7b72b4802ae120fb945331b6 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Fri, 21 Aug 2026 20:42:42 -0400 Subject: [PATCH 41/70] Stop counting arrows inside binder-notation bodies as parameters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `conclusionBinders` recognises a leading `∀` by the character after it, so `∀ᶠ n in atTop, p n → q n` was not a forall to it. The text then reached the arrow scan, which counted the two top-level arrows inside the `∀ᶠ` body as Pi binders. The elaborated type is `Filter.Eventually (fun n => …)`, an application with an empty telescope, and alignment failed for erdos_100.variants.strong: "source conclusion has 2 binders, but the elaborated type has only 0". An arrow that follows a top-level comma belongs to the body of a binder notation that is not a Pi (`∀ᶠ`, `∃!`, `∀ᵐ`, …), so the scan now stops there. The self-test covers `∀ᶠ`, `∃!`, and an arrow before an `∀ᶠ`. --- .../adapter/ComparatorFacts/Binders.lean | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/comparator/adapter/ComparatorFacts/Binders.lean b/comparator/adapter/ComparatorFacts/Binders.lean index 017759f179..c73753382c 100644 --- a/comparator/adapter/ComparatorFacts/Binders.lean +++ b/comparator/adapter/ComparatorFacts/Binders.lean @@ -290,8 +290,16 @@ partial def conclusionBinders (env : Environment) (text : String) : Except Strin c.isWhitespace || c == '(' || c == '{' || c == '[' || c == '⦃' if unicodeExists || asciiExists then return #[] + -- A top-level comma before the first arrow means the arrow sits inside the + -- body of a binder notation that is not a Pi (`∀ᶠ n in l, p n → q n`, + -- `∃! x, p x → q x`, `∀ᵐ x ∂μ, …`): the elaborated type is an application + -- there, and its telescope is empty. + let commaBeforeArrow : Bool := match findTopLevelToken .arrow text, findTopLevelToken .comma text with + | some (arrow, _), some (comma, _) => comma < arrow + | _, _ => false match findTopLevelToken .arrow text, findTopLevelToken .iff text with | some (arrow, width), none => + if commaBeforeArrow then return #[] let rest := sliceChars text (arrow + width) text.length return #[{ name? := none, info := .default }] ++ (← conclusionBinders env rest) | _, _ => return #[] @@ -410,7 +418,15 @@ def binderBoundarySelfTest (env : Environment) : IO UInt32 := do "∃ f : Nat → Nat, ∀ n, True → f n = f n", #[], 0), ("theorem t : True → (∀ n : Nat, 1 < n → True) := by simp", "True → (∀ n : Nat, 1 < n → True)", - #[mkLocal 0 `h₁ .default, mkLocal 1 `n .default, mkLocal 2 `h₂ .default], 0) + #[mkLocal 0 `h₁ .default, mkLocal 1 `n .default, mkLocal 2 `h₂ .default], 0), + -- Binder notations that elaborate to applications, not Pis: the arrows in + -- their bodies are not declaration parameters (erdos_100.variants.strong). + ("theorem t : ∀ᶠ n in Filter.atTop, 1 < n → True := by simp", + "∀ᶠ n in Filter.atTop, 1 < n → True", #[], 0), + ("theorem t : ∃! n : Nat, 1 < n → True := by simp", + "∃! n : Nat, 1 < n → True", #[], 0), + ("theorem t : True → ∀ᶠ n in Filter.atTop, 1 < n → True := by simp", + "True → ∀ᶠ n in Filter.atTop, 1 < n → True", #[mkLocal 0 `h .default], 0) ] for (source, resultType, elaborated, expected) in alignmentCases do let result := do From 2654e42de2026de6cdb248ad5ed0f1c7d659c8fa Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Fri, 21 Aug 2026 23:54:03 -0400 Subject: [PATCH 42/70] Name the generator contract by its schema version lean-eval reserves unqualified "v1" and "v2" for problem sets. The wire format this adapter speaks is "generator schema version 1" (a "schema-version-1 request"), and the hypothetical successor is "a future generator contract revision". Frozen identifiers such as request-v1.schema.json keep their names. --- comparator/OWNERSHIP.md | 21 ++++++++++--------- comparator/README.md | 2 +- comparator/adapter/leaneval_generator_cli.py | 4 ++-- comparator/adapter/leaneval_interface.py | 12 +++++------ .../adapter/make_comparator_workspace.py | 10 ++++----- comparator/adapter/test_leaneval_interface.py | 2 +- .../adapter/test_make_comparator_workspace.py | 2 +- 7 files changed, 27 insertions(+), 26 deletions(-) diff --git a/comparator/OWNERSHIP.md b/comparator/OWNERSHIP.md index 721f4aeec4..1aafaabd53 100644 --- a/comparator/OWNERSHIP.md +++ b/comparator/OWNERSHIP.md @@ -10,7 +10,7 @@ and scope fidelity work from deterministic Lean CLI with a frozen, versioned JSON contract, consumed at the exact revision `comparator/tools.toml` pins under `[generator]`. **The Formal Conjectures importer does not fork the generation logic.** It maps FC -declarations and metadata to a v1 request, and records the FC source commit +declarations and metadata to a schema-version-1 request, and records the FC source commit and declaration id for every problem. ## The seam @@ -40,7 +40,7 @@ revision is normative). Per problem it carries: | `group` | for a frozen-set import, the set itself: the list is immutable while its members keep getting solved, so every member stays in the open-conjectures display and the category rides along as a tag. For a single import, the declaration's `@[category ...]` tag decides; a declaration that is not a problem is refused either way | | `leanToolchain`, `mathlib` | LeanEval's pins, from `[target]` in `tools.toml` — the consumer's, never this repository's | | `templates.workspaceTest` | `comparator/templates/WorkspaceTest.lean`, which stays FC-supplied: the contract requires the consumer to provide it | -| `contextRoot` | a directory this side materialises: the module file the generator byte-checks against `moduleContent`, and a synthesised `.ilean` carrying the spans above, because v1 still resolves declaration spans from compiled metadata | +| `contextRoot` | a directory this side materialises: the module file the generator byte-checks against `moduleContent`, and a synthesised `.ilean` carrying the spans above, because generator schema version 1 still resolves declaration spans from compiled metadata | The module carries no markers of any kind. `@[eval_problem]` does not exist outside lean-eval, so a module carrying it could not elaborate under @@ -58,10 +58,10 @@ file, and every digest is checked before a byte lands on disk. ### Provenance rides beside the request, not in it lean-eval#536 requires each imported problem to record the FC source commit -and declaration id. The v1 wire format has no field for either — its optional +and declaration id. The schema-version-1 wire format has no field for either — its optional `source` is one free-text line — and lean-eval keeps that format frozen on -purpose, so the sidecar is the v1 provenance boundary **by design**, not a -stopgap (kim-em on #4951, 2026-08-21); a typed provenance object is a v2 +purpose, so the sidecar is the schema-version-1 provenance boundary **by design**, not a +stopgap (kim-em on #4951, 2026-08-21); a typed provenance object is a future-contract-revision matter and does not gate the FC100 import. The manifest this repository builds (`ProblemManifest`: commit, path, blob, module, declaration, copied dependencies, the pins the hole types were read at) is written beside the @@ -85,7 +85,7 @@ Conjectures corrects a misformalisation upstream. | `comparator/adapter/fc_source.py` | reads this repository's own Lean: where a declaration is, the file-scoped directives in force where it was written, the FC-defined notation it uses, its `answer(sorry)` slots and their elaborated types, and the pins the text was read at | | `comparator/adapter/fc_leaneval_importer.py` | assembles the marked-up module and the provenance record from those answers: resolves the declaration against an exact FC commit, copies the FC-local closure, hoists each slot, and records the provenance | | `comparator/adapter/ComparatorFacts/` and `comparator_facts.lean` | the Lean extractor (a small library — `Binders.lean` recovers declaration-header binder boundaries from source syntax, `Extract.lean` reads the elaborated environment — and a thin executable): source ranges, declaration-header binder boundaries, elaborated binder names/explicitness, answer-slot types, and the `@[category ...]` tag. The parsed source distinguishes header parameters from `∀` binders in the conclusion; every emitted binder fact still comes from the elaborated environment. | -| `comparator/adapter/leaneval_interface.py` | the request builder and response checker — the FC side of the wire format, permanently, since the consumer owns hole resolution under the v1 contract | +| `comparator/adapter/leaneval_interface.py` | the request builder and response checker — the FC side of the wire format, permanently, since the consumer owns hole resolution under the schema-version-1 contract | | `comparator/adapter/leaneval_generator_cli.py` | plumbing for the pinned binary | | `comparator/adapter/make_comparator_workspace.py` | the command, the emitted seam artifact, and the whole-set batch run | | `comparator/templates/WorkspaceTest.lean` | the workspace test template the contract requires the consumer to supply | @@ -121,10 +121,11 @@ request; it keeps no state about what happened to one. ## What this side cannot settle alone -1. **Provenance fields in the contract.** v1 has no home for the FC source - commit and declaration id that §10 requires by name, so they travel as a - sidecar. A v2 passthrough or provenance field would let a generated - workspace carry its own origin. Related: +1. **Provenance fields in the contract.** Generator schema version 1 has no + home for the FC source commit and declaration id that §10 requires by + name, so they travel as a sidecar. A passthrough or provenance field in a + future generator contract revision would let a generated workspace carry + its own origin. Related: `mathlib-initiative/formalization.yaml` already standardises a source repository, revision, declaration and Comparator config — `Paul-Lez/hadamard-668-comparator` uses it to describe a wrapper around FC diff --git a/comparator/README.md b/comparator/README.md index c24971951a..5fd76b9380 100644 --- a/comparator/README.md +++ b/comparator/README.md @@ -82,7 +82,7 @@ python3 comparator/adapter/make_comparator_workspace.py erdos_1038.parts.i \ ``` This writes the exact bytes that cross the seam — `request.json`, the -`context/` directory the v1 contract reads, and the provenance sidecar — and +`context/` directory the schema-version-1 contract reads, and the provenance sidecar — and generates no workspace. Running the pinned binary on that request from inside the emitted directory yields the same file map generation would have written, which is what makes the seam checkable rather than asserted. diff --git a/comparator/adapter/leaneval_generator_cli.py b/comparator/adapter/leaneval_generator_cli.py index 5238d24b4c..122a1722ef 100644 --- a/comparator/adapter/leaneval_generator_cli.py +++ b/comparator/adapter/leaneval_generator_cli.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Run the pinned `lean-eval-generator` binary on a v1 request. +"""Run the pinned `lean-eval-generator` binary on a schema-version-1 request. `leanprover/lean-eval#536` extracts lean-eval's generator core into `leanprover/lean-eval-generator`, a deterministic Lean CLI: one JSON request @@ -12,7 +12,7 @@ is cheap — the package depends on nothing — so CI clones the pinned revision and runs `lake build`; `comparator/README.md` shows the same for a local run. -The context root exists because the v1 contract still resolves two things +The context root exists because the schema-version-1 contract still resolves two things from a benchmark checkout rather than from the request: the module source (which must byte-match the request's `moduleContent`) and each declaration's span from compiled `.ilean` metadata. This consumer is not a benchmark diff --git a/comparator/adapter/leaneval_interface.py b/comparator/adapter/leaneval_interface.py index db373a8cce..dd9d95599d 100644 --- a/comparator/adapter/leaneval_interface.py +++ b/comparator/adapter/leaneval_interface.py @@ -17,9 +17,9 @@ ProblemManifest the facts about the problem that the module's text does not carry, including the FC source commit and the FC declaration id; written beside the generated workspace - as `fc-provenance.json`, because the v1 contract has no + as `fc-provenance.json`, because the schema-version-1 contract has no provenance fields of its own - build_request (module, manifest) pairs -> the v1 request object + build_request (module, manifest) pairs -> the schema-version-1 request object parse_response response text -> file maps, digests checked `comparator/adapter/fc_leaneval_importer.py` produces the pairs. @@ -147,7 +147,7 @@ class TargetRecord: bumped anything, with nothing here to notice. Formal Conjectures keeps the full pin set under `[target]` in - `comparator/tools.toml`; this record carries only the two fields the v1 + `comparator/tools.toml`; this record carries only the two fields the schema-version-1 request consumes. The comparator and lean4export pins are read from the TOML directly by the CI job that runs them. """ @@ -186,7 +186,7 @@ class ProblemManifest: # the seam, and every generated file the response returned for it. With # them a workspace carries its own chain — FC commit → module bytes → # generated bytes — and a reader can check each link without this - # repository. The sidecar is the v1 provenance boundary by design + # repository. The sidecar is the schema-version-1 provenance boundary by design # (lean-eval-generator keeps its wire format frozen), so it has to be # strict and deterministic as well: unknown keys are refused on load and # serialisation is key-sorted. @@ -464,7 +464,7 @@ def line_of(offset): def build_problem(marked_up, manifest, module_name=None, group=None): - """One problem entry of the v1 request, and its `.ilean` declaration map. + """One problem entry of the schema-version-1 request, and its `.ilean` declaration map. The module name is a single identifier on purpose: the generator resolves module names to paths by splitting on every dot, guillemets included, so @@ -537,7 +537,7 @@ def build_problem(marked_up, manifest, module_name=None, group=None): def build_request(problems, target, workspace_test, context_root): - """The complete v1 request for a batch of `(problem, ilean)` pairs. + """The complete schema-version-1 request for a batch of `(problem, ilean)` pairs. `problems` are the entries `build_problem` returned; the ilean halves go to whoever writes the context root, not into the request. Ids must be diff --git a/comparator/adapter/make_comparator_workspace.py b/comparator/adapter/make_comparator_workspace.py index e5a40b4628..e8c0a36466 100644 --- a/comparator/adapter/make_comparator_workspace.py +++ b/comparator/adapter/make_comparator_workspace.py @@ -7,7 +7,7 @@ declaration, in the two steps `leanprover/lean-eval#536` separates: fc_leaneval_importer FC declaration -> marked-up module + manifest - lean-eval-generator v1 request -> workspace file map + lean-eval-generator schema-version-1 request -> workspace file map The first half is Formal Conjectures'. The second is the pinned `leanprover/lean-eval-generator` binary — a deterministic Lean CLI with a @@ -48,7 +48,7 @@ recorded ones: an unexpected failure and a silently fixed one both count, because a gate that only ever passes proves nothing. -`--emit-import` writes the exact bytes that cross the seam — the v1 request, +`--emit-import` writes the exact bytes that cross the seam — the schema-version-1 request, with its context directory — and generates no workspace; running the pinned binary on that request from inside the emitted directory yields the same file map this command would have written. @@ -111,9 +111,9 @@ def seam_files(pairs, group=None): """The request and context for `(marked_up, manifest)` pairs, as files. This is the artifact the FC importer contributes once lean-eval consumes - the shared generator: the request bytes, the context directory the v1 + the shared generator: the request bytes, the context directory the schema-version-1 contract still reads, and one provenance record per problem — the FC - source commit and declaration id §10 requires, which the v1 wire format + source commit and declaration id §10 requires, which the schema-version-1 wire format has no field for, so they travel beside it rather than through it. """ problems = [ @@ -322,7 +322,7 @@ def main(argv): "--emit-import", default=None, metavar="DIR", - help="write only the v1 request and its context, the bytes this " + help="write only the schema-version-1 request and its context, the bytes this " "repository hands the pinned generator, and generate no workspace", ) ap.add_argument( diff --git a/comparator/adapter/test_leaneval_interface.py b/comparator/adapter/test_leaneval_interface.py index 67326281d4..bce1925fcc 100644 --- a/comparator/adapter/test_leaneval_interface.py +++ b/comparator/adapter/test_leaneval_interface.py @@ -312,7 +312,7 @@ def test_an_unknown_schema_version_is_refused(self): class ProvenanceSidecarTest(unittest.TestCase): - """The sidecar is the v1 provenance boundary: strict, deterministic, digested.""" + """The sidecar is the schema-version-1 provenance boundary: strict, deterministic, digested.""" def test_digests_round_trip(self): bound = a_manifest().with_digests("a" * 64, {"Challenge.lean": "b" * 64, "A.lean": "c" * 64}) diff --git a/comparator/adapter/test_make_comparator_workspace.py b/comparator/adapter/test_make_comparator_workspace.py index 2e97aa581c..6e41cdacb5 100644 --- a/comparator/adapter/test_make_comparator_workspace.py +++ b/comparator/adapter/test_make_comparator_workspace.py @@ -78,7 +78,7 @@ def test_the_ilean_carries_every_declaration(self): ) def test_the_provenance_sidecar_is_the_manifest(self): - # The v1 wire format has no provenance fields, so the FC source + # The schema-version-1 wire format has no provenance fields, so the FC source # commit and declaration id §10 requires travel beside the request. _, files = seam_files([(A_MODULE, a_manifest())]) payload = json.loads(files["fc-provenance-erdos_940.json"]) From 37fd9ab3c5d8011e60ca8f1ff351a31bbf608848 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Sat, 22 Aug 2026 04:40:33 -0400 Subject: [PATCH 43/70] Advance the generator pin to 77373a53 and retire the Erdos125 entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit leanprover/lean-eval-generator merged its first three changes on 2026-08-22: #1 preserves active options around extracted syntax context, #2 resolves quoted («…») module paths, and #3 synchronises the standalone generator with LeanEval. The wire format is unchanged apart from the schema titles, which now say "schema version 1". With #1 the generated Challenge for Erdos125.erdos_125.variants.positive_unequal_density carries the `set_option quotPrecheck false` its local notation needs, so the ledger entry that waited on that fix is removed. The two toolchain-drift entries stay until Formal Conjectures' bump. --- comparator/known_failures.toml | 13 ------------- comparator/tools.toml | 2 +- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/comparator/known_failures.toml b/comparator/known_failures.toml index 7084ebf409..68c5f2a09e 100644 --- a/comparator/known_failures.toml +++ b/comparator/known_failures.toml @@ -32,16 +32,3 @@ dependency copied faithfully from source. Retired by the repository's planned toolchain bump (to Lean 4.33; #4428 stages v4.32.0) — kim-em verified on 2026-08-21 at LeanEval's target pins on a merge with #4428. Re-run the audit after the bump merges and remove this entry.""" - -[[failure]] -declaration = "Erdos125.erdos_125.variants.positive_unequal_density" -workspace = "Erdos125_erdos_125_variants_positive_unequal_density" -stage = "target" -reason = """The statement's `local notation` uses set-builder syntax and needs -`set_option quotPrecheck false`, which the source states file-scoped and the -emitted module carries attached to the notation command. The generator's -Challenge reconstruction re-extracts the notation line without its -set_option, and `Challenge.lean` is generator-owned text this side cannot -amend. Fixed upstream in leanprover/lean-eval-generator#1 (verified by kim-em at -this PR's head); remove this entry when `[generator]` in tools.toml advances to a -revision containing that fix.""" diff --git a/comparator/tools.toml b/comparator/tools.toml index cb8184aba5..a385dcdfde 100644 --- a/comparator/tools.toml +++ b/comparator/tools.toml @@ -28,4 +28,4 @@ lean4export = "15f6055e299ad5b89345e533cc2192f4cc00f659" # and has to survive the seam round-trip test. [generator] repository = "https://github.com/leanprover/lean-eval-generator" -rev = "a726789593eeac5c32ad82760061cd5bf6cae662" +rev = "77373a539b31f8f304c852f288d7d8469cceebff" From e51535ae2caeab6c7493450a5d86a5a8651fa82d Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Sat, 22 Aug 2026 08:52:26 -0400 Subject: [PATCH 44/70] Write import_problem as a pipeline of named stages The 157-line function is now six: locate_target (find the declaration, ask the elaborated environment, merge the problem file's explicit copy dependencies), restate (strip, sorry, flatten a dotted name, hoist and unwrap answers), explicit_arguments, scope_region, place_notations, and import_problem itself, which runs them in order and assembles the module and the manifest. Every line of logic moved unchanged; the generated workspaces for the CI smoke list and the whole FC100 set are byte-identical before and after. --- comparator/adapter/fc_leaneval_importer.py | 125 ++++++++++++++------- 1 file changed, 82 insertions(+), 43 deletions(-) diff --git a/comparator/adapter/fc_leaneval_importer.py b/comparator/adapter/fc_leaneval_importer.py index abe63ee112..e8fb0f9524 100644 --- a/comparator/adapter/fc_leaneval_importer.py +++ b/comparator/adapter/fc_leaneval_importer.py @@ -367,12 +367,12 @@ def source_record( ) -def import_problem(problem, answer_type=None, module=None): - """Map one declaration to a marked-up module and a manifest. +def locate_target(problem, module=None): + """Find the declaration and ask the elaborated environment about it. - Importing a closure out of a repository full of `sorry` is safe because - Comparator checks axioms. A solution closing the goal with a copied - statement reports `sorryAx`, which `permitted_axioms` does not allow. + Returns the problem file, the qualified FC declaration, the source path, + the FC module name, the module docstring, and the elaborator facts with + the problem file's explicit copy dependencies merged in. """ problem_file = load_manifest(problem) declaration = problem_file.get("declaration", problem) @@ -391,21 +391,18 @@ def import_problem(problem, answer_type=None, module=None): facts["generatedDependencies"] = list( dict.fromkeys(explicit_generated + facts.get("generatedDependencies", [])) ) + return problem_file, declaration, path, fc_module, module_doc, facts - source_lines = path.read_text(encoding="utf-8").split("\n") - original, lo = slice_range(source_lines, facts["range"]) - statement = original - preamble, namespaces_at_target = file_scoped_preamble(source_lines, lo) - dependencies, copied = closure_region( - facts.get("dependencies", []), - facts.get("generatedDependencies", []), - declaration, - namespaces_at_target, - target_name=facts.get("name"), - ) +def restate(original, declaration, facts, answer_type=None): + """Turn the sliced declaration into the workspace statement. - statement = strip_decorations(statement) + Strips decorations, replaces the proof with `sorry`, flattens a dotted + name to its slug, hoists every `answer(sorry)` into a hole and unwraps + the answers a solved statement carries. Returns the statement, the name + it declares, the name it declared in the source, and the holes. + """ + statement = strip_decorations(original) statement = replace_proof_with_sorry(statement) declared = None for line in statement.split("\n"): @@ -431,7 +428,11 @@ def import_problem(problem, answer_type=None, module=None): # slot, so nothing above removed it and `answer(` would reach a workspace # that cannot parse it. statement = unwrap_answers(statement) + return statement, declared, original_declared, holes + +def explicit_arguments(facts, declared): + """The explicit binders the Solution adapter applies by name.""" args = [b["name"] for b in facts["binders"] if b["explicit"]] bad = [a for a in args if "✝" in a or "._" in a] if bad: @@ -439,7 +440,11 @@ def import_problem(problem, answer_type=None, module=None): f"{declared} has inaccessible explicit binders {bad}; the " "Solution adapter cannot apply them by name" ) + return args + +def scope_region(namespaces_at_target, copied, preamble): + """The `open`s and file-scoped preamble the statement elaborates under.""" # `open A`, then `open A.B`: opening the inner namespace does not open the # outer one, and a statement may name siblings from either. With nothing # copied there are no siblings to name and nothing declares the @@ -453,13 +458,16 @@ def import_problem(problem, answer_type=None, module=None): if copied else [] ) + return "\n".join(opens + localise_notation(preamble)) - mathlib_rev, fc_rev = pins(path.relative_to(ROOT)) - preamble = localise_notation(preamble) - scope_text = "\n".join(opens + preamble) - # Notation is text, not a constant: a statement or copied declaration - # spelled with an FC-defined token needs the defining command copied too, - # and the elaborated closure cannot say so. + +def place_notations(dependencies, scope_text, statement, copied): + """Copy the FC notation commands the text needs, on the right side of the closure. + + Notation is text, not a constant: a statement or copied declaration + spelled with an FC-defined token needs the defining command copied too, + and the elaborated closure cannot say so. + """ # Namespaces the module opens, at its scope and inside every copied # block: a scoped notation can only have been in force where one of # these opens it. @@ -471,26 +479,56 @@ def import_problem(problem, answer_type=None, module=None): notations = notation_blocks( [dependencies, scope_text, statement], opened_for_notation ) - if notations: - # A notation whose right-hand side names a copied declaration must - # come after the block declaring it; every other notation comes - # first, because copied declarations may use its token textually. A - # single notation needing both would need interleaving; none does, - # and `--verify` is what says so. - copied_last_components = {name.rsplit(".", 1)[-1] for name, _ in copied} - before, after = [], [] - for block in notations: - rhs = block.split("=>", 1)[-1] - names = set(re.findall(r"[\w«»'.]+", rhs)) - names |= {name.rsplit(".", 1)[-1] for name in names} - if names & copied_last_components: - after.append(block) - else: - before.append(block) - if before: - dependencies = "\n\n".join(before) + "\n\n" + dependencies - if after: - dependencies = dependencies + "\n\n" + "\n\n".join(after) + if not notations: + return dependencies + # A notation whose right-hand side names a copied declaration must + # come after the block declaring it; every other notation comes + # first, because copied declarations may use its token textually. A + # single notation needing both would need interleaving; none does, + # and `--verify` is what says so. + copied_last_components = {name.rsplit(".", 1)[-1] for name, _ in copied} + before, after = [], [] + for block in notations: + rhs = block.split("=>", 1)[-1] + names = set(re.findall(r"[\w«»'.]+", rhs)) + names |= {name.rsplit(".", 1)[-1] for name in names} + if names & copied_last_components: + after.append(block) + else: + before.append(block) + if before: + dependencies = "\n\n".join(before) + "\n\n" + dependencies + if after: + dependencies = dependencies + "\n\n" + "\n\n".join(after) + return dependencies + + +def import_problem(problem, answer_type=None, module=None): + """Map one declaration to a marked-up module and a manifest. + + Importing a closure out of a repository full of `sorry` is safe because + Comparator checks axioms. A solution closing the goal with a copied + statement reports `sorryAx`, which `permitted_axioms` does not allow. + """ + problem_file, declaration, path, fc_module, module_doc, facts = locate_target( + problem, module + ) + source_lines = path.read_text(encoding="utf-8").split("\n") + original, lo = slice_range(source_lines, facts["range"]) + preamble, namespaces_at_target = file_scoped_preamble(source_lines, lo) + dependencies, copied = closure_region( + facts.get("dependencies", []), + facts.get("generatedDependencies", []), + declaration, + namespaces_at_target, + target_name=facts.get("name"), + ) + statement, declared, original_declared, holes = restate( + original, declaration, facts, answer_type + ) + args = explicit_arguments(facts, declared) + scope_text = scope_region(namespaces_at_target, copied, preamble) + dependencies = place_notations(dependencies, scope_text, statement, copied) marked_up = MarkedUpModule( dependencies=dependencies, scope=scope_text, @@ -502,6 +540,7 @@ def import_problem(problem, answer_type=None, module=None): # workspace statement may carry the flattened `declared` instead, and # this is what ties the two together. qualified = ".".join(namespaces_at_target + [original_declared]) + mathlib_rev, fc_rev = pins(path.relative_to(ROOT)) manifest = ProblemManifest( # The default id is the qualified name: two modules declaring # `conjecture` in different namespaces must not share a workspace. From 29d04d641d9060e18fe809ca32c1d2f8ec7b6f35 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Mon, 24 Aug 2026 02:04:22 -0400 Subject: [PATCH 45/70] Retire the toolchain-drift ledger on the 4.33.1 bump Formal Conjectures now builds at Lean 4.33.1 / Mathlib 0df444a (#4428, merged as fd5c0ab5), which is the bump the two remaining ledger entries waited on: kim-em verified both declarations compile at LeanEval's target pins on a merge with that bump. The expected-failure ledger is now empty, and the whole-set audit asserts exactly that; any new failure fails closed. The prose that said this repository elaborates at Lean 4.27 now says 4.33.1, the interface-test fixtures carry the real toolchain, and [tools].lean4export points at the v4.33.0 tag with a note that no v4.33.1 tag exists upstream yet (patch releases share the export format; build the tag under this repository's toolchain). --- .github/workflows/comparator-lean-4-33.yml | 6 ++--- comparator/adapter/test_leaneval_interface.py | 4 ++-- comparator/known_failures.toml | 24 ------------------- comparator/tools.toml | 6 +++-- 4 files changed, 9 insertions(+), 31 deletions(-) diff --git a/.github/workflows/comparator-lean-4-33.yml b/.github/workflows/comparator-lean-4-33.yml index 2312a753ff..e55e9e005b 100644 --- a/.github/workflows/comparator-lean-4-33.yml +++ b/.github/workflows/comparator-lean-4-33.yml @@ -10,7 +10,7 @@ name: Generated workspace at LeanEval pins # # The two toolchains in this job are the point, not an accident. The importer # reads the declaration's source range, binders and `answer(sorry)` slot types -# from an environment elaborated at this repository's Lean 4.27; the workspace +# from an environment elaborated at this repository's Lean 4.33.1; the workspace # is built and checked at LeanEval's Lean 4.33. If those two disagree about a # statement or a slot type, this job is where it shows. @@ -74,7 +74,7 @@ jobs: - name: Build the pinned lean-eval-generator uses: ./.github/actions/build-lean-eval-generator - # `--verify` elaborates the marked-up module here, at 4.27. It is not a + # `--verify` elaborates the marked-up module here, at 4.33.1. It is not a # substitute for the 4.33 build below; it is what keeps an FC-side # copying defect from being reported as a LeanEval build failure. - name: Import and generate two workspaces @@ -88,7 +88,7 @@ jobs: python3 comparator/adapter/make_comparator_workspace.py "$d" \ --out .comparator --verify done - # One plain theorem and one `answer(sorry)` slot typed at 4.27. + # One plain theorem and one `answer(sorry)` slot typed at 4.33.1. grep -q "isSumOfThreeCubes_iff_mod_9_answer : Prop" \ .comparator/SumOfThreeCubes_isSumOfThreeCubes_iff_mod_9/Challenge.lean # Generated for LeanEval, not for here. diff --git a/comparator/adapter/test_leaneval_interface.py b/comparator/adapter/test_leaneval_interface.py index bce1925fcc..e5c7c36479 100644 --- a/comparator/adapter/test_leaneval_interface.py +++ b/comparator/adapter/test_leaneval_interface.py @@ -49,7 +49,7 @@ def a_source(**overrides): "declaration": "erdos_940", "copied_dependencies": ("Foo.bar",), "original_declaration": "theorem erdos_940 : True := by\n sorry", - "lean_toolchain": "leanprover/lean4:v4.27.0", + "lean_toolchain": "leanprover/lean4:v4.33.1", "mathlib_revision": "c" * 40, } fields.update(overrides) @@ -108,7 +108,7 @@ def test_the_pins_the_hole_types_were_read_at_are_recorded(self): # The consumer supplies its own pins, but it cannot know where these # hole types were read unless the manifest says so. payload = a_manifest().to_json_object() - self.assertEqual(payload["source"]["lean_toolchain"], "leanprover/lean4:v4.27.0") + self.assertEqual(payload["source"]["lean_toolchain"], "leanprover/lean4:v4.33.1") def test_the_manifest_survives_a_round_trip(self): manifest = a_manifest() diff --git a/comparator/known_failures.toml b/comparator/known_failures.toml index 68c5f2a09e..622a029ee8 100644 --- a/comparator/known_failures.toml +++ b/comparator/known_failures.toml @@ -8,27 +8,3 @@ # stage "source" (import or --verify at FC pins) or # "target" (compile at LeanEval pins) # reason what fails and why it is not fixed here - -[[failure]] -declaration = "Erdos1092.f_asymptotic_general" -workspace = "Erdos1092_f_asymptotic_general" -stage = "target" -reason = """The copied `f` definition synthesizes `Fintype ↑H.verts` and -`Fintype ↑H.coe.edgeSet` at this repository's Mathlib but not at LeanEval's -(observed at target pin 6f1ef4e5): instance drift between the two revisions, -not a copying defect. Formal Conjectures' planned toolchain bump (to Lean 4.33; -#4428 stages v4.32.0) adapts the source and retires the gap — kim-em verified on -2026-08-21 that the generated Challenge compiles at LeanEval's target pins on a -merge with #4428. Re-run the audit after the bump merges and remove this entry.""" - -[[failure]] -declaration = "EllipticCurveRank.RatEllipticCurve.twentyone_le_rank_height_count_asymptotic" -workspace = "EllipticCurveRank_RatEllipticCurve_twentyone_le_rank_height_count_asymptotic" -stage = "target" -reason = """The copied `toWeierstrass⟮ℚ⟯` Mordell-Weil notation elaborates at -this repository's Mathlib but not at LeanEval's (observed at target pin -6f1ef4e5): the notation's shape changed between the two revisions, in a -dependency copied faithfully from source. Retired by the repository's planned -toolchain bump (to Lean 4.33; #4428 stages v4.32.0) — kim-em verified on -2026-08-21 at LeanEval's target pins on a merge with #4428. Re-run the audit -after the bump merges and remove this entry.""" diff --git a/comparator/tools.toml b/comparator/tools.toml index a385dcdfde..5d21e83308 100644 --- a/comparator/tools.toml +++ b/comparator/tools.toml @@ -5,8 +5,10 @@ [tools] comparator = "71b52ec29e06d4b7d882726553b1ceb99a2499e0" landrun = "5ed4a3db3a4ad930d577215c6b9abaa19df7f99f" -# lean4export: the tag matching this repository's lean-toolchain, v4.27.0 today. -lean4export = "v4.27.0" +# lean4export: no upstream tag exists for this repository's v4.33.1 yet; +# build the v4.33.0 tag under this repository's toolchain (patch releases +# share the export format). +lean4export = "v4.33.0" # Where a generated workspace is built and checked. These are LeanEval's pins, # not this repository's: a workspace is vendored into lean-eval and built From 3445ed6b8ecbf1e031ce70788b3ee1a17e2f81e3 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:31:12 -0400 Subject: [PATCH 46/70] Retire dead code the review found The unreachable `elif` arm in hoist_answers (its condition implies the first arm already fired), the uncalled ProblemManifest.hole_names, and a duplicated import triple in comparator_facts.lean. --- comparator/adapter/comparator_facts.lean | 4 ---- comparator/adapter/fc_source.py | 5 +---- comparator/adapter/leaneval_interface.py | 13 +++++++++++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/comparator/adapter/comparator_facts.lean b/comparator/adapter/comparator_facts.lean index f10da35e0c..80a00c1e4e 100644 --- a/comparator/adapter/comparator_facts.lean +++ b/comparator/adapter/comparator_facts.lean @@ -20,10 +20,6 @@ import FormalConjecturesUtil.Attributes.Basic import ComparatorFacts.Binders import ComparatorFacts.Extract -import Lean -import FormalConjecturesUtil.Answer -import FormalConjecturesUtil.Attributes.Basic - /-! The executable over `ComparatorFacts/`: the elaborator-side facts `comparator/adapter/fc_leaneval_importer.py` would otherwise get by reading Lean with regular expressions. diff --git a/comparator/adapter/fc_source.py b/comparator/adapter/fc_source.py index 59b3f8266d..9151fc22d0 100644 --- a/comparator/adapter/fc_source.py +++ b/comparator/adapter/fc_source.py @@ -668,8 +668,7 @@ def hoist_answers(statement, basename, slot_types, override=None): position, and that reading wins. For the rest, the types come from the elaborated environment, where the `answer` elaborator ran with the expected type in hand; the old surface-syntax guess (an `↔` beside the - slot means `Prop`) and the FC problem file's hand-kept `answer_type` - both survive only as overrides. Unascribed slots of differing types are + slot means `Prop`) survives only as the `--answer-type` override. Unascribed slots of differing types are refused: the environment reports the types as a set, and matching them to positions would be a guess. """ @@ -705,8 +704,6 @@ def hoist_answers(statement, basename, slot_types, override=None): elif missing == 0 and remaining and len(set(remaining_env)) == 1: for i in remaining: types[i] = remaining_env[0] - elif missing == 0 and not remaining: - pass elif missing == 0: raise SystemExit( f"{basename} has {len(remaining)} answer slots of differing " diff --git a/comparator/adapter/leaneval_interface.py b/comparator/adapter/leaneval_interface.py index dd9d95599d..a95eb715b8 100644 --- a/comparator/adapter/leaneval_interface.py +++ b/comparator/adapter/leaneval_interface.py @@ -67,6 +67,17 @@ MANIFEST_SCHEMA_VERSION = 1 +def lean_errors(output): + """The error lines of a Lean build's output. + + Only errors fail a check: `sorry` warnings are the importer working, and + linter warnings come from copied source. The predicate lives here because + the source-side elaboration gate and the target-side compile gate must + agree on what a failing build is. + """ + return [line for line in output.splitlines() if "error:" in line] + + def sha256_text(text): """The digest the generator response uses: SHA-256 of the UTF-8 bytes.""" return hashlib.sha256(text.encode("utf-8")).hexdigest() @@ -213,8 +224,6 @@ def __post_init__(self): if not self.source.declaration: raise SystemExit(f"manifest {self.id} records no FC declaration id") - def hole_names(self): - return [hole.name for hole in self.holes] def to_json_object(self): payload = { From a12b0d2bcd9a76e6a439dafb7fd78605c19e21ca Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:31:12 -0400 Subject: [PATCH 47/70] One ledger loader and one build-failure predicate for both gates The target gate read known_failures.toml with a bare tomllib.load and silently dropped a target entry missing `workspace`; the source gate's load_known_failures validated everything except `workspace`. The loader now requires `workspace` on target entries and both gates read through it. The "what counts as a failing Lean build" predicate had two copies; `lean_errors` in leaneval_interface is now the only one. Also fail closed on problem-file keys nothing reads: two files carried a dead `source` key and three doc sites claimed an `answer_type` field the code never consulted. The keys are gone, the docs say what the code does, and load_manifest refuses unknown keys. Empty [target] pins are refused instead of flowing into a request as empty strings. --- comparator/adapter/compile_fc100_target.py | 19 +++++++++---------- comparator/adapter/fc_leaneval_importer.py | 12 +++++++++++- .../adapter/make_comparator_workspace.py | 9 +++++++-- .../adapter/test_fc_leaneval_importer.py | 9 ++++++++- .../arithmetic_sum_s_conjecture_1_1.toml | 1 - .../problems/margulis_conjecture_1_1.toml | 1 - 6 files changed, 35 insertions(+), 16 deletions(-) diff --git a/comparator/adapter/compile_fc100_target.py b/comparator/adapter/compile_fc100_target.py index 95847cbb0a..f906400404 100644 --- a/comparator/adapter/compile_fc100_target.py +++ b/comparator/adapter/compile_fc100_target.py @@ -29,7 +29,8 @@ import sys import tomllib -from leaneval_interface import dump_json +from leaneval_interface import lean_errors, dump_json +from make_comparator_workspace import load_known_failures def arrange_project(workspaces_dir, project_dir): @@ -123,11 +124,7 @@ def build(project_dir, modules): capture_output=True, text=True, ) - errors = [ - line - for line in (completed.stdout + completed.stderr).splitlines() - if "error:" in line - ] + errors = lean_errors(completed.stdout + completed.stderr) ok = completed.returncode == 0 and not errors results.append( { @@ -168,14 +165,16 @@ def main(argv): print(f"{report['ok']}/{report['total']} Challenges compile at target pins") if args.known_failures: - with open(args.known_failures, "rb") as handle: - recorded = tomllib.load(handle) # Known failures are recorded by declaration; workspaces are named by # the slugged id, which the `workspace` field of each entry supplies. + # The loader is the source gate's: one validator, one reading of the + # ledger, and a target entry without a `workspace` is refused there + # rather than silently dropped here. + recorded = load_known_failures(args.known_failures) expected = { entry["workspace"] - for entry in recorded.get("failure", []) - if entry.get("stage") == "target" and "workspace" in entry + for entry in recorded.values() + if entry["stage"] == "target" } unexpected = sorted(failed - expected) fixed = sorted(expected - failed) diff --git a/comparator/adapter/fc_leaneval_importer.py b/comparator/adapter/fc_leaneval_importer.py index e8fb0f9524..0fd7f3dae2 100644 --- a/comparator/adapter/fc_leaneval_importer.py +++ b/comparator/adapter/fc_leaneval_importer.py @@ -37,6 +37,7 @@ ProblemManifest, SourceRecord, TargetRecord, + lean_errors, ) from fc_source import ( DECL_START, @@ -83,6 +84,9 @@ def target_pins(): read and where they will be used a readable fact rather than an assumption. """ target = _tools_file()["target"] + for key in ("lean_toolchain", "mathlib_revision"): + if not target.get(key): + raise SystemExit(f"comparator/tools.toml [target] has an empty `{key}`") return TargetRecord( lean_toolchain=target["lean_toolchain"], mathlib_revision=target["mathlib_revision"], @@ -186,6 +190,12 @@ def load_manifest(problem_id): ) if "declaration" not in data: raise SystemExit(f"{path} has no `declaration` field") + unknown = sorted(set(data) - {"id", "declaration", "module", "copy_dependencies"}) + if unknown: + raise SystemExit( + f"{path} has keys nothing reads: {', '.join(unknown)}; a field no " + "code consumes is a record nobody can check" + ) return data @@ -604,7 +614,7 @@ def elaborate(marked_up): # those warnings are the importer working. Linter warnings such as # `unused variable` come from the copied source and say nothing about # whether the copy is faithful. - errors = [line for line in output.splitlines() if "error:" in line] + errors = lean_errors(output) if proc.returncode != 0 or errors: raise SystemExit( "the marked-up module does not elaborate:\n" diff --git a/comparator/adapter/make_comparator_workspace.py b/comparator/adapter/make_comparator_workspace.py index e8c0a36466..7afb3e959c 100644 --- a/comparator/adapter/make_comparator_workspace.py +++ b/comparator/adapter/make_comparator_workspace.py @@ -206,6 +206,11 @@ def load_known_failures(path): f"{path}: {entry['declaration']} has stage {entry['stage']!r}; " "expected source or target" ) + if entry["stage"] == "target" and "workspace" not in entry: + raise SystemExit( + f"{path}: {entry['declaration']} is a target failure without a " + "`workspace`; the target gate matches by workspace id" + ) failures[entry["declaration"]] = entry return failures @@ -303,8 +308,8 @@ def main(argv): ap.add_argument( "--answer-type", default=None, - help="type of a non-Prop answer(sorry) slot; " - "the problem file's `answer_type` is used when absent", + help="type of a non-Prop answer(sorry) slot, for the rare " + "statement whose slots the elaborated environment cannot type apart", ) ap.add_argument( "--module", diff --git a/comparator/adapter/test_fc_leaneval_importer.py b/comparator/adapter/test_fc_leaneval_importer.py index f709387fda..06c9a8af23 100644 --- a/comparator/adapter/test_fc_leaneval_importer.py +++ b/comparator/adapter/test_fc_leaneval_importer.py @@ -210,8 +210,15 @@ def test_absent_problem_file_is_not_an_error(self): self.assertEqual(load_manifest("no_such_problem"), {}) def test_fields_are_read(self): + self.write("p.toml", 'id = "p"\ndeclaration = "d"\nmodule = "m.lean"\n') + self.assertEqual(load_manifest("p")["module"], "m.lean") + + def test_unknown_keys_are_refused(self): + # A field no code consumes is a record nobody can check: it reads as + # configuration while silently doing nothing. self.write("p.toml", 'id = "p"\ndeclaration = "d"\nanswer_type = "ENNReal"\n') - self.assertEqual(load_manifest("p")["answer_type"], "ENNReal") + with self.assertRaisesRegex(SystemExit, "keys nothing reads"): + load_manifest("p") def test_id_must_match_the_filename(self): # The filename is what the importer looks up, so a disagreeing `id` diff --git a/comparator/problems/arithmetic_sum_s_conjecture_1_1.toml b/comparator/problems/arithmetic_sum_s_conjecture_1_1.toml index cf71b6e68b..49dcb6ff29 100644 --- a/comparator/problems/arithmetic_sum_s_conjecture_1_1.toml +++ b/comparator/problems/arithmetic_sum_s_conjecture_1_1.toml @@ -1,4 +1,3 @@ id = "arithmetic_sum_s_conjecture_1_1" declaration = "conjecture_1_1" module = "FormalConjectures/Arxiv/2501.03234/ArithmeticSumS.lean" -source = "https://arxiv.org/abs/2501.03234" diff --git a/comparator/problems/margulis_conjecture_1_1.toml b/comparator/problems/margulis_conjecture_1_1.toml index 23e682c381..49e821b185 100644 --- a/comparator/problems/margulis_conjecture_1_1.toml +++ b/comparator/problems/margulis_conjecture_1_1.toml @@ -4,4 +4,3 @@ id = "margulis_conjecture_1_1" declaration = "conjecture_1_1" module = "FormalConjectures/Arxiv/2504.17644/Margulis.lean" -source = "https://arxiv.org/abs/2504.17644" From 5f5af059f30365426353761adb90d520906e9637 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:31:41 -0400 Subject: [PATCH 48/70] Read *References:* too, and say when a citation is missing Formal Conjectures writes the plural in most modules; the singular split silently dropped the citation from about half the FC100 sidecars. The importer now matches either spelling and warns on stderr when a module docstring yields no link, instead of omitting source_url without a trace. --- comparator/adapter/fc_leaneval_importer.py | 9 ++++++++- comparator/adapter/fc_source.py | 12 ++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/comparator/adapter/fc_leaneval_importer.py b/comparator/adapter/fc_leaneval_importer.py index 0fd7f3dae2..ee5f7ff2d2 100644 --- a/comparator/adapter/fc_leaneval_importer.py +++ b/comparator/adapter/fc_leaneval_importer.py @@ -551,6 +551,13 @@ def import_problem(problem, answer_type=None, module=None): # this is what ties the two together. qualified = ".".join(namespaces_at_target + [original_declared]) mathlib_rev, fc_rev = pins(path.relative_to(ROOT)) + source_url = docstring_reference(module_doc) + if not source_url: + print( + f"{declaration}: the module docstring has no *Reference:* link; " + "the provenance sidecar will carry no source_url", + file=sys.stderr, + ) manifest = ProblemManifest( # The default id is the qualified name: two modules declaring # `conjecture` in different namespaces must not share a workspace. @@ -569,7 +576,7 @@ def import_problem(problem, answer_type=None, module=None): original, mathlib_rev, ), - source_url=docstring_reference(module_doc), + source_url=source_url, category=facts.get("category") or "", ) return marked_up, manifest diff --git a/comparator/adapter/fc_source.py b/comparator/adapter/fc_source.py index 9151fc22d0..7a8376be81 100644 --- a/comparator/adapter/fc_source.py +++ b/comparator/adapter/fc_source.py @@ -122,16 +122,16 @@ def file_scoped_preamble(lines, start_line): def docstring_reference(module_doc): """The source citation Formal Conjectures already writes in the module. - Module docstrings carry a `*Reference:*` line naming where the problem - comes from, sometimes with several links under it. The first is the - problem's own; later ones are commentary and proof notes. + Module docstrings carry a `*Reference:*` or `*References:*` line naming + where the problem comes from, sometimes with several links under it. The + first is the problem's own; later ones are commentary and proof notes. """ if not module_doc: return "" - after = module_doc.split("*Reference:*", 1) - if len(after) != 2: + marker = re.search(r"\*References?:\*", module_doc) + if not marker: return "" - link = re.search(r"\]\((https?://[^)\s]+)\)", after[1]) + link = re.search(r"\]\((https?://[^)\s]+)\)", module_doc[marker.end() :]) return link.group(1) if link else "" def module_name(rel_path): From a60314791a2923353eff12420b8047aa66339fd9 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:36:45 -0400 Subject: [PATCH 49/70] Say each rule once, and scan the tree once `answer_spans` carried its comment-and-string scanner twice; whenever the scan is at code the lexical state is empty, so one `_next_code` helper serves both loops. `closure_region` read and parsed every dependency twice; one pass computes the tuple both consumers share. The seam is one `_seam` helper: the emit path assembles its file map from it, and the generate path stages the context files directly through the same `_write_files` loop `write_tree` uses, instead of filtering the emit map back apart by path prefix. `find_declaration` rglobbed and read every source file per lookup; a cached one-pass index of declared name tokens answers the same question with the same matching rule. The Mathlib revision and the FC merge-base are read once per run; the per-path dirty check stays per call. Shared-library notations with ASCII-only tokens (J(, L(, e) are now copy candidates; the existing gates do the filtering the removed non-ASCII heuristic approximated. --- comparator/adapter/fc_leaneval_importer.py | 20 ++- comparator/adapter/fc_source.py | 161 ++++++++++-------- .../adapter/make_comparator_workspace.py | 49 +++--- 3 files changed, 130 insertions(+), 100 deletions(-) diff --git a/comparator/adapter/fc_leaneval_importer.py b/comparator/adapter/fc_leaneval_importer.py index ee5f7ff2d2..53bc86d345 100644 --- a/comparator/adapter/fc_leaneval_importer.py +++ b/comparator/adapter/fc_leaneval_importer.py @@ -278,14 +278,19 @@ def covered_by_another(dep): # harmless. This covers the statement's own namespace stack and every # namespace a copied preamble opens. created = [] + # One read and preamble parse per dependency; the namespace pre-creation + # and the copied blocks below both consume the same tuple, so the two + # can never disagree about what a dependency's preamble says. + sliced = [] for dep in dependencies: if dep["range"] is None: + sliced.append(None) continue dep_path = module_source_path(dep["module"]) dep_lines = dep_path.read_text(encoding="utf-8").split("\n") - dep_preamble, dep_namespaces = file_scoped_preamble( - dep_lines, slice_range(dep_lines, dep["range"])[1] - ) + dep_text, dep_start = slice_range(dep_lines, dep["range"]) + dep_preamble, dep_namespaces = file_scoped_preamble(dep_lines, dep_start) + sliced.append((dep_path, dep_text, dep_preamble, dep_namespaces)) for entry in dep_preamble: words = entry.split("\n")[0].split() if not words or words[0] != "open": @@ -310,13 +315,10 @@ def covered_by_another(dep): continue seen_namespaces.add(namespace) blocks.append(f"namespace {namespace}\nend {namespace}") - for dep in dependencies: - if dep["range"] is None: + for dep, cut in zip(dependencies, sliced): + if cut is None: raise SystemExit(f"{declaration}: {dep['name']} has no source range") - path = module_source_path(dep["module"]) - lines = path.read_text(encoding="utf-8").split("\n") - text, start = slice_range(lines, dep["range"]) - preamble, namespaces = file_scoped_preamble(lines, start) + path, text, preamble, namespaces = cut body = strip_fc_attributes(text).strip("\n") if not body: raise SystemExit(f"{declaration}: {dep['name']} sliced to nothing") diff --git a/comparator/adapter/fc_source.py b/comparator/adapter/fc_source.py index 7a8376be81..d091a2bb2a 100644 --- a/comparator/adapter/fc_source.py +++ b/comparator/adapter/fc_source.py @@ -10,6 +10,7 @@ """ +import functools import json import pathlib import re @@ -147,17 +148,35 @@ def module_name(rel_path): ] return ".".join(parts) -def _declaring_files(name): - """The files whose text declares `name` as a theorem or lemma.""" - pattern = re.compile( - rf"(?:theorem|lemma)\s+(?:[\w.«»]*\.)?{re.escape(name)}[\s:]" - ) - hits = [] +@functools.lru_cache(maxsize=1) +def _declared_names(): + """Every `theorem`/`lemma` name token in the tree, one pass, cached. + + `{path: [name, ...]}` in sorted path order. A batch import looks up + hundreds of declarations; one scan of the tree replaces a full rglob and + re-read per lookup. + """ + token = re.compile(r"(?:theorem|lemma)\s+([\w.«»]+)[\s:]") + index = {} for src in SOURCE_DIRS: for path in sorted(src.rglob("*.lean")): - if pattern.search(path.read_text(encoding="utf-8")): - hits.append(path) - return hits + index[path] = token.findall(path.read_text(encoding="utf-8")) + return index + + +def _declaring_files(name): + """The files whose text declares `name` as a theorem or lemma. + + A declared token matches when it equals `name` or ends in `.name` — + the same reading as the old per-file regex, whose optional prefix was + `[\w.«»]*\.` over the identical character class. + """ + dotted = "." + name + return [ + path + for path, names in _declared_names().items() + if any(t == name or t.endswith(dotted) for t in names) + ] def _declares_namespaces(text, components): """True if the file opens namespaces spelling out `components` in order. @@ -375,11 +394,13 @@ def fc_notation_commands(): text.append(lines[follow]) follow += 1 command = "\n".join(text) - tokens = [ - token - for token in re.findall(r'"([^"]+)"', command) - if any(ord(c) > 127 for c in token) - ] + # Every string-literal token is a candidate. The gates that + # decide whether a command is actually copied — the token must + # appear in the module text, and a scoped notation's namespace + # must be among the opens — do the filtering; an ASCII-token + # notation such as `J(` in the shared library is otherwise + # invisible here and its consumers fail to elaborate. + tokens = re.findall(r'"([^"]+)"', command) if not tokens: continue bracket = re.match(r"^(?:@\[[^\]]*\]\s*)?scoped\[([\w.«»]+)\]", line) @@ -507,15 +528,14 @@ def replace_proof_with_sorry(text): return text[: m.start()].rstrip() + " := by\n sorry" return text.rstrip() + " := by\n sorry" -def answer_spans(text): - """Return the source spans of syntactic `answer(...)` calls. +def _next_code(text, i): + """The first index at or after `i` holding code, skipping comments and strings. - This small lexer skips strings and nested line/block comments and balances - parentheses, so an answer term may itself contain parentheses. It is not a - Lean parser; malformed or unterminated syntax is refused. + Whenever the scan is at code, the lexical state is empty by construction, + so no state threads between calls. Returns `(index, unterminated)`, with + `index == len(text)` at the end and `unterminated` reporting a comment or + string still open there. """ - spans = [] - i = 0 block_depth = 0 in_string = False escaped = False @@ -552,6 +572,27 @@ def answer_spans(text): in_string = True i += 1 continue + return i, False + return i, bool(block_depth or in_string) + + +def answer_spans(text): + """Return the source spans of syntactic `answer(...)` calls. + + This small lexer skips strings and nested line/block comments and balances + parentheses, so an answer term may itself contain parentheses. It is not a + Lean parser; malformed or unterminated syntax is refused. + """ + spans = [] + i = 0 + while i < len(text): + i, unterminated = _next_code(text, i) + if i >= len(text): + if unterminated: + raise SystemExit( + "unterminated comment or string while reading answers" + ) + break if text.startswith("answer", i) and ( i == 0 or not (text[i - 1].isalnum() or text[i - 1] in "_.'") ): @@ -561,55 +602,24 @@ def answer_spans(text): if j < len(text) and text[j] == "(": depth = 1 k = j + 1 - nested_string = False - nested_escaped = False - nested_comment = 0 while k < len(text) and depth: - nested_pair = text[k : k + 2] - if nested_comment: - if nested_pair == "/-": - nested_comment += 1 - k += 2 - elif nested_pair == "-/": - nested_comment -= 1 - k += 2 - else: - k += 1 - continue - if nested_string: - if nested_escaped: - nested_escaped = False - elif text[k] == "\\": - nested_escaped = True - elif text[k] == '"': - nested_string = False - k += 1 - continue - if nested_pair == "/-": - nested_comment = 1 - k += 2 - elif nested_pair == "--": - newline = text.find("\n", k + 2) - k = len(text) if newline < 0 else newline + 1 - elif text[k] == '"': - nested_string = True - k += 1 - else: - if text[k] == "(": - depth += 1 - elif text[k] == ")": - depth -= 1 - k += 1 + k, _ = _next_code(text, k) + if k >= len(text): + break + if text[k] == "(": + depth += 1 + elif text[k] == ")": + depth -= 1 + k += 1 if depth: raise SystemExit("unterminated answer(...) term") spans.append((i, k, text[j + 1 : k - 1])) i = k continue i += 1 - if block_depth or in_string: - raise SystemExit("unterminated comment or string while reading answers") return spans + def unwrap_answers(statement): """Replace any surviving `answer(t)` with `(t)`. @@ -727,15 +737,13 @@ def hoist_answers(statement, basename, slot_types, override=None): statement = statement[:start] + name + statement[end:] return statement, holes -def pins(source_path=None): - """Revisions the workspace's own build can actually fetch. +@functools.lru_cache(maxsize=1) +def _base_pins(): + """The Mathlib revision and the FC merge-base, invariant for one run. - The FC pin must be reachable from the upstream repository the lakefile - names, so it is the merge-base with `origin/main`, not HEAD: a local - branch commit would generate a workspace whose build fails at fetch time. - The importer stops if the selected source differs from that revision. - Otherwise it could combine a working-tree statement with an older imported - context. + Only the per-path dirty check in `pins` varies between calls, so the + subprocess and manifest parse run once per batch rather than once per + declaration. """ manifest = json.loads((ROOT / "lake-manifest.json").read_text()) mathlib_rev = next(p["rev"] for p in manifest["packages"] if p["name"] == "mathlib") @@ -746,7 +754,20 @@ def pins(source_path=None): ) if merge_base.returncode != 0 or not merge_base.stdout.strip(): raise SystemExit("cannot resolve the Formal Conjectures source revision") - fc_rev = merge_base.stdout.strip() + return mathlib_rev, merge_base.stdout.strip() + + +def pins(source_path=None): + """Revisions the workspace's own build can actually fetch. + + The FC pin must be reachable from the upstream repository the lakefile + names, so it is the merge-base with `origin/main`, not HEAD: a local + branch commit would generate a workspace whose build fails at fetch time. + The importer stops if the selected source differs from that revision. + Otherwise it could combine a working-tree statement with an older imported + context. + """ + mathlib_rev, fc_rev = _base_pins() if source_path is not None: comparison = subprocess.run( ["git", "-C", str(ROOT), "diff", "--quiet", fc_rev, "--", str(source_path)] diff --git a/comparator/adapter/make_comparator_workspace.py b/comparator/adapter/make_comparator_workspace.py index 7afb3e959c..4827fb2235 100644 --- a/comparator/adapter/make_comparator_workspace.py +++ b/comparator/adapter/make_comparator_workspace.py @@ -80,6 +80,14 @@ CONTEXT_DIR = "context" +def _write_files(directory, files): + """Materialise a `{relative path: content}` mapping under `directory`.""" + for relative, content in files.items(): + destination = directory / relative + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(content, encoding="utf-8") + + def write_tree(target, files): """Write a complete directory without overwriting or leaving a partial one. @@ -96,10 +104,7 @@ def write_tree(target, files): tempfile.mkdtemp(prefix=f".{target.name}.", dir=target.parent) ) try: - for relative, content in files.items(): - destination = staging / relative - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text(content, encoding="utf-8") + _write_files(staging, files) staging.rename(target) except BaseException: shutil.rmtree(staging, ignore_errors=True) @@ -107,15 +112,8 @@ def write_tree(target, files): return target -def seam_files(pairs, group=None): - """The request and context for `(marked_up, manifest)` pairs, as files. - - This is the artifact the FC importer contributes once lean-eval consumes - the shared generator: the request bytes, the context directory the schema-version-1 - contract still reads, and one provenance record per problem — the FC - source commit and declaration id §10 requires, which the schema-version-1 wire format - has no field for, so they travel beside it rather than through it. - """ +def _seam(pairs, group=None): + """The request and its `build_problem` outputs for `(marked_up, manifest)` pairs.""" problems = [ build_problem(marked_up, manifest, group=group) for marked_up, manifest in pairs @@ -127,6 +125,19 @@ def seam_files(pairs, group=None): request = build_request( [problem for problem, _ in problems], target, template, CONTEXT_DIR ) + return request, problems + + +def seam_files(pairs, group=None): + """The request and context for `(marked_up, manifest)` pairs, as files. + + This is the artifact the FC importer contributes once lean-eval consumes + the shared generator: the request bytes, the context directory the schema-version-1 + contract still reads, and one provenance record per problem — the FC + source commit and declaration id §10 requires, which the schema-version-1 wire format + has no field for, so they travel beside it rather than through it. + """ + request, problems = _seam(pairs, group=group) files = {"request.json": dump_json(request)} for path, content in generator_cli.context_files(problems).items(): files[f"{CONTEXT_DIR}/{path}"] = content @@ -140,17 +151,13 @@ def seam_files(pairs, group=None): def generate_workspaces(pairs, out_dir, group=None): """Generate one workspace per pair under `out_dir`, via the pinned binary.""" - request, files = seam_files(pairs, group=group) + request, problems = _seam(pairs, group=group) staging = pathlib.Path(tempfile.mkdtemp(prefix=".fc-seam.")) try: # Only the context crosses to the binary; the request goes on stdin - # and the provenance sidecars are for the written workspaces. - for relative, content in files.items(): - if not relative.startswith(f"{CONTEXT_DIR}/"): - continue - destination = staging / relative - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text(content, encoding="utf-8") + # and the provenance sidecars belong to the written workspaces, so + # neither is staged here. + _write_files(staging / CONTEXT_DIR, generator_cli.context_files(problems)) request["contextRoot"] = str(staging / CONTEXT_DIR) workspaces = generator_cli.generate(request) finally: From d75f2f9717f7bb8b7e82f87cb268860f6b82986f Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:36:45 -0400 Subject: [PATCH 50/70] Stop triggering the comparator job on a file it refuses to import `pins()` refuses to generate from a source file that differs from the merge-base with upstream main, so a pull request editing SumOfThreeCubes.lean could never pass the job its own edit triggered. The trigger now covers comparator/** and the workflow itself; the whole-set audit covers statement drift once an edit lands. The four paths comparator/** already covered are gone. tools.toml now names the comparator repository beside its revision, the workflow clones what the TOML says instead of a hardcoded URL, and the build asserts the bundled lean4export is the revision [target] declares rather than printing it into an output nothing read. --- .github/workflows/comparator-lean-4-33.yml | 25 +++++++++++++++------- comparator/tools.toml | 1 + 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.github/workflows/comparator-lean-4-33.yml b/.github/workflows/comparator-lean-4-33.yml index e55e9e005b..6eb4e94137 100644 --- a/.github/workflows/comparator-lean-4-33.yml +++ b/.github/workflows/comparator-lean-4-33.yml @@ -22,14 +22,12 @@ concurrency: on: pull_request: + # Not the imported statement's source file: `pins()` refuses to generate + # from a file that differs from the merge-base with upstream main, so a + # pull request editing it could never pass this job; the whole-set audit + # covers statement drift once the edit lands. paths: - - 'comparator/adapter/fc_leaneval_importer.py' - - 'comparator/adapter/leaneval_generator_cli.py' - - 'comparator/adapter/leaneval_interface.py' - - 'comparator/adapter/make_comparator_workspace.py' - - 'comparator/adapter/comparator_facts.lean' - 'comparator/**' - - 'FormalConjectures/Wikipedia/SumOfThreeCubes.lean' - '.github/workflows/comparator-lean-4-33.yml' workflow_dispatch: @@ -60,7 +58,12 @@ jobs: with open("comparator/tools.toml", "rb") as handle: target = tomllib.load(handle)["target"] - for key in ("comparator", "lean4export", "lean_toolchain"): + for key in ( + "comparator", + "comparator_repository", + "lean4export", + "lean_toolchain", + ): print(f"{key}={target[key]}") PY @@ -105,10 +108,16 @@ jobs: - name: Build the pinned Lean 4.33 verifier stack env: COMPARATOR_REV: ${{ steps.target.outputs.comparator }} + COMPARATOR_REPOSITORY: ${{ steps.target.outputs.comparator_repository }} + LEAN4EXPORT_REV: ${{ steps.target.outputs.lean4export }} run: | - git clone https://github.com/leanprover/comparator.git "$RUNNER_TEMP/comparator" + git clone "$COMPARATOR_REPOSITORY" "$RUNNER_TEMP/comparator" git -C "$RUNNER_TEMP/comparator" checkout "$COMPARATOR_REV" (cd "$RUNNER_TEMP/comparator" && lake build comparator lean4export) + # The lean4export this stack bundles must be the one tools.toml + # declares, or a green run certifies a different exporter. + BUNDLED="$(git -C "$RUNNER_TEMP/comparator/.lake/packages/lean4export" rev-parse HEAD)" + test "$BUNDLED" = "$LEAN4EXPORT_REV" # Three Comparator verdicts, each of which has to come out the stated way. # A run that only ever accepts proves nothing about the gate. diff --git a/comparator/tools.toml b/comparator/tools.toml index 5d21e83308..657bea84d5 100644 --- a/comparator/tools.toml +++ b/comparator/tools.toml @@ -18,6 +18,7 @@ lean4export = "v4.33.0" # the gap between the two readable rather than assumed. [target] repository = "leanprover/lean-eval" +comparator_repository = "https://github.com/leanprover/comparator" commit = "7699436464052268e6c04b41554bfbc2c6908ec5" lean_toolchain = "leanprover/lean4:v4.33.0" mathlib_revision = "6f1ef4e5dd604a435bddba4747b13970cd65d2a1" From ee62fce814e06e2c3da98be000357bb0f35d359a Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:42:54 -0400 Subject: [PATCH 51/70] Read statements the way Lean does, not the way the corpus happens to be MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four fixes the review's finders confirmed against the tree: - `omit` and `include` are file-scoped context; the keyword list did not know them, so a statement written under `omit [...]` was copied with the omitted instances silently restored — an elaborating module stating a different theorem, which no gate could see. Corpus witnesses in ErdosProblems/80, Wikipedia/Kaplansky and two Arxiv files. - `answerTypes` was read off the conclusion after the telescope, so a slot inside a hypothesis binder was invisible and mistyped Prop; the binder types come from the same telescope and now contribute (witness: erdos_975.variants.quadratic, whose slot is ℝ). - An ascribed slot retired its environment entry only when the source spelling string-equalled ppExpr output, so `Set <| Triangle ℝ ℝ²` produced the refusal "-1 Prop slot(s) ... cannot be matched" (witness: erdos_633, which now imports). A statement whose every slot is ascribed is fully typed; leftover entries are respellings. - The proof cut and the block-comment depth count were textual in ways Lean is not: the cut now happens at the first bracket-depth-zero `:= by` (an autoParam default or a structure literal is statement text), and `/-` inside a string, after `--`, or as part of `/--` no longer moves the comment depth. Also: strip_fc_attributes removes the one blank line an emptied attribute leaves, not every blank line in the block. --- comparator/adapter/comparator_facts.lean | 11 ++- comparator/adapter/fc_source.py | 83 ++++++++++++++----- .../adapter/test_fc_leaneval_importer.py | 23 ++++- 3 files changed, 91 insertions(+), 26 deletions(-) diff --git a/comparator/adapter/comparator_facts.lean b/comparator/adapter/comparator_facts.lean index 80a00c1e4e..7fbd398ac0 100644 --- a/comparator/adapter/comparator_facts.lean +++ b/comparator/adapter/comparator_facts.lean @@ -68,8 +68,15 @@ where -- `findAnswerExprs` is the repository's own detection: it reads the -- annotation the `answer` elaborator leaves, rather than guessing from -- `sorryAx` applications. - let answerTypes ← forallTelescope info.type fun _ body => do - let found := Google.findAnswerExprs body + let answerTypes ← forallTelescope info.type fun xs body => do + -- The slots live anywhere in the statement: a hypothesis binder + -- `(h : c = answer(sorry))` carries one just as the conclusion can. + -- Binder types come from the telescope's local declarations, so the + -- expressions are closed in the local context and inferType works. + let mut found := #[] + for x in xs do + found := found ++ Google.findAnswerExprs (← x.fvarId!.getDecl).type + found := found ++ Google.findAnswerExprs body found.mapM fun a => do pure (toString (← ppExpr (← inferType a))) let sourceResult ← declarationSource modName ranges let declarationText ← match sourceResult with diff --git a/comparator/adapter/fc_source.py b/comparator/adapter/fc_source.py index d091a2bb2a..dba3f13852 100644 --- a/comparator/adapter/fc_source.py +++ b/comparator/adapter/fc_source.py @@ -39,6 +39,7 @@ # stack and a compilation mode for everything inside it. r"^(?:(?:local|scoped)\s+)?" r"(open|variable|universe|section|namespace|end|attribute|set_option" + r"|omit|include" r"|notation|postfix|prefix|infixl|infixr|infix|macro|syntax|macro_rules)\b" r"|^noncomputable section\b" ) @@ -108,7 +109,17 @@ def file_scoped_preamble(lines, start_line): index += 1 text.append(lines[index]) preamble.append(("\n".join(text), list(stack))) - depth += len(re.findall(r"/-", line)) - len(re.findall(r"-/", line)) + code = line + if depth == 0: + # A `/-` inside a string or after `--` is not a comment opener; + # inside an open block comment the raw line is what counts. + code = re.sub(r'"(?:[^"\\]|\\.)*"', '""', code) + # `--` opens a line comment unless it is the tail of the + # doc-comment opener `/--`. + m = re.search(r"(? 1: + openers = "([{⟨" + closers = ")]}⟩" + depth = 0 + assigns = [] + i = 0 + while i < len(text): + i, _ = _next_code(text, i) + if i >= len(text): + break + char = text[i] + if char in openers: + depth += 1 + elif char in closers: + depth = max(depth - 1, 0) + elif depth == 0 and text.startswith(":=", i): + j = i + 2 + while j < len(text) and text[j].isspace(): + j += 1 + tactic = text.startswith("by", j) and ( + j + 2 >= len(text) + or not (text[j + 2].isalnum() or text[j + 2] in "_'") + ) + if tactic: + return text[: i].rstrip() + " := by\n sorry" + assigns.append(i) + i = j + continue + i += 1 + if len(assigns) > 1: raise SystemExit( - "the declaration has a term-mode proof and more than one `:=`, so " - "the start of the proof cannot be read off the text" + "the declaration has a term-mode proof and more than one " + "top-level `:=`, so the start of the proof cannot be read off " + "the text" ) - m = re.search(r":=", text) - if m: - return text[: m.start()].rstrip() + " := by\n sorry" + if assigns: + return text[: assigns[0]].rstrip() + " := by\n sorry" return text.rstrip() + " := by\n sorry" def _next_code(text, i): @@ -708,7 +746,12 @@ def hoist_answers(statement, basename, slot_types, override=None): # carries no annotation for is a `Prop` slot by the elaborator's own # rule, not by guesswork, and no postpone build is needed. missing = len(remaining) - len(remaining_env) - if missing == len(remaining): + if not remaining: + # Every slot stated its own type. An environment entry that + # survived is one of those same slots spelled the way the + # elaborator prints types, not an extra slot to place. + pass + elif missing == len(remaining): for i in remaining: types[i] = "Prop" elif missing == 0 and remaining and len(set(remaining_env)) == 1: diff --git a/comparator/adapter/test_fc_leaneval_importer.py b/comparator/adapter/test_fc_leaneval_importer.py index 06c9a8af23..311f526d69 100644 --- a/comparator/adapter/test_fc_leaneval_importer.py +++ b/comparator/adapter/test_fc_leaneval_importer.py @@ -159,11 +159,26 @@ def test_term_mode_proof_is_replaced_too(self): self.assertNotIn("trivial", out) self.assertTrue(out.rstrip().endswith("sorry")) - def test_term_proof_with_a_structure_literal_is_refused(self): - # The statement's own `:=` cannot be told from the proof's, and - # cutting at the wrong one truncates the statement. + def test_structure_literal_assign_is_statement_text(self): + # `{ a := 1 }` lives inside brackets; only the top-level `:=` starts + # the proof, so the statement survives intact. + out = replace_proof_with_sorry("theorem t : F { a := 1 } := ⟨rfl⟩") + self.assertIn("F { a := 1 }", out) + self.assertNotIn("⟨rfl⟩", out) + self.assertTrue(out.rstrip().endswith("sorry")) + + def test_autoparam_default_is_statement_text(self): + # An autoParam binder carries `:= by` inside its parentheses; the + # proof is the top-level one. + out = replace_proof_with_sorry( + "theorem t (h : Fact (1 < 2) := by norm_num) : True := by trivial" + ) + self.assertIn(":= by norm_num", out) + self.assertNotIn("trivial", out) + + def test_two_top_level_assigns_are_refused(self): with self.assertRaises(SystemExit): - replace_proof_with_sorry("theorem t : F { a := 1 } := ⟨rfl⟩") + replace_proof_with_sorry("def t : Nat := f := g") def test_a_line_comment_between_docstring_and_attribute_is_stripped(self): # Erdos 918 writes a `--` formalisation note there. One anchored pass From ca4839e2d6f1621fe3c2d6d20b70309c38350f0e Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:47:50 -0400 Subject: [PATCH 52/70] Pay the Mathlib import once per set `comparator_facts` gains a `--batch` arm: `module declaration` pairs on stdin, one environment importing every module in the batch, one JSON object per line in input order. `resolveIn` already filters candidates by module, so the shared environment answers each pair exactly as a per-module import does; the heartbeat budget scales with the batch so each pair keeps its single-run allowance. `import_set` prefetches the whole set through it into a cache that `elaborator_facts` consults first. A pair the batch reports an error for is not cached: the per-declaration run remains the arbiter of what fails and with what message. A hundred-declaration audit run drops from a hundred Mathlib imports to one. --- .../adapter/ComparatorFacts/Extract.lean | 8 ++-- comparator/adapter/comparator_facts.lean | 44 ++++++++++++++++--- comparator/adapter/fc_leaneval_importer.py | 9 ++++ comparator/adapter/fc_source.py | 40 ++++++++++++++++- .../adapter/make_comparator_workspace.py | 11 +++++ 5 files changed, 101 insertions(+), 11 deletions(-) diff --git a/comparator/adapter/ComparatorFacts/Extract.lean b/comparator/adapter/ComparatorFacts/Extract.lean index 33f5718a65..0b0aaf762c 100644 --- a/comparator/adapter/ComparatorFacts/Extract.lean +++ b/comparator/adapter/ComparatorFacts/Extract.lean @@ -74,7 +74,7 @@ partial def fcOrder (env : Environment) (n : Name) (seen, acc.push n) unsafe def runWithImports {α : Type} (moduleNames : Array Name) - (actionToRun : MetaM α) : IO α := do + (actionToRun : MetaM α) (heartbeats : Nat := 400000000) : IO α := do initSearchPath (← getBuildDir) let imports := moduleNames.map fun n => { module := n } Lean.enableInitializersExecution @@ -82,8 +82,10 @@ unsafe def runWithImports {α : Type} (moduleNames : Array Name) -- Twice the default budget, in the context's raw units, which are a -- thousand times the `maxHeartbeats` option's: 800000 here meant "800" and -- killed the first query. Finite, so a pathological statement errors and is - -- caught rather than grinding forever, which maxHeartbeats := 0 did. - let ctx := { fileName := "", fileMap := default, maxHeartbeats := 400000000 } + -- caught rather than grinding forever, which maxHeartbeats := 0 did. A + -- batch caller scales the budget by its pair count, since one context + -- meters the whole action. + let ctx := { fileName := "", fileMap := default, maxHeartbeats := heartbeats } let (result, _) ← Core.CoreM.toIO (actionToRun.run' {} {}) ctx { env := env } return result diff --git a/comparator/adapter/comparator_facts.lean b/comparator/adapter/comparator_facts.lean index 7fbd398ac0..80e1b5144b 100644 --- a/comparator/adapter/comparator_facts.lean +++ b/comparator/adapter/comparator_facts.lean @@ -49,20 +49,51 @@ unsafe def main (args : List String) : IO UInt32 := do match args with | ["--self-test"] => runWithImports #[`Mathlib] do binderBoundarySelfTest (← getEnv) + | ["--batch"] => + -- One `module declaration` pair per stdin line, one environment for all + -- of them: the Mathlib import dominates a run, and `resolveIn` filters + -- by module, so a shared environment answers each pair exactly as a + -- per-module import does. One JSON object per line, in input order. + let stdin ← IO.getStdin + let lines := (← stdin.readToEnd).splitOn "\n" |>.filter (· ≠ "") + let pairs ← lines.mapM fun line => do + match line.splitOn " " with + | [modName, declName] => pure (modName, declName) + | _ => throw <| IO.userError s!"malformed batch line: {line}" + let modules := pairs.foldl (init := #[]) fun acc (m, _) => + if acc.contains m.toName then acc else acc.push m.toName + -- The heartbeat budget is shared by the whole action, so it scales with + -- the batch; each pair keeps the single-run allowance. + runWithImports modules (heartbeats := pairs.length * 400000000) do + let env ← getEnv + for (modName, declName) in pairs do + let tagged (rest : List (String × Json)) := Json.mkObj <| + [("module", Json.str modName), ("declaration", Json.str declName)] ++ rest + match resolveIn env modName.toName declName with + | .error msg => IO.println (tagged [("error", Json.str msg)]).compress + | .ok n => + try + let payload ← factsPayload env modName.toName n declName + IO.println (tagged [("facts", payload)]).compress + catch e => + IO.println (tagged [("error", Json.str (← e.toMessageData.toString))]).compress + return 0 | [modName, declName] => runWithImports #[modName.toName] do let env ← getEnv match resolveIn env modName.toName declName with | .error msg => IO.eprintln msg; return 1 - | .ok n => emit env modName.toName n declName + | .ok n => + IO.println (← factsPayload env modName.toName n declName).pretty + return 0 | _ => - IO.eprintln "usage: comparator_facts | --self-test" + IO.eprintln "usage: comparator_facts | --batch | --self-test" return 1 where - emit (env : Environment) (modName name : Name) (decl : String) : MetaM UInt32 := do - let some info := env.find? name | IO.eprintln "vanished"; return 1 + factsPayload (env : Environment) (modName name : Name) (decl : String) : MetaM Json := do + let some info := env.find? name | throwError "{name} vanished from the environment" let some ranges ← findDeclarationRanges? name - | IO.eprintln s!"{name} has no source range"; return 1 + | throwError "{name} has no source range" -- The statement's sorries are `answer(sorry)` slots; a proof's sorry is -- not in the *type*, so everything found here is a slot. -- `findAnswerExprs` is the repository's own detection: it reads the @@ -146,8 +177,7 @@ where ("answerTypes", toJson answerTypes.toList), ("dependencies", toJson deps.toList), ("generatedDependencies", toJson generated.toList)] - IO.println payload.pretty - return 0 + return payload rangeToJson (ranges : Option DeclarationRanges) : Json := match ranges with | some r => Json.mkObj [ diff --git a/comparator/adapter/fc_leaneval_importer.py b/comparator/adapter/fc_leaneval_importer.py index 53bc86d345..41a56a8c7e 100644 --- a/comparator/adapter/fc_leaneval_importer.py +++ b/comparator/adapter/fc_leaneval_importer.py @@ -515,6 +515,15 @@ def place_notations(dependencies, scope_text, statement, copied): return dependencies +def statement_pair(problem, module=None): + """The `(module, declaration)` pair `import_problem` will ask the elaborator about.""" + problem_file = load_manifest(problem) + declaration = problem_file.get("declaration", problem) + module = module or problem_file.get("module") + path, _imports, _module_doc, _body = find_declaration(declaration, module) + return module_name(path.relative_to(ROOT)), declaration + + def import_problem(problem, answer_type=None, module=None): """Map one declaration to a marked-up module and a manifest. diff --git a/comparator/adapter/fc_source.py b/comparator/adapter/fc_source.py index dba3f13852..6cebdda82a 100644 --- a/comparator/adapter/fc_source.py +++ b/comparator/adapter/fc_source.py @@ -44,6 +44,40 @@ r"|^noncomputable section\b" ) +_FACTS_CACHE = {} + + +def prefetch_elaborator_facts(pairs): + """Fill the facts cache from one batched extractor run. + + `pairs` are `(module, declaration)` tuples. The Mathlib import dominates + a `comparator_facts` launch, so a batch pays it once for the whole set. + A pair the batch reports an error for is left out of the cache: the + caller's own `elaborator_facts` call re-runs it singly and fails with + exactly the message a single run always produced. + """ + wanted = [pair for pair in dict.fromkeys(pairs) if pair not in _FACTS_CACHE] + if not wanted: + return + proc = subprocess.run( + ["lake", "exe", "comparator_facts", "--batch"], + input="".join(f"{module} {declaration}\n" for module, declaration in wanted), + capture_output=True, + text=True, + cwd=ROOT, + ) + if proc.returncode != 0: + # The batch is an optimisation; the per-declaration path is the + # arbiter of what fails and how it is reported. + return + for line in proc.stdout.splitlines(): + if not line.startswith("{"): + continue + entry = json.loads(line) + if "facts" in entry: + _FACTS_CACHE[(entry["module"], entry["declaration"])] = entry["facts"] + + def elaborator_facts(module, declaration): """What the elaborated environment knows about a declaration. @@ -51,8 +85,12 @@ def elaborator_facts(module, declaration): declaration's source range, its binders with real explicitness, and the inferred type of each `answer(sorry)` slot. Every one of these used to be reconstructed from text, and each reconstruction had failure modes the - elaborator does not. + elaborator does not. A batch import fills `_FACTS_CACHE` first, so a set + run pays the Mathlib import once. """ + cached = _FACTS_CACHE.get((module, declaration)) + if cached is not None: + return json.loads(json.dumps(cached)) proc = subprocess.run( ["lake", "exe", "comparator_facts", module, declaration], capture_output=True, diff --git a/comparator/adapter/make_comparator_workspace.py b/comparator/adapter/make_comparator_workspace.py index 4827fb2235..4718a417b2 100644 --- a/comparator/adapter/make_comparator_workspace.py +++ b/comparator/adapter/make_comparator_workspace.py @@ -67,6 +67,7 @@ import tomllib import fc_leaneval_importer as importer +import fc_source import leaneval_generator_cli as generator_cli from leaneval_interface import build_problem, build_request, dump_json, sha256_text, slug @@ -231,6 +232,16 @@ def import_set(set_name, out_dir, verify=False, known_failures=None): lean-eval#536 gates the FC import on this audit being reproducible. """ declarations = subset_declarations(set_name) + # One batched extractor run pays the Mathlib import once for the whole + # set. A declaration whose module cannot even be located is skipped here + # and fails in its own import below, exactly as it always did. + statement_pairs = [] + for declaration in declarations: + try: + statement_pairs.append(importer.statement_pair(declaration)) + except SystemExit: + continue + fc_source.prefetch_elaborator_facts(statement_pairs) pairs, results = [], [] for declaration in declarations: try: From 1b4b05a513bd391124ec7aa0bdb91ece8cf5029d Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:53:00 -0400 Subject: [PATCH 53/70] Qualify notation commands by their distinctive tokens Dropping the non-ASCII rule wholesale let a bare ")" literal qualify a command against every text. Delimiter-only literals are out; tokens with letters or non-ASCII stay, which is what admits J(, L( and e from the shared library. --- comparator/adapter/fc_source.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/comparator/adapter/fc_source.py b/comparator/adapter/fc_source.py index 6cebdda82a..9c6cad7c0e 100644 --- a/comparator/adapter/fc_source.py +++ b/comparator/adapter/fc_source.py @@ -444,13 +444,17 @@ def fc_notation_commands(): text.append(lines[follow]) follow += 1 command = "\n".join(text) - # Every string-literal token is a candidate. The gates that - # decide whether a command is actually copied — the token must - # appear in the module text, and a scoped notation's namespace - # must be among the opens — do the filtering; an ASCII-token - # notation such as `J(` in the shared library is otherwise - # invisible here and its consumers fail to elaborate. - tokens = re.findall(r'"([^"]+)"', command) + # A command's distinctive tokens decide whether a module + # uses it: `J(` or `α(` says something, the closing `")"` + # matches every text. So delimiter-only literals are dropped, + # while ASCII tokens with letters stay candidates — the old + # non-ASCII rule hid `J(`, `L(` and `e` from the shared + # library and their consumers failed to elaborate. + tokens = [ + token + for token in re.findall(r'"([^"]+)"', command) + if any(c.isalnum() or ord(c) > 127 for c in token) + ] if not tokens: continue bracket = re.match(r"^(?:@\[[^\]]*\]\s*)?scoped\[([\w.«»]+)\]", line) From c674e024c7da9624d1f6ee445cfcc637938a7286 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:25:03 -0400 Subject: [PATCH 54/70] Hold the extractor payload to the wire format The adapter's other three JSON boundaries refuse keys nothing reads; the extractor payload was the one consumed as a raw dict. FactsRecord validates the payload's keys, binder and dependency shapes at the seam, so a drift between comparator_facts and this side fails there instead of surfacing as a missing default downstream. `_resolve` is the one statement of problem-file-to-declaration resolution, shared by statement_pair and locate_target. OWNERSHIP and the README now describe the batch arm and the prefetch. --- comparator/OWNERSHIP.md | 4 +- comparator/README.md | 6 +- comparator/adapter/fc_leaneval_importer.py | 62 +++++++++------- comparator/adapter/fc_source.py | 72 ++++++++++++++++++- .../adapter/test_fc_leaneval_importer.py | 31 ++++---- 5 files changed, 131 insertions(+), 44 deletions(-) diff --git a/comparator/OWNERSHIP.md b/comparator/OWNERSHIP.md index 1aafaabd53..2d2952de2f 100644 --- a/comparator/OWNERSHIP.md +++ b/comparator/OWNERSHIP.md @@ -82,9 +82,9 @@ Conjectures corrects a misformalisation upstream. | File | Why it cannot move | |---|---| -| `comparator/adapter/fc_source.py` | reads this repository's own Lean: where a declaration is, the file-scoped directives in force where it was written, the FC-defined notation it uses, its `answer(sorry)` slots and their elaborated types, and the pins the text was read at | +| `comparator/adapter/fc_source.py` | reads this repository's own Lean: where a declaration is, the file-scoped directives in force where it was written, the FC-defined notation it uses, its `answer(sorry)` slots and their elaborated types, and the pins the text was read at. Holds the strict `FactsRecord` boundary for the extractor's payload, and the prefetch cache a set run fills through the extractor's `--batch` arm so the Mathlib import is paid once | | `comparator/adapter/fc_leaneval_importer.py` | assembles the marked-up module and the provenance record from those answers: resolves the declaration against an exact FC commit, copies the FC-local closure, hoists each slot, and records the provenance | -| `comparator/adapter/ComparatorFacts/` and `comparator_facts.lean` | the Lean extractor (a small library — `Binders.lean` recovers declaration-header binder boundaries from source syntax, `Extract.lean` reads the elaborated environment — and a thin executable): source ranges, declaration-header binder boundaries, elaborated binder names/explicitness, answer-slot types, and the `@[category ...]` tag. The parsed source distinguishes header parameters from `∀` binders in the conclusion; every emitted binder fact still comes from the elaborated environment. | +| `comparator/adapter/ComparatorFacts/` and `comparator_facts.lean` | the Lean extractor (a small library — `Binders.lean` recovers declaration-header binder boundaries from source syntax, `Extract.lean` reads the elaborated environment — and a thin executable): source ranges, declaration-header binder boundaries, elaborated binder names/explicitness, answer-slot types (anywhere in the statement, hypothesis binders included), and the `@[category ...]` tag. One `module declaration` pair per invocation, or `--batch` pairs on stdin sharing a single environment, one JSON object per line in input order. The parsed source distinguishes header parameters from `∀` binders in the conclusion; every emitted binder fact still comes from the elaborated environment. | | `comparator/adapter/leaneval_interface.py` | the request builder and response checker — the FC side of the wire format, permanently, since the consumer owns hole resolution under the schema-version-1 contract | | `comparator/adapter/leaneval_generator_cli.py` | plumbing for the pinned binary | | `comparator/adapter/make_comparator_workspace.py` | the command, the emitted seam artifact, and the whole-set batch run | diff --git a/comparator/README.md b/comparator/README.md index 5fd76b9380..3951eafff5 100644 --- a/comparator/README.md +++ b/comparator/README.md @@ -96,7 +96,11 @@ python3 comparator/adapter/make_comparator_workspace.py --set FC100OpenSet1 \ ``` One request carries every declaration that imports; failures are recorded per -declaration in the report instead of aborting the run. With +declaration in the report instead of aborting the run. The set is prefetched +through `comparator_facts --batch`, so the Mathlib environment is imported +once for the whole run rather than once per declaration; a declaration the +batch cannot answer falls back to its own single run and fails with that +run's message. With `--known-failures`, the run fails unless the failures are exactly the recorded ones — an unexpected failure and a silently fixed one both count. diff --git a/comparator/adapter/fc_leaneval_importer.py b/comparator/adapter/fc_leaneval_importer.py index 41a56a8c7e..9532de8fa6 100644 --- a/comparator/adapter/fc_leaneval_importer.py +++ b/comparator/adapter/fc_leaneval_importer.py @@ -25,6 +25,7 @@ that dependency unrecoverable from the compiled environment. """ +import dataclasses import pathlib import re import subprocess @@ -126,15 +127,15 @@ def explicit_copy_dependencies(problem_file): raise SystemExit(f"copy dependency module does not exist: {relative}") module = module_name(relative) facts = elaborator_facts(module, entry["declaration"]) - records.extend(facts.get("dependencies", [])) + records.extend(facts.dependencies) records.append( { - "name": facts["name"], + "name": facts.name, "module": module, - "range": facts["range"], + "range": facts.range, } ) - generated.extend(facts.get("generatedDependencies", [])) + generated.extend(facts.generated_dependencies) return records, generated @@ -386,22 +387,21 @@ def locate_target(problem, module=None): the FC module name, the module docstring, and the elaborator facts with the problem file's explicit copy dependencies merged in. """ - problem_file = load_manifest(problem) - declaration = problem_file.get("declaration", problem) - # An argument given on the command line is explicit, so it wins over the - # problem file; the file is the durable record of the same choice. - module = module or problem_file.get("module") - path, _imports, module_doc, _body = find_declaration(declaration, module) + problem_file, declaration, located = _resolve(problem, module) + path, _imports, module_doc, _body = located fc_module = module_name(path.relative_to(ROOT)) facts = elaborator_facts(fc_module, declaration) - if facts["range"] is None: + if facts.range is None: raise SystemExit(f"{declaration}: no source range recorded") explicit_dependencies, explicit_generated = explicit_copy_dependencies(problem_file) - facts["dependencies"] = merge_dependency_records( - explicit_dependencies, facts.get("dependencies", []) - ) - facts["generatedDependencies"] = list( - dict.fromkeys(explicit_generated + facts.get("generatedDependencies", [])) + facts = dataclasses.replace( + facts, + dependencies=tuple( + merge_dependency_records(explicit_dependencies, list(facts.dependencies)) + ), + generated_dependencies=tuple( + dict.fromkeys(explicit_generated + list(facts.generated_dependencies)) + ), ) return problem_file, declaration, path, fc_module, module_doc, facts @@ -434,7 +434,7 @@ def restate(original, declaration, facts, answer_type=None): # still meaningful, and the provenance sidecar records the FC name. declared, statement = flatten_declared_name(declared, statement) statement, holes = hoist_answers( - statement, declared, facts.get("answerTypes", []), answer_type + statement, declared, list(facts.answer_types), answer_type ) # A `research solved` statement carries its answer rather than a `sorry` # slot, so nothing above removed it and `answer(` would reach a workspace @@ -445,7 +445,7 @@ def restate(original, declaration, facts, answer_type=None): def explicit_arguments(facts, declared): """The explicit binders the Solution adapter applies by name.""" - args = [b["name"] for b in facts["binders"] if b["explicit"]] + args = [b["name"] for b in facts.binders if b["explicit"]] bad = [a for a in args if "✝" in a or "._" in a] if bad: raise SystemExit( @@ -515,12 +515,20 @@ def place_notations(dependencies, scope_text, statement, copied): return dependencies -def statement_pair(problem, module=None): - """The `(module, declaration)` pair `import_problem` will ask the elaborator about.""" +def _resolve(problem, module=None): + """The problem file, declaration and source location one import works on.""" problem_file = load_manifest(problem) declaration = problem_file.get("declaration", problem) + # An argument given on the command line is explicit, so it wins over the + # problem file; the file is the durable record of the same choice. module = module or problem_file.get("module") - path, _imports, _module_doc, _body = find_declaration(declaration, module) + located = find_declaration(declaration, module) + return problem_file, declaration, located + + +def statement_pair(problem, module=None): + """The `(module, declaration)` pair `import_problem` will ask the elaborator about.""" + _, declaration, (path, _imports, _doc, _body) = _resolve(problem, module) return module_name(path.relative_to(ROOT)), declaration @@ -535,14 +543,14 @@ def import_problem(problem, answer_type=None, module=None): problem, module ) source_lines = path.read_text(encoding="utf-8").split("\n") - original, lo = slice_range(source_lines, facts["range"]) + original, lo = slice_range(source_lines, facts.range) preamble, namespaces_at_target = file_scoped_preamble(source_lines, lo) dependencies, copied = closure_region( - facts.get("dependencies", []), - facts.get("generatedDependencies", []), + list(facts.dependencies), + list(facts.generated_dependencies), declaration, namespaces_at_target, - target_name=facts.get("name"), + target_name=facts.name, ) statement, declared, original_declared, holes = restate( original, declaration, facts, answer_type @@ -583,12 +591,12 @@ def import_problem(problem, answer_type=None, module=None): fc_module, path.relative_to(ROOT), fc_rev, - [dep["name"] for dep in facts.get("dependencies", [])], + [dep["name"] for dep in facts.dependencies], original, mathlib_rev, ), source_url=source_url, - category=facts.get("category") or "", + category=facts.category or "", ) return marked_up, manifest diff --git a/comparator/adapter/fc_source.py b/comparator/adapter/fc_source.py index 9c6cad7c0e..c19da70725 100644 --- a/comparator/adapter/fc_source.py +++ b/comparator/adapter/fc_source.py @@ -10,6 +10,7 @@ """ +import dataclasses import functools import json import pathlib @@ -44,6 +45,73 @@ r"|^noncomputable section\b" ) +@dataclasses.dataclass(frozen=True) +class FactsRecord: + """One declaration's elaborator facts, held to the payload the extractor emits. + + The fourth JSON boundary in the adapter, made as strict as the other + three: the provenance sidecar, the problem files and the failure ledger + all refuse keys nothing reads, and the extractor payload now does too, so + a drift between `comparator_facts` and this side fails at the seam + instead of surfacing as a missing default somewhere downstream. + """ + + declaration: str + name: str + category: str + range: dict + binders: tuple + answer_types: tuple + dependencies: tuple + generated_dependencies: tuple + + PAYLOAD_KEYS = frozenset( + { + "declaration", + "name", + "category", + "range", + "binders", + "answerTypes", + "dependencies", + "generatedDependencies", + } + ) + BINDER_KEYS = frozenset({"name", "explicit"}) + DEPENDENCY_KEYS = frozenset({"name", "module", "range"}) + + @classmethod + def from_payload(cls, payload, declaration): + unknown = sorted(set(payload) - cls.PAYLOAD_KEYS) + missing = sorted(cls.PAYLOAD_KEYS - set(payload)) + if unknown or missing: + raise SystemExit( + f"comparator_facts {declaration}: payload keys do not match the " + f"wire format (unknown: {unknown or 'none'}, " + f"missing: {missing or 'none'})" + ) + for binder in payload["binders"]: + if set(binder) != cls.BINDER_KEYS: + raise SystemExit( + f"comparator_facts {declaration}: malformed binder {binder}" + ) + for dep in payload["dependencies"]: + if set(dep) != cls.DEPENDENCY_KEYS: + raise SystemExit( + f"comparator_facts {declaration}: malformed dependency {dep}" + ) + return cls( + declaration=payload["declaration"], + name=payload["name"], + category=payload["category"], + range=payload["range"], + binders=tuple(payload["binders"]), + answer_types=tuple(payload["answerTypes"]), + dependencies=tuple(payload["dependencies"]), + generated_dependencies=tuple(payload["generatedDependencies"]), + ) + + _FACTS_CACHE = {} @@ -90,7 +158,7 @@ def elaborator_facts(module, declaration): """ cached = _FACTS_CACHE.get((module, declaration)) if cached is not None: - return json.loads(json.dumps(cached)) + return FactsRecord.from_payload(cached, declaration) proc = subprocess.run( ["lake", "exe", "comparator_facts", module, declaration], capture_output=True, @@ -105,7 +173,7 @@ def elaborator_facts(module, declaration): out = proc.stdout if "{" not in out: raise SystemExit(f"comparator_facts {declaration}: no JSON in output") - return json.loads(out[out.index("{") :]) + return FactsRecord.from_payload(json.loads(out[out.index("{") :]), declaration) def file_scoped_preamble(lines, start_line): """Directives in force at `start_line`, and the namespace stack there. diff --git a/comparator/adapter/test_fc_leaneval_importer.py b/comparator/adapter/test_fc_leaneval_importer.py index 311f526d69..600456dae2 100644 --- a/comparator/adapter/test_fc_leaneval_importer.py +++ b/comparator/adapter/test_fc_leaneval_importer.py @@ -352,18 +352,25 @@ def test_a_generated_constant_under_a_copied_parent_is_accepted(self): self.assertEqual(copied, [("Foo.bar", "def Foo.bar := 1")]) def test_an_explicit_source_only_dependency_carries_its_closure(self): - facts = { - "name": "Foo.opaqueLemma", - "range": {"startLine": 2, "endLine": 2, "endColumn": None}, - "dependencies": [ - { - "name": "Foo.Predicate", - "module": "FormalConjectures.Example", - "range": {"startLine": 1, "endLine": 1, "endColumn": None}, - } - ], - "generatedDependencies": ["Foo.opaqueLemma._proof_1"], - } + facts = fc_source.FactsRecord.from_payload( + { + "declaration": "Foo.opaqueLemma", + "name": "Foo.opaqueLemma", + "category": None, + "range": {"startLine": 2, "endLine": 2, "endColumn": None}, + "binders": [], + "answerTypes": [], + "dependencies": [ + { + "name": "Foo.Predicate", + "module": "FormalConjectures.Example", + "range": {"startLine": 1, "endLine": 1, "endColumn": None}, + } + ], + "generatedDependencies": ["Foo.opaqueLemma._proof_1"], + }, + "Foo.opaqueLemma", + ) with tempfile.TemporaryDirectory() as tmp, _root_at(tmp): module = pathlib.Path(tmp) / "FormalConjectures" / "Example.lean" module.parent.mkdir(parents=True) From bd282283515efaeeb7eaa0903379f8fb2a2e4357 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:31:55 -0400 Subject: [PATCH 55/70] Give the ledger loader its own module; split tests by module `compile_fc100_target.py` read the known-failures ledger by importing the other command, which dragged the whole importer graph into the target gate. The format's loader now lives in `known_failures.py`, imported by both. The test file covered two modules; `test_fc_source.py` now holds the source-reading cases, the wire-format cases join `test_leaneval_interface.py`, and `test_fc_leaneval_importer.py` keeps the assembly cases. No test changed, only its file. --- comparator/OWNERSHIP.md | 3 +- comparator/adapter/compile_fc100_target.py | 2 +- comparator/adapter/fc_source.py | 2 +- comparator/adapter/known_failures.py | 45 ++ .../adapter/make_comparator_workspace.py | 23 +- .../adapter/test_fc_leaneval_importer.py | 479 +----------------- comparator/adapter/test_fc_source.py | 452 +++++++++++++++++ comparator/adapter/test_leaneval_interface.py | 51 ++ .../adapter/test_make_comparator_workspace.py | 2 +- 9 files changed, 557 insertions(+), 502 deletions(-) create mode 100644 comparator/adapter/known_failures.py create mode 100644 comparator/adapter/test_fc_source.py diff --git a/comparator/OWNERSHIP.md b/comparator/OWNERSHIP.md index 2d2952de2f..4224a6d01f 100644 --- a/comparator/OWNERSHIP.md +++ b/comparator/OWNERSHIP.md @@ -87,7 +87,8 @@ Conjectures corrects a misformalisation upstream. | `comparator/adapter/ComparatorFacts/` and `comparator_facts.lean` | the Lean extractor (a small library — `Binders.lean` recovers declaration-header binder boundaries from source syntax, `Extract.lean` reads the elaborated environment — and a thin executable): source ranges, declaration-header binder boundaries, elaborated binder names/explicitness, answer-slot types (anywhere in the statement, hypothesis binders included), and the `@[category ...]` tag. One `module declaration` pair per invocation, or `--batch` pairs on stdin sharing a single environment, one JSON object per line in input order. The parsed source distinguishes header parameters from `∀` binders in the conclusion; every emitted binder fact still comes from the elaborated environment. | | `comparator/adapter/leaneval_interface.py` | the request builder and response checker — the FC side of the wire format, permanently, since the consumer owns hole resolution under the schema-version-1 contract | | `comparator/adapter/leaneval_generator_cli.py` | plumbing for the pinned binary | -| `comparator/adapter/make_comparator_workspace.py` | the command, the emitted seam artifact, and the whole-set batch run | +| `comparator/adapter/make_comparator_workspace.py` | the command, the emitted seam artifact, and the whole-set batch run| +| `comparator/adapter/known_failures.py` | the known-failures ledger's format and loader, shared by the set run and the target-stage compile | | `comparator/templates/WorkspaceTest.lean` | the workspace test template the contract requires the consumer to supply | | `comparator/problems/*.toml` | the rare source-boundary facts the compiled environment cannot recover: which module when two declare the same name, and an explicit copied proof dependency when opaque theorem-value erasure removes it from the compiled dependency graph | | `comparator/tools.toml` | the pins, in one machine-readable place: this repository's under `[tools]`, LeanEval's under `[target]`, the generator revision under `[generator]` | diff --git a/comparator/adapter/compile_fc100_target.py b/comparator/adapter/compile_fc100_target.py index f906400404..fc8005e2db 100644 --- a/comparator/adapter/compile_fc100_target.py +++ b/comparator/adapter/compile_fc100_target.py @@ -30,7 +30,7 @@ import tomllib from leaneval_interface import lean_errors, dump_json -from make_comparator_workspace import load_known_failures +from known_failures import load_known_failures def arrange_project(workspaces_dir, project_dir): diff --git a/comparator/adapter/fc_source.py b/comparator/adapter/fc_source.py index c19da70725..1b941fce3c 100644 --- a/comparator/adapter/fc_source.py +++ b/comparator/adapter/fc_source.py @@ -282,7 +282,7 @@ def _declared_names(): def _declaring_files(name): - """The files whose text declares `name` as a theorem or lemma. + r"""The files whose text declares `name` as a theorem or lemma. A declared token matches when it equals `name` or ends in `.name` — the same reading as the old per-file regex, whose optional prefix was diff --git a/comparator/adapter/known_failures.py b/comparator/adapter/known_failures.py new file mode 100644 index 0000000000..15a7031bd3 --- /dev/null +++ b/comparator/adapter/known_failures.py @@ -0,0 +1,45 @@ +# Copyright 2026 The Formal Conjectures Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The known-failures ledger: `comparator/known_failures.toml` and its loader. + +Both gates read the ledger — the source-side set run and the target-side +compile — and each matches it exactly: an unexpected failure and a silently +fixed one both fail. The loader lives here so neither command has to import +the other to read the format. +""" + +import tomllib + +def load_known_failures(path): + """The recorded failures, `{declaration: {stage, reason}}`.""" + with open(path, "rb") as handle: + data = tomllib.load(handle) + failures = {} + for entry in data.get("failure", []): + for field in ("declaration", "stage", "reason"): + if field not in entry: + raise SystemExit(f"{path}: a failure entry has no `{field}`") + if entry["stage"] not in ("source", "target"): + raise SystemExit( + f"{path}: {entry['declaration']} has stage {entry['stage']!r}; " + "expected source or target" + ) + if entry["stage"] == "target" and "workspace" not in entry: + raise SystemExit( + f"{path}: {entry['declaration']} is a target failure without a " + "`workspace`; the target gate matches by workspace id" + ) + failures[entry["declaration"]] = entry + return failures diff --git a/comparator/adapter/make_comparator_workspace.py b/comparator/adapter/make_comparator_workspace.py index 4718a417b2..58d4495322 100644 --- a/comparator/adapter/make_comparator_workspace.py +++ b/comparator/adapter/make_comparator_workspace.py @@ -64,8 +64,8 @@ import shutil import sys import tempfile -import tomllib +from known_failures import load_known_failures import fc_leaneval_importer as importer import fc_source import leaneval_generator_cli as generator_cli @@ -200,27 +200,6 @@ def subset_declarations(set_name): return names -def load_known_failures(path): - """The recorded failures, `{declaration: {stage, reason}}`.""" - with open(path, "rb") as handle: - data = tomllib.load(handle) - failures = {} - for entry in data.get("failure", []): - for field in ("declaration", "stage", "reason"): - if field not in entry: - raise SystemExit(f"{path}: a failure entry has no `{field}`") - if entry["stage"] not in ("source", "target"): - raise SystemExit( - f"{path}: {entry['declaration']} has stage {entry['stage']!r}; " - "expected source or target" - ) - if entry["stage"] == "target" and "workspace" not in entry: - raise SystemExit( - f"{path}: {entry['declaration']} is a target failure without a " - "`workspace`; the target gate matches by workspace id" - ) - failures[entry["declaration"]] = entry - return failures def import_set(set_name, out_dir, verify=False, known_failures=None): diff --git a/comparator/adapter/test_fc_leaneval_importer.py b/comparator/adapter/test_fc_leaneval_importer.py index 600456dae2..1687223297 100644 --- a/comparator/adapter/test_fc_leaneval_importer.py +++ b/comparator/adapter/test_fc_leaneval_importer.py @@ -29,180 +29,8 @@ import fc_leaneval_importer as importer import fc_source -from leaneval_interface import MarkedUpModule, problem_group -from test_leaneval_interface import a_manifest from fc_leaneval_importer import closure_region, load_manifest -from fc_source import ( - answer_spans, - file_scoped_preamble, - hoist_answers, - pins, - replace_proof_with_sorry, - strip_decorations, - strip_fc_attributes, - unwrap_answers, -) - - -class HoistTest(unittest.TestCase): - """Slot types come from the elaborated environment.""" - - def test_slot_takes_the_environment_type(self): - stmt, holes = hoist_answers( - "theorem t : answer(sorry) ↔ ∀ n, n ≤ n := by\n sorry", "t", ["Prop"] - ) - self.assertIn("t_answer", stmt) - self.assertEqual( - holes[0].declaration(), "noncomputable def t_answer : Prop := sorry" - ) - - def test_erased_slot_is_prop_by_the_elaborators_rule(self): - # The default `alwaysTrue` setting erases a slot iff its expected - # type is Prop, so a missing annotation names the type exactly. - _, holes = hoist_answers("theorem t : answer(sorry) ↔ P := by\n sorry", "t", []) - self.assertEqual( - holes[0].declaration(), "noncomputable def t_answer : Prop := sorry" - ) - - def test_mixed_prop_and_typed_slots_are_refused(self): - with self.assertRaises(SystemExit): - hoist_answers( - "theorem t : answer(sorry) ∧ (answer(sorry) = 3) := by\n sorry", - "t", - ["Nat"], - ) - - def test_non_prop_type_is_read_not_guessed(self): - _, holes = hoist_answers( - "theorem t : sSup S = answer(sorry) := by\n sorry", "t", ["ENNReal"] - ) - self.assertEqual(holes[0].type, "ENNReal") - - def test_override_wins(self): - _, holes = hoist_answers( - "theorem t : sSup S = answer(sorry) := by\n sorry", "t", ["ENNReal"], "ℝ" - ) - self.assertEqual(holes[0].type, "ℝ") - - def test_differing_slot_types_are_refused(self): - # Matching types to positions would be a guess. - with self.assertRaises(SystemExit): - hoist_answers( - "theorem t : answer(sorry) = answer(sorry) := by\n sorry", - "t", - ["Nat", "Int"], - ) - - def test_no_slot_is_left_alone(self): - stmt, holes = hoist_answers("theorem t : True := by\n sorry", "t", []) - self.assertEqual(holes, []) - - def test_fixed_answer_is_not_turned_into_a_hole(self): - original = "theorem t : IsGLB S answer(2) := by\n sorry" - unchanged, holes = hoist_answers(original, "t", ["ENNReal"]) - self.assertEqual(unchanged, original) - self.assertEqual(holes, []) - - def test_nested_answer_term_is_one_balanced_slot(self): - calls = answer_spans("theorem t : f answer((fun x => x) (g 2)) := by\n sorry") - self.assertEqual(len(calls), 1) - self.assertEqual(calls[0][2], "(fun x => x) (g 2)") - - def test_answer_text_in_comments_and_strings_is_ignored(self): - calls = answer_spans( - '-- answer(1)\ntheorem t : p "answer(2)" answer(3) := by sorry' - ) - self.assertEqual(len(calls), 1) - self.assertEqual(calls[0][2], "3") - - -class PreambleTest(unittest.TestCase): - """Only directives in force at the statement are carried.""" - - def test_variable_in_a_closed_section_is_dropped(self): - lines = [ - "section S", - "variable {n : Nat}", - "end S", - "", - "open Nat", - "", - "theorem t : True := trivial", - ] - pre, ns = file_scoped_preamble(lines, 7) - self.assertEqual(pre, ["open Nat"]) - self.assertEqual(ns, []) - - def test_namespace_stack_is_reported(self): - lines = ["namespace A", "open Nat", "theorem t : True := trivial"] - pre, ns = file_scoped_preamble(lines, 3) - self.assertEqual(pre, ["open Nat"]) - self.assertEqual(ns, ["A"]) - - def test_directive_inside_a_comment_is_not_a_directive(self): - lines = ["/--", "open the door", "-/", "theorem t : True := trivial"] - pre, _ = file_scoped_preamble(lines, 4) - self.assertEqual(pre, []) - - -class StatementTest(unittest.TestCase): - def test_proof_is_replaced_but_statement_kept(self): - out = replace_proof_with_sorry( - "theorem t : True := by\n have h := trivial\n exact h" - ) - self.assertIn("theorem t : True", out) - self.assertNotIn("have h", out) - self.assertTrue(out.rstrip().endswith("sorry")) - - def test_term_mode_proof_is_replaced_too(self): - out = replace_proof_with_sorry("theorem t : True := trivial") - self.assertNotIn("trivial", out) - self.assertTrue(out.rstrip().endswith("sorry")) - - def test_structure_literal_assign_is_statement_text(self): - # `{ a := 1 }` lives inside brackets; only the top-level `:=` starts - # the proof, so the statement survives intact. - out = replace_proof_with_sorry("theorem t : F { a := 1 } := ⟨rfl⟩") - self.assertIn("F { a := 1 }", out) - self.assertNotIn("⟨rfl⟩", out) - self.assertTrue(out.rstrip().endswith("sorry")) - - def test_autoparam_default_is_statement_text(self): - # An autoParam binder carries `:= by` inside its parentheses; the - # proof is the top-level one. - out = replace_proof_with_sorry( - "theorem t (h : Fact (1 < 2) := by norm_num) : True := by trivial" - ) - self.assertIn(":= by norm_num", out) - self.assertNotIn("trivial", out) - - def test_two_top_level_assigns_are_refused(self): - with self.assertRaises(SystemExit): - replace_proof_with_sorry("def t : Nat := f := g") - - def test_a_line_comment_between_docstring_and_attribute_is_stripped(self): - # Erdos 918 writes a `--` formalisation note there. One anchored pass - # each left `@[category research open]` on the statement, and Lean - # parsed as far as the `open` inside it. - out = strip_decorations( - "/-- doc -/\n-- note\n@[category research open, AMS 5]\n" - "theorem t : True := by\n sorry" - ) - self.assertTrue(out.startswith("theorem")) - - def test_open_in_survives_stripping(self): - # It binds to the declaration, and it sits above the docstring. - out = strip_decorations( - "open scoped Classical in\n/-- doc -/\n@[category research open]\n" - "theorem t : True := by\n sorry" - ) - self.assertTrue(out.startswith("open scoped Classical in\ntheorem")) - - def test_decorations_are_stripped_from_the_target(self): - out = strip_decorations( - "/-- doc -/\n@[category research open]\ntheorem t : True := by\n sorry" - ) - self.assertTrue(out.startswith("theorem")) +from fc_source import pins, strip_fc_attributes, unwrap_answers class ProblemFileTest(unittest.TestCase): @@ -498,306 +326,5 @@ def test_the_closure_region_does_not_carry_the_import(self): unittest.main() -class DocstringReferenceTest(unittest.TestCase): - """The source citation is read from Formal Conjectures, not copied. - - A hand-kept copy drifts. The Margulis module's docstring pins - `arxiv/2504.17644v3`; the problem file that used to carry the same - citation had the unversioned URL, so the copy was already less exact - than the docstring it was copied from. - """ - - def test_the_first_reference_link_is_the_citation(self): - doc = ( - "/-!\n# Erdős Problem 1038\n\n*Reference:*\n" - " - [erdosproblems.com/1038](https://www.erdosproblems.com/1038)\n" - " - [Tao25] a blog post (https://example.com/other)\n-/" - ) - self.assertEqual( - fc_source.docstring_reference(doc), "https://www.erdosproblems.com/1038" - ) - - def test_an_arxiv_version_suffix_is_preserved(self): - doc = "/-!\n*Reference:* [arxiv/2504.17644v3](https://arxiv.org/abs/2504.17644v3)\n-/" - self.assertEqual( - fc_source.docstring_reference(doc), "https://arxiv.org/abs/2504.17644v3" - ) - - def test_links_above_the_reference_line_are_not_the_citation(self): - doc = "/-!\n# A problem\n\nSee [Mathlib](https://leanprover-community.github.io).\n-/" - self.assertEqual(fc_source.docstring_reference(doc), "") - - def test_a_module_without_a_docstring_has_no_citation(self): - self.assertEqual(fc_source.docstring_reference(""), "") - - -class ModuleNameCodecTest(unittest.TestCase): - """`module_name` and `module_source_path` are inverse on real modules.""" - - def test_a_guillemet_component_keeps_its_dots(self): - self.assertEqual( - fc_source.split_module( - "FormalConjectures.Arxiv.«0912.2382».CurlingNumberConjecture" - ), - ["FormalConjectures", "Arxiv", "0912.2382", "CurlingNumberConjecture"], - ) - - def test_a_dotted_final_component_keeps_its_tail(self): - # `with_suffix` would have turned `«2501.03234»` into `«2501.lean`. - path = fc_source.module_source_path( - "FormalConjectures.Arxiv.«2501.03234».ArithmeticSumS" - ) - self.assertEqual(path.name, "ArithmeticSumS.lean") - self.assertEqual(path.parent.name, "2501.03234") - - def test_a_malformed_name_is_refused(self): - with self.assertRaises(SystemExit): - fc_source.split_module("FormalConjectures.«unterminated") - - def test_every_real_module_round_trips(self): - # The property that keeps the codec from drifting again: for every - # file the importer can name, decoding the name reaches the file. - for src in fc_source.SOURCE_DIRS: - for path in src.rglob("*.lean"): - rel = path.relative_to(importer.ROOT) - with self.subTest(module=str(rel)): - self.assertEqual( - fc_source.module_source_path(fc_source.module_name(rel)), path - ) - - -class QualifiedResolutionTest(unittest.TestCase): - """Qualified requests resolve through the namespace stack.""" - - def test_the_bare_colliding_name_is_ambiguous(self): - with self.assertRaises(SystemExit) as ctx: - fc_source.find_declaration("conjecture") - self.assertIn("ambiguous", str(ctx.exception)) - - def test_each_qualified_name_reaches_its_own_file(self): - for qualified, filename in ( - ("OeisA303656.conjecture", "303656.lean"), - ("OeisA308734.conjecture", "308734.lean"), - ): - with self.subTest(qualified=qualified): - path, _, _, _ = fc_source.find_declaration(qualified) - self.assertEqual(path.name, filename) - - def test_a_declared_name_with_dots_still_resolves(self): - # The declared name itself contains dots; no namespace split applies. - path, _, _, _ = fc_source.find_declaration( - "erdos_125.variants.positive_unequal_density" - ) - self.assertEqual(path.name, "125.lean") - - def test_longest_declared_suffix_wins(self): - # `Erdos125.erdos_125.variants.positive_unequal_density`: the first - # component is the namespace, the rest is the declared name. - path, _, _, _ = fc_source.find_declaration( - "Erdos125.erdos_125.variants.positive_unequal_density" - ) - self.assertEqual(path.name, "125.lean") - - -class ProblemGroupTest(unittest.TestCase): - """Categories map to lean-eval groups; non-problems are refused.""" - - def _manifest(self, category): - manifest = mock.Mock() - manifest.category = category - manifest.id = "some_problem" - return manifest - - def test_open_research_is_an_open_conjecture(self): - self.assertEqual( - problem_group(self._manifest("research open")), - "open-conjectures", - ) - - def test_settled_statements_are_evaluation_material(self): - for category in ("research solved", "textbook", "test"): - with self.subTest(category=category): - self.assertEqual( - problem_group(self._manifest(category)), - "formalization-evaluation", - ) - - def test_api_and_untagged_declarations_are_refused(self): - for category in ("API", ""): - with self.subTest(category=category): - with self.assertRaises(SystemExit): - problem_group(self._manifest(category)) - - -class FlattenDeclaredNameTest(unittest.TestCase): - """Dotted declaration names are restated as slugs for the generator.""" - - def test_the_declaring_occurrence_is_renamed(self): - name, statement = fc_source.flatten_declared_name( - "erdos_100.variants.strong", - "theorem erdos_100.variants.strong : True := by\n sorry", - ) - self.assertEqual(name, "erdos_100_variants_strong") - self.assertEqual( - statement, "theorem erdos_100_variants_strong : True := by\n sorry" - ) - - def test_a_prefix_line_does_not_confuse_the_rename(self): - # `open X in` binds to the declaration below and travels with the - # slice; the declaring line is not the first line. - name, statement = fc_source.flatten_declared_name( - "a.b", "open Nat in\ntheorem a.b : True := by\n sorry" - ) - self.assertEqual(name, "a_b") - self.assertIn("theorem a_b :", statement) - - def test_an_absent_declaration_is_refused(self): - with self.assertRaises(SystemExit): - fc_source.flatten_declared_name("a.b", "theorem c.d : True := sorry") - - -class PreambleNotationTest(unittest.TestCase): - """File-scoped notation and macros travel with the preamble.""" - - def test_local_notation_is_kept(self): - # Irrational.lean: dropping `local notation "e" => exp 1` left `e` - # to auto-bind as an implicit at FC pins and fail at LeanEval's. - lines = ['local notation "e" => exp 1', "theorem t : True := trivial"] - pre, _ = file_scoped_preamble(lines, 2) - self.assertEqual(pre, ['local notation "e" => exp 1']) - - def test_a_macro_keeps_its_indented_body(self): - # Poincare.lean: the 𝕊ⁿ macro's body is on the next line; one kept - # line would be broken syntax. - lines = [ - 'local macro:max "𝕊" noWs n:superscript(term) : term =>', - " `(Metric.sphere 0 1)", - "theorem t : True := trivial", - ] - pre, _ = file_scoped_preamble(lines, 3) - self.assertEqual(len(pre), 1) - self.assertIn("`(Metric.sphere 0 1)", pre[0]) - - def test_noncomputable_section_is_restated(self): - # OpenQuantumProblems/23: a copied def that was total inside - # `noncomputable section` fails to compile outside it. - lines = ["noncomputable section", "theorem t : True := trivial"] - pre, _ = file_scoped_preamble(lines, 2) - self.assertIn("noncomputable section", pre) - - def test_a_closed_noncomputable_section_is_not_restated(self): - lines = ["noncomputable section", "end", "theorem t : True := trivial"] - pre, _ = file_scoped_preamble(lines, 3) - self.assertNotIn("noncomputable section", pre) - - -class AscribedSlotTest(unittest.TestCase): - """`(answer(sorry) : T)` states its own type at its own position.""" - - def test_the_ascription_wins_over_the_erasure_rule(self): - # Erdos332: the annotation for an ascribed-and-applied slot does not - # survive elaboration, so the environment reports nothing and the - # erasure rule would call it Prop. - statement = ( - "theorem erdos_332 (A : Set ℕ) : " - "(answer(sorry) : Set ℕ → Prop) A → True := by\n sorry" - ) - _, holes = hoist_answers(statement, "erdos_332", []) - self.assertEqual(holes[0].type, "Set ℕ → Prop") - - def test_a_nested_paren_type_stays_whole(self): - statement = "theorem t : (answer(sorry) : (ℕ → ℕ) → Prop) f := by\n sorry" - _, holes = hoist_answers(statement, "t", []) - self.assertEqual(holes[0].type, "(ℕ → ℕ) → Prop") - - def test_an_unascribed_slot_still_follows_the_erasure_rule(self): - statement = "theorem t : answer(sorry) ↔ True := by\n sorry" - _, holes = hoist_answers(statement, "t", []) - self.assertEqual(holes[0].type, "Prop") - - -class NotationBlocksTest(unittest.TestCase): - """FC-defined notation is copied only where it was in force.""" - - def _with_commands(self, commands): - return mock.patch.object( - fc_source, "fc_notation_commands", return_value=commands - ) - - def test_a_scoped_notation_needs_its_namespace_opened(self): - commands = [ - (["ℝ²"], 'scoped[EuclideanGeometry] notation "ℝ²" => E', "EuclideanGeometry", True), - ] - with self._with_commands(commands): - self.assertEqual( - fc_source.notation_blocks(["def f : ℝ² := sorry"], {"EuclideanGeometry"}), - ['scoped[EuclideanGeometry] notation "ℝ²" => E'], - ) - # Green9's `⊆` false positive: same token, namespace never opened. - self.assertEqual( - fc_source.notation_blocks(["def f : ℝ² := sorry"], set()), [] - ) - - def test_a_shared_global_notation_is_copied_as_local(self): - # Global would be declared in ChallengeDeps and re-extracted into - # the importing file too; `local` keeps each copy to its own file. - commands = [(["≪"], 'notation g " ≪ " f => IsBigO g f', None, True)] - with self._with_commands(commands): - self.assertEqual( - fc_source.notation_blocks(["theorem t : a ≪ b := sorry"], set()), - ['local notation g " ≪ " f => IsBigO g f'], - ) - - def test_a_problem_module_global_notation_is_never_copied(self): - commands = [(["≪"], 'notation g " ≪ " f => X g f', None, False)] - with self._with_commands(commands): - self.assertEqual( - fc_source.notation_blocks(["theorem t : a ≪ b := sorry"], set()), [] - ) - - def test_an_unused_token_is_not_copied(self): - commands = [(["ℝ²"], 'notation "ℝ²" => E', None, True)] - with self._with_commands(commands): - self.assertEqual(fc_source.notation_blocks(["theorem t : True"], set()), []) - - -class LocaliseNotationTest(unittest.TestCase): - def test_a_global_notation_becomes_local(self): - self.assertEqual( - fc_source.localise_notation(['notation "R(" k ")" => f k']), - ['local notation "R(" k ")" => f k'], - ) - - def test_quot_precheck_travels_with_the_notation(self): - out = fc_source.localise_notation( - ["set_option quotPrecheck false", 'local notation "A" => s'] - ) - self.assertEqual( - out[1], - 'set_option quotPrecheck false in\nlocal notation "A" => s', - ) - - def test_other_preamble_lines_pass_through(self): - self.assertEqual( - fc_source.localise_notation(["open Nat", "variable (n : Nat)"]), - ["open Nat", "variable (n : Nat)"], - ) - - -class StatementPrefixSpanTest(unittest.TestCase): - def test_the_span_starts_at_the_declaration_keyword(self): - from leaneval_interface import module_declarations - - module = MarkedUpModule( - dependencies="def Foo.bar := 1", - scope="", - holes="", - statement="open scoped Classical in\ntheorem t : True := by\n sorry", - dependency_declarations=(("Foo.bar", "def Foo.bar := 1"),), - ) - manifest = a_manifest( - theorem="t", qualified_theorem="t", holes=(), apply_arguments=() - ) - *_, statement_entry = module_declarations(module, manifest) - self.assertTrue(statement_entry[1].startswith("theorem t")) - self.assertNotIn("Classical in", statement_entry[1]) +if __name__ == "__main__": + unittest.main() diff --git a/comparator/adapter/test_fc_source.py b/comparator/adapter/test_fc_source.py new file mode 100644 index 0000000000..5d149ed1b0 --- /dev/null +++ b/comparator/adapter/test_fc_source.py @@ -0,0 +1,452 @@ +# Copyright 2026 The Formal Conjectures Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline tests for reading this repository's own Lean source. + +Every case pins a failure a real import produced, or a rule whose violation +would copy source that elaborates but poses the wrong problem. +""" + +import unittest +from unittest import mock + +import fc_source +from fc_source import ( + answer_spans, + file_scoped_preamble, + hoist_answers, + replace_proof_with_sorry, + strip_decorations, +) + + +class HoistTest(unittest.TestCase): + """Slot types come from the elaborated environment.""" + + def test_slot_takes_the_environment_type(self): + stmt, holes = hoist_answers( + "theorem t : answer(sorry) ↔ ∀ n, n ≤ n := by\n sorry", "t", ["Prop"] + ) + self.assertIn("t_answer", stmt) + self.assertEqual( + holes[0].declaration(), "noncomputable def t_answer : Prop := sorry" + ) + + def test_erased_slot_is_prop_by_the_elaborators_rule(self): + # The default `alwaysTrue` setting erases a slot iff its expected + # type is Prop, so a missing annotation names the type exactly. + _, holes = hoist_answers("theorem t : answer(sorry) ↔ P := by\n sorry", "t", []) + self.assertEqual( + holes[0].declaration(), "noncomputable def t_answer : Prop := sorry" + ) + + def test_mixed_prop_and_typed_slots_are_refused(self): + with self.assertRaises(SystemExit): + hoist_answers( + "theorem t : answer(sorry) ∧ (answer(sorry) = 3) := by\n sorry", + "t", + ["Nat"], + ) + + def test_non_prop_type_is_read_not_guessed(self): + _, holes = hoist_answers( + "theorem t : sSup S = answer(sorry) := by\n sorry", "t", ["ENNReal"] + ) + self.assertEqual(holes[0].type, "ENNReal") + + def test_override_wins(self): + _, holes = hoist_answers( + "theorem t : sSup S = answer(sorry) := by\n sorry", "t", ["ENNReal"], "ℝ" + ) + self.assertEqual(holes[0].type, "ℝ") + + def test_differing_slot_types_are_refused(self): + # Matching types to positions would be a guess. + with self.assertRaises(SystemExit): + hoist_answers( + "theorem t : answer(sorry) = answer(sorry) := by\n sorry", + "t", + ["Nat", "Int"], + ) + + def test_no_slot_is_left_alone(self): + stmt, holes = hoist_answers("theorem t : True := by\n sorry", "t", []) + self.assertEqual(holes, []) + + def test_fixed_answer_is_not_turned_into_a_hole(self): + original = "theorem t : IsGLB S answer(2) := by\n sorry" + unchanged, holes = hoist_answers(original, "t", ["ENNReal"]) + self.assertEqual(unchanged, original) + self.assertEqual(holes, []) + + def test_nested_answer_term_is_one_balanced_slot(self): + calls = answer_spans("theorem t : f answer((fun x => x) (g 2)) := by\n sorry") + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0][2], "(fun x => x) (g 2)") + + def test_answer_text_in_comments_and_strings_is_ignored(self): + calls = answer_spans( + '-- answer(1)\ntheorem t : p "answer(2)" answer(3) := by sorry' + ) + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0][2], "3") + + +class PreambleTest(unittest.TestCase): + """Only directives in force at the statement are carried.""" + + def test_variable_in_a_closed_section_is_dropped(self): + lines = [ + "section S", + "variable {n : Nat}", + "end S", + "", + "open Nat", + "", + "theorem t : True := trivial", + ] + pre, ns = file_scoped_preamble(lines, 7) + self.assertEqual(pre, ["open Nat"]) + self.assertEqual(ns, []) + + def test_namespace_stack_is_reported(self): + lines = ["namespace A", "open Nat", "theorem t : True := trivial"] + pre, ns = file_scoped_preamble(lines, 3) + self.assertEqual(pre, ["open Nat"]) + self.assertEqual(ns, ["A"]) + + def test_directive_inside_a_comment_is_not_a_directive(self): + lines = ["/--", "open the door", "-/", "theorem t : True := trivial"] + pre, _ = file_scoped_preamble(lines, 4) + self.assertEqual(pre, []) + + +class StatementTest(unittest.TestCase): + def test_proof_is_replaced_but_statement_kept(self): + out = replace_proof_with_sorry( + "theorem t : True := by\n have h := trivial\n exact h" + ) + self.assertIn("theorem t : True", out) + self.assertNotIn("have h", out) + self.assertTrue(out.rstrip().endswith("sorry")) + + def test_term_mode_proof_is_replaced_too(self): + out = replace_proof_with_sorry("theorem t : True := trivial") + self.assertNotIn("trivial", out) + self.assertTrue(out.rstrip().endswith("sorry")) + + def test_structure_literal_assign_is_statement_text(self): + # `{ a := 1 }` lives inside brackets; only the top-level `:=` starts + # the proof, so the statement survives intact. + out = replace_proof_with_sorry("theorem t : F { a := 1 } := ⟨rfl⟩") + self.assertIn("F { a := 1 }", out) + self.assertNotIn("⟨rfl⟩", out) + self.assertTrue(out.rstrip().endswith("sorry")) + + def test_autoparam_default_is_statement_text(self): + # An autoParam binder carries `:= by` inside its parentheses; the + # proof is the top-level one. + out = replace_proof_with_sorry( + "theorem t (h : Fact (1 < 2) := by norm_num) : True := by trivial" + ) + self.assertIn(":= by norm_num", out) + self.assertNotIn("trivial", out) + + def test_two_top_level_assigns_are_refused(self): + with self.assertRaises(SystemExit): + replace_proof_with_sorry("def t : Nat := f := g") + + def test_a_line_comment_between_docstring_and_attribute_is_stripped(self): + # Erdos 918 writes a `--` formalisation note there. One anchored pass + # each left `@[category research open]` on the statement, and Lean + # parsed as far as the `open` inside it. + out = strip_decorations( + "/-- doc -/\n-- note\n@[category research open, AMS 5]\n" + "theorem t : True := by\n sorry" + ) + self.assertTrue(out.startswith("theorem")) + + def test_open_in_survives_stripping(self): + # It binds to the declaration, and it sits above the docstring. + out = strip_decorations( + "open scoped Classical in\n/-- doc -/\n@[category research open]\n" + "theorem t : True := by\n sorry" + ) + self.assertTrue(out.startswith("open scoped Classical in\ntheorem")) + + def test_decorations_are_stripped_from_the_target(self): + out = strip_decorations( + "/-- doc -/\n@[category research open]\ntheorem t : True := by\n sorry" + ) + self.assertTrue(out.startswith("theorem")) + + +class DocstringReferenceTest(unittest.TestCase): + """The source citation is read from Formal Conjectures, not copied. + + A hand-kept copy drifts. The Margulis module's docstring pins + `arxiv/2504.17644v3`; the problem file that used to carry the same + citation had the unversioned URL, so the copy was already less exact + than the docstring it was copied from. + """ + + def test_the_first_reference_link_is_the_citation(self): + doc = ( + "/-!\n# Erdős Problem 1038\n\n*Reference:*\n" + " - [erdosproblems.com/1038](https://www.erdosproblems.com/1038)\n" + " - [Tao25] a blog post (https://example.com/other)\n-/" + ) + self.assertEqual( + fc_source.docstring_reference(doc), "https://www.erdosproblems.com/1038" + ) + + def test_an_arxiv_version_suffix_is_preserved(self): + doc = "/-!\n*Reference:* [arxiv/2504.17644v3](https://arxiv.org/abs/2504.17644v3)\n-/" + self.assertEqual( + fc_source.docstring_reference(doc), "https://arxiv.org/abs/2504.17644v3" + ) + + def test_links_above_the_reference_line_are_not_the_citation(self): + doc = "/-!\n# A problem\n\nSee [Mathlib](https://leanprover-community.github.io).\n-/" + self.assertEqual(fc_source.docstring_reference(doc), "") + + def test_a_module_without_a_docstring_has_no_citation(self): + self.assertEqual(fc_source.docstring_reference(""), "") + + +class ModuleNameCodecTest(unittest.TestCase): + """`module_name` and `module_source_path` are inverse on real modules.""" + + def test_a_guillemet_component_keeps_its_dots(self): + self.assertEqual( + fc_source.split_module( + "FormalConjectures.Arxiv.«0912.2382».CurlingNumberConjecture" + ), + ["FormalConjectures", "Arxiv", "0912.2382", "CurlingNumberConjecture"], + ) + + def test_a_dotted_final_component_keeps_its_tail(self): + # `with_suffix` would have turned `«2501.03234»` into `«2501.lean`. + path = fc_source.module_source_path( + "FormalConjectures.Arxiv.«2501.03234».ArithmeticSumS" + ) + self.assertEqual(path.name, "ArithmeticSumS.lean") + self.assertEqual(path.parent.name, "2501.03234") + + def test_a_malformed_name_is_refused(self): + with self.assertRaises(SystemExit): + fc_source.split_module("FormalConjectures.«unterminated") + + def test_every_real_module_round_trips(self): + # The property that keeps the codec from drifting again: for every + # file the importer can name, decoding the name reaches the file. + for src in fc_source.SOURCE_DIRS: + for path in src.rglob("*.lean"): + rel = path.relative_to(fc_source.ROOT) + with self.subTest(module=str(rel)): + self.assertEqual( + fc_source.module_source_path(fc_source.module_name(rel)), path + ) + + +class QualifiedResolutionTest(unittest.TestCase): + """Qualified requests resolve through the namespace stack.""" + + def test_the_bare_colliding_name_is_ambiguous(self): + with self.assertRaises(SystemExit) as ctx: + fc_source.find_declaration("conjecture") + self.assertIn("ambiguous", str(ctx.exception)) + + def test_each_qualified_name_reaches_its_own_file(self): + for qualified, filename in ( + ("OeisA303656.conjecture", "303656.lean"), + ("OeisA308734.conjecture", "308734.lean"), + ): + with self.subTest(qualified=qualified): + path, _, _, _ = fc_source.find_declaration(qualified) + self.assertEqual(path.name, filename) + + def test_a_declared_name_with_dots_still_resolves(self): + # The declared name itself contains dots; no namespace split applies. + path, _, _, _ = fc_source.find_declaration( + "erdos_125.variants.positive_unequal_density" + ) + self.assertEqual(path.name, "125.lean") + + def test_longest_declared_suffix_wins(self): + # `Erdos125.erdos_125.variants.positive_unequal_density`: the first + # component is the namespace, the rest is the declared name. + path, _, _, _ = fc_source.find_declaration( + "Erdos125.erdos_125.variants.positive_unequal_density" + ) + self.assertEqual(path.name, "125.lean") + + +class FlattenDeclaredNameTest(unittest.TestCase): + """Dotted declaration names are restated as slugs for the generator.""" + + def test_the_declaring_occurrence_is_renamed(self): + name, statement = fc_source.flatten_declared_name( + "erdos_100.variants.strong", + "theorem erdos_100.variants.strong : True := by\n sorry", + ) + self.assertEqual(name, "erdos_100_variants_strong") + self.assertEqual( + statement, "theorem erdos_100_variants_strong : True := by\n sorry" + ) + + def test_a_prefix_line_does_not_confuse_the_rename(self): + # `open X in` binds to the declaration below and travels with the + # slice; the declaring line is not the first line. + name, statement = fc_source.flatten_declared_name( + "a.b", "open Nat in\ntheorem a.b : True := by\n sorry" + ) + self.assertEqual(name, "a_b") + self.assertIn("theorem a_b :", statement) + + def test_an_absent_declaration_is_refused(self): + with self.assertRaises(SystemExit): + fc_source.flatten_declared_name("a.b", "theorem c.d : True := sorry") + + +class PreambleNotationTest(unittest.TestCase): + """File-scoped notation and macros travel with the preamble.""" + + def test_local_notation_is_kept(self): + # Irrational.lean: dropping `local notation "e" => exp 1` left `e` + # to auto-bind as an implicit at FC pins and fail at LeanEval's. + lines = ['local notation "e" => exp 1', "theorem t : True := trivial"] + pre, _ = file_scoped_preamble(lines, 2) + self.assertEqual(pre, ['local notation "e" => exp 1']) + + def test_a_macro_keeps_its_indented_body(self): + # Poincare.lean: the 𝕊ⁿ macro's body is on the next line; one kept + # line would be broken syntax. + lines = [ + 'local macro:max "𝕊" noWs n:superscript(term) : term =>', + " `(Metric.sphere 0 1)", + "theorem t : True := trivial", + ] + pre, _ = file_scoped_preamble(lines, 3) + self.assertEqual(len(pre), 1) + self.assertIn("`(Metric.sphere 0 1)", pre[0]) + + def test_noncomputable_section_is_restated(self): + # OpenQuantumProblems/23: a copied def that was total inside + # `noncomputable section` fails to compile outside it. + lines = ["noncomputable section", "theorem t : True := trivial"] + pre, _ = file_scoped_preamble(lines, 2) + self.assertIn("noncomputable section", pre) + + def test_a_closed_noncomputable_section_is_not_restated(self): + lines = ["noncomputable section", "end", "theorem t : True := trivial"] + pre, _ = file_scoped_preamble(lines, 3) + self.assertNotIn("noncomputable section", pre) + + +class AscribedSlotTest(unittest.TestCase): + """`(answer(sorry) : T)` states its own type at its own position.""" + + def test_the_ascription_wins_over_the_erasure_rule(self): + # Erdos332: the annotation for an ascribed-and-applied slot does not + # survive elaboration, so the environment reports nothing and the + # erasure rule would call it Prop. + statement = ( + "theorem erdos_332 (A : Set ℕ) : " + "(answer(sorry) : Set ℕ → Prop) A → True := by\n sorry" + ) + _, holes = hoist_answers(statement, "erdos_332", []) + self.assertEqual(holes[0].type, "Set ℕ → Prop") + + def test_a_nested_paren_type_stays_whole(self): + statement = "theorem t : (answer(sorry) : (ℕ → ℕ) → Prop) f := by\n sorry" + _, holes = hoist_answers(statement, "t", []) + self.assertEqual(holes[0].type, "(ℕ → ℕ) → Prop") + + def test_an_unascribed_slot_still_follows_the_erasure_rule(self): + statement = "theorem t : answer(sorry) ↔ True := by\n sorry" + _, holes = hoist_answers(statement, "t", []) + self.assertEqual(holes[0].type, "Prop") + + +class NotationBlocksTest(unittest.TestCase): + """FC-defined notation is copied only where it was in force.""" + + def _with_commands(self, commands): + return mock.patch.object( + fc_source, "fc_notation_commands", return_value=commands + ) + + def test_a_scoped_notation_needs_its_namespace_opened(self): + commands = [ + (["ℝ²"], 'scoped[EuclideanGeometry] notation "ℝ²" => E', "EuclideanGeometry", True), + ] + with self._with_commands(commands): + self.assertEqual( + fc_source.notation_blocks(["def f : ℝ² := sorry"], {"EuclideanGeometry"}), + ['scoped[EuclideanGeometry] notation "ℝ²" => E'], + ) + # Green9's `⊆` false positive: same token, namespace never opened. + self.assertEqual( + fc_source.notation_blocks(["def f : ℝ² := sorry"], set()), [] + ) + + def test_a_shared_global_notation_is_copied_as_local(self): + # Global would be declared in ChallengeDeps and re-extracted into + # the importing file too; `local` keeps each copy to its own file. + commands = [(["≪"], 'notation g " ≪ " f => IsBigO g f', None, True)] + with self._with_commands(commands): + self.assertEqual( + fc_source.notation_blocks(["theorem t : a ≪ b := sorry"], set()), + ['local notation g " ≪ " f => IsBigO g f'], + ) + + def test_a_problem_module_global_notation_is_never_copied(self): + commands = [(["≪"], 'notation g " ≪ " f => X g f', None, False)] + with self._with_commands(commands): + self.assertEqual( + fc_source.notation_blocks(["theorem t : a ≪ b := sorry"], set()), [] + ) + + def test_an_unused_token_is_not_copied(self): + commands = [(["ℝ²"], 'notation "ℝ²" => E', None, True)] + with self._with_commands(commands): + self.assertEqual(fc_source.notation_blocks(["theorem t : True"], set()), []) + + +class LocaliseNotationTest(unittest.TestCase): + def test_a_global_notation_becomes_local(self): + self.assertEqual( + fc_source.localise_notation(['notation "R(" k ")" => f k']), + ['local notation "R(" k ")" => f k'], + ) + + def test_quot_precheck_travels_with_the_notation(self): + out = fc_source.localise_notation( + ["set_option quotPrecheck false", 'local notation "A" => s'] + ) + self.assertEqual( + out[1], + 'set_option quotPrecheck false in\nlocal notation "A" => s', + ) + + def test_other_preamble_lines_pass_through(self): + self.assertEqual( + fc_source.localise_notation(["open Nat", "variable (n : Nat)"]), + ["open Nat", "variable (n : Nat)"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/comparator/adapter/test_leaneval_interface.py b/comparator/adapter/test_leaneval_interface.py index e5c7c36479..7696021617 100644 --- a/comparator/adapter/test_leaneval_interface.py +++ b/comparator/adapter/test_leaneval_interface.py @@ -22,6 +22,7 @@ """ import unittest +from unittest import mock from leaneval_interface import ( DefinitionHole, @@ -35,6 +36,7 @@ declaration_spans, module_declarations, parse_response, + problem_group, slug, ) @@ -336,3 +338,52 @@ def test_unknown_digest_keys_are_refused(self): payload["digests"]["request"] = "d" * 64 with self.assertRaises(SystemExit): ProblemManifest.from_json_object(payload) + + +class ProblemGroupTest(unittest.TestCase): + """Categories map to lean-eval groups; non-problems are refused.""" + + def _manifest(self, category): + manifest = mock.Mock() + manifest.category = category + manifest.id = "some_problem" + return manifest + + def test_open_research_is_an_open_conjecture(self): + self.assertEqual( + problem_group(self._manifest("research open")), + "open-conjectures", + ) + + def test_settled_statements_are_evaluation_material(self): + for category in ("research solved", "textbook", "test"): + with self.subTest(category=category): + self.assertEqual( + problem_group(self._manifest(category)), + "formalization-evaluation", + ) + + def test_api_and_untagged_declarations_are_refused(self): + for category in ("API", ""): + with self.subTest(category=category): + with self.assertRaises(SystemExit): + problem_group(self._manifest(category)) + + +class StatementPrefixSpanTest(unittest.TestCase): + def test_the_span_starts_at_the_declaration_keyword(self): + from leaneval_interface import module_declarations + + module = MarkedUpModule( + dependencies="def Foo.bar := 1", + scope="", + holes="", + statement="open scoped Classical in\ntheorem t : True := by\n sorry", + dependency_declarations=(("Foo.bar", "def Foo.bar := 1"),), + ) + manifest = a_manifest( + theorem="t", qualified_theorem="t", holes=(), apply_arguments=() + ) + *_, statement_entry = module_declarations(module, manifest) + self.assertTrue(statement_entry[1].startswith("theorem t")) + self.assertNotIn("Classical in", statement_entry[1]) diff --git a/comparator/adapter/test_make_comparator_workspace.py b/comparator/adapter/test_make_comparator_workspace.py index 6e41cdacb5..5bd488ec95 100644 --- a/comparator/adapter/test_make_comparator_workspace.py +++ b/comparator/adapter/test_make_comparator_workspace.py @@ -195,7 +195,7 @@ def test_a_missing_subset_is_refused(self): class KnownFailuresTest(unittest.TestCase): def _load(self, text): - from make_comparator_workspace import load_known_failures + from known_failures import load_known_failures with tempfile.NamedTemporaryFile("w", suffix=".toml", delete=False) as f: f.write(text) From b68dc53ef4aba4d64736c083911494734a33b383 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:55:19 -0400 Subject: [PATCH 56/70] Hold every read file to the source pin; record what was copied and by what MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The source pin checked only the statement's own file, while the closure copier read dependency and notation files straight from the working tree: a workspace could mix a pinned statement with branch-local copied text and record one commit for all of it. `pins` now takes every path whose text reached the workspace — statement, dependencies, notation commands, and the pin files the record quotes — refusing a change or an untracked file in any of them. The sidecar's provenance closes the same chain: each copied dependency is recorded as the slice actually emitted (declaration, module, path, range, digest), the statement travels as range plus digest instead of its full text — a trusted-statement record has no business carrying the source's proof body — and a `producer` section names the importer commit, the pinned generator with its contract version, and the target pins the artifact was generated for. --- comparator/adapter/fc_leaneval_importer.py | 94 +++++++++++++-- comparator/adapter/fc_source.py | 87 +++++++++++--- comparator/adapter/leaneval_interface.py | 112 +++++++++++++++++- .../adapter/make_comparator_workspace.py | 8 +- .../adapter/test_fc_leaneval_importer.py | 88 ++++++++++---- comparator/adapter/test_fc_source.py | 18 ++- comparator/adapter/test_leaneval_interface.py | 56 ++++++++- 7 files changed, 402 insertions(+), 61 deletions(-) diff --git a/comparator/adapter/fc_leaneval_importer.py b/comparator/adapter/fc_leaneval_importer.py index 9532de8fa6..5455c82d50 100644 --- a/comparator/adapter/fc_leaneval_importer.py +++ b/comparator/adapter/fc_leaneval_importer.py @@ -34,11 +34,14 @@ import tomllib from leaneval_interface import ( + CONTRACT_VERSION, MarkedUpModule, ProblemManifest, + ProducerRecord, SourceRecord, TargetRecord, lean_errors, + sha256_text, ) from fc_source import ( DECL_START, @@ -48,6 +51,7 @@ find_declaration, flatten_declared_name, hoist_answers, + importer_state, localise_notation, module_name, module_source_path, @@ -94,6 +98,36 @@ def target_pins(): ) +def producer_record(): + """What produced a generated artifact: this adapter, that generator, those pins. + + A workspace is immutable once written, so its record names the producers + as they were at generation time — the importer's own commit and whether + its files carried uncommitted edits, the pinned generator and its + contract version, and the target pins the workspace was generated for. + These are facts about this artifact, not assertions about what any + consumer uses forever. + """ + tools = _tools_file() + generator = tools.get("generator", {}) + target = tools.get("target", {}) + for table, key in (("generator", "repository"), ("generator", "rev")): + if not tools.get(table, {}).get(key): + raise SystemExit(f"comparator/tools.toml [{table}] has an empty `{key}`") + commit, dirty = importer_state() + return ProducerRecord( + importer_commit=commit, + importer_dirty=dirty, + generator_repository=generator["repository"], + generator_rev=generator["rev"], + contract_version=CONTRACT_VERSION, + target_lean_toolchain=target.get("lean_toolchain", ""), + target_mathlib_revision=target.get("mathlib_revision", ""), + target_comparator=target.get("comparator", ""), + target_lean4export=target.get("lean4export", ""), + ) + + def explicit_copy_dependencies(problem_file): """Source-only dependencies the compiled environment cannot retain. @@ -270,7 +304,7 @@ def covered_by_another(dep): subsumed = [dep["name"] for dep in dependencies if covered_by_another(dep)] dependencies = [dep for dep in dependencies if dep["name"] not in subsumed] - blocks, provenance = [], [] + blocks, provenance, records = [], [], [] # `open X` on a namespace nothing has declared yet is an error, and a # copied preamble may open a namespace whose declaring block comes later # in the copy, or never: with the problem's module no longer imported, @@ -337,6 +371,15 @@ def covered_by_another(dep): chunk.append("end") blocks.append("\n".join(chunk)) provenance.append((dep["name"], body)) + records.append( + { + "declaration": dep["name"], + "module": dep["module"], + "path": str(path.relative_to(ROOT)), + "range": dep["range"], + "content_sha256": sha256_text(body), + } + ) listing = "\n".join(f"* `{name}`" for name, _ in provenance) return ( @@ -346,11 +389,18 @@ def covered_by_another(dep): "come before the declarations that use them:\n\n" f"{listing}\n" "-/\n\n" + "\n\n".join(blocks) + "\n" - ), provenance + ), provenance, records def source_record( - declaration, module, source_path, fc_rev, dependencies, original, mathlib_rev + declaration, + module, + source_path, + fc_rev, + copied_records, + original, + original_range, + mathlib_rev, ): """Where the copied statement and its dependencies came from. @@ -359,6 +409,14 @@ def source_record( generator sees a Lean module, not a repository. They are also what makes the importer's regeneration duty possible — when Formal Conjectures fixes a misformalisation upstream, this record says which problem to redo. + + Each copied dependency is recorded as the slice that was actually + emitted — declaration, module, path, range and a digest of the copied + text — and the statement itself as its range and digest. The original + text is not carried: `pins` holds every recorded path to `commit`, so + range plus digest identify the bytes exactly, and a trusted-statement + record has no business shipping the source's proof body into a benchmark + workspace. """ blob = subprocess.run( ["git", "-C", str(ROOT), "rev-parse", f"{fc_rev}:{source_path}"], @@ -373,8 +431,9 @@ def source_record( blob_sha=blob.stdout.strip() or "", module=module, declaration=declaration, - copied_dependencies=tuple(dependencies), - original_declaration=original, + copied_dependencies=tuple(copied_records), + original_range=dict(original_range), + original_sha256=sha256_text(original), lean_toolchain=(ROOT / "lean-toolchain").read_text(encoding="utf-8").strip(), mathlib_revision=mathlib_rev, ) @@ -492,7 +551,7 @@ def place_notations(dependencies, scope_text, statement, copied): [dependencies, scope_text, statement], opened_for_notation ) if not notations: - return dependencies + return dependencies, [] # A notation whose right-hand side names a copied declaration must # come after the block declaring it; every other notation comes # first, because copied declarations may use its token textually. A @@ -500,7 +559,7 @@ def place_notations(dependencies, scope_text, statement, copied): # and `--verify` is what says so. copied_last_components = {name.rsplit(".", 1)[-1] for name, _ in copied} before, after = [], [] - for block in notations: + for block, _path in notations: rhs = block.split("=>", 1)[-1] names = set(re.findall(r"[\w«»'.]+", rhs)) names |= {name.rsplit(".", 1)[-1] for name in names} @@ -512,7 +571,7 @@ def place_notations(dependencies, scope_text, statement, copied): dependencies = "\n\n".join(before) + "\n\n" + dependencies if after: dependencies = dependencies + "\n\n" + "\n\n".join(after) - return dependencies + return dependencies, [path for _, path in notations] def _resolve(problem, module=None): @@ -545,7 +604,7 @@ def import_problem(problem, answer_type=None, module=None): source_lines = path.read_text(encoding="utf-8").split("\n") original, lo = slice_range(source_lines, facts.range) preamble, namespaces_at_target = file_scoped_preamble(source_lines, lo) - dependencies, copied = closure_region( + dependencies, copied, copied_records = closure_region( list(facts.dependencies), list(facts.generated_dependencies), declaration, @@ -557,7 +616,9 @@ def import_problem(problem, answer_type=None, module=None): ) args = explicit_arguments(facts, declared) scope_text = scope_region(namespaces_at_target, copied, preamble) - dependencies = place_notations(dependencies, scope_text, statement, copied) + dependencies, notation_paths = place_notations( + dependencies, scope_text, statement, copied + ) marked_up = MarkedUpModule( dependencies=dependencies, scope=scope_text, @@ -569,7 +630,15 @@ def import_problem(problem, answer_type=None, module=None): # workspace statement may carry the flattened `declared` instead, and # this is what ties the two together. qualified = ".".join(namespaces_at_target + [original_declared]) - mathlib_rev, fc_rev = pins(path.relative_to(ROOT)) + # Every file whose text reached the workspace is held to the one source + # revision: the statement's own file, each copied dependency's, each + # copied notation command's, and the pin files the record quotes. + read_paths = ( + [path.relative_to(ROOT), "lean-toolchain", "lake-manifest.json"] + + [record["path"] for record in copied_records] + + list(notation_paths) + ) + mathlib_rev, fc_rev = pins(read_paths) source_url = docstring_reference(module_doc) if not source_url: print( @@ -591,8 +660,9 @@ def import_problem(problem, answer_type=None, module=None): fc_module, path.relative_to(ROOT), fc_rev, - [dep["name"] for dep in facts.dependencies], + copied_records, original, + facts.range, mathlib_rev, ), source_url=source_url, diff --git a/comparator/adapter/fc_source.py b/comparator/adapter/fc_source.py index 1b941fce3c..675278e9c2 100644 --- a/comparator/adapter/fc_source.py +++ b/comparator/adapter/fc_source.py @@ -488,10 +488,11 @@ def fc_notation_commands(): `local` notations are file-scoped at their origin and cannot be in force in a problem file, so they are not candidates. - Returns `[(tokens, command, namespaces)]`, where `tokens` are the - command's string literals that contain a non-ASCII character — the - distinctive ones worth matching on — and `namespaces` is the stack a - plain `scoped` command needs restated around it. + Returns `[(tokens, command, scope, shared, path)]`, where `tokens` are + the command's distinctive string literals, `scope` is the namespace a + `scoped` command needs restated around it, and `path` is the defining + file relative to ROOT — a copied command's text is a read source input, + so the snapshot check needs to know where it came from. """ global _NOTATION_CACHE if _NOTATION_CACHE is not None: @@ -539,7 +540,9 @@ def fc_notation_commands(): # problem files do not import each other, and the problem # file's own notations travel with the preamble. shared = src.name != "FormalConjectures" - commands.append((tokens, command, scope, shared)) + commands.append( + (tokens, command, scope, shared, path.relative_to(ROOT)) + ) _NOTATION_CACHE = commands return commands @@ -580,10 +583,13 @@ def notation_blocks(module_texts, opened): imports was never in force either, but the corpus keeps global notation in the problem file itself, which the preamble already carries, so unscoped commands from other files are not candidates at all. + + Returns `[(block, path)]`: each copyable block with the file it was read + from, so the caller can hold that file to the source pin. """ combined = "\n".join(module_texts) blocks, seen = [], set() - for tokens, command, scope, shared in fc_notation_commands(): + for tokens, command, scope, shared, path in fc_notation_commands(): if scope: if scope not in opened: continue @@ -602,7 +608,7 @@ def notation_blocks(module_texts, opened): command = f"namespace {scope}\n{command}\nend {scope}" elif not scope: command = "local " + command - blocks.append(command) + blocks.append((command, path)) return blocks def flatten_declared_name(declared, statement): @@ -910,26 +916,75 @@ def _base_pins(): return mathlib_rev, merge_base.stdout.strip() -def pins(source_path=None): +def pins(source_paths=None): """Revisions the workspace's own build can actually fetch. The FC pin must be reachable from the upstream repository the lakefile names, so it is the merge-base with `origin/main`, not HEAD: a local branch commit would generate a workspace whose build fails at fetch time. - The importer stops if the selected source differs from that revision. - Otherwise it could combine a working-tree statement with an older imported - context. + The importer stops if any file it read differs from that revision — + `source_paths` is every file whose text reached the workspace, not just + the statement's own — so the record names one revision and the copied + text all comes from it. A path the revision does not track fails too: + `git diff` is silent about untracked files, so tracking is checked first. """ mathlib_rev, fc_rev = _base_pins() - if source_path is not None: + if source_paths is not None: + if isinstance(source_paths, (str, pathlib.Path)): + source_paths = [source_paths] + paths = sorted({str(path) for path in source_paths}) + tracked = subprocess.run( + ["git", "-C", str(ROOT), "ls-tree", "-r", "--name-only", fc_rev, "--"] + + paths, + capture_output=True, + text=True, + ) + if tracked.returncode != 0: + raise SystemExit(f"cannot list {fc_rev[:12]}: {tracked.stderr.strip()}") + missing = sorted(set(paths) - set(tracked.stdout.split("\n"))) + if missing: + raise SystemExit( + f"{', '.join(missing)}: not tracked at pinned revision " + f"{fc_rev[:12]}; land the source on upstream main before " + "generating" + ) comparison = subprocess.run( - ["git", "-C", str(ROOT), "diff", "--quiet", fc_rev, "--", str(source_path)] + ["git", "-C", str(ROOT), "diff", "--quiet", fc_rev, "--"] + paths ) if comparison.returncode not in (0, 1): - raise SystemExit(f"cannot compare {source_path} with {fc_rev[:12]}") + raise SystemExit(f"cannot compare {', '.join(paths)} with {fc_rev[:12]}") if comparison.returncode == 1: + changed = subprocess.run( + ["git", "-C", str(ROOT), "diff", "--name-only", fc_rev, "--"] + paths, + capture_output=True, + text=True, + ) raise SystemExit( - f"{source_path} differs from pinned revision {fc_rev[:12]}; " - "land the source on upstream main before generating" + f"{changed.stdout.strip() or ', '.join(paths)} differs from " + f"pinned revision {fc_rev[:12]}; land the source on upstream " + "main before generating" ) return mathlib_rev, fc_rev + + +def importer_state(): + """The commit this importer ran as, and whether its own files were edited. + + The generated record separates two questions the source pin cannot answer: + which adapter produced the artifact (HEAD, not the merge-base — the + adapter itself is allowed to be branch work), and whether that adapter was + running with uncommitted edits under `comparator/`. + """ + head = subprocess.run( + ["git", "-C", str(ROOT), "rev-parse", "HEAD"], + capture_output=True, + text=True, + ) + if head.returncode != 0 or not head.stdout.strip(): + raise SystemExit("cannot resolve the importer's own commit") + status = subprocess.run( + ["git", "-C", str(ROOT), "status", "--porcelain", "--", "comparator"], + capture_output=True, + text=True, + ) + return head.stdout.strip(), bool(status.stdout.strip()) diff --git a/comparator/adapter/leaneval_interface.py b/comparator/adapter/leaneval_interface.py index a95eb715b8..f1a110fa89 100644 --- a/comparator/adapter/leaneval_interface.py +++ b/comparator/adapter/leaneval_interface.py @@ -132,6 +132,13 @@ class SourceRecord: hole types in the manifest were read from an environment elaborated at them, so a reader comparing them with `TargetRecord` can see whether the types were read where they will be used. + + `copied_dependencies` holds one record per emitted slice — declaration, + module, path, source range and a digest of the copied text — and the + statement itself travels as `original_range` plus `original_sha256` + rather than as text: with every recorded path held to `commit`, range and + digest identify the bytes exactly, and the record never carries the + source's proof body into a workspace. """ repository: str @@ -141,11 +148,91 @@ class SourceRecord: module: str declaration: str copied_dependencies: tuple - original_declaration: str + original_range: dict + original_sha256: str lean_toolchain: str mathlib_revision: str +@dataclasses.dataclass(frozen=True) +class ProducerRecord: + """What produced one generated workspace, as it was at generation time. + + Immutable output deserves a record of its producers: the importer commit + (and whether `comparator/` carried uncommitted edits), the pinned + generator with its contract version, and the target pins the workspace + was generated for. These describe this artifact; the consumer's live + regime stays the consumer's, and `TargetRecord` remains the request-side + statement of it. + """ + + importer_commit: str + importer_dirty: bool + generator_repository: str + generator_rev: str + contract_version: int + target_lean_toolchain: str + target_mathlib_revision: str + target_comparator: str + target_lean4export: str + + def to_json_object(self): + return { + "importer": { + "commit": self.importer_commit, + "dirty": self.importer_dirty, + }, + "generator": { + "repository": self.generator_repository, + "rev": self.generator_rev, + "contract_version": self.contract_version, + }, + "target": { + "lean_toolchain": self.target_lean_toolchain, + "mathlib_revision": self.target_mathlib_revision, + "comparator": self.target_comparator, + "lean4export": self.target_lean4export, + }, + } + + @classmethod + def from_json_object(cls, payload): + sections = { + "importer": {"commit", "dirty"}, + "generator": {"repository", "rev", "contract_version"}, + "target": { + "lean_toolchain", + "mathlib_revision", + "comparator", + "lean4export", + }, + } + unknown = sorted(set(payload) - set(sections)) + if unknown: + raise SystemExit(f"producer record has unknown keys: {', '.join(unknown)}") + for section, keys in sections.items(): + entries = payload.get(section, {}) + unknown = sorted(set(entries) - keys) + if unknown: + raise SystemExit( + f"producer {section} has unknown keys: {', '.join(unknown)}" + ) + importer = payload.get("importer", {}) + generator = payload.get("generator", {}) + target = payload.get("target", {}) + return cls( + importer_commit=importer.get("commit", ""), + importer_dirty=bool(importer.get("dirty", False)), + generator_repository=generator.get("repository", ""), + generator_rev=generator.get("rev", ""), + contract_version=generator.get("contract_version", 0), + target_lean_toolchain=target.get("lean_toolchain", ""), + target_mathlib_revision=target.get("mathlib_revision", ""), + target_comparator=target.get("comparator", ""), + target_lean4export=target.get("lean4export", ""), + ) + + @dataclasses.dataclass(frozen=True) class TargetRecord: """The pins a generated workspace is built and checked with. @@ -203,6 +290,9 @@ class ProblemManifest: # serialisation is key-sorted. module_sha256: str = "" file_sha256: tuple = () + # What produced the artifact — importer commit, pinned generator, target + # pins — bound at generation time like the digests, absent before it. + producer: ProducerRecord = None def with_digests(self, module_sha256, files): """The same manifest, bound to the module bytes and generated files.""" @@ -212,6 +302,10 @@ def with_digests(self, module_sha256, files): file_sha256=tuple(sorted((path, digest) for path, digest in files.items())), ) + def with_producer(self, producer): + """The same manifest, naming what produced the artifact.""" + return dataclasses.replace(self, producer=producer) + def __post_init__(self): for field in ("id", "theorem", "qualified_theorem"): if not getattr(self, field): @@ -247,13 +341,15 @@ def to_json_object(self): "module": self.module_sha256, "files": dict(self.file_sha256), } + if self.producer is not None: + payload["producer"] = self.producer.to_json_object() return payload KNOWN_KEYS = frozenset( { "schema_version", "id", "theorem", "qualified_theorem", "category", "apply_arguments", "holes", "permitted_axioms", "source", - "source_url", "digests", + "source_url", "digests", "producer", } ) @@ -270,6 +366,13 @@ def from_json_object(cls, payload): raise SystemExit(f"provenance record has unknown keys: {', '.join(unknown)}") source = dict(payload["source"]) source["copied_dependencies"] = tuple(source["copied_dependencies"]) + copied_keys = {"declaration", "module", "path", "range", "content_sha256"} + for entry in source["copied_dependencies"]: + unknown = sorted(set(entry) - copied_keys) + if unknown: + raise SystemExit( + f"a copied dependency has unknown keys: {', '.join(unknown)}" + ) unknown = sorted(set(source) - {f.name for f in dataclasses.fields(SourceRecord)}) if unknown: raise SystemExit(f"provenance source has unknown keys: {', '.join(unknown)}") @@ -289,6 +392,11 @@ def from_json_object(cls, payload): category=payload.get("category", ""), module_sha256=digests.get("module", ""), file_sha256=tuple(sorted(dict(digests.get("files", {})).items())), + producer=( + ProducerRecord.from_json_object(payload["producer"]) + if "producer" in payload + else None + ), ) def to_json(self): diff --git a/comparator/adapter/make_comparator_workspace.py b/comparator/adapter/make_comparator_workspace.py index 58d4495322..89eb0bd2eb 100644 --- a/comparator/adapter/make_comparator_workspace.py +++ b/comparator/adapter/make_comparator_workspace.py @@ -139,13 +139,16 @@ def seam_files(pairs, group=None): has no field for, so they travel beside it rather than through it. """ request, problems = _seam(pairs, group=group) + producer = importer.producer_record() files = {"request.json": dump_json(request)} for path, content in generator_cli.context_files(problems).items(): files[f"{CONTEXT_DIR}/{path}"] = content for (problem, _), (_, manifest) in zip(problems, pairs): # Before generation the record binds the module bytes only; the # workspace's copy adds the generated files. - bound = manifest.with_digests(sha256_text(problem["moduleContent"]), {}) + bound = manifest.with_digests( + sha256_text(problem["moduleContent"]), {} + ).with_producer(producer) files[f"{PROVENANCE_STEM}-{problem['id']}.json"] = bound.to_json() return request, files @@ -164,6 +167,7 @@ def generate_workspaces(pairs, out_dir, group=None): finally: shutil.rmtree(staging, ignore_errors=True) module_content = {p["id"]: p["moduleContent"] for p in request["problems"]} + producer = importer.producer_record() written = [] for _, manifest in pairs: problem_id = slug(manifest.id) @@ -176,7 +180,7 @@ def generate_workspaces(pairs, out_dir, group=None): bound = manifest.with_digests( sha256_text(module_content[problem_id]), {path: sha256_text(content) for path, content in workspace.items()}, - ) + ).with_producer(producer) workspace[PROVENANCE_FILE] = bound.to_json() written.append(write_tree(pathlib.Path(out_dir) / problem_id, workspace)) return written diff --git a/comparator/adapter/test_fc_leaneval_importer.py b/comparator/adapter/test_fc_leaneval_importer.py index 1687223297..f0049f3512 100644 --- a/comparator/adapter/test_fc_leaneval_importer.py +++ b/comparator/adapter/test_fc_leaneval_importer.py @@ -77,29 +77,66 @@ def test_declaration_is_required(self): class PinTest(unittest.TestCase): - def test_changed_source_is_refused(self): - saved_root = importer.ROOT + """Every path the importer read is held to the one source revision.""" + + @contextlib.contextmanager + def _pins_repo(self, git_results): + saved_root = fc_source.ROOT + fc_source._base_pins.cache_clear() with tempfile.TemporaryDirectory() as tmp: root = pathlib.Path(tmp) - importer.ROOT = root + fc_source.ROOT = root (root / "lake-manifest.json").write_text( - json.dumps( - { - "packages": [{"name": "mathlib", "rev": "b" * 40}], - } - ), + json.dumps({"packages": [{"name": "mathlib", "rev": "b" * 40}]}), encoding="utf-8", ) - results = [ - subprocess.CompletedProcess([], 0, stdout="a" * 40 + "\n"), - subprocess.CompletedProcess([], 1), - ] try: - with mock.patch.object(importer.subprocess, "run", side_effect=results): - with self.assertRaisesRegex(SystemExit, "differs from pinned"): - pins(pathlib.Path("FormalConjectures/Example.lean")) + with mock.patch.object( + fc_source.subprocess, "run", side_effect=git_results + ): + yield finally: - importer.ROOT = saved_root + fc_source.ROOT = saved_root + fc_source._base_pins.cache_clear() + + def test_changed_source_is_refused(self): + path = "FormalConjectures/Example.lean" + results = [ + subprocess.CompletedProcess([], 0, stdout="a" * 40 + "\n"), + subprocess.CompletedProcess([], 0, stdout=path + "\n"), + subprocess.CompletedProcess([], 1), + subprocess.CompletedProcess([], 0, stdout=path + "\n"), + ] + with self._pins_repo(results): + with self.assertRaisesRegex(SystemExit, "differs from pinned"): + pins(pathlib.Path(path)) + + def test_a_path_the_revision_does_not_track_is_refused(self): + # `git diff` is silent about untracked files, so a dependency read + # from a file the pinned revision has never seen must fail on the + # tracking check, not pass by omission. + results = [ + subprocess.CompletedProcess([], 0, stdout="a" * 40 + "\n"), + subprocess.CompletedProcess([], 0, stdout="\n"), + ] + with self._pins_repo(results): + with self.assertRaisesRegex(SystemExit, "not tracked at pinned"): + pins(pathlib.Path("FormalConjectures/New.lean")) + + def test_a_dirty_dependency_fails_even_with_a_clean_target(self): + # The reviewer's mixed state: target at the pin, dependency edited in + # the working tree. One diff over every read path refuses it. + target = "FormalConjectures/Target.lean" + dep = "FormalConjectures/Dep.lean" + results = [ + subprocess.CompletedProcess([], 0, stdout="a" * 40 + "\n"), + subprocess.CompletedProcess([], 0, stdout=f"{dep}\n{target}\n"), + subprocess.CompletedProcess([], 1), + subprocess.CompletedProcess([], 0, stdout=dep + "\n"), + ] + with self._pins_repo(results): + with self.assertRaisesRegex(SystemExit, "Dep.lean differs from pinned"): + pins([pathlib.Path(target), pathlib.Path(dep)]) @contextlib.contextmanager @@ -174,10 +211,19 @@ def test_a_generated_constant_under_a_copied_parent_is_accepted(self): source = pathlib.Path(tmp) / "Example.lean" source.write_text("def Foo.bar := 1\n", encoding="utf-8") resolve.return_value = source - out, copied = closure_region(deps, ["Foo.bar._proof_1"], "t") + out, copied, records = closure_region(deps, ["Foo.bar._proof_1"], "t") self.assertIn("def Foo.bar := 1", out) self.assertIn("noncomputable section", out) self.assertEqual(copied, [("Foo.bar", "def Foo.bar := 1")]) + # The sidecar's record of the same copy: the emitted slice, located + # and digested, so the provenance chain reaches each dependency. + self.assertEqual(len(records), 1) + record = records[0] + self.assertEqual(record["declaration"], "Foo.bar") + self.assertEqual(record["module"], "FormalConjectures.Example") + self.assertEqual(record["path"], "Example.lean") + self.assertEqual(record["range"]["startLine"], 1) + self.assertEqual(len(record["content_sha256"]), 64) def test_an_explicit_source_only_dependency_carries_its_closure(self): facts = fc_source.FactsRecord.from_payload( @@ -277,7 +323,7 @@ def span(name, lo, hi): encoding="utf-8", ) resolve.return_value = source - out, _copied = closure_region(deps, [], "t") + out, _copied, _records = closure_region(deps, [], "t") self.assertIn("Foo.EdgeN`", out) self.assertNotIn("Foo.EdgeN.mk`", out) self.assertIn("Foo.aux`", out) @@ -287,7 +333,7 @@ def test_an_opened_namespace_no_dependency_declares_is_created(self): # The statement reopens the namespace stack its target sat in. With # the problem's module no longer imported, `open Grimm` is an error # unless something declares that namespace. - out, _copied = closure_region([], [], "grimm_conjecture", ["Grimm"]) + out, _copied, _records = closure_region([], [], "grimm_conjecture", ["Grimm"]) self.assertIn("namespace Grimm\nend Grimm", out) def test_namespaces_exist_before_any_copied_block_opens_them(self): @@ -306,7 +352,7 @@ def test_namespaces_exist_before_any_copied_block_opens_them(self): source = pathlib.Path(tmp) / "Example.lean" source.write_text("def Grimm.helper := 1\n", encoding="utf-8") resolve.return_value = source - out, _copied = closure_region(deps, [], "t", ["Grimm"]) + out, _copied, _records = closure_region(deps, [], "t", ["Grimm"]) # The empty block that makes the namespace exist comes before any # copied block: a copied preamble may `open` it before anything # declares it. Redundant creation is harmless. @@ -318,7 +364,7 @@ def test_the_closure_region_does_not_carry_the_import(self): # `import Mathlib` belongs to the module as a whole, and the generator # is what decides which emitted file carries it. A region that # restated it would put an import in the middle of a Lean file. - out, _copied = closure_region([], [], "t") + out, _copied, _records = closure_region([], [], "t") self.assertNotIn("import", out) diff --git a/comparator/adapter/test_fc_source.py b/comparator/adapter/test_fc_source.py index 5d149ed1b0..50625726ff 100644 --- a/comparator/adapter/test_fc_source.py +++ b/comparator/adapter/test_fc_source.py @@ -18,6 +18,7 @@ would copy source that elaborates but poses the wrong problem. """ +import pathlib import unittest from unittest import mock @@ -390,12 +391,13 @@ def _with_commands(self, commands): def test_a_scoped_notation_needs_its_namespace_opened(self): commands = [ - (["ℝ²"], 'scoped[EuclideanGeometry] notation "ℝ²" => E', "EuclideanGeometry", True), + (["ℝ²"], 'scoped[EuclideanGeometry] notation "ℝ²" => E', "EuclideanGeometry", True, pathlib.Path("FormalConjecturesForMathlib/Geometry.lean")), ] with self._with_commands(commands): self.assertEqual( fc_source.notation_blocks(["def f : ℝ² := sorry"], {"EuclideanGeometry"}), - ['scoped[EuclideanGeometry] notation "ℝ²" => E'], + [('scoped[EuclideanGeometry] notation "ℝ²" => E', + pathlib.Path("FormalConjecturesForMathlib/Geometry.lean"))], ) # Green9's `⊆` false positive: same token, namespace never opened. self.assertEqual( @@ -405,22 +407,26 @@ def test_a_scoped_notation_needs_its_namespace_opened(self): def test_a_shared_global_notation_is_copied_as_local(self): # Global would be declared in ChallengeDeps and re-extracted into # the importing file too; `local` keeps each copy to its own file. - commands = [(["≪"], 'notation g " ≪ " f => IsBigO g f', None, True)] + commands = [(["≪"], 'notation g " ≪ " f => IsBigO g f', None, True, + pathlib.Path("FormalConjecturesForMathlib/Order.lean"))] with self._with_commands(commands): self.assertEqual( fc_source.notation_blocks(["theorem t : a ≪ b := sorry"], set()), - ['local notation g " ≪ " f => IsBigO g f'], + [('local notation g " ≪ " f => IsBigO g f', + pathlib.Path("FormalConjecturesForMathlib/Order.lean"))], ) def test_a_problem_module_global_notation_is_never_copied(self): - commands = [(["≪"], 'notation g " ≪ " f => X g f', None, False)] + commands = [(["≪"], 'notation g " ≪ " f => X g f', None, False, + pathlib.Path("FormalConjectures/Wikipedia/X.lean"))] with self._with_commands(commands): self.assertEqual( fc_source.notation_blocks(["theorem t : a ≪ b := sorry"], set()), [] ) def test_an_unused_token_is_not_copied(self): - commands = [(["ℝ²"], 'notation "ℝ²" => E', None, True)] + commands = [(["ℝ²"], 'notation "ℝ²" => E', None, True, + pathlib.Path("FormalConjecturesForMathlib/Geometry.lean"))] with self._with_commands(commands): self.assertEqual(fc_source.notation_blocks(["theorem t : True"], set()), []) diff --git a/comparator/adapter/test_leaneval_interface.py b/comparator/adapter/test_leaneval_interface.py index 7696021617..50a91df6d0 100644 --- a/comparator/adapter/test_leaneval_interface.py +++ b/comparator/adapter/test_leaneval_interface.py @@ -28,6 +28,7 @@ DefinitionHole, MarkedUpModule, ProblemManifest, + ProducerRecord, SourceRecord, TargetRecord, _utf16_column, @@ -49,8 +50,17 @@ def a_source(**overrides): "blob_sha": "b" * 40, "module": "FormalConjectures.Example", "declaration": "erdos_940", - "copied_dependencies": ("Foo.bar",), - "original_declaration": "theorem erdos_940 : True := by\n sorry", + "copied_dependencies": ( + { + "declaration": "Foo.bar", + "module": "FormalConjectures.Example", + "path": "FormalConjectures/Example.lean", + "range": {"startLine": 3, "startColumn": 0, "endLine": 4, "endColumn": 11}, + "content_sha256": "d" * 64, + }, + ), + "original_range": {"startLine": 20, "startColumn": 0, "endLine": 22, "endColumn": 7}, + "original_sha256": "e" * 64, "lean_toolchain": "leanprover/lean4:v4.33.1", "mathlib_revision": "c" * 40, } @@ -387,3 +397,45 @@ def test_the_span_starts_at_the_declaration_keyword(self): *_, statement_entry = module_declarations(module, manifest) self.assertTrue(statement_entry[1].startswith("theorem t")) self.assertNotIn("Classical in", statement_entry[1]) + +def a_producer(**overrides): + fields = { + "importer_commit": "9" * 40, + "importer_dirty": False, + "generator_repository": "https://github.com/leanprover/lean-eval-generator", + "generator_rev": "7" * 40, + "contract_version": 1, + "target_lean_toolchain": "leanprover/lean4:v4.33.0", + "target_mathlib_revision": "6" * 40, + "target_comparator": "c" * 40, + "target_lean4export": "1" * 40, + } + fields.update(overrides) + return ProducerRecord(**fields) + + +class ProducerRecordTest(unittest.TestCase): + """The sidecar names what produced the artifact, and only that.""" + + def test_the_producer_survives_a_round_trip(self): + bound = a_manifest().with_producer(a_producer(importer_dirty=True)) + loaded = ProblemManifest.from_json(bound.to_json()) + self.assertEqual(loaded.producer, bound.producer) + self.assertTrue(loaded.producer.importer_dirty) + + def test_a_manifest_without_a_producer_stays_without_one(self): + loaded = ProblemManifest.from_json(a_manifest().to_json()) + self.assertIsNone(loaded.producer) + + def test_unknown_producer_keys_are_refused(self): + payload = a_manifest().with_producer(a_producer()).to_json_object() + payload["producer"]["generator"]["binary_path"] = "/tmp/gen" + with self.assertRaisesRegex(SystemExit, "unknown keys"): + ProblemManifest.from_json_object(payload) + + def test_unknown_copied_dependency_keys_are_refused(self): + payload = a_manifest().to_json_object() + payload["source"]["copied_dependencies"][0]["blob"] = "f" * 40 + with self.assertRaisesRegex(SystemExit, "unknown keys"): + ProblemManifest.from_json_object(payload) + From ee9ccf98695a7d2cfe19d4668d9ffec26a7c932d Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:57:24 -0400 Subject: [PATCH 57/70] One request serialisation, piped and emitted and digested alike MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The emitted artifact claimed to be "the exact bytes crossing the seam" while generation piped a different serialisation with an absolute machine-local context root swapped in. The request is now serialised once — relative `context` root, resolved by running the binary with the staging directory as its working directory — and that one string is what the binary reads, what `--emit-import` writes, and what the sidecar digests as `digests.request`. The seam test feeds the emitted file back byte-for-byte instead of reparsing it. --- .github/workflows/build-and-docs.yml | 9 ++++----- comparator/adapter/leaneval_generator_cli.py | 11 ++++++++--- comparator/adapter/leaneval_interface.py | 13 ++++++++++--- .../adapter/make_comparator_workspace.py | 19 ++++++++++++++----- comparator/adapter/test_leaneval_interface.py | 7 ++++++- .../adapter/test_make_comparator_workspace.py | 14 +++++++------- 6 files changed, 49 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build-and-docs.yml b/.github/workflows/build-and-docs.yml index 19c0135133..d71dec5913 100644 --- a/.github/workflows/build-and-docs.yml +++ b/.github/workflows/build-and-docs.yml @@ -253,12 +253,11 @@ jobs: assert len(manifest.source.commit) == 40, manifest.source.commit assert manifest.source.declaration, "no FC declaration id" - request = json.loads( - (handed_over / "request.json").read_text(encoding="utf-8") - ) + # The emitted request is fed back as its exact bytes — no reparse, + # no reserialise — which is the claim the seam test exists to check. + request_text = (handed_over / "request.json").read_text(encoding="utf-8") workspace = pathlib.Path(".comparator/Erdos1038_erdos_1038_parts_i").resolve() - os.chdir(handed_over) - regenerated = generator_cli.generate(request) + regenerated = generator_cli.generate(request_text, cwd=handed_over) files = regenerated["Erdos1038_erdos_1038_parts_i"] for name, content in files.items(): expected = (workspace / name).read_text(encoding="utf-8") diff --git a/comparator/adapter/leaneval_generator_cli.py b/comparator/adapter/leaneval_generator_cli.py index 122a1722ef..3711358416 100644 --- a/comparator/adapter/leaneval_generator_cli.py +++ b/comparator/adapter/leaneval_generator_cli.py @@ -71,16 +71,21 @@ def context_files(problems): return files -def generate(request): +def generate(request_text, cwd=None): """The generator's verified file maps for one request. - Returns `{problem_id: {path: content}}`. + Takes the request as its exact serialised bytes, not a dict: the string + piped to the binary is the same string `--emit-import` writes and the + sidecar digests, so "the exact bytes crossing the seam" is a fact rather + than a paraphrase. The request's `contextRoot` stays the relative + `context`, resolved against `cwd`. Returns `{problem_id: {path: content}}`. """ proc = subprocess.run( [binary()], - input=json.dumps(request), + input=request_text, capture_output=True, text=True, + cwd=cwd, ) if proc.returncode != 0: raise SystemExit( diff --git a/comparator/adapter/leaneval_interface.py b/comparator/adapter/leaneval_interface.py index f1a110fa89..ee7e770661 100644 --- a/comparator/adapter/leaneval_interface.py +++ b/comparator/adapter/leaneval_interface.py @@ -290,16 +290,20 @@ class ProblemManifest: # serialisation is key-sorted. module_sha256: str = "" file_sha256: tuple = () + # The exact request bytes that crossed the seam, as their digest: with + # it, "the emitted request is what generation ran" is checkable. + request_sha256: str = "" # What produced the artifact — importer commit, pinned generator, target # pins — bound at generation time like the digests, absent before it. producer: ProducerRecord = None - def with_digests(self, module_sha256, files): - """The same manifest, bound to the module bytes and generated files.""" + def with_digests(self, module_sha256, files, request_sha256=""): + """The same manifest, bound to the request, module bytes and files.""" return dataclasses.replace( self, module_sha256=module_sha256, file_sha256=tuple(sorted((path, digest) for path, digest in files.items())), + request_sha256=request_sha256, ) def with_producer(self, producer): @@ -341,6 +345,8 @@ def to_json_object(self): "module": self.module_sha256, "files": dict(self.file_sha256), } + if self.request_sha256: + payload["digests"]["request"] = self.request_sha256 if self.producer is not None: payload["producer"] = self.producer.to_json_object() return payload @@ -377,7 +383,7 @@ def from_json_object(cls, payload): if unknown: raise SystemExit(f"provenance source has unknown keys: {', '.join(unknown)}") digests = dict(payload.get("digests", {})) - unknown = sorted(set(digests) - {"module", "files"}) + unknown = sorted(set(digests) - {"module", "files", "request"}) if unknown: raise SystemExit(f"provenance digests have unknown keys: {', '.join(unknown)}") return cls( @@ -392,6 +398,7 @@ def from_json_object(cls, payload): category=payload.get("category", ""), module_sha256=digests.get("module", ""), file_sha256=tuple(sorted(dict(digests.get("files", {})).items())), + request_sha256=digests.get("request", ""), producer=( ProducerRecord.from_json_object(payload["producer"]) if "producer" in payload diff --git a/comparator/adapter/make_comparator_workspace.py b/comparator/adapter/make_comparator_workspace.py index 89eb0bd2eb..ec0b192f33 100644 --- a/comparator/adapter/make_comparator_workspace.py +++ b/comparator/adapter/make_comparator_workspace.py @@ -139,15 +139,18 @@ def seam_files(pairs, group=None): has no field for, so they travel beside it rather than through it. """ request, problems = _seam(pairs, group=group) + request_text = dump_json(request) producer = importer.producer_record() - files = {"request.json": dump_json(request)} + files = {"request.json": request_text} for path, content in generator_cli.context_files(problems).items(): files[f"{CONTEXT_DIR}/{path}"] = content for (problem, _), (_, manifest) in zip(problems, pairs): # Before generation the record binds the module bytes only; the # workspace's copy adds the generated files. bound = manifest.with_digests( - sha256_text(problem["moduleContent"]), {} + sha256_text(problem["moduleContent"]), + {}, + request_sha256=sha256_text(request_text), ).with_producer(producer) files[f"{PROVENANCE_STEM}-{problem['id']}.json"] = bound.to_json() return request, files @@ -156,14 +159,18 @@ def seam_files(pairs, group=None): def generate_workspaces(pairs, out_dir, group=None): """Generate one workspace per pair under `out_dir`, via the pinned binary.""" request, problems = _seam(pairs, group=group) + # One serialisation, used everywhere: the string piped to the binary is + # the string `--emit-import` writes and the sidecar digests. The request + # keeps its relative `contextRoot`; the binary runs with the staging + # directory as its working directory, which is where that root resolves. + request_text = dump_json(request) staging = pathlib.Path(tempfile.mkdtemp(prefix=".fc-seam.")) try: # Only the context crosses to the binary; the request goes on stdin # and the provenance sidecars belong to the written workspaces, so # neither is staged here. _write_files(staging / CONTEXT_DIR, generator_cli.context_files(problems)) - request["contextRoot"] = str(staging / CONTEXT_DIR) - workspaces = generator_cli.generate(request) + workspaces = generator_cli.generate(request_text, cwd=staging) finally: shutil.rmtree(staging, ignore_errors=True) module_content = {p["id"]: p["moduleContent"] for p in request["problems"]} @@ -176,10 +183,12 @@ def generate_workspaces(pairs, out_dir, group=None): workspace = dict(workspaces[problem_id]) # The provenance sidecar rides in the workspace directory, not in the # generator's file map: the generator neither knows nor checks it. It - # binds the exact module bytes sent and every file received. + # binds the exact request and module bytes sent and every file + # received. bound = manifest.with_digests( sha256_text(module_content[problem_id]), {path: sha256_text(content) for path, content in workspace.items()}, + request_sha256=sha256_text(request_text), ).with_producer(producer) workspace[PROVENANCE_FILE] = bound.to_json() written.append(write_tree(pathlib.Path(out_dir) / problem_id, workspace)) diff --git a/comparator/adapter/test_leaneval_interface.py b/comparator/adapter/test_leaneval_interface.py index 50a91df6d0..05266c64f6 100644 --- a/comparator/adapter/test_leaneval_interface.py +++ b/comparator/adapter/test_leaneval_interface.py @@ -345,10 +345,15 @@ def test_unknown_keys_are_refused(self): def test_unknown_digest_keys_are_refused(self): payload = a_manifest().with_digests("a" * 64, {}).to_json_object() - payload["digests"]["request"] = "d" * 64 + payload["digests"]["blake3"] = "d" * 64 with self.assertRaises(SystemExit): ProblemManifest.from_json_object(payload) + def test_the_request_digest_survives_a_round_trip(self): + bound = a_manifest().with_digests("a" * 64, {}, request_sha256="d" * 64) + loaded = ProblemManifest.from_json(bound.to_json()) + self.assertEqual(loaded.request_sha256, "d" * 64) + class ProblemGroupTest(unittest.TestCase): """Categories map to lean-eval groups; non-problems are refused.""" diff --git a/comparator/adapter/test_make_comparator_workspace.py b/comparator/adapter/test_make_comparator_workspace.py index 5bd488ec95..698929c372 100644 --- a/comparator/adapter/test_make_comparator_workspace.py +++ b/comparator/adapter/test_make_comparator_workspace.py @@ -35,6 +35,7 @@ seam_files, write_tree, ) +from leaneval_interface import dump_json from test_leaneval_interface import A_MODULE, a_manifest @@ -165,13 +166,12 @@ def test_the_emitted_request_yields_identical_digests(self): path = root / relative path.parent.mkdir(parents=True, exist_ok=True) path.write_text(content, encoding="utf-8") - cwd = os.getcwd() - try: - os.chdir(root) - first = generator_cli.generate(request) - second = generator_cli.generate(request) - finally: - os.chdir(cwd) + # The emitted request IS what generation runs: the same bytes, + # resolved from inside the emitted directory. + request_text = (root / "request.json").read_text(encoding="utf-8") + self.assertEqual(request_text, dump_json(request)) + first = generator_cli.generate(request_text, cwd=root) + second = generator_cli.generate(request_text, cwd=root) self.assertEqual(first, second) self.assertEqual(sorted(first), ["erdos_940"]) From d7c23fd18fe71061b4857c7e483b01ee71a98f55 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:59:33 -0400 Subject: [PATCH 58/70] Fail closed on everything the response could get wrong MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `parse_response` now refuses what it previously let through: unknown fields at either level, a missing field (a refusal, not a KeyError), paths that are absolute, traverse upward, or carry dot/empty/backslash components — which also makes each path its own normal form, so two spellings of one file cannot dodge the duplicate check — a response naming this side's `fc-provenance.json`, and, given the requested ids, any workspace missing or extra. `_write_files` re-checks containment at the write. A `--set` batch validates everything, sidecars and target collisions included, before the first workspace lands, so a refused batch writes nothing rather than a prefix of itself. --- comparator/adapter/leaneval_generator_cli.py | 7 +- comparator/adapter/leaneval_interface.py | 83 +++++++++++++++++-- .../adapter/make_comparator_workspace.py | 44 +++++++--- .../adapter/test_fc_leaneval_importer.py | 4 - comparator/adapter/test_leaneval_interface.py | 73 ++++++++++++---- 5 files changed, 167 insertions(+), 44 deletions(-) diff --git a/comparator/adapter/leaneval_generator_cli.py b/comparator/adapter/leaneval_generator_cli.py index 3711358416..a96244a416 100644 --- a/comparator/adapter/leaneval_generator_cli.py +++ b/comparator/adapter/leaneval_generator_cli.py @@ -71,14 +71,15 @@ def context_files(problems): return files -def generate(request_text, cwd=None): +def generate(request_text, cwd=None, expected_ids=None): """The generator's verified file maps for one request. Takes the request as its exact serialised bytes, not a dict: the string piped to the binary is the same string `--emit-import` writes and the sidecar digests, so "the exact bytes crossing the seam" is a fact rather than a paraphrase. The request's `contextRoot` stays the relative - `context`, resolved against `cwd`. Returns `{problem_id: {path: content}}`. + `context`, resolved against `cwd`. With `expected_ids`, the response must + cover exactly those workspaces. Returns `{problem_id: {path: content}}`. """ proc = subprocess.run( [binary()], @@ -91,4 +92,4 @@ def generate(request_text, cwd=None): raise SystemExit( f"lean-eval-generator failed:\n{proc.stderr.strip() or proc.stdout.strip()}" ) - return parse_response(proc.stdout) + return parse_response(proc.stdout, expected_ids=expected_ids) diff --git a/comparator/adapter/leaneval_interface.py b/comparator/adapter/leaneval_interface.py index ee7e770661..ce1bd9a24f 100644 --- a/comparator/adapter/leaneval_interface.py +++ b/comparator/adapter/leaneval_interface.py @@ -687,31 +687,96 @@ def build_request(problems, target, workspace_test, context_root): } -def parse_response(text): - """The generator's file maps, with every digest checked. +def safe_workspace_path(path): + """A response path fit to join under a directory, or a refusal. + + The pinned generator is still an external process across a versioned + boundary; a future defect there must fail closed here, not write outside + the staging tree. Strictly relative POSIX, every component a real name: + no absolute paths, no drive letters, no backslashes, no NUL, no empty or + `.`/`..` components — which also makes the supplied spelling its own + normal form, so two spellings of one file cannot slip past the duplicate + check. + """ + if not path or "\x00" in path or "\\" in path: + raise SystemExit(f"response path {path!r} is not a plain relative path") + if path.startswith("/") or re.match(r"^[A-Za-z]:", path): + raise SystemExit(f"response path {path!r} is not relative") + if any(part in ("", ".", "..") for part in path.split("/")): + raise SystemExit(f"response path {path!r} has an empty or dot component") + return path + + +# The provenance sidecar's name inside a workspace. It is this side's file: +# a generator response naming it would silently lose to the sidecar written +# after it, so the response parser refuses it outright. +PROVENANCE_STEM = "fc-provenance" +PROVENANCE_FILE = f"{PROVENANCE_STEM}.json" + + +def parse_response(text, expected_ids=None): + """The generator's file maps, with every byte and identity checked. Returns `{problem_id: {path: content}}`. A digest mismatch means the bytes were damaged in transit or the pinned generator is not the one this code was written against; either way the workspace cannot be - trusted, so refuse. + trusted, so refuse. The same goes for shape: unknown fields, missing + fields, unsafe paths and — when `expected_ids` is given — any mismatch + between the workspaces requested and the workspaces returned. An extra + workspace dropped on the floor is as wrong as a missing one. """ - payload = json.loads(text) + try: + payload = json.loads(text) + except json.JSONDecodeError as error: + raise SystemExit(f"generator response is not JSON: {error}") from None + if not isinstance(payload, dict): + raise SystemExit("generator response is not a JSON object") + unknown = sorted(set(payload) - {"schemaVersion", "files"}) + if unknown: + raise SystemExit(f"generator response has unknown keys: {', '.join(unknown)}") version = payload.get("schemaVersion") if version != CONTRACT_VERSION: raise SystemExit( f"generator response schema version {version!r} is not {CONTRACT_VERSION}" ) + if "files" not in payload: + raise SystemExit("generator response has no files") workspaces = {} for entry in payload["files"]: + unknown = sorted(set(entry) - {"problemId", "path", "sha256", "content"}) + if unknown: + raise SystemExit( + f"a response entry has unknown keys: {', '.join(unknown)}" + ) + missing = sorted({"problemId", "path", "sha256", "content"} - set(entry)) + if missing: + raise SystemExit(f"a response entry has no {', '.join(missing)}") + path = safe_workspace_path(entry["path"]) + if path == PROVENANCE_FILE: + raise SystemExit( + f"{entry['problemId']}: the response names {PROVENANCE_FILE}, " + "which is this side's provenance sidecar" + ) if sha256_text(entry["content"]) != entry["sha256"]: raise SystemExit( - f"{entry['problemId']}/{entry['path']}: content does not match " - "its digest" + f"{entry['problemId']}/{path}: content does not match its digest" ) files = workspaces.setdefault(entry["problemId"], {}) - if entry["path"] in files: + if path in files: + raise SystemExit(f"{entry['problemId']}/{path}: appears twice in response") + files[path] = entry["content"] + if expected_ids is not None: + expected = set(expected_ids) + returned = set(workspaces) + missing = sorted(expected - returned) + extra = sorted(returned - expected) + if missing: + raise SystemExit( + f"the generator returned no files for {', '.join(missing)}" + ) + if extra: raise SystemExit( - f"{entry['problemId']}/{entry['path']}: appears twice in response" + "the generator returned workspaces nothing requested: " + f"{', '.join(extra)}" ) - files[entry["path"]] = entry["content"] return workspaces diff --git a/comparator/adapter/make_comparator_workspace.py b/comparator/adapter/make_comparator_workspace.py index ec0b192f33..b33f778bcb 100644 --- a/comparator/adapter/make_comparator_workspace.py +++ b/comparator/adapter/make_comparator_workspace.py @@ -69,22 +69,36 @@ import fc_leaneval_importer as importer import fc_source import leaneval_generator_cli as generator_cli -from leaneval_interface import build_problem, build_request, dump_json, sha256_text, slug +from leaneval_interface import ( + PROVENANCE_FILE, + PROVENANCE_STEM, + build_problem, + build_request, + dump_json, + sha256_text, + slug, +) ROOT = importer.ROOT -PROVENANCE_STEM = "fc-provenance" -PROVENANCE_FILE = f"{PROVENANCE_STEM}.json" - # The request's context directory, relative to the request file, so an # emitted seam artifact is self-contained and reproducible from any path. CONTEXT_DIR = "context" def _write_files(directory, files): - """Materialise a `{relative path: content}` mapping under `directory`.""" + """Materialise a `{relative path: content}` mapping under `directory`. + + Every destination must land inside `directory`: the mapping may contain + paths from a response, and `parse_response` already refuses unsafe ones, + but the writer is the last line and checks for itself. + """ + directory = pathlib.Path(directory) + base = directory.resolve() for relative, content in files.items(): destination = directory / relative + if not destination.resolve().is_relative_to(base): + raise SystemExit(f"{relative}: escapes the output directory") destination.parent.mkdir(parents=True, exist_ok=True) destination.write_text(content, encoding="utf-8") @@ -170,16 +184,21 @@ def generate_workspaces(pairs, out_dir, group=None): # and the provenance sidecars belong to the written workspaces, so # neither is staged here. _write_files(staging / CONTEXT_DIR, generator_cli.context_files(problems)) - workspaces = generator_cli.generate(request_text, cwd=staging) + workspaces = generator_cli.generate( + request_text, + cwd=staging, + expected_ids=[p["id"] for p in request["problems"]], + ) finally: shutil.rmtree(staging, ignore_errors=True) module_content = {p["id"]: p["moduleContent"] for p in request["problems"]} producer = importer.producer_record() - written = [] + # Every refusal — digest, identity, sidecar, target-already-exists — + # happens before the first workspace lands, so a refused batch writes + # nothing rather than a prefix of itself. + outputs = [] for _, manifest in pairs: problem_id = slug(manifest.id) - if problem_id not in workspaces: - raise SystemExit(f"the generator returned no files for {problem_id}") workspace = dict(workspaces[problem_id]) # The provenance sidecar rides in the workspace directory, not in the # generator's file map: the generator neither knows nor checks it. It @@ -191,8 +210,11 @@ def generate_workspaces(pairs, out_dir, group=None): request_sha256=sha256_text(request_text), ).with_producer(producer) workspace[PROVENANCE_FILE] = bound.to_json() - written.append(write_tree(pathlib.Path(out_dir) / problem_id, workspace)) - return written + outputs.append((pathlib.Path(out_dir) / problem_id, workspace)) + for target, _ in outputs: + if target.exists(): + raise SystemExit(f"refusing to overwrite existing workspace: {target}") + return [write_tree(target, files) for target, files in outputs] def subset_declarations(set_name): diff --git a/comparator/adapter/test_fc_leaneval_importer.py b/comparator/adapter/test_fc_leaneval_importer.py index f0049f3512..bf080f26d8 100644 --- a/comparator/adapter/test_fc_leaneval_importer.py +++ b/comparator/adapter/test_fc_leaneval_importer.py @@ -370,7 +370,3 @@ def test_the_closure_region_does_not_carry_the_import(self): if __name__ == "__main__": unittest.main() - - -if __name__ == "__main__": - unittest.main() diff --git a/comparator/adapter/test_leaneval_interface.py b/comparator/adapter/test_leaneval_interface.py index 05266c64f6..431db854cc 100644 --- a/comparator/adapter/test_leaneval_interface.py +++ b/comparator/adapter/test_leaneval_interface.py @@ -187,9 +187,6 @@ def test_a_qualified_declaration_becomes_an_identifier(self): ) -if __name__ == "__main__": - unittest.main() - class DeclarationSpanTest(unittest.TestCase): """Spans are computed from the rendered text, exactly.""" @@ -293,23 +290,62 @@ def test_duplicate_ids_are_refused(self): class ParseResponseTest(unittest.TestCase): - def _response(self, content="hello"): + def _response(self, content="hello", **entry_overrides): import hashlib import json - return json.dumps( - { - "schemaVersion": 1, - "files": [ - { - "problemId": "p", - "path": "a.txt", - "sha256": hashlib.sha256(content.encode()).hexdigest(), - "content": "hello", - } - ], - } - ) + entry = { + "problemId": "p", + "path": "a.txt", + "sha256": hashlib.sha256(content.encode()).hexdigest(), + "content": "hello", + } + entry.update(entry_overrides) + return json.dumps({"schemaVersion": 1, "files": [entry]}) + + def test_a_traversal_path_is_refused(self): + for path in ("../outside.txt", "a/../../b.txt", "/etc/x", "a//b.txt", + "./a.txt", "a\\b.txt", "C:whatever"): + with self.subTest(path=path): + with self.assertRaisesRegex(SystemExit, "response path"): + parse_response(self._response(path=path)) + + def test_the_sidecar_name_is_refused(self): + # A response naming fc-provenance.json would silently lose to the + # sidecar written after it — and its recorded digest would then + # describe bytes no longer on disk. + with self.assertRaisesRegex(SystemExit, "provenance sidecar"): + parse_response(self._response(path="fc-provenance.json")) + + def test_unknown_response_keys_are_refused(self): + import json + + payload = json.loads(self._response()) + payload["extra"] = True + with self.assertRaisesRegex(SystemExit, "unknown keys"): + parse_response(json.dumps(payload)) + + def test_unknown_entry_keys_are_refused(self): + with self.assertRaisesRegex(SystemExit, "unknown keys"): + parse_response(self._response(mode="0755")) + + def test_a_missing_entry_field_is_a_refusal_not_a_crash(self): + import json + + payload = json.loads(self._response()) + del payload["files"][0]["sha256"] + with self.assertRaisesRegex(SystemExit, "has no sha256"): + parse_response(json.dumps(payload)) + + def test_non_json_is_a_refusal_not_a_crash(self): + with self.assertRaisesRegex(SystemExit, "not JSON"): + parse_response("lake build output\n{") + + def test_the_returned_workspaces_must_be_the_requested_ones(self): + with self.assertRaisesRegex(SystemExit, "returned no files for q"): + parse_response(self._response(), expected_ids=["p", "q"]) + with self.assertRaisesRegex(SystemExit, "nothing requested"): + parse_response(self._response(), expected_ids=[]) def test_a_good_response_yields_the_file_map(self): self.assertEqual(parse_response(self._response()), {"p": {"a.txt": "hello"}}) @@ -444,3 +480,6 @@ def test_unknown_copied_dependency_keys_are_refused(self): with self.assertRaisesRegex(SystemExit, "unknown keys"): ProblemManifest.from_json_object(payload) + +if __name__ == "__main__": + unittest.main() From 58e51ff785d726bc4b6deb29988ef0201e0bff5e Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:01:44 -0400 Subject: [PATCH 59/70] State the intake policy explicitly; check the recorded axiom policy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Destination group, lifecycle status, visibility, statement revision and submitter are LeanEval's decisions, but they lived as constants inside the wire-format module. They now arrive as an explicit `ImportPolicy` the command constructs — the draft-intake instantiation in one place, a real LeanEval intake free to build its own. The Mathlib repository URL moves to `tools.toml [target]` with the pins it belongs beside. The sidecar's `permitted_axioms` was an assertion nothing checked: the generator owns the Comparator config. Generation now compares the config actually produced against the recorded tuple, so the two cannot drift silently. `slug` and `DefinitionHole` move to `fc_source`, the domain module, so the dependency runs wire-on-source rather than the reverse and fc_source's own "knows nothing of requests" claim is true. --- comparator/adapter/fc_leaneval_importer.py | 3 +- comparator/adapter/fc_source.py | 30 +++++-- comparator/adapter/leaneval_interface.py | 78 +++++++++---------- .../adapter/make_comparator_workspace.py | 35 ++++++++- comparator/adapter/test_leaneval_interface.py | 30 +++++-- comparator/tools.toml | 1 + 6 files changed, 119 insertions(+), 58 deletions(-) diff --git a/comparator/adapter/fc_leaneval_importer.py b/comparator/adapter/fc_leaneval_importer.py index 5455c82d50..ff528fcec8 100644 --- a/comparator/adapter/fc_leaneval_importer.py +++ b/comparator/adapter/fc_leaneval_importer.py @@ -89,12 +89,13 @@ def target_pins(): read and where they will be used a readable fact rather than an assumption. """ target = _tools_file()["target"] - for key in ("lean_toolchain", "mathlib_revision"): + for key in ("lean_toolchain", "mathlib_revision", "mathlib_git"): if not target.get(key): raise SystemExit(f"comparator/tools.toml [target] has an empty `{key}`") return TargetRecord( lean_toolchain=target["lean_toolchain"], mathlib_revision=target["mathlib_revision"], + mathlib_git=target["mathlib_git"], ) diff --git a/comparator/adapter/fc_source.py b/comparator/adapter/fc_source.py index 675278e9c2..bd8a01ec26 100644 --- a/comparator/adapter/fc_source.py +++ b/comparator/adapter/fc_source.py @@ -17,9 +17,31 @@ import re import subprocess -from leaneval_interface import ( - DefinitionHole, -) + +def slug(name): + """A Lake package name and directory name for a problem id. + + A Lake package name is an identifier, so the dots in a qualified + declaration cannot go into one verbatim. + """ + return re.sub(r"[^0-9A-Za-z_]", "_", name) + + +@dataclasses.dataclass(frozen=True) +class DefinitionHole: + """One `answer(sorry)` slot, hoisted into a definition the solver fills. + + `name` is the unqualified definition name as it appears in the module's + `holes` region; `type` is the type the elaborated environment reported for + the slot, which surface syntax does not carry. + """ + + name: str + type: str + + def declaration(self): + return f"noncomputable def {self.name} : {self.type} := sorry" + ROOT = pathlib.Path(__file__).resolve().parent.parent.parent @@ -619,8 +641,6 @@ def flatten_declared_name(declared, statement): rewrite is refused rather than guessed if the name cannot be found where the declaration keyword put it. """ - from leaneval_interface import slug - flattened = slug(declared) lines = statement.split("\n") for index, line in enumerate(lines): diff --git a/comparator/adapter/leaneval_interface.py b/comparator/adapter/leaneval_interface.py index ce1bd9a24f..47576c30e9 100644 --- a/comparator/adapter/leaneval_interface.py +++ b/comparator/adapter/leaneval_interface.py @@ -60,6 +60,8 @@ import json import re +from fc_source import DefinitionHole, slug + REGIONS = ("dependencies", "scope", "holes", "statement") MODULE_PREAMBLE = "import Mathlib\n" @@ -92,31 +94,6 @@ def dump_json(obj, sort_keys=False): CONTRACT_VERSION = 1 -def slug(name): - """A Lake package name and directory name for a problem id. - - A Lake package name is an identifier, so the dots in a qualified - declaration cannot go into one verbatim. - """ - return re.sub(r"[^0-9A-Za-z_]", "_", name) - - -@dataclasses.dataclass(frozen=True) -class DefinitionHole: - """One `answer(sorry)` slot, hoisted into a definition the solver fills. - - `name` is the unqualified definition name as it appears in the module's - `holes` region; `type` is the type the elaborated environment reported for - the slot, which surface syntax does not carry. - """ - - name: str - type: str - - def declaration(self): - return f"noncomputable def {self.name} : {self.type} := sorry" - - @dataclasses.dataclass(frozen=True) class SourceRecord: """Where the marked-up module's text came from. @@ -252,6 +229,26 @@ class TargetRecord: lean_toolchain: str mathlib_revision: str + mathlib_git: str = "https://github.com/leanprover-community/mathlib4.git" + + +@dataclasses.dataclass(frozen=True) +class ImportPolicy: + """LeanEval-side catalog policy, stated explicitly per import run. + + Group, lifecycle status, visibility, statement revision, submitter and + tags are the consumer's decisions, not source facts; the schema-version-1 + request happens to require them inline, so the command constructs a + policy and hands it in rather than this module hard-coding one. An empty + `group` means "derive from the category" via `CATEGORY_GROUPS`. + """ + + group: str + status: str + visible: bool + statement_revision: int + submitter: str + tags: tuple @dataclasses.dataclass(frozen=True) @@ -475,11 +472,6 @@ def problem_group(manifest): return group -MATHLIB_GIT = "https://github.com/leanprover-community/mathlib4.git" - -SUBMITTER = "formal-conjectures-importer" - - def module_declarations(marked_up, manifest): """Every declaration in the rendered module, in order. @@ -587,7 +579,7 @@ def line_of(offset): return spans -def build_problem(marked_up, manifest, module_name=None, group=None): +def build_problem(marked_up, manifest, policy, module_name=None): """One problem entry of the schema-version-1 request, and its `.ilean` declaration map. The module name is a single identifier on purpose: the generator resolves @@ -595,11 +587,11 @@ def build_problem(marked_up, manifest, module_name=None, group=None): a dotted or quoted name would trip the same decoder defect this repository fixed on its own side. - `group` overrides the category-derived group for members of a frozen - set: the set decides the display tab, because the list is immutable - while its members keep getting solved, and the category rides along as - a tag. The category is still validated either way — a declaration that - is not a problem has no business in any group. + `policy` is the LeanEval-side intake policy: a frozen set's explicit + `group` overrides the category-derived one, because the list is + immutable while its members keep getting solved, and the category rides + along as a tag. The category is still validated either way — a + declaration that is not a problem has no business in any group. Returns `(problem, ilean_decls)`. The `.ilean` payload exists because the generator reads helper-declaration spans from compiled metadata it @@ -643,14 +635,14 @@ def build_problem(marked_up, manifest, module_name=None, group=None): problem = { "id": slug(manifest.id), "title": manifest.qualified_theorem, - "group": group or category_group, - "status": "draft", - "visible": True, - "statementRevision": 1, - "tags": ["formal-conjectures", manifest.category.replace(" ", "-")], + "group": policy.group or category_group, + "status": policy.status, + "visible": policy.visible, + "statementRevision": policy.statement_revision, + "tags": list(policy.tags) + [manifest.category.replace(" ", "-")], "moduleName": module_name, "holes": [entry["declarationName"] for entry in resolved], - "submitter": SUBMITTER, + "submitter": policy.submitter, "notes": None, "source": manifest.source_url or None, "informalSolution": None, @@ -679,7 +671,7 @@ def build_request(problems, target, workspace_test, context_root): "leanToolchain": target.lean_toolchain, "mathlib": { "name": "mathlib", - "git": MATHLIB_GIT, + "git": target.mathlib_git, "rev": target.mathlib_revision, }, "templates": {"workspaceTest": workspace_test}, diff --git a/comparator/adapter/make_comparator_workspace.py b/comparator/adapter/make_comparator_workspace.py index b33f778bcb..3e3dad043b 100644 --- a/comparator/adapter/make_comparator_workspace.py +++ b/comparator/adapter/make_comparator_workspace.py @@ -59,6 +59,7 @@ """ import argparse +import json import pathlib import re import shutil @@ -70,6 +71,7 @@ import fc_source import leaneval_generator_cli as generator_cli from leaneval_interface import ( + ImportPolicy, PROVENANCE_FILE, PROVENANCE_STEM, build_problem, @@ -127,10 +129,30 @@ def write_tree(target, files): return target +def import_policy(group=None): + """The intake policy this command submits under, stated in one place. + + Destination group, lifecycle status, visibility, statement revision and + submitter are LeanEval's decisions; the schema-version-1 request requires + them inline, so the command instantiates the draft-intake values + explicitly rather than leaving them as wire-module constants. A real + LeanEval intake would build this from the catalog's own state. + """ + return ImportPolicy( + group=group or "", + status="draft", + visible=True, + statement_revision=1, + submitter="formal-conjectures-importer", + tags=("formal-conjectures",), + ) + + def _seam(pairs, group=None): """The request and its `build_problem` outputs for `(marked_up, manifest)` pairs.""" + policy = import_policy(group) problems = [ - build_problem(marked_up, manifest, group=group) + build_problem(marked_up, manifest, policy) for marked_up, manifest in pairs ] target = importer.target_pins() @@ -204,6 +226,17 @@ def generate_workspaces(pairs, out_dir, group=None): # generator's file map: the generator neither knows nor checks it. It # binds the exact request and module bytes sent and every file # received. + # The sidecar's `permitted_axioms` is an assertion about the + # generated Comparator config; check it against the config actually + # produced, so the recorded policy cannot drift from the enforced one. + config = json.loads(workspace.get("config.json", "{}")) + generated_axioms = tuple(config.get("permitted_axioms", ())) + if sorted(generated_axioms) != sorted(manifest.permitted_axioms): + raise SystemExit( + f"{problem_id}: the generated config permits axioms " + f"{sorted(generated_axioms)}, but the manifest records " + f"{sorted(manifest.permitted_axioms)}" + ) bound = manifest.with_digests( sha256_text(module_content[problem_id]), {path: sha256_text(content) for path, content in workspace.items()}, diff --git a/comparator/adapter/test_leaneval_interface.py b/comparator/adapter/test_leaneval_interface.py index 431db854cc..7f7d11cb10 100644 --- a/comparator/adapter/test_leaneval_interface.py +++ b/comparator/adapter/test_leaneval_interface.py @@ -26,6 +26,7 @@ from leaneval_interface import ( DefinitionHole, + ImportPolicy, MarkedUpModule, ProblemManifest, ProducerRecord, @@ -77,6 +78,19 @@ def a_target(**overrides): return TargetRecord(**fields) +def a_policy(**overrides): + fields = { + "group": "", + "status": "draft", + "visible": True, + "statement_revision": 1, + "submitter": "formal-conjectures-importer", + "tags": ("formal-conjectures",), + } + fields.update(overrides) + return ImportPolicy(**fields) + + def a_manifest(**overrides): fields = { "id": "erdos_940", @@ -226,7 +240,7 @@ def test_utf16_columns_count_supplementary_plane_pairs(self): class BuildProblemTest(unittest.TestCase): def test_the_problem_satisfies_the_contract_shape(self): - problem, ilean = build_problem(A_MODULE, a_manifest()) + problem, ilean = build_problem(A_MODULE, a_manifest(), a_policy()) self.assertEqual(problem["id"], "erdos_940") self.assertEqual(problem["group"], "open-conjectures") self.assertEqual(problem["moduleName"], "erdos_940") @@ -243,17 +257,17 @@ def test_the_problem_satisfies_the_contract_shape(self): ) def test_the_theorem_hole_carries_the_copied_dependencies(self): - problem, _ = build_problem(A_MODULE, a_manifest()) + problem, _ = build_problem(A_MODULE, a_manifest(), a_policy()) theorem = problem["resolvedHoles"][-1] self.assertEqual(theorem["sameModuleDependencies"], ["Foo.bar"]) self.assertEqual(problem["resolvedHoles"][0]["sameModuleDependencies"], []) def test_a_non_problem_category_is_refused(self): with self.assertRaises(SystemExit): - build_problem(A_MODULE, a_manifest(category="API")) + build_problem(A_MODULE, a_manifest(category="API"), a_policy()) def test_the_category_rides_along_as_a_tag(self): - problem, _ = build_problem(A_MODULE, a_manifest(category="research solved")) + problem, _ = build_problem(A_MODULE, a_manifest(category="research solved"), a_policy()) self.assertIn("research-solved", problem["tags"]) def test_a_set_override_keeps_a_solved_member_in_its_set(self): @@ -263,20 +277,20 @@ def test_a_set_override_keeps_a_solved_member_in_its_set(self): problem, _ = build_problem( A_MODULE, a_manifest(category="research solved"), - group="open-conjectures", + a_policy(group="open-conjectures"), ) self.assertEqual(problem["group"], "open-conjectures") def test_a_set_override_does_not_admit_a_non_problem(self): with self.assertRaises(SystemExit): build_problem( - A_MODULE, a_manifest(category="API"), group="open-conjectures" + A_MODULE, a_manifest(category="API"), a_policy(group="open-conjectures") ) class BuildRequestTest(unittest.TestCase): def test_the_request_carries_the_targets_pins(self): - problem, _ = build_problem(A_MODULE, a_manifest()) + problem, _ = build_problem(A_MODULE, a_manifest(), a_policy()) request = build_request([problem], a_target(), "-- test", "context") self.assertEqual(request["schemaVersion"], 1) self.assertEqual(request["leanToolchain"], "leanprover/lean4:v4.33.0") @@ -284,7 +298,7 @@ def test_the_request_carries_the_targets_pins(self): self.assertEqual(request["templates"]["workspaceTest"], "-- test") def test_duplicate_ids_are_refused(self): - problem, _ = build_problem(A_MODULE, a_manifest()) + problem, _ = build_problem(A_MODULE, a_manifest(), a_policy()) with self.assertRaisesRegex(SystemExit, "duplicate workspace id"): build_request([problem, problem], a_target(), "", "context") diff --git a/comparator/tools.toml b/comparator/tools.toml index 657bea84d5..1aa65b3d46 100644 --- a/comparator/tools.toml +++ b/comparator/tools.toml @@ -22,6 +22,7 @@ comparator_repository = "https://github.com/leanprover/comparator" commit = "7699436464052268e6c04b41554bfbc2c6908ec5" lean_toolchain = "leanprover/lean4:v4.33.0" mathlib_revision = "6f1ef4e5dd604a435bddba4747b13970cd65d2a1" +mathlib_git = "https://github.com/leanprover-community/mathlib4.git" comparator = "c0c5a52d2aff92b457c3e5ed4a68c1ebc5795809" lean4export = "15f6055e299ad5b89345e533cc2192f4cc00f659" From 9b7c8aac6c7903904b4868b6dd1957a36218de99 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:03:37 -0400 Subject: [PATCH 60/70] JSON lines for the batch seam; a strict ledger; timeouts everywhere they wait MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The batch protocol's space-delimited lines could not carry every legal Lean name — a guillemet component may contain anything — so both sides now speak JSON lines. The Python side refuses an answer for a pair nothing asked about instead of caching whatever echoes back, and both extractor invocations carry timeouts, so a hang ends the run instead of the day. The ledger loader refuses unknown keys and duplicate declarations: a typo'd field rode along invisibly, and last-one-wins would shadow an entry and quietly defeat the exact match the gates promise. --- comparator/adapter/comparator_facts.lean | 19 ++++--- comparator/adapter/fc_source.py | 55 ++++++++++++++----- comparator/adapter/known_failures.py | 21 ++++++- .../adapter/test_make_comparator_workspace.py | 17 ++++++ 4 files changed, 89 insertions(+), 23 deletions(-) diff --git a/comparator/adapter/comparator_facts.lean b/comparator/adapter/comparator_facts.lean index 80e1b5144b..dc0f2fda47 100644 --- a/comparator/adapter/comparator_facts.lean +++ b/comparator/adapter/comparator_facts.lean @@ -50,16 +50,21 @@ unsafe def main (args : List String) : IO UInt32 := do | ["--self-test"] => runWithImports #[`Mathlib] do binderBoundarySelfTest (← getEnv) | ["--batch"] => - -- One `module declaration` pair per stdin line, one environment for all - -- of them: the Mathlib import dominates a run, and `resolveIn` filters - -- by module, so a shared environment answers each pair exactly as a - -- per-module import does. One JSON object per line, in input order. + -- One `{"module": M, "declaration": D}` object per stdin line, one + -- environment for all of them: the Mathlib import dominates a run, and + -- `resolveIn` filters by module, so a shared environment answers each + -- pair exactly as a per-module import does. JSON lines rather than + -- space-delimited fields, because a guillemet name may contain anything. + -- One JSON object per line out, in input order. let stdin ← IO.getStdin let lines := (← stdin.readToEnd).splitOn "\n" |>.filter (· ≠ "") let pairs ← lines.mapM fun line => do - match line.splitOn " " with - | [modName, declName] => pure (modName, declName) - | _ => throw <| IO.userError s!"malformed batch line: {line}" + match Json.parse line with + | .error msg => throw <| IO.userError s!"malformed batch line: {line} ({msg})" + | .ok json => + match json.getObjValAs? String "module", json.getObjValAs? String "declaration" with + | .ok modName, .ok declName => pure (modName, declName) + | _, _ => throw <| IO.userError s!"malformed batch line: {line}" let modules := pairs.foldl (init := #[]) fun acc (m, _) => if acc.contains m.toName then acc else acc.push m.toName -- The heartbeat budget is shared by the whole action, so it scales with diff --git a/comparator/adapter/fc_source.py b/comparator/adapter/fc_source.py index bd8a01ec26..da74d2869f 100644 --- a/comparator/adapter/fc_source.py +++ b/comparator/adapter/fc_source.py @@ -149,23 +149,42 @@ def prefetch_elaborator_facts(pairs): wanted = [pair for pair in dict.fromkeys(pairs) if pair not in _FACTS_CACHE] if not wanted: return - proc = subprocess.run( - ["lake", "exe", "comparator_facts", "--batch"], - input="".join(f"{module} {declaration}\n" for module, declaration in wanted), - capture_output=True, - text=True, - cwd=ROOT, - ) - if proc.returncode != 0: + try: + proc = subprocess.run( + ["lake", "exe", "comparator_facts", "--batch"], + input="".join( + json.dumps({"module": module, "declaration": declaration}) + "\n" + for module, declaration in wanted + ), + capture_output=True, + text=True, + cwd=ROOT, + # Generous: a cold run imports Mathlib and may build the + # extractor first. A hang should end the run, not the day. + timeout=1800 + 30 * len(wanted), + ) + except subprocess.TimeoutExpired: # The batch is an optimisation; the per-declaration path is the # arbiter of what fails and how it is reported. return + if proc.returncode != 0: + return + requested = set(wanted) for line in proc.stdout.splitlines(): if not line.startswith("{"): + # `lake` progress lines share stdout with the payload; anything + # non-JSON is theirs. Error entries are also left uncached, so + # the single re-run reproduces the exact message. continue entry = json.loads(line) + key = (entry["module"], entry["declaration"]) + if key not in requested: + raise SystemExit( + f"comparator_facts --batch answered for {key[1]} in {key[0]}, " + "which nothing asked about" + ) if "facts" in entry: - _FACTS_CACHE[(entry["module"], entry["declaration"])] = entry["facts"] + _FACTS_CACHE[key] = entry["facts"] def elaborator_facts(module, declaration): @@ -181,12 +200,18 @@ def elaborator_facts(module, declaration): cached = _FACTS_CACHE.get((module, declaration)) if cached is not None: return FactsRecord.from_payload(cached, declaration) - proc = subprocess.run( - ["lake", "exe", "comparator_facts", module, declaration], - capture_output=True, - text=True, - cwd=ROOT, - ) + try: + proc = subprocess.run( + ["lake", "exe", "comparator_facts", module, declaration], + capture_output=True, + text=True, + cwd=ROOT, + timeout=1800, + ) + except subprocess.TimeoutExpired: + raise SystemExit( + f"comparator_facts {declaration}: no answer within 30 minutes" + ) from None if proc.returncode != 0: raise SystemExit( f"comparator_facts {declaration}: " diff --git a/comparator/adapter/known_failures.py b/comparator/adapter/known_failures.py index 15a7031bd3..fc0f2ee185 100644 --- a/comparator/adapter/known_failures.py +++ b/comparator/adapter/known_failures.py @@ -22,8 +22,17 @@ import tomllib +KNOWN_KEYS = frozenset({"declaration", "stage", "reason", "workspace"}) + + def load_known_failures(path): - """The recorded failures, `{declaration: {stage, reason}}`.""" + """The recorded failures, `{declaration: {stage, reason}}`. + + Strict like every other boundary here: a key nothing reads is refused + rather than carried, and a declaration recorded twice is refused rather + than last-one-wins — a shadowed entry would quietly defeat the exact + match the gates promise. + """ with open(path, "rb") as handle: data = tomllib.load(handle) failures = {} @@ -31,6 +40,12 @@ def load_known_failures(path): for field in ("declaration", "stage", "reason"): if field not in entry: raise SystemExit(f"{path}: a failure entry has no `{field}`") + unknown = sorted(set(entry) - KNOWN_KEYS) + if unknown: + raise SystemExit( + f"{path}: {entry['declaration']} has unknown keys: " + f"{', '.join(unknown)}" + ) if entry["stage"] not in ("source", "target"): raise SystemExit( f"{path}: {entry['declaration']} has stage {entry['stage']!r}; " @@ -41,5 +56,9 @@ def load_known_failures(path): f"{path}: {entry['declaration']} is a target failure without a " "`workspace`; the target gate matches by workspace id" ) + if entry["declaration"] in failures: + raise SystemExit( + f"{path}: {entry['declaration']} is recorded twice" + ) failures[entry["declaration"]] = entry return failures diff --git a/comparator/adapter/test_make_comparator_workspace.py b/comparator/adapter/test_make_comparator_workspace.py index 698929c372..d32a4eff4b 100644 --- a/comparator/adapter/test_make_comparator_workspace.py +++ b/comparator/adapter/test_make_comparator_workspace.py @@ -221,6 +221,23 @@ def test_a_missing_field_is_refused(self): with self.assertRaises(SystemExit): self._load('[[failure]]\ndeclaration = "A.b"\nstage = "source"\n') + def test_an_unknown_key_is_refused(self): + # A typo'd key would otherwise ride along invisibly. + with self.assertRaisesRegex(SystemExit, "unknown keys: workspce"): + self._load( + '[[failure]]\ndeclaration = "A.b"\nstage = "source"\n' + 'reason = "x"\nworkspce = "A_b"\n' + ) + + def test_a_duplicate_declaration_is_refused(self): + # Last-one-wins would shadow an entry and quietly defeat the exact + # match the gates promise. + with self.assertRaisesRegex(SystemExit, "recorded twice"): + self._load( + '[[failure]]\ndeclaration = "A.b"\nstage = "source"\nreason = "x"\n' + '[[failure]]\ndeclaration = "A.b"\nstage = "source"\nreason = "y"\n' + ) + if __name__ == "__main__": unittest.main() From 7bd576a7744065a3639f8d62d58a0a7ea7c4f210 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:04:24 -0400 Subject: [PATCH 61/70] A lock entry nothing reads is confidence without control; delete five MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `[tools]` (comparator, landrun, lean4export) and `[target]`'s repository and commit had no consumer anywhere in the adapter or CI — the CI landrun is the cloned comparator repo's own stub, and the lean-eval commit predates the standalone-generator consolidation. The resolved component pins are now stated to be authoritative, the lean4export local-build note moves to the README beside the local-run instructions, and the file's header commits to the rule: every key has a consumer, or it goes. --- comparator/OWNERSHIP.md | 2 +- comparator/README.md | 17 ++++++++++------- comparator/tools.toml | 22 ++++++---------------- 3 files changed, 17 insertions(+), 24 deletions(-) diff --git a/comparator/OWNERSHIP.md b/comparator/OWNERSHIP.md index 4224a6d01f..3c6ead1b5d 100644 --- a/comparator/OWNERSHIP.md +++ b/comparator/OWNERSHIP.md @@ -91,7 +91,7 @@ Conjectures corrects a misformalisation upstream. | `comparator/adapter/known_failures.py` | the known-failures ledger's format and loader, shared by the set run and the target-stage compile | | `comparator/templates/WorkspaceTest.lean` | the workspace test template the contract requires the consumer to supply | | `comparator/problems/*.toml` | the rare source-boundary facts the compiled environment cannot recover: which module when two declare the same name, and an explicit copied proof dependency when opaque theorem-value erasure removes it from the compiled dependency graph | -| `comparator/tools.toml` | the pins, in one machine-readable place: this repository's under `[tools]`, LeanEval's under `[target]`, the generator revision under `[generator]` | +| `comparator/tools.toml` | the pins, in one machine-readable place: LeanEval's under `[target]`, the generator revision under `[generator]`; every key has a consumer in the adapter or CI | The tests beside each file pin real defects: the importer suite covers extraction, the interface suite covers the wire shapes, and the command suite diff --git a/comparator/README.md b/comparator/README.md index 3951eafff5..ec841c3866 100644 --- a/comparator/README.md +++ b/comparator/README.md @@ -150,13 +150,16 @@ python3 comparator/adapter/make_comparator_workspace.py --validate ## Tool pins -`tools.toml` is the one machine-readable source. `[tools]` are the revisions a -local run uses under this repository's toolchain. `[target]` are LeanEval's: -the Lean toolchain and Mathlib revision every generated workspace is pinned to, -and the Comparator and `lean4export` commits that check it. `[generator]` is -the extracted generator revision every request is written against; bumping it -is a contract change and has to survive the seam round-trip test. Generation -itself does not run Comparator. +`tools.toml` is the one machine-readable source, and every key in it has a +consumer in the adapter or in CI — a pin nothing reads is confidence without +control, and gets deleted. `[target]` are LeanEval's pins: the Lean toolchain +and Mathlib revision every generated workspace is pinned to, and the +Comparator and `lean4export` commits that check it. `[generator]` is the +extracted generator revision every request is written against; bumping it is +a contract change and has to survive the seam round-trip test. Generation +itself does not run Comparator. For a local Comparator run under this +repository's own toolchain, build `lean4export` from its `v4.33.0` tag — +upstream has no v4.33.1 tag yet, and patch releases share the export format. ## Conformance before a public import diff --git a/comparator/tools.toml b/comparator/tools.toml index 1aa65b3d46..8e6ff7e6b3 100644 --- a/comparator/tools.toml +++ b/comparator/tools.toml @@ -1,25 +1,15 @@ # The pinned external tools, one machine-readable source of truth. "At or -# after" prose is not a lock; CI, local setup and documentation read this. - -# What a local run of the importer uses, under this repository's toolchain. -[tools] -comparator = "71b52ec29e06d4b7d882726553b1ceb99a2499e0" -landrun = "5ed4a3db3a4ad930d577215c6b9abaa19df7f99f" -# lean4export: no upstream tag exists for this repository's v4.33.1 yet; -# build the v4.33.0 tag under this repository's toolchain (patch releases -# share the export format). -lean4export = "v4.33.0" +# after" prose is not a lock — and neither is a key nothing reads: every key +# in this file has a consumer in the adapter or in CI, and a key that loses +# its last consumer is deleted rather than kept for confidence. # Where a generated workspace is built and checked. These are LeanEval's pins, # not this repository's: a workspace is vendored into lean-eval and built -# there, so it has to be buildable where it is going. The generator writes -# `lean_toolchain` and `mathlib_revision` into every workspace, and every -# manifest records the pair beside Formal Conjectures' own, which is what makes -# the gap between the two readable rather than assumed. +# there, so it has to be buildable where it is going. The resolved component +# pins below are authoritative; there is deliberately no lean-eval commit +# here, because nothing could check the two statements against each other. [target] -repository = "leanprover/lean-eval" comparator_repository = "https://github.com/leanprover/comparator" -commit = "7699436464052268e6c04b41554bfbc2c6908ec5" lean_toolchain = "leanprover/lean4:v4.33.0" mathlib_revision = "6f1ef4e5dd604a435bddba4747b13970cd65d2a1" mathlib_git = "https://github.com/leanprover-community/mathlib4.git" From 9466e4039ad4516c36e242b1d3acb3d5b93dd1d4 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:05:45 -0400 Subject: [PATCH 62/70] Run the audit when what it measures changes; upload what produced the result MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The whole-set audit's pull-request trigger watched only its own file and the ledger, so an adapter, pin, template, action or toolchain change could bypass the gate it exists to be. It now triggers on everything its outcome depends on, with ordinary statement edits still excluded — the weekly run is their backstop. The artifact grows from two reports into the review evidence: the exact request bytes the set run piped to the generator, every workspace with its provenance sidecar, and a manifest digesting each file. The Comparator job's third-verdict fixture edits gain the same fired-or-fail asserts the second verdict already had. --- .github/workflows/comparator-lean-4-33.yml | 11 +++- .github/workflows/fc100-audit.yml | 65 ++++++++++++++++--- .../adapter/make_comparator_workspace.py | 20 +++++- 3 files changed, 82 insertions(+), 14 deletions(-) diff --git a/.github/workflows/comparator-lean-4-33.yml b/.github/workflows/comparator-lean-4-33.yml index 6eb4e94137..f56243c7f7 100644 --- a/.github/workflows/comparator-lean-4-33.yml +++ b/.github/workflows/comparator-lean-4-33.yml @@ -165,12 +165,17 @@ jobs: "noncomputable def isSumOfThreeCubes_iff_mod_9_answer : Prop :=\n" " ∀ n : ℤ, IsSumOfThreeCubes n ↔ ¬(n ≡ 4 [ZMOD 9] ∨ n ≡ 5 [ZMOD 9])" ) - text = text.replace( + # Each replacement must actually fire: a generated shape change + # that made either a no-op would let this verdict pass vacuously. + filled = text.replace( "noncomputable def isSumOfThreeCubes_iff_mod_9_answer : Prop := sorry", answer, ) - text = text.replace(":= by\n sorry", ":=\n Iff.rfl") - submission.write_text(text, encoding="utf-8") + assert filled != text, "the answer hole was not where this step expects" + text = filled + filled = text.replace(":= by\n sorry", ":=\n Iff.rfl") + assert filled != text, "the proof hole was not where this step expects" + submission.write_text(filled, encoding="utf-8") PY (cd .comparator/SumOfThreeCubes_isSumOfThreeCubes_iff_mod_9 && lake build && lake test) echo "Comparator accepted a gamed definition hole; hole values need a human." diff --git a/.github/workflows/fc100-audit.yml b/.github/workflows/fc100-audit.yml index e04b2c7c91..b6996a97c8 100644 --- a/.github/workflows/fc100-audit.yml +++ b/.github/workflows/fc100-audit.yml @@ -23,8 +23,10 @@ name: FC100 whole-set audit # gate that only ever passes proves nothing. # # Two full Mathlib builds make this far too heavy for every pull request, so -# it runs on demand and weekly; the per-PR jobs cover representative -# declarations for each defect class instead. +# it runs on demand, weekly, and on pull requests that change what it +# measures — the adapter, pins, ledger, templates or tooling. Ordinary +# statement edits rely on the per-PR jobs' representative declarations and +# the weekly run as backstop. concurrency: group: fc100-audit-${{ github.ref }} @@ -36,13 +38,26 @@ on: # Weekly, early Monday UTC. - cron: '17 4 * * 1' # Dispatch and schedule only reach a workflow on the default branch, so a - # pull request introducing or reconfiguring the audit could never show its - # run. Trigger on the audit's own configuration instead: these paths change - # when the audit changes, and the artifact is the review evidence. + # pull request that changes what the audit measures could never show its + # run. Trigger on everything the audit's outcome depends on: the adapter, + # the templates and problem overrides, the pins, the ledger, the composite + # actions that build its tools, and the toolchain the modules elaborate + # under. Ordinary statement edits stay out — the weekly run is their + # backstop, and a whole-set run per problem PR would be two Mathlib builds + # of noise. pull_request: paths: - - '.github/workflows/fc100-audit.yml' + - 'comparator/adapter/**' + - 'comparator/templates/**' + - 'comparator/problems/**' + - 'comparator/tools.toml' - 'comparator/known_failures.toml' + - '.github/actions/build-lean-eval-generator/**' + - '.github/actions/prepare-extractor/**' + - '.github/workflows/fc100-audit.yml' + - 'FormalConjectures/Subsets/FC100OpenSet1.lean' + - 'lean-toolchain' + - 'lake-manifest.json' permissions: contents: read @@ -110,12 +125,46 @@ jobs: --report fc100-target-report.json \ --known-failures comparator/known_failures.toml - - name: Upload the audit report + # The artifact is the review evidence, so it carries what produced the + # result, not just the result: the exact request bytes, every + # workspace with its provenance sidecar, both reports, and a manifest + # digesting each file so the bundle can vouch for itself. + - name: Digest the audit bundle + if: always() + run: | + python3 - <<'PY' + import hashlib + import json + import pathlib + + manifest = {} + for pattern in ("fc100-report.json", "fc100-target-report.json"): + path = pathlib.Path(pattern) + if path.is_file(): + manifest[str(path)] = hashlib.sha256(path.read_bytes()).hexdigest() + root = pathlib.Path(".fc100") + if root.is_dir(): + for path in sorted(root.rglob("*")): + if path.is_file(): + manifest[str(path)] = hashlib.sha256( + path.read_bytes() + ).hexdigest() + pathlib.Path("bundle-manifest.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print(f"{len(manifest)} files digested") + PY + + - name: Upload the audit bundle if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: fc100-audit-report + name: fc100-audit-bundle path: | fc100-report.json fc100-target-report.json + bundle-manifest.json + .fc100/ if-no-files-found: warn + include-hidden-files: true diff --git a/comparator/adapter/make_comparator_workspace.py b/comparator/adapter/make_comparator_workspace.py index 3e3dad043b..82598f45f2 100644 --- a/comparator/adapter/make_comparator_workspace.py +++ b/comparator/adapter/make_comparator_workspace.py @@ -192,8 +192,13 @@ def seam_files(pairs, group=None): return request, files -def generate_workspaces(pairs, out_dir, group=None): - """Generate one workspace per pair under `out_dir`, via the pinned binary.""" +def generate_workspaces(pairs, out_dir, group=None, emit_request=None): + """Generate one workspace per pair under `out_dir`, via the pinned binary. + + With `emit_request`, the exact request bytes piped to the binary are also + written to that path — a set audit's artifact should carry the request + that produced it, not just the reports about it. + """ request, problems = _seam(pairs, group=group) # One serialisation, used everywhere: the string piped to the binary is # the string `--emit-import` writes and the sidecar digests. The request @@ -213,6 +218,10 @@ def generate_workspaces(pairs, out_dir, group=None): ) finally: shutil.rmtree(staging, ignore_errors=True) + if emit_request is not None: + emit_request = pathlib.Path(emit_request) + emit_request.parent.mkdir(parents=True, exist_ok=True) + emit_request.write_text(request_text, encoding="utf-8") module_content = {p["id"]: p["moduleContent"] for p in request["problems"]} producer = importer.producer_record() # Every refusal — digest, identity, sidecar, target-already-exists — @@ -317,7 +326,12 @@ def import_set(set_name, out_dir, verify=False, known_failures=None): # solved members marked by their category tag, so every member goes to # the open-conjectures group (google-deepmind/formal-conjectures#5075). written = ( - generate_workspaces(pairs, out_dir, group="open-conjectures") + generate_workspaces( + pairs, + out_dir, + group="open-conjectures", + emit_request=pathlib.Path(out_dir) / "request.json", + ) if pairs else [] ) From 20e3051d583ec9e53a71e1e6396a6a4df41aadf9 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:06:34 -0400 Subject: [PATCH 63/70] Say what the guarantees are now that they hold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit README: the snapshot check covers every read file, not just the selected source; "exact bytes" is now literal and says why; the 92/8 paragraph caught up with the frozen-set policy #5075 settled. OWNERSHIP records the two boundaries this series drew — catalog policy arrives as an explicit ImportPolicy, and the import header is faithful for this corpus because every problem file already elaborates under all of Mathlib via FormalConjecturesUtil's public import, with the Mathlib revision as the one recorded source-versus-target gap. The same fact sits on MODULE_PREAMBLE itself, where the next reader will look first. --- comparator/OWNERSHIP.md | 12 +++++++++++- comparator/README.md | 21 +++++++++++++-------- comparator/adapter/leaneval_interface.py | 9 +++++++++ 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/comparator/OWNERSHIP.md b/comparator/OWNERSHIP.md index 3c6ead1b5d..1272250758 100644 --- a/comparator/OWNERSHIP.md +++ b/comparator/OWNERSHIP.md @@ -37,7 +37,7 @@ revision is normative). Per problem it carries: | `moduleContent` | the rendered marked-up module: the statement's copied FC-local closure, the scope directives in force where it was written, one `noncomputable def : := sorry` per `answer(sorry)` slot, and the statement with its proof replaced by `sorry` — in that order, requiring Mathlib and nothing else | | `resolvedHoles` | a source span, kind, and explicit parameters for each hole, computed from the rendered text — exactly, because this side rendered it | | `holes`, `id`, `moduleName` | the qualified declaration name, slugged; two modules declaring `conjecture` in different namespaces must not share a workspace | -| `group` | for a frozen-set import, the set itself: the list is immutable while its members keep getting solved, so every member stays in the open-conjectures display and the category rides along as a tag. For a single import, the declaration's `@[category ...]` tag decides; a declaration that is not a problem is refused either way | +| `group`, `status`, `visible`, `statementRevision`, `submitter`, `tags` | LeanEval catalog policy, not source facts: they arrive as an explicit `ImportPolicy` the command constructs, with the draft-intake values stated in `make_comparator_workspace.import_policy`. For a frozen-set import the policy's group is the set itself — the list is immutable while its members keep getting solved, so every member stays in the open-conjectures display and the category rides along as a tag; for a single import the declaration's `@[category ...]` tag decides. A declaration that is not a problem is refused either way | | `leanToolchain`, `mathlib` | LeanEval's pins, from `[target]` in `tools.toml` — the consumer's, never this repository's | | `templates.workspaceTest` | `comparator/templates/WorkspaceTest.lean`, which stays FC-supplied: the contract requires the consumer to provide it | | `contextRoot` | a directory this side materialises: the module file the generator byte-checks against `moduleContent`, and a synthesised `.ilean` carrying the spans above, because generator schema version 1 still resolves declaration spans from compiled metadata | @@ -46,6 +46,16 @@ The module carries no markers of any kind. `@[eval_problem]` does not exist outside lean-eval, so a module carrying it could not elaborate under `--verify`; the ranges in the request already say where the holes are. +The module's `import Mathlib` header is faithful, not convenient. Every +problem file under `FormalConjectures/` imports exactly +`FormalConjecturesUtil`, which `public import`s all of Mathlib, so each +statement already elaborates under the full library; no file in the corpus +carries a narrow or third-party import the header could widen away. What the +header drops — the FC-local layer — travels as the copied closure, and +`--verify` elaborates under exactly the header that ships. The remaining +source-versus-target gap is the Mathlib revision, which the sidecar records +on both sides and the cross-pin CI job builds. + The module is one file rather than four strings because the importer can then elaborate exactly what it is about to hand over: `--verify` runs the module through this checkout's Mathlib, so an FC-side defect — a lost `open`, an diff --git a/comparator/README.md b/comparator/README.md index ec841c3866..0b961bd497 100644 --- a/comparator/README.md +++ b/comparator/README.md @@ -57,9 +57,11 @@ Use `--out` to choose the parent directory. Generation refuses to overwrite an existing workspace: it writes into a temporary directory and renames the complete workspace into place. -The importer stops when the selected source differs from the pinned upstream -revision. This prevents a workspace from combining a working-tree statement -with an older imported context. +The importer stops when any file it read differs from the pinned upstream +revision — the statement's own file, every copied dependency's, every copied +notation command's, and the pin files the record quotes. One workspace, one +source revision: a pinned statement cannot be combined with working-tree +copied text, and an untracked file cannot slip in unrecorded. `--verify` elaborates the marked-up module before anything is written, so an FC-side copying defect fails here rather than in LeanEval CI. It runs at this @@ -83,7 +85,10 @@ python3 comparator/adapter/make_comparator_workspace.py erdos_1038.parts.i \ This writes the exact bytes that cross the seam — `request.json`, the `context/` directory the schema-version-1 contract reads, and the provenance sidecar — and -generates no workspace. Running the pinned binary on that request from inside +generates no workspace. "Exact" is literal: generation pipes the same +serialisation to the binary that this writes to disk, the sidecar records its +SHA-256 under `digests.request`, and the seam test feeds the emitted file +back byte-for-byte. Running the pinned binary on that request from inside the emitted directory yields the same file map generation would have written, which is what makes the seam checkable rather than asserted. @@ -178,7 +183,7 @@ and `comparator-lean-4-33.yml` builds two of them at LeanEval's pins and runs Comparator on them. They validate extraction and adapter behaviour, not mathematical correctness or maintainer acceptance. -The first public open-conjectures import also needs a corrected source set. -`FC100OpenSet1` currently verifies itself as 92 `research open` entries and 8 -`research solved` entries, so it must not be imported wholesale as one hundred -open conjectures. +`FC100OpenSet1` is a frozen list whose members keep getting solved — designed +lifecycle, not an anomaly (formal-conjectures#5075). The whole set imports +into the open-conjectures group, with each member's current category riding +along as a tag; the audit report states the split as it stands on each run. diff --git a/comparator/adapter/leaneval_interface.py b/comparator/adapter/leaneval_interface.py index 47576c30e9..5abfed4c35 100644 --- a/comparator/adapter/leaneval_interface.py +++ b/comparator/adapter/leaneval_interface.py @@ -64,6 +64,15 @@ REGIONS = ("dependencies", "scope", "holes", "statement") +# Faithful for this corpus, not a convenience: every problem file under +# FormalConjectures/ imports exactly `FormalConjecturesUtil`, which +# `public import`s all of Mathlib, so each statement already elaborates +# under the full library and no file carries a narrow or third-party +# import this header could widen away. The FC-local layer — the one part +# of the source environment this header drops — travels as the copied +# closure, and `--verify` elaborates under exactly this header. The +# remaining source-versus-target gap is the Mathlib revision, which the +# sidecar records on both sides and the cross-pin CI job builds. MODULE_PREAMBLE = "import Mathlib\n" MANIFEST_SCHEMA_VERSION = 1 From 9d2cb7964d3bff3e32d0085ce3d83e0315c064ad Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Tue, 25 Aug 2026 13:38:20 -0400 Subject: [PATCH 64/70] Do not hold the environment to the source pin Adding every read file to the snapshot check swept in `lean-toolchain` and `lake-manifest.json`, which made a toolchain bump the one change that could not pass it: a bump edits exactly those two files, so `pins()` would refuse every declaration and the comparator jobs would fail on the pull request that most needs them to run. They were never source text. The record states them as the environment the facts were read in, which is an observation, not a claim that they came from the pinned commit. The statement, its copied dependencies and its copied notation stay held to one revision. --- comparator/adapter/fc_leaneval_importer.py | 11 ++++++++--- comparator/adapter/test_fc_leaneval_importer.py | 12 ++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/comparator/adapter/fc_leaneval_importer.py b/comparator/adapter/fc_leaneval_importer.py index ff528fcec8..ba41a7e87d 100644 --- a/comparator/adapter/fc_leaneval_importer.py +++ b/comparator/adapter/fc_leaneval_importer.py @@ -632,10 +632,15 @@ def import_problem(problem, answer_type=None, module=None): # this is what ties the two together. qualified = ".".join(namespaces_at_target + [original_declared]) # Every file whose text reached the workspace is held to the one source - # revision: the statement's own file, each copied dependency's, each - # copied notation command's, and the pin files the record quotes. + # revision: the statement's own file, each copied dependency's, and each + # copied notation command's. The toolchain and manifest are deliberately + # not among them. They are the environment the facts were read in, which + # the record states as an observation rather than as a claim about the + # pinned commit, and holding them to the merge base would make a + # toolchain bump — the change that most needs this check to run — the one + # change that cannot pass it. read_paths = ( - [path.relative_to(ROOT), "lean-toolchain", "lake-manifest.json"] + [path.relative_to(ROOT)] + [record["path"] for record in copied_records] + list(notation_paths) ) diff --git a/comparator/adapter/test_fc_leaneval_importer.py b/comparator/adapter/test_fc_leaneval_importer.py index bf080f26d8..d740687b44 100644 --- a/comparator/adapter/test_fc_leaneval_importer.py +++ b/comparator/adapter/test_fc_leaneval_importer.py @@ -20,6 +20,7 @@ """ import contextlib +import inspect import json import pathlib import subprocess @@ -123,6 +124,17 @@ def test_a_path_the_revision_does_not_track_is_refused(self): with self.assertRaisesRegex(SystemExit, "not tracked at pinned"): pins(pathlib.Path("FormalConjectures/New.lean")) + def test_the_toolchain_is_not_held_to_the_pin(self): + # A toolchain bump edits `lean-toolchain` and `lake-manifest.json`. + # Those describe the environment the facts were read in, which the + # record states as an observation. Holding them to the merge base + # would make a bump the one change that cannot pass this check. + import fc_leaneval_importer as importer_module + + source = inspect.getsource(importer_module.import_problem) + self.assertNotIn("lean-toolchain", source) + self.assertNotIn("lake-manifest.json", source) + def test_a_dirty_dependency_fails_even_with_a_clean_target(self): # The reviewer's mixed state: target at the pin, dependency edited in # the working tree. One diff over every read path refuses it. From 60bc734877114a8ca2a8715ea2a36ca9610d218e Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Tue, 25 Aug 2026 13:42:25 -0400 Subject: [PATCH 65/70] Put the adapter's checks where the adapter changes The generation smoke test and the seam round-trip ran inside the corpus build, on every pull request. A statement pull request paid for building the pinned generator and importing eight declarations it had nothing to do with, a failure in either reported as the corpus build failing at minute eighty, and a pull request editing one of those eight source files could not pass: `pins()` refuses a source file that differs from the merge base, which is exactly what such a pull request changes. The Comparator job already had the right triggers, the extractor and the generator, and the comment explaining why it excludes statement files. Both checks move there, and its module list grows to cover the smoke set. The corpus build goes back to building the corpus. --- .github/workflows/build-and-docs.yml | 75 --------------- .github/workflows/comparator-lean-4-33.yml | 107 +++++++++++++++++++-- 2 files changed, 98 insertions(+), 84 deletions(-) diff --git a/.github/workflows/build-and-docs.yml b/.github/workflows/build-and-docs.yml index d71dec5913..d07d4479cd 100644 --- a/.github/workflows/build-and-docs.yml +++ b/.github/workflows/build-and-docs.yml @@ -190,81 +190,6 @@ jobs: action: remove linters: lean - - name: Build the pinned lean-eval-generator - if: steps.mode.outputs.website_only != 'true' - uses: ./.github/actions/build-lean-eval-generator - - # The elaborator-to-importer boundary, exercised on the oleans the - # build above produced: one plain theorem, a Prop answer slot, a - # non-Prop answer slot, a forall-conclusion (whose binder must not be - # applied), explicit parameters (which must be), and the qualified-id - # collision pair. The comparator run itself needs landrun and stays in - # a separate Linux job. - - name: Comparator generation smoke test - if: steps.mode.outputs.website_only != 'true' - run: | - lake build comparator_facts - lake exe comparator_facts --self-test - for d in exists_hadamard_zero erdos_940.variants.large_integers \ - erdos_1038.parts.i erdos_100.variants.strong \ - KotherConjecture.variants.le_KotherRadical \ - OeisA303656.conjecture OeisA308734.conjecture \ - curling_number_conjecture; do - python3 comparator/adapter/make_comparator_workspace.py "$d" --out .comparator - done - grep -q "large_integers_answer : Prop" .comparator/Erdos940_erdos_940_variants_large_integers/Challenge.lean - grep -q "i_answer : ENNReal" .comparator/Erdos1038_erdos_1038_parts_i/Challenge.lean - grep -q "Submission.erdos_100_variants_strong$" .comparator/Erdos100_erdos_100_variants_strong/Solution.lean - grep -q "le_KotherRadical hI" .comparator/Koethe_KotherConjecture_variants_le_KotherRadical/Solution.lean - # Two modules declare `conjecture`; qualified default ids keep the - # workspaces apart, and a guillemet module path decodes correctly. - test -d .comparator/OeisA303656_conjecture - test -d .comparator/OeisA308734_conjecture - test -d .comparator/Arxiv__0912_2382__curling_number_conjecture - - # The importer-to-generator seam, on a real declaration. - # `--emit-import` writes the exact bytes that cross it — the v1 - # request and its context directory — and running the pinned binary on - # those bytes from inside the emitted directory has to reproduce the - # workspace exactly; if it does not, the emitted artifact is not the - # whole interface. See comparator/OWNERSHIP.md. - - name: Importer to generator seam - if: steps.mode.outputs.website_only != 'true' - run: | - python3 comparator/adapter/make_comparator_workspace.py erdos_1038.parts.i \ - --emit-import .comparator-import - python3 - <<'PY' - import json - import os - import pathlib - import sys - - sys.path.insert(0, "comparator/adapter") - import leaneval_generator_cli as generator_cli - from leaneval_interface import ProblemManifest - - handed_over = pathlib.Path( - ".comparator-import/Erdos1038_erdos_1038_parts_i" - ).resolve() - manifest = ProblemManifest.from_json( - (handed_over / "fc-provenance-Erdos1038_erdos_1038_parts_i.json") - .read_text(encoding="utf-8") - ) - assert len(manifest.source.commit) == 40, manifest.source.commit - assert manifest.source.declaration, "no FC declaration id" - - # The emitted request is fed back as its exact bytes — no reparse, - # no reserialise — which is the claim the seam test exists to check. - request_text = (handed_over / "request.json").read_text(encoding="utf-8") - workspace = pathlib.Path(".comparator/Erdos1038_erdos_1038_parts_i").resolve() - regenerated = generator_cli.generate(request_text, cwd=handed_over) - files = regenerated["Erdos1038_erdos_1038_parts_i"] - for name, content in files.items(): - expected = (workspace / name).read_text(encoding="utf-8") - assert content == expected, name - print(f"{len(files)} files reproduced from the emitted request") - PY - - name: Build literate source pages if: steps.mode.outputs.website_only != 'true' && steps.mode.outputs.site == 'true' run: | diff --git a/.github/workflows/comparator-lean-4-33.yml b/.github/workflows/comparator-lean-4-33.yml index f56243c7f7..40f8113b3c 100644 --- a/.github/workflows/comparator-lean-4-33.yml +++ b/.github/workflows/comparator-lean-4-33.yml @@ -1,13 +1,25 @@ name: Generated workspace at LeanEval pins -# End to end: import a declaration from this repository's source, generate a -# workspace, build it at LeanEval's Lean 4.33 and Mathlib, and run Comparator -# on it. Nothing here is checked in. A workspace that lives in the tree is a -# copy of generator output that drifts from the generator and says nothing -# about the importer, because a human wrote it; this job says the importer, the +# Everything that crosses the adapter's boundaries, in one job: +# +# 1. the elaborator to the importer, over a smoke set that covers each +# defect class the whole-set audit found; +# 2. the importer to the generator, by feeding the emitted request bytes +# back through the pinned binary and comparing the file map; +# 3. the generated workspace to Comparator, built at LeanEval's pins. +# +# Nothing here is checked in. A workspace that lives in the tree is a copy of +# generator output that drifts from the generator and says nothing about the +# importer, because a human wrote it; this job says the importer, the # generator and Comparator work together, which is what # `leanprover/lean-eval#536` needs before FC opens problem pull requests. # +# These checks live here rather than beside the corpus build because they are +# about the adapter, not the corpus. A pull request that only edits a +# statement should not pay for them, and could not pass them: `pins()` refuses +# a source file that differs from the merge base, which is exactly what such a +# pull request changes. +# # The two toolchains in this job are the point, not an accident. The importer # reads the declaration's source range, binders and `answer(sorry)` slot types # from an environment elaborated at this repository's Lean 4.33.1; the workspace @@ -67,16 +79,93 @@ jobs: print(f"{key}={target[key]}") PY - # At this repository's toolchain: the declaration's facts come from an - # elaborated environment, so the module it lives in has to be built. - - name: Prepare the extractor and the source module + # At this repository's toolchain. The facts the importer reads come + # from an elaborated environment, not from the text, so every module a + # step below imports from has to be built first. + - name: Prepare the extractor and the source modules uses: ./.github/actions/prepare-extractor with: - modules: FormalConjectures.Wikipedia.SumOfThreeCubes + modules: >- + FormalConjectures.Wikipedia.SumOfThreeCubes + FormalConjectures.Wikipedia.Hadamard + FormalConjectures.Wikipedia.Koethe + FormalConjectures.ErdosProblems.«100» + FormalConjectures.ErdosProblems.«940» + FormalConjectures.ErdosProblems.«1038» + FormalConjectures.OEIS.«303656» + FormalConjectures.OEIS.«308734» + FormalConjectures.Arxiv.«0912.2382».CurlingNumberConjecture - name: Build the pinned lean-eval-generator uses: ./.github/actions/build-lean-eval-generator + # The elaborator-to-importer boundary, exercised on the oleans the + # build above produced: one plain theorem, a Prop answer slot, a + # non-Prop answer slot, a forall-conclusion (whose binder must not be + # applied), explicit parameters (which must be), and the qualified-id + # collision pair. The comparator run itself needs landrun and stays in + # a separate Linux job. + - name: Comparator generation smoke test + run: | + for d in exists_hadamard_zero erdos_940.variants.large_integers \ + erdos_1038.parts.i erdos_100.variants.strong \ + KotherConjecture.variants.le_KotherRadical \ + OeisA303656.conjecture OeisA308734.conjecture \ + curling_number_conjecture; do + python3 comparator/adapter/make_comparator_workspace.py "$d" --out .comparator + done + grep -q "large_integers_answer : Prop" .comparator/Erdos940_erdos_940_variants_large_integers/Challenge.lean + grep -q "i_answer : ENNReal" .comparator/Erdos1038_erdos_1038_parts_i/Challenge.lean + grep -q "Submission.erdos_100_variants_strong$" .comparator/Erdos100_erdos_100_variants_strong/Solution.lean + grep -q "le_KotherRadical hI" .comparator/Koethe_KotherConjecture_variants_le_KotherRadical/Solution.lean + # Two modules declare `conjecture`; qualified default ids keep the + # workspaces apart, and a guillemet module path decodes correctly. + test -d .comparator/OeisA303656_conjecture + test -d .comparator/OeisA308734_conjecture + test -d .comparator/Arxiv__0912_2382__curling_number_conjecture + + # The importer-to-generator seam, on a real declaration. + # `--emit-import` writes the exact bytes that cross it — the v1 + # request and its context directory — and running the pinned binary on + # those bytes from inside the emitted directory has to reproduce the + # workspace exactly; if it does not, the emitted artifact is not the + # whole interface. See comparator/OWNERSHIP.md. + - name: Importer to generator seam + run: | + python3 comparator/adapter/make_comparator_workspace.py erdos_1038.parts.i \ + --emit-import .comparator-import + python3 - <<'PY' + import json + import os + import pathlib + import sys + + sys.path.insert(0, "comparator/adapter") + import leaneval_generator_cli as generator_cli + from leaneval_interface import ProblemManifest + + handed_over = pathlib.Path( + ".comparator-import/Erdos1038_erdos_1038_parts_i" + ).resolve() + manifest = ProblemManifest.from_json( + (handed_over / "fc-provenance-Erdos1038_erdos_1038_parts_i.json") + .read_text(encoding="utf-8") + ) + assert len(manifest.source.commit) == 40, manifest.source.commit + assert manifest.source.declaration, "no FC declaration id" + + # The emitted request is fed back as its exact bytes — no reparse, + # no reserialise — which is the claim the seam test exists to check. + request_text = (handed_over / "request.json").read_text(encoding="utf-8") + workspace = pathlib.Path(".comparator/Erdos1038_erdos_1038_parts_i").resolve() + regenerated = generator_cli.generate(request_text, cwd=handed_over) + files = regenerated["Erdos1038_erdos_1038_parts_i"] + for name, content in files.items(): + expected = (workspace / name).read_text(encoding="utf-8") + assert content == expected, name + print(f"{len(files)} files reproduced from the emitted request") + PY + # `--verify` elaborates the marked-up module here, at 4.33.1. It is not a # substitute for the 4.33 build below; it is what keeps an FC-side # copying defect from being reported as a LeanEval build failure. From 2010a9ff627bd0220df65e942bd331ba910dadd5 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Tue, 25 Aug 2026 13:53:36 -0400 Subject: [PATCH 66/70] Fail closed where the fallback was the reassuring answer Three provenance and timeout defects, each the same shape: the failure path produced the answer a reader wants rather than the one that is true. `blob_sha` took an unchecked `git rev-parse` and stored `""` beside a real commit, so a record that had lost its anchor looked like a record that had one. `importer_dirty` took an unchecked `git status`, whose empty output means both "clean tree" and "command failed", and reported clean for both. Both now refuse. The generator is the only call crossing into an external binary and the only one with no timeout, while the extractor's two are bounded and say why; it gets the same bound. A collapsed batch prefetch stays a fallback, not a failure, but it now says so on stderr: re-running a hundred declarations one at a time was otherwise visible only as a job that took hours. Two documentation claims were false. The README credited a job that no longer runs those steps and miscounted the declarations. OWNERSHIP said a test asserts the CLI plumbing never imports the importer; no such test existed, so this adds it rather than softening the claim. --- comparator/README.md | 8 +++--- comparator/adapter/fc_leaneval_importer.py | 10 +++++++- comparator/adapter/fc_source.py | 25 ++++++++++++++++++- comparator/adapter/leaneval_generator_cli.py | 24 ++++++++++++------ comparator/adapter/test_leaneval_interface.py | 24 ++++++++++++++++++ 5 files changed, 78 insertions(+), 13 deletions(-) diff --git a/comparator/README.md b/comparator/README.md index 0b961bd497..3530da6715 100644 --- a/comparator/README.md +++ b/comparator/README.md @@ -177,10 +177,10 @@ The adapter should cover these boundary cases before importing a frozen set: - trusted helper dependencies requiring `ChallengeDeps` or multiple trusted files. -The two CI jobs exercise those distinctions: `build-and-docs.yml` generates -five declarations covering each case and checks the importer-to-generator seam, -and `comparator-lean-4-33.yml` builds two of them at LeanEval's pins and runs -Comparator on them. They validate extraction and adapter behaviour, not +`comparator-lean-4-33.yml` exercises those distinctions: it generates eight +declarations covering each case, checks the importer-to-generator seam by +regenerating from the emitted request, and builds two of them at LeanEval's +pins to run Comparator on. It validates extraction and adapter behaviour, not mathematical correctness or maintainer acceptance. `FC100OpenSet1` is a frozen list whose members keep getting solved — designed diff --git a/comparator/adapter/fc_leaneval_importer.py b/comparator/adapter/fc_leaneval_importer.py index ba41a7e87d..a02124d212 100644 --- a/comparator/adapter/fc_leaneval_importer.py +++ b/comparator/adapter/fc_leaneval_importer.py @@ -425,11 +425,19 @@ def source_record( text=True, check=False, ) + # An empty digest beside a real commit reads as "recorded", so a failure + # here has to be one. `pins` already proved the path is tracked at this + # revision, which is why this is a refusal and not a fallback. + if blob.returncode != 0 or not blob.stdout.strip(): + raise SystemExit( + f"cannot resolve {source_path} at {fc_rev[:12]}: " + f"{blob.stderr.strip() or 'no object returned'}" + ) return SourceRecord( repository=SOURCE_REPOSITORY, commit=fc_rev, path=str(source_path), - blob_sha=blob.stdout.strip() or "", + blob_sha=blob.stdout.strip(), module=module, declaration=declaration, copied_dependencies=tuple(copied_records), diff --git a/comparator/adapter/fc_source.py b/comparator/adapter/fc_source.py index da74d2869f..5a612eb32a 100644 --- a/comparator/adapter/fc_source.py +++ b/comparator/adapter/fc_source.py @@ -16,6 +16,7 @@ import pathlib import re import subprocess +import sys def slug(name): @@ -165,9 +166,23 @@ def prefetch_elaborator_facts(pairs): ) except subprocess.TimeoutExpired: # The batch is an optimisation; the per-declaration path is the - # arbiter of what fails and how it is reported. + # arbiter of what fails and how it is reported. It is still worth + # saying so: a collapsed batch re-runs the whole set one declaration + # at a time, which is otherwise visible only as a job that takes + # hours instead of minutes. + print( + f"comparator_facts --batch: no answer for {len(wanted)} pairs " + "within the timeout; falling back to one run per declaration", + file=sys.stderr, + ) return if proc.returncode != 0: + print( + f"comparator_facts --batch failed for {len(wanted)} pairs; " + "falling back to one run per declaration: " + f"{proc.stderr.strip().splitlines()[-1] if proc.stderr.strip() else 'no output'}", + file=sys.stderr, + ) return requested = set(wanted) for line in proc.stdout.splitlines(): @@ -1032,4 +1047,12 @@ def importer_state(): capture_output=True, text=True, ) + # Empty output means a clean tree, and a failed command also produces + # empty output. Reporting the reassuring answer for both would make the + # field worth less than not recording it. + if status.returncode != 0: + raise SystemExit( + f"cannot tell whether the importer is dirty: " + f"{status.stderr.strip() or 'git status failed'}" + ) return head.stdout.strip(), bool(status.stdout.strip()) diff --git a/comparator/adapter/leaneval_generator_cli.py b/comparator/adapter/leaneval_generator_cli.py index a96244a416..12da25d91a 100644 --- a/comparator/adapter/leaneval_generator_cli.py +++ b/comparator/adapter/leaneval_generator_cli.py @@ -81,13 +81,23 @@ def generate(request_text, cwd=None, expected_ids=None): `context`, resolved against `cwd`. With `expected_ids`, the response must cover exactly those workspaces. Returns `{problem_id: {path: content}}`. """ - proc = subprocess.run( - [binary()], - input=request_text, - capture_output=True, - text=True, - cwd=cwd, - ) + try: + proc = subprocess.run( + [binary()], + input=request_text, + capture_output=True, + text=True, + cwd=cwd, + # The generator is deterministic and does no I/O beyond the + # context root, so a run that has not answered by now is stuck. + # This is the only call in the adapter that crosses into an + # external binary; the extractor's two are bounded the same way. + timeout=1800, + ) + except subprocess.TimeoutExpired: + raise SystemExit( + "lean-eval-generator did not answer within 30 minutes" + ) from None if proc.returncode != 0: raise SystemExit( f"lean-eval-generator failed:\n{proc.stderr.strip() or proc.stdout.strip()}" diff --git a/comparator/adapter/test_leaneval_interface.py b/comparator/adapter/test_leaneval_interface.py index 7f7d11cb10..bf880b7377 100644 --- a/comparator/adapter/test_leaneval_interface.py +++ b/comparator/adapter/test_leaneval_interface.py @@ -495,5 +495,29 @@ def test_unknown_copied_dependency_keys_are_refused(self): ProblemManifest.from_json_object(payload) +class SeamDirectionTest(unittest.TestCase): + """The arrow points one way, and this is the assertion OWNERSHIP.md cites.""" + + def test_the_cli_plumbing_does_not_import_the_importer(self): + # `leaneval_generator_cli` runs the pinned binary and nothing else. If + # it grew an importer import, the seam would still work and the + # ownership split would quietly stop being true. + import ast + import pathlib + + source = ( + pathlib.Path(__file__).with_name("leaneval_generator_cli.py") + ).read_text(encoding="utf-8") + imported = set() + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module) + self.assertNotIn("fc_leaneval_importer", imported) + self.assertNotIn("fc_source", imported) + self.assertIn("leaneval_interface", imported) + + if __name__ == "__main__": unittest.main() From 9e189f49711b0f89574620b770211f5b5ce5ba58 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:02:46 -0400 Subject: [PATCH 67/70] Say each rule once, and bound everything that waits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The known-failures gate was written twice, once per stage, differing in the key field and one word; it now lives beside the loader it reads, so the two stages cannot drift about what the ledger means. "Which trees are ours" was spelled three incompatible ways — the copy-dependency allowlist excluded `FormalConjecturesUtil` while Lean's `isFCLocal` included it — and is now one tuple, with the notation scan's ordering kept explicit because it decides output. Every subprocess is bounded. The generator was the last unbounded crossing into an external binary, and the git and lake calls had no bound at all. The limits live in their own module: four callers need them and none of them needs another, which the new seam test noticed when a timeout import created exactly that edge. Three defaults pointed the wrong way and now refuse: a missing `endColumn` silently sliced to the end of the line, a producer record missing whole sections loaded as empty strings, and the ledger gate's own guard defaulted to green. A `--set` run with no ledger says that it gated nothing. A target build that fails without an `error:` line keeps its output instead of recording the words "build failed". In Lean, the heartbeat budget is named once and the batch scales it. A constant whose module the environment cannot name now answers "copy it" rather than "Mathlib has it", and one the environment cannot find at all is kept rather than dropped: both then reach the generated-dependency check, which asks about them, instead of leaving ChallengeDeps quietly short. --- .../adapter/ComparatorFacts/Extract.lean | 29 +++++++++-- comparator/adapter/comparator_facts.lean | 3 +- comparator/adapter/compile_fc100_target.py | 43 ++++++++------- comparator/adapter/fc_leaneval_importer.py | 11 ++-- comparator/adapter/fc_source.py | 52 +++++++++++++++---- comparator/adapter/known_failures.py | 37 +++++++++++-- comparator/adapter/leaneval_generator_cli.py | 8 +-- comparator/adapter/leaneval_interface.py | 38 +++++++++----- comparator/adapter/limits.py | 34 ++++++++++++ .../adapter/make_comparator_workspace.py | 31 ++++------- 10 files changed, 205 insertions(+), 81 deletions(-) create mode 100644 comparator/adapter/limits.py diff --git a/comparator/adapter/ComparatorFacts/Extract.lean b/comparator/adapter/ComparatorFacts/Extract.lean index 0b0aaf762c..a0eb49fa1b 100644 --- a/comparator/adapter/ComparatorFacts/Extract.lean +++ b/comparator/adapter/ComparatorFacts/Extract.lean @@ -39,9 +39,22 @@ def moduleOf (env : Environment) (n : Name) : String := | some idx => (env.header.moduleNames[idx.toNat]?.getD Name.anonymous).toString | none => "" -/-- Declared by this repository, as opposed to arriving with `import Mathlib`. -/ +/-- Declared by this repository, as opposed to arriving with `import Mathlib`. + +A constant with a module index the environment cannot name answers `true` +here rather than `false`. Both answers are wrong in that case, but they fail +in opposite directions: `false` means "arrives with Mathlib, copy nothing", +which drops it from `ChallengeDeps` and is caught only by elaborating the +result, while `true` means "copy it", and a constant with no source range is +reported as a generated dependency, which the importer refuses unless a +copied ancestor explains it. Fail towards the check that exists. -/ def isFCLocal (env : Environment) (n : Name) : Bool := - (moduleOf env n).startsWith "FormalConjectures" + match env.getModuleIdxFor? n with + | some idx => + match env.header.moduleNames[idx.toNat]? with + | some name => name.toString.startsWith "FormalConjectures" + | none => true + | none => false /-- The FC-local constants a declaration needs, dependencies before dependents. @@ -55,7 +68,11 @@ partial def fcOrder (env : Environment) (n : Name) if seen.contains n then (seen, acc) else let seen := seen.insert n match env.find? n with - | none => (seen, acc) + -- Reached as a used constant but absent from the environment. Keeping it + -- costs a generated-dependency entry the importer will question; dropping + -- it costs a `ChallengeDeps` that is short by one declaration and says + -- nothing about it. + | none => (seen, acc.push n) | some info => let fromValue := match info.value? with | some v => v.getUsedConstants @@ -73,8 +90,12 @@ partial def fcOrder (env : Environment) (n : Name) let (seen, acc) := children.foldl (fun p c => fcOrder env c p.1 p.2) (seen, acc) (seen, acc.push n) +/-- One declaration's heartbeat budget, in the context's raw units. A batch +caller multiplies this by its pair count rather than restating it. -/ +def heartbeatsPerDeclaration : Nat := 400000000 + unsafe def runWithImports {α : Type} (moduleNames : Array Name) - (actionToRun : MetaM α) (heartbeats : Nat := 400000000) : IO α := do + (actionToRun : MetaM α) (heartbeats : Nat := heartbeatsPerDeclaration) : IO α := do initSearchPath (← getBuildDir) let imports := moduleNames.map fun n => { module := n } Lean.enableInitializersExecution diff --git a/comparator/adapter/comparator_facts.lean b/comparator/adapter/comparator_facts.lean index dc0f2fda47..7b0708baf4 100644 --- a/comparator/adapter/comparator_facts.lean +++ b/comparator/adapter/comparator_facts.lean @@ -69,7 +69,8 @@ unsafe def main (args : List String) : IO UInt32 := do if acc.contains m.toName then acc else acc.push m.toName -- The heartbeat budget is shared by the whole action, so it scales with -- the batch; each pair keeps the single-run allowance. - runWithImports modules (heartbeats := pairs.length * 400000000) do + runWithImports modules + (heartbeats := pairs.length * heartbeatsPerDeclaration) do let env ← getEnv for (modName, declName) in pairs do let tagged (rest : List (String × Json)) := Json.mkObj <| diff --git a/comparator/adapter/compile_fc100_target.py b/comparator/adapter/compile_fc100_target.py index fc8005e2db..e7494e3aa3 100644 --- a/comparator/adapter/compile_fc100_target.py +++ b/comparator/adapter/compile_fc100_target.py @@ -30,7 +30,8 @@ import tomllib from leaneval_interface import lean_errors, dump_json -from known_failures import load_known_failures +from limits import LEAN_TIMEOUT_SECONDS +from known_failures import gate, load_known_failures def arrange_project(workspaces_dir, project_dir): @@ -113,7 +114,12 @@ def build(project_dir, modules): """Build each Challenge target, attributing failures per workspace.""" project_dir = pathlib.Path(project_dir) for command in (["lake", "update"], ["lake", "exe", "cache", "get"]): - completed = subprocess.run(command, cwd=project_dir) + # These two fetch over the network, so they get the same bound as a + # build rather than none: a stalled fetch would otherwise sit until + # the job's own timeout. + completed = subprocess.run( + command, cwd=project_dir, timeout=LEAN_TIMEOUT_SECONDS + ) if completed.returncode != 0: raise SystemExit(f"{' '.join(command)} failed in {project_dir}") results = [] @@ -123,6 +129,7 @@ def build(project_dir, modules): cwd=project_dir, capture_output=True, text=True, + timeout=LEAN_TIMEOUT_SECONDS, ) errors = lean_errors(completed.stdout + completed.stderr) ok = completed.returncode == 0 and not errors @@ -130,7 +137,20 @@ def build(project_dir, modules): { "workspace": workspace_id, "status": "ok" if ok else "target-failed", - **({} if ok else {"reason": "\n".join(errors[:10]) or "build failed"}), + # A failure with no `error:` line is lake dying, not Lean + # rejecting: keep its output rather than recording the word + # "build failed", which the report cannot act on. + **( + {} + if ok + else { + "reason": "\n".join(errors[:10]) + or "\n".join( + (completed.stdout + completed.stderr).splitlines()[-20:] + ) + or f"lake exited {completed.returncode} with no output" + } + ), } ) print(f"{workspace_id}: {'ok' if ok else 'FAILED'}", flush=True) @@ -171,22 +191,7 @@ def main(argv): # ledger, and a target entry without a `workspace` is refused there # rather than silently dropped here. recorded = load_known_failures(args.known_failures) - expected = { - entry["workspace"] - for entry in recorded.values() - if entry["stage"] == "target" - } - unexpected = sorted(failed - expected) - fixed = sorted(expected - failed) - for name in unexpected: - print(f"unexpected target failure: {name}", file=sys.stderr) - for name in fixed: - print( - f"{name} is recorded as a known target failure but compiled; " - "remove it from the record", - file=sys.stderr, - ) - if unexpected or fixed: + if not gate(recorded, failed, "target", "workspace"): return 1 return 0 diff --git a/comparator/adapter/fc_leaneval_importer.py b/comparator/adapter/fc_leaneval_importer.py index a02124d212..8a1729fe51 100644 --- a/comparator/adapter/fc_leaneval_importer.py +++ b/comparator/adapter/fc_leaneval_importer.py @@ -33,6 +33,7 @@ import tempfile import tomllib +from limits import GIT_TIMEOUT_SECONDS, LEAN_TIMEOUT_SECONDS from leaneval_interface import ( CONTRACT_VERSION, MarkedUpModule, @@ -45,6 +46,7 @@ ) from fc_source import ( DECL_START, + FC_SOURCE_TREES, docstring_reference, elaborator_facts, file_scoped_preamble, @@ -151,8 +153,7 @@ def explicit_copy_dependencies(problem_file): relative.is_absolute() or ".." in relative.parts or not relative.parts - or relative.parts[0] - not in {"FormalConjectures", "FormalConjecturesForMathlib"} + or relative.parts[0] not in FC_SOURCE_TREES ): raise SystemExit( f"copy dependency module must stay under a source tree: {relative}" @@ -424,6 +425,7 @@ def source_record( capture_output=True, text=True, check=False, + timeout=GIT_TIMEOUT_SECONDS, ) # An empty digest beside a real commit reads as "recorded", so a failure # here has to be one. `pins` already proved the path is tracked at this @@ -594,9 +596,9 @@ def _resolve(problem, module=None): return problem_file, declaration, located -def statement_pair(problem, module=None): +def statement_pair(problem): """The `(module, declaration)` pair `import_problem` will ask the elaborator about.""" - _, declaration, (path, _imports, _doc, _body) = _resolve(problem, module) + _, declaration, (path, _imports, _doc, _body) = _resolve(problem) return module_name(path.relative_to(ROOT)), declaration @@ -715,6 +717,7 @@ def elaborate(marked_up): text=True, cwd=ROOT, check=False, + timeout=LEAN_TIMEOUT_SECONDS, ) finally: pathlib.Path(combined).unlink(missing_ok=True) diff --git a/comparator/adapter/fc_source.py b/comparator/adapter/fc_source.py index 5a612eb32a..81c11b8299 100644 --- a/comparator/adapter/fc_source.py +++ b/comparator/adapter/fc_source.py @@ -18,6 +18,12 @@ import subprocess import sys +from limits import ( + BATCH_TIMEOUT_PER_PAIR_SECONDS, + GIT_TIMEOUT_SECONDS, + LEAN_TIMEOUT_SECONDS, +) + def slug(name): """A Lake package name and directory name for a problem id. @@ -46,6 +52,18 @@ def declaration(self): ROOT = pathlib.Path(__file__).resolve().parent.parent.parent +# The trees this repository's own Lean lives in. `Extract.lean`'s `isFCLocal` +# tests the same set with a `FormalConjectures` module-name prefix, so a +# declaration Lean calls FC-local is one of these, and everything here that +# asks "is this ours" asks it the same way. +FC_SOURCE_TREES = ( + "FormalConjectures", + "FormalConjecturesForMathlib", + "FormalConjecturesUtil", +) + +# Problem statements live in the first tree; the other two are the support +# layer that statements are written against. SOURCE_DIRS = [ROOT / "FormalConjectures"] DECL_START = re.compile( @@ -160,9 +178,8 @@ def prefetch_elaborator_facts(pairs): capture_output=True, text=True, cwd=ROOT, - # Generous: a cold run imports Mathlib and may build the - # extractor first. A hang should end the run, not the day. - timeout=1800 + 30 * len(wanted), + timeout=LEAN_TIMEOUT_SECONDS + + BATCH_TIMEOUT_PER_PAIR_SECONDS * len(wanted), ) except subprocess.TimeoutExpired: # The batch is an optimisation; the per-declaration path is the @@ -221,11 +238,12 @@ def elaborator_facts(module, declaration): capture_output=True, text=True, cwd=ROOT, - timeout=1800, + timeout=LEAN_TIMEOUT_SECONDS, ) except subprocess.TimeoutExpired: raise SystemExit( - f"comparator_facts {declaration}: no answer within 30 minutes" + f"comparator_facts {declaration}: no answer within " + f"{LEAN_TIMEOUT_SECONDS // 60} minutes" ) from None if proc.returncode != 0: raise SystemExit( @@ -521,7 +539,12 @@ def slice_range(lines, source_range): range covers in some toolchains, so it is pulled in when present. """ lo, hi = source_range["startLine"], source_range["endLine"] - end_column = source_range.get("endColumn") + # Every other field of the payload is held to the wire format; the range + # object is read positionally here, so it says so when it is malformed + # rather than slicing to the end of the line. + if "endColumn" not in source_range: + raise SystemExit(f"source range has no endColumn: {source_range}") + end_column = source_range["endColumn"] while ( lo > 1 and lines[lo - 2].rstrip().endswith(" in") @@ -560,8 +583,11 @@ def fc_notation_commands(): if _NOTATION_CACHE is not None: return _NOTATION_CACHE commands = [] - roots = [ROOT / "FormalConjecturesForMathlib", ROOT / "FormalConjecturesUtil"] - for src in roots + SOURCE_DIRS: + # Support trees first, then the problems tree. The order decides the + # order copied notation appears in a generated module, so it is fixed + # rather than incidental. + support = [tree for tree in FC_SOURCE_TREES if tree != SOURCE_DIRS[0].name] + for src in [ROOT / tree for tree in support] + SOURCE_DIRS: for path in sorted(src.rglob("*.lean")): lines = path.read_text(encoding="utf-8").split("\n") for index, line in enumerate(lines): @@ -601,7 +627,7 @@ def fc_notation_commands(): # which imports both; one in a problem module is not, since # problem files do not import each other, and the problem # file's own notations travel with the preamble. - shared = src.name != "FormalConjectures" + shared = src.name != SOURCE_DIRS[0].name commands.append( (tokens, command, scope, shared, path.relative_to(ROOT)) ) @@ -970,6 +996,7 @@ def _base_pins(): ["git", "-C", str(ROOT), "merge-base", "HEAD", "origin/main"], capture_output=True, text=True, + timeout=GIT_TIMEOUT_SECONDS, ) if merge_base.returncode != 0 or not merge_base.stdout.strip(): raise SystemExit("cannot resolve the Formal Conjectures source revision") @@ -998,6 +1025,7 @@ def pins(source_paths=None): + paths, capture_output=True, text=True, + timeout=GIT_TIMEOUT_SECONDS, ) if tracked.returncode != 0: raise SystemExit(f"cannot list {fc_rev[:12]}: {tracked.stderr.strip()}") @@ -1009,7 +1037,8 @@ def pins(source_paths=None): "generating" ) comparison = subprocess.run( - ["git", "-C", str(ROOT), "diff", "--quiet", fc_rev, "--"] + paths + ["git", "-C", str(ROOT), "diff", "--quiet", fc_rev, "--"] + paths, + timeout=GIT_TIMEOUT_SECONDS, ) if comparison.returncode not in (0, 1): raise SystemExit(f"cannot compare {', '.join(paths)} with {fc_rev[:12]}") @@ -1018,6 +1047,7 @@ def pins(source_paths=None): ["git", "-C", str(ROOT), "diff", "--name-only", fc_rev, "--"] + paths, capture_output=True, text=True, + timeout=GIT_TIMEOUT_SECONDS, ) raise SystemExit( f"{changed.stdout.strip() or ', '.join(paths)} differs from " @@ -1039,6 +1069,7 @@ def importer_state(): ["git", "-C", str(ROOT), "rev-parse", "HEAD"], capture_output=True, text=True, + timeout=GIT_TIMEOUT_SECONDS, ) if head.returncode != 0 or not head.stdout.strip(): raise SystemExit("cannot resolve the importer's own commit") @@ -1046,6 +1077,7 @@ def importer_state(): ["git", "-C", str(ROOT), "status", "--porcelain", "--", "comparator"], capture_output=True, text=True, + timeout=GIT_TIMEOUT_SECONDS, ) # Empty output means a clean tree, and a failed command also produces # empty output. Reporting the reassuring answer for both would make the diff --git a/comparator/adapter/known_failures.py b/comparator/adapter/known_failures.py index fc0f2ee185..5204730b30 100644 --- a/comparator/adapter/known_failures.py +++ b/comparator/adapter/known_failures.py @@ -12,14 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""The known-failures ledger: `comparator/known_failures.toml` and its loader. +"""The known-failures ledger: `comparator/known_failures.toml`, its loader and its gate. -Both gates read the ledger — the source-side set run and the target-side +Both stages read the ledger — the source-side set run and the target-side compile — and each matches it exactly: an unexpected failure and a silently -fixed one both fail. The loader lives here so neither command has to import -the other to read the format. +fixed one both fail. Format, loader and comparison live here so the two +stages cannot drift into disagreeing about what the ledger means, and so +neither command has to import the other to read it. """ +import sys import tomllib KNOWN_KEYS = frozenset({"declaration", "stage", "reason", "workspace"}) @@ -62,3 +64,30 @@ def load_known_failures(path): ) failures[entry["declaration"]] = entry return failures + + +def expected_failures(recorded, stage, key): + """The names the ledger expects to fail at `stage`, read from `key`.""" + return {entry[key] for entry in recorded.values() if entry["stage"] == stage} + + +def gate(recorded, actual, stage, key): + """Whether the observed failures are exactly the recorded ones. + + Both directions fail. An unexpected failure is the obvious one; a + recorded failure that no longer happens is the one a gate without this + check would never mention, and a ledger nobody prunes stops describing + the run it guards. + """ + expected = expected_failures(recorded, stage, key) + unexpected = sorted(set(actual) - expected) + fixed = sorted(expected - set(actual)) + for name in unexpected: + print(f"unexpected {stage} failure: {name}", file=sys.stderr) + for name in fixed: + print( + f"{name} is recorded as a known {stage} failure but did not fail; " + "remove it from the record", + file=sys.stderr, + ) + return not (unexpected or fixed) diff --git a/comparator/adapter/leaneval_generator_cli.py b/comparator/adapter/leaneval_generator_cli.py index 12da25d91a..3f335aa9f7 100644 --- a/comparator/adapter/leaneval_generator_cli.py +++ b/comparator/adapter/leaneval_generator_cli.py @@ -28,6 +28,7 @@ import shutil import subprocess +from limits import LEAN_TIMEOUT_SECONDS from leaneval_interface import parse_response BINARY_ENV = "LEAN_EVAL_GENERATOR_BIN" @@ -90,13 +91,12 @@ def generate(request_text, cwd=None, expected_ids=None): cwd=cwd, # The generator is deterministic and does no I/O beyond the # context root, so a run that has not answered by now is stuck. - # This is the only call in the adapter that crosses into an - # external binary; the extractor's two are bounded the same way. - timeout=1800, + timeout=LEAN_TIMEOUT_SECONDS, ) except subprocess.TimeoutExpired: raise SystemExit( - "lean-eval-generator did not answer within 30 minutes" + "lean-eval-generator did not answer within " + f"{LEAN_TIMEOUT_SECONDS // 60} minutes" ) from None if proc.returncode != 0: raise SystemExit( diff --git a/comparator/adapter/leaneval_interface.py b/comparator/adapter/leaneval_interface.py index 5abfed4c35..81b6dcb412 100644 --- a/comparator/adapter/leaneval_interface.py +++ b/comparator/adapter/leaneval_interface.py @@ -203,19 +203,29 @@ def from_json_object(cls, payload): raise SystemExit( f"producer {section} has unknown keys: {', '.join(unknown)}" ) - importer = payload.get("importer", {}) - generator = payload.get("generator", {}) - target = payload.get("target", {}) + missing = sorted( + f"{section}.{key}" + for section, keys in sections.items() + for key in keys + if key not in payload.get(section, {}) + ) + if missing: + raise SystemExit( + f"producer record has no {', '.join(missing)}" + ) + importer = payload["importer"] + generator = payload["generator"] + target = payload["target"] return cls( - importer_commit=importer.get("commit", ""), - importer_dirty=bool(importer.get("dirty", False)), - generator_repository=generator.get("repository", ""), - generator_rev=generator.get("rev", ""), - contract_version=generator.get("contract_version", 0), - target_lean_toolchain=target.get("lean_toolchain", ""), - target_mathlib_revision=target.get("mathlib_revision", ""), - target_comparator=target.get("comparator", ""), - target_lean4export=target.get("lean4export", ""), + importer_commit=importer["commit"], + importer_dirty=bool(importer["dirty"]), + generator_repository=generator["repository"], + generator_rev=generator["rev"], + contract_version=generator["contract_version"], + target_lean_toolchain=target["lean_toolchain"], + target_mathlib_revision=target["mathlib_revision"], + target_comparator=target["comparator"], + target_lean4export=target["lean4export"], ) @@ -588,7 +598,7 @@ def line_of(offset): return spans -def build_problem(marked_up, manifest, policy, module_name=None): +def build_problem(marked_up, manifest, policy): """One problem entry of the schema-version-1 request, and its `.ilean` declaration map. The module name is a single identifier on purpose: the generator resolves @@ -608,7 +618,7 @@ def build_problem(marked_up, manifest, policy, module_name=None): metadata from the spans it computed, which it can do exactly because it rendered the module. """ - module_name = module_name or slug(manifest.id) + module_name = slug(manifest.id) text = marked_up.render() spans = declaration_spans(text, module_declarations(marked_up, manifest)) resolved, ilean = [], {} diff --git a/comparator/adapter/limits.py b/comparator/adapter/limits.py new file mode 100644 index 0000000000..494eda9061 --- /dev/null +++ b/comparator/adapter/limits.py @@ -0,0 +1,34 @@ +# Copyright 2026 The Formal Conjectures Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""How long the adapter waits for anything it does not control. + +Every subprocess here is bounded, and the bounds are stated once. They live +in their own module because the four callers — source reading, the importer, +the generator plumbing and the target compile — otherwise have no reason to +import one another, and a shared constant is not a reason to grow the +dependency graph. +""" + +# A Lean run that has not answered by now is stuck, not slow: a cold run +# imports Mathlib and may build the extractor first, and that is minutes. +# A hang should end the run, not the day. +LEAN_TIMEOUT_SECONDS = 1800 + +# Each extra pair in a batch shares one environment, so it costs elaboration +# but not another import. +BATCH_TIMEOUT_PER_PAIR_SECONDS = 30 + +# Git here is local and reads the index; a minute means a stuck lock file. +GIT_TIMEOUT_SECONDS = 60 diff --git a/comparator/adapter/make_comparator_workspace.py b/comparator/adapter/make_comparator_workspace.py index 82598f45f2..516dfd380a 100644 --- a/comparator/adapter/make_comparator_workspace.py +++ b/comparator/adapter/make_comparator_workspace.py @@ -66,7 +66,7 @@ import sys import tempfile -from known_failures import load_known_failures +from known_failures import gate, load_known_failures import fc_leaneval_importer as importer import fc_source import leaneval_generator_cli as generator_cli @@ -349,30 +349,14 @@ def import_set(set_name, out_dir, verify=False, known_failures=None): "declarations": results, } if known_failures is not None: - expected = { - name - for name, entry in known_failures.items() - if entry["stage"] == "source" - } actual = { entry["declaration"] for entry in results if entry["status"] == "source-failed" } - unexpected = sorted(actual - expected) - fixed = sorted(expected - actual) - if unexpected or fixed: - for name in unexpected: - print(f"unexpected source failure: {name}", file=sys.stderr) - for name in fixed: - print( - f"{name} is recorded as a known source failure but " - "imported; remove it from the record", - file=sys.stderr, - ) - report["known_failures_match"] = False - else: - report["known_failures_match"] = True + report["known_failures_match"] = gate( + known_failures, actual, "source", "declaration" + ) return report @@ -448,7 +432,12 @@ def main(argv): if args.report: pathlib.Path(args.report).write_text(text, encoding="utf-8") print(text, end="") - if known is not None and not report.get("known_failures_match", True): + if known is None: + print( + "no --known-failures: this run reports failures but gates nothing", + file=sys.stderr, + ) + if known is not None and not report["known_failures_match"]: return 1 return 0 if not args.declaration: From 43a36f5e1dee9250ab0652ed28049bd19f1d8631 Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:05:46 -0400 Subject: [PATCH 68/70] Let CI ask where the workspace went instead of respelling the rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Twenty lines of YAML spelled workspace ids the code derives, so three rules — the slug character class, the qualified-name join, and the default id — were encoded in a second place that could not be checked against the first. The command already prints the directory it wrote, so the jobs capture it. A change to how an id is built now fails in the code that owns it. The same for the sidecar: the seam step globs the file the command named rather than composing the name again, and it passes the expected id through, so the response-identity check runs where it matters most. Three smaller duplications go with them. The elan release was installed by two jobs from two copies of the same four lines and is now one composite action, which is what the extractor action already existed to be. The audit's bundle manifest hand-rolled JSON and hashing that differed from the adapter's in encoding, so its digests could not be compared with the ones inside the artifacts it lists; it uses the adapter's now. The Comparator workflow was the only file here without a licence header. --- .github/actions/install-elan/action.yml | 31 ++++++ .github/actions/prepare-extractor/action.yml | 7 +- .github/workflows/build-and-docs.yml | 6 +- .github/workflows/comparator-lean-4-33.yml | 111 ++++++++++++------- .github/workflows/fc100-audit.yml | 25 +++-- 5 files changed, 120 insertions(+), 60 deletions(-) create mode 100644 .github/actions/install-elan/action.yml diff --git a/.github/actions/install-elan/action.yml b/.github/actions/install-elan/action.yml new file mode 100644 index 0000000000..8483728b4c --- /dev/null +++ b/.github/actions/install-elan/action.yml @@ -0,0 +1,31 @@ +# Copyright 2026 The Formal Conjectures Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# https://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Install elan +description: >- + Install elan and put it on PATH, with no default toolchain: every job here + builds in a checkout whose `lean-toolchain` decides the version. The elan + release is pinned in one place so two jobs cannot install two different + installers. + +runs: + using: composite + steps: + - name: Install elan + shell: bash + run: | + set -o pipefail + curl -sSfL https://github.com/leanprover/elan/releases/download/v1.4.2/elan-x86_64-unknown-linux-gnu.tar.gz | tar xz + ./elan-init -y --default-toolchain none + echo "$HOME/.elan/bin" >> "$GITHUB_PATH" diff --git a/.github/actions/prepare-extractor/action.yml b/.github/actions/prepare-extractor/action.yml index d99038d93d..1484d29e4a 100644 --- a/.github/actions/prepare-extractor/action.yml +++ b/.github/actions/prepare-extractor/action.yml @@ -31,12 +31,7 @@ runs: using: composite steps: - name: Install elan - shell: bash - run: | - set -o pipefail - curl -sSfL https://github.com/leanprover/elan/releases/download/v1.4.2/elan-x86_64-unknown-linux-gnu.tar.gz | tar xz - ./elan-init -y --default-toolchain none - echo "$HOME/.elan/bin" >> "$GITHUB_PATH" + uses: ./.github/actions/install-elan - name: Build the extractor and the source modules shell: bash diff --git a/.github/workflows/build-and-docs.yml b/.github/workflows/build-and-docs.yml index d07d4479cd..63f7dea6d8 100644 --- a/.github/workflows/build-and-docs.yml +++ b/.github/workflows/build-and-docs.yml @@ -113,11 +113,7 @@ jobs: - name: Install elan if: steps.mode.outputs.website_only != 'true' - run: | - set -o pipefail - curl -sSfL https://github.com/leanprover/elan/releases/download/v1.4.2/elan-x86_64-unknown-linux-gnu.tar.gz | tar xz - ./elan-init -y --default-toolchain none - echo "$HOME/.elan/bin" >> "$GITHUB_PATH" + uses: ./.github/actions/install-elan - name: Restore ~/.cache/mathlib if: steps.mode.outputs.website_only != 'true' diff --git a/.github/workflows/comparator-lean-4-33.yml b/.github/workflows/comparator-lean-4-33.yml index 40f8113b3c..1d47a044b5 100644 --- a/.github/workflows/comparator-lean-4-33.yml +++ b/.github/workflows/comparator-lean-4-33.yml @@ -1,3 +1,17 @@ +# Copyright 2026 The Formal Conjectures Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# https://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + name: Generated workspace at LeanEval pins # Everything that crosses the adapter's boundaries, in one job: @@ -107,22 +121,32 @@ jobs: # a separate Linux job. - name: Comparator generation smoke test run: | - for d in exists_hadamard_zero erdos_940.variants.large_integers \ - erdos_1038.parts.i erdos_100.variants.strong \ - KotherConjecture.variants.le_KotherRadical \ - OeisA303656.conjecture OeisA308734.conjecture \ - curling_number_conjecture; do - python3 comparator/adapter/make_comparator_workspace.py "$d" --out .comparator - done - grep -q "large_integers_answer : Prop" .comparator/Erdos940_erdos_940_variants_large_integers/Challenge.lean - grep -q "i_answer : ENNReal" .comparator/Erdos1038_erdos_1038_parts_i/Challenge.lean - grep -q "Submission.erdos_100_variants_strong$" .comparator/Erdos100_erdos_100_variants_strong/Solution.lean - grep -q "le_KotherRadical hI" .comparator/Koethe_KotherConjecture_variants_le_KotherRadical/Solution.lean + # The command prints the directory it wrote. Asking it removes the + # only other place that would have to know how a qualified name + # becomes a workspace id, and a change to that rule then fails in + # the code that owns it rather than in twenty lines of YAML. + ws() { + python3 comparator/adapter/make_comparator_workspace.py "$1" --out .comparator + } + hadamard=$(ws exists_hadamard_zero) + large_integers=$(ws erdos_940.variants.large_integers) + parts_i=$(ws erdos_1038.parts.i) + strong=$(ws erdos_100.variants.strong) + koethe=$(ws KotherConjecture.variants.le_KotherRadical) + oeis_a=$(ws OeisA303656.conjecture) + oeis_b=$(ws OeisA308734.conjecture) + curling=$(ws curling_number_conjecture) + echo "PARTS_I=$parts_i" >> "$GITHUB_ENV" + test -d "$hadamard" + grep -q "large_integers_answer : Prop" "$large_integers/Challenge.lean" + grep -q "i_answer : ENNReal" "$parts_i/Challenge.lean" + grep -q "Submission.erdos_100_variants_strong$" "$strong/Solution.lean" + grep -q "le_KotherRadical hI" "$koethe/Solution.lean" # Two modules declare `conjecture`; qualified default ids keep the # workspaces apart, and a guillemet module path decodes correctly. - test -d .comparator/OeisA303656_conjecture - test -d .comparator/OeisA308734_conjecture - test -d .comparator/Arxiv__0912_2382__curling_number_conjecture + test "$oeis_a" != "$oeis_b" + test -d "$oeis_a" && test -d "$oeis_b" + case "$curling" in *0912_2382*) ;; *) echo "guillemet module lost its digits: $curling"; exit 1 ;; esac # The importer-to-generator seam, on a real declaration. # `--emit-import` writes the exact bytes that cross it — the v1 @@ -131,11 +155,14 @@ jobs: # workspace exactly; if it does not, the emitted artifact is not the # whole interface. See comparator/OWNERSHIP.md. - name: Importer to generator seam + env: + WORKSPACE: ${{ env.PARTS_I }} run: | - python3 comparator/adapter/make_comparator_workspace.py erdos_1038.parts.i \ - --emit-import .comparator-import + HANDED_OVER=$(python3 comparator/adapter/make_comparator_workspace.py \ + erdos_1038.parts.i --emit-import .comparator-import) + export HANDED_OVER python3 - <<'PY' - import json + import glob import os import pathlib import sys @@ -144,12 +171,14 @@ jobs: import leaneval_generator_cli as generator_cli from leaneval_interface import ProblemManifest - handed_over = pathlib.Path( - ".comparator-import/Erdos1038_erdos_1038_parts_i" - ).resolve() + handed_over = pathlib.Path(os.environ["HANDED_OVER"]).resolve() + workspace = pathlib.Path(os.environ["WORKSPACE"]).resolve() + problem_id = handed_over.name + + # One sidecar, named by the command, found rather than spelled. + (sidecar,) = glob.glob(str(handed_over / "fc-provenance-*.json")) manifest = ProblemManifest.from_json( - (handed_over / "fc-provenance-Erdos1038_erdos_1038_parts_i.json") - .read_text(encoding="utf-8") + pathlib.Path(sidecar).read_text(encoding="utf-8") ) assert len(manifest.source.commit) == 40, manifest.source.commit assert manifest.source.declaration, "no FC declaration id" @@ -157,9 +186,10 @@ jobs: # The emitted request is fed back as its exact bytes — no reparse, # no reserialise — which is the claim the seam test exists to check. request_text = (handed_over / "request.json").read_text(encoding="utf-8") - workspace = pathlib.Path(".comparator/Erdos1038_erdos_1038_parts_i").resolve() - regenerated = generator_cli.generate(request_text, cwd=handed_over) - files = regenerated["Erdos1038_erdos_1038_parts_i"] + regenerated = generator_cli.generate( + request_text, cwd=handed_over, expected_ids=[problem_id] + ) + files = regenerated[problem_id] for name, content in files.items(): expected = (workspace / name).read_text(encoding="utf-8") assert content == expected, name @@ -176,21 +206,22 @@ jobs: env: TARGET_TOOLCHAIN: ${{ steps.target.outputs.lean_toolchain }} run: | - for d in isSumOfThreeCubes_2 isSumOfThreeCubes_iff_mod_9; do - python3 comparator/adapter/make_comparator_workspace.py "$d" \ + gen() { + python3 comparator/adapter/make_comparator_workspace.py "$1" \ --out .comparator --verify - done + } + PLAIN=$(gen isSumOfThreeCubes_2) + HOLED=$(gen isSumOfThreeCubes_iff_mod_9) + echo "PLAIN=$PLAIN" >> "$GITHUB_ENV" + echo "HOLED=$HOLED" >> "$GITHUB_ENV" # One plain theorem and one `answer(sorry)` slot typed at 4.33.1. - grep -q "isSumOfThreeCubes_iff_mod_9_answer : Prop" \ - .comparator/SumOfThreeCubes_isSumOfThreeCubes_iff_mod_9/Challenge.lean + grep -q "isSumOfThreeCubes_iff_mod_9_answer : Prop" "$HOLED/Challenge.lean" # Generated for LeanEval, not for here. - grep -q "$TARGET_TOOLCHAIN" \ - .comparator/SumOfThreeCubes_isSumOfThreeCubes_2/lean-toolchain + grep -q "$TARGET_TOOLCHAIN" "$PLAIN/lean-toolchain" - name: Build both workspaces at Lean 4.33 run: | - for ws in .comparator/SumOfThreeCubes_isSumOfThreeCubes_2 \ - .comparator/SumOfThreeCubes_isSumOfThreeCubes_iff_mod_9; do + for ws in "$PLAIN" "$HOLED"; do (cd "$ws" && lake update && lake exe cache get && lake build) done @@ -220,7 +251,7 @@ jobs: # adds `sorryAx`, which `permitted_axioms` does not allow. A # generated workspace that passed before anyone proved anything # would be worthless. - if (cd .comparator/SumOfThreeCubes_isSumOfThreeCubes_2 && lake test); then + if (cd "$PLAIN" && lake test); then echo "::error::Comparator accepted an unproved generated workspace" exit 1 fi @@ -228,15 +259,16 @@ jobs: # 2. The same workspace with the statement actually proved. The # witness is the one this repository's own source gives. python3 - <<'PY' + import os import pathlib - submission = pathlib.Path(".comparator/SumOfThreeCubes_isSumOfThreeCubes_2/Submission.lean") + submission = pathlib.Path(os.environ["PLAIN"]) / "Submission.lean" text = submission.read_text(encoding="utf-8") filled = text.replace(":= by\n sorry", ":= by\n exact ⟨1, 1, 0, by norm_num⟩") assert filled != text, "nothing to fill in the generated submission" submission.write_text(filled, encoding="utf-8") PY - (cd .comparator/SumOfThreeCubes_isSumOfThreeCubes_2 && lake build && lake test) + (cd "$PLAIN" && lake build && lake test) # 3. The `answer(sorry)` workspace, with the hole filled by the # proposition on the other side of the iff and the bridge closed @@ -245,9 +277,10 @@ jobs: # stand in for a human reading the answer. The generated README # tells the solver so; this asserts it. python3 - <<'PY' + import os import pathlib - root = pathlib.Path(".comparator/SumOfThreeCubes_isSumOfThreeCubes_iff_mod_9") + root = pathlib.Path(os.environ["HOLED"]) submission = root / "Submission.lean" text = submission.read_text(encoding="utf-8") answer = ( @@ -266,5 +299,5 @@ jobs: assert filled != text, "the proof hole was not where this step expects" submission.write_text(filled, encoding="utf-8") PY - (cd .comparator/SumOfThreeCubes_isSumOfThreeCubes_iff_mod_9 && lake build && lake test) + (cd "$HOLED" && lake build && lake test) echo "Comparator accepted a gamed definition hole; hole values need a human." diff --git a/.github/workflows/fc100-audit.yml b/.github/workflows/fc100-audit.yml index b6996a97c8..dd3ed2d916 100644 --- a/.github/workflows/fc100-audit.yml +++ b/.github/workflows/fc100-audit.yml @@ -133,25 +133,30 @@ jobs: if: always() run: | python3 - <<'PY' - import hashlib - import json import pathlib + import sys + + sys.path.insert(0, "comparator/adapter") + from leaneval_interface import dump_json, sha256_text + + # The same digest and the same serialisation the sidecars use, so a + # reader can compare a manifest entry with the digest recorded + # inside the artifact it names. + def digest(path): + return sha256_text(path.read_text(encoding="utf-8")) manifest = {} - for pattern in ("fc100-report.json", "fc100-target-report.json"): - path = pathlib.Path(pattern) + for name in ("fc100-report.json", "fc100-target-report.json"): + path = pathlib.Path(name) if path.is_file(): - manifest[str(path)] = hashlib.sha256(path.read_bytes()).hexdigest() + manifest[str(path)] = digest(path) root = pathlib.Path(".fc100") if root.is_dir(): for path in sorted(root.rglob("*")): if path.is_file(): - manifest[str(path)] = hashlib.sha256( - path.read_bytes() - ).hexdigest() + manifest[str(path)] = digest(path) pathlib.Path("bundle-manifest.json").write_text( - json.dumps(manifest, indent=2, sort_keys=True) + "\n", - encoding="utf-8", + dump_json(manifest, sort_keys=True), encoding="utf-8" ) print(f"{len(manifest)} files digested") PY From e9392d043f1cb35deab0423d51911b4a3ad9a6de Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:09:39 -0400 Subject: [PATCH 69/70] Key every cache on the root it read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three modules each held their own copy of the root, so patching one patched a third of the adapter, and the tests only worked because they also mocked whatever the other two would have reached. Every module now reads `fc_source.ROOT`, and the directories derived from it are derived on each read rather than snapshotted at import. The caches were the reason that mattered. `_declared_names`, `fc_notation_commands` and `_base_pins` each cached a value computed from the root while keyed on nothing, and the notation cache stored root-relative paths that reach `pins`. They take the root as an argument now, so a moved root cannot collide with an entry from the old one, and the `cache_clear` calls the tests needed — one of them present, two missing — are gone with the reason for them. The two leaky helpers go too: a `setUp` that assigned before it could register cleanup, and a root assigned outside the `try` that undid it. --- comparator/adapter/fc_leaneval_importer.py | 43 ++++++++------- comparator/adapter/fc_source.py | 53 ++++++++++--------- .../adapter/make_comparator_workspace.py | 7 ++- .../adapter/test_fc_leaneval_importer.py | 42 +++++++-------- comparator/adapter/test_fc_source.py | 2 +- 5 files changed, 79 insertions(+), 68 deletions(-) diff --git a/comparator/adapter/fc_leaneval_importer.py b/comparator/adapter/fc_leaneval_importer.py index 8a1729fe51..4e168c939c 100644 --- a/comparator/adapter/fc_leaneval_importer.py +++ b/comparator/adapter/fc_leaneval_importer.py @@ -44,6 +44,7 @@ lean_errors, sha256_text, ) +import fc_source from fc_source import ( DECL_START, FC_SOURCE_TREES, @@ -60,14 +61,20 @@ notation_blocks, pins, replace_proof_with_sorry, - ROOT, slice_range, strip_decorations, strip_fc_attributes, unwrap_answers, ) -COMPARATOR_DIR = ROOT / "comparator" -MANIFEST_DIR = COMPARATOR_DIR / "problems" +# Derived on each read, like every other path here, so that a root which +# moves takes them with it rather than leaving a snapshot from import time. +def comparator_dir(): + return fc_source.ROOT / "comparator" + + +def manifest_dir(): + """Where the per-problem source-boundary files live.""" + return comparator_dir() / "problems" SOURCE_REPOSITORY = "https://github.com/google-deepmind/formal-conjectures" @@ -77,7 +84,7 @@ def _tools_file(): """comparator/tools.toml is the one machine-readable source of pins, and this module refuses to restate it.""" - with (COMPARATOR_DIR / "tools.toml").open("rb") as handle: + with (comparator_dir() / "tools.toml").open("rb") as handle: return tomllib.load(handle) @@ -158,7 +165,7 @@ def explicit_copy_dependencies(problem_file): raise SystemExit( f"copy dependency module must stay under a source tree: {relative}" ) - path = ROOT / relative + path = fc_source.ROOT / relative if not path.is_file() or relative.suffix != ".lean": raise SystemExit(f"copy dependency module does not exist: {relative}") module = module_name(relative) @@ -215,7 +222,7 @@ def load_manifest(problem_id): ambiguous answer-slot type is a `--answer-type` argument: it is rare, and a field no problem uses is a format nobody can check. """ - path = MANIFEST_DIR / f"{problem_id}.toml" + path = manifest_dir() / f"{problem_id}.toml" if not path.exists(): return {} with path.open("rb") as handle: @@ -237,7 +244,7 @@ def load_manifest(problem_id): def manifest_ids(): - return sorted(p.stem for p in MANIFEST_DIR.glob("*.toml")) + return sorted(p.stem for p in manifest_dir().glob("*.toml")) def closure_region( @@ -361,7 +368,7 @@ def covered_by_another(dep): raise SystemExit(f"{declaration}: {dep['name']} sliced to nothing") namespace = ".".join(namespaces) chunk = [ - f"-- {dep['name']}, from {path.relative_to(ROOT)}", + f"-- {dep['name']}, from {path.relative_to(fc_source.ROOT)}", "noncomputable section", ] chunk += preamble @@ -377,7 +384,7 @@ def covered_by_another(dep): { "declaration": dep["name"], "module": dep["module"], - "path": str(path.relative_to(ROOT)), + "path": str(path.relative_to(fc_source.ROOT)), "range": dep["range"], "content_sha256": sha256_text(body), } @@ -421,7 +428,7 @@ def source_record( workspace. """ blob = subprocess.run( - ["git", "-C", str(ROOT), "rev-parse", f"{fc_rev}:{source_path}"], + ["git", "-C", str(fc_source.ROOT), "rev-parse", f"{fc_rev}:{source_path}"], capture_output=True, text=True, check=False, @@ -445,7 +452,7 @@ def source_record( copied_dependencies=tuple(copied_records), original_range=dict(original_range), original_sha256=sha256_text(original), - lean_toolchain=(ROOT / "lean-toolchain").read_text(encoding="utf-8").strip(), + lean_toolchain=(fc_source.ROOT / "lean-toolchain").read_text(encoding="utf-8").strip(), mathlib_revision=mathlib_rev, ) @@ -459,7 +466,7 @@ def locate_target(problem, module=None): """ problem_file, declaration, located = _resolve(problem, module) path, _imports, module_doc, _body = located - fc_module = module_name(path.relative_to(ROOT)) + fc_module = module_name(path.relative_to(fc_source.ROOT)) facts = elaborator_facts(fc_module, declaration) if facts.range is None: raise SystemExit(f"{declaration}: no source range recorded") @@ -599,7 +606,7 @@ def _resolve(problem, module=None): def statement_pair(problem): """The `(module, declaration)` pair `import_problem` will ask the elaborator about.""" _, declaration, (path, _imports, _doc, _body) = _resolve(problem) - return module_name(path.relative_to(ROOT)), declaration + return module_name(path.relative_to(fc_source.ROOT)), declaration def import_problem(problem, answer_type=None, module=None): @@ -650,7 +657,7 @@ def import_problem(problem, answer_type=None, module=None): # toolchain bump — the change that most needs this check to run — the one # change that cannot pass it. read_paths = ( - [path.relative_to(ROOT)] + [path.relative_to(fc_source.ROOT)] + [record["path"] for record in copied_records] + list(notation_paths) ) @@ -674,7 +681,7 @@ def import_problem(problem, answer_type=None, module=None): source=source_record( qualified, fc_module, - path.relative_to(ROOT), + path.relative_to(fc_source.ROOT), fc_rev, copied_records, original, @@ -715,7 +722,7 @@ def elaborate(marked_up): ["lake", "env", "lean", combined], capture_output=True, text=True, - cwd=ROOT, + cwd=fc_source.ROOT, check=False, timeout=LEAN_TIMEOUT_SECONDS, ) @@ -748,12 +755,12 @@ def validate(): problem_file = load_manifest(problem_id) declaration = problem_file["declaration"] path, _i, _d, _b = find_declaration(declaration, problem_file.get("module")) - elaborator_facts(module_name(path.relative_to(ROOT)), declaration) + elaborator_facts(module_name(path.relative_to(fc_source.ROOT)), declaration) except SystemExit as exc: print(f"{problem_id}: {exc}", file=sys.stderr) bad += 1 continue - print(f"{problem_id}: {declaration} in {path.relative_to(ROOT)}") + print(f"{problem_id}: {declaration} in {path.relative_to(fc_source.ROOT)}") if bad: print(f"{bad} problem file(s) do not resolve", file=sys.stderr) return 1 if bad else 0 diff --git a/comparator/adapter/fc_source.py b/comparator/adapter/fc_source.py index 81c11b8299..de9258b766 100644 --- a/comparator/adapter/fc_source.py +++ b/comparator/adapter/fc_source.py @@ -64,7 +64,17 @@ def declaration(self): # Problem statements live in the first tree; the other two are the support # layer that statements are written against. -SOURCE_DIRS = [ROOT / "FormalConjectures"] +PROBLEM_TREE = FC_SOURCE_TREES[0] + + +def source_dirs(root): + """The trees problem statements live in, under `root`. + + A function rather than a constant because every cache below is keyed on + the root it read, and a module-level list derived from `ROOT` at import + would not follow a root that moved. + """ + return [root / PROBLEM_TREE] DECL_START = re.compile( # `local notation` and `scoped notation` carry the modifier before the @@ -345,8 +355,8 @@ def module_name(rel_path): ] return ".".join(parts) -@functools.lru_cache(maxsize=1) -def _declared_names(): +@functools.lru_cache(maxsize=4) +def _declared_names(root): """Every `theorem`/`lemma` name token in the tree, one pass, cached. `{path: [name, ...]}` in sorted path order. A batch import looks up @@ -355,13 +365,13 @@ def _declared_names(): """ token = re.compile(r"(?:theorem|lemma)\s+([\w.«»]+)[\s:]") index = {} - for src in SOURCE_DIRS: + for src in source_dirs(root): for path in sorted(src.rglob("*.lean")): index[path] = token.findall(path.read_text(encoding="utf-8")) return index -def _declaring_files(name): +def _declaring_files(root, name): r"""The files whose text declares `name` as a theorem or lemma. A declared token matches when it equals `name` or ends in `.name` — @@ -371,7 +381,7 @@ def _declaring_files(name): dotted = "." + name return [ path - for path, names in _declared_names().items() + for path, names in _declared_names(root).items() if any(t == name or t.endswith(dotted) for t in names) ] @@ -405,7 +415,7 @@ def find_declaration(basename, module=None): if not named.exists(): raise SystemExit(f"manifest names {module}, which does not exist") return _read_source(named) - hits = _declaring_files(basename) + hits = _declaring_files(ROOT, basename) if not hits and "." in basename: # A fully qualified request such as `OeisA303656.conjecture` names a # declaration whose file spells only `conjecture`, the prefix coming @@ -419,7 +429,7 @@ def find_declaration(basename, module=None): prefix, suffix = parts[:cut], ".".join(parts[cut:]) hits = [ path - for path in _declaring_files(suffix) + for path in _declaring_files(ROOT, suffix) if _declares_namespaces(path.read_text(encoding="utf-8"), prefix) ] if hits: @@ -561,9 +571,8 @@ def slice_range(lines, source_range): r"(?:notation[0-9]*|postfix|prefix|infixl|infixr|infix)[:\s]" ) -_NOTATION_CACHE = None - -def fc_notation_commands(): +@functools.lru_cache(maxsize=4) +def fc_notation_commands(root): """Every exportable notation command an FC module defines, with its token. A notation is not a constant, so the elaborated closure never reports it: @@ -579,15 +588,12 @@ def fc_notation_commands(): file relative to ROOT — a copied command's text is a read source input, so the snapshot check needs to know where it came from. """ - global _NOTATION_CACHE - if _NOTATION_CACHE is not None: - return _NOTATION_CACHE commands = [] # Support trees first, then the problems tree. The order decides the # order copied notation appears in a generated module, so it is fixed # rather than incidental. - support = [tree for tree in FC_SOURCE_TREES if tree != SOURCE_DIRS[0].name] - for src in [ROOT / tree for tree in support] + SOURCE_DIRS: + support = [tree for tree in FC_SOURCE_TREES if tree != PROBLEM_TREE] + for src in [root / tree for tree in support] + source_dirs(root): for path in sorted(src.rglob("*.lean")): lines = path.read_text(encoding="utf-8").split("\n") for index, line in enumerate(lines): @@ -627,12 +633,11 @@ def fc_notation_commands(): # which imports both; one in a problem module is not, since # problem files do not import each other, and the problem # file's own notations travel with the preamble. - shared = src.name != SOURCE_DIRS[0].name + shared = src.name != PROBLEM_TREE commands.append( - (tokens, command, scope, shared, path.relative_to(ROOT)) + (tokens, command, scope, shared, path.relative_to(root)) ) - _NOTATION_CACHE = commands - return commands + return tuple(commands) NOTATION_FAMILY = re.compile(r"^(?:notation[0-9]*|postfix|prefix|infixl|infixr|infix)[:\s]") @@ -677,7 +682,7 @@ def notation_blocks(module_texts, opened): """ combined = "\n".join(module_texts) blocks, seen = [], set() - for tokens, command, scope, shared, path in fc_notation_commands(): + for tokens, command, scope, shared, path in fc_notation_commands(ROOT): if scope: if scope not in opened: continue @@ -982,8 +987,8 @@ def hoist_answers(statement, basename, slot_types, override=None): statement = statement[:start] + name + statement[end:] return statement, holes -@functools.lru_cache(maxsize=1) -def _base_pins(): +@functools.lru_cache(maxsize=4) +def _base_pins(root): """The Mathlib revision and the FC merge-base, invariant for one run. Only the per-path dirty check in `pins` varies between calls, so the @@ -1015,7 +1020,7 @@ def pins(source_paths=None): text all comes from it. A path the revision does not track fails too: `git diff` is silent about untracked files, so tracking is checked first. """ - mathlib_rev, fc_rev = _base_pins() + mathlib_rev, fc_rev = _base_pins(ROOT) if source_paths is not None: if isinstance(source_paths, (str, pathlib.Path)): source_paths = [source_paths] diff --git a/comparator/adapter/make_comparator_workspace.py b/comparator/adapter/make_comparator_workspace.py index 516dfd380a..5bc893edc5 100644 --- a/comparator/adapter/make_comparator_workspace.py +++ b/comparator/adapter/make_comparator_workspace.py @@ -81,7 +81,6 @@ slug, ) -ROOT = importer.ROOT # The request's context directory, relative to the request file, so an # emitted seam artifact is self-contained and reproducible from any path. @@ -157,7 +156,7 @@ def _seam(pairs, group=None): ] target = importer.target_pins() template = ( - importer.COMPARATOR_DIR / "templates" / "WorkspaceTest.lean" + importer.comparator_dir() / "templates" / "WorkspaceTest.lean" ).read_text(encoding="utf-8") request = build_request( [problem for problem, _ in problems], target, template, CONTEXT_DIR @@ -266,7 +265,7 @@ def subset_declarations(set_name): `decl_name%` elaborator is what guarantees each name resolves, so the text layer can read the list without re-proving that. """ - path = ROOT / "FormalConjectures" / "Subsets" / f"{set_name}.lean" + path = fc_source.ROOT / "FormalConjectures" / "Subsets" / f"{set_name}.lean" if not path.is_file(): raise SystemExit(f"no subset module at {path}") names = re.findall( @@ -367,7 +366,7 @@ def main(argv): nargs="?", help="a problem id, or a declaration name such as erdos_940", ) - ap.add_argument("--out", default=str(ROOT / ".comparator")) + ap.add_argument("--out", default=str(fc_source.ROOT / ".comparator")) ap.add_argument( "--answer-type", default=None, diff --git a/comparator/adapter/test_fc_leaneval_importer.py b/comparator/adapter/test_fc_leaneval_importer.py index d740687b44..8588bf2f1e 100644 --- a/comparator/adapter/test_fc_leaneval_importer.py +++ b/comparator/adapter/test_fc_leaneval_importer.py @@ -38,16 +38,19 @@ class ProblemFileTest(unittest.TestCase): """An FC problem file supplies what the Lean source cannot.""" def setUp(self): + # addCleanup rather than tearDown: a setUp that raises after the + # first line would otherwise leave the root pointing at a directory + # this test is about to delete. self._dir = tempfile.TemporaryDirectory() - self._saved = importer.MANIFEST_DIR - importer.MANIFEST_DIR = pathlib.Path(self._dir.name) - - def tearDown(self): - importer.MANIFEST_DIR = self._saved - self._dir.cleanup() + self.addCleanup(self._dir.cleanup) + root = pathlib.Path(self._dir.name) + (root / "comparator" / "problems").mkdir(parents=True) + saved = fc_source.ROOT + self.addCleanup(setattr, fc_source, "ROOT", saved) + fc_source.ROOT = root def write(self, name, body): - (importer.MANIFEST_DIR / name).write_text(body) + (importer.manifest_dir() / name).write_text(body) def test_absent_problem_file_is_not_an_error(self): # Most statements need none, and the importer works without one. @@ -82,23 +85,19 @@ class PinTest(unittest.TestCase): @contextlib.contextmanager def _pins_repo(self, git_results): - saved_root = fc_source.ROOT - fc_source._base_pins.cache_clear() with tempfile.TemporaryDirectory() as tmp: root = pathlib.Path(tmp) - fc_source.ROOT = root (root / "lake-manifest.json").write_text( json.dumps({"packages": [{"name": "mathlib", "rev": "b" * 40}]}), encoding="utf-8", ) - try: + # No cache_clear: `_base_pins` is keyed on the root it read, so a + # fixture root and the real one cannot share an entry. + with _root_at(root): with mock.patch.object( fc_source.subprocess, "run", side_effect=git_results ): yield - finally: - fc_source.ROOT = saved_root - fc_source._base_pins.cache_clear() def test_changed_source_is_refused(self): path = "FormalConjectures/Example.lean" @@ -153,17 +152,18 @@ def test_a_dirty_dependency_fails_even_with_a_clean_target(self): @contextlib.contextmanager def _root_at(directory): - """Point the module's ROOT at a fixture tree. + """Point the adapter's root at a fixture tree. - `closure_region` records each copied declaration's path relative to ROOT, - so a fixture written outside it cannot be described. + Every module reads `fc_source.ROOT` rather than a copy of it, and every + cache is keyed on the root it read, so this is the one place a test has + to move and there is nothing to invalidate afterwards. """ - saved = importer.ROOT - importer.ROOT = pathlib.Path(directory) + saved = fc_source.ROOT + fc_source.ROOT = pathlib.Path(directory) try: - yield + yield fc_source.ROOT finally: - importer.ROOT = saved + fc_source.ROOT = saved class MathlibOnlyClosureTest(unittest.TestCase): diff --git a/comparator/adapter/test_fc_source.py b/comparator/adapter/test_fc_source.py index 50625726ff..7d68b9029a 100644 --- a/comparator/adapter/test_fc_source.py +++ b/comparator/adapter/test_fc_source.py @@ -252,7 +252,7 @@ def test_a_malformed_name_is_refused(self): def test_every_real_module_round_trips(self): # The property that keeps the codec from drifting again: for every # file the importer can name, decoding the name reaches the file. - for src in fc_source.SOURCE_DIRS: + for src in fc_source.source_dirs(fc_source.ROOT): for path in src.rglob("*.lean"): rel = path.relative_to(fc_source.ROOT) with self.subTest(module=str(rel)): From aae0e3d116b43bc641ee189ba7069482bf6e9dba Mon Sep 17 00:00:00 2001 From: Will Blair <85643015+williamjblair@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:18:14 -0400 Subject: [PATCH 70/70] Split the audit along the line where its cost is The whole-set audit was one job doing two things that fail for different reasons and cost differently. Importing, verifying and generating the set is one Mathlib, this repository's, and it is where an FC-side defect shows: that half stays on every pull request touching the adapter. Compiling the result at LeanEval's pins is a second Mathlib at a second toolchain, most of the runtime, and what it catches is the other repository bumping rather than anything a pull request here did: that half runs weekly and on demand, against the set the first half generated rather than regenerating it. `merge_group` stays off deliberately. It does not support path filters, so adding it would put a two-Mathlib job on every merge in a repository with three hundred open pull requests. --- .github/workflows/fc100-audit.yml | 105 +++++++++++++++++++++--------- 1 file changed, 73 insertions(+), 32 deletions(-) diff --git a/.github/workflows/fc100-audit.yml b/.github/workflows/fc100-audit.yml index dd3ed2d916..6b1fe3cfb8 100644 --- a/.github/workflows/fc100-audit.yml +++ b/.github/workflows/fc100-audit.yml @@ -14,19 +14,26 @@ name: FC100 whole-set audit -# The whole-set run lean-eval#536 gates the FC import on, as a CI artifact: -# import every declaration of the frozen set, elaborate each marked-up module -# at this repository's pins, generate every workspace through the pinned -# generator, and compile every generated Challenge at LeanEval's pins in one -# shared project. Failures must match comparator/known_failures.toml exactly — -# an unexpected failure and a silently fixed one both fail the job, because a -# gate that only ever passes proves nothing. +# The whole-set run lean-eval#536 gates the FC import on, as a CI artifact. +# It is two halves, and they are split because they cost differently. # -# Two full Mathlib builds make this far too heavy for every pull request, so -# it runs on demand, weekly, and on pull requests that change what it -# measures — the adapter, pins, ledger, templates or tooling. Ordinary -# statement edits rely on the per-PR jobs' representative declarations and -# the weekly run as backstop. +# source import every declaration of the frozen set, elaborate each +# marked-up module at this repository's pins, generate every +# workspace through the pinned generator, and classify the set. +# One Mathlib, this repository's. This is where an FC-side defect +# shows, so it runs on every pull request that changes what it +# measures — the adapter, pins, ledger, templates or tooling. +# +# target compile every generated Challenge at LeanEval's pins in one +# shared project. A second Mathlib at a second toolchain, which is +# most of the cost, and it catches target-pin drift rather than an +# FC-side defect. Drift comes from the other repository bumping, +# not from a pull request here, so it runs weekly and on demand. +# +# Failures on either side must match comparator/known_failures.toml exactly: +# an unexpected failure and a silently fixed one both fail, because a gate +# that only ever passes proves nothing. Ordinary statement edits rely on the +# per-PR jobs' representative declarations and the weekly run as backstop. concurrency: group: fc100-audit-${{ github.ref }} @@ -63,10 +70,10 @@ permissions: contents: read jobs: - audit: + source: runs-on: ubuntu-latest - name: Import, verify, generate and compile FC100 - timeout-minutes: 300 + name: Import, verify and generate FC100 + timeout-minutes: 120 steps: - name: Checkout Formal Conjectures uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 @@ -115,20 +122,11 @@ jobs: assert sum(categories.values()) + report["source_failed"] == 100, categories PY - # Kim's shared-project arrangement: every generated ChallengeDeps and - # Challenge compiles in one Lake project at LeanEval's pins, so Mathlib - # is built once for the whole set rather than once per workspace. - - name: Compile every generated Challenge at LeanEval pins - run: | - python3 comparator/adapter/compile_fc100_target.py .fc100 \ - --project "$RUNNER_TEMP/fc100-target" \ - --report fc100-target-report.json \ - --known-failures comparator/known_failures.toml - # The artifact is the review evidence, so it carries what produced the - # result, not just the result: the exact request bytes, every - # workspace with its provenance sidecar, both reports, and a manifest - # digesting each file so the bundle can vouch for itself. + # result, not just the result: the exact request bytes, every workspace + # with its provenance sidecar, the report, and a manifest digesting + # each file so the bundle can vouch for itself. The target job below + # consumes this same artifact rather than generating its own. - name: Digest the audit bundle if: always() run: | @@ -146,7 +144,7 @@ jobs: return sha256_text(path.read_text(encoding="utf-8")) manifest = {} - for name in ("fc100-report.json", "fc100-target-report.json"): + for name in ("fc100-report.json",): path = pathlib.Path(name) if path.is_file(): manifest[str(path)] = digest(path) @@ -161,15 +159,58 @@ jobs: print(f"{len(manifest)} files digested") PY - - name: Upload the audit bundle + - name: Upload the generated set if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: fc100-audit-bundle + name: fc100-source-bundle path: | fc100-report.json - fc100-target-report.json bundle-manifest.json .fc100/ if-no-files-found: warn include-hidden-files: true + + # The second Mathlib. It answers "does LeanEval's toolchain still accept + # what we generate", and that changes when the other repository bumps, not + # when a pull request here edits the adapter. So it runs weekly and on + # demand, against the set the job above generated rather than regenerating + # it. `merge_group` is deliberately not a trigger: it does not support path + # filters, so this would then run on every merge in the repository. + target: + runs-on: ubuntu-latest + name: Compile FC100 at LeanEval pins + timeout-minutes: 240 + needs: source + if: github.event_name != 'pull_request' + steps: + - name: Checkout Formal Conjectures + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + + - name: Install elan + uses: ./.github/actions/install-elan + + - name: Download the generated set + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: fc100-source-bundle + + # Kim's shared-project arrangement: every generated ChallengeDeps and + # Challenge compiles in one Lake project at LeanEval's pins, so Mathlib + # is built once for the whole set rather than once per workspace. + - name: Compile every generated Challenge at LeanEval pins + run: | + python3 comparator/adapter/compile_fc100_target.py .fc100 \ + --project "$RUNNER_TEMP/fc100-target" \ + --report fc100-target-report.json \ + --known-failures comparator/known_failures.toml + + - name: Upload the target report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: fc100-target-report + path: fc100-target-report.json + if-no-files-found: warn