diff --git a/README.md b/README.md index 125973e..d8c2a15 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,31 @@ paper-format-agent-mcp Tools: `format_paper`, `extract_format_rules`, `score_paper`. Client config in [docs/MCP.md](docs/MCP.md). +## regression manifest + +[`docs/regression_manifest.sample.json`](docs/regression_manifest.sample.json) +declares synthetic end-to-end checks for the formatter. Each case names a fake +fixture, optional strict flags, and expected score thresholds. It is a sample +only: it references fixture names that you create locally, never real papers or +private templates. The `note` field explains each case and is ignored by the +runner. + +To run the regression suite against fixtures you have created: + +```bash +python tools/regression_runner.py \ + --manifest docs/regression_manifest.sample.json \ + --out-dir ./regression_out +``` + +Expected output: a `regression_summary.json` in `--out-dir` with one result per +case (`ok`, `score_before`, `score_after`, `content_changed`, `failed_checks`). +A case passes when it exits 0, the engine reported success, content stayed +unchanged (unless `allow_content_change`), and the score thresholds hold. The +command exits 0 only if every case passes. You must validate the JSON yourself +(`python -c "import json; json.load(open('docs/regression_manifest.sample.json'))"`) +because the sample contains documentation notes, not code comments. + ## contributing Small PRs are welcome. Adding a synthetic test for a school or journal diff --git a/docs/regression_manifest.sample.json b/docs/regression_manifest.sample.json index 7eea5b8..a597332 100644 --- a/docs/regression_manifest.sample.json +++ b/docs/regression_manifest.sample.json @@ -1,4 +1,5 @@ { + "_about": "Synthetic regression manifest for Paper Format Agent. Each entry points at fake fixture names only; no real papers or private templates. The 'note' field is informational and ignored by tools/regression_runner.py. Replace fixture paths with your own synthetic .docx files before running.", "cases": [ { "name": "synthetic_thesis_case", @@ -9,7 +10,55 @@ "require_content_unchanged": true, "require_engine_success": true, "min_score_after": 90, - "min_score_improvement": 3 + "min_score_improvement": 3, + "note": "Baseline synthetic thesis: formatting must finish on the python engine, keep the body text unchanged, and lift the score by at least 3 points to stay at or above 90." + }, + { + "name": "synthetic_journal_strict_sections", + "format_file": "examples/fixtures/synthetic_journal_format_guide.docx", + "paper_file": "examples/fixtures/synthetic_journal_paper.docx", + "engine": "python", + "strict_required_sections": true, + "require_content_unchanged": true, + "require_engine_success": true, + "min_score_after": 85, + "min_score_improvement": 2, + "note": "Journal-style case with strict required-section enforcement (abstract, keywords, headings). Expected output: required sections present, score_after >= 85, score_improvement >= 2, content_changed == false." + }, + { + "name": "synthetic_ieee_conference_case", + "format_file": "examples/fixtures/synthetic_ieee_format_guide.docx", + "paper_file": "examples/fixtures/synthetic_ieee_paper.docx", + "engine": "python", + "strict_required_sections": true, + "require_content_unchanged": true, + "require_engine_success": true, + "min_score_after": 90, + "min_score_improvement": 5, + "note": "IEEE-style conference: tight formatting rules. Expected output: score_after >= 90, score jumps by at least 5 points while leaving prose untouched." + }, + { + "name": "synthetic_toc_refresh_case", + "format_file": "examples/fixtures/synthetic_format_guide.docx", + "paper_file": "examples/fixtures/synthetic_toc_paper.docx", + "engine": "word-com", + "strict_required_sections": false, + "allow_content_change": true, + "require_content_unchanged": false, + "require_engine_success": true, + "min_score_after": 80, + "note": "Shows an opt-in case: the word-com engine refreshes the table of contents, so content_changed may be true and require_content_unchanged must be disabled. Only the score_after floor is enforced." + }, + { + "name": "synthetic_minimal_reliability_case", + "format_file": "examples/fixtures/synthetic_format_guide.docx", + "paper_file": "examples/fixtures/synthetic_paper.docx", + "engine": "python", + "strict_required_sections": false, + "require_content_unchanged": false, + "require_engine_success": true, + "min_score_after": 60, + "note": "Minimal reliability guard: only requires the engine to succeed and produce a report with score_after >= 60. No content/improvement constraints." } ] -} +} \ No newline at end of file