feat(analyzer): Add golden-dataset evaluation harness with per-PR quality report

.github/workflows/ci.yml

@@ -204,6 +204,64 @@ jobs:
           pip install --require-hashes -r ${{ github.workspace }}/.github/pipelines/requirements-build.txt
           python -m build --wheel
 
+  analyzer-evaluation:
+    name: Analyzer Evaluation Report
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      UV_CACHE_DIR: /mnt/uv-cache
+      UV_PROJECT_ENVIRONMENT: /mnt/uv-venv
+      UV_LINK_MODE: hardlink
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.13'
+
+      - name: Install uv
+        run: python -m pip install uv==0.11.6
+
+      - name: Prepare uv directories on /mnt
+        run: |
+          sudo mkdir -p "$UV_CACHE_DIR" "$UV_PROJECT_ENVIRONMENT"
+          sudo chown -R "$(id -u):$(id -g)" "$UV_CACHE_DIR" "$UV_PROJECT_ENVIRONMENT"
+
+      - name: Cache uv downloads
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.0
+        with:
+          path: ${{ env.UV_CACHE_DIR }}
+          key: uv-eval-${{ runner.os }}-${{ hashFiles('presidio-analyzer/uv.lock') }}
+          restore-keys: |
+            uv-eval-${{ runner.os }}-
+
+      - name: Install dependencies
+        working-directory: presidio-analyzer
+        run: |
+          # The evaluation runs the default (spaCy) recipe only, so install the
+          # base analyzer plus the `evaluation` group (presidio-evaluator) and
+          # `dev` (pip, for spaCy's model download) — no `transformers`/`gliner`
+          # extras, which conflict with presidio-evaluator's transformers>=5.
+          # presidio-evaluator tokenizes with en_core_web_sm and the analyzer's
+          # default NER uses en_core_web_lg.
+          uv sync --locked --group dev --group evaluation --python 3.13
+          uv run --no-sync python -m spacy download en_core_web_sm
+          uv run --no-sync python -m spacy download en_core_web_lg
+
+      - name: Run golden-dataset evaluation
+        working-directory: presidio-analyzer
+        run: |
+          # Report-only for now: the presidio-evaluator report is published to
+          # the workflow summary; regression gating against baselines is a
+          # follow-up (the --fail-on-regression flag exists but is not passed).
+          uv run --no-sync python -m tests.evaluation.run_evaluation \
+            --output evaluation_report.md
+          cat evaluation_report.md >> "$GITHUB_STEP_SUMMARY"
+
   build-platform-images:
     name: Build ${{ matrix.image }} (${{ matrix.platform }})
     runs-on: ${{ matrix.runner }}

presidio-analyzer/pyproject.toml

@@ -84,6 +84,31 @@ dev = [
     "pre-commit",
     "diff-cover",
 ]
+# Detection-quality evaluation harness (tests/evaluation). Kept in its own
+# group — not the `dev` group and not an extra — so the general test matrix
+# does not pull presidio-evaluator's heavy deps into every Python cell. Only
+# the dedicated CI evaluation job installs it (`--group evaluation`).
+# presidio-evaluator supports Python 3.11-3.13; the marker keeps `uv lock`
+# resolvable across the analyzer's wider 3.10-3.14 range.
+evaluation = [
+    "presidio-evaluator (>=0.3,<0.4) ; python_version >= '3.11' and python_version < '3.14'",
+]
+
+[tool.uv]
+# presidio-evaluator (evaluation group) requires transformers>=5, while the
+# `transformers` extra pins transformers<5. They are never installed together
+# — the evaluation CI job uses the default spaCy recipe — so declare them
+# mutually exclusive and let uv resolve them in separate forks.
+conflicts = [
+    [
+        { extra = "transformers" },
+        { group = "evaluation" },
+    ],
+    [
+        { extra = "gliner" },
+        { group = "evaluation" },
+    ],
+]
 
 [tool.coverage.run]
 relative_files = true

presidio-analyzer/tests/evaluation/README.md

@@ -0,0 +1,159 @@
+# Analyzer evaluation harness
+
+A CI harness that measures analyzer detection quality (per-entity
+precision / recall / F2) and latency against a curated, hand-annotated golden
+dataset, using the org's own
+[`presidio-evaluator`](https://github.com/data-privacy-stack/presidio-research)
+package as the scoring engine. It addresses
+[#1639](https://github.com/data-privacy-stack/presidio/issues/1639) (evaluate
+precision/recall/latency during CI) and
+[#1810](https://github.com/data-privacy-stack/presidio/issues/1810) (curated
+PII/PHI benchmark dataset), and lays the groundwork for the recipes comparison
+in [#1809](https://github.com/data-privacy-stack/presidio/issues/1809).
+
+## Installing
+
+`presidio-evaluator` is in the `evaluation` dependency group (not installed by
+the default test matrix). From the `presidio-analyzer` directory:
+
+```sh
+uv sync --group dev --group evaluation
+uv run python -m spacy download en_core_web_sm   # tokenization
+uv run python -m spacy download en_core_web_lg   # default NER
+```
+
+## Running it
+
+```sh
+python -m tests.evaluation.run_evaluation            # report to stdout
+python -m tests.evaluation.run_evaluation --output report.md
+python -m tests.evaluation.run_evaluation --iou 0.75 # stricter span overlap
+
+# evaluate another analyzer configuration (e.g. transformers):
+python -m tests.evaluation.run_evaluation \
+    --analyzer-conf path/to/analyzer_conf.yaml \
+    --write-baseline tests/evaluation/baselines/my_config.json
+
+# enforce the baseline locally (CI does not do this yet):
+python -m tests.evaluation.run_evaluation --fail-on-regression
+```
+
+The end-to-end smoke run also executes under `pytest tests/evaluation` when the
+`evaluation` group is installed (it is skipped otherwise), and a CI job
+publishes the report to the workflow step summary on every PR.
+
+## Design decisions
+
+**Scoring engine is `presidio-evaluator`.** Rather than a bespoke evaluator,
+scoring uses `presidio-evaluator`'s span-based `SpanEvaluator` (character IoU)
+wrapped around `PresidioAnalyzerWrapper`, with F-beta = 2 (recall-weighted),
+matching Presidio's documented evaluation convention. `presidio-evaluator` is
+the org's own sibling package (`data-privacy-stack/presidio-research`), already
+referenced by `docs/evaluation/index.md`, so this reuses the standard data
+model (`InputSample`/`Span`), metrics and error analysis instead of reinventing
+them. This harness is the thin CI layer on top: dataset, baseline ratchet,
+markdown report and gating flags.
+
+**The `CanonicalMapper` step is skipped.** `presidio-evaluator`'s entity
+hierarchy/mapper reconciles differing label spaces between a model and a
+dataset. Our golden dataset is annotated with the analyzer's own entity names,
+so predictions and annotations already share a label space — mapping is
+unnecessary, and skipping it keeps the report keyed by raw Presidio entity
+types and the run deterministic (no interactive resolution).
+
+**Enforcement is built in but switched off.** The report compares every run
+against the checked-in baseline (`baselines/spacy_en.json`) and shows
+per-entity F2 deltas; `--fail-on-regression` exits non-zero when overall or
+per-entity F2 drops more than `--f2-tolerance` (default 0.02) below the
+baseline. CI runs report-only: switching enforcement on is a one-line CI
+change, deliberately left as a maintainer decision once the numbers have proven
+stable across real PRs (a new spaCy model release can shift NER results with no
+code change).
+
+**Updating the baseline** is part of the PR that changes behavior:
+`python -m tests.evaluation.run_evaluation --write-baseline
+tests/evaluation/baselines/spacy_en.json`, committed alongside the code, so
+reviewers see the metric change explicitly in the diff.
+
+**Curated golden set now, synthetic generation later.** A checked-in,
+hand-annotated dataset is deterministic, reviewable in diffs, and cheap to run
+per-PR. The template + Faker synthetic generation described in #1639 (available
+in `presidio-evaluator` as `PresidioSentenceFaker`) is the right tool for
+scaling coverage with every new recognizer, and is planned as a follow-up
+phase — its output is `InputSample`s, the same type this harness already
+consumes.
+
+**Evaluated entities are a fixed allowlist.** Predictions for entity types not
+declared in the dataset (`entities` in `golden_en.json`) are excluded via
+`entities_to_keep`, so recognizers without golden coverage don't pollute the
+report with unreviewable false positives. Adding coverage for a new entity =
+adding annotated samples + the entity to the allowlist.
+
+**Per-PR CI evaluates the default configuration only.** That covers spaCy NER
+(`PERSON`, `LOCATION`, `DATE_TIME`) plus the predefined pattern/checksum
+recognizers. HuggingFace, GLiNER and transformers recognizers are optional
+extras with large model downloads — and their `transformers>=?` pins conflict
+with `presidio-evaluator`'s, so they are declared mutually exclusive in
+`[tool.uv] conflicts` and not run per-PR. The runner already supports them via
+`--analyzer-conf <yaml>` with a separate baseline per configuration; wiring
+them into a scheduled nightly job is roadmap step 4.
+
+## Dataset
+
+`datasets/golden_en.json` — 46 English samples, 93 annotated spans across 12
+entity types (`PERSON`, `LOCATION`, `DATE_TIME`, `EMAIL_ADDRESS`,
+`PHONE_NUMBER`, `CREDIT_CARD`, `US_SSN`, `IP_ADDRESS`, `URL`, `IBAN_CODE`,
+`CRYPTO`, `UK_NHS`), organized in categories per #1810:
+
+- `simple` — single-entity one-liners
+- `medium` — multi-entity texts (support tickets, clinical notes, logs)
+- `long` — full documents (discharge summary, incident report, KYC)
+- `edge` — lowercase/inverted names, hyphenated names, adjacent entities,
+  entities at text boundaries, inline IDs
+- `negative` — no PII, but tempting lookalikes (version numbers, room
+  numbers, quantities)
+
+Entity values are checksum-valid where recognizers validate (Luhn for cards,
+mod-97 for IBANs, NHS check digit, SSN excluded ranges), reusing values from
+the unit tests where possible. Span offsets are validated by
+`test_golden_dataset.py::TestDatasetIntegrity`, so every annotation is
+guaranteed to match its text slice.
+
+> Note: per-entity `support` in the report is computed by `presidio-evaluator`
+> from its token/span reconstruction and may differ slightly from the raw gold
+> span count.
+
+### Extending the dataset
+
+`generate_golden_en.py` is the source of truth — samples are defined as
+interleaved text parts and `(value, entity_type)` tuples, so offsets are
+computed rather than hand-counted. To add coverage (e.g. for a new recognizer):
+
+1. Add samples to `SAMPLES` in `generate_golden_en.py` (and the entity type
+   to `ENTITIES` if new).
+2. Regenerate: `python -m tests.evaluation.generate_golden_en`
+3. Regenerate the baseline: `... run_evaluation --write-baseline
+   tests/evaluation/baselines/spacy_en.json`
+4. Commit all three; a sync test fails if the JSON drifts from the generator,
+   so the JSON can never be hand-edited.
+
+### Other languages
+
+The dataset format is language-agnostic: one file per language
+(`golden_en.json` today, e.g. `golden_de.json` later), each declaring its
+`language` and evaluated entity list. What is still English-only is the runner,
+which builds a default `AnalyzerEngine`; another language means adding a
+per-language engine configuration and installing its model in the CI job.
+Planned alongside the nightly matrix (roadmap step 4).
+
+## Roadmap
+
+1. **(this PR)** `presidio-evaluator`-based harness + golden dataset +
+   baselines and regression detection (enforcement off) + report-only CI job.
+2. Switch `--fail-on-regression` on in CI once metrics have proven stable —
+   a one-line CI change plus a baseline-update note in CONTRIBUTING.
+3. Synthetic data generation via `PresidioSentenceFaker` (templates + Faker
+   providers); contributing a recognizer requires contributing templates (#1639).
+4. Nightly multi-configuration matrix (spaCy / transformers / GLiNER / LLM)
+   with per-configuration baselines and non-English datasets, feeding the
+   fast/balanced/accurate recipes comparison (#1809).

presidio-analyzer/tests/evaluation/__init__.py

@@ -0,0 +1,24 @@
+"""Analyzer detection-quality evaluation for CI.
+
+A thin wrapper around ``presidio-evaluator`` that scores the analyzer against
+a curated golden dataset and produces a per-entity precision/recall/F2 report
+plus baseline regression comparison. See tests/evaluation/README.md.
+"""
+
+from tests.evaluation.evaluation import (
+    EntityScore,
+    EvaluationReport,
+    Mismatch,
+    default_dataset_path,
+    load_input_samples,
+    run_evaluation,
+)
+
+__all__ = [
+    "EntityScore",
+    "EvaluationReport",
+    "Mismatch",
+    "default_dataset_path",
+    "load_input_samples",
+    "run_evaluation",
+]

presidio-analyzer/tests/evaluation/baseline.py

@@ -0,0 +1,97 @@
+"""Baseline persistence and regression comparison for evaluation reports.
+
+A baseline is a checked-in snapshot of per-entity F2 metrics for one analyzer
+configuration. Comparing a fresh report against it turns absolute numbers into
+deltas and, when enforcement is switched on via ``--fail-on-regression``, lets
+CI fail when F2 drops more than a tolerance below the baseline.
+
+Throughput is deliberately not part of the baseline: it depends on the machine
+running the evaluation and would make regressions non-reproducible.
+"""
+
+import json
+from pathlib import Path
+from typing import Dict, List
+
+from tests.evaluation.evaluation import EvaluationReport
+
+DEFAULT_F2_TOLERANCE = 0.02
+
+
+def default_baseline_path() -> Path:
+    """Path of the checked-in baseline for the default (spaCy) config."""
+    return Path(__file__).parent / "baselines" / "spacy_en.json"
+
+
+def report_to_baseline(report: EvaluationReport, config_name: str) -> Dict:
+    """Snapshot an evaluation report as a baseline dict."""
+    return {
+        "config": config_name,
+        "overall": {
+            "precision": round(report.overall_precision, 4),
+            "recall": round(report.overall_recall, 4),
+            "f2": round(report.overall_f2, 4),
+        },
+        "per_entity": {
+            entity: {
+                "support": score.support,
+                "precision": round(score.precision, 4),
+                "recall": round(score.recall, 4),
+                "f2": round(score.f2, 4),
+            }
+            for entity, score in sorted(report.per_entity.items())
+        },
+    }
+
+
+def save_baseline(report: EvaluationReport, config_name: str, path: Path) -> None:
+    """Write a baseline snapshot to disk."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    baseline = report_to_baseline(report, config_name)
+    path.write_text(json.dumps(baseline, indent=2) + "\n", encoding="utf-8")
+
+
+def load_baseline(path: Path) -> Dict:
+    """Load a baseline snapshot from disk."""
+    with open(path, encoding="utf-8") as f:
+        return json.load(f)
+
+
+def compare_to_baseline(
+    report: EvaluationReport,
+    baseline: Dict,
+    f2_tolerance: float = DEFAULT_F2_TOLERANCE,
+) -> List[str]:
+    """Compare a report against a baseline and describe regressions.
+
+    :param report: Fresh evaluation report.
+    :param baseline: Baseline dict, as produced by :func:`report_to_baseline`.
+    :param f2_tolerance: Maximum allowed F2 drop before it counts as a
+        regression, both overall and per entity type.
+    :return: Human-readable regression descriptions; empty when the report
+        is within tolerance. Entities absent from the baseline are skipped
+        (they are new coverage, not regressions).
+    """
+    regressions = []
+
+    overall_drop = baseline["overall"]["f2"] - report.overall_f2
+    if overall_drop > f2_tolerance:
+        regressions.append(
+            f"overall F2 dropped {overall_drop:.3f} "
+            f"(baseline {baseline['overall']['f2']:.3f}, "
+            f"current {report.overall_f2:.3f}, tolerance {f2_tolerance})"
+        )
+
+    for entity, score in sorted(report.per_entity.items()):
+        baseline_entity = baseline["per_entity"].get(entity)
+        if baseline_entity is None:
+            continue
+        drop = baseline_entity["f2"] - score.f2
+        if drop > f2_tolerance:
+            regressions.append(
+                f"{entity} F2 dropped {drop:.3f} "
+                f"(baseline {baseline_entity['f2']:.3f}, "
+                f"current {score.f2:.3f}, tolerance {f2_tolerance})"
+            )
+
+    return regressions