test(notebooks): run the demo notebook end to end on both backends

.github/workflows/notebook-execution.yml

@@ -0,0 +1,148 @@
+name: Nightly notebook execution check
+
+# Runs every cell of the demo notebook against both backends, on the
+# checked-out TabPFN rather than the released one, and checks the headline
+# metrics it prints still clear their floors.
+#
+# The notebook installs its own dependencies. Seeding the venv with an
+# editable install of this checkout first is what keeps them: `uv pip install
+# tabpfn` audits an already-satisfied requirement instead of pulling PyPI's
+# wheel over the top.
+#
+# Runs daily on a cron, on manual dispatch, and on pull requests that touch
+# the notebook or this workflow. Scheduled failures open or comment on a
+# tracking issue, since a cron failure notifies nobody by default.
+
+on:
+  schedule:
+    - cron: 41 5 * * *   # 05:41 UTC daily, before the install check
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - examples/notebooks/TabPFN_Demo_Local.ipynb
+      - tests/test_notebook_execution.py
+      - .github/workflows/notebook-execution.yml
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  execute:
+    name: Run the demo notebook (${{ matrix.backend }} backend)
+    # Forks cannot read TABPFN_TOKEN, and every cell needs it.
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-22.04-4core-gpu
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        backend: [local, client]
+
+    env:
+      TABPFN_TOKEN: ${{ secrets.TABPFN_TOKEN }}
+      TABPFN_DEMO_BACKEND: ${{ matrix.backend }}
+      RUN_NOTEBOOK_EXECUTION_CHECK: "1"
+      # No display on the runner, so every plotting cell needs a headless
+      # matplotlib backend.
+      MPLBACKEND: Agg
+      HF_HUB_DISABLE_XET: "1"
+      VIRTUAL_ENV: ${{ github.workspace }}/.notebook-venv
+      NOTEBOOK_EXECUTION_OUT: ${{ github.workspace }}/executed.ipynb
+
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4.3.1
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7.3.0
+        with:
+          enable-cache: true
+
+      # uv provides the interpreter rather than `actions/setup-python`. Some
+      # of the notebook's dependencies build from source, and a uv-managed
+      # CPython keeps its headers and its pkg-config prefix in the same tree;
+      # the tool-cache Python on this runner image reports an include path
+      # that is not where it was installed.
+      - name: Install Python
+        run: uv python install 3.12
+
+      # The notebook's own `!uv pip install` cells write into $VIRTUAL_ENV, so
+      # the kernel and those cells must share one environment.
+      - name: Create the notebook environment
+        run: |
+          uv venv --python 3.12 "$VIRTUAL_ENV"
+          uv pip install -e .
+          uv pip install ipykernel nbclient nbformat pytest
+          # Cells that shell out to an installed command (`!gdown`) resolve it
+          # on PATH, which is how it works in Colab.
+          echo "$VIRTUAL_ENV/bin" >> "$GITHUB_PATH"
+
+      - name: Run the notebook
+        shell: bash
+        run: |
+          set -o pipefail
+          "$VIRTUAL_ENV/bin/python" -m pytest \
+            --noconftest tests/test_notebook_execution.py -v 2>&1 \
+            | tee pytest_output.txt
+
+      - name: Upload the executed notebook
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: executed-notebook-${{ matrix.backend }}
+          path: executed.ipynb
+          if-no-files-found: ignore
+
+      - name: Open or update tracking issue on failure
+        if: failure() && github.event_name == 'schedule'
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        env:
+          BACKEND: ${{ matrix.backend }}
+        with:
+          script: |
+            // Both matrix cells may file concurrently. The worst case is a
+            // duplicate issue on the same nightly — acceptable, the
+            // maintainer closes one.
+            const fs = require("fs");
+            let failedLines = "(no FAILED lines parsed; see logs)";
+            try {
+              const text = fs.readFileSync("pytest_output.txt", "utf-8");
+              const lines = text.split("\n").filter(l => l.startsWith("FAILED "));
+              if (lines.length > 0) failedLines = lines.join("\n");
+            } catch (e) { /* output file missing — fall through to default */ }
+
+            // Per-backend title so each backend owns its own tracking issue
+            // and can be closed independently when fixed.
+            const title =
+              `Nightly: demo notebook failed (${process.env.BACKEND} backend)`;
+            const { data: issues } = await github.rest.issues.listForRepo({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              state: "open",
+              labels: ["nightly-failure"],
+            });
+            const runUrl =
+              `${context.serverUrl}/${context.repo.owner}/` +
+              `${context.repo.repo}/actions/runs/${context.runId}`;
+            const body =
+              `Run: ${runUrl}\n` +
+              `Backend: ${process.env.BACKEND}\n\n` +
+              `\`\`\`\n${failedLines}\n\`\`\`\n\n` +
+              `The executed notebook is attached to the run as an artifact.`;
+            const existing = issues.find(i => i.title === title);
+            if (existing) {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: existing.number,
+                body,
+              });
+            } else {
+              await github.rest.issues.create({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                title,
+                body,
+                labels: ["nightly-failure"],
+              });
+            }

changelog/1214.fixed.md

@@ -0,0 +1 @@
+Set `TABPFN_DEMO_BACKEND` to `local` or `client` to pick the demo notebook's backend without being prompted, and let its local setup use a `TABPFN_TOKEN` already present in the environment instead of asking for one.

examples/notebooks/TabPFN_Demo_Local.ipynb

@@ -129,6 +129,9 @@
     "# Install rich for better and more readable printing\n",
     "!uv pip install rich\n",
     "\n",
+    "# Install gdown to fetch the text dataset used later in the notebook\n",
+    "!uv pip install gdown\n",
+    "\n",
     "## TabPFN Installation optimized for Google Colab\n",
     "# Install the TabPFN Client library\n",
     "!uv pip install tabpfn-client\n",
@@ -264,7 +267,8 @@
     "If you select **local** and it's your first time using the model, you'll need to accept the license agreement and log into Hugging Face.\n",
     "You'll be prompted with instructions on how to do this the first time you fit the model.\n",
     "\n",
-    "For demonstration purposes, the cell below provides an interactive way to switch between local mode and the client:\n"
+    "For demonstration purposes, the cell below provides an interactive way to switch between local mode and the client.\n",
+    "Set the `TABPFN_DEMO_BACKEND` environment variable to `local` or `client` to choose without being asked.\n"
    ]
   },
   {
@@ -522,6 +526,8 @@
    "source": [
     "# Authentication code - No need to read it!\n",
     "\n",
+    "import os\n",
+    "\n",
     "console = Console()\n",
     "\n",
     "console.print(Panel.fit(\"[bold magenta]TabPFN Demo: Backend Selection[/bold magenta]\"))\n",
@@ -531,7 +537,12 @@
     "    \"  [bold]client:[/bold] Uses the TabPFN API. Requires an internet connection and a free account.\"\n",
     ")\n",
     "\n",
-    "backend = None\n",
+    "# Set TABPFN_DEMO_BACKEND to \"local\" or \"client\" to pick the backend without\n",
+    "# being asked. Anything else falls through to the prompt below.\n",
+    "backend = os.environ.get(\"TABPFN_DEMO_BACKEND\")\n",
+    "if backend not in [\"local\", \"client\"]:\n",
+    "  backend = None\n",
+    "\n",
     "while backend is None:\n",
     "  console.print(\n",
     "      \"\\n[bold]Choose your backend[/bold]: - If no text box is shown, restart the cell.\",\n",
@@ -573,19 +584,23 @@
     "    import os\n",
     "    import getpass\n",
     "\n",
-    "    tabpfn_token = None\n",
+    "    # 1. A token already in the environment wins\n",
+    "    tabpfn_token = os.environ.get(\"TABPFN_TOKEN\")\n",
+    "    if tabpfn_token:\n",
+    "        console.print(\"[bold green]\u2705 Found TABPFN_TOKEN in the environment.[/bold green]\")\n",
     "\n",
-    "    # 1. Try Colab secret TABPFN_TOKEN\n",
-    "    try:\n",
-    "        from google.colab import userdata\n",
-    "        tabpfn_token = userdata.get(\"TABPFN_TOKEN\")\n",
-    "        if tabpfn_token:\n",
-    "            os.environ[\"TABPFN_TOKEN\"] = tabpfn_token\n",
-    "            console.print(\"[bold green]\u2705 Found TABPFN_TOKEN in Colab secrets.[/bold green]\")\n",
-    "    except Exception:\n",
-    "        pass\n",
+    "    # 2. Try Colab secret TABPFN_TOKEN\n",
+    "    if not tabpfn_token:\n",
+    "        try:\n",
+    "            from google.colab import userdata\n",
+    "            tabpfn_token = userdata.get(\"TABPFN_TOKEN\")\n",
+    "            if tabpfn_token:\n",
+    "                os.environ[\"TABPFN_TOKEN\"] = tabpfn_token\n",
+    "                console.print(\"[bold green]\u2705 Found TABPFN_TOKEN in Colab secrets.[/bold green]\")\n",
+    "        except Exception:\n",
+    "            pass\n",
     "\n",
-    "    # 2. If no token found, prompt the user\n",
+    "    # 3. If no token found, prompt the user\n",
     "    if not tabpfn_token:\n",
     "        console.print(\n",
     "            Panel(\n",
@@ -2511,7 +2526,7 @@
     "    )\n",
     "else:\n",
     "    # We will use gdown to download the dataset\n",
-    "    !gdown --fuzzy \"17bJekFGIAxbrdcBeBIvcyZjneweY581E\"\n",
+    "    !gdown -O cloth.csv \"17bJekFGIAxbrdcBeBIvcyZjneweY581E\"\n",
     "\n",
     "    # Load the clothing review dataset\n",
     "    # We restrict to 500 rows to make the example faster\n",

tests/test_notebook_execution.py

@@ -0,0 +1,114 @@
+#  Copyright (c) Prior Labs GmbH 2026.
+
+"""Execute the demo notebook end to end against one of the TabPFN backends.
+
+Runs every cell of `examples/notebooks/TabPFN_Demo_Local.ipynb` in a real
+Jupyter kernel with `TABPFN_DEMO_BACKEND` set, so a cell that raises fails the
+test, then checks that the headline metrics the notebook prints still clear
+their floors.
+
+Skipped by default. Set `RUN_NOTEBOOK_EXECUTION_CHECK=1` to enable and
+`TABPFN_DEMO_BACKEND` to `local` or `client` to pick the backend. Intended to
+run from `.github/workflows/notebook-execution.yml`, not from the regular PR
+test matrix: a full pass installs several GB of packages, downloads public
+datasets and takes tens of minutes on a GPU.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from pathlib import Path
+
+import pytest
+
+if not os.environ.get("RUN_NOTEBOOK_EXECUTION_CHECK"):
+    pytest.skip(
+        "set RUN_NOTEBOOK_EXECUTION_CHECK=1 to enable (intended for nightly CI)",
+        allow_module_level=True,
+    )
+
+NOTEBOOK = (
+    Path(__file__).parents[1] / "examples" / "notebooks" / "TabPFN_Demo_Local.ipynb"
+)
+
+BACKEND = os.environ.get("TABPFN_DEMO_BACKEND", "")
+
+# Per-cell wall clock. The slowest cells fit a model per cross-validation fold
+# for four estimators, so this is generous by design; the workflow's job
+# timeout is what bounds a hung run.
+CELL_TIMEOUT_S = 60 * 30
+
+# Floors on the metrics the notebook prints, as
+# `label -> (regex over cell output, lowest acceptable value)`. They sit far
+# enough below the observed values to absorb the run-to-run spread of an
+# unseeded server-side ensemble, and still catch a backend that has stopped
+# learning. Metrics where lower is better are negated before comparison.
+METRIC_FLOORS: dict[str, tuple[str, float]] = {
+    "parkinsons roc auc": (r"TabPFN ROC AUC Score: ([0-9.]+)", 0.85),
+    "boston neg rmse": (r"TabPFN RMSE: (-?[0-9.]+)", -4.5),
+}
+NEGATED_METRICS = {"boston neg rmse"}
+
+
+def _cell_text(notebook: dict) -> str:
+    """Return every stream and text/plain output in the notebook, concatenated."""
+    chunks = []
+    for cell in notebook["cells"]:
+        for output in cell.get("outputs", []):
+            if output.get("output_type") == "stream":
+                chunks.append("".join(output.get("text", [])))
+            data = output.get("data", {})
+            if "text/plain" in data:
+                chunks.append("".join(data["text/plain"]))
+    return "\n".join(chunks)
+
+
+@pytest.fixture(scope="module")
+def executed_notebook(tmp_path_factory: pytest.TempPathFactory) -> dict:
+    """Run the notebook in a fresh working directory and return it."""
+    nbformat = pytest.importorskip("nbformat")
+    nbclient = pytest.importorskip("nbclient")
+
+    if BACKEND not in {"local", "client"}:
+        pytest.fail(f"TABPFN_DEMO_BACKEND must be 'local' or 'client', got {BACKEND!r}")
+
+    workdir = tmp_path_factory.mktemp("notebook-run")
+    notebook = nbformat.read(NOTEBOOK, as_version=4)
+    nbclient.NotebookClient(
+        notebook,
+        timeout=CELL_TIMEOUT_S,
+        kernel_name="python3",
+        resources={"metadata": {"path": str(workdir)}},
+    ).execute()
+
+    # Keep the executed copy so a CI failure can be inspected from the run's
+    # artifacts rather than only from the traceback.
+    out = Path(os.environ.get("NOTEBOOK_EXECUTION_OUT", workdir / "executed.ipynb"))
+    out.parent.mkdir(parents=True, exist_ok=True)
+    nbformat.write(notebook, out)
+    return notebook
+
+
+def test_notebook_runs_end_to_end(executed_notebook: dict) -> None:
+    """Every code cell produced output; none raised."""
+    errors = [
+        output
+        for cell in executed_notebook["cells"]
+        for output in cell.get("outputs", [])
+        if output.get("output_type") == "error"
+    ]
+    assert not errors, json.dumps(errors, indent=2)[:4000]
+
+
+@pytest.mark.parametrize("metric", sorted(METRIC_FLOORS))
+def test_headline_metric_did_not_regress(executed_notebook: dict, metric: str) -> None:
+    pattern, floor = METRIC_FLOORS[metric]
+    text = _cell_text(executed_notebook)
+    match = re.search(pattern, text)
+    assert match, f"{metric}: no cell output matched {pattern!r}"
+    value = float(match.group(1))
+    if metric in NEGATED_METRICS:
+        value = -value
+    assert value >= floor, f"{metric}: {value} fell below {floor}"