Expand package installation and prediction validation

.github/workflows/ci.yml

@@ -47,3 +47,19 @@ jobs:
         run: uv build --all-packages
       - name: Verify required package data
         run: python workflows/verify_distributions.py
+
+  install:
+    name: Isolated wheel installations
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.12"
+      - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+        with:
+          enable-cache: true
+      - name: Build wheels and test each installation path
+        env:
+          OMP_NUM_THREADS: "1"
+        run: python workflows/check_package_split.py --output "$RUNNER_TEMP/package-split"

packages/tabpfn-rel/README.md

@@ -103,4 +103,5 @@ contains ordinary version requirements. The candidate distributions must be
 released before the index-only installation commands above are available.
 
 The tests exercise feature and context behavior without downloading model weights
-or making hosted API requests.
+or making hosted API requests. Integration tests use real DFS and a small test
+estimator to cover the predictive interface and temporal tuning.

packages/tabpfn-rel/tests/test_integration.py

@@ -0,0 +1,126 @@
+"""Real DFS, temporal tuning and prediction with an in-memory estimator backend."""
+
+from __future__ import annotations
+
+import importlib.util
+from collections.abc import Callable
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import pytest
+from relarena_core.tfm import TFMSpec
+
+from tabpfn_rel import PredictiveQuery, PredictiveQuerySpec, tfm
+
+
+class _Estimator:
+    def fit(self, X: pd.DataFrame, y: np.ndarray) -> _Estimator:
+        assert len(X) == len(y) > 0
+        assert X.shape[1] > 0
+        self.classes_ = np.unique(y)
+        self.mean_ = float(np.mean(y))
+        self.columns_ = list(X.columns)
+        return self
+
+    def predict_proba(self, X: pd.DataFrame) -> np.ndarray:
+        return np.full((len(X), len(self.classes_)), 1 / len(self.classes_))
+
+    def predict(self, X: pd.DataFrame) -> np.ndarray:
+        return np.full(len(X), self.mean_)
+
+
+def _make_estimator(**kwargs: object) -> _Estimator:
+    return _Estimator()
+
+
+_EXAMPLE = Path(__file__).resolve().parents[1] / "examples" / "tiny_database.py"
+
+
+@pytest.fixture(scope="session")
+def write_database() -> Callable[..., Path]:
+    """The generated-database writer from the example script, loaded by path."""
+    spec = importlib.util.spec_from_file_location("tiny_database", _EXAMPLE)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module.write_database
+
+
+@pytest.fixture
+def query(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    request: pytest.FixtureRequest,
+    write_database: Callable[..., Path],
+) -> PredictiveQuery:
+    for name in ("tabpfn-v3", "tabpfn-v3-api"):
+        monkeypatch.setitem(
+            tfm.TFM_REGISTRY,
+            name,
+            TFMSpec(_make_estimator, _make_estimator, 100_000, supports_text=True),
+        )
+    task_path = write_database(
+        tmp_path, getattr(request, "param", "binary_classification")
+    )
+    spec = PredictiveQuerySpec.from_yaml(str(task_path), data_dir=str(tmp_path))
+    return PredictiveQuery(spec, data_version="test-v1")
+
+
+@pytest.mark.parametrize(
+    "query", ["binary_classification", "regression"], indirect=True
+)
+@pytest.mark.parametrize("backend", ["local", "client"])
+@pytest.mark.parametrize("n_trials", [0, 2])
+def test_rpi_fits_tunes_and_reuses_prediction_cache(
+    query: PredictiveQuery, backend: str, n_trials: int, tmp_path: Path
+) -> None:
+    query.fit(f"tabpfn-rel-{backend}", n_trials=n_trials, cache_dir=tmp_path / "cache")
+    predictions = query.predict()
+    pd.testing.assert_frame_equal(predictions, query.predict())
+    assert sorted(predictions["customer_id"]) == ["a", "b", "c", "d"]
+    assert predictions["y_pred"].notna().all()
+    assert len(query.compute_test_labels()) == 4
+    assert query.config["max_depth"] in (2, 3)
+    if n_trials:
+        assert len(query.trials) == 2
+        assert all(trial.val_score is not None for trial in query.trials)
+    else:
+        assert query.trials is None
+    if backend == "client":
+        assert "description__raw_text" in query._model._fitted.estimator.columns_
+    assert list((tmp_path / "cache").rglob("*.parquet"))
+
+
+def test_cli_benchmarks_discovered_model(
+    query: PredictiveQuery, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    cli = pytest.importorskip("relarena.cli")
+    runner = pytest.importorskip("relarena.runner")
+    from relarena.tasks import TaskSpec
+
+    source = query._source
+    monkeypatch.setattr(runner, "RelBenchDatasetTask", lambda *args, **kwargs: source)
+    monkeypatch.setattr(
+        cli,
+        "list_entity_tasks",
+        lambda datasets: [TaskSpec("tiny", "customers", query.task.task_type)],
+    )
+    output = tmp_path / "results.csv"
+    assert (
+        cli.main(
+            [
+                "--model",
+                "tabpfn-rel-local",
+                "--datasets",
+                "tiny",
+                "--n-trials",
+                "1",
+                "--output",
+                str(output),
+            ]
+        )
+        == 0
+    )
+    results = pd.read_csv(output)
+    assert results["test_score"].notna().any()

packages/tabpfn-rel/workflows/check_backends.py

@@ -0,0 +1,54 @@
+"""Check installed estimator constructors without fitting or making API requests."""
+
+from __future__ import annotations
+
+import argparse
+from importlib.metadata import version
+
+import numpy as np
+
+from tabpfn_rel.tfm import TFM_REGISTRY
+
+
+def main() -> None:
+    """Construct classification and regression estimators with context overrides."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("backend", choices=["local", "api", "all"])
+    args = parser.parse_args()
+    for name, dependency in (
+        ("tabpfn-v3", "tabpfn"),
+        ("tabpfn-v3-api", "tabpfn-client"),
+    ):
+        if args.backend == "local" and name.endswith("api"):
+            continue
+        if args.backend == "api" and not name.endswith("api"):
+            continue
+        spec = TFM_REGISTRY[name]
+        for make in (spec.make_classifier, spec.make_regressor):
+            estimator = make(
+                device="cpu",
+                seed=7,
+                n_estimators=2,
+                inference_config={
+                    "SUBSAMPLE_SAMPLES": [np.array([0, 1]), np.array([1, 2])]
+                },
+            )
+            assert estimator.random_state == 7
+            assert estimator.n_estimators == 2
+            if name.endswith("api"):
+                assert estimator.model_path in {
+                    "v3_default",
+                    "tabpfn-v3-classifier-v3_default.ckpt",
+                    "tabpfn-v3-regressor-v3_default.ckpt",
+                }
+                assert estimator.inference_config["SUBSAMPLE_SAMPLES"] == [
+                    [0, 1],
+                    [1, 2],
+                ]
+        print(
+            f"{dependency} {version(dependency)}: classifier and regressor constructors passed"
+        )
+
+
+if __name__ == "__main__":
+    main()

packages/tabpfn-rel/workflows/check_inference.py

@@ -0,0 +1,80 @@
+"""Exercise real TabPFN inference over the generated relational example.
+
+From this checkout, run ``python -m workflows.check_inference --output PATH``.
+The local backend needs model weights. The client backend needs authentication
+and consumes service quota. Only generated customer/event data are used.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from importlib.metadata import version
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+from examples.tiny_database import write_database
+from tabpfn_rel import PredictiveQuery, PredictiveQuerySpec
+
+
+def main() -> None:
+    """Fit classification and regression models, then check warm-cache predictions and tuning."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--backend", choices=["local", "client"], default="local")
+    parser.add_argument("--n-trials", type=int, default=2)
+    args = parser.parse_args()
+    output = args.output.resolve()
+    output.mkdir(parents=True, exist_ok=True)
+    results = []
+    for task_type in ("binary_classification", "regression"):
+        directory = output / task_type
+        task_path = write_database(directory, task_type)
+        spec = PredictiveQuerySpec.from_yaml(str(task_path), data_dir=str(directory))
+        query = PredictiveQuery(spec, data_version="generated-v1").fit(
+            f"tabpfn-rel-{args.backend}",
+            n_trials=args.n_trials,
+            cache_dir=directory / "cache",
+        )
+        cold_predictions = query.predict()
+        predictions = query.predict()
+        repeated = query.predict()
+        assert sorted(predictions["customer_id"]) == ["a", "b", "c", "d"]
+        assert np.isfinite(predictions["y_pred"]).all()
+        pd.testing.assert_frame_equal(predictions, repeated, rtol=1e-5, atol=1e-7)
+        if task_type == "binary_classification":
+            assert predictions["y_pred"].between(0, 1).all()
+        labels = query.compute_test_labels()
+        assert len(labels) == len(predictions) == 4
+        assert list((directory / "cache").rglob("*.parquet"))
+        if args.n_trials:
+            assert query.trials and all(
+                t.ok and np.isfinite(t.val_score) for t in query.trials
+            )
+        predictions.to_csv(directory / "predictions.csv", index=False)
+        labels.to_csv(directory / "labels.csv", index=False)
+        result = {
+            "task_type": task_type,
+            "backend": args.backend,
+            "backend_version": version(
+                "tabpfn" if args.backend == "local" else "tabpfn-client"
+            ),
+            "config": query.config,
+            "validation_scores": [t.val_score for t in query.trials or []],
+            "rows": len(predictions),
+            "cold_warm_max_difference": float(
+                np.max(np.abs(cold_predictions["y_pred"] - predictions["y_pred"]))
+            ),
+            "passed": True,
+        }
+        results.append(result)
+        (output / "results.json").write_text(json.dumps(results, indent=2) + "\n")
+        print(
+            f"{task_type}: passed with {len(predictions)} real predictions", flush=True
+        )
+
+
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -98,6 +98,5 @@ ignore-decorators = ["typing.override", "typing_extensions.override"]
 members = ["packages/relarena", "packages/relarena-core", "packages/tabpfn-rel"]
 
 [tool.pytest.ini_options]
-pythonpath = ["packages/tabpfn-rel"]
 testpaths = ["packages/relarena/tests", "packages/relarena-core/tests", "packages/tabpfn-rel/tests"]
 addopts = "--import-mode=importlib"