diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb70755..9ec2944 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,3 +47,19 @@ jobs: run: uv build --all-packages - name: Verify required package data run: python workflows/verify_distributions.py + + install: + name: Isolated wheel installations + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + enable-cache: true + - name: Build wheels and test each installation path + env: + OMP_NUM_THREADS: "1" + run: python workflows/check_package_split.py --output "$RUNNER_TEMP/package-split" diff --git a/packages/tabpfn-rel/README.md b/packages/tabpfn-rel/README.md index e06a183..c881019 100644 --- a/packages/tabpfn-rel/README.md +++ b/packages/tabpfn-rel/README.md @@ -103,4 +103,5 @@ contains ordinary version requirements. The candidate distributions must be released before the index-only installation commands above are available. The tests exercise feature and context behavior without downloading model weights -or making hosted API requests. +or making hosted API requests. Integration tests use real DFS and a small test +estimator to cover the predictive interface and temporal tuning. diff --git a/packages/tabpfn-rel/tests/test_integration.py b/packages/tabpfn-rel/tests/test_integration.py new file mode 100644 index 0000000..c7d6e90 --- /dev/null +++ b/packages/tabpfn-rel/tests/test_integration.py @@ -0,0 +1,126 @@ +"""Real DFS, temporal tuning and prediction with an in-memory estimator backend.""" + +from __future__ import annotations + +import importlib.util +from collections.abc import Callable +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +from relarena_core.tfm import TFMSpec + +from tabpfn_rel import PredictiveQuery, PredictiveQuerySpec, tfm + + +class _Estimator: + def fit(self, X: pd.DataFrame, y: np.ndarray) -> _Estimator: + assert len(X) == len(y) > 0 + assert X.shape[1] > 0 + self.classes_ = np.unique(y) + self.mean_ = float(np.mean(y)) + self.columns_ = list(X.columns) + return self + + def predict_proba(self, X: pd.DataFrame) -> np.ndarray: + return np.full((len(X), len(self.classes_)), 1 / len(self.classes_)) + + def predict(self, X: pd.DataFrame) -> np.ndarray: + return np.full(len(X), self.mean_) + + +def _make_estimator(**kwargs: object) -> _Estimator: + return _Estimator() + + +_EXAMPLE = Path(__file__).resolve().parents[1] / "examples" / "tiny_database.py" + + +@pytest.fixture(scope="session") +def write_database() -> Callable[..., Path]: + """The generated-database writer from the example script, loaded by path.""" + spec = importlib.util.spec_from_file_location("tiny_database", _EXAMPLE) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.write_database + + +@pytest.fixture +def query( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + request: pytest.FixtureRequest, + write_database: Callable[..., Path], +) -> PredictiveQuery: + for name in ("tabpfn-v3", "tabpfn-v3-api"): + monkeypatch.setitem( + tfm.TFM_REGISTRY, + name, + TFMSpec(_make_estimator, _make_estimator, 100_000, supports_text=True), + ) + task_path = write_database( + tmp_path, getattr(request, "param", "binary_classification") + ) + spec = PredictiveQuerySpec.from_yaml(str(task_path), data_dir=str(tmp_path)) + return PredictiveQuery(spec, data_version="test-v1") + + +@pytest.mark.parametrize( + "query", ["binary_classification", "regression"], indirect=True +) +@pytest.mark.parametrize("backend", ["local", "client"]) +@pytest.mark.parametrize("n_trials", [0, 2]) +def test_rpi_fits_tunes_and_reuses_prediction_cache( + query: PredictiveQuery, backend: str, n_trials: int, tmp_path: Path +) -> None: + query.fit(f"tabpfn-rel-{backend}", n_trials=n_trials, cache_dir=tmp_path / "cache") + predictions = query.predict() + pd.testing.assert_frame_equal(predictions, query.predict()) + assert sorted(predictions["customer_id"]) == ["a", "b", "c", "d"] + assert predictions["y_pred"].notna().all() + assert len(query.compute_test_labels()) == 4 + assert query.config["max_depth"] in (2, 3) + if n_trials: + assert len(query.trials) == 2 + assert all(trial.val_score is not None for trial in query.trials) + else: + assert query.trials is None + if backend == "client": + assert "description__raw_text" in query._model._fitted.estimator.columns_ + assert list((tmp_path / "cache").rglob("*.parquet")) + + +def test_cli_benchmarks_discovered_model( + query: PredictiveQuery, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + cli = pytest.importorskip("relarena.cli") + runner = pytest.importorskip("relarena.runner") + from relarena.tasks import TaskSpec + + source = query._source + monkeypatch.setattr(runner, "RelBenchDatasetTask", lambda *args, **kwargs: source) + monkeypatch.setattr( + cli, + "list_entity_tasks", + lambda datasets: [TaskSpec("tiny", "customers", query.task.task_type)], + ) + output = tmp_path / "results.csv" + assert ( + cli.main( + [ + "--model", + "tabpfn-rel-local", + "--datasets", + "tiny", + "--n-trials", + "1", + "--output", + str(output), + ] + ) + == 0 + ) + results = pd.read_csv(output) + assert results["test_score"].notna().any() diff --git a/packages/tabpfn-rel/workflows/check_backends.py b/packages/tabpfn-rel/workflows/check_backends.py new file mode 100644 index 0000000..6869cc8 --- /dev/null +++ b/packages/tabpfn-rel/workflows/check_backends.py @@ -0,0 +1,54 @@ +"""Check installed estimator constructors without fitting or making API requests.""" + +from __future__ import annotations + +import argparse +from importlib.metadata import version + +import numpy as np + +from tabpfn_rel.tfm import TFM_REGISTRY + + +def main() -> None: + """Construct classification and regression estimators with context overrides.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("backend", choices=["local", "api", "all"]) + args = parser.parse_args() + for name, dependency in ( + ("tabpfn-v3", "tabpfn"), + ("tabpfn-v3-api", "tabpfn-client"), + ): + if args.backend == "local" and name.endswith("api"): + continue + if args.backend == "api" and not name.endswith("api"): + continue + spec = TFM_REGISTRY[name] + for make in (spec.make_classifier, spec.make_regressor): + estimator = make( + device="cpu", + seed=7, + n_estimators=2, + inference_config={ + "SUBSAMPLE_SAMPLES": [np.array([0, 1]), np.array([1, 2])] + }, + ) + assert estimator.random_state == 7 + assert estimator.n_estimators == 2 + if name.endswith("api"): + assert estimator.model_path in { + "v3_default", + "tabpfn-v3-classifier-v3_default.ckpt", + "tabpfn-v3-regressor-v3_default.ckpt", + } + assert estimator.inference_config["SUBSAMPLE_SAMPLES"] == [ + [0, 1], + [1, 2], + ] + print( + f"{dependency} {version(dependency)}: classifier and regressor constructors passed" + ) + + +if __name__ == "__main__": + main() diff --git a/packages/tabpfn-rel/workflows/check_inference.py b/packages/tabpfn-rel/workflows/check_inference.py new file mode 100644 index 0000000..15d6f1b --- /dev/null +++ b/packages/tabpfn-rel/workflows/check_inference.py @@ -0,0 +1,80 @@ +"""Exercise real TabPFN inference over the generated relational example. + +From this checkout, run ``python -m workflows.check_inference --output PATH``. +The local backend needs model weights. The client backend needs authentication +and consumes service quota. Only generated customer/event data are used. +""" + +from __future__ import annotations + +import argparse +import json +from importlib.metadata import version +from pathlib import Path + +import numpy as np +import pandas as pd + +from examples.tiny_database import write_database +from tabpfn_rel import PredictiveQuery, PredictiveQuerySpec + + +def main() -> None: + """Fit classification and regression models, then check warm-cache predictions and tuning.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--backend", choices=["local", "client"], default="local") + parser.add_argument("--n-trials", type=int, default=2) + args = parser.parse_args() + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=True) + results = [] + for task_type in ("binary_classification", "regression"): + directory = output / task_type + task_path = write_database(directory, task_type) + spec = PredictiveQuerySpec.from_yaml(str(task_path), data_dir=str(directory)) + query = PredictiveQuery(spec, data_version="generated-v1").fit( + f"tabpfn-rel-{args.backend}", + n_trials=args.n_trials, + cache_dir=directory / "cache", + ) + cold_predictions = query.predict() + predictions = query.predict() + repeated = query.predict() + assert sorted(predictions["customer_id"]) == ["a", "b", "c", "d"] + assert np.isfinite(predictions["y_pred"]).all() + pd.testing.assert_frame_equal(predictions, repeated, rtol=1e-5, atol=1e-7) + if task_type == "binary_classification": + assert predictions["y_pred"].between(0, 1).all() + labels = query.compute_test_labels() + assert len(labels) == len(predictions) == 4 + assert list((directory / "cache").rglob("*.parquet")) + if args.n_trials: + assert query.trials and all( + t.ok and np.isfinite(t.val_score) for t in query.trials + ) + predictions.to_csv(directory / "predictions.csv", index=False) + labels.to_csv(directory / "labels.csv", index=False) + result = { + "task_type": task_type, + "backend": args.backend, + "backend_version": version( + "tabpfn" if args.backend == "local" else "tabpfn-client" + ), + "config": query.config, + "validation_scores": [t.val_score for t in query.trials or []], + "rows": len(predictions), + "cold_warm_max_difference": float( + np.max(np.abs(cold_predictions["y_pred"] - predictions["y_pred"])) + ), + "passed": True, + } + results.append(result) + (output / "results.json").write_text(json.dumps(results, indent=2) + "\n") + print( + f"{task_type}: passed with {len(predictions)} real predictions", flush=True + ) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 5dd165f..bf1885a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,6 +98,5 @@ ignore-decorators = ["typing.override", "typing_extensions.override"] members = ["packages/relarena", "packages/relarena-core", "packages/tabpfn-rel"] [tool.pytest.ini_options] -pythonpath = ["packages/tabpfn-rel"] testpaths = ["packages/relarena/tests", "packages/relarena-core/tests", "packages/tabpfn-rel/tests"] addopts = "--import-mode=importlib" diff --git a/workflows/check_dependency_floors.py b/workflows/check_dependency_floors.py new file mode 100644 index 0000000..ff2f790 --- /dev/null +++ b/workflows/check_dependency_floors.py @@ -0,0 +1,106 @@ +"""Check core dependency wheel floors and supported backend constructors. + +Build the workspace wheels first, then run with --wheels PATH --output PATH. +Installs dependencies into three clean environments without making API requests. +Core uses the lowest direct versions with compatible wheels for the interpreter. +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from pathlib import Path + +import tomllib + + +def main() -> None: + """Install lowest-direct core and both backend floors, then run smoke checks.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--wheels", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + root = Path(__file__).resolve().parents[1] + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + core = tomllib.loads((root / "packages/relarena-core/pyproject.toml").read_text()) + core_requirements = [ + requirement + for requirement in core["project"]["dependencies"] + if any(operator in requirement for operator in ("<", ">", "=")) + ] + model_version = tomllib.loads( + (root / "packages/tabpfn-rel/pyproject.toml").read_text() + )["project"]["version"] + cases = ( + ("core", [f"relarena-core=={core['project']['version']}", *core_requirements]), + ("local", [f"tabpfn-rel[local]=={model_version}", "tabpfn==8.0.0"]), + ("api", [f"tabpfn-rel[api]=={model_version}", "tabpfn-client==0.3.2"]), + ) + env = dict(os.environ, OMP_NUM_THREADS="1") + env.pop("PYTHONPATH", None) + for name, requirements in cases: + directory = output / name + python = str(directory / "bin/python") + commands = [ + ["uv", "venv", "--seed", "--python", sys.executable, str(directory)], + [ + "uv", + "pip", + "install", + "--python", + python, + "--find-links", + str(args.wheels.resolve()), + *( + ["--resolution", "lowest-direct", "--only-binary", ":all:"] + if name == "core" + else [] + ), + *requirements, + ], + [python, "-m", "pip", "check"], + ] + if name == "core": + commands.extend( + [ + ["uv", "pip", "install", "--python", python, "pytest>=9.1.1"], + [ + python, + "-m", + "pytest", + "-q", + "--import-mode=importlib", + str( + root + / "packages/relarena-core/tests/test_package_boundary.py" + ), + str( + root + / "packages/relarena-core/tests/test_standalone_runtime.py" + ), + ], + ] + ) + else: + commands.append( + [ + python, + str(root / "packages/tabpfn-rel/workflows/check_backends.py"), + name, + ] + ) + with (output / f"{name}.log").open("w") as log: + for command in commands: + log.write("$ " + " ".join(command) + "\n") + log.flush() + subprocess.run( + command, cwd=output, env=env, stdout=log, stderr=log, check=True + ) + print(f"{name}: passed", flush=True) + + +if __name__ == "__main__": + main() diff --git a/workflows/check_package_split.py b/workflows/check_package_split.py new file mode 100644 index 0000000..4928b79 --- /dev/null +++ b/workflows/check_package_split.py @@ -0,0 +1,243 @@ +"""Build three distributions and test isolated core, model and benchmark installs. + +Run from the workspace with ``python workflows/check_package_split.py --output PATH``. +Downloads dependencies, but does not download model weights or call an API. +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import tarfile +import zipfile +from email.parser import BytesParser +from pathlib import Path + +CASES = ( + ("core", "relarena-core", None), + ("base", "relarena", None), + ("host-local", "relarena[tabpfn-rel-local]", "tabpfn"), + ("host-api", "relarena[tabpfn-rel-api]", "tabpfn-client"), + ("direct-local", "tabpfn-rel[local]", "tabpfn"), + ("direct-api", "tabpfn-rel[api]", "tabpfn-client"), +) + + +def run(command: list[str], cwd: Path, env: dict[str, str], log: Path) -> None: + """Run a checked command and retain its output for review.""" + with log.open("a") as output: + output.write("\n$ " + " ".join(command) + "\n") + output.flush() + result = subprocess.run( + command, cwd=cwd, env=env, stdout=output, stderr=subprocess.STDOUT + ) + if result.returncode: + raise RuntimeError(f"Command failed ({result.returncode}); see {log}") + + +def check_artifacts(wheels: Path) -> list[str]: + """Verify dependency direction, package ownership, schemas and notices.""" + pins = [] + for name in ("relarena", "relarena_core", "tabpfn_rel"): + wheel = next(wheels.glob(f"{name}-*.whl")) + with zipfile.ZipFile(wheel) as archive: + names = archive.namelist() + metadata = BytesParser().parsebytes( + archive.read(next(n for n in names if n.endswith("/METADATA"))) + ) + pins.append(f"{metadata['Name']}=={metadata['Version']}") + requirements = metadata.get_all("Requires-Dist", []) + assert not any(" @ " in req for req in requirements), requirements + for namespace in ("relarena", "relarena_core", "tabpfn_rel"): + if namespace != name: + assert not any(n.startswith(namespace + "/") for n in names) + for notice in ("LICENSE", "NOTICE"): + assert any(n.endswith("/licenses/" + notice) for n in names) + if name == "relarena": + assert "tabpfn-rel" not in metadata.get_all("Provides-Extra", []) + plugin_requirements = [ + r for r in requirements if r.startswith("tabpfn-rel") + ] + assert len(plugin_requirements) == 2, plugin_requirements + assert all("extra ==" in r for r in plugin_requirements) + assert any(r.startswith("relarena-core") for r in requirements) + assert "relarena/models/rdblearn/tfm.py" in names + assert "relarena/refit.py" in names + assert "relarena/models/_shared/gbdt/lgb.py" in names + assert "relarena/tfm.py" not in names + assert "relarena/tuner.py" not in names + assert "relarena/checksums/relbench_v1_checksums.json" in names + assert "relarena/models/VENDORED-LICENSES" in names + assert any(n.endswith("/db.yaml") for n in names) + else: + assert not any( + r.startswith(("relarena>", "relarena=", "relarena[", "relarena ")) + for r in requirements + ) + if name == "relarena_core": + assert not any(r.startswith("tabpfn") for r in requirements) + for schema in ("database", "task"): + assert f"relarena_core/userdb/{schema}.schema.json" in names + else: + assert any(r.startswith("relarena-core") for r in requirements) + if name != "relarena_core": + entrypoints = archive.read( + next(n for n in names if n.endswith("/entry_points.txt")) + ).decode() + assert "[relarena.models]" in entrypoints + target = "relarena.models" if name == "relarena" else "tabpfn_rel.model" + assert target in entrypoints + with tarfile.open(next(wheels.glob(f"{name}-*.tar.gz"))) as archive: + names = archive.getnames() + assert any(n.endswith("/pyproject.toml") for n in names) + assert any(n.endswith(f"/src/{name}/__init__.py") for n in names) + return pins + + +def main() -> None: + """Build candidates, rebuild sdists and verify each installed dependency closure.""" + root = Path(__file__).resolve().parents[1] + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--companion", type=Path, default=root / "packages/tabpfn-rel") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--python", default=sys.executable) + args = parser.parse_args() + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + wheels = output / "wheels" + wheels.mkdir() + log = output / "commands.log" + env = { + **os.environ, + "OMP_NUM_THREADS": "1", + "UV_CACHE_DIR": os.environ.get("UV_CACHE_DIR", str(output / "uv-cache")), + } + env.pop("PYTHONPATH", None) + for source in ( + root / "packages/relarena-core", + root / "packages/relarena", + args.companion.resolve(), + ): + run(["uv", "build", "--out-dir", str(wheels)], source, env, log) + pins = check_artifacts(wheels) + constraints = output / "constraints.txt" + constraints.write_text("\n".join(pins) + "\n") + for sdist in sorted(wheels.glob("*.tar.gz")): + run( + [ + "uv", + "build", + "--wheel", + str(sdist), + "--out-dir", + str(output / "rebuilt"), + ], + output, + env, + log, + ) + results = [] + for name, requirement, backend in CASES: + directory = output / name + print(f"Testing {name}", flush=True) + run( + ["uv", "venv", "--seed", "--python", args.python, str(directory)], + output, + env, + log, + ) + python = str(directory / "bin/python") + install = ["uv", "pip", "install", "--python", python] + run( + [ + *install, + "--find-links", + str(wheels), + "--constraint", + str(constraints), + requirement, + ], + output, + env, + log, + ) + run([python, "-m", "pip", "check"], output, env, log) + host = name in {"base", "host-local", "host-api"} + code = f""" +import importlib.metadata as metadata +import importlib.util +import inspect +import sys +import relarena_core +from relarena_core.userdb import PredictiveQuery +from relarena_core.userdb._schema import load_schema +installed = {{d.metadata['Name'].lower().replace('_', '-') for d in metadata.distributions()}} +assert 'relarena-core' in installed +assert ('relarena' in installed) == {host!r} +assert ('tabpfn-rel' in installed) == {bool(backend)!r} +assert not relarena_core.registry.names() +assert list(inspect.signature(PredictiveQuery.precompute_cache).parameters) == ['self', 'cache_dir'] +assert load_schema('task.schema.json')['type'] == 'object' +assert load_schema('database.schema.json')['type'] == 'object' +if not {host!r}: + assert importlib.util.find_spec('relarena') is None +if {host!r}: + import relarena + from relarena.userdb import PredictiveQuery as HostQuery + assert HostQuery is PredictiveQuery + assert relarena.RelArenaModel is relarena_core.RelArenaModel + assert relarena.registry is relarena_core.registry +if {bool(backend)!r}: + import tabpfn_rel + assert tabpfn_rel.PredictiveQuery is PredictiveQuery + assert {backend!r} in installed + assert 'fastdfs' in installed + if {backend!r} == 'tabpfn-client': + assert 'tabpfn' not in installed +relarena_core.discover_models() +relarena_core.discover_models() +assert ('tabpfn-rel-local' in relarena_core.registry) == {bool(backend)!r} +assert ('rdblearn' in relarena_core.registry) == {host!r} +assert 'tabpfn' not in sys.modules +assert 'tabpfn_client' not in sys.modules +if {bool(backend)!r}: + assert relarena_core.registry.get('tabpfn-rel-local') is tabpfn_rel.TabPFNRelLocalModel +print('Verified', sys.executable, sorted(installed)) +""" + run([python, "-c", code], output, env, log) + if host: + run( + [python, "-m", "relarena.cli", "--list", "--datasets", "rel-f1"], + output, + env, + log, + ) + if name == "core": + run([*install, "pytest>=9.1.1"], output, env, log) + run( + [ + python, + "-m", + "pytest", + "-q", + "--import-mode=importlib", + str(root / "packages/relarena-core/tests"), + "--ignore", + str(root / "packages/relarena-core/tests/featurization"), + ], + output, + env, + log, + ) + results.append({"case": name, "passed": True}) + (output / "results.json").write_text(json.dumps(results, indent=2) + "\n") + print( + f"All {len(results)} installation paths passed. Results: {output / 'results.json'}" + ) + + +if __name__ == "__main__": + main()