From e7d29d79966afe683a172ae422f78eea61c8d353 Mon Sep 17 00:00:00 2001 From: Benjamin Jaeger Date: Fri, 21 Aug 2026 16:41:00 +0200 Subject: [PATCH] Apply regression target transforms to the unnormalized target [RES-2639] Swaps the two steps of a member's target pipeline: the preset now reshapes the target in its original units, and the result is standardized afterwards. Applied to a z-normalised target, `1_plus_log` was `log1p` of a z-score, which is undefined below -1: for a symmetric target roughly 15% of the rows came out non-finite and reached the model, since the log presets carry no scaler of their own. `safepower` was fitted on standardized values, where yeo-johnson cannot reach the log limit that actually removes skew from a heavy right tail. Also resolves the `"none"` preset to no transform at all -- it is the identity, so composing it only costs a pass over the target -- and fixes the docstring of `_transform_labels_one`, which called its input "unprocessed". This changes the predictions of the v2, v2.5 and v3 defaults, whose ensembles use `safepower` for half of their estimators. The v2.6 default (`"none"`) and every estimator without a target transform are unaffected. Validated on TabArena regression: 13 datasets, 222 splits per arm, 8 estimators, 1332 fits, -0.031% rmse / -0.069% mae / -0.062% 1-r2, better on 9 of 13 datasets, with the gains concentrated on heavy-tailed positive targets (healthcare_insurance_expenses -0.23%, diamonds -0.16%, physiochemical_protein -0.13%). That run measured the same semantics through an earlier implementation of this change; the two agree to 1e-7 relative on the five datasets used to cross-check them. The caveat from that report carries over: dropping the power transform entirely costs only +0.069% rmse on those 13 datasets, so the view caps what this can be worth and the effect sits below its noise floor. Follow-up suggested on the ticket is to validate on skewed positive-target data. Co-Authored-By: Claude Opus 5 --- changelog/1199.changed.md | 1 + src/tabpfn/preprocessing/target_transform.py | 8 +- src/tabpfn/preprocessing/transform.py | 5 +- src/tabpfn/regressor.py | 4 +- .../test_target_transform.py | 43 +++++++++-- tests/test_regressor_interface.py | 77 +++++++++++++++++++ 6 files changed, 125 insertions(+), 13 deletions(-) create mode 100644 changelog/1199.changed.md diff --git a/changelog/1199.changed.md b/changelog/1199.changed.md new file mode 100644 index 000000000..ee9f58291 --- /dev/null +++ b/changelog/1199.changed.md @@ -0,0 +1 @@ +Regression target transforms (`REGRESSION_Y_PREPROCESS_TRANSFORMS`) now reshape the target in its original units and standardize the result, instead of reshaping the already z-normalized target. A transform such as `1_plus_log` therefore does what its name suggests; previously it was `log1p` of a z-score, which is undefined below -1 and left the model fitted on non-finite targets for roughly half of a symmetric target. This changes the predictions of the v2, v2.5 and v3 defaults, whose ensembles use `safepower` for half of their estimators; the v2.6 default (`"none"`) and any estimator without a target transform are unaffected. diff --git a/src/tabpfn/preprocessing/target_transform.py b/src/tabpfn/preprocessing/target_transform.py index 0ce061055..42e5bec5c 100644 --- a/src/tabpfn/preprocessing/target_transform.py +++ b/src/tabpfn/preprocessing/target_transform.py @@ -90,10 +90,12 @@ def make_target_transform(transform: Transformer | Pipeline | None) -> Pipeline: return Pipeline(steps=[(STANDARDIZE_STEP, StandardizeTarget())]) return Pipeline( steps=[ - # The preset reshapes the standardized target, as it always has; - # the ordering of these two steps is what RES-2639 changes. - (STANDARDIZE_STEP, StandardizeTarget()), + # The preset reshapes the target in its own units, and only then is + # the result standardized. Reshaping a standardized target instead + # means something else entirely: `1_plus_log` was `log1p` of a + # z-score, undefined wherever that dropped below -1. (TARGET_TRANSFORM_STEP, transform), + (STANDARDIZE_STEP, StandardizeTarget()), ], ) diff --git a/src/tabpfn/preprocessing/transform.py b/src/tabpfn/preprocessing/transform.py index 1bb4ba25a..bf731b617 100644 --- a/src/tabpfn/preprocessing/transform.py +++ b/src/tabpfn/preprocessing/transform.py @@ -95,7 +95,10 @@ def _transform_labels_one( Args: config: Ensemble config. - y_train: The unprocessed labels. + y_train: The labels as the estimator handed them over: encoded class + indices for classification, and the target in its original units + for regression, which the config's target pipeline reshapes and + standardizes (see `tabpfn.preprocessing.target_transform`). Return: The processed labels. """ diff --git a/src/tabpfn/regressor.py b/src/tabpfn/regressor.py index d91d2728d..0d06856ab 100644 --- a/src/tabpfn/regressor.py +++ b/src/tabpfn/regressor.py @@ -899,10 +899,12 @@ def _initialize_dataset_preprocessing( ) # Every member standardizes the target itself, so the members are handed # the target in its original units and no statistic of it is needed here. + # `"none"` is the identity, so it is resolved to no transform at all + # rather than to a preset that only costs a pass over the target. target_preprocessors: list[TransformerMixin | Pipeline | None] = [ make_target_transform( None - if y_target_preprocessor is None + if y_target_preprocessor in (None, "none") else possible_target_transforms[y_target_preprocessor] ) for y_target_preprocessor in ( diff --git a/tests/test_preprocessing/test_target_transform.py b/tests/test_preprocessing/test_target_transform.py index ece2e6409..4c4de820b 100644 --- a/tests/test_preprocessing/test_target_transform.py +++ b/tests/test_preprocessing/test_target_transform.py @@ -75,22 +75,49 @@ def test__make_target_transform__without_a_transform_only_standardizes() -> None @pytest.mark.parametrize("name", TRANSFORM_NAMES) -def test__make_target_transform__transform_sees_the_standardized_target( +def test__make_target_transform__transform_sees_the_unnormalized_target( name: str, ) -> None: - """The preset still reshapes the standardized target. - - This is what RES-2639 goes on to change; pinning it here keeps the move of - the standardization to a separate, reviewable step. + """The preset reshapes the target in its own units, and only then is it + standardized -- the point of RES-2639. """ y = _target() transform = _get_transform(name, len(y)) got = make_target_transform(transform).fit_transform(y.reshape(-1, 1)).astype(float) - standardized = (y - np.mean(y)) / (np.std(y) + StandardizeTarget.EPSILON) - expected = _get_transform(name, len(y)).fit_transform(standardized.reshape(-1, 1)) - np.testing.assert_allclose(got, np.asarray(expected, dtype=float), rtol=1e-12) + reshaped = np.asarray( + _get_transform(name, len(y)).fit_transform(y.reshape(-1, 1)), dtype=float + ) + expected = (reshaped - reshaped.mean()) / ( + reshaped.std() + StandardizeTarget.EPSILON + ) + np.testing.assert_allclose(got, expected, rtol=1e-10) + + +def test__make_target_transform__log_of_the_target_is_defined() -> None: + """The bug the ticket is about: `log1p` of a z-score is mostly NaN. + + Roughly half of a z-normalized target is negative, and `log1p` is undefined + below -1, so the model used to be fitted on imputed or non-finite targets. + """ + rng = np.random.default_rng(0) + y = rng.normal(100.0, 10.0, size=500) # symmetric, strictly positive + + standardized = (y - np.mean(y)) / np.std(y) + assert not np.isfinite(np.log1p(standardized)).all() + + got = ( + make_target_transform(_get_transform("1_plus_log", len(y))) + .fit_transform(y.reshape(-1, 1)) + .ravel() + ) + + assert np.isfinite(got).all() + expected = np.log1p(y) + np.testing.assert_allclose( + got, (expected - expected.mean()) / expected.std(), rtol=1e-10 + ) @pytest.mark.parametrize("name", TRANSFORM_NAMES) diff --git a/tests/test_regressor_interface.py b/tests/test_regressor_interface.py index 4a5c2ef39..2d0ca6bd3 100644 --- a/tests/test_regressor_interface.py +++ b/tests/test_regressor_interface.py @@ -1414,6 +1414,83 @@ def test__fit__member_target_is_the_znormalized_target() -> None: assert np.array_equal(member_y, (y - np.mean(y)) / (np.std(y) + 1e-20)) +def _mk_skewed_reg_dataset( + seed: int, n: int = 200, f: int = 5 +) -> tuple[np.ndarray, np.ndarray]: + """A strictly positive, right-skewed target -- the log-transform case.""" + r = np.random.RandomState(seed) + X = r.randn(n, f) + y = np.exp(1.5 + X @ np.array([1.0, -0.5, 0.3, 0.0, 0.2]) + 0.3 * r.randn(n)) + return X, y + + +def test__fit__target_transform__applied_to_the_unnormalized_target() -> None: + """A target transform must see the target in its original units. + + The estimator's own z-normalization used to come first, so a transform such + as `1_plus_log` operated on standardized values, where it means something + entirely different and is undefined below -1. + """ + X, y = _mk_skewed_reg_dataset(0) + reg = TabPFNRegressor( + n_estimators=1, + device="cpu", + random_state=42, + inference_config={"REGRESSION_Y_PREPROCESS_TRANSFORMS": ("1_plus_log",)}, + ) + reg.fit(X, y) + + expected = np.log1p(y) + expected = (expected - expected.mean()) / expected.std() + member_y = np.asarray(reg.executor_.ensemble_members[0].y_train) + + np.testing.assert_allclose(member_y, expected, rtol=1e-8) + + +def test__fit__none_preset_is_resolved_to_no_transform() -> None: + """`"none"` is the identity, so it must not cost an extra pass.""" + X, y = _mk_skewed_reg_dataset(0) + reg = TabPFNRegressor( + n_estimators=1, + device="cpu", + random_state=42, + inference_config={"REGRESSION_Y_PREPROCESS_TRANSFORMS": ("none",)}, + ) + reg.fit(X, y) + + pipeline = reg.executor_.ensemble_members[0].config.target_transform + assert list(pipeline.named_steps) == ["standardize_target"] + assert np.array_equal( + np.asarray(reg.executor_.ensemble_members[0].y_train), + (y - np.mean(y)) / (np.std(y) + 1e-20), + ) + + +@pytest.mark.parametrize("transform", ["1_plus_log", "log", "safepower"]) +def test__predict__target_transform__predictions_are_finite_and_accurate( + transform: str, +) -> None: + """Transforming a skewed target must yield usable predictions. + + Guards the whole round trip: the pipeline is fitted on the unnormalized + target, while the model's borders are mapped back through it and then into + the frame the ensemble is aggregated in. + """ + X, y = _mk_skewed_reg_dataset(0) + X_train, X_test, y_train, y_test = X[:150], X[150:], y[:150], y[150:] + reg = TabPFNRegressor( + n_estimators=2, + device="cpu", + random_state=42, + inference_config={"REGRESSION_Y_PREPROCESS_TRANSFORMS": (transform,)}, + ) + reg.fit(X_train, y_train) + pred = reg.predict(X_test) + + assert np.isfinite(pred).all() + assert r2_score(y_test, pred) > 0.7 + + def test__fit__target_frame_is_independent_of_the_target_units() -> None: """`y_train_mean_`/`y_train_std_` define the frame, and only that.