Apply regression target transforms to the unnormalized target [RES-2639]

changelog/1199.changed.md

@@ -0,0 +1 @@
+Regression target transforms (`REGRESSION_Y_PREPROCESS_TRANSFORMS`) now reshape the target in its original units and standardize the result, instead of reshaping the already z-normalized target. A transform such as `1_plus_log` therefore does what its name suggests; previously it was `log1p` of a z-score, which is undefined below -1 and left the model fitted on non-finite targets for roughly half of a symmetric target. This changes the predictions of the v2, v2.5 and v3 defaults, whose ensembles use `safepower` for half of their estimators; the v2.6 default (`"none"`) and any estimator without a target transform are unaffected.

src/tabpfn/preprocessing/target_transform.py

@@ -90,10 +90,12 @@ def make_target_transform(transform: Transformer | Pipeline | None) -> Pipeline:
         return Pipeline(steps=[(STANDARDIZE_STEP, StandardizeTarget())])
     return Pipeline(
         steps=[
-            # The preset reshapes the standardized target, as it always has;
-            # the ordering of these two steps is what RES-2639 changes.
-            (STANDARDIZE_STEP, StandardizeTarget()),
+            # The preset reshapes the target in its own units, and only then is
+            # the result standardized. Reshaping a standardized target instead
+            # means something else entirely: `1_plus_log` was `log1p` of a
+            # z-score, undefined wherever that dropped below -1.
             (TARGET_TRANSFORM_STEP, transform),
+            (STANDARDIZE_STEP, StandardizeTarget()),
         ],
     )
 

src/tabpfn/preprocessing/transform.py

@@ -95,7 +95,10 @@ def _transform_labels_one(
 
     Args:
         config: Ensemble config.
-        y_train: The unprocessed labels.
+        y_train: The labels as the estimator handed them over: encoded class
+            indices for classification, and the target in its original units
+            for regression, which the config's target pipeline reshapes and
+            standardizes (see `tabpfn.preprocessing.target_transform`).
 
     Return: The processed labels.
     """

src/tabpfn/regressor.py

@@ -899,10 +899,12 @@ def _initialize_dataset_preprocessing(
         )
         # Every member standardizes the target itself, so the members are handed
         # the target in its original units and no statistic of it is needed here.
+        # `"none"` is the identity, so it is resolved to no transform at all
+        # rather than to a preset that only costs a pass over the target.
         target_preprocessors: list[TransformerMixin | Pipeline | None] = [
             make_target_transform(
                 None
-                if y_target_preprocessor is None
+                if y_target_preprocessor in (None, "none")
                 else possible_target_transforms[y_target_preprocessor]
             )
             for y_target_preprocessor in (

tests/test_preprocessing/test_target_transform.py

@@ -75,22 +75,49 @@ def test__make_target_transform__without_a_transform_only_standardizes() -> None
 
 
 @pytest.mark.parametrize("name", TRANSFORM_NAMES)
-def test__make_target_transform__transform_sees_the_standardized_target(
+def test__make_target_transform__transform_sees_the_unnormalized_target(
     name: str,
 ) -> None:
-    """The preset still reshapes the standardized target.
-
-    This is what RES-2639 goes on to change; pinning it here keeps the move of
-    the standardization to a separate, reviewable step.
+    """The preset reshapes the target in its own units, and only then is it
+    standardized -- the point of RES-2639.
     """
     y = _target()
     transform = _get_transform(name, len(y))
 
     got = make_target_transform(transform).fit_transform(y.reshape(-1, 1)).astype(float)
 
-    standardized = (y - np.mean(y)) / (np.std(y) + StandardizeTarget.EPSILON)
-    expected = _get_transform(name, len(y)).fit_transform(standardized.reshape(-1, 1))
-    np.testing.assert_allclose(got, np.asarray(expected, dtype=float), rtol=1e-12)
+    reshaped = np.asarray(
+        _get_transform(name, len(y)).fit_transform(y.reshape(-1, 1)), dtype=float
+    )
+    expected = (reshaped - reshaped.mean()) / (
+        reshaped.std() + StandardizeTarget.EPSILON
+    )
+    np.testing.assert_allclose(got, expected, rtol=1e-10)
+
+
+def test__make_target_transform__log_of_the_target_is_defined() -> None:
+    """The bug the ticket is about: `log1p` of a z-score is mostly NaN.
+
+    Roughly half of a z-normalized target is negative, and `log1p` is undefined
+    below -1, so the model used to be fitted on imputed or non-finite targets.
+    """
+    rng = np.random.default_rng(0)
+    y = rng.normal(100.0, 10.0, size=500)  # symmetric, strictly positive
+
+    standardized = (y - np.mean(y)) / np.std(y)
+    assert not np.isfinite(np.log1p(standardized)).all()
+
+    got = (
+        make_target_transform(_get_transform("1_plus_log", len(y)))
+        .fit_transform(y.reshape(-1, 1))
+        .ravel()
+    )
+
+    assert np.isfinite(got).all()
+    expected = np.log1p(y)
+    np.testing.assert_allclose(
+        got, (expected - expected.mean()) / expected.std(), rtol=1e-10
+    )
 
 
 @pytest.mark.parametrize("name", TRANSFORM_NAMES)

tests/test_regressor_interface.py

@@ -1414,6 +1414,83 @@ def test__fit__member_target_is_the_znormalized_target() -> None:
     assert np.array_equal(member_y, (y - np.mean(y)) / (np.std(y) + 1e-20))
 
 
+def _mk_skewed_reg_dataset(
+    seed: int, n: int = 200, f: int = 5
+) -> tuple[np.ndarray, np.ndarray]:
+    """A strictly positive, right-skewed target -- the log-transform case."""
+    r = np.random.RandomState(seed)
+    X = r.randn(n, f)
+    y = np.exp(1.5 + X @ np.array([1.0, -0.5, 0.3, 0.0, 0.2]) + 0.3 * r.randn(n))
+    return X, y
+
+
+def test__fit__target_transform__applied_to_the_unnormalized_target() -> None:
+    """A target transform must see the target in its original units.
+
+    The estimator's own z-normalization used to come first, so a transform such
+    as `1_plus_log` operated on standardized values, where it means something
+    entirely different and is undefined below -1.
+    """
+    X, y = _mk_skewed_reg_dataset(0)
+    reg = TabPFNRegressor(
+        n_estimators=1,
+        device="cpu",
+        random_state=42,
+        inference_config={"REGRESSION_Y_PREPROCESS_TRANSFORMS": ("1_plus_log",)},
+    )
+    reg.fit(X, y)
+
+    expected = np.log1p(y)
+    expected = (expected - expected.mean()) / expected.std()
+    member_y = np.asarray(reg.executor_.ensemble_members[0].y_train)
+
+    np.testing.assert_allclose(member_y, expected, rtol=1e-8)
+
+
+def test__fit__none_preset_is_resolved_to_no_transform() -> None:
+    """`"none"` is the identity, so it must not cost an extra pass."""
+    X, y = _mk_skewed_reg_dataset(0)
+    reg = TabPFNRegressor(
+        n_estimators=1,
+        device="cpu",
+        random_state=42,
+        inference_config={"REGRESSION_Y_PREPROCESS_TRANSFORMS": ("none",)},
+    )
+    reg.fit(X, y)
+
+    pipeline = reg.executor_.ensemble_members[0].config.target_transform
+    assert list(pipeline.named_steps) == ["standardize_target"]
+    assert np.array_equal(
+        np.asarray(reg.executor_.ensemble_members[0].y_train),
+        (y - np.mean(y)) / (np.std(y) + 1e-20),
+    )
+
+
+@pytest.mark.parametrize("transform", ["1_plus_log", "log", "safepower"])
+def test__predict__target_transform__predictions_are_finite_and_accurate(
+    transform: str,
+) -> None:
+    """Transforming a skewed target must yield usable predictions.
+
+    Guards the whole round trip: the pipeline is fitted on the unnormalized
+    target, while the model's borders are mapped back through it and then into
+    the frame the ensemble is aggregated in.
+    """
+    X, y = _mk_skewed_reg_dataset(0)
+    X_train, X_test, y_train, y_test = X[:150], X[150:], y[:150], y[150:]
+    reg = TabPFNRegressor(
+        n_estimators=2,
+        device="cpu",
+        random_state=42,
+        inference_config={"REGRESSION_Y_PREPROCESS_TRANSFORMS": (transform,)},
+    )
+    reg.fit(X_train, y_train)
+    pred = reg.predict(X_test)
+
+    assert np.isfinite(pred).all()
+    assert r2_score(y_test, pred) > 0.7
+
+
 def test__fit__target_frame_is_independent_of_the_target_units() -> None:
     """`y_train_mean_`/`y_train_std_` define the frame, and only that.