From cf1cd0d62587276f6a805159de4e1760b3220874 Mon Sep 17 00:00:00 2001 From: Pedro Marques Date: Tue, 1 Apr 2025 16:02:34 +0100 Subject: [PATCH 1/3] Fix #61208: OverflowError when fillna on DataFrame with a pd.Timestamp - Now correctly raises OutOfBoundsDatetime - Added test_fillna_out_of_bounds_datetime() --- pandas/core/internals/blocks.py | 2 ++ pandas/tests/frame/methods/test_fillna.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index d1a9081b234de..f985acca7b323 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -1749,6 +1749,8 @@ def putmask(self, mask, new) -> list[Block]: try: # Caller is responsible for ensuring matching lengths values._putmask(mask, new) + except OutOfBoundsDatetime as e: + raise except (TypeError, ValueError): if self.ndim == 1 or self.shape[0] == 1: if isinstance(self.dtype, IntervalDtype): diff --git a/pandas/tests/frame/methods/test_fillna.py b/pandas/tests/frame/methods/test_fillna.py index 67d1d45af1cb3..92fe1a534ed00 100644 --- a/pandas/tests/frame/methods/test_fillna.py +++ b/pandas/tests/frame/methods/test_fillna.py @@ -15,6 +15,7 @@ ) import pandas._testing as tm from pandas.tests.frame.common import _check_mixed_float +from pandas.errors import OutOfBoundsDatetime class TestFillNA: @@ -781,3 +782,19 @@ def test_fillna_with_none_object(test_frame, dtype): if test_frame: expected = expected.to_frame() tm.assert_equal(result, expected) + + +def test_fillna_out_of_bounds_datetime(): + # GH#61208 + df = DataFrame({ + 'datetime': date_range('1/1/2011', periods=3, freq='h'), + 'value': [1, 2, 3] + }) + df.iloc[0, 0] = None + + msg="Cannot cast 0001-01-01 00:00:00 to unit='ns' without overflow" + with pytest.raises( + OutOfBoundsDatetime, + match=msg + ): + df.fillna(Timestamp('0001-01-01'), inplace=True) From 0a9a54241d1f6bf6719647668ddf384e89eadeda Mon Sep 17 00:00:00 2001 From: Pedro Marques Date: Wed, 2 Apr 2025 20:06:10 +0100 Subject: [PATCH 2/3] Comply with pre-commit and added an entry in v3.0.0.rst --- doc/source/whatsnew/v3.0.0.rst | 1 + pandas/core/internals/blocks.py | 2 +- pandas/tests/frame/methods/test_fillna.py | 23 ++++++++++------------- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst index 4a6cf117fd196..2b07901ba0e86 100644 --- a/doc/source/whatsnew/v3.0.0.rst +++ b/doc/source/whatsnew/v3.0.0.rst @@ -644,6 +644,7 @@ Datetimelike - Bug in :func:`date_range` where using a negative frequency value would not include all points between the start and end values (:issue:`56147`) - Bug in :func:`tseries.api.guess_datetime_format` would fail to infer time format when "%Y" == "%H%M" (:issue:`57452`) - Bug in :func:`tseries.frequencies.to_offset` would fail to parse frequency strings starting with "LWOM" (:issue:`59218`) +- Bug in :meth:`DataFrame.fillna` raising an ``AssertionError`` instead of ``OutOfBoundsDatetime`` when filling a ``datetime64[ns]`` column with an out-of-bounds timestamp. Now correctly raises ``OutOfBoundsDatetime``. (:issue:`61208`) - Bug in :meth:`DataFrame.min` and :meth:`DataFrame.max` casting ``datetime64`` and ``timedelta64`` columns to ``float64`` and losing precision (:issue:`60850`) - Bug in :meth:`Dataframe.agg` with df with missing values resulting in IndexError (:issue:`58810`) - Bug in :meth:`DatetimeIndex.is_year_start` and :meth:`DatetimeIndex.is_quarter_start` does not raise on Custom business days frequencies bigger then "1C" (:issue:`58664`) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index f985acca7b323..10848090cc206 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -1749,7 +1749,7 @@ def putmask(self, mask, new) -> list[Block]: try: # Caller is responsible for ensuring matching lengths values._putmask(mask, new) - except OutOfBoundsDatetime as e: + except OutOfBoundsDatetime: raise except (TypeError, ValueError): if self.ndim == 1 or self.shape[0] == 1: diff --git a/pandas/tests/frame/methods/test_fillna.py b/pandas/tests/frame/methods/test_fillna.py index 92fe1a534ed00..a1e51babf5fba 100644 --- a/pandas/tests/frame/methods/test_fillna.py +++ b/pandas/tests/frame/methods/test_fillna.py @@ -1,6 +1,8 @@ import numpy as np import pytest +from pandas.errors import OutOfBoundsDatetime + from pandas import ( Categorical, DataFrame, @@ -15,7 +17,6 @@ ) import pandas._testing as tm from pandas.tests.frame.common import _check_mixed_float -from pandas.errors import OutOfBoundsDatetime class TestFillNA: @@ -782,19 +783,15 @@ def test_fillna_with_none_object(test_frame, dtype): if test_frame: expected = expected.to_frame() tm.assert_equal(result, expected) - - + + def test_fillna_out_of_bounds_datetime(): # GH#61208 - df = DataFrame({ - 'datetime': date_range('1/1/2011', periods=3, freq='h'), - 'value': [1, 2, 3] - }) + df = DataFrame( + {"datetime": date_range("1/1/2011", periods=3, freq="h"), "value": [1, 2, 3]} + ) df.iloc[0, 0] = None - msg="Cannot cast 0001-01-01 00:00:00 to unit='ns' without overflow" - with pytest.raises( - OutOfBoundsDatetime, - match=msg - ): - df.fillna(Timestamp('0001-01-01'), inplace=True) + msg = "Cannot cast 0001-01-01 00:00:00 to unit='ns' without overflow" + with pytest.raises(OutOfBoundsDatetime, match=msg): + df.fillna(Timestamp("0001-01-01"), inplace=True) From 311bb726d2eb2d501191d81ec0bc6fc1966a1188 Mon Sep 17 00:00:00 2001 From: Pedro Marques Date: Sat, 12 Apr 2025 16:21:10 +0100 Subject: [PATCH 3/3] Removed flag 'inplace=True' from test and fixed the bug for this case. --- pandas/core/internals/blocks.py | 2 ++ pandas/tests/frame/methods/test_fillna.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 10848090cc206..84fab6c03cf75 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -1682,6 +1682,8 @@ def where(self, other, cond) -> list[Block]: try: res_values = arr._where(cond, other).T + except OutOfBoundsDatetime: + raise except (ValueError, TypeError): if self.ndim == 1 or self.shape[0] == 1: if isinstance(self.dtype, (IntervalDtype, StringDtype)): diff --git a/pandas/tests/frame/methods/test_fillna.py b/pandas/tests/frame/methods/test_fillna.py index a1e51babf5fba..8915d6f205d65 100644 --- a/pandas/tests/frame/methods/test_fillna.py +++ b/pandas/tests/frame/methods/test_fillna.py @@ -794,4 +794,4 @@ def test_fillna_out_of_bounds_datetime(): msg = "Cannot cast 0001-01-01 00:00:00 to unit='ns' without overflow" with pytest.raises(OutOfBoundsDatetime, match=msg): - df.fillna(Timestamp("0001-01-01"), inplace=True) + df.fillna(Timestamp("0001-01-01"))