Merge branch 'main' into shiny-new-feature

Author: Chiwendaiyue

doc/source/whatsnew/v3.0.0.rst

@@ -808,6 +808,7 @@ Other Deprecations
 - Deprecated option "future.no_silent_downcasting", as it is no longer used. In a future version accessing this option will raise (:issue:`59502`)
 - Deprecated passing non-Index types to :meth:`Index.join`; explicitly convert to Index first (:issue:`62897`)
 - Deprecated silent casting of non-datetime 'other' to datetime in :meth:`Series.combine_first` (:issue:`62931`)
+- Deprecated silently casting strings to :class:`Timedelta` in binary operations with :class:`Timedelta` (:issue:`59653`)
 - Deprecated slicing on a :class:`Series` or :class:`DataFrame` with a :class:`DatetimeIndex` using a ``datetime.date`` object, explicitly cast to :class:`Timestamp` instead (:issue:`35830`)
 - Deprecated support for the Dataframe Interchange Protocol (:issue:`56732`)
 - Deprecated the 'inplace' keyword from :meth:`Resampler.interpolate`, as passing ``True`` raises ``AttributeError`` (:issue:`58690`)

pandas/_libs/tslibs/timedeltas.pyx

@@ -333,58 +333,39 @@ cdef convert_to_timedelta64(object ts, str unit):
 
     Handle these types of objects:
         - timedelta/Timedelta
-        - timedelta64
-        - an offset
-        - np.int64 (with unit providing a possible modifier)
-        - None/NaT
 
-    Return an ns based int64
+    Return an timedelta64[ns] object
     """
     # Caller is responsible for checking unit not in ["Y", "y", "M"]
-    if checknull_with_nat_and_na(ts):
-        return np.timedelta64(NPY_NAT, "ns")
-    elif isinstance(ts, _Timedelta):
+    if isinstance(ts, _Timedelta):
         # already in the proper format
         if ts._creso != NPY_FR_ns:
             ts = ts.as_unit("ns").asm8
         else:
             ts = np.timedelta64(ts._value, "ns")
-    elif cnp.is_timedelta64_object(ts):
-        ts = ensure_td64ns(ts)
-    elif is_integer_object(ts):
-        if ts == NPY_NAT:
-            return np.timedelta64(NPY_NAT, "ns")
-        else:
-            ts = _maybe_cast_from_unit(ts, unit)
-    elif is_float_object(ts):
-        ts = _maybe_cast_from_unit(ts, unit)
-    elif isinstance(ts, str):
-        if (len(ts) > 0 and ts[0] == "P") or (len(ts) > 1 and ts[:2] == "-P"):
-            ts = parse_iso_format_string(ts)
-        else:
-            ts = parse_timedelta_string(ts)
-        ts = np.timedelta64(ts, "ns")
-    elif is_tick_object(ts):
-        ts = np.timedelta64(ts.nanos, "ns")
 
-    if PyDelta_Check(ts):
+    elif PyDelta_Check(ts):
         ts = np.timedelta64(delta_to_nanoseconds(ts), "ns")
     elif not cnp.is_timedelta64_object(ts):
         raise TypeError(f"Invalid type for timedelta scalar: {type(ts)}")
     return ts.astype("timedelta64[ns]")
 
 
-cdef _maybe_cast_from_unit(ts, str unit):
+cdef _numeric_to_td64ns(object item, str unit):
     # caller is responsible for checking
     #  assert unit not in ["Y", "y", "M"]
+    #  assert is_integer_object(item) or is_float_object(item)
+    if is_integer_object(item) and item == NPY_NAT:
+        return np.timedelta64(NPY_NAT, "ns")
+
     try:
-        ts = cast_from_unit(ts, unit)
+        item = cast_from_unit(item, unit)
     except OutOfBoundsDatetime as err:
         raise OutOfBoundsTimedelta(
-            f"Cannot cast {ts} from {unit} to 'ns' without overflow."
+            f"Cannot cast {item} from {unit} to 'ns' without overflow."
         ) from err
 
-    ts = np.timedelta64(ts, "ns")
+    ts = np.timedelta64(item, "ns")
     return ts
 
 
@@ -408,10 +389,11 @@ def array_to_timedelta64(
     cdef:
         Py_ssize_t i, n = values.size
         ndarray result = np.empty((<object>values).shape, dtype="m8[ns]")
-        object item
+        object item, td64ns_obj
         int64_t ival
         cnp.broadcast mi = cnp.PyArray_MultiIterNew2(result, values)
         cnp.flatiter it
+        str parsed_unit = parse_timedelta_unit(unit or "ns")
 
     if values.descr.type_num != cnp.NPY_OBJECT:
         # raise here otherwise we segfault below
@@ -431,70 +413,63 @@ def array_to_timedelta64(
                 )
             cnp.PyArray_ITER_NEXT(it)
 
-    # Usually, we have all strings. If so, we hit the fast path.
-    # If this path fails, we try conversion a different way, and
-    # this is where all of the error handling will take place.
-    try:
-        for i in range(n):
-            # Analogous to: item = values[i]
-            item = <object>(<PyObject**>cnp.PyArray_MultiIter_DATA(mi, 1))[0]
+    for i in range(n):
+        item = <object>(<PyObject**>cnp.PyArray_MultiIter_DATA(mi, 1))[0]
 
-            ival = _item_to_timedelta64_fastpath(item)
+        try:
+            if checknull_with_nat_and_na(item):
+                ival = NPY_NAT
 
-            # Analogous to: iresult[i] = ival
-            (<int64_t*>cnp.PyArray_MultiIter_DATA(mi, 0))[0] = ival
+            elif cnp.is_timedelta64_object(item):
+                td64ns_obj = ensure_td64ns(item)
+                ival = cnp.get_timedelta64_value(td64ns_obj)
 
-            cnp.PyArray_MultiIter_NEXT(mi)
+            elif isinstance(item, _Timedelta):
+                if item._creso != NPY_FR_ns:
+                    ival = item.as_unit("ns")._value
+                else:
+                    ival = item._value
+
+            elif PyDelta_Check(item):
+                # i.e. isinstance(item, timedelta)
+                ival = delta_to_nanoseconds(item)
+
+            elif isinstance(item, str):
+                if (
+                    (len(item) > 0 and item[0] == "P")
+                    or (len(item) > 1 and item[:2] == "-P")
+                ):
+                    ival = parse_iso_format_string(item)
+                else:
+                    ival = parse_timedelta_string(item)
 
-    except (TypeError, ValueError):
-        cnp.PyArray_MultiIter_RESET(mi)
+            elif is_tick_object(item):
+                ival = item.nanos
 
-        parsed_unit = parse_timedelta_unit(unit or "ns")
-        for i in range(n):
-            item = <object>(<PyObject**>cnp.PyArray_MultiIter_DATA(mi, 1))[0]
+            elif is_integer_object(item) or is_float_object(item):
+                td64ns_obj = _numeric_to_td64ns(item, parsed_unit)
+                ival = cnp.get_timedelta64_value(td64ns_obj)
 
-            ival = _item_to_timedelta64(item, parsed_unit, errors)
+            else:
+                raise TypeError(f"Invalid type for timedelta scalar: {type(item)}")
+
+        except ValueError as err:
+            if errors == "coerce":
+                ival = NPY_NAT
+            elif "unit abbreviation w/o a number" in str(err):
+                # re-raise with more pertinent message
+                msg = f"Could not convert '{item}' to NumPy timedelta"
+                raise ValueError(msg) from err
+            else:
+                raise
 
-            (<int64_t*>cnp.PyArray_MultiIter_DATA(mi, 0))[0] = ival
+        (<int64_t*>cnp.PyArray_MultiIter_DATA(mi, 0))[0] = ival
 
-            cnp.PyArray_MultiIter_NEXT(mi)
+        cnp.PyArray_MultiIter_NEXT(mi)
 
     return result
 
 
-cdef int64_t _item_to_timedelta64_fastpath(object item) except? -1:
-    """
-    See array_to_timedelta64.
-    """
-    if item is NaT:
-        # we allow this check in the fast-path because NaT is a C-object
-        #  so this is an inexpensive check
-        return NPY_NAT
-    else:
-        return parse_timedelta_string(item)
-
-
-cdef int64_t _item_to_timedelta64(
-    object item,
-    str parsed_unit,
-    str errors
-) except? -1:
-    """
-    See array_to_timedelta64.
-    """
-    try:
-        return cnp.get_timedelta64_value(convert_to_timedelta64(item, parsed_unit))
-    except ValueError as err:
-        if errors == "coerce":
-            return NPY_NAT
-        elif "unit abbreviation w/o a number" in str(err):
-            # re-raise with more pertinent message
-            msg = f"Could not convert '{item}' to NumPy timedelta"
-            raise ValueError(msg) from err
-        else:
-            raise
-
-
 @cython.cpow(True)
 cdef int64_t parse_timedelta_string(str ts) except? -1:
     """
@@ -798,7 +773,7 @@ def _binary_op_method_timedeltalike(op, name):
             return NotImplemented
 
         try:
-            other = Timedelta(other)
+            other = _wrapped_to_timedelta(other)
         except ValueError:
             # failed to parse as timedelta
             return NotImplemented
@@ -2154,12 +2129,14 @@ class Timedelta(_Timedelta):
             new_value = delta_to_nanoseconds(value, reso=new_reso)
             return cls._from_value_and_reso(new_value, reso=new_reso)
 
+        elif checknull_with_nat_and_na(value):
+            return NaT
+
         elif is_integer_object(value) or is_float_object(value):
             # unit=None is de-facto 'ns'
             unit = parse_timedelta_unit(unit)
-            value = convert_to_timedelta64(value, unit)
-        elif checknull_with_nat_and_na(value):
-            return NaT
+            value = _numeric_to_td64ns(value, unit)
+
         else:
             raise ValueError(
                 "Value must be Timedelta, string, integer, "
@@ -2341,7 +2318,7 @@ class Timedelta(_Timedelta):
     def __truediv__(self, other):
         if _should_cast_to_timedelta(other):
             # We interpret NaT as timedelta64("NaT")
-            other = Timedelta(other)
+            other = _wrapped_to_timedelta(other)
             if other is NaT:
                 return np.nan
             if other._creso != self._creso:
@@ -2374,7 +2351,7 @@ class Timedelta(_Timedelta):
     def __rtruediv__(self, other):
         if _should_cast_to_timedelta(other):
             # We interpret NaT as timedelta64("NaT")
-            other = Timedelta(other)
+            other = _wrapped_to_timedelta(other)
             if other is NaT:
                 return np.nan
             if self._creso != other._creso:
@@ -2402,7 +2379,7 @@ class Timedelta(_Timedelta):
         # just defer
         if _should_cast_to_timedelta(other):
             # We interpret NaT as timedelta64("NaT")
-            other = Timedelta(other)
+            other = _wrapped_to_timedelta(other)
             if other is NaT:
                 return np.nan
             if self._creso != other._creso:
@@ -2457,7 +2434,7 @@ class Timedelta(_Timedelta):
         # just defer
         if _should_cast_to_timedelta(other):
             # We interpret NaT as timedelta64("NaT")
-            other = Timedelta(other)
+            other = _wrapped_to_timedelta(other)
             if other is NaT:
                 return np.nan
             if self._creso != other._creso:
@@ -2525,6 +2502,7 @@ def truediv_object_array(ndarray left, ndarray right):
         if cnp.get_timedelta64_value(td64) == NPY_NAT:
             # td here should be interpreted as a td64 NaT
             if _should_cast_to_timedelta(obj):
+                _wrapped_to_timedelta(obj)  # deprecate if allowing string
                 res_value = np.nan
             else:
                 # if its a number then let numpy handle division, otherwise
@@ -2554,6 +2532,7 @@ def floordiv_object_array(ndarray left, ndarray right):
         if cnp.get_timedelta64_value(td64) == NPY_NAT:
             # td here should be interpreted as a td64 NaT
             if _should_cast_to_timedelta(obj):
+                _wrapped_to_timedelta(obj)  # deprecate allowing string
                 res_value = np.nan
             else:
                 # if its a number then let numpy handle division, otherwise
@@ -2585,6 +2564,23 @@ cdef bint is_any_td_scalar(object obj):
     )
 
 
+cdef inline _wrapped_to_timedelta(object other):
+    # Helper for deprecating cases where we cast str to Timedelta
+    td = Timedelta(other)
+    if isinstance(other, str):
+        from pandas.errors import Pandas4Warning
+        warnings.warn(
+            # GH#59653
+            "Scalar operations between Timedelta and string are "
+            "deprecated and will raise in a future version. "
+            "Explicitly cast to Timedelta first.",
+            Pandas4Warning,
+            stacklevel=find_stack_level(),
+        )
+        # When this is enforced, remove str from _should_cast_to_timedelta
+    return td
+
+
 cdef bint _should_cast_to_timedelta(object obj):
     """
     Should we treat this object as a Timedelta for the purpose of a binary op

pandas/conftest.py

@@ -722,7 +722,7 @@ def index(request):
         - ...
     """
     # copy to avoid mutation, e.g. setting .name
-    return indices_dict[request.param].copy()
+    return indices_dict[request.param].copy(deep=False)
 
 
 @pytest.fixture(
@@ -735,7 +735,7 @@ def index_flat(request):
     index fixture, but excluding MultiIndex cases.
     """
     key = request.param
-    return indices_dict[key].copy()
+    return indices_dict[key].copy(deep=False)
 
 
 @pytest.fixture(
@@ -758,18 +758,15 @@ def index_with_missing(request):
 
     MultiIndex is excluded because isna() is not defined for MultiIndex.
     """
-
-    # GH 35538. Use deep copy to avoid illusive bug on np-dev
-    # GHA pipeline that writes into indices_dict despite copy
-    ind = indices_dict[request.param].copy(deep=True)
-    vals = ind.values.copy()
+    ind = indices_dict[request.param]
     if request.param in ["tuples", "mi-with-dt64tz-level", "multi"]:
         # For setting missing values in the top level of MultiIndex
         vals = ind.tolist()
         vals[0] = (None,) + vals[0][1:]
         vals[-1] = (None,) + vals[-1][1:]
         return MultiIndex.from_tuples(vals)
     else:
+        vals = ind.values.copy()
         vals[0] = None
         vals[-1] = None
         return type(ind)(vals)
@@ -850,7 +847,7 @@ def index_or_series_obj(request):
     Fixture for tests on indexes, series and series with a narrow dtype
     copy to avoid mutation, e.g. setting .name
     """
-    return _index_or_series_objs[request.param].copy(deep=True)
+    return _index_or_series_objs[request.param].copy(deep=False)
 
 
 _typ_objects_series = {
@@ -873,7 +870,7 @@ def index_or_series_memory_obj(request):
     series with empty objects type
     copy to avoid mutation, e.g. setting .name
     """
-    return _index_or_series_memory_objs[request.param].copy(deep=True)
+    return _index_or_series_memory_objs[request.param].copy(deep=False)
 
 
 # ----------------------------------------------------------------