Merge remote-tracking branch 'upstream' into perf/groupby-arrow-native

Author: fangchenli

.github/workflows/unit-tests.yml

@@ -31,6 +31,7 @@ jobs:
         # Prevent the include jobs from overriding other jobs
         pattern: [""]
         pandas_future_infer_string: ["1"]
+        pandas_future_python_scalars: ["0"]
         include:
           - name: "Downstream Compat"
             env_file: actions-313-downstream_compat.yaml
@@ -75,6 +76,10 @@ jobs:
             env_file: actions-313.yaml
             pandas_future_infer_string: "0"
             platform: ubuntu-24.04
+          - name: "PANDAS_FUTURE_PYTHON_SCALARS=1"
+            env_file: actions-313.yaml
+            pandas_future_python_scalars: "1"
+            platform: ubuntu-24.04
           - name: "Numpy Dev"
             env_file: actions-313-numpydev.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -92,6 +97,7 @@ jobs:
       LC_ALL: ${{ matrix.lc_all || '' }}
       PANDAS_CI: '1'
       PANDAS_FUTURE_INFER_STRING: ${{ matrix.pandas_future_infer_string || '1' }}
+      PANDAS_FUTURE_PYTHON_SCALARS: ${{ matrix.pandas_future_python_scalars || '0' }}
       TEST_ARGS: ${{ matrix.test_args || '' }}
       PYTEST_WORKERS: 'auto'
       PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }}

doc/source/whatsnew/v3.0.0.rst

@@ -1160,6 +1160,7 @@ Timedelta
 - Accuracy improvement in :meth:`Timedelta.to_pytimedelta` to round microseconds consistently for large nanosecond based Timedelta (:issue:`57841`)
 - Bug in :class:`Timedelta` constructor failing to raise when passed an invalid keyword (:issue:`53801`)
 - Bug in :meth:`DataFrame.cumsum` which was raising ``IndexError`` if dtype is ``timedelta64[ns]`` (:issue:`57956`)
+- Bug in adding or subtracting a :class:`Timedelta` object with non-nanosecond unit to a python ``datetime.datetime`` object giving incorrect results; this now works correctly for Timedeltas inside the ``datetime.timedelta`` implementation bounds (:issue:`53643`)
 - Bug in multiplication operations with ``timedelta64`` dtype failing to raise ``TypeError`` when multiplying by ``bool`` objects or dtypes (:issue:`58054`)
 - Bug in multiplication operations with ``timedelta64`` dtype incorrectly raising when multiplying by numpy-nullable dtypes or pyarrow integer dtypes (:issue:`58054`)
 

pandas/_config/__init__.py

@@ -35,6 +35,11 @@ def using_string_dtype() -> bool:
     return _mode_options["infer_string"]
 
 
+def using_python_scalars() -> bool:
+    _mode_options = _global_config["future"]
+    return _mode_options["python_scalars"]
+
+
 def is_nan_na() -> bool:
     _mode_options = _global_config["future"]
     return not _mode_options["distinguish_nan_and_na"]

pandas/_libs/meson.build

@@ -160,7 +160,7 @@ foreach ext_name, ext_dict : libs_sources
         ext_dict.get('sources'),
         cython_args: cython_args,
         include_directories: [inc_np, inc_pd],
-        dependencies: ext_dict.get('deps', ''),
+        dependencies: ext_dict.get('deps', []),
         subdir: 'pandas/_libs',
         install: true,
     )

pandas/_libs/tslibs/meson.build

@@ -40,7 +40,7 @@ foreach ext_name, ext_dict : tslibs_sources
         ext_dict.get('sources'),
         cython_args: cython_args,
         include_directories: [inc_np, inc_pd],
-        dependencies: ext_dict.get('deps', ''),
+        dependencies: ext_dict.get('deps', []),
         subdir: 'pandas/_libs/tslibs',
         install: true,
     )

pandas/_libs/tslibs/timedeltas.pyx

@@ -1022,9 +1022,23 @@ cdef _timedelta_from_value_and_reso(cls, int64_t value, NPY_DATETIMEUNIT reso):
     elif reso == NPY_DATETIMEUNIT.NPY_FR_us:
         td_base = _Timedelta.__new__(cls, microseconds=int(value))
     elif reso == NPY_DATETIMEUNIT.NPY_FR_ms:
-        td_base = _Timedelta.__new__(cls, milliseconds=0)
+        if -86_399_999_913_600_000 <= value < 86_400_000_000_000_000:
+            # i.e. we are in range for pytimedelta. By passing the
+            #  'correct' value here we can
+            #   make pydatetime + Timedelta operations work correctly,
+            #  xref GH#53643
+            td_base = _Timedelta.__new__(cls, milliseconds=value)
+        else:
+            td_base = _Timedelta.__new__(cls, milliseconds=0)
     elif reso == NPY_DATETIMEUNIT.NPY_FR_s:
-        td_base = _Timedelta.__new__(cls, seconds=0)
+        if -86_399_999_913_600 <= value < 86_400_000_000_000:
+            # i.e. we are in range for pytimedelta. By passing the
+            #  'correct' value here we can
+            #   make pydatetime + Timedelta operations work correctly,
+            #  xref GH#53643
+            td_base = _Timedelta.__new__(cls, seconds=value)
+        else:
+            td_base = _Timedelta.__new__(cls, seconds=0)
     # Other resolutions are disabled but could potentially be implemented here:
     # elif reso == NPY_DATETIMEUNIT.NPY_FR_m:
     #    td_base = _Timedelta.__new__(Timedelta, minutes=int(value))

pandas/conftest.py

@@ -2094,6 +2094,11 @@ def using_infer_string() -> bool:
     return pd.options.future.infer_string is True
 
 
+@pytest.fixture
+def using_python_scalars() -> bool:
+    return pd.options.future.python_scalars is True
+
+
 _warsaws: list[Any] = ["Europe/Warsaw", "dateutil/Europe/Warsaw"]
 if pytz is not None:
     _warsaws.append(pytz.timezone("Europe/Warsaw"))

pandas/core/algorithms.py

@@ -898,18 +898,15 @@ def value_counts_internal(
             result = result.iloc[0:0]
 
         # normalizing is by len of all (regardless of dropna)
-        counts = np.array([len(ii)])
+        normalize_denominator = len(ii)
 
     else:
+        normalize_denominator = None
         if is_extension_array_dtype(values):
             # handle Categorical and sparse,
             result = Series(values, copy=False)._values.value_counts(dropna=dropna)
             result.name = name
             result.index.name = index_name
-            counts = result._values
-            if not isinstance(counts, np.ndarray):
-                # e.g. ArrowExtensionArray
-                counts = np.asarray(counts)
 
         elif isinstance(values, ABCMultiIndex):
             # GH49558
@@ -920,10 +917,6 @@ def value_counts_internal(
                 .size()
             )
             result.index.names = values.names
-            # error: Incompatible types in assignment (expression has type
-            # "ndarray[Any, Any] | DatetimeArray | TimedeltaArray | PeriodArray | Any",
-            # variable has type "ndarray[tuple[int, ...], dtype[Any]]")
-            counts = result._values  # type: ignore[assignment]
 
         else:
             values = _ensure_arraylike(values, func_name="value_counts")
@@ -936,8 +929,7 @@ def value_counts_internal(
             idx = Index(keys, dtype=keys.dtype, name=index_name)
 
             if (
-                bins is None
-                and not sort
+                not sort
                 and isinstance(values, (DatetimeIndex, TimedeltaIndex))
                 and idx.equals(values)
                 and values.inferred_freq is not None
@@ -951,7 +943,10 @@ def value_counts_internal(
         result = result.sort_values(ascending=ascending, kind="stable")
 
     if normalize:
-        result = result / counts.sum()
+        if normalize_denominator is not None:
+            result = result / normalize_denominator
+        else:
+            result = result / result.sum()
 
     return result
 

pandas/core/arraylike.py

@@ -15,6 +15,7 @@
 from pandas._libs import lib
 from pandas._libs.ops_dispatch import maybe_dispatch_ufunc_to_dunder_op
 
+from pandas.core.dtypes.cast import maybe_unbox_numpy_scalar
 from pandas.core.dtypes.generic import ABCNDFrame
 
 from pandas.core import roperator
@@ -529,4 +530,6 @@ def dispatch_reduction_ufunc(self, ufunc: np.ufunc, method: str, *inputs, **kwar
 
     # By default, numpy's reductions do not skip NaNs, so we have to
     #  pass skipna=False
-    return getattr(self, method_name)(skipna=False, **kwargs)
+    result = getattr(self, method_name)(skipna=False, **kwargs)
+    result = maybe_unbox_numpy_scalar(result)
+    return result

pandas/core/arrays/_arrow_string_mixins.py

@@ -203,12 +203,16 @@ def _str_swapcase(self) -> Self:
         return self._from_pyarrow_array(pc.utf8_swapcase(self._pa_array))
 
     def _str_removeprefix(self, prefix: str):
+        if prefix == "":
+            return self._from_pyarrow_array(self._pa_array)
         starts_with = pc.starts_with(self._pa_array, pattern=prefix)
         removed = pc.utf8_slice_codeunits(self._pa_array, len(prefix))
         result = pc.if_else(starts_with, removed, self._pa_array)
         return self._from_pyarrow_array(result)
 
     def _str_removesuffix(self, suffix: str):
+        if suffix == "":
+            return self._from_pyarrow_array(self._pa_array)
         ends_with = pc.ends_with(self._pa_array, pattern=suffix)
         removed = pc.utf8_slice_codeunits(self._pa_array, 0, stop=-len(suffix))
         result = pc.if_else(ends_with, removed, self._pa_array)