Merge branch 'pandas-dev:main' into main

Author: sdhjebngc

doc/source/whatsnew/v3.0.0.rst

@@ -820,6 +820,7 @@ Other API changes
   :meth:`~DataFrame.ffill`, :meth:`~DataFrame.bfill`, :meth:`~DataFrame.interpolate`,
   :meth:`~DataFrame.where`, :meth:`~DataFrame.mask`, :meth:`~DataFrame.clip`) now return
   the modified DataFrame or Series (``self``) instead of ``None`` when ``inplace=True`` (:issue:`63207`)
+- All Index constructors now copy ``numpy.ndarray`` and ``ExtensionArray`` inputs by default when ``copy=None``, consistent with :class:`Series` behavior (:issue:`63388`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_300.deprecations:

pandas/_libs/meson.build

@@ -160,7 +160,7 @@ foreach ext_name, ext_dict : libs_sources
         ext_dict.get('sources'),
         cython_args: cython_args,
         include_directories: [inc_np, inc_pd],
-        dependencies: ext_dict.get('deps', ''),
+        dependencies: ext_dict.get('deps', []),
         subdir: 'pandas/_libs',
         install: true,
     )

pandas/_libs/tslibs/meson.build

@@ -40,7 +40,7 @@ foreach ext_name, ext_dict : tslibs_sources
         ext_dict.get('sources'),
         cython_args: cython_args,
         include_directories: [inc_np, inc_pd],
-        dependencies: ext_dict.get('deps', ''),
+        dependencies: ext_dict.get('deps', []),
         subdir: 'pandas/_libs/tslibs',
         install: true,
     )

pandas/core/algorithms.py

@@ -878,18 +878,15 @@ def value_counts_internal(
             result = result.iloc[0:0]
 
         # normalizing is by len of all (regardless of dropna)
-        counts = np.array([len(ii)])
+        normalize_denominator = len(ii)
 
     else:
+        normalize_denominator = None
         if is_extension_array_dtype(values):
             # handle Categorical and sparse,
             result = Series(values, copy=False)._values.value_counts(dropna=dropna)
             result.name = name
             result.index.name = index_name
-            counts = result._values
-            if not isinstance(counts, np.ndarray):
-                # e.g. ArrowExtensionArray
-                counts = np.asarray(counts)
 
         elif isinstance(values, ABCMultiIndex):
             # GH49558
@@ -900,10 +897,6 @@ def value_counts_internal(
                 .size()
             )
             result.index.names = values.names
-            # error: Incompatible types in assignment (expression has type
-            # "ndarray[Any, Any] | DatetimeArray | TimedeltaArray | PeriodArray | Any",
-            # variable has type "ndarray[tuple[int, ...], dtype[Any]]")
-            counts = result._values  # type: ignore[assignment]
 
         else:
             values = _ensure_arraylike(values, func_name="value_counts")
@@ -916,8 +909,7 @@ def value_counts_internal(
             idx = Index(keys, dtype=keys.dtype, name=index_name)
 
             if (
-                bins is None
-                and not sort
+                not sort
                 and isinstance(values, (DatetimeIndex, TimedeltaIndex))
                 and idx.equals(values)
                 and values.inferred_freq is not None
@@ -931,7 +923,10 @@ def value_counts_internal(
         result = result.sort_values(ascending=ascending, kind="stable")
 
     if normalize:
-        result = result / counts.sum()
+        if normalize_denominator is not None:
+            result = result / normalize_denominator
+        else:
+            result = result / result.sum()
 
     return result
 

pandas/core/arrays/_arrow_string_mixins.py

@@ -203,12 +203,16 @@ def _str_swapcase(self) -> Self:
         return self._from_pyarrow_array(pc.utf8_swapcase(self._pa_array))
 
     def _str_removeprefix(self, prefix: str):
+        if prefix == "":
+            return self._from_pyarrow_array(self._pa_array)
         starts_with = pc.starts_with(self._pa_array, pattern=prefix)
         removed = pc.utf8_slice_codeunits(self._pa_array, len(prefix))
         result = pc.if_else(starts_with, removed, self._pa_array)
         return self._from_pyarrow_array(result)
 
     def _str_removesuffix(self, suffix: str):
+        if suffix == "":
+            return self._from_pyarrow_array(self._pa_array)
         ends_with = pc.ends_with(self._pa_array, pattern=suffix)
         removed = pc.utf8_slice_codeunits(self._pa_array, 0, stop=-len(suffix))
         result = pc.if_else(ends_with, removed, self._pa_array)

pandas/core/arrays/datetimes.py

@@ -226,13 +226,16 @@ class DatetimeArray(dtl.TimelikeOps, dtl.DatelikeOps):
     """
 
     _typ = "datetimearray"
-    _internal_fill_value = np.datetime64("NaT", "ns")
     _recognized_scalars = (datetime, np.datetime64)
     _is_recognized_dtype: Callable[[DtypeObj], bool] = lambda x: lib.is_np_dtype(
         x, "M"
     ) or isinstance(x, DatetimeTZDtype)
     _infer_matches = ("datetime", "datetime64", "date")
 
+    @property
+    def _internal_fill_value(self) -> np.datetime64:
+        return np.datetime64("NaT", self.unit)
+
     @property
     def _scalar_type(self) -> type[Timestamp]:
         return Timestamp

pandas/core/arrays/timedeltas.py

@@ -154,11 +154,14 @@ class TimedeltaArray(dtl.TimelikeOps):
     """
 
     _typ = "timedeltaarray"
-    _internal_fill_value = np.timedelta64("NaT", "ns")
     _recognized_scalars = (timedelta, np.timedelta64, Tick)
     _is_recognized_dtype: Callable[[DtypeObj], bool] = lambda x: lib.is_np_dtype(x, "m")
     _infer_matches = ("timedelta", "timedelta64")
 
+    @property
+    def _internal_fill_value(self) -> np.timedelta64:
+        return np.timedelta64("NaT", self.unit)
+
     @property
     def _scalar_type(self) -> type[Timedelta]:
         return Timedelta

pandas/core/indexes/base.py

@@ -505,12 +505,8 @@ def __new__(
         if not copy and isinstance(data, (ABCSeries, Index)):
             refs = data._references
 
-        if isinstance(data, (ExtensionArray, np.ndarray)):
-            # GH 63306
-            if copy is not False:
-                if dtype is None or astype_is_view(data.dtype, dtype):
-                    data = data.copy()
-                    copy = False
+        # GH 63306, GH 63388
+        data, copy = cls._maybe_copy_array_input(data, copy, dtype)
 
         # range
         if isinstance(data, (range, RangeIndex)):
@@ -5197,6 +5193,21 @@ def _raise_scalar_data_error(cls, data):
             "was passed"
         )
 
+    @classmethod
+    def _maybe_copy_array_input(
+        cls, data, copy: bool | None, dtype
+    ) -> tuple[Any, bool]:
+        """
+        Ensure that the input data is copied if necessary.
+        GH#63388
+        """
+        if isinstance(data, (ExtensionArray, np.ndarray)):
+            if copy is not False:
+                if dtype is None or astype_is_view(data.dtype, pandas_dtype(dtype)):
+                    data = data.copy()
+                    copy = False
+        return data, bool(copy)
+
     def _validate_fill_value(self, value):
         """
         Check if the value can be inserted into our array without casting,
@@ -6865,12 +6876,15 @@ def get_slice_bound(self, label, side: Literal["left", "right"]) -> int:
         # we need to look up the label
         try:
             slc = self.get_loc(label)
-        except KeyError as err:
+        except KeyError:
             try:
                 return self._searchsorted_monotonic(label, side)
             except ValueError:
-                # raise the original KeyError
-                raise err from None
+                raise KeyError(
+                    f"Cannot get {side} slice bound for non-monotonic index "
+                    f"with a missing label {original_label!r}. "
+                    "Either sort the index or specify an existing label."
+                ) from None
 
         if isinstance(slc, np.ndarray):
             # get_loc may return a boolean array, which

pandas/core/indexes/datetimes.py

@@ -181,8 +181,13 @@ class DatetimeIndex(DatetimeTimedeltaMixin):
         If True parse dates in `data` with the year first order.
     dtype : numpy.dtype or DatetimeTZDtype or str, default None
         Note that the only NumPy dtype allowed is `datetime64[ns]`.
-    copy : bool, default False
-        Make a copy of input ndarray.
+    copy : bool, default None
+        Whether to copy input data, only relevant for array, Series, and Index
+        inputs (for other input, e.g. a list, a new array is created anyway).
+        Defaults to True for array input and False for Index/Series.
+        Set to False to avoid copying array input at your own risk (if you
+        know the input data won't be modified elsewhere).
+        Set to True to force copying Series/Index up front.
     name : label, default None
         Name to be stored in the index.
 
@@ -669,7 +674,7 @@ def __new__(
         dayfirst: bool = False,
         yearfirst: bool = False,
         dtype: Dtype | None = None,
-        copy: bool = False,
+        copy: bool | None = None,
         name: Hashable | None = None,
     ) -> Self:
         if is_scalar(data):
@@ -679,6 +684,9 @@ def __new__(
 
         name = maybe_extract_name(name, data, cls)
 
+        # GH#63388
+        data, copy = cls._maybe_copy_array_input(data, copy, dtype)
+
         if (
             isinstance(data, DatetimeArray)
             and freq is lib.no_default

pandas/core/indexes/interval.py

@@ -169,8 +169,13 @@ class IntervalIndex(ExtensionIndex):
         neither.
     dtype : dtype or None, default None
         If None, dtype will be inferred.
-    copy : bool, default False
-        Copy the input data.
+    copy : bool, default None
+        Whether to copy input data, only relevant for array, Series, and Index
+        inputs (for other input, e.g. a list, a new array is created anyway).
+        Defaults to True for array input and False for Index/Series.
+        Set to False to avoid copying array input at your own risk (if you
+        know the input data won't be modified elsewhere).
+        Set to True to force copying Series/Index input up front.
     name : object, optional
          Name to be stored in the index.
     verify_integrity : bool, default True
@@ -252,12 +257,15 @@ def __new__(
         data,
         closed: IntervalClosedType | None = None,
         dtype: Dtype | None = None,
-        copy: bool = False,
+        copy: bool | None = None,
         name: Hashable | None = None,
         verify_integrity: bool = True,
     ) -> Self:
         name = maybe_extract_name(name, data, cls)
 
+        # GH#63388
+        data, copy = cls._maybe_copy_array_input(data, copy, dtype)
+
         with rewrite_exception("IntervalArray", cls.__name__):
             array = IntervalArray(
                 data,