diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index f000c562c6b11..490de66ce50ad 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -44,6 +44,7 @@ from pandas._libs.tslibs.conversion cimport ( cast_from_unit, ) from pandas._libs.tslibs.dtypes cimport ( + abbrev_to_npy_unit, c_DEPR_UNITS, get_supported_reso, is_supported_unit, @@ -359,11 +360,16 @@ def array_to_timedelta64( cnp.broadcast mi = cnp.PyArray_MultiIterNew2(result, values) cnp.flatiter it str parsed_unit = parse_timedelta_unit(unit or "ns") - NPY_DATETIMEUNIT item_reso + NPY_DATETIMEUNIT item_reso, int_reso ResoState state = ResoState(creso) bint infer_reso = creso == NPY_DATETIMEUNIT.NPY_FR_GENERIC ndarray iresult = result.view("i8") + if unit is None: + int_reso = NPY_FR_ns + else: + int_reso = get_supported_reso(abbrev_to_npy_unit(parsed_unit)) + if values.descr.type_num != cnp.NPY_OBJECT: # raise here otherwise we segfault below raise TypeError("array_to_timedelta64 'values' must have object dtype") @@ -472,7 +478,18 @@ def array_to_timedelta64( creso = state.creso ival = delta_to_nanoseconds(item, reso=creso) - elif is_integer_object(item) or is_float_object(item): + elif is_integer_object(item): + if item == NPY_NAT: + ival = NPY_NAT + else: + ival = _numeric_to_td64ns(item, parsed_unit, int_reso) + item_reso = int_reso + + state.update_creso(item_reso) + if infer_reso: + creso = state.creso + + elif is_float_object(item): ival = _numeric_to_td64ns(item, parsed_unit, NPY_FR_ns) item_reso = NPY_FR_ns diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index d4ada8360d645..2f5dbe0faf9ca 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -1656,7 +1656,7 @@ def mean(self, *, skipna: bool = True, axis: AxisInt | None = 0): >>> tdelta_idx = pd.to_timedelta([1, 2, 3], unit="D") >>> tdelta_idx TimedeltaIndex(['1 days', '2 days', '3 days'], - dtype='timedelta64[ns]', freq=None) + dtype='timedelta64[s]', freq=None) >>> tdelta_idx.mean() Timedelta('2 days 00:00:00') """ diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index ff2e2e6429936..1a8f7bb9442fd 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -850,7 +850,7 @@ def to_pytimedelta(self) -> npt.NDArray[np.object_]: >>> tdelta_idx = pd.to_timedelta([1, 2, 3], unit="D") >>> tdelta_idx TimedeltaIndex(['1 days', '2 days', '3 days'], - dtype='timedelta64[ns]', freq=None) + dtype='timedelta64[s]', freq=None) >>> tdelta_idx.to_pytimedelta() array([datetime.timedelta(days=1), datetime.timedelta(days=2), datetime.timedelta(days=3)], dtype=object) @@ -883,7 +883,7 @@ def to_pytimedelta(self) -> npt.NDArray[np.object_]: 0 1 days 1 2 days 2 3 days - dtype: timedelta64[ns] + dtype: timedelta64[s] >>> ser.dt.days 0 1 1 2 @@ -918,7 +918,7 @@ def to_pytimedelta(self) -> npt.NDArray[np.object_]: 0 0 days 00:00:01 1 0 days 00:00:02 2 0 days 00:00:03 - dtype: timedelta64[ns] + dtype: timedelta64[s] >>> ser.dt.seconds 0 1 1 2 @@ -930,7 +930,7 @@ def to_pytimedelta(self) -> npt.NDArray[np.object_]: >>> tdelta_idx = pd.to_timedelta([1, 2, 3], unit='s') >>> tdelta_idx TimedeltaIndex(['0 days 00:00:01', '0 days 00:00:02', '0 days 00:00:03'], - dtype='timedelta64[ns]', freq=None) + dtype='timedelta64[s]', freq=None) >>> tdelta_idx.seconds Index([1, 2, 3], dtype='int32')""" ) @@ -958,7 +958,7 @@ def to_pytimedelta(self) -> npt.NDArray[np.object_]: 0 0 days 00:00:00.000001 1 0 days 00:00:00.000002 2 0 days 00:00:00.000003 - dtype: timedelta64[ns] + dtype: timedelta64[us] >>> ser.dt.microseconds 0 1 1 2 @@ -971,7 +971,7 @@ def to_pytimedelta(self) -> npt.NDArray[np.object_]: >>> tdelta_idx TimedeltaIndex(['0 days 00:00:00.000001', '0 days 00:00:00.000002', '0 days 00:00:00.000003'], - dtype='timedelta64[ns]', freq=None) + dtype='timedelta64[us]', freq=None) >>> tdelta_idx.microseconds Index([1, 2, 3], dtype='int32')""" ) @@ -1208,7 +1208,9 @@ def _ints_to_td64ns(data, unit: str = "ns") -> tuple[np.ndarray, bool]: dtype_str = f"timedelta64[{unit}]" data = data.view(dtype_str) - data = astype_overflowsafe(data, dtype=TD64NS_DTYPE) + new_dtype = get_supported_dtype(data.dtype) + if new_dtype != data.dtype: + data = astype_overflowsafe(data, dtype=new_dtype) # the astype conversion makes a copy, so we can avoid re-copying later copy_made = True diff --git a/pandas/core/indexes/accessors.py b/pandas/core/indexes/accessors.py index 9ead85cb1b2e7..3029a304cdbbe 100644 --- a/pandas/core/indexes/accessors.py +++ b/pandas/core/indexes/accessors.py @@ -489,7 +489,7 @@ def to_pytimedelta(self) -> np.ndarray: 2 2 days 3 3 days 4 4 days - dtype: timedelta64[ns] + dtype: timedelta64[s] >>> s.dt.to_pytimedelta() array([datetime.timedelta(0), datetime.timedelta(days=1), @@ -535,7 +535,7 @@ def components(self) -> DataFrame: 2 0 days 00:00:02 3 0 days 00:00:03 4 0 days 00:00:04 - dtype: timedelta64[ns] + dtype: timedelta64[s] >>> s.dt.components days hours minutes seconds milliseconds microseconds nanoseconds 0 0 0 0 0 0 0 0 diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index fd061666c1f00..921b67ca1ed69 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -132,7 +132,7 @@ def mean(self, *, skipna: bool = True, axis: int | None = 0): >>> tdelta_idx = pd.to_timedelta([1, 2, 3], unit="D") >>> tdelta_idx TimedeltaIndex(['1 days', '2 days', '3 days'], - dtype='timedelta64[ns]', freq=None) + dtype='timedelta64[s]', freq=None) >>> tdelta_idx.mean() Timedelta('2 days 00:00:00') """ diff --git a/pandas/core/tools/timedeltas.py b/pandas/core/tools/timedeltas.py index 6789eb85b6c4b..5f2f9ff1ea265 100644 --- a/pandas/core/tools/timedeltas.py +++ b/pandas/core/tools/timedeltas.py @@ -167,10 +167,10 @@ def to_timedelta( >>> pd.to_timedelta(np.arange(5), unit="s") TimedeltaIndex(['0 days 00:00:00', '0 days 00:00:01', '0 days 00:00:02', '0 days 00:00:03', '0 days 00:00:04'], - dtype='timedelta64[ns]', freq=None) + dtype='timedelta64[s]', freq=None) >>> pd.to_timedelta(np.arange(5), unit="D") TimedeltaIndex(['0 days', '1 days', '2 days', '3 days', '4 days'], - dtype='timedelta64[ns]', freq=None) + dtype='timedelta64[s]', freq=None) """ if unit is not None: unit = parse_timedelta_unit(unit) diff --git a/pandas/tests/arithmetic/test_timedelta64.py b/pandas/tests/arithmetic/test_timedelta64.py index ed8c7fd2ed2ec..89a9148bed557 100644 --- a/pandas/tests/arithmetic/test_timedelta64.py +++ b/pandas/tests/arithmetic/test_timedelta64.py @@ -735,11 +735,12 @@ def test_tdi_add_overflow(self): ts + pd.to_timedelta(106580, "D") _NaT = NaT._value + 1 + td = pd.to_timedelta([106580], "D").as_unit("ns") msg = "Overflow in int64 addition" with pytest.raises(OverflowError, match=msg): - pd.to_timedelta([106580], "D") + Timestamp("2000") + td + Timestamp("2000") with pytest.raises(OverflowError, match=msg): - Timestamp("2000") + pd.to_timedelta([106580], "D") + Timestamp("2000") + td with pytest.raises(OverflowError, match=msg): pd.to_timedelta([_NaT]) - Timedelta("1 days") with pytest.raises(OverflowError, match=msg): diff --git a/pandas/tests/groupby/methods/test_quantile.py b/pandas/tests/groupby/methods/test_quantile.py index 1b4a26919af44..5d9f0063df1e6 100644 --- a/pandas/tests/groupby/methods/test_quantile.py +++ b/pandas/tests/groupby/methods/test_quantile.py @@ -363,15 +363,14 @@ def test_groupby_quantile_allNA_column(dtype): def test_groupby_timedelta_quantile(): # GH: 29485 - df = DataFrame( - {"value": pd.to_timedelta(np.arange(4), unit="s"), "group": [1, 1, 2, 2]} - ) + tdi = pd.to_timedelta(np.arange(4), unit="s").as_unit("us") + df = DataFrame({"value": tdi, "group": [1, 1, 2, 2]}) result = df.groupby("group").quantile(0.99) expected = DataFrame( { "value": [ - pd.Timedelta("0 days 00:00:00.990000").as_unit("ns"), - pd.Timedelta("0 days 00:00:02.990000").as_unit("ns"), + pd.Timedelta("0 days 00:00:00.990000"), + pd.Timedelta("0 days 00:00:02.990000"), ] }, index=Index([1, 2], name="group"), diff --git a/pandas/tests/indexes/timedeltas/methods/test_shift.py b/pandas/tests/indexes/timedeltas/methods/test_shift.py index 297887596dc82..166eeb671867f 100644 --- a/pandas/tests/indexes/timedeltas/methods/test_shift.py +++ b/pandas/tests/indexes/timedeltas/methods/test_shift.py @@ -49,7 +49,7 @@ def test_tdi_shift_int(self): "5 days 01:00:00", ], freq="D", - dtype="m8[ns]", + dtype="m8[s]", ) tm.assert_index_equal(result, expected) @@ -67,7 +67,7 @@ def test_tdi_shift_nonstandard_freq(self): "10 days 01:00:03", ], freq="D", - dtype="m8[ns]", + dtype="m8[s]", ) tm.assert_index_equal(result, expected) diff --git a/pandas/tests/indexes/timedeltas/test_constructors.py b/pandas/tests/indexes/timedeltas/test_constructors.py index 1a2f7053a98ab..a0be8f4b3c74c 100644 --- a/pandas/tests/indexes/timedeltas/test_constructors.py +++ b/pandas/tests/indexes/timedeltas/test_constructors.py @@ -267,4 +267,5 @@ def test_unit_deprecated(self, unit, unit_depr): with tm.assert_produces_warning(Pandas4Warning, match=msg): tdi = to_timedelta([1, 2], unit=unit_depr) - tm.assert_index_equal(tdi, expected.as_unit("ns")) + exp_unit = unit if unit in ["s", "ms", "us"] else "s" + tm.assert_index_equal(tdi, expected.as_unit(exp_unit)) diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index 87d5bfa78c8ab..5bb2770d15493 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -1156,7 +1156,8 @@ def test_timedelta(self): with tm.assert_produces_warning(Pandas4Warning, match=msg): json = frame.to_json() - tm.assert_frame_equal(frame, read_json(StringIO(json)).apply(converter)) + result = read_json(StringIO(json)).apply(converter) + tm.assert_frame_equal(frame.astype("m8[ms]"), result) def test_timedelta2(self): frame = DataFrame( diff --git a/pandas/tests/resample/test_timedelta.py b/pandas/tests/resample/test_timedelta.py index e0b4248fab210..3bec66e3a1aa2 100644 --- a/pandas/tests/resample/test_timedelta.py +++ b/pandas/tests/resample/test_timedelta.py @@ -101,9 +101,7 @@ def test_resample_categorical_data_with_timedeltaindex(): df = DataFrame({"Group_obj": "A"}, index=pd.to_timedelta(list(range(20)), unit="s")) df["Group"] = df["Group_obj"].astype("category") result = df.resample("10s").agg(lambda x: (x.value_counts().index[0])) - exp_tdi = pd.TimedeltaIndex(np.array([0, 10], dtype="m8[s]"), freq="10s").as_unit( - "ns" - ) + exp_tdi = pd.TimedeltaIndex(np.array([0, 10], dtype="m8[s]"), freq="10s") expected = DataFrame( {"Group_obj": ["A", "A"], "Group": ["A", "A"]}, index=exp_tdi, diff --git a/pandas/tests/scalar/timedelta/test_constructors.py b/pandas/tests/scalar/timedelta/test_constructors.py index be990d25f08d6..28ef2d8079106 100644 --- a/pandas/tests/scalar/timedelta/test_constructors.py +++ b/pandas/tests/scalar/timedelta/test_constructors.py @@ -161,9 +161,10 @@ def test_unit_deprecated(self, unit, unit_depr): def test_unit_parser(self, unit, np_unit, wrapper): # validate all units, GH 6855, GH 21762 # array-likes + exp_unit = np_unit if np_unit not in ["W", "D", "m"] else "s" expected = TimedeltaIndex( [np.timedelta64(i, np_unit) for i in np.arange(5).tolist()], - dtype="m8[ns]", + dtype=f"m8[{exp_unit}]", ) result = to_timedelta(wrapper(range(5)), unit=unit) diff --git a/pandas/tests/tools/test_to_datetime.py b/pandas/tests/tools/test_to_datetime.py index c786c01b58292..1ed82e3f68517 100644 --- a/pandas/tests/tools/test_to_datetime.py +++ b/pandas/tests/tools/test_to_datetime.py @@ -2116,7 +2116,7 @@ def test_dataframe_field_aliases_column_subset(self, df, cache, unit): result = to_datetime(df[list(unit.keys())].rename(columns=unit), cache=cache) expected = Series( [Timestamp("20150204 06:58:10"), Timestamp("20160305 07:59:11")], - dtype="M8[ns]", + dtype="M8[us]", ) tm.assert_series_equal(result, expected) diff --git a/pandas/tests/tools/test_to_timedelta.py b/pandas/tests/tools/test_to_timedelta.py index 878f9ecf79ef1..ad63106c36392 100644 --- a/pandas/tests/tools/test_to_timedelta.py +++ b/pandas/tests/tools/test_to_timedelta.py @@ -104,7 +104,7 @@ def test_to_timedelta_units(self): result = TimedeltaIndex( [np.timedelta64(0, "ns"), np.timedelta64(10, "s").astype("m8[ns]")] ) - expected = to_timedelta([0, 10], unit="s") + expected = to_timedelta([0, 10], unit="s").as_unit("ns") tm.assert_index_equal(result, expected) @pytest.mark.parametrize( @@ -121,7 +121,7 @@ def test_to_timedelta_units_dtypes(self, dtype, unit): # arrays of various dtypes arr = np.array([1] * 5, dtype=dtype) result = to_timedelta(arr, unit=unit) - exp_dtype = "m8[ns]" if dtype == "int64" else "m8[s]" + exp_dtype = "m8[s]" expected = TimedeltaIndex([np.timedelta64(1, unit)] * 5, dtype=exp_dtype) tm.assert_index_equal(result, expected) @@ -277,7 +277,7 @@ def test_to_timedelta_coerce_strings_unit(self): ) def test_to_timedelta_nullable_int64_dtype(self, expected_val, result_val): # GH 35574 - expected = Series([timedelta(days=1), expected_val], dtype="m8[ns]") + expected = Series([timedelta(days=1), expected_val], dtype="m8[s]") result = to_timedelta(Series([1, result_val], dtype="Int64"), unit="days") tm.assert_series_equal(result, expected)