diff --git a/.github/workflows/test-pypi-release.yml b/.github/workflows/test-pypi-release.yml new file mode 100644 index 0000000000..a1f5fdb3ec --- /dev/null +++ b/.github/workflows/test-pypi-release.yml @@ -0,0 +1,105 @@ +--- +name: Test PyPI Release + +on: + workflow_dispatch: + push: + tags: + - 'v*' + - 'test-release-*' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build_artifacts: + name: Build distribution files + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + submodules: true + fetch-depth: 0 + + - uses: actions/setup-python@v6 + name: Install Python + with: + python-version: '3.11' + + - name: Install Hatch + uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc + with: + version: '1.16.5' + + - name: Build wheel and sdist + run: hatch build + + - uses: actions/upload-artifact@v7 + with: + name: distribution + path: dist + + test_testpypi_upload: + needs: build_artifacts + runs-on: ubuntu-latest + environment: + name: testpypi + url: https://test.pypi.org/p/zarr + + steps: + - uses: actions/download-artifact@v7 + with: + name: distribution + path: dist + + - name: List artifacts + run: ls -la dist/ + + - name: Publish package to TestPyPI + uses: pypa/gh-action-pypi-publish@v1.13.0 + with: + repository-url: https://test.pypi.org/legacy/ + password: ${{ secrets.TESTPYPI_API_TOKEN }} + + test_testpypi_install: + needs: test_testpypi_upload + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.9', '3.11'] + fail-fast: true + + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + + - name: Install from TestPyPI + run: | + python -m pip install --index-url https://test.pypi.org/simple/ \ + --extra-index-url https://pypi.org/simple/ \ + zarr + + - name: Smoke test + run: | + python -c " + import zarr + print(f'zarr version: {zarr.__version__}') + print(f'zarr location: {zarr.__file__}') + + # Basic functionality test + store = zarr.MemoryStore() + root = zarr.open_group(store=store, mode='w') + array = root.create_dataset('test', data=[1, 2, 3]) + assert len(array) == 3, 'Failed to create/read dataset' + print('✓ Basic zarr operations work correctly') + " + + - name: Print success message + run: echo "✓ TestPyPI installation and smoke tests passed!" diff --git a/changes/3796.bugfix.md b/changes/3796.bugfix.md new file mode 100644 index 0000000000..bd41113293 --- /dev/null +++ b/changes/3796.bugfix.md @@ -0,0 +1,3 @@ +Fix ValueError when matching generic dtype instances (UIntDtype, IntDtype) to specific Zarr integer data types on Windows. + +When numpy's bitwise operations produce generic unsigned/signed integer dtype instances instead of specifically-sized ones (e.g., UIntDtype instead of UInt32DType), the dtype matching logic now correctly identifies and handles these cases. This resolves the issue where `zarr.array(np.array([1, 2], dtype=np.uint32) & 1)` would fail with "No Zarr data type found that matches dtype" on Windows. diff --git a/changes/3798.feature.md b/changes/3798.feature.md new file mode 100644 index 0000000000..43c7e386ce --- /dev/null +++ b/changes/3798.feature.md @@ -0,0 +1 @@ +Add GitHub Actions workflow to test distributions on TestPyPI before releases. This workflow validates the package build process, ensures uploads work correctly, and confirms installation from TestPyPI succeeds across multiple Python versions, catching packaging issues early. diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index b82c77fa9c..330c62aab1 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -1039,6 +1039,38 @@ def shape(self) -> tuple[int, ...]: """ return self.metadata.shape + def __len__(self) -> int: + """Return the length of the first dimension. + + Matches numpy behavior: returns shape[0] for dimensioned arrays, + raises TypeError for 0-dimensional arrays. + + Returns + ------- + int + The size of the first dimension. + + Raises + ------ + TypeError + If the array is 0-dimensional (empty shape). + + Examples + -------- + >>> import zarr + >>> a = zarr.zeros((5, 10)) + >>> len(a) + 5 + >>> b = zarr.zeros(()) + >>> len(b) # doctest: +SKIP + Traceback (most recent call last): + ... + TypeError: len() of unsized object + """ + if self.ndim == 0: + raise TypeError("len() of unsized object") + return self.shape[0] + @property def chunks(self) -> tuple[int, ...]: """Returns the chunk shape of the Array. @@ -2263,6 +2295,36 @@ def shape(self, value: tuple[int, ...]) -> None: """Sets the shape of the array by calling resize.""" self.resize(value) + def __len__(self) -> int: + """Return the length of the first dimension. + + Matches numpy behavior: returns shape[0] for dimensioned arrays, + raises TypeError for 0-dimensional arrays. + + Returns + ------- + int + The size of the first dimension. + + Raises + ------ + TypeError + If the array is 0-dimensional (empty shape). + + Examples + -------- + >>> import zarr + >>> a = zarr.zeros((5, 10)) + >>> len(a) + 5 + >>> b = zarr.zeros(()) + >>> len(b) # doctest: +SKIP + Traceback (most recent call last): + ... + TypeError: len() of unsized object + """ + return self.async_array.__len__() + @property def chunks(self) -> tuple[int, ...]: """Returns a tuple of integers describing the length of each dimension of a chunk of the array. diff --git a/src/zarr/core/dtype/npy/int.py b/src/zarr/core/dtype/npy/int.py index f71f535abb..e6288ae054 100644 --- a/src/zarr/core/dtype/npy/int.py +++ b/src/zarr/core/dtype/npy/int.py @@ -563,6 +563,32 @@ class Int16(BaseInt[np.dtypes.Int16DType, np.int16], HasEndianness): _zarr_v3_name: ClassVar[Literal["int16"]] = "int16" _zarr_v2_names: ClassVar[tuple[Literal[">i2"], Literal["i2", " TypeGuard[np.dtypes.Int16DType]: + """ + A type guard that checks if the input is assignable to the type of ``cls.dtype_class`` + + This method is overridden for this particular data type because of a Windows-specific issue + where np.dtype('i') can create an instance of ``np.dtypes.IntDtype``, rather than an + instance of ``np.dtypes.Int16DType``, even though both represent 16-bit signed integers. + + Parameters + ---------- + dtype : TDType + The dtype to check. + + Returns + ------- + Bool + True if the dtype matches, False otherwise. + """ + return super()._check_native_dtype(dtype) or ( + hasattr(dtype, "itemsize") + and hasattr(dtype, "kind") + and dtype.itemsize == 2 + and dtype.kind == "i" + ) + @classmethod def from_native_dtype(cls, dtype: TBaseDType) -> Self: """ @@ -725,6 +751,32 @@ class UInt16(BaseInt[np.dtypes.UInt16DType, np.uint16], HasEndianness): _zarr_v3_name: ClassVar[Literal["uint16"]] = "uint16" _zarr_v2_names: ClassVar[tuple[Literal[">u2"], Literal["u2", " TypeGuard[np.dtypes.UInt16DType]: + """ + A type guard that checks if the input is assignable to the type of ``cls.dtype_class`` + + This method is overridden for this particular data type because of a Windows-specific issue + where np.dtype('u') can create an instance of ``np.dtypes.UIntDtype``, rather than an + instance of ``np.dtypes.UInt16DType``, even though both represent 16-bit unsigned integers. + + Parameters + ---------- + dtype : TDType + The dtype to check. + + Returns + ------- + Bool + True if the dtype matches, False otherwise. + """ + return super()._check_native_dtype(dtype) or ( + hasattr(dtype, "itemsize") + and hasattr(dtype, "kind") + and dtype.itemsize == 2 + and dtype.kind == "u" + ) + @classmethod def from_native_dtype(cls, dtype: TBaseDType) -> Self: """ @@ -906,7 +958,12 @@ def _check_native_dtype(cls: type[Self], dtype: TBaseDType) -> TypeGuard[np.dtyp Bool True if the dtype matches, False otherwise. """ - return super()._check_native_dtype(dtype) or dtype == np.dtypes.Int32DType() + return super()._check_native_dtype(dtype) or ( + hasattr(dtype, "itemsize") + and hasattr(dtype, "kind") + and dtype.itemsize == 4 + and dtype.kind == "i" + ) @classmethod def from_native_dtype(cls: type[Self], dtype: TBaseDType) -> Self: @@ -1070,6 +1127,32 @@ class UInt32(BaseInt[np.dtypes.UInt32DType, np.uint32], HasEndianness): _zarr_v3_name: ClassVar[Literal["uint32"]] = "uint32" _zarr_v2_names: ClassVar[tuple[Literal[">u4"], Literal["u4", " TypeGuard[np.dtypes.UInt32DType]: + """ + A type guard that checks if the input is assignable to the type of ``cls.dtype_class`` + + This method is overridden for this particular data type because of a Windows-specific issue + where np.dtype('u') can create an instance of ``np.dtypes.UIntDtype``, rather than an + instance of ``np.dtypes.UInt32DType``, even though both represent 32-bit unsigned integers. + + Parameters + ---------- + dtype : TDType + The dtype to check. + + Returns + ------- + Bool + True if the dtype matches, False otherwise. + """ + return super()._check_native_dtype(dtype) or ( + hasattr(dtype, "itemsize") + and hasattr(dtype, "kind") + and dtype.itemsize == 4 + and dtype.kind == "u" + ) + @classmethod def from_native_dtype(cls, dtype: TBaseDType) -> Self: """ @@ -1228,6 +1311,32 @@ class Int64(BaseInt[np.dtypes.Int64DType, np.int64], HasEndianness): _zarr_v3_name: ClassVar[Literal["int64"]] = "int64" _zarr_v2_names: ClassVar[tuple[Literal[">i8"], Literal["i8", " TypeGuard[np.dtypes.Int64DType]: + """ + A type guard that checks if the input is assignable to the type of ``cls.dtype_class`` + + This method is overridden for this particular data type because of a Windows-specific issue + where np.dtype('i') can create an instance of ``np.dtypes.IntDtype``, rather than an + instance of ``np.dtypes.Int64DType``, even though both represent 64-bit signed integers. + + Parameters + ---------- + dtype : TDType + The dtype to check. + + Returns + ------- + Bool + True if the dtype matches, False otherwise. + """ + return super()._check_native_dtype(dtype) or ( + hasattr(dtype, "itemsize") + and hasattr(dtype, "kind") + and dtype.itemsize == 8 + and dtype.kind == "i" + ) + @classmethod def from_native_dtype(cls, dtype: TBaseDType) -> Self: """ @@ -1481,6 +1590,32 @@ def to_json( return self._zarr_v3_name raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + @classmethod + def _check_native_dtype(cls: type[Self], dtype: TBaseDType) -> TypeGuard[np.dtypes.UInt64DType]: + """ + A type guard that checks if the input is assignable to the type of ``cls.dtype_class`` + + This method is overridden for this particular data type because of a Windows-specific issue + where np.dtype('u') can create an instance of ``np.dtypes.UIntDtype``, rather than an + instance of ``np.dtypes.UInt64DType``, even though both represent 64-bit unsigned integers. + + Parameters + ---------- + dtype : TDType + The dtype to check. + + Returns + ------- + Bool + True if the dtype matches, False otherwise. + """ + return super()._check_native_dtype(dtype) or ( + hasattr(dtype, "itemsize") + and hasattr(dtype, "kind") + and dtype.itemsize == 8 + and dtype.kind == "u" + ) + @classmethod def from_native_dtype(cls, dtype: TBaseDType) -> Self: """ diff --git a/tests/test_array.py b/tests/test_array.py index 5b85c6ba1d..a8d912b48c 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -2299,3 +2299,31 @@ def test_with_config_polymorphism() -> None: arr_source_config_dict = arr.with_config(source_config_dict) assert arr_source_config.config == arr_source_config_dict.config + + +@pytest.mark.parametrize("shape", [(10,), (5, 10), (3, 4, 5), (2, 3, 4, 5)]) +def test_array_len_dimensioned(shape: tuple[int, ...]) -> None: + """Test __len__ for dimensioned arrays returns shape[0].""" + arr = zarr.create_array({}, shape=shape, dtype="uint8") + assert len(arr) == shape[0] + + +@pytest.mark.parametrize("shape", [(10,), (5, 10), (3, 4, 5)]) +async def test_array_len_dimensioned_async(shape: tuple[int, ...]) -> None: + """Test __len__ for async dimensioned arrays returns shape[0].""" + arr = await AsyncArray.create({}, shape=shape, dtype="uint8") + assert len(arr) == shape[0] + + +def test_array_len_0d_raises() -> None: + """Test __len__ raises TypeError for 0-dimensional arrays.""" + arr = zarr.create_array({}, shape=(), dtype="uint8") + with pytest.raises(TypeError, match="len\\(\\) of unsized object"): + len(arr) + + +async def test_array_len_0d_raises_async() -> None: + """Test __len__ raises TypeError for async 0-dimensional arrays.""" + arr = await AsyncArray.create({}, shape=(), dtype="uint8") + with pytest.raises(TypeError, match="len\\(\\) of unsized object"): + len(arr) diff --git a/tests/test_issue_3796_dtype_matching.py b/tests/test_issue_3796_dtype_matching.py new file mode 100644 index 0000000000..1c92744d33 --- /dev/null +++ b/tests/test_issue_3796_dtype_matching.py @@ -0,0 +1,175 @@ +""" +Tests for issue #3796: ValueError on dtype matching (Windows-specific issue with generic dtypes). + +This test suite verifies that the dtype matching logic correctly handles cases where +numpy's bitwise operations produce generic dtype classes (like UIntDtype, IntDtype) +instead of specific sized types (like UInt32DType, Int32DType), which happens on Windows. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +from zarr.core.dtype.npy.int import Int16, Int32, Int64, UInt16, UInt32, UInt64 + + +class TestDtypeMatching: + """Test dtype matching for integer types with generic numpy dtypes (Windows issue).""" + + def test_uint32_from_normal_array(self) -> None: + """Test that UInt32 correctly matches a normal uint32 numpy array.""" + arr = np.array([1, 2], dtype=np.uint32) + zdtype = UInt32.from_native_dtype(arr.dtype) + assert isinstance(zdtype, UInt32) + assert zdtype.to_native_dtype().itemsize == 4 + + def test_uint32_from_bitwise_operation(self) -> None: + """ + Test that UInt32 correctly matches uint32 from bitwise operations. + + On Windows, bitwise operations on uint32 can produce UIntDtype instead of UInt32DType. + This test verifies that our fix handles this case. + """ + arr = np.array([1, 2], dtype=np.uint32) & 1 + # The dtype might be UInt32DType or UIntDtype depending on OS/numpy version + assert arr.dtype.itemsize == 4 + assert np.issubdtype(arr.dtype, np.unsignedinteger) + + # This should not raise ValueError + zdtype = UInt32.from_native_dtype(arr.dtype) + assert isinstance(zdtype, UInt32) + + def test_uint16_from_bitwise_operation(self) -> None: + """Test that UInt16 correctly matches uint16 from bitwise operations.""" + arr = np.array([1, 2], dtype=np.uint16) & 1 + assert arr.dtype.itemsize == 2 + assert np.issubdtype(arr.dtype, np.unsignedinteger) + + zdtype = UInt16.from_native_dtype(arr.dtype) + assert isinstance(zdtype, UInt16) + + def test_uint64_from_bitwise_operation(self) -> None: + """Test that UInt64 correctly matches uint64 from bitwise operations.""" + arr = np.array([1, 2], dtype=np.uint64) & 1 + assert arr.dtype.itemsize == 8 + assert np.issubdtype(arr.dtype, np.unsignedinteger) + + zdtype = UInt64.from_native_dtype(arr.dtype) + assert isinstance(zdtype, UInt64) + + def test_int32_from_bitwise_operation(self) -> None: + """Test that Int32 correctly matches int32 from bitwise operations.""" + arr = np.array([1, 2], dtype=np.int32) & 1 + assert arr.dtype.itemsize == 4 + assert np.issubdtype(arr.dtype, np.signedinteger) + + zdtype = Int32.from_native_dtype(arr.dtype) + assert isinstance(zdtype, Int32) + + def test_int16_from_bitwise_operation(self) -> None: + """Test that Int16 correctly matches int16 from bitwise operations.""" + arr = np.array([1, 2], dtype=np.int16) & 1 + assert arr.dtype.itemsize == 2 + assert np.issubdtype(arr.dtype, np.signedinteger) + + zdtype = Int16.from_native_dtype(arr.dtype) + assert isinstance(zdtype, Int16) + + def test_int64_from_bitwise_operation(self) -> None: + """Test that Int64 correctly matches int64 from bitwise operations.""" + arr = np.array([1, 2], dtype=np.int64) & 1 + assert arr.dtype.itemsize == 8 + assert np.issubdtype(arr.dtype, np.signedinteger) + + zdtype = Int64.from_native_dtype(arr.dtype) + assert isinstance(zdtype, Int64) + + def test_uint32_with_different_endianness(self) -> None: + """Test that UInt32 correctly matches uint32 with different endianness.""" + # Test native endianness + arr_native = np.array([1, 2], dtype=np.uint32) + zdtype_native = UInt32.from_native_dtype(arr_native.dtype) + assert isinstance(zdtype_native, UInt32) + + # Test little-endian + arr_le = np.array([1, 2], dtype=" None: + """Test that creating and converting back to native dtype works for UInt32.""" + zdtype = UInt32() + native_dtype = zdtype.to_native_dtype() + zdtype_again = UInt32.from_native_dtype(native_dtype) + assert isinstance(zdtype_again, UInt32) + assert zdtype_again.to_native_dtype().itemsize == 4 + + +class TestDtypeMatchingWithZarr: + """Test dtype matching through the zarr.array() API.""" + + def test_zarr_array_from_uint32_bitwise(self) -> None: + """Test that zarr.array() works with uint32 from bitwise operations.""" + import zarr + + arr = np.array([1, 2], dtype=np.uint32) & 1 + # This should not raise ValueError + z = zarr.array(arr) + assert z.dtype == np.dtype("uint32") + assert z.shape == (2,) + + def test_zarr_array_from_uint16_bitwise(self) -> None: + """Test that zarr.array() works with uint16 from bitwise operations.""" + import zarr + + arr = np.array([1, 2], dtype=np.uint16) & 1 + z = zarr.array(arr) + assert z.dtype == np.dtype("uint16") + assert z.shape == (2,) + + def test_zarr_array_from_int32_bitwise(self) -> None: + """Test that zarr.array() works with int32 from bitwise operations.""" + import zarr + + arr = np.array([1, 2], dtype=np.int32) & 1 + z = zarr.array(arr) + assert z.dtype == np.dtype("int32") + assert z.shape == (2,) + + +class TestErrorCases: + """Test that invalid dtypes still raise appropriate errors.""" + + def test_uint32_rejects_wrong_size(self) -> None: + """Test that UInt32 rejects dtypes with wrong itemsize.""" + # Create a dtype with wrong size - this is artificial, + # as numpy doesn't naturally create such dtypes + arr_correct = np.array([1, 2], dtype=np.uint32) + arr_wrong = np.array([1, 2], dtype=np.uint16) + + # This should work + UInt32.from_native_dtype(arr_correct.dtype) + + # This should raise + with pytest.raises(Exception): # Could be DataTypeValidationError or ValueError + UInt32.from_native_dtype(arr_wrong.dtype) + + def test_uint32_rejects_signed_integer(self) -> None: + """Test that UInt32 rejects signed integer dtypes.""" + arr_signed = np.array([1, 2], dtype=np.int32) + + with pytest.raises(Exception): # Could be DataTypeValidationError or ValueError + UInt32.from_native_dtype(arr_signed.dtype) + + def test_int32_rejects_unsigned_integer(self) -> None: + """Test that Int32 rejects unsigned integer dtypes.""" + arr_unsigned = np.array([1, 2], dtype=np.uint32) + + with pytest.raises(Exception): # Could be DataTypeValidationError or ValueError + Int32.from_native_dtype(arr_unsigned.dtype)