From 48ee7d56dffcaf276fdd0f3f76ba65e7b4bbe4ad Mon Sep 17 00:00:00 2001 From: James Parrott <80779630+JamesParrott@users.noreply.github.com> Date: Wed, 22 Jul 2026 10:38:51 +0100 Subject: [PATCH 1/4] Follow all .strftime("%Y%m%d")s with .zfill(8) --- .github/workflows/run_checks_build_and_test.yml | 11 ++++++++++- README.md | 8 ++++++-- changelog.txt | 4 ++++ src/shapefile.py | 8 +++++--- tests/hypothesis_tests.py | 9 ++++++--- 5 files changed, 31 insertions(+), 9 deletions(-) diff --git a/.github/workflows/run_checks_build_and_test.yml b/.github/workflows/run_checks_build_and_test.yml index 05657c00..3067d3c1 100644 --- a/.github/workflows/run_checks_build_and_test.yml +++ b/.github/workflows/run_checks_build_and_test.yml @@ -58,10 +58,20 @@ jobs: matrix: python-version: [ "3.14", + "3.13", + "3.12", + "3.15-dev", + "3.11", + "3.10", + "3.9", ] os: [ "ubuntu-24.04", ] + include: + - python-version: "3.14" + - os: "ubuntu-latest" + runs-on: ${{ matrix.os }} steps: - uses: actions/setup-python@v6 @@ -73,7 +83,6 @@ jobs: path: ./Pyshp - name: "Hypothesis tests" - if: uses: ./Pyshp/.github/actions/test with: extra_args: '-m hypothesis' diff --git a/README.md b/README.md index 8899510c..9ad10069 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ The Python Shapefile Library (PyShp) reads and writes ESRI Shapefiles in pure Py - **Author**: [Joel Lawhead](https://github.com/GeospatialPython) - **Maintainers**: [James Parrott](https://github.com/JamesParrott) & [Karim Bahgat](https://github.com/karimbahgat) -- **Version**: 3.1.4 -- **Date**: 29th June 2026 +- **Version**: 3.1.5 +- **Date**: 22nd July 2026 - **License**: [MIT](https://github.com/GeospatialPython/pyshp/blob/master/LICENSE.TXT) ## Contents @@ -93,6 +93,10 @@ part of your geospatial project. # Version Changes +## 3.1.5 +### Bug fix + - Fixed another bug causing dates before the year 1000 to be encoded as less than 8 chars under "%Y%m%d (found by [Thomas Beierlein](https://github.com/GeospatialPython/pyshp/issues/435)) + ## 3.1.4 ### Bug fix - Fix bug causing dates supplied as length 8 strings of digits to be encoded by the custom encoding, not ascii. diff --git a/changelog.txt b/changelog.txt index 3d46e338..3e0b2b32 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,7 @@ +VERSION 3.1.5 +2026-07-22 + * Fixed another bug causing dates before the year 1000 to be encoded as less than 8 chars under "%Y%m%d (found by [Thomas Beierlein](https://github.com/GeospatialPython/pyshp/issues/435)) + VERSION 3.1.4 2026-06-29 diff --git a/src/shapefile.py b/src/shapefile.py index 547a4c0a..2b942a0b 100644 --- a/src/shapefile.py +++ b/src/shapefile.py @@ -8,7 +8,7 @@ from __future__ import annotations -__version__ = "3.1.4" +__version__ = "3.1.5" import abc import array @@ -4430,8 +4430,10 @@ def _record(self, record: list[RecordValue]) -> None: # date: 8 bytes - date stored as a string in the format YYYYMMDD. if isinstance(value, list) and len(value) == 3: value = date(*value) - if isinstance(value, date): - str_val = value.strftime("%Y%m%d") + if isinstance(value, date): + # In Pythons using certain glibc versions. + # date.strftime does not preppend zeros. + str_val = value.strftime("%Y%m%d").zfill(8) # b"".join(ord(c).to_bytes() for c in s) elif value in MISSING: str_val = "0" * 8 # QGIS NULL for date type diff --git a/tests/hypothesis_tests.py b/tests/hypothesis_tests.py index 8b6381b8..7fb44bcc 100644 --- a/tests/hypothesis_tests.py +++ b/tests/hypothesis_tests.py @@ -688,6 +688,9 @@ def test_dbf_Field_roundtrips(encoding_and_dbf_field: dict) -> None: ascii_printable = string.ascii_letters + string.digits + string.punctuation + " " +def date_to_str(d: datetime.date) -> str: + return d.strftime("%Y%m%d").zfill(8) + def record_value_for_field(name: str, field_type: str, size: int, decimal: int, encoding: str): if field_type == "C": @@ -715,7 +718,7 @@ def record_value_for_field(name: str, field_type: str, size: int, decimal: int, if field_type == "L": return sampled_from([True, False, None]) if field_type == "D": - return one_of(dates(), dates().map(lambda d: d.strftime("%Y%m%d"))) + return one_of(dates(), dates().map(date_to_str)) raise ValueError(f"Unsupported: {field_type=}") @@ -771,9 +774,9 @@ def _assert_reader_matches_expected_records(r, fields, written_records): decimal = field["decimal"] if field_type == "D": if isinstance(expected, datetime.date): - expected = expected.strftime("%Y%m%d") + expected = date_to_str(expected) if isinstance(actual, datetime.date): - actual = actual.strftime("%Y%m%d") + actual = date_to_str(actual) elif field_type in ("N", "F") and decimal >= 1: expected = float(format(expected, f".{decimal}f")) assert actual == expected, f"{actual=}, {expected=}, {field_type=}, {type(actual)=}, {type(expected)=}" From f2c8b27e441cbe6a02c4258d33cf2a14de5701e3 Mon Sep 17 00:00:00 2001 From: James Parrott <80779630+JamesParrott@users.noreply.github.com> Date: Wed, 22 Jul 2026 11:16:23 +0100 Subject: [PATCH 2/4] Try testing text fields against all valid code points for the codec --- .../workflows/run_checks_build_and_test.yml | 2 +- src/shapefile.py | 4 +- tests/hypothesis_tests.py | 37 +++++++++---------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/.github/workflows/run_checks_build_and_test.yml b/.github/workflows/run_checks_build_and_test.yml index 3067d3c1..429c97cf 100644 --- a/.github/workflows/run_checks_build_and_test.yml +++ b/.github/workflows/run_checks_build_and_test.yml @@ -71,7 +71,7 @@ jobs: include: - python-version: "3.14" - os: "ubuntu-latest" - + runs-on: ${{ matrix.os }} steps: - uses: actions/setup-python@v6 diff --git a/src/shapefile.py b/src/shapefile.py index 2b942a0b..718f2ef4 100644 --- a/src/shapefile.py +++ b/src/shapefile.py @@ -4430,9 +4430,9 @@ def _record(self, record: list[RecordValue]) -> None: # date: 8 bytes - date stored as a string in the format YYYYMMDD. if isinstance(value, list) and len(value) == 3: value = date(*value) - if isinstance(value, date): + if isinstance(value, date): # In Pythons using certain glibc versions. - # date.strftime does not preppend zeros. + # date.strftime does not preppend zeros. str_val = value.strftime("%Y%m%d").zfill(8) # b"".join(ord(c).to_bytes() for c in s) elif value in MISSING: diff --git a/tests/hypothesis_tests.py b/tests/hypothesis_tests.py index 7fb44bcc..67904573 100644 --- a/tests/hypothesis_tests.py +++ b/tests/hypothesis_tests.py @@ -66,6 +66,7 @@ def ignore_warnings(category=None): ) + def coords_2D_list( min_size: int = 1, max_size: int | None = None, @@ -76,6 +77,17 @@ def coords_2D_list( max_size=max_size, ) +def strings_of_supported_code_points(encoding: str, min_size: int=1, max_size: int=10): + return text( + alphabet=characters( + codec=encoding, + # https://en.wikipedia.org/wiki/Unicode_character_property#General_Category + exclude_categories=["Cs", "Co", "Cn"], # Cs - surrogates + # exclude_characters=[" "], + ), + min_size=min_size, + max_size=max_size, + ) @pytest.mark.hypothesis @given(expected=point_2D, i=integers(min_value=1)) @@ -570,6 +582,8 @@ def _encodings() -> set[str]: for enc in aliases.values(): if enc in encs: continue + # I'm not sure why any encoding would fail on an empty string, + # but I'd rather not have the tests get bogged by any that do exist. try: "".encode(enc) except (UnicodeEncodeError, LookupError): @@ -593,18 +607,7 @@ def _encodings() -> set[str]: def _dbf_fields_strategy(draw, encoding: str) -> dict[str, str | int]: field_type, bounds_dict = draw(sampled_from(list(DBF_FIELD_TYPES.items()))) - name = draw( - text( - alphabet=characters( - codec=encoding, - # https://en.wikipedia.org/wiki/Unicode_character_property#General_Category - exclude_categories=["Cs", "Co", "Cn"], # Cs - surrogates - # exclude_characters=[" "], - ), - min_size=1, - max_size=10, - ) - ) + name = draw(strings_of_supported_code_points(encoding)) max_length = bounds_dict.get("max_length", 254) min_length = bounds_dict.get("min_length", 1) @@ -691,14 +694,10 @@ def test_dbf_Field_roundtrips(encoding_and_dbf_field: dict) -> None: def date_to_str(d: datetime.date) -> str: return d.strftime("%Y%m%d").zfill(8) -def record_value_for_field(name: str, field_type: str, size: int, decimal: int, encoding: str): +def record_value_strat_for_field(name: str, field_type: str, size: int, decimal: int, encoding: str): if field_type == "C": - return text( - alphabet=ascii_printable, - min_size=0, - max_size=size, - ) + return strings_of_supported_code_points(encoding, 0, size) if field_type in {"N", "F"}: int_digits = size if decimal == 0 else size - decimal - 1 @@ -732,7 +731,7 @@ def _dbf_encoding_fields_and_record_strategy( fields = draw(lists(_dbf_fields_strategy(encoding), min_size=1, max_size=max_fields)) - record_strategy = tuples(*(record_value_for_field(encoding=encoding, **field) for field in fields)) + record_strategy = tuples(*(record_value_strat_for_field(encoding=encoding, **field) for field in fields)) return encoding, fields, record_strategy From 267a00ca5d28c652a2007a39ce09864735188c87 Mon Sep 17 00:00:00 2001 From: James Parrott <80779630+JamesParrott@users.noreply.github.com> Date: Wed, 22 Jul 2026 12:24:29 +0100 Subject: [PATCH 3/4] Exclude ISO 2022 control char --- tests/hypothesis_tests.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/tests/hypothesis_tests.py b/tests/hypothesis_tests.py index 67904573..8d61b5fd 100644 --- a/tests/hypothesis_tests.py +++ b/tests/hypothesis_tests.py @@ -77,13 +77,42 @@ def coords_2D_list( max_size=max_size, ) +EXCLUDE_CHARS = [ + ("iso2022", "\x1b") +] +""" Pending solution to (not sure if bug in hypothesis generation or core library): +Python 3.13.14 (main, Jul 18 2026, 17:02:37) [Clang 22.1.3 ] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> enc="iso2022_jp_1" +>>> s="\x1b" +>>> b=s.encode(enc) +>>> b +b'\x1b' +>>> b.decode(enc) +Traceback (most recent call last): + File "", line 1, in + b.decode(enc) + ~~~~~~~~^^^^^ +UnicodeDecodeError: 'iso2022_jp_1' codec can't decode byte 0x1b in position 0: incomplete multibyte sequence +decoding with 'iso2022_jp_1' codec failed + +""" + + def strings_of_supported_code_points(encoding: str, min_size: int=1, max_size: int=10): + + for prefix, exclude_chars in EXCLUDE_CHARS: + if encoding.startswith(prefix): + break + else: + exclude_chars = "" + return text( alphabet=characters( codec=encoding, # https://en.wikipedia.org/wiki/Unicode_character_property#General_Category exclude_categories=["Cs", "Co", "Cn"], # Cs - surrogates - # exclude_characters=[" "], + exclude_characters=exclude_chars, ), min_size=min_size, max_size=max_size, From 15e221b0a3c4e1775ef0867937c7d36ec22fc466 Mon Sep 17 00:00:00 2001 From: James Parrott <80779630+JamesParrott@users.noreply.github.com> Date: Wed, 22 Jul 2026 13:40:00 +0100 Subject: [PATCH 4/4] Don't include known non-round trippable chars in round trip tests --- tests/hypothesis_tests.py | 144 ++++++++++++++++++++++++++------------ 1 file changed, 98 insertions(+), 46 deletions(-) diff --git a/tests/hypothesis_tests.py b/tests/hypothesis_tests.py index 8d61b5fd..3b200952 100644 --- a/tests/hypothesis_tests.py +++ b/tests/hypothesis_tests.py @@ -2,6 +2,7 @@ import contextlib import datetime +import functools import io import itertools import os @@ -77,46 +78,7 @@ def coords_2D_list( max_size=max_size, ) -EXCLUDE_CHARS = [ - ("iso2022", "\x1b") -] -""" Pending solution to (not sure if bug in hypothesis generation or core library): -Python 3.13.14 (main, Jul 18 2026, 17:02:37) [Clang 22.1.3 ] on linux -Type "help", "copyright", "credits" or "license" for more information. ->>> enc="iso2022_jp_1" ->>> s="\x1b" ->>> b=s.encode(enc) ->>> b -b'\x1b' ->>> b.decode(enc) -Traceback (most recent call last): - File "", line 1, in - b.decode(enc) - ~~~~~~~~^^^^^ -UnicodeDecodeError: 'iso2022_jp_1' codec can't decode byte 0x1b in position 0: incomplete multibyte sequence -decoding with 'iso2022_jp_1' codec failed - -""" - - -def strings_of_supported_code_points(encoding: str, min_size: int=1, max_size: int=10): - - for prefix, exclude_chars in EXCLUDE_CHARS: - if encoding.startswith(prefix): - break - else: - exclude_chars = "" - return text( - alphabet=characters( - codec=encoding, - # https://en.wikipedia.org/wiki/Unicode_character_property#General_Category - exclude_categories=["Cs", "Co", "Cn"], # Cs - surrogates - exclude_characters=exclude_chars, - ), - min_size=min_size, - max_size=max_size, - ) @pytest.mark.hypothesis @given(expected=point_2D, i=integers(min_value=1)) @@ -604,12 +566,53 @@ def test_shx_reader_writer_roundtrip(codes_and_shapes)-> None: "utf-32-le", "cp1140", ] +POSSIBLY_NONINJECTIVE_CODECS = frozenset({ + 'big5', 'big5hkscs', 'cp932', 'cp936', 'cp949', 'cp950', + 'euc_jp', 'euc_jis_2004', 'euc_kr', 'gb2312', 'gbk', 'gb18030', + 'hz', 'iso2022_jp', 'iso2022_kr', 'shift_jis', 'shift_jis_2004' +}) + +@functools.cache +def _exclude_chars() -> dict[str,list[str]]: + + exclude_chars = {} + exclude_chars["iso2022"].append("\x1b") + """ + Not sure if bug in hypothesis generation, in core library, or just the way it is: + + Python taking a short cut with Control characters in ISO_2022 stateful codecs + Python 3.13.14 (main, Jul 18 2026, 17:02:37) [Clang 22.1.3 ] on linux + Type "help", "copyright", "credits" or "license" for more information. + >>> enc="iso2022_jp_1" + >>> s="\x1b" + >>> b=s.encode(enc) + >>> b + b'\x1b' + >>> b.decode(enc) + Traceback (most recent call last): + File "", line 1, in + b.decode(enc) + ~~~~~~~~^^^^^ + UnicodeDecodeError: 'iso2022_jp_1' codec can't decode byte 0x1b in position 0: incomplete multibyte sequence + decoding with 'iso2022_jp_1' codec failed + + Non injective encodings (dealt with below) + >>> enc="cp950" + >>> "•".encode(enc) + b'\xa1E' + >>> b=_ + >>> b.decode(enc) + '‧' + >>> s = _ + >>> s.encode(enc) + b'\xa1E' + + """ + -def _encodings() -> set[str]: from encodings.aliases import aliases - encs = set() for enc in aliases.values(): - if enc in encs: + if enc in exclude_chars: continue # I'm not sure why any encoding would fail on an empty string, # but I'd rather not have the tests get bogged by any that do exist. @@ -617,9 +620,37 @@ def _encodings() -> set[str]: "".encode(enc) except (UnicodeEncodeError, LookupError): continue - encs.add(enc) - return encs -# assert _encodings() == {'utf_16_le', 'iso8859_7', 'cp437', 'iso2022_jp_3', 'shift_jis', 'cp775', 'cp1140', + + exclude_chars[enc] = [] + + if enc not in POSSIBLY_NONINJECTIVE_CODECS: + continue + + # find collisions of non-injective codecs + # Iterate over BMP + for code_point in range(0x10000): + char = chr(code_point) + try: + b = char.encode(enc) + except (UnicodeEncodeError, LookupError): + exclude_chars[enc].append(char) + continue + + decoded = None + try: + decoded = b.decode(enc) + except (UnicodeDecodeError, LookupError): + exclude_chars[enc].append(char) + continue + + if decoded != char: + exclude_chars[enc].append(char) + + + + + return exclude_chars +# assert set(_encodings()) == {'utf_16_le', 'iso8859_7', 'cp437', 'iso2022_jp_3', 'shift_jis', 'cp775', 'cp1140', # 'cp861', 'iso8859_11', 'iso8859_9', 'euc_jp', 'utf_16', 'cp950', 'mac_cyrillic', 'mac_turkish', 'iso2022_jp_1', 'iso8859_10', # 'iso2022_jp_2004', 'cp866', 'mac_greek', 'hz', 'cp1257', 'cp037', 'cp863', 'iso8859_4', 'utf_16_be', 'gb18030', 'cp1250', # 'cp850', 'iso8859_5', 'shift_jisx0213', 'iso8859_8', 'cp273', 'euc_jisx0213', 'cp932', 'cp862', 'tis_620', 'cp1125', 'koi8_r', @@ -629,7 +660,28 @@ def _encodings() -> set[str]: # 'iso2022_kr', 'cp1251', 'cp1255', 'mac_iceland', 'kz1048', 'iso8859_14', 'utf_32_be', 'ptcp154', 'iso8859_6', 'mac_roman', # 'utf_32', 'iso2022_jp_2', 'iso8859_16', 'mbcs', 'cp500', 'iso8859_2', 'cp949', 'cp852', 'utf_7', 'big5hkscs', 'johab'} -encodings = sampled_from(list(_encodings())) # if IN_CI else ENCODINGS) + +def strings_of_supported_code_points(encoding: str, min_size: int=1, max_size: int=10): + + for prefix, exclude_chars in _exclude_chars().items(): + if encoding.startswith(prefix): + break + else: + exclude_chars = [] + + return text( + alphabet=characters( + codec=encoding, + # https://en.wikipedia.org/wiki/Unicode_character_property#General_Category + exclude_categories=["Cs", "Co", "Cn"], # Cs - surrogates + exclude_characters=exclude_chars, + ), + min_size=min_size, + max_size=max_size, + ) + + +encodings = sampled_from(list(_exclude_chars())) # if IN_CI else ENCODINGS) @composite