diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000000..6ad67b721fee --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "packages/google-crc32c/google_crc32c"] + path = packages/google-crc32c/google_crc32c + url = https://github.com/google/crc32c.git diff --git a/.librarian/state.yaml b/.librarian/state.yaml index c863eb88ee08..dfacd6bdb703 100644 --- a/.librarian/state.yaml +++ b/.librarian/state.yaml @@ -3549,6 +3549,15 @@ libraries: remove_regex: - packages/google-cloud-workstations/ tag_format: '{id}-v{version}' + - id: google-crc32c + version: 1.8.0 + last_generated_commit: "" + apis: [] + source_roots: + - packages/google-crc32c + preserve_regex: [] + remove_regex: [] + tag_format: '{id}-v{version}' - id: google-geo-type version: 0.5.0 last_generated_commit: 3322511885371d2b2253f209ccc3aa60d4100cfd diff --git a/packages/google-crc32c/.coveragerc b/packages/google-crc32c/.coveragerc new file mode 100644 index 000000000000..195287608806 --- /dev/null +++ b/packages/google-crc32c/.coveragerc @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generated by synthtool. DO NOT EDIT! +[run] +branch = True +omit = + google/__init__.py + google/cloud/__init__.py + +[report] +fail_under = 100 +show_missing = True +exclude_lines = + # Re-enable the standard pragma + pragma: NO COVER + # Ignore debug-only repr + def __repr__ + # Ignore abstract methods + raise NotImplementedError +omit = + */gapic/*.py + */proto/*.py + */core/*.py + */site-packages/*.py + google/cloud/__init__.py diff --git a/packages/google-crc32c/.flake8 b/packages/google-crc32c/.flake8 new file mode 100644 index 000000000000..32986c79287a --- /dev/null +++ b/packages/google-crc32c/.flake8 @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generated by synthtool. DO NOT EDIT! +[flake8] +ignore = E203, E231, E266, E501, W503 +exclude = + # Exclude generated code. + **/proto/** + **/gapic/** + **/services/** + **/types/** + *_pb2.py + + # Standard linting exemptions. + **/.nox/** + __pycache__, + .git, + *.pyc, + conf.py diff --git a/packages/google-crc32c/.repo-metadata.json b/packages/google-crc32c/.repo-metadata.json new file mode 100644 index 000000000000..f853e7b029b4 --- /dev/null +++ b/packages/google-crc32c/.repo-metadata.json @@ -0,0 +1,14 @@ +{ + "name": "google-crc32c", + "name_pretty": "A python wrapper of the C library 'Google CRC32C'", + "product_documentation": "", + "client_documentation": "https://github.com/googleapis/python-crc32c", + "issue_tracker": "https://github.com/googleapis/python-crc32c/issues", + "release_level": "stable", + "language": "python", + "library_type": "OTHER", + "repo": "googleapis/google-cloud-python", + "distribution_name": "google-crc32c", + "default_version": "", + "codeowner_team": "" +} diff --git a/packages/google-crc32c/BUILDING.md b/packages/google-crc32c/BUILDING.md new file mode 100644 index 000000000000..bdc19ad9e040 --- /dev/null +++ b/packages/google-crc32c/BUILDING.md @@ -0,0 +1,92 @@ + +# Building + +## Be sure to check out all submodules: + +``` +$ git clone --recursive https://github.com/googleapis/python-crc32c +``` + +## Building and Testing Wheels + +The scripts directory contains multiple scripts for building. They are also +used by the CI workflows which can be found at .github/workflows. + +On Linux: + +``` +./scripts/manylinux/build.sh + + +# Install the wheel that was built as a result +pip install --no-index --find-links=wheels google-crc32c + +# Check the package, try and load the native library. +python ./scripts/check_crc32c_extension.py +``` + +On OS X: + +``` +# Build the C library and wheel +./scripts/osx/build.sh + +# Install the wheel that was built as a result +pip install --no-index --find-links=wheels google-crc32c + +# Check the package, try and load the native library. +python ./scripts/check_crc32c_extension.py +``` + +On Windows: + +``` +.\scripts\windows\build.bat +.\scripts\windows\test.bat +``` + + +## Installing locally for testing + +Initialize the submodules and build the main `libcrc32c.so` shared +library using `cmake` / `make`: + +```bash +$ cd python-crc32c +$ git submodule update --init --recursive +$ python -m venv venv +$ venv/bin/pip install --upgrade setuptools pip wheel +$ venv/bin/pip install cmake +$ mkdir usr +$ export CRC32C_INSTALL_PREFIX=$(pwd)/usr +$ mkdir google_crc32c/build +$ cd google_crc32c/build +$ ../../venv/bin/cmake \ +> -DCMAKE_BUILD_TYPE=Release \ +> -DCRC32C_BUILD_TESTS=no \ +> -DCRC32C_BUILD_BENCHMARKS=no \ +> -DBUILD_SHARED_LIBS=yes \ +> -DCMAKE_INSTALL_PREFIX:PATH=${CRC32C_INSTALL_PREFIX} \ +> .. +$ make all install +$ cd ../.. +``` + +Now, run the tests: + +```bash +$ venv/bin/python setup.py build_ext \ + --include-dirs=$(pwd)/usr/include \ + --library-dirs=$(pwd)/usr/lib \ + --rpath=$(pwd)/usr/lib +$ venv/bin/pip install -e .[testing] +$ venv/bin/py.test tests/ +============================= test session starts ============================== +platform linux -- Python 3.6.7, pytest-3.10.0, py-1.7.0, pluggy-0.8.0 +rootdir: ..., inifile: +collected 9 items + +tests/test___init__.py ......... [100%] + +=========================== 9 passed in 0.03 seconds =========================== +``` diff --git a/packages/google-crc32c/CHANGELOG.md b/packages/google-crc32c/CHANGELOG.md new file mode 100644 index 000000000000..aa9140456f8b --- /dev/null +++ b/packages/google-crc32c/CHANGELOG.md @@ -0,0 +1,209 @@ +# Changelog + +[PyPI History][1] + +[1]: https://pypi.org/project/google-crc32c/#history + +## [1.8.0](https://github.com/googleapis/python-crc32c/compare/v1.7.1...v1.8.0) (2025-12-15) + + +### Features + +* Add support for Python 3.14 ([d3118d9](https://github.com/googleapis/python-crc32c/commit/d3118d938b0acabc68fbc9d009aeb1e00445bc4c)) +* Support Python 3.14 ([#315](https://github.com/googleapis/python-crc32c/issues/315)) ([49d0a36](https://github.com/googleapis/python-crc32c/commit/49d0a36f3334048833d0201788fd864d3342d1b2)) + + +### Bug Fixes + +* Update toml file with setup.cfg content ([#319](https://github.com/googleapis/python-crc32c/issues/319)) ([42e5268](https://github.com/googleapis/python-crc32c/commit/42e5268c7e6077a71e54b342ada4ed2843dbc01e)) + +## [1.8.0](https://github.com/googleapis/python-crc32c/compare/v1.7.1...v1.8.0) (2025-12-15) + + +### Features + +* support Python 3.14 (#315) ([49d0a36f3334048833d0201788fd864d3342d1b2](https://github.com/googleapis/python-crc32c/commit/49d0a36f3334048833d0201788fd864d3342d1b2)) + + +### Bug Fixes + +* update toml file with setup.cfg content (#319) ([42e5268c7e6077a71e54b342ada4ed2843dbc01e](https://github.com/googleapis/python-crc32c/commit/42e5268c7e6077a71e54b342ada4ed2843dbc01e)) + +## [1.7.1](https://github.com/googleapis/python-crc32c/compare/v1.7.0...v1.7.1) (2025-03-25) + + +### Bug Fixes + +* Add Python 3.13 windows wheels ([fbb0e26](https://github.com/googleapis/python-crc32c/commit/fbb0e26b576fb82b8677b14505e702313cbcfe4e)) + +## [1.7.0](https://github.com/googleapis/python-crc32c/compare/v1.6.0...v1.7.0) (2025-03-17) + + +### Features + +* Add Python 3.13 trove classifier ([#233](https://github.com/googleapis/python-crc32c/issues/233)) ([b125257](https://github.com/googleapis/python-crc32c/commit/b1252575f21d0dde1e131f5251a989f56fb2c0d5)) +* Add support for python 3.13 ([#239](https://github.com/googleapis/python-crc32c/issues/239)) ([6b918f4](https://github.com/googleapis/python-crc32c/commit/6b918f4b081daf3cbee0cbd778b27c98670fb3a4)) + +## [1.6.0](https://github.com/googleapis/python-crc32c/compare/v1.5.0...v1.6.0) (2024-08-29) + + +### Features + +* Add support for python 3.12 ([#177](https://github.com/googleapis/python-crc32c/issues/177)) ([5ff1207](https://github.com/googleapis/python-crc32c/commit/5ff1207e7b60256e7a32932324ccb9ad4ec265d2)) +* Build using Visual Studio 17 2022 instead of Visual Studio 16 2019 ([c1c8c59](https://github.com/googleapis/python-crc32c/commit/c1c8c597d07e573406d76765022a837b007f9074)) + + +### Bug Fixes + +* Drop support for python 3.7 and 3.8 ([c1c8c59](https://github.com/googleapis/python-crc32c/commit/c1c8c597d07e573406d76765022a837b007f9074)) +* Drop support for Windows 32bit which is not supported in Visual Studio 17 2022 ([c1c8c59](https://github.com/googleapis/python-crc32c/commit/c1c8c597d07e573406d76765022a837b007f9074)) +* Remove manylinux1 which is no longer supported by PyPA ([#186](https://github.com/googleapis/python-crc32c/issues/186)) ([79edb3f](https://github.com/googleapis/python-crc32c/commit/79edb3fd3cda0e4193a6fb6a8346058398df43de)) + +## [1.5.0](https://github.com/googleapis/python-crc32c/compare/v1.4.0...v1.5.0) (2022-08-31) + + +### Features + +* add support for Python 3.11 ([#144](https://github.com/googleapis/python-crc32c/issues/144)) ([63f5166](https://github.com/googleapis/python-crc32c/commit/63f5166815682d3430a771c51c7fd5a81d2fd259)) + + +### Bug Fixes + +* drop support for python 3.6 ([#145](https://github.com/googleapis/python-crc32c/issues/145)) ([f9745b8](https://github.com/googleapis/python-crc32c/commit/f9745b8089fa2645be7b8823c979f5192df3eec7)) + + +### Documentation + +* fix changelog header to consistent size ([#136](https://github.com/googleapis/python-crc32c/issues/136)) ([0fb3b88](https://github.com/googleapis/python-crc32c/commit/0fb3b88131aeee316e6d6f607fa32ec4767e4386)) + +## [1.4.0](https://github.com/googleapis/python-crc32c/compare/v1.3.0...v1.4.0) (2022-03-10) + + +### Features + +* add 'py.typed' declaration ([#119](https://github.com/googleapis/python-crc32c/issues/119)) ([f1c91c2](https://github.com/googleapis/python-crc32c/commit/f1c91c277f1baac2ca8d60dd1e45361551ad3140)) + + +### Bug Fixes + +* add empty slots to CommonChecksum ([#121](https://github.com/googleapis/python-crc32c/issues/121)) ([c21cb89](https://github.com/googleapis/python-crc32c/commit/c21cb8923728801d732c11f6531a2415a1e5cd3f)) + +## [1.3.0](https://www.github.com/googleapis/python-crc32c/compare/v1.2.0...v1.3.0) (2021-10-05) + + +### Features + +* add support for Python 3.10 ([#116](https://www.github.com/googleapis/python-crc32c/issues/116)) ([01ee9c0](https://www.github.com/googleapis/python-crc32c/commit/01ee9c0b4d6a992ddcf1fbbeaaea9d107c972b74)) + +## [1.2.0](https://www.github.com/googleapis/python-crc32c/compare/v1.1.5...v1.2.0) (2021-09-17) + + +### Features + +* build wheels using CIBuildWheel ([#103](https://www.github.com/googleapis/python-crc32c/issues/103)) ([55eb731](https://www.github.com/googleapis/python-crc32c/commit/55eb7310b0a0f424da33f4b6d3b4b50e02c323eb)) + + +### Documentation + +* update list of supported wheel platforms / architectures ([#105](https://www.github.com/googleapis/python-crc32c/issues/105)) ([edc8d2d](https://www.github.com/googleapis/python-crc32c/commit/edc8d2dbe643f0c2bf1180f855e1585b0e81bdba)) + +## [1.1.5](https://www.github.com/googleapis/python-crc32c/compare/v1.1.4...v1.1.5) (2021-09-07) + + +### Bug Fixes + +* revert to silent fallback to pure-Python build ([#93](https://www.github.com/googleapis/python-crc32c/issues/93)) ([789a420](https://www.github.com/googleapis/python-crc32c/commit/789a4203648d1b43f060332510177cf3867f82c4)) +* fix segfault on MacOS 11 ("Big Sur") for Python < 3.9 ([#93](https://www.github.com/googleapis/python-crc32c/issues/93)) ([789a420](https://www.github.com/googleapis/python-crc32c/commit/789a4203648d1b43f060332510177cf3867f82c4)) + +## [1.1.4](https://www.github.com/googleapis/python-crc32c/compare/v1.1.4...v1.1.4) (2021-09-02) + + +### Bug Fixes + +* advise setting 'CRC32C_PURE_PYTHON' after build failure ([#84](https://www.github.com/googleapis/python-crc32c/issues/84)) ([6aa1cd6](https://www.github.com/googleapis/python-crc32c/commit/6aa1cd69376b57fbc9bc2c470ed63a270279623d)) +* restore building 'manylinux1' wheels ([#87](https://www.github.com/googleapis/python-crc32c/issues/87)) ([ebb9c68](https://www.github.com/googleapis/python-crc32c/commit/ebb9c68aca66e6b89d832e9e237679ac8b9ad344)) +* use correct Python 3.10 specifier ([#88](https://www.github.com/googleapis/python-crc32c/issues/88)) ([0c1b740](https://www.github.com/googleapis/python-crc32c/commit/0c1b740c195caed8ac1e67fc38d87073223a6b3d)) + +## [1.1.4](https://www.github.com/googleapis/python-crc32c/compare/v1.1.3...v1.1.4) (2021-09-01) + + +### Bug Fixes + +* advise setting 'CRC32C_PURE_PYTHON' after build failure ([#84](https://www.github.com/googleapis/python-crc32c/issues/84)) ([6aa1cd6](https://www.github.com/googleapis/python-crc32c/commit/6aa1cd69376b57fbc9bc2c470ed63a270279623d)) +* restore building 'manylinux1' wheels ([#87](https://www.github.com/googleapis/python-crc32c/issues/87)) ([ebb9c68](https://www.github.com/googleapis/python-crc32c/commit/ebb9c68aca66e6b89d832e9e237679ac8b9ad344)) + +## [1.1.3](https://www.github.com/googleapis/python-crc32c/compare/v1.1.2...v1.1.3) (2021-08-30) + + +### Performance Improvements + +* replace CFFI with a native C extension ([#76](https://www.github.com/googleapis/python-crc32c/issues/76)) ([b1bf461](https://www.github.com/googleapis/python-crc32c/commit/b1bf461cc0539962ac16a62860cae3cd2384cb4f)) + +## [1.1.2](https://www.github.com/googleapis/python-crc32c/compare/v1.1.1...v1.1.2) (2021-01-20) + + +### Bug Fixes + +* add manylinux2014_aarch64 wheels ([#61](https://www.github.com/googleapis/python-crc32c/issues/61)) ([6387658](https://www.github.com/googleapis/python-crc32c/commit/63876582aec715100f61581657f9d994a1ace1bc)) +* Add manylinux2014_x86_64 wheels ([#57](https://www.github.com/googleapis/python-crc32c/issues/57)) ([74cb457](https://www.github.com/googleapis/python-crc32c/commit/74cb457255a81d0aa5bee16425675140ed637410)) + + +### Documentation + +* add aarch64 to the readme as produced wheels ([#62](https://www.github.com/googleapis/python-crc32c/issues/62)) ([4ef317d](https://www.github.com/googleapis/python-crc32c/commit/4ef317d0efcd654842d17e03749b801303c8bc30)) + +## [1.1.1](https://www.github.com/googleapis/python-crc32c/compare/v1.1.0...v1.1.1) (2021-01-14) + + +### Bug Fixes + +* Update CI to use GitHub actions and build for osx, windows, linux ([#51](https://www.github.com/googleapis/python-crc32c/issues/51)) ([66f49b8](https://www.github.com/googleapis/python-crc32c/commit/66f49b889ad66f7ecd5d6aeaf840f2c8f2ac131e)) + + +### Documentation + +* simplify main readme that is shown on pypi, add additional BUILDING.md ([#54](https://www.github.com/googleapis/python-crc32c/issues/54)) ([93e021f](https://www.github.com/googleapis/python-crc32c/commit/93e021fe8bc55fb046317b884ca21cb75e131e4f)) + +## [1.1.0](https://www.github.com/googleapis/python-crc32c/compare/v0.1.1...v1.1.0) (2020-12-14) + + +### Features + +* add Python 3.9 support, drop Python 3.5 support ([#42](https://www.github.com/googleapis/python-crc32c/issues/42)) ([1d7fe63](https://www.github.com/googleapis/python-crc32c/commit/1d7fe6338fbcb0e74245f84c2034ac5371f7782a)), closes [#38](https://www.github.com/googleapis/python-crc32c/issues/38) + + +### Bug Fixes + +* add LICENSE to package manifest ([#34](https://www.github.com/googleapis/python-crc32c/issues/34)) ([6c8883b](https://www.github.com/googleapis/python-crc32c/commit/6c8883b2c41aaa6f0dd5991896ad58e73f516182)) + +## [1.0.0](https://www.github.com/googleapis/python-crc32c/compare/v0.1.0...v1.0.0) (2020-08-07) + +### Breaking Changes +* prefix module name with 'google_' to avoid conflict with ICRAR version ([#30](https://www.github.com/googleapis/python-crc32c/issues/30)) ([b50f43e](https://www.github.com/googleapis/python-crc32c/commit/b50f43e7bc40d91ccdade9ccc577a93c0ed05f3a)), closes [#29](https://www.github.com/googleapis/python-crc32c/issues/29) + +### Documentation + +* rewrap long line ([#32](https://www.github.com/googleapis/python-crc32c/issues/32)) ([30479d4](https://www.github.com/googleapis/python-crc32c/commit/30479d41997a09115aa0152b39ffef09bc97b13a)) + +## [0.1.0](https://www.github.com/googleapis/python-crc32c/compare/v0.0.2...v0.1.0) (2020-03-20) + + +### Features + +* add pure python implementation ([#20](https://www.github.com/googleapis/python-crc32c/issues/20)) ([97cf381](https://www.github.com/googleapis/python-crc32c/commit/97cf3819035486628b2dcc2ad03e3b427fbf8046)) + + +### Bug Fixes + +* retrieve package version at runtime ([#24](https://www.github.com/googleapis/python-crc32c/issues/24)) ([f365e47](https://www.github.com/googleapis/python-crc32c/commit/f365e471c9ae90238ded65456635ccdb6cd33ca2)) + +## 0.0.2 + +03-09-2020 15:33 PDT + + +### Packaging Changes +- fix: add a manifest to specify other files ([#14](https://github.com/googleapis/python-crc32c/pull/14)) + +### Documentation +- docs: README.md improvements ([#15](https://github.com/googleapis/python-crc32c/pull/15)) diff --git a/packages/google-crc32c/CONTRIBUTING.md b/packages/google-crc32c/CONTRIBUTING.md new file mode 100644 index 000000000000..ebbb59e5310f --- /dev/null +++ b/packages/google-crc32c/CONTRIBUTING.md @@ -0,0 +1,28 @@ +# How to Contribute + +We'd love to accept your patches and contributions to this project. There are +just a few small guidelines you need to follow. + +## Contributor License Agreement + +Contributions to this project must be accompanied by a Contributor License +Agreement. You (or your employer) retain the copyright to your contribution; +this simply gives us permission to use and redistribute your contributions as +part of the project. Head over to to see +your current agreements on file or to sign a new one. + +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again. + +## Code reviews + +All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult +[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more +information on using pull requests. + +## Community Guidelines + +This project follows +[Google's Open Source Community Guidelines](https://opensource.google.com/conduct/). \ No newline at end of file diff --git a/packages/google-crc32c/LICENSE b/packages/google-crc32c/LICENSE new file mode 100644 index 000000000000..d64569567334 --- /dev/null +++ b/packages/google-crc32c/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/google-crc32c/MANIFEST.in b/packages/google-crc32c/MANIFEST.in new file mode 100644 index 000000000000..2fb6b766f58a --- /dev/null +++ b/packages/google-crc32c/MANIFEST.in @@ -0,0 +1,2 @@ +include src/google_crc32c_build.py +include LICENSE diff --git a/packages/google-crc32c/README.md b/packages/google-crc32c/README.md new file mode 100644 index 000000000000..78088ce883b1 --- /dev/null +++ b/packages/google-crc32c/README.md @@ -0,0 +1,37 @@ +# `google-crc32c` +![GA](https://img.shields.io/badge/support-GA-gold.svg) [](https://pypi.org/project/google-crc32c) ![Python Versions](https://img.shields.io/pypi/pyversions/google-crc32c) + +This package wraps the [`google/crc32c`](https://github.com/google/crc32c) +hardware-based implementation of the CRC32C hashing algorithm. Multiple wheels +are distributed as well as source. If a wheel is not published for the python +version and platform you are using, you will need to compile crc32c using a +C toolchain. + +# Currently Published Wheels + +Wheels are currently published for CPython 3.9, 3.10, 3.11, 3.12, 3.13 and 3.14 +for multiple architectures. PyPy 3.9 and 3.10 are also supported for Linux. +For information on building your own wheels please view [BUILDING.md](BUILDING.md). + + +## Linux + +Wheels are published for the following platforms / architectures: + +- `manylinux2010` platform, `x86_64` and `1686` architectures +- `manylinux2014` platform, `aarch64` architecture + +### Unsupported Platforms + +- `manylinux1` platform, `x86_64` architecture support has ended. + +See https://github.com/pypa/manylinux/issues/994. + +## Mac OS + +Wheels are published for `x86_64` and `arm64` architectures. + + +## Windows + +Wheels are published for the `win_amd64` architecture. diff --git a/packages/google-crc32c/google_crc32c b/packages/google-crc32c/google_crc32c new file mode 160000 index 000000000000..02e65f4fd306 --- /dev/null +++ b/packages/google-crc32c/google_crc32c @@ -0,0 +1 @@ +Subproject commit 02e65f4fd3065d27b2e29324800ca6d04df16126 diff --git a/packages/google-crc32c/mypy.ini b/packages/google-crc32c/mypy.ini new file mode 100644 index 000000000000..8efb6f7241c8 --- /dev/null +++ b/packages/google-crc32c/mypy.ini @@ -0,0 +1,3 @@ +[mypy] +python_version = 3.9 +exclude = tests/unit/resources/ diff --git a/packages/google-crc32c/noxfile.py b/packages/google-crc32c/noxfile.py new file mode 100644 index 000000000000..0a859e12c7aa --- /dev/null +++ b/packages/google-crc32c/noxfile.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generated by synthtool. DO NOT EDIT! + +import os +import sys + +import nox + +HERE = os.path.dirname(__file__) + +# Constants +DEFAULT_PYTHON_VERSION = "3.14" +UNIT_TEST_PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] +ALL_PYTHON = list(UNIT_TEST_PYTHON_VERSIONS) +ALL_PYTHON.extend(["3.7"]) + +FLAKE8_VERSION = "flake8==6.1.0" +BLACK_VERSION = "black[jupyter]==23.7.0" +ISORT_VERSION = "isort==5.11.0" +LINT_PATHS = ["src", "tests", "noxfile.py", "setup.py"] + +nox.options.sessions = [ + "build_libcrc32c", + "check", + "lint", + "blacken", + "format", + "lint_setup_py", + "mypy", + "prerelease_deps", + "core_deps_from_source", + "unit", + "docs", + "docfx", +] + + +@nox.session(python=UNIT_TEST_PYTHON_VERSIONS) +def build_libcrc32c(session): + session.env["PY_BIN"] = f"python{session.python}" + session.env["REPO_ROOT"] = HERE + + if sys.platform.startswith("win"): + session.run("cmd", "-c", "scripts\\windows\\build.bat") + elif sys.platform == "linux": + session.run("bash", "scripts/local-linux/build_libcrc32c.sh") + elif sys.platform == "darwin": + session.run("bash", "scripts/osx/build.sh") + else: + raise Exception("Unsupported") + + +@nox.session(python=UNIT_TEST_PYTHON_VERSIONS) +def check(session): + session.install("pytest") + session.install("--no-index", f"--find-links={HERE}/wheels", "google-crc32c") + + # Run py.test against the unit tests. + session.run("py.test", "tests") + session.run("python", f"{HERE}/scripts/check_crc32c_extension.py", *session.posargs) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def mypy(session): + """Verify type hints are mypy compatible.""" + session.install( + "mypy", + "types-mock", + "types-setuptools", + ) + session.env["MYPYPATH"] = "src" + session.run("mypy", "src/google_crc32c/", "tests/") + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def lint(session): + """Run linters. + + Returns a failure if the linters find linting errors or sufficiently + serious code quality issues. + """ + session.install(FLAKE8_VERSION, BLACK_VERSION) + session.run( + "black", + "--check", + *LINT_PATHS, + ) + session.run("flake8", *LINT_PATHS) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def blacken(session): + """Run black. Format code to uniform standard.""" + session.install(BLACK_VERSION) + session.run( + "black", + *LINT_PATHS, + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def format(session): + """ + Run isort to sort imports. Then run black + to format code to uniform standard. + """ + session.install(BLACK_VERSION, ISORT_VERSION) + # Use the --fss option to sort imports using strict alphabetical order. + # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections + session.run( + "isort", + "--fss", + *LINT_PATHS, + ) + session.run( + "black", + *LINT_PATHS, + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def lint_setup_py(session): + """Verify that setup.py is valid (including RST check).""" + session.install("docutils", "pygments", "setuptools") + session.run("python", "setup.py", "check", "--restructuredtext", "--strict") + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def prerelease_deps(session): + """Run all tests with prerelease versions of dependencies installed.""" + # TODO(https://github.com/googleapis/google-cloud-python/issues/16014): + # Add prerelease deps tests + session.skip("prerelease deps tests are not yet supported") + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def core_deps_from_source(session): + """Run all tests with core dependencies installed from source + rather than pulling the dependencies from PyPI. + """ + # TODO(https://github.com/googleapis/google-cloud-python/issues/16014): + # Add core deps from source tests + session.skip("Core deps from source tests are not yet supported") + + +@nox.session(python=ALL_PYTHON) +def unit(session): + """Run all unit tests.""" + session.skip("Unit tests are not supported") + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def docs(session): + """Run all docs tests.""" + session.skip("Docs tests are not supported") + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def docfx(session): + """Run all docfx tests.""" + session.skip("Docfx tests are not supported") diff --git a/packages/google-crc32c/pyproject.toml b/packages/google-crc32c/pyproject.toml new file mode 100644 index 000000000000..ddff644d32e6 --- /dev/null +++ b/packages/google-crc32c/pyproject.toml @@ -0,0 +1,24 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "google-crc32c" +version = "1.8.0" +description = "A python wrapper of the C library 'Google CRC32C'" +readme = "README.md" +requires-python = ">=3.8" diff --git a/packages/google-crc32c/scripts/check_crc32c_extension.py b/packages/google-crc32c/scripts/check_crc32c_extension.py new file mode 100644 index 000000000000..dc4a85aa349f --- /dev/null +++ b/packages/google-crc32c/scripts/check_crc32c_extension.py @@ -0,0 +1,24 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google_crc32c import _crc32c + + +def main(): + print("_crc32c: {}".format(_crc32c)) + print("dir(_crc32c): {}".format(dir(_crc32c))) + + +if __name__ == "__main__": + main() diff --git a/packages/google-crc32c/scripts/dev-requirements.txt b/packages/google-crc32c/scripts/dev-requirements.txt new file mode 100644 index 000000000000..7479544b333a --- /dev/null +++ b/packages/google-crc32c/scripts/dev-requirements.txt @@ -0,0 +1,5 @@ +auditwheel >= 2.1.1; sys_platform == 'linux' or sys_platform == 'linux2' +delocate >= 0.8.0; sys_platform == 'darwin' +setuptools >= 42.0.2 +# See: https://github.com/pypa/auditwheel/issues/102 +wheel >= 0.34 diff --git a/packages/google-crc32c/scripts/local-linux/build_libcrc32c.sh b/packages/google-crc32c/scripts/local-linux/build_libcrc32c.sh new file mode 100755 index 000000000000..705f24771903 --- /dev/null +++ b/packages/google-crc32c/scripts/local-linux/build_libcrc32c.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -x + +PY_BIN=${PY_BIN:-python3.9} +REPO_ROOT=${REPO_ROOT:-$(pwd)} + +CRC32C_INSTALL_PREFIX=${REPO_ROOT}/usr + +# NOTE: This assumes the local install has an up-to-date `pip`. +# Create a virtualenv where we can install `cmake`. +VENV=${REPO_ROOT}/venv +${PY_BIN} -m venv ${VENV} +${VENV}/bin/python -m pip install --upgrade setuptools pip wheel +${VENV}/bin/python -m pip install "cmake >= 3.12.0" +rm -rf ${REPO_ROOT}/build +rm -rf ${CRC32C_INSTALL_PREFIX} +# Build `libcrc32c` +cd ${REPO_ROOT}/google_crc32c +rm -rf build +mkdir build +cd build/ +${VENV}/bin/cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DCRC32C_BUILD_TESTS=no \ + -DCRC32C_BUILD_BENCHMARKS=no \ + -DBUILD_SHARED_LIBS=yes \ + -DCMAKE_INSTALL_PREFIX:PATH=${CRC32C_INSTALL_PREFIX} \ + .. +# Install `libcrc32c` into CRC32C_INSTALL_PREFIX. +make all install + +cd ${REPO_ROOT} + +${VENV}/bin/python setup.py build_ext \ + --include-dirs=${REPO_ROOT}/usr/include \ + --library-dirs=${REPO_ROOT}/usr/lib \ + --rpath=${REPO_ROOT}/usr/lib +${VENV}/bin/python -m pip wheel . --wheel-dir=wheels + +# Clean up. +rm -fr ${REPO_ROOT}/google_crc32c/build +rm -fr ${VENV} diff --git a/packages/google-crc32c/scripts/manylinux/build.sh b/packages/google-crc32c/scripts/manylinux/build.sh new file mode 100755 index 000000000000..91397c8eba00 --- /dev/null +++ b/packages/google-crc32c/scripts/manylinux/build.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -x +echo "BUILDING ON LINUX" +export BUILD_PYTHON=${BUILD_PYTHON} + +MANYLINUX_DIR=$(echo $(cd $(dirname ${0}); pwd)) +SCRIPTS_DIR=$(dirname ${MANYLINUX_DIR}) +REPO_ROOT=$(dirname ${SCRIPTS_DIR}) + +sudo apt-get install -y software-properties-common +sudo add-apt-repository -y ppa:deadsnakes/ppa +sudo apt-get update +sudo apt-get install -y python3.12 + +cd $REPO_ROOT +# Add directory as safe to avoid "detected dubious ownership" fatal issue1 +git config --global --add safe.directory $REPO_ROOT +git config --global --add safe.directory $REPO_ROOT/google_crc32c +git submodule update --init --recursive + + + +docker pull quay.io/pypa/manylinux2014_x86_64 +docker run \ + --rm \ + --interactive \ + --volume ${REPO_ROOT}:/var/code/python-crc32c/ \ + --env BUILD_PYTHON=${BUILD_PYTHON} \ + quay.io/pypa/manylinux2014_x86_64 \ + /var/code/python-crc32c/scripts/manylinux/build_on_centos.sh + +docker run --rm --privileged hypriot/qemu-register +docker pull quay.io/pypa/manylinux2014_aarch64 +docker run \ + --rm \ + --interactive \ + --volume ${REPO_ROOT}:/var/code/python-crc32c/ \ + --env BUILD_PYTHON=${BUILD_PYTHON} \ + quay.io/pypa/manylinux2014_aarch64 \ + /var/code/python-crc32c/scripts/manylinux/build_on_centos.sh + +if [[ "${PUBLISH_WHEELS}" == "true" ]]; then + . /${MANYLINUX_DIR}/publish_python_wheel.sh +fi diff --git a/packages/google-crc32c/scripts/manylinux/build_on_centos.sh b/packages/google-crc32c/scripts/manylinux/build_on_centos.sh new file mode 100755 index 000000000000..9507dda09262 --- /dev/null +++ b/packages/google-crc32c/scripts/manylinux/build_on_centos.sh @@ -0,0 +1,119 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -x +MAIN_PYTHON_BIN="/opt/python/cp39-cp39/bin/" +echo "BUILD_PYTHON: ${BUILD_PYTHON}" +REPO_ROOT=/var/code/python-crc32c/ + +# Install `openssl-devel` so that `cmake` can be built. +yum install -y openssl-devel + +# Upgrade `pip` before using it. +${MAIN_PYTHON_BIN}/python -m pip install --upgrade pip +# Install `cmake` (i.e. non-Python build dependency). +${MAIN_PYTHON_BIN}/python -m pip install "cmake >= 3.12.0" +# Install Python build dependencies. +${MAIN_PYTHON_BIN}/python -m pip install \ + --requirement ${REPO_ROOT}/scripts/dev-requirements.txt + +# Build and install `crc32c` +cd ${REPO_ROOT}/google_crc32c/ +rm -rf build +mkdir build +cd build/ +cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DCRC32C_BUILD_TESTS=no \ + -DCRC32C_BUILD_BENCHMARKS=no \ + -DBUILD_SHARED_LIBS=yes \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.12 \ + .. +make all install + +PYTHON_VERSIONS="" +if [[ -z ${BUILD_PYTHON} ]]; then + # Collect all target Python versions. + for PYTHON_BIN in /opt/python/*/bin; do + # H/T: https://stackoverflow.com/a/229606/1068170 + if [[ "${PYTHON_BIN}" == *"39"* ]]; then + PYTHON_VERSIONS="${PYTHON_VERSIONS} ${PYTHON_BIN}" + continue + elif [[ "${PYTHON_BIN}" == *"310"* ]]; then + PYTHON_VERSIONS="${PYTHON_VERSIONS} ${PYTHON_BIN}" + continue + elif [[ "${PYTHON_BIN}" == *"311"* ]]; then + PYTHON_VERSIONS="${PYTHON_VERSIONS} ${PYTHON_BIN}" + continue + elif [[ "${PYTHON_BIN}" == *"312"* ]]; then + PYTHON_VERSIONS="${PYTHON_VERSIONS} ${PYTHON_BIN}" + continue + elif [[ "${PYTHON_BIN}" == *"313"* && "${PYTHON_BIN}" != *"313t"* ]]; then + PYTHON_VERSIONS="${PYTHON_VERSIONS} ${PYTHON_BIN}" + continue + elif [[ "${PYTHON_BIN}" == *"314"* && "${PYTHON_BIN}" != *"314t"* ]]; then + PYTHON_VERSIONS="${PYTHON_VERSIONS} ${PYTHON_BIN}" + continue + else + echo "Ignoring unsupported version: ${PYTHON_BIN}" + echo "=====================================" + fi + done +else + STRIPPED_PYTHON=$(echo ${BUILD_PYTHON} | sed -e "s/\.//g" | sed -e "s/-dev$//") + for PYTHON_BIN in /opt/python/*/bin; do + if [[ "${PYTHON_BIN}" == *"${STRIPPED_PYTHON}"* ]]; then + PYTHON_VERSIONS="${PYTHON_VERSIONS} ${PYTHON_BIN}" + fi + done +fi + +# Build the wheels. +cd ${REPO_ROOT} +for PYTHON_BIN in ${PYTHON_VERSIONS}; do + ${PYTHON_BIN}/python -m pip install --upgrade pip + ${PYTHON_BIN}/python -m pip install \ + --requirement ${REPO_ROOT}/scripts/dev-requirements.txt + ${PYTHON_BIN}/python -m pip wheel . --wheel-dir dist_wheels/ +done + +# Bundle external shared libraries into the wheels +for whl in dist_wheels/google_crc32c*.whl; do + "${MAIN_PYTHON_BIN}/auditwheel" repair "${whl}" --wheel-dir wheels/ +done + +# Install and test wheels +for PYTHON_BIN in ${PYTHON_VERSIONS}; do + # Identify the short python version e.g. "39", "310" + # Get the ABI tag from the Python binary's path, e.g., "cp310-cp310" + ABI_TAG=$(basename $(dirname ${PYTHON_BIN})) + ARCH=$(uname -m) + # Create a virtual environment to install and test the wheel + ${PYTHON_BIN}/python -m venv /tmp/venv + + # Find the correct wheel file using the precise ABI tag and architecture. + WHEEL_FILE=$(ls ${REPO_ROOT}/wheels/google_crc32c-*-${ABI_TAG}-*manylinux*${ARCH}*.whl) # Install the wheel + /tmp/venv/bin/pip install "${WHEEL_FILE}" + + # Verify that the module is installed and peek at contents. + /tmp/venv/bin/python ${REPO_ROOT}/scripts/check_crc32c_extension.py + + # Clean up the virtual environment + rm -rf /tmp/venv +done + +# Clean up. +rm -fr ${REPO_ROOT}/google_crc32c/build/ +rm -fr ${REPO_ROOT}/dist_wheels/ diff --git a/packages/google-crc32c/scripts/manylinux/publish_python_wheel.sh b/packages/google-crc32c/scripts/manylinux/publish_python_wheel.sh new file mode 100755 index 000000000000..7b5bdf6d3520 --- /dev/null +++ b/packages/google-crc32c/scripts/manylinux/publish_python_wheel.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +python -m pip install "setuptools<71" + +# Start the releasetool reporter +python -m pip install --require-hashes -r ${REPO_ROOT}/.kokoro/requirements.txt +python -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script + +ls ${REPO_ROOT}/wheels/ +# Disable logging +set +x +TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-3") +python -m twine upload --skip-existing --username __token__ --password "${TWINE_PASSWORD}" ${REPO_ROOT}/wheels/* diff --git a/packages/google-crc32c/scripts/osx/build.sh b/packages/google-crc32c/scripts/osx/build.sh new file mode 100755 index 000000000000..7000337c70b4 --- /dev/null +++ b/packages/google-crc32c/scripts/osx/build.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -x +echo "BUILDING FOR OSX" + +# set deployment target +export MACOSX_DEPLOYMENT_TARGET=12 + +# ``readlink -f`` is not our friend on OS X. This relies on **some** +# ``python`` being installed. +SCRIPT_FI=$(python3 -c "import os; print(os.path.realpath('${0}'))") +OSX_DIR=$(dirname ${SCRIPT_FI}) +SCRIPTS_DIR=$(dirname ${OSX_DIR}) +export REPO_ROOT=$(dirname ${SCRIPTS_DIR}) + +# Build and install `libcrc32c` +export PY_BIN="python3" +export CRC32C_INSTALL_PREFIX="${REPO_ROOT}/usr" + +cd ${REPO_ROOT} +# Add directory as safe to avoid "detected dubious ownership" fatal issue +git config --global --add safe.directory $REPO_ROOT +git config --global --add safe.directory $REPO_ROOT/google_crc32c +git submodule update --init --recursive + +${OSX_DIR}/build_c_lib.sh + +# reinstall pyenv +rm -rf /Users/kbuilder/.pyenv +git clone https://github.com/pyenv/pyenv.git /Users/kbuilder/.pyenv + +SUPPORTED_PYTHON_VERSIONS=("3.9" "3.10" "3.11" "3.12" "3.13" "3.14") + +for PYTHON_VERSION in ${SUPPORTED_PYTHON_VERSIONS[@]}; do + echo "Build wheel for Python ${PYTHON_VERSION}" + export PY_BIN=$PYTHON_VERSION + export PY_TAG="cp${PYTHON_VERSION//.}-cp${PYTHON_VERSION//.}" + . /${OSX_DIR}/build_python_wheel.sh +done + +# Clean up. +rm -fr ${CRC32C_INSTALL_PREFIX} diff --git a/packages/google-crc32c/scripts/osx/build_c_lib.sh b/packages/google-crc32c/scripts/osx/build_c_lib.sh new file mode 100755 index 000000000000..4d9ae0e6543c --- /dev/null +++ b/packages/google-crc32c/scripts/osx/build_c_lib.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build and install `libcrc32c` + +set -e -x + +# Check that the install prefix is set. Exit early if the +# directory already exists. +if [[ -z "${CRC32C_INSTALL_PREFIX}" ]]; then + echo "CRC32C_INSTALL_PREFIX environment variable should be set by the caller." + exit 1 +fi +if [[ -d "${CRC32C_INSTALL_PREFIX}" ]]; then + echo "CRC32C_INSTALL_PREFIX=${CRC32C_INSTALL_PREFIX} already exists." + exit 0 +fi + +# Check that the REPO_ROOT and PY_BIN environment variables are set. +if [[ -z "${REPO_ROOT}" ]]; then + echo "REPO_ROOT environment variable should be set by the caller." + exit 1 +fi + +if [[ -z "${PY_BIN}" ]]; then + echo "PY_BIN environment variable should be set by the caller." + exit 1 +fi + +# Precreate install directories for crc32 lib. +mkdir -p ${CRC32C_INSTALL_PREFIX} +mkdir -p ${CRC32C_INSTALL_PREFIX}/lib + +# Create a virtualenv where we can install `cmake`. +VENV=${REPO_ROOT}/venv_build_libcrc32c +${PY_BIN} -m venv ${VENV} +${VENV}/bin/python -m pip install --upgrade pip +${VENV}/bin/python -m pip install "cmake >= 3.12.0" + +# Build `libcrc32c` +cd ${REPO_ROOT}/google_crc32c +mkdir -p build +cd build +ls + +# We don't build i386 anymore as XCode no longer supports. +${VENV}/bin/cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=10.9 \ + -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64" \ + -DCRC32C_BUILD_TESTS=no \ + -DCRC32C_BUILD_BENCHMARKS=no \ + -DBUILD_SHARED_LIBS=yes \ + -DCMAKE_INSTALL_PREFIX:PATH=${CRC32C_INSTALL_PREFIX} \ + -DCMAKE_INSTALL_NAME_DIR:PATH=${CRC32C_INSTALL_PREFIX}/lib \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.12 \ + .. + +# Install `libcrc32c` into CRC32C_INSTALL_PREFIX. +make all install + +# Clean up. +cd .. +rm -fr ${REPO_ROOT}/google_crc32c/build +rm -fr ${VENV} diff --git a/packages/google-crc32c/scripts/osx/build_python_wheel.sh b/packages/google-crc32c/scripts/osx/build_python_wheel.sh new file mode 100755 index 000000000000..c63241677364 --- /dev/null +++ b/packages/google-crc32c/scripts/osx/build_python_wheel.sh @@ -0,0 +1,104 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build a **single** Python wheel for a specified version. The version and +# associated paths should be set as environment variables; the expected +# environment variables will be verified below. + +set -e -x + +# Check that the REPO_ROOT, PY_BIN and PY_TAG environment variables are set. +if [[ -z "${REPO_ROOT}" ]]; then + echo "REPO_ROOT environment variable should be set by the caller." + exit 1 +fi +if [[ -z "${PY_BIN}" ]]; then + echo "PY_BIN environment variable should be set by the caller." + exit 1 +fi +if [[ -z "${PY_TAG}" ]]; then + echo "PY_TAG environment variable should be set by the caller." + exit 1 +fi + +# set up pyenv & shell environment for switching across python versions +eval "$(pyenv init -)" +eval "$(pyenv init --path)" + +install_python_pyenv() { + version=$1 + + if [ -z "$(pyenv versions --bare | grep $version)" ]; then + echo "Python $version is not installed. Installing..." + pyenv install $version + echo "Python $version installed." + else + echo "Python $version is already installed." + fi + pyenv shell $version +} +install_python_pyenv ${PY_BIN} + + +# ``readlink -f`` is not our friend on OS X. This relies on **some** +# ``python`` being installed. +SCRIPT_FI=$(python3 -c "import os; print(os.path.realpath('${0}'))") +OSX_DIR=$(dirname ${SCRIPT_FI}) +SCRIPTS_DIR=$(dirname ${OSX_DIR}) +export REPO_ROOT=$(dirname ${SCRIPTS_DIR}) + +# Create a virtualenv where we can install Python build dependencies. +VENV=${REPO_ROOT}/venv${PY_BIN} +"python${PY_BIN}" -m venv ${VENV} + +curl https://bootstrap.pypa.io/get-pip.py | ${VENV}/bin/python +${VENV}/bin/python -m pip install \ + --requirement ${REPO_ROOT}/scripts/dev-requirements.txt + +# Create the wheel. +DIST_WHEELS="${REPO_ROOT}/dist_wheels" +mkdir -p ${DIST_WHEELS} +cd ${REPO_ROOT} +${VENV}/bin/python setup.py build_ext \ + --include-dirs=${REPO_ROOT}/usr/include \ + --library-dirs=${REPO_ROOT}/usr/lib \ + --rpath=${REPO_ROOT}/usr/lib +${VENV}/bin/python -m pip wheel ${REPO_ROOT} --wheel-dir ${DIST_WHEELS} + +# Delocate the wheel. +FIXED_WHEELS="${REPO_ROOT}/wheels" +mkdir -p ${FIXED_WHEELS} +${VENV}/bin/delocate-wheel \ + --wheel-dir ${FIXED_WHEELS} \ + --verbose \ + --check-archs \ + ${DIST_WHEELS}/google_crc32c*${PY_TAG}*.whl + +if [[ "${PUBLISH_WHEELS}" == "true" ]]; then + . /${OSX_DIR}/publish_python_wheel.sh +fi + +# test wheel +${VENV}/bin/pip install \ + --no-index --find-links=${REPO_ROOT}/wheels google-crc32c --force-reinstall +${VENV}/bin/pip install pytest +${VENV}/bin/py.test ${REPO_ROOT}/tests +${VENV}/bin/python ${REPO_ROOT}/scripts/check_crc32c_extension.py + +ls ${REPO_ROOT}/wheels/ + +# Clean up. +rm -fr ${DIST_WHEELS} +rm -fr ${VENV} diff --git a/packages/google-crc32c/scripts/osx/publish_python_wheel.sh b/packages/google-crc32c/scripts/osx/publish_python_wheel.sh new file mode 100755 index 000000000000..c5288eac09fc --- /dev/null +++ b/packages/google-crc32c/scripts/osx/publish_python_wheel.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +if [ -z "$(pyenv versions --bare | grep 3.8)" ]; then + echo "Python 3.8 is not installed. Installing..." + pyenv install 3.8 +fi +pyenv shell 3.8 + +python -m pip install "setuptools<71" + +# Start the releasetool reporter +python -m pip install --require-hashes -r ${REPO_ROOT}/.kokoro/requirements.txt +python -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script + +ls ${REPO_ROOT}/wheels/ +# Disable logging +set +x +TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-3") +python -m twine upload --skip-existing --username __token__ --password "${TWINE_PASSWORD}" ${REPO_ROOT}/wheels/* diff --git a/packages/google-crc32c/scripts/requirements.in b/packages/google-crc32c/scripts/requirements.in new file mode 100644 index 000000000000..0508fa5e096b --- /dev/null +++ b/packages/google-crc32c/scripts/requirements.in @@ -0,0 +1,4 @@ +setuptools +pip +wheel +cmake \ No newline at end of file diff --git a/packages/google-crc32c/scripts/requirements.txt b/packages/google-crc32c/scripts/requirements.txt new file mode 100644 index 000000000000..ca66848ed4f1 --- /dev/null +++ b/packages/google-crc32c/scripts/requirements.txt @@ -0,0 +1,41 @@ +# +# This file is autogenerated by pip-compile with python 3.10 +# To update, run: +# +# pip-compile --allow-unsafe --generate-hashes requirements.in +# +cmake==3.31.6 \ + --hash=sha256:024a79ca3d2c355f75875b6cc92d907afd710d1a4ffde2f20a7da712a2f4b1c3 \ + --hash=sha256:112b36427e59bd26145b705a49d5f70b16433a655ce807cb8fdd81dd4d0e60c2 \ + --hash=sha256:13f2e636dc27834fe096f53301d6efb913b4b501fdc0ed03f386c0a7e7ec1a21 \ + --hash=sha256:1c8b05df0602365da91ee6a3336fe57525b137706c4ab5675498f662ae1dbcec \ + --hash=sha256:2297e9591307d9c61e557efe737bcf4d7c13a30f1f860732f684a204fee24dca \ + --hash=sha256:42d9883b8958da285d53d5f69d40d9650c2d1bcf922d82b3ebdceb2b3a7d4521 \ + --hash=sha256:4326f6c6f39867a60e2822fea8e6aedbcac09c9f59ad3f0f3386a890a2c8d89d \ + --hash=sha256:547efc1d0e27a194da819a0392fe645a9b8f1485bc2c3f34ae4f1e682cfd3153 \ + --hash=sha256:689441fc74fbb03673c67e20d4636614a231634d5e803387cd213d2cdf9675fc \ + --hash=sha256:6cb97adae7e5390ce68f8b7f38e1be1c72bf19e9f6727f31f8fa1c095b39be88 \ + --hash=sha256:6f77db820af725bb92fab60c4c9d67f64442ac0ea9b933aca4cd4586219cbd1f \ + --hash=sha256:8b67bf9613dfb59c12ce643c6be582c49c981e6eee28c4c244aeb3248b33f05e \ + --hash=sha256:8edddfbf367fa1bcf4b9f3064470bc0e1022f70609c0cf69c863961897826205 \ + --hash=sha256:9eed74a1f2a29a7cd92a9f071a35d64645b19802beb393ec250d6e7c09441314 \ + --hash=sha256:9f170e3c6933dba64f333cb456823bbb1d0ac126f94aa4a577e40855d2b1ca49 \ + --hash=sha256:bbaed969cef3c427f4f17591feb28db4ae595e3a4bbd45cb35522cee14df6a32 \ + --hash=sha256:ce5fc0299ecafe489b2614daa6176c3c2baacea6bc3b359bac9aa25b46ed43e9 \ + --hash=sha256:cefb910be81e1b4fdc3b89ef61819c3e848b3906ed56ac36d090f37cfa05666b \ + --hash=sha256:da9d4fd9abd571fd016ddb27da0428b10277010b23bb21e3678f8b9e96e1686e + # via -r requirements.in +wheel==0.45.1 \ + --hash=sha256:661e1abd9198507b1409a20c02106d9670b2576e916d58f520316666abca6729 \ + --hash=sha256:708e7481cc80179af0e556bbf0cc00b8444c7321e2700b8d8580231d13017248 + # via -r requirements.in + +# The following packages are considered to be unsafe in a requirements file: +pip==25.0.1 \ + --hash=sha256:88f96547ea48b940a3a385494e181e29fb8637898f88d88737c5049780f196ea \ + --hash=sha256:c46efd13b6aa8279f33f2864459c8ce587ea6a1a59ee20de055868d8f7688f7f + # via -r requirements.in +setuptools==78.1.0 \ + --hash=sha256:18fd474d4a82a5f83dac888df697af65afa82dec7323d09c3e37d1f14288da54 \ + --hash=sha256:3e386e96793c8702ae83d17b853fb93d3e09ef82ec62722e61da5cd22376dcd8 + # via -r requirements.in diff --git a/packages/google-crc32c/scripts/windows/build.bat b/packages/google-crc32c/scripts/windows/build.bat new file mode 100644 index 000000000000..c2b6b6fcb779 --- /dev/null +++ b/packages/google-crc32c/scripts/windows/build.bat @@ -0,0 +1,83 @@ +@rem Copyright 2019 Google LLC. All rights reserved. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + + +setlocal ENABLEDELAYEDEXPANSION + +set CMAKE_GENERATOR="Visual Studio 17 2022" +set CONFIGURATION=RelWithDebInfo +set CRC32C_INSTALL_PREFIX=%cd%\build\%CONFIGURATION% + +@rem Iterate through supported Python versions. +@rem Unfortunately pyenv for Windows has an out-of-date versions list. Choco's +@rem installer seems to have some problems with installing multiple versions at +@rem once, so as a workaround, we will install and then uninstall every version. +FOR %%P IN (3.9, 3.10, 3.11, 3.12, 3.13.1, 3.14.0) DO ( + + echo "Installing Python version %%P" + choco install python --version=%%P -y --no-progress + + echo "Listing available Python versions' + py -0 + + set python_version=%%P + set python_version_trimmed=!python_version:~0,4! + + py -!python_version_trimmed!-64 -m pip install --upgrade pip + + echo "Installing cmake for Python %%P" + py -!python_version_trimmed!-64 -m pip install cmake + + @rem Add directory as safe to avoid "detected dubious ownership" fatal issue + git config --global --add safe.directory %cd% + git config --global --add safe.directory C:/tmpfs/src/github/python-crc32c + git submodule update --init --recursive + + git config --global --add safe.directory %cd%\google_crc32c + git config --global --add safe.directory C:/tmpfs/src/github/python-crc32c/google_crc32c + pushd google_crc32c + @rem reset hard to cleanup any changes done by a previous build. + git reset --hard + git clean -fxd + + del /s /q CMakeFiles\ + del CMakeCache.txt + + mkdir build + cd build + + echo "Running cmake with Generator: %CMAKE_GENERATOR%, Platform: x64, Install Prefix: %CRC32C_INSTALL_PREFIX%" + + py -!python_version_trimmed!-64 -m cmake -G "Visual Studio 17 2022" -A x64 -DCMAKE_POLICY_VERSION_MINIMUM=3.12 -DCRC32C_BUILD_BENCHMARKS=no -DCRC32C_BUILD_TESTS=no -DBUILD_SHARED_LIBS=yes -DCMAKE_WINDOWS_EXPORT_ALL_SYMBOLS=yes -DCRC32C_USE_GLOG=0 -DCMAKE_INSTALL_PREFIX:PATH=C:\tmpfs\src\github\python-crc32c\build\RelWithDebInfo .. + + py -!python_version_trimmed!-64 -m cmake --build . --config "%CONFIGURATION%" --target install + + dir %CRC32C_INSTALL_PREFIX% /b /s + popd + + dir %CRC32C_INSTALL_PREFIX%\bin + echo "Copying Binary to root: %CRC32C_INSTALL_PREFIX%\bin\crc32c.dll" + copy %CRC32C_INSTALL_PREFIX%\bin\crc32c.dll . + + py -!python_version_trimmed!-64 -m pip install --upgrade pip setuptools wheel + echo "Building C extension" + py -!python_version_trimmed!-64 setup.py build_ext -v --include-dirs=%CRC32C_INSTALL_PREFIX%\include --library-dirs=%CRC32C_INSTALL_PREFIX%\lib + echo "Building Wheel" + py -!python_version_trimmed!-64 -m pip wheel . --wheel-dir wheels/ + + echo "Built wheel, now running tests." + call %~dp0/test.bat !python_version_trimmed! || goto :error + + echo "Finished with Python version %P" +) diff --git a/packages/google-crc32c/scripts/windows/test.bat b/packages/google-crc32c/scripts/windows/test.bat new file mode 100644 index 000000000000..c7f28501d6bb --- /dev/null +++ b/packages/google-crc32c/scripts/windows/test.bat @@ -0,0 +1,32 @@ +@rem Copyright 2019 Google LLC. All rights reserved. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +@rem This test file runs for one Python version at a time, and is intended to +@rem be called from within the build loop. + +set PYTHON_VERSION=%1 +if "%PYTHON_VERSION%"=="" ( + echo "Python version was not provided, using Python 3.10" + set PYTHON_VERSION=3.10 +) + +py -%PYTHON_VERSION%-64 -m pip install --no-index --find-links=wheels google-crc32c --force-reinstall + +py -%PYTHON_VERSION%-64 ./scripts/check_crc32c_extension.py + +@rem pyreadline is removed here because pytest will opportunistically use it if +@rem available, but the installed version is too old to work. +py -%PYTHON_VERSION%-64 -m pip uninstall -y pyreadline +py -%PYTHON_VERSION%-64 -m pip install pytest +py -%PYTHON_VERSION%-64 -m pytest tests diff --git a/packages/google-crc32c/setup.cfg b/packages/google-crc32c/setup.cfg new file mode 100644 index 000000000000..9e97011dd7ad --- /dev/null +++ b/packages/google-crc32c/setup.cfg @@ -0,0 +1,49 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[metadata] +name = google-crc32c +description = A python wrapper of the C library 'Google CRC32C' +url = https://github.com/googleapis/python-crc32c +long_description = file: README.md +long_description_content_type = text/markdown +author = Google LLC +author_email = googleapis-packages@google.com + +license = Apache 2.0 +license_files = LICENSE +platforms = Posix, MacOS X, Windows +classifiers = + Development Status :: 4 - Beta + Intended Audience :: Developers + Operating System :: OS Independent + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 + Programming Language :: Python :: 3.13 + Programming Language :: Python :: 3.14 + +[options] +zip_safe = True +python_requires = >=3.8 + +[options.extras_require] +testing = pytest + +[options.package_data] +google_crc32c = + py.typed diff --git a/packages/google-crc32c/setup.py b/packages/google-crc32c/setup.py new file mode 100644 index 000000000000..cebbca973d22 --- /dev/null +++ b/packages/google-crc32c/setup.py @@ -0,0 +1,134 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import platform +import shutil + +import setuptools +import setuptools.command.build_ext + +_EXTRA_DLL = "extra-dll" +_DLL_FILENAME = "crc32c.dll" + +# Explicit environment variable disables pure-Python fallback +CRC32C_PURE_PYTHON_EXPLICIT = "CRC32C_PURE_PYTHON" in os.environ +_FALSE_OPTIONS = ("0", "false", "no", "False", "No", None) +CRC32C_PURE_PYTHON = os.getenv("CRC32C_PURE_PYTHON") not in _FALSE_OPTIONS + + +def copy_dll(build_lib): + install_prefix = os.environ.get("CRC32C_INSTALL_PREFIX") + + if os.name == "nt" and install_prefix is not None: + assert os.path.isdir(install_prefix) + + installed_dll = os.path.join(install_prefix, "bin", _DLL_FILENAME) + assert os.path.isfile(installed_dll) + + lib_dlls = os.path.join(build_lib, "google_crc32c", _EXTRA_DLL) + os.makedirs(lib_dlls, exist_ok=True) + relocated_dll = os.path.join(lib_dlls, _DLL_FILENAME) + + shutil.copyfile(installed_dll, relocated_dll) + assert os.path.isfile(relocated_dll) + + +class BuildExtWithDLL(setuptools.command.build_ext.build_ext): + def run(self): + copy_dll(self.build_lib) + result = setuptools.command.build_ext.build_ext.run(self) + return result + + +def build_pure_python(): + setuptools.setup( + packages=["google_crc32c"], + package_dir={"": "src"}, + ext_modules=[], + ) + + +def build_c_extension(): + install_prefix = os.getenv("CRC32C_INSTALL_PREFIX") + if install_prefix is not None: + install_prefix = os.path.normcase(install_prefix) + print(f"#### using local install of 'crc32c': {install_prefix!r}") + # assert os.path.isdir(install_prefix) + install_prefix = os.path.realpath(install_prefix) + include_dirs = [os.path.join(install_prefix, "include")] + library_dirs = [os.path.join(install_prefix, "lib")] + + if platform.system() == "Linux": + library_dirs.append(os.path.join(install_prefix, "lib64")) + + if os.name == "nt": + library_dirs.append(os.path.join(install_prefix, "bin")) + kwargs = { + "include_dirs": include_dirs, + "library_dirs": library_dirs, + } + else: + runtime_library_dirs = library_dirs[:] + kwargs = { + "include_dirs": include_dirs, + "library_dirs": library_dirs, + "runtime_library_dirs": runtime_library_dirs, + } + else: + print("#### using global install of 'crc32c'") + kwargs = {} + + module_path = os.path.join("src", "google_crc32c", "_crc32c.c") + sources = [os.path.normcase(module_path)] + print(f"##### sources: {sources}") + print(f"##### module kwargs: {kwargs}") + module = setuptools.Extension( + "google_crc32c._crc32c", sources=sources, libraries=["crc32c"], **kwargs + ) + + setuptools.setup( + packages=["google_crc32c"], + package_dir={"": "src"}, + ext_modules=[module], + cmdclass={"build_ext": BuildExtWithDLL}, + install_requires=[ + "importlib_resources>=1.3 ; python_version < '3.9' and os_name == 'nt'" + ], + ) + + +if CRC32C_PURE_PYTHON: + build_pure_python() +else: + try: + build_c_extension() + except SystemExit: + if CRC32C_PURE_PYTHON_EXPLICIT: + # If build / install fails, it is likely a compilation error with + # the C extension: advise user how to enable the pure-Python + # build. + logging.error( + "Compiling the C Extension for the crc32c library failed. " + "To enable building / installing a pure-Python-only version, " + "set 'CRC32C_PURE_PYTHON=1' in the environment." + ) + raise + + logging.info( + "Compiling the C Extension for the crc32c library failed. " + "Falling back to pure Python build." + ) + build_pure_python() diff --git a/packages/google-crc32c/src/google_crc32c/__config__.py b/packages/google-crc32c/src/google_crc32c/__config__.py new file mode 100644 index 000000000000..536f0307e7f2 --- /dev/null +++ b/packages/google-crc32c/src/google_crc32c/__config__.py @@ -0,0 +1,39 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +def modify_path(): + """Modify the module search path.""" + # Only modify path on Windows. + if os.name != "nt": + return + + path = os.environ.get("PATH") + if path is None: + return + + try: + from importlib.resources import files as _resources_files + + extra_dll_dir = str(_resources_files("google_crc32c") / "extra-dll") + if os.path.isdir(extra_dll_dir): + # Python 3.8+ uses add_dll_directory. + os.add_dll_directory(extra_dll_dir) + except ImportError: + pass + + +modify_path() diff --git a/packages/google-crc32c/src/google_crc32c/__init__.py b/packages/google-crc32c/src/google_crc32c/__init__.py new file mode 100644 index 000000000000..bba10d5c4b1c --- /dev/null +++ b/packages/google-crc32c/src/google_crc32c/__init__.py @@ -0,0 +1,39 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import warnings + +_SLOW_CRC32C_WARNING = ( + "As the c extension couldn't be imported, `google-crc32c` is using a " + "pure python implementation that is significantly slower. If possible, " + "please configure a c build environment and compile the extension" +) + +# Default to C exstension Implementation, falling back to pure python. +try: + from google_crc32c import cext as impl + + implementation = "c" +except ImportError: + from google_crc32c import python as impl # type: ignore + + warnings.warn(_SLOW_CRC32C_WARNING, RuntimeWarning) + implementation = "python" + +extend = impl.extend +value = impl.value + +Checksum = impl.Checksum + +__all__ = ["extend", "value", "Checksum", "implementation"] diff --git a/packages/google-crc32c/src/google_crc32c/_checksum.py b/packages/google-crc32c/src/google_crc32c/_checksum.py new file mode 100644 index 000000000000..0aaf0854ce34 --- /dev/null +++ b/packages/google-crc32c/src/google_crc32c/_checksum.py @@ -0,0 +1,88 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + + +class CommonChecksum(object): + """Hashlib-alike helper for CRC32C operations. + + This class should not be used directly and requires an update implementation. + + Args: + initial_value (Optional[bytes]): the initial chunk of data from + which the CRC32C checksum is computed. Defaults to b''. + """ + + __slots__ = () + + def __init__(self, initial_value=b""): + self._crc = 0 + if initial_value != b"": + self.update(initial_value) + + def update(self, data): + """Update the checksum with a new chunk of data. + + Args: + chunk (Optional[bytes]): a chunk of data used to extend + the CRC32C checksum. + """ + raise NotImplementedError + + def digest(self): + """Big-endian order, per RFC 4960. + + See: https://cloud.google.com/storage/docs/json_api/v1/objects#crc32c + + Returns: + bytes: An eight-byte digest string. + """ + return struct.pack(">L", self._crc) + + def hexdigest(self): + """Like :meth:`digest` except returns as a bytestring of double length. + + Returns + bytes: A sixteen byte digest string, contaiing only hex digits. + """ + return "{:08x}".format(self._crc).encode("ascii") + + def copy(self): + """Create another checksum with the same CRC32C value. + + Returns: + Checksum: the new instance. + """ + clone = self.__class__() + clone._crc = self._crc + return clone + + def consume(self, stream, chunksize): + """Consume chunks from a stream, extending our CRC32 checksum. + + Args: + stream (BinaryIO): the stream to consume. + chunksize (int): the size of the read to perform + + Returns: + Generator[bytes, None, None]: Iterable of the chunks read from the + stream. + """ + while True: + chunk = stream.read(chunksize) + if not chunk: + break + self.update(chunk) + yield chunk diff --git a/packages/google-crc32c/src/google_crc32c/_crc32c.c b/packages/google-crc32c/src/google_crc32c/_crc32c.c new file mode 100644 index 000000000000..c84b8839376a --- /dev/null +++ b/packages/google-crc32c/src/google_crc32c/_crc32c.c @@ -0,0 +1,62 @@ +#define PY_SSIZE_T_CLEAN +#include +#include + + +static PyObject * +_crc32c_extend(PyObject *self, PyObject *args) +{ + unsigned long crc_input; + uint32_t crc; + const char *chunk; + Py_ssize_t length; + + if (!PyArg_ParseTuple(args, "ky#", &crc_input, &chunk, &length)) + return NULL; + + crc = crc32c_extend((uint32_t)crc_input, (const uint8_t*)chunk, length); + + return PyLong_FromUnsignedLong(crc); +} + + +static PyObject * +_crc32c_value(PyObject *self, PyObject *args) +{ + uint32_t crc; + const char *chunk; + Py_ssize_t length; + + if (!PyArg_ParseTuple(args, "y#", &chunk, &length)) + return NULL; + + crc = crc32c_value((const uint8_t*)chunk, length); + + return PyLong_FromUnsignedLong(crc); +} + + +static PyMethodDef Crc32cMethods[] = { + {"extend", _crc32c_extend, METH_VARARGS, + "Return an updated CRC32C checksum."}, + {"value", _crc32c_value, METH_VARARGS, + "Return an initial CRC32C checksum."}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + + +static struct PyModuleDef crc32cmodule = { + PyModuleDef_HEAD_INIT, + "_crc32c", /* name of module */ + NULL, /* module documentation, may be NULL */ + -1, /* size of per-interpreter state of the module, + or -1 if the module keeps state in global variables. */ + Crc32cMethods +}; + + +PyMODINIT_FUNC +PyInit__crc32c(void) +{ + return PyModule_Create(&crc32cmodule); +} diff --git a/packages/google-crc32c/src/google_crc32c/cext.py b/packages/google-crc32c/src/google_crc32c/cext.py new file mode 100644 index 000000000000..1ace01c48106 --- /dev/null +++ b/packages/google-crc32c/src/google_crc32c/cext.py @@ -0,0 +1,43 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE: ``__config__`` **must** be the first import because it (may) +# modify the search path used to locate shared libraries. +import google_crc32c.__config__ # noqa: F401 +from google_crc32c._checksum import CommonChecksum +from google_crc32c._crc32c import extend # type: ignore +from google_crc32c._crc32c import value # type: ignore + + +class Checksum(CommonChecksum): + """Hashlib-alike helper for CRC32C operations. + + Args: + initial_value (Optional[bytes]): the initial chunk of data from + which the CRC32C checksum is computed. Defaults to b''. + """ + + __slots__ = ("_crc",) + + def __init__(self, initial_value=b""): + self._crc = value(initial_value) + + def update(self, chunk): + """Update the checksum with a new chunk of data. + + Args: + chunk (Optional[bytes]): a chunk of data used to extend + the CRC32C checksum. + """ + self._crc = extend(self._crc, chunk) diff --git a/packages/google-crc32c/src/google_crc32c/py.typed b/packages/google-crc32c/src/google_crc32c/py.typed new file mode 100644 index 000000000000..076325e24604 --- /dev/null +++ b/packages/google-crc32c/src/google_crc32c/py.typed @@ -0,0 +1,2 @@ +# Marker file for PEP 561. +# The google_crc32c package uses inline types. diff --git a/packages/google-crc32c/src/google_crc32c/python.py b/packages/google-crc32c/src/google_crc32c/python.py new file mode 100644 index 000000000000..04cedabffe26 --- /dev/null +++ b/packages/google-crc32c/src/google_crc32c/python.py @@ -0,0 +1,123 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import array + +from google_crc32c._checksum import CommonChecksum + + +def extend(crc, chunk): + """Update an existing CRC checksum with new chunk of data. + + Args + crc (int): An existing CRC check sum. + chunk (Union[bytes, List[int], Tuple[int]]): A new chunk of data. + Intended to be a byte string or similar. + + Returns + int: New CRC checksum computed by extending existing CRC + with ``chunk``. + """ + c = Checksum() + c._crc = crc + c.update(chunk) + return c._crc + + +def value(chunk): + """Compute a CRC checksum for a chunk of data. + + Args + chunk (Union[bytes, List[int], Tuple[int]]): A new chunk of data. + Intended to be a byte string or similar. + + Returns + int: New CRC checksum computed for ``chunk``. + """ + c = Checksum() + c.update(chunk) + return c._crc + + +class Checksum(CommonChecksum): + """Hashlib-alike helper for CRC32C operations. + + Args: + initial_value (Optional[bytes]): the initial chunk of data from + which the CRC32C checksum is computed. Defaults to b''. + """ + + def __init__(self, initial_value=b""): + self._crc = 0 + if initial_value != b"": + self.update(initial_value) + + def update(self, data): + """Update the checksum with a new chunk of data. + + Args: + chunk (Optional[bytes]): a chunk of data used to extend + the CRC32C checksum. + """ + if not isinstance(data, array.array) or data.itemsize != 1: + buffer = array.array("B", data) + else: + buffer = data + self._crc = self._crc ^ 0xFFFFFFFF + for b in buffer: + table_poly = _TABLE[(b ^ self._crc) & 0xFF] + self._crc = table_poly ^ ((self._crc >> 8) & 0xFFFFFFFF) + self._crc = self._crc ^ 0xFFFFFFFF + + +# fmt:off +_TABLE = [ + 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, 0xc79a971f, 0x35f1141c, 0x26a1e7e8, + 0xd4ca64eb, 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, 0x4d43cfd0, 0xbf284cd3, + 0xac78bf27, 0x5e133c24, 0x105ec76f, 0xe235446c, 0xf165b798, 0x30e349b, 0xd7c45070, + 0x25afd373, 0x36ff2087, 0xc494a384, 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, + 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, + 0x33ed7d2a, 0xe72719c1, 0x154c9ac2, 0x61c6936, 0xf477ea35, 0xaa64d611, 0x580f5512, + 0x4b5fa6e6, 0xb93425e5, 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, 0x30e349b1, + 0xc288cab2, 0xd1d83946, 0x23b3ba45, 0xf779deae, 0x5125dad, 0x1642ae59, 0xe4292d5a, + 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, 0x7da08661, 0x8fcb0562, 0x9c9bf696, + 0x6ef07595, 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, 0x86e18aa3, 0x748a09a0, + 0x67dafa54, 0x95b17957, 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, 0xc38d26c, + 0xfe53516f, 0xed03a29b, 0x1f682198, 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, + 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, 0xdbfc821c, 0x2997011f, 0x3ac7f2eb, + 0xc8ac71e8, 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0xf36e6f7, 0x61c69362, 0x93ad1061, + 0x80fde395, 0x72966096, 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, 0xeb1fcbad, + 0x197448ae, 0xa24bb5a, 0xf84f3859, 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, + 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, 0xb602c312, 0x44694011, 0x5739b3e5, + 0xa55230e6, 0xfb410cc2, 0x92a8fc1, 0x1a7a7c35, 0xe811ff36, 0x3cdb9bdd, 0xceb018de, + 0xdde0eb2a, 0x2f8b6829, 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, 0x456cac67, + 0xb7072f64, 0xa457dc90, 0x563c5f93, 0x82f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, + 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, 0x92a8fc17, 0x60c37f14, 0x73938ce0, + 0x81f80fe3, 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, 0x1871a4d8, 0xea1a27db, + 0xf94ad42f, 0xb21572c, 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, 0xa24bb5a6, + 0x502036a5, 0x4370c551, 0xb11b4652, 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, + 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, 0xef087a76, 0x1d63f975, 0xe330a81, + 0xfc588982, 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, 0x758fe5d6, 0x87e466d5, + 0x94b49521, 0x66df1622, 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, 0xff56bd19, + 0xd3d3e1a, 0x1e6dcdee, 0xec064eed, 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, + 0x417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f, 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, + 0x5a048dff, 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, 0xd3d3e1ab, 0x21b862a8, + 0x32e8915c, 0xc083125f, 0x144976b4, 0xe622f5b7, 0xf5720643, 0x7198540, 0x590ab964, + 0xab613a67, 0xb831c993, 0x4a5a4a90, 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, + 0xe330a81a, 0x115b2b19, 0x20bd8ed, 0xf0605bee, 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, + 0x37faccf1, 0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321, 0xae7367ca, 0x5c18e4c9, + 0x4f48173d, 0xbd23943e, 0xf36e6f75, 0x105ec76, 0x12551f82, 0xe03e9c81, 0x34f4f86a, + 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, + 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351 +] diff --git a/packages/google-crc32c/tests/test___init__.py b/packages/google-crc32c/tests/test___init__.py new file mode 100644 index 000000000000..68bed918ffa4 --- /dev/null +++ b/packages/google-crc32c/tests/test___init__.py @@ -0,0 +1,289 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import functools +import itertools +from unittest import mock + +import pytest # type: ignore + +import google_crc32c + +EMPTY = b"" +EMPTY_CRC = 0x00000000 + +# From: https://tools.ietf.org/html/rfc3720#appendix-B.4 +# +# 32 bytes of zeroes: +# +# Byte: 0 1 2 3 +# +# 0: 00 00 00 00 +# ... +# 28: 00 00 00 00 +# +# CRC: aa 36 91 8a + +ALL_ZEROS = b"\x00" * 32 +ALL_ZEROS_CRC = 0x8A9136AA + +# 32 bytes of ones: +# +# Byte: 0 1 2 3 +# +# 0: ff ff ff ff +# ... +# 28: ff ff ff ff +# +# CRC: 43 ab a8 62 + +ALL_ONES = b"\xff" * 32 +ALL_ONES_CRC = 0x62A8AB43 + +# +# 32 bytes of incrementing 00..1f: +# +# Byte: 0 1 2 3 +# +# 0: 00 01 02 03 +# ... +# 28: 1c 1d 1e 1f +# +# CRC: 4e 79 dd 46 + +INCREMENTING = bytes(range(32)) +INCREMENTING_CRC = 0x46DD794E + +# +# 32 bytes of decrementing 1f..00: +# +# Byte: 0 1 2 3 +# +# 0: 1f 1e 1d 1c +# ... +# 28: 03 02 01 00 +# +# CRC: 5c db 3f 11 + +DECREMENTING = bytes(reversed(range(32))) +DECREMENTING_CRC = 0x113FDB5C + +# +# An iSCSI - SCSI Read (10) Command PDU +# +# Byte: 0 1 2 3 +# +# 0: 01 c0 00 00 +# 4: 00 00 00 00 +# 8: 00 00 00 00 +# 12: 00 00 00 00 +# 16: 14 00 00 00 +# 20: 00 00 04 00 +# 24: 00 00 00 14 +# 28: 00 00 00 18 +# 32: 28 00 00 00 +# 36: 00 00 00 00 +# 40: 02 00 00 00 +# 44: 00 00 00 00 +# +# CRC: 56 3a 96 d9 + +ISCSI_SCSI_READ_10_COMMAND_PDU = [ + 0x01, + 0xC0, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x14, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x04, + 0x00, + 0x00, + 0x00, + 0x00, + 0x14, + 0x00, + 0x00, + 0x00, + 0x18, + 0x28, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x02, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, +] +ISCSI_LENGTH = len(ISCSI_SCSI_READ_10_COMMAND_PDU) +ISCSI_BYTES = bytes(ISCSI_SCSI_READ_10_COMMAND_PDU) +ISCSI_CRC = 0xD9963A56 + + +def iscsi_chunks(chunksize): + for index in itertools.islice(range(ISCSI_LENGTH), 0, None, chunksize): + yield ISCSI_SCSI_READ_10_COMMAND_PDU[index : index + chunksize] + + +_EXPECTED = [ + (EMPTY, EMPTY_CRC), + (ALL_ZEROS, ALL_ZEROS_CRC), + (ALL_ONES, ALL_ONES_CRC), + (INCREMENTING, INCREMENTING_CRC), + (DECREMENTING, DECREMENTING_CRC), + (ISCSI_SCSI_READ_10_COMMAND_PDU, ISCSI_CRC), + (ISCSI_BYTES, ISCSI_CRC), +] + + +def test_extend_w_empty_chunk(): + crc = 123 + assert google_crc32c.extend(crc, b"") == crc + + +def test_extend_w_multiple_chunks(): + crc = 0 + + for chunk in iscsi_chunks(7): + chunk_bytes = bytes(chunk) + crc = google_crc32c.extend(crc, chunk_bytes) + + assert crc == ISCSI_CRC + + +def test_extend_w_reduce(): + chunks = [bytes(chunk) for chunk in iscsi_chunks(3)] + assert functools.reduce(google_crc32c.extend, chunks, 0) == ISCSI_CRC + + +@pytest.mark.parametrize("chunk, expected", _EXPECTED) +def test_value(chunk, expected): + assert google_crc32c.value(bytes(chunk)) == expected + + +def pytest_generate_tests(metafunc): + if "_crc32c" in metafunc.fixturenames: + metafunc.parametrize("_crc32c", ["python", "cext"], indirect=True) + + +@pytest.fixture +def _crc32c(request): + if request.param == "python": + from google_crc32c import python + + return python + elif request.param == "cext": + from google_crc32c import cext + + return cext + else: + raise ValueError("invalid internal test config") + + +class TestChecksum(object): + @staticmethod + def test_ctor_defaults(_crc32c): + helper = google_crc32c.Checksum() + assert helper._crc == 0 + + @staticmethod + def test_ctor_explicit(_crc32c): + chunk = b"DEADBEEF" + helper = google_crc32c.Checksum(chunk) + assert helper._crc == google_crc32c.value(chunk) + + @staticmethod + def test_update(_crc32c): + chunk = b"DEADBEEF" + helper = google_crc32c.Checksum() + helper.update(chunk) + assert helper._crc == google_crc32c.value(chunk) + + @staticmethod + def test_update_w_multiple_chunks(_crc32c): + helper = google_crc32c.Checksum() + + for index in itertools.islice(range(ISCSI_LENGTH), 0, None, 7): + chunk = ISCSI_SCSI_READ_10_COMMAND_PDU[index : index + 7] + helper.update(bytes(chunk)) + + assert helper._crc == ISCSI_CRC + + @staticmethod + def test_digest_zero(_crc32c): + helper = google_crc32c.Checksum() + assert helper.digest() == b"\x00" * 4 + + @staticmethod + def test_digest_nonzero(_crc32c): + helper = google_crc32c.Checksum() + helper._crc = 0x01020304 + assert helper.digest() == b"\x01\x02\x03\x04" + + @staticmethod + def test_hexdigest_zero(_crc32c): + helper = google_crc32c.Checksum() + assert helper.hexdigest() == b"00" * 4 + + @staticmethod + def test_hexdigest_nonzero(_crc32c): + helper = google_crc32c.Checksum() + helper._crc = 0x091A3B2C + assert helper.hexdigest() == b"091a3b2c" + + @staticmethod + def test_copy(_crc32c): + chunk = b"DEADBEEF" + helper = google_crc32c.Checksum(chunk) + clone = helper.copy() + before = helper._crc + helper.update(b"FACEDACE") + assert clone._crc == before + + @staticmethod + @pytest.mark.parametrize("chunksize", [1, 3, 5, 7, 11, 13, ISCSI_LENGTH]) + def test_consume_stream(_crc32c, chunksize): + helper = google_crc32c.Checksum() + expected = [bytes(chunk) for chunk in iscsi_chunks(chunksize)] + stream = mock.Mock(spec=["read"]) + stream.read.side_effect = expected + [b""] + + found = list(helper.consume(stream, chunksize)) + + assert helper._crc == ISCSI_CRC + assert found == expected + for call in stream.read.call_args_list: + assert call == mock.call(chunksize) diff --git a/scripts/split_repo_migration/single-library.git-migrate-history.sh b/scripts/split_repo_migration/single-library.git-migrate-history.sh index 9f926e1d030e..650abac5ca26 100755 --- a/scripts/split_repo_migration/single-library.git-migrate-history.sh +++ b/scripts/split_repo_migration/single-library.git-migrate-history.sh @@ -70,7 +70,7 @@ echo "Created working directory: ${WORKDIR}" pushd "${WORKDIR}" # cd into workdir echo "Cloning source repository: ${SOURCE_REPO}" -git clone --recurse-submodules "git@github.com:${SOURCE_REPO}.git" source-repo +git clone --recurse-submodules --recurse-submodules "git@github.com:${SOURCE_REPO}.git" source-repo pushd source-repo @@ -124,7 +124,7 @@ git filter-branch \ --force \ --prune-empty \ --tree-filter \ - "git submodule update --init --recursive; find . -mindepth 2 -name .git -exec rm -rf {} +; shopt -s dotglob; mkdir -p ${WORKDIR}/migrated-source; mv * ${WORKDIR}/migrated-source; mkdir -p ${TARGET_PATH}; { mv ${WORKDIR}/migrated-source/* ${TARGET_PATH} || echo 'No files to move' ; }" + "git submodule update --init --recursive; find . -mindepth 2 -name .git -exec rm -rf {} +; git submodule update --init --recursive; find . -mindepth 2 -name .git -exec rm -rf {} +; shopt -s dotglob; mkdir -p ${WORKDIR}/migrated-source; mv * ${WORKDIR}/migrated-source; mkdir -p ${TARGET_PATH}; { mv ${WORKDIR}/migrated-source/* ${TARGET_PATH} || echo 'No files to move' ; }" # back to workdir popd @@ -142,7 +142,7 @@ echo "Success" popd # back to workdir # Do a diff between source code split repo and migrated code. -git clone --recurse-submodules "git@github.com:${SOURCE_REPO}.git" source-repo-validation # Not ideal to clone again. +git clone --recurse-submodules --recurse-submodules "git@github.com:${SOURCE_REPO}.git" source-repo-validation # Not ideal to clone again. find source-repo-validation -name .git -exec rm -rf {} + # That folder is not needed for validation. DIFF_FILE="${WORKDIR}/diff.txt"