From 7b62011174e030d696d7ca42eb11ad82f2b19a3d Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Wed, 20 Dec 2023 15:44:45 +0000 Subject: [PATCH] [stable-only] remove pkg_resouces from reqiurements-check this is a squash of 3 commits and a cap of setuptools to < 82 in setup.py Depends-on: https://review.opendev.org/c/openstack/devstack/+/976106 Replace use of distutils.version This is gone in Python 3.12. Ordering is slightly different with release candidates now (correctly, IMO) placed ahead of general releases so we have to update a single entry. codesearch.o.o [1] suggests this shouldn't affect anyone. [1] https://codesearch.opendev.org/?q=pysaml2!%3D4.0.3 Change-Id: I4cde27f048889dd557c0473138bd8a75c1e1cc58 Signed-off-by: Stephen Finucane (cherry picked from commit 2d88770535d5a985a461625fe6443039932671c3) Remove check-conflicts script This script made sense back before pip had a proper dependency resolver, but since pip 20.3 [1] this is no longer the case. Our jobs to test this can simply install the dependencies and let pip complain if it cannot resolve things. [1] https://discuss.python.org/t/announcement-pip-20-3-release/5948 Change-Id: Ifc30478dfffd52f126e0ad7535468cdd0b1cd36a Signed-off-by: Stephen Finucane (cherry picked from commit 63a179077672327ccd02e34010a74f15e9d9eea9) Add pyproject.toml to fix install issues After capping setuptools in setup.py, there are issues when installing our library in a venv. Add a pyproject.toml file to avoid this. Signed-off-by: Dr. Jens Harbott Change-Id: Ib9b7ecde0210a3b11bcb70e96e295cf73f2e0954 (cherry picked from commit ad78d8346bd5e3abd4ba5a901867baab6581f0d8) Replace pkg_resources This will be removed after 2025-11-30. Get ahead of that breakage. Change-Id: Ib97eea1037fac541c8ebadc526ae9c3bb160f685 Signed-off-by: Stephen Finucane (cherry picked from commit 7f008f902d5470093c67d7b03cd3a896712702f0) (cherry picked from commit ed0f971305bee6b48fb803e479c62ab0118c15be) --- detail.py | 14 ++-- global-requirements.txt | 2 +- .../cmds/check_conflicts.py | 75 ------------------- openstack_requirements/constraints.py | 4 +- openstack_requirements/requirement.py | 15 ++-- .../tests/test_requirement.py | 12 +-- pyproject.toml | 3 + setup.cfg | 1 - setup.py | 2 +- tools/cap.py | 9 +-- tools/what-broke.py | 4 +- tox.ini | 6 +- 12 files changed, 38 insertions(+), 109 deletions(-) delete mode 100644 openstack_requirements/cmds/check_conflicts.py create mode 100644 pyproject.toml diff --git a/detail.py b/detail.py index b48d4c44f..a98065a20 100644 --- a/detail.py +++ b/detail.py @@ -20,7 +20,7 @@ import urllib.parse as urlparse import urllib.request as urlreq -import pkg_resources +import packaging.requirement try: PYPI_LOCATION = os.environ['PYPI_LOCATION'] @@ -40,11 +40,11 @@ def iter_names(req): - for k in (req.key, req.project_name): - yield k - yield k.title() - yield k.replace("-", "_") - yield k.replace("-", "_").title() + yield req.name + yield req.name.lower() + yield req.name.title() + yield req.name.replace("-", "_") + yield req.name.replace("-", "_").title() def release_data(req): @@ -76,7 +76,7 @@ def main(): line = line.strip() if line.startswith("#") or not line: continue - req = pkg_resources.Requirement.parse(line) + req = packaging.requirement.Requirement(line) print(" - processing: %s" % (req)) try: raw_req_data = release_data(req) diff --git a/global-requirements.txt b/global-requirements.txt index b2fc87271..8b4ba159b 100644 --- a/global-requirements.txt +++ b/global-requirements.txt @@ -125,7 +125,7 @@ PyMySQL # MIT License pyOpenSSL # Apache-2.0 pyparsing # MIT pyroute2!=0.5.4,!=0.5.5,!=0.7.1;sys_platform!='win32' # Apache-2.0 (+ dual licensed GPL2) -pysaml2!=4.0.3,!=4.0.4,!=4.0.5,!=4.0.5rc1,!=4.1.0,!=4.2.0,!=4.3.0,!=4.4.0,!=4.6.0 # Apache-2.0 +pysaml2!=4.0.3,!=4.0.4,!=4.0.5rc1,!=4.0.5,!=4.1.0,!=4.2.0,!=4.3.0,!=4.4.0,!=4.6.0 # Apache-2.0 pysnmp-lextudio # BSD pystache # MIT # Only required for sasl/binary protocol diff --git a/openstack_requirements/cmds/check_conflicts.py b/openstack_requirements/cmds/check_conflicts.py deleted file mode 100644 index fbea72b6f..000000000 --- a/openstack_requirements/cmds/check_conflicts.py +++ /dev/null @@ -1,75 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Apply validation rules to the various requirements lists. - -""" - -import argparse -import sys -import traceback - -import pkg_resources - -from openstack_requirements.utils import read_requirements_file - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - 'upper_constraints', - default='upper-constraints.txt', - help='path to the upper-constraints.txt file') - parser.add_argument( - 'uc_xfails', - default='upper-constraints-xfails.txt', - help='Path to the upper-constraints-xfails.txt file', - ) - args = parser.parse_args() - - error_count = 0 - - print('\nChecking %s' % args.upper_constraints) - upper_constraints = read_requirements_file(args.upper_constraints) - xfails = read_requirements_file(args.uc_xfails) - for name, spec_list in upper_constraints.items(): - try: - if name: - pyver = "python_version=='%s.%s'" % (sys.version_info[0], - sys.version_info[1]) - for req, original_line in spec_list: - if req.markers in ["", pyver]: - pkg_resources.require(name) - except pkg_resources.ContextualVersionConflict as e: - if e.dist.key in xfails: - xfail_requirement = xfails[e.dist.key][0][0] - xfail_denylists = set(xfail_requirement.markers.split(',')) - conflict = e.dist.as_requirement() - conflict_specifiers = ''.join(conflict.specs[0]) - conflict_name = conflict.name.lower() - - if (e.required_by.issubset(xfail_denylists) and - xfail_requirement.package == conflict_name and - conflict_specifiers == xfail_requirement.specifiers): - - print('XFAIL while checking conflicts ' - 'for %s: %s conflicts with %s' % - (name, e.dist, str(e.req))) - continue - - print('Checking conflicts for %s:\n' - 'ContextualVersionConflict: %s' % (name, str(e))) - - traceback.print_exc(file=sys.stdout) - error_count += 1 - - return 1 if error_count else 0 diff --git a/openstack_requirements/constraints.py b/openstack_requirements/constraints.py index f374f242c..6680d3a20 100644 --- a/openstack_requirements/constraints.py +++ b/openstack_requirements/constraints.py @@ -12,6 +12,8 @@ from packaging import specifiers +from openstack_requirements import requirement + # FIXME(dhellmann): These items were not in the constraints list but # should not be denylisted. We don't know yet what versions they @@ -102,7 +104,7 @@ def satisfied(reqs, name, version, failures): failures = [] for pkg_constraints in constraints.values(): for constraint, _ in pkg_constraints: - name = constraint.package + name = requirement.canonical_name(constraint.package) version = constraint.specifiers[3:] satisfied(global_reqs, name, version, failures) return failures diff --git a/openstack_requirements/requirement.py b/openstack_requirements/requirement.py index 287d063f5..e0e25a4bf 100644 --- a/openstack_requirements/requirement.py +++ b/openstack_requirements/requirement.py @@ -15,9 +15,10 @@ # This module has no IO at all, and none should be added. import collections -import distutils.version +import packaging.requirements import packaging.specifiers -import pkg_resources +import packaging.utils +import packaging.version import re @@ -37,7 +38,7 @@ def key_specifier(a): '===': 1, '==': 1, '~=': 1, '!=': 1, '<': 2, '<=': 2} a = a._spec - return (weight[a[0]], distutils.version.LooseVersion(a[1])) + return (weight[a[0]], packaging.version.parse(a[1])) class Requirement(collections.namedtuple('Requirement', @@ -81,7 +82,7 @@ def to_line(self, marker_sep=';', line_prefix='', comment_prefix=' ', def canonical_name(req_name): """Return the canonical form of req_name.""" - return pkg_resources.safe_name(req_name).lower() + return packaging.utils.canonicalize_name(req_name) def parse(content, permit_urls=False): @@ -127,7 +128,7 @@ def parse_line(req_line, permit_urls=False): hash_pos = hash_pos + parse_start else: # Trigger an early failure before we look for ':' - pkg_resources.Requirement.parse(req_line) + packaging.requirements.Requirement(req_line) else: parse_start = 0 location = '' @@ -149,8 +150,8 @@ def parse_line(req_line, permit_urls=False): specifier = '' elif req_line: # Pulled out a requirement - parsed = pkg_resources.Requirement.parse(req_line) - name = parsed.project_name + parsed = packaging.requirements.Requirement(req_line) + name = parsed.name extras = parsed.extras specifier = str(parsed.specifier) else: diff --git a/openstack_requirements/tests/test_requirement.py b/openstack_requirements/tests/test_requirement.py index 97c5130c2..3e1d972bb 100644 --- a/openstack_requirements/tests/test_requirement.py +++ b/openstack_requirements/tests/test_requirement.py @@ -156,7 +156,7 @@ def test_multiline(self): """) reqs = requirement.parse(content) self.assertEqual( - set(['oslo.config', 'oslo.concurrency', 'oslo.context']), + {'oslo-config', 'oslo-concurrency', 'oslo-context'}, set(reqs.keys()), ) @@ -168,16 +168,16 @@ def test_extras(self): """) reqs = requirement.parse(content) self.assertEqual( - set(['oslo.config', 'oslo.concurrency', 'oslo.db']), + {'oslo-config', 'oslo-concurrency', 'oslo-db'}, set(reqs.keys()), ) - self.assertEqual(reqs['oslo.config'][0][0].extras, frozenset(())) - self.assertEqual(reqs['oslo.concurrency'][0][0].extras, + self.assertEqual(reqs['oslo-config'][0][0].extras, frozenset(())) + self.assertEqual(reqs['oslo-concurrency'][0][0].extras, frozenset(('fixtures',))) - self.assertEqual(reqs['oslo.db'][0][0].extras, + self.assertEqual(reqs['oslo-db'][0][0].extras, frozenset(('fixtures', 'mysql'))) self.assertCountEqual(reqs, - ['oslo.config', 'oslo.concurrency', 'oslo.db']) + ['oslo-config', 'oslo-concurrency', 'oslo-db']) class TestCanonicalName(testtools.TestCase): diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..b59feab27 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["pbr>=6.1.1","setuptools<82"] +build-backend = "pbr.build" diff --git a/setup.cfg b/setup.cfg index 6c17f7c47..514fa070a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,6 @@ packages = console_scripts = edit-constraints = openstack_requirements.cmds.edit_constraint:main generate-constraints = openstack_requirements.cmds.generate:main - check-conflicts = openstack_requirements.cmds.check_conflicts:main validate-constraints = openstack_requirements.cmds.validate:main validate-projects = openstack_requirements.cmds.validate_projects:main normalize-requirements = openstack_requirements.cmds.normalize_requirements:main diff --git a/setup.py b/setup.py index f63cc23c5..0b0e54c04 100644 --- a/setup.py +++ b/setup.py @@ -17,5 +17,5 @@ import setuptools setuptools.setup( - setup_requires=['pbr>=2.0.0'], + setup_requires=['pbr>=2.0.0', 'setuptools<82'], pbr=True) diff --git a/tools/cap.py b/tools/cap.py index cd6ba5675..82ed8e617 100755 --- a/tools/cap.py +++ b/tools/cap.py @@ -12,11 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. - import argparse import re -import pkg_resources +import packaging.requirements overrides = dict() # List of overrides needed. Ignore version in pip-freeze and use the one here @@ -38,7 +37,7 @@ def cap(requirements, frozen): output = [] for line in requirements: try: - req = pkg_resources.Requirement.parse(line) + req = packaging.requirements.Requirement(line) specifier = str(req.specifier) if any(op in specifier for op in ['==', '~=', '<']): # if already capped, continue @@ -67,7 +66,7 @@ def cap(requirements, frozen): def pin(line, new_cap): """Add new cap into existing line - Don't use pkg_resources so we can preserve the comments. + Don't use packaging.requirements so we can preserve the comments. """ end = None use_comma = False @@ -109,7 +108,7 @@ def freeze(lines): for line in lines: try: - req = pkg_resources.Requirement.parse(line) + req = packaging.requirements.Requirement(line) freeze[req.project_name] = req.specifier except ValueError: # not a valid requirement, can be a comment, blank line etc diff --git a/tools/what-broke.py b/tools/what-broke.py index 01194341f..bb2d74e1e 100755 --- a/tools/what-broke.py +++ b/tools/what-broke.py @@ -38,7 +38,7 @@ import sys import urllib.request as urlreq -import pkg_resources +import packaging.requirements class Release(object): @@ -62,7 +62,7 @@ def _parse_pypi_released(datestr): def _package_name(line): - return pkg_resources.Requirement.parse(line).project_name + return packaging.requirements.Requirement(line).name def get_requirements(): diff --git a/tox.ini b/tox.ini index e7671866d..6bdd5145a 100644 --- a/tox.ini +++ b/tox.ini @@ -22,17 +22,17 @@ commands = check-conflicts {toxinidir}/upper-constraints.txt {toxinidir}/upper-c [testenv:py310-check-uc] basepython = python3.10 deps = -r{toxinidir}/upper-constraints.txt -commands = check-conflicts {toxinidir}/upper-constraints.txt {toxinidir}/upper-constraints-xfails.txt +commands = python -c 'print("done")' [testenv:py311-check-uc] basepython = python3.11 deps = -r{toxinidir}/upper-constraints.txt -commands = check-conflicts {toxinidir}/upper-constraints.txt {toxinidir}/upper-constraints-xfails.txt +commands = python -c 'print("done")' [testenv:py312-check-uc] basepython = python3.12 deps = -r{toxinidir}/upper-constraints.txt -commands = check-conflicts {toxinidir}/upper-constraints.txt {toxinidir}/upper-constraints-xfails.txt +commands = python -c 'print("done")' [testenv:venv] commands = {posargs}