util.py: include non prod versions in constraints.txt (#1234)

requirements file still contains only prod versions (from versions-prod.cfg) but constraints is extended to also contain versions from versions.cfg, so that during tests we can install extras with pip, like we do for universal2 on OSX. This fixes a problem of test failing on OSX because the version pin for MarkupSafe was missing and when installing it with pip, the latest version was installed, which might not be the version defined in versions.cfg, causing buildout to install again MarkupSafe at the version defined in versions.cfg and failing. Co-authored-by: Jens Vagelpohl <[email protected]>
zopefoundation · Nov 3, 2024 · d54a989 · d54a989
1 parent f89f522
commit d54a989
Show file tree

Hide file tree

Showing 3 changed files with 77 additions and 19 deletions.
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -14,14 +14,18 @@ https://github.com/zopefoundation/Zope/blob/4.x/CHANGES.rst
 5.11.1 (2024-11-03)
 -------------------
 
+- Include versions constraints for production and non-production dependencies
+  in ``constraints.txt``.
+  (`#1234 <https://github.com/zopefoundation/Zope/pull/1234>`_)
+
 - Update to newest compatible versions of dependencies.
 
 - Define ``request["PARENTS"]`` before request parameter processing
   such that request failure subscribers (such as that of
   ``Products.SiteErrorLog``) can determine the error context
   for exceptions raised during request parameter processing
   (see `#1235 <https://github.com/zopefoundation/Zope/issues/1235>`_).
-   
+
 
 5.11 (2024-10-11)
 -----------------

diff --git a/constraints.txt b/constraints.txt
@@ -2,39 +2,91 @@ AccessControl==7.2
 Acquisition==6.1
 AuthEncoding==5.0
 BTrees==6.1
+Babel==2.16.0
 Chameleon==4.4.4; python_version == '3.8'
 Chameleon==4.4.4; python_version > '3.8'
 DateTime==5.5
 DocumentTemplate==4.6
 ExtensionClass==6.0
+Jinja2==3.1.4
+MarkupSafe==2.1.5; python_version == '3.8'
+MarkupSafe==3.0.2; python_version > '3.8'
+Missing==5.0
 MultiMapping==5.0
 Paste==3.10.1
 PasteDeploy==3.1.0
 Persistence==5.1
+Products.BTreeFolder2==5.1
+Products.ZCatalog==7.1
+Pygments==2.18.0
+Record==4.1
 RestrictedPython==7.4
+Sphinx==7.1.2; python_version == '3.8'
+Sphinx==7.4.7; python_version == '3.9'
+Sphinx==8.1.3; python_version > '3.9'
 WSGIProxy2==0.5.1
 WebOb==1.8.9
 WebTest==3.0.1
 ZConfig==4.1
 ZODB==6.0
 Zope2==4.0
+alabaster==0.7.13; python_version == '3.8'
+alabaster==0.7.14; python_version == '3.9'
+alabaster==1.0.0; python_version > '3.9'
 beautifulsoup4==4.12.3
+certifi==2024.8.30
 cffi==1.17.1
+charset-normalizer==3.4.0
+collective.recipe.template==2.2
+colorama==0.4.6
+docutils==0.20.1; python_version == '3.8'
+docutils==0.20.1; python_version == '3.9'
+docutils==0.21.2; python_version > '3.9'
+five.localsitemanager==4.0
+furo==2024.8.6
+idna==3.10
+imagesize==1.4.1
+importlib-metadata==8.5.0
+importlib-resources==6.4.0; python_version == '3.8'
+legacy-cgi==2.6.1
+mr.developer==2.0.2
 multipart==0.2.5
+packaging==24.1
 persistent==6.1
+plone.recipe.command==1.1
 pycparser==2.22
 python-gettext==5.0
 pytz==2024.2
+requests==2.32.3
 roman==4.2
 six==1.16.0
+snowballstemmer==2.2.0
 soupsieve==2.6
+sphinx-basic-ng==1.0.0b2
+sphinxcontrib-applehelp==1.0.4; python_version == '3.8'
+sphinxcontrib-applehelp==2.0.0; python_version > '3.8'
+sphinxcontrib-devhelp==1.0.2; python_version == '3.8'
+sphinxcontrib-devhelp==2.0.0; python_version > '3.8'
+sphinxcontrib-htmlhelp==2.0.1; python_version == '3.8'
+sphinxcontrib-htmlhelp==2.1.0; python_version > '3.8'
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3; python_version == '3.8'
+sphinxcontrib-qthelp==2.0.0; python_version > '3.8'
+sphinxcontrib-serializinghtml==1.1.5; python_version == '3.8'
+sphinxcontrib-serializinghtml==2.0.0; python_version > '3.8'
+tempstorage==6.0
+tomli==2.0.2
 transaction==5.0
+urllib3==2.2.3
 waitress==3.0.0; python_version == '3.8'
 waitress==3.0.1; python_version > '3.8'
+z3c.checkversions==2.1
 z3c.pt==4.4
 zExceptions==5.0
 zc.lockfile==3.0.post1
 zc.recipe.egg==2.0.7
+zc.recipe.testrunner==3.1
+zipp==3.20.2
 zodbpickle==4.1.1
 zope.annotation==5.0
 zope.browser==3.0
@@ -76,5 +128,6 @@ zope.tal==5.0.1
 zope.tales==6.0
 zope.testbrowser==7.0
 zope.testing==5.0.1
+zope.testrunner==6.6
 zope.traversing==5.0
 zope.viewlet==5.0
diff --git a/util.py b/util.py
@@ -16,51 +16,48 @@ def optionxform(self, value):
         return value
 
 
-def generate(in_, requirements_file, constraints_file):
-    in_file = os.path.join(HERE, in_)
+def generate(
+        buildout_file: str,
+        requirements_file: str,
+        zope_requirement: str):
     out_file_requirements = os.path.join(HERE, requirements_file)
-    out_file_constraints = os.path.join(HERE, constraints_file)
     parser = CaseSensitiveParser()
-    parser.read(in_file)
+    parser.read(os.path.join(HERE, buildout_file))
+    for extend in parser.get('buildout', 'extends', fallback='').splitlines():
+        extend = extend.strip()
+        if extend:
+            parser.read(os.path.join(HERE, extend))
 
     requirements = []
-    constraints = []
-    zope_requirement = (
-        '-e git+https://github.com/zopefoundation/Zope.git@master#egg=Zope\n')
 
     # Try to include sections for all currently supported Python versions
-    for py_version in ('3.8', '3.9', '3.10', '3.11', '3.12'):
+    for py_version in ('3.8', '3.9', '3.10', '3.11', '3.12', '3.13'):
         short_version = py_version.replace('.', '')
         try:
             zope_requirement = _generate(
                 parser.items(f'versions:python{short_version}'), py_version,
-                requirements, constraints, zope_requirement)
+                requirements, zope_requirement)
         except NoSectionError:
             continue
 
     # "Unversioned" pins must come last, how they are handled depends on
     # Python version qualifiers for dependencies of the same name.
     zope_requirement = _generate(
-        parser.items('versions'), None, requirements, constraints,
-        zope_requirement)
+        parser.items('versions'), None, requirements, zope_requirement)
 
     with open(out_file_requirements, 'w') as fd:
         fd.write(zope_requirement)
         for req in sorted(requirements):
             fd.write(req)
-    with open(out_file_constraints, 'w') as fcon:
-        for con in sorted(constraints):
-            fcon.write(con)
 
 
 def _generate(
     versions: List[Tuple[str, str]],
     python_version: Optional[str],
     requirements: List[str],
-    constraints: List[str],
     zope_requirement: str
 ) -> str:
-    """Generate requirements and constraints for a specific Python version.
+    """Generate requirements or constraints file for a specific Python version.
 
     If ``python_version`` is false, generate for all python versions.
     Returns a probably changed ``zope_requirement``.
@@ -90,12 +87,16 @@ def _generate(
                 spec = f"{spec}; python_version > '{versions[-1]}'"
 
         requirements.append(spec + '\n')
-        constraints.append(spec + '\n')
     return zope_requirement
 
 
 def main():
-    generate('versions-prod.cfg', 'requirements-full.txt', 'constraints.txt')
+    generate(
+        'versions-prod.cfg',
+        'requirements-full.txt',
+        '-e git+https://github.com/zopefoundation/Zope.git@master#egg=Zope\n',
+    )
+    generate('versions.cfg', 'constraints.txt', '')
 
 
 if __name__ == '__main__':