Refactor cog scripts to avoid regular expression hacks.

Author: egparedes

constraints.txt

@@ -7,38 +7,38 @@
 aenum==3.1.15             # via dace
 alabaster==0.7.13         # via sphinx
 asttokens==2.4.1          # via devtools, stack-data
-astunparse==1.6.3 ; python_version < "3.9"  # via dace, gt4py (pyproject.toml)
-attrs==23.2.0             # via flake8-bugbear, flake8-eradicate, gt4py (pyproject.toml), hypothesis, jsonschema, referencing
+astunparse==1.6.3 ; python_version < "3.9"  # via dace, gt4py, gt4py (pyproject.toml)
+attrs==23.2.0             # via flake8-bugbear, flake8-eradicate, gt4py, gt4py (pyproject.toml), hypothesis, jsonschema, referencing
 babel==2.14.0             # via sphinx
 backcall==0.2.0           # via ipython
-black==24.1.1             # via gt4py (pyproject.toml)
+black==24.1.1             # via gt4py, gt4py (pyproject.toml)
 blinker==1.7.0            # via flask
-boltons==23.1.1           # via gt4py (pyproject.toml)
+boltons==23.1.1           # via gt4py, gt4py (pyproject.toml)
 build==1.0.3              # via pip-tools
-cached-property==1.5.2    # via gt4py (pyproject.toml)
+cached-property==1.5.2    # via gt4py, gt4py (pyproject.toml)
 cachetools==5.3.2         # via tox
 certifi==2024.2.2         # via requests
 cffi==1.16.0              # via cryptography
 cfgv==3.4.0               # via pre-commit
 chardet==5.2.0            # via tox
 charset-normalizer==3.3.2  # via requests
-clang-format==17.0.6      # via -r requirements-dev.in, gt4py (pyproject.toml)
-click==8.1.7              # via black, flask, gt4py (pyproject.toml), pip-tools
-cmake==3.28.1             # via gt4py (pyproject.toml)
+clang-format==17.0.6      # via -r requirements-dev.in, gt4py
+click==8.1.7              # via black, flask, gt4py, gt4py (pyproject.toml), pip-tools
+cmake==3.28.1             # via gt4py, gt4py (pyproject.toml)
 cogapp==3.3.0             # via -r requirements-dev.in
 colorama==0.4.6           # via tox
 comm==0.2.1               # via ipykernel
 contourpy==1.1.1          # via matplotlib
 coverage==7.4.1           # via -r requirements-dev.in, pytest-cov
 cryptography==42.0.2      # via types-paramiko, types-pyopenssl, types-redis
 cycler==0.12.1            # via matplotlib
-cytoolz==0.12.3           # via gt4py (pyproject.toml)
-dace==0.15.1              # via gt4py (pyproject.toml)
+cytoolz==0.12.3           # via gt4py, gt4py (pyproject.toml)
+dace==0.15.1              # via gt4py
 darglint==1.8.1           # via -r requirements-dev.in
 debugpy==1.8.0            # via ipykernel
 decorator==5.1.1          # via ipython
-deepdiff==6.7.1           # via gt4py (pyproject.toml)
-devtools==0.12.2          # via gt4py (pyproject.toml)
+deepdiff==6.7.1           # via gt4py, gt4py (pyproject.toml)
+devtools==0.12.2          # via gt4py, gt4py (pyproject.toml)
 dill==0.3.8               # via dace
 distlib==0.3.8            # via virtualenv
 docutils==0.20.1          # via restructuredtext-lint, sphinx, sphinx-rtd-theme
@@ -59,54 +59,59 @@ flake8-eradicate==1.5.0   # via -r requirements-dev.in
 flake8-mutable==1.2.0     # via -r requirements-dev.in
 flake8-pyproject==1.2.3   # via -r requirements-dev.in
 flake8-rst-docstrings==0.3.0  # via -r requirements-dev.in
-flask==3.0.1              # via dace
+flask==3.0.2              # via dace
 fonttools==4.47.2         # via matplotlib
 fparser==0.1.4            # via dace
-frozendict==2.4.0         # via gt4py (pyproject.toml)
-gridtools-cpp==2.3.2      # via gt4py (pyproject.toml)
-hypothesis==6.97.4        # via -r requirements-dev.in, gt4py (pyproject.toml)
+frozendict==2.4.0         # via gt4py, gt4py (pyproject.toml)
+gridtools-cpp==2.3.2      # via gt4py, gt4py (pyproject.toml)
+gt4py @ file:///home/enriqueg/Projects/GT/gt4py/wip-settings  # via gt4py (pyproject.toml)
+hypothesis==6.98.1        # via -r requirements-dev.in, gt4py
 identify==2.5.33          # via pre-commit
 idna==3.6                 # via requests
 imagesize==1.4.1          # via sphinx
-importlib-metadata==7.0.1  # via build, flask, jupyter-client, sphinx
-importlib-resources==6.1.1 ; python_version < "3.9"  # via gt4py (pyproject.toml), jsonschema, jsonschema-specifications, matplotlib
+importlib-metadata==7.0.1  # via build, flask, jax, jupyter-client, sphinx
+importlib-resources==6.1.1 ; python_version < "3.9"  # via gt4py, gt4py (pyproject.toml), jsonschema, jsonschema-specifications, matplotlib
 inflection==0.5.1         # via pytest-factoryboy
 iniconfig==2.0.0          # via pytest
 ipykernel==6.29.0         # via nbmake
 ipython==8.12.3           # via ipykernel
 isort==5.13.2             # via -r requirements-dev.in
 itsdangerous==2.1.2       # via flask
+jax==0.4.13               # via gt4py
+jaxlib==0.4.13            # via jax
 jedi==0.19.1              # via ipython
-jinja2==3.1.3             # via flask, gt4py (pyproject.toml), sphinx
+jinja2==3.1.3             # via flask, gt4py, gt4py (pyproject.toml), sphinx
 jsonschema==4.21.1        # via nbformat
 jsonschema-specifications==2023.12.1  # via jsonschema
 jupyter-client==8.6.0     # via ipykernel, nbclient
 jupyter-core==5.7.1       # via ipykernel, jupyter-client, nbformat
 jupytext==1.16.1          # via -r requirements-dev.in
 kiwisolver==1.4.5         # via matplotlib
-lark==1.1.9               # via gt4py (pyproject.toml)
-mako==1.3.2               # via gt4py (pyproject.toml)
+lark==1.1.9               # via gt4py, gt4py (pyproject.toml)
+mako==1.3.2               # via gt4py, gt4py (pyproject.toml)
 markdown-it-py==3.0.0     # via jupytext, mdit-py-plugins
-markupsafe==2.1.4         # via jinja2, mako, werkzeug
+markupsafe==2.1.5         # via jinja2, mako, werkzeug
 matplotlib==3.7.4         # via -r requirements-dev.in
 matplotlib-inline==0.1.6  # via ipykernel, ipython
 mccabe==0.7.0             # via flake8
 mdit-py-plugins==0.4.0    # via jupytext
 mdurl==0.1.2              # via markdown-it-py
+ml-dtypes==0.2.0          # via jax, jaxlib
 mpmath==1.3.0             # via sympy
 mypy==1.8.0               # via -r requirements-dev.in
 mypy-extensions==1.0.0    # via black, mypy
-nanobind==1.8.0           # via gt4py (pyproject.toml)
+nanobind==1.8.0           # via gt4py, gt4py (pyproject.toml)
 nbclient==0.6.8           # via nbmake
 nbformat==5.9.2           # via jupytext, nbclient, nbmake
 nbmake==1.5.0             # via -r requirements-dev.in
 nest-asyncio==1.6.0       # via ipykernel, nbclient
 networkx==3.1             # via dace
-ninja==1.11.1.1           # via gt4py (pyproject.toml)
+ninja==1.11.1.1           # via gt4py, gt4py (pyproject.toml)
 nodeenv==1.8.0            # via pre-commit
-numpy==1.24.4             # via contourpy, dace, gt4py (pyproject.toml), matplotlib, types-jack-client
+numpy==1.24.4             # via contourpy, dace, gt4py, gt4py (pyproject.toml), jax, jaxlib, matplotlib, ml-dtypes, opt-einsum, scipy, types-jack-client
+opt-einsum==3.3.0         # via jax
 ordered-set==4.1.0        # via deepdiff
-packaging==23.2           # via black, build, gt4py (pyproject.toml), ipykernel, jupytext, matplotlib, pyproject-api, pytest, setuptools-scm, sphinx, tox
+packaging==23.2           # via black, build, gt4py, gt4py (pyproject.toml), ipykernel, jupytext, matplotlib, pyproject-api, pytest, setuptools-scm, sphinx, tox
 parso==0.8.3              # via jedi
 pathspec==0.12.1          # via black
 pexpect==4.9.0            # via ipython
@@ -123,7 +128,7 @@ prompt-toolkit==3.0.43    # via ipython
 psutil==5.9.8             # via -r requirements-dev.in, ipykernel, pytest-xdist
 ptyprocess==0.7.0         # via pexpect
 pure-eval==0.2.2          # via stack-data
-pybind11==2.11.1          # via gt4py (pyproject.toml)
+pybind11==2.11.1          # via gt4py, gt4py (pyproject.toml)
 pycodestyle==2.11.1       # via flake8, flake8-debugger
 pycparser==2.21           # via cffi
 pydocstyle==6.3.0         # via flake8-docstrings
@@ -132,7 +137,7 @@ pygments==2.17.2          # via -r requirements-dev.in, devtools, flake8-rst-doc
 pyparsing==3.1.1          # via matplotlib
 pyproject-api==1.6.1      # via tox
 pyproject-hooks==1.0.0    # via build
-pytest==8.0.0             # via -r requirements-dev.in, gt4py (pyproject.toml), nbmake, pytest-cache, pytest-cov, pytest-factoryboy, pytest-xdist
+pytest==8.0.0             # via -r requirements-dev.in, gt4py, nbmake, pytest-cache, pytest-cov, pytest-factoryboy, pytest-xdist
 pytest-cache==1.0         # via -r requirements-dev.in
 pytest-cov==4.1.0         # via -r requirements-dev.in
 pytest-factoryboy==2.6.0  # via -r requirements-dev.in
@@ -146,6 +151,7 @@ requests==2.31.0          # via dace, sphinx
 restructuredtext-lint==1.4.0  # via flake8-rst-docstrings
 rpds-py==0.17.1           # via jsonschema, referencing
 ruff==0.2.0               # via -r requirements-dev.in
+scipy==1.10.1             # via gt4py, jax, jaxlib
 setuptools-scm==8.0.4     # via fparser
 six==1.16.0               # via asttokens, astunparse, python-dateutil
 snowballstemmer==2.2.0    # via pydocstyle, sphinx
@@ -160,8 +166,8 @@ sphinxcontrib-jsmath==1.0.1  # via sphinx
 sphinxcontrib-qthelp==1.0.3  # via sphinx
 sphinxcontrib-serializinghtml==1.1.5  # via sphinx
 stack-data==0.6.3         # via ipython
-sympy==1.9                # via dace, gt4py (pyproject.toml)
-tabulate==0.9.0           # via gt4py (pyproject.toml)
+sympy==1.9                # via dace, gt4py
+tabulate==0.9.0           # via gt4py, gt4py (pyproject.toml)
 toml==0.10.2              # via jupytext
 tomli==2.0.1 ; python_version < "3.11"  # via -r requirements-dev.in, black, build, coverage, flake8-pyproject, mypy, pip-tools, pyproject-api, pyproject-hooks, pytest, setuptools-scm, tox
 toolz==0.12.1             # via cytoolz
@@ -175,15 +181,15 @@ types-atomicwrites==1.4.5.1  # via types-all
 types-backports==0.1.3    # via types-all
 types-backports-abc==0.5.2  # via types-all
 types-bleach==6.1.0.1     # via types-all
-types-boto==2.49.18.9     # via types-all
+types-boto==2.49.18.20240205  # via types-all
 types-cachetools==5.3.0.7  # via types-all
 types-certifi==2021.10.8.3  # via types-all
 types-cffi==1.16.0.20240106  # via types-jack-client
 types-characteristic==14.3.7  # via types-all
 types-chardet==5.0.4.6    # via types-all
 types-click==7.1.8        # via types-all, types-flask
 types-click-spinner==0.1.13.20240106  # via types-all
-types-colorama==0.4.15.20240106  # via types-all
+types-colorama==0.4.15.20240205  # via types-all
 types-contextvars==2.4.7.3  # via types-all
 types-croniter==2.0.0.20240106  # via types-all
 types-cryptography==3.3.23.2  # via types-all, types-openssl-python, types-pyjwt
@@ -217,7 +223,7 @@ types-mypy-extensions==1.0.0.5  # via types-all
 types-nmap==0.1.6         # via types-all
 types-openssl-python==0.1.3  # via types-all
 types-orjson==3.6.2       # via types-all
-types-paramiko==3.4.0.20240120  # via types-all, types-pysftp
+types-paramiko==3.4.0.20240205  # via types-all, types-pysftp
 types-pathlib2==2.3.0     # via types-all
 types-pillow==10.2.0.20240125  # via types-all
 types-pkg-resources==0.1.3  # via types-all
@@ -235,7 +241,7 @@ types-pysftp==0.2.17.20240106  # via types-all
 types-python-dateutil==2.8.19.20240106  # via types-all, types-datetimerange
 types-python-gflags==3.1.7.3  # via types-all
 types-python-slugify==8.0.2.20240127  # via types-all
-types-pytz==2023.4.0.20240130  # via types-all, types-tzlocal
+types-pytz==2024.1.0.20240203  # via types-all, types-tzlocal
 types-pyvmomi==8.0.0.6    # via types-all
 types-pyyaml==6.0.12.12   # via types-all
 types-redis==4.6.0.20240106  # via types-all
@@ -257,16 +263,16 @@ types-ujson==5.9.0.0      # via types-all
 types-waitress==2.1.4.20240106  # via types-all
 types-werkzeug==1.0.9     # via types-all, types-flask
 types-xxhash==3.0.5.2     # via types-all
-typing-extensions==4.5.0  # via black, faker, gt4py (pyproject.toml), ipython, mypy, pytest-factoryboy, setuptools-scm
+typing-extensions==4.5.0  # via black, faker, gt4py, gt4py (pyproject.toml), ipython, mypy, pytest-factoryboy, setuptools-scm
 urllib3==2.2.0            # via requests, types-requests
 virtualenv==20.25.0       # via pre-commit, tox
 wcwidth==0.2.13           # via prompt-toolkit
 websockets==12.0          # via dace
 werkzeug==3.0.1           # via flask
 wheel==0.42.0             # via astunparse, pip-tools
-xxhash==3.0.0             # via gt4py (pyproject.toml)
+xxhash==3.0.0             # via gt4py, gt4py (pyproject.toml)
 zipp==3.17.0              # via importlib-metadata, importlib-resources
 
 # The following packages are considered to be unsafe in a requirements file:
-pip==23.3.2               # via pip-tools
-setuptools==69.0.3        # via gt4py (pyproject.toml), nodeenv, pip-tools, setuptools-scm
+pip==24.0                 # via pip-tools
+setuptools==69.0.3        # via gt4py, gt4py (pyproject.toml), nodeenv, pip-tools, setuptools-scm

min-extra-requirements-test.txt

@@ -1,24 +1,55 @@
+#
+# Generated automatically by cog from pyproject.toml and requirements-dev.in
+# Run: 
+#       tox r -e requirements-common
+#
+
 ##[[[cog
-## import re, sys
+## import copy, sys
+## from packaging import requirements as reqs, specifiers as specs
 ## if sys.version_info >= (3, 11):
-##    import tomllib
+##     import tomllib
 ## else:
-##    import tomli as tomllib
+##     import tomli as tomllib
+##
+## def make_min_req(r: reqs.Requirement) -> reqs.Requirement:
+##     for s in r.specifier:
+##        if (ss := str(s)).startswith(">"):
+##            min_spec = f"=={ss[1:]}" if ss[1] != "=" else f"=={ss[2:]}"
+##            spec = specs.SpecifierSet(min_spec)
+##            break
+##     min_r = copy.deepcopy(r)
+##     min_r.specifier = spec
+##     return min_r
+##
 ## project = tomllib.loads(open("pyproject.toml").read())
-## requirements = set(
-##     rr
-##     for r in (
-##         project["project"]["dependencies"]
-##         + project["project"]["optional-dependencies"]["full"]
-##         + open("requirements-dev.in").readlines()
-##     )
-##     if (rr := r.strip()) and not rr.startswith('#')
-## )
-## for r in sorted(requirements):
-##     m = re.match("^([\w-][\w\d\[\]-]*)[=>~][=]([^,]+)", r)
-##     print(f"{m[1]}=={m[2].strip()}")
+## all_cpu_extra = project["project"]["optional-dependencies"]["all-cpu"]
+## assert len(all_cpu_extra) == 1 and all_cpu_extra[0].startswith("gt4py[")
+## opt_req_versions = {
+##     reqs.Requirement(r).name: reqs.Requirement(r)
+##     for e in reqs.Requirement(all_cpu_extra[0]).extras
+##     for r in project["project"]["optional-dependencies"][e]
+## }
+## requirements = [
+##     reqs.Requirement(rr)
+##     for r in (project["project"]["dependencies"] + open("requirements-dev.in").readlines())
+##     if (rr := (r[: r.find("#")] if "#" in r else r))
+## ]
+## processed = set()
+## result = []
+## for r in requirements:
+##     assert r.name not in processed
+##     processed.add(r.name)
+##     if not r.specifier:
+##         assert r.name in opt_req_versions, f"Missing contraints for '{r.name}'"
+##         r = opt_req_versions[r.name]
+##     result.append(str(make_min_req(r)))
+## for r_name, r in opt_req_versions.items():
+##     if r_name not in processed:
+##         result.append(str(make_min_req(r)))
+## print("\n".join(sorted(result)))
 ##]]]
-astunparse==1.6.3;python_version<'3.9'
+astunparse==1.6.3; python_version < "3.9"
 attrs==21.3
 black==22.3
 boltons==20.1
@@ -46,7 +77,7 @@ flake8==5.0.4
 frozendict==2.3
 gridtools-cpp==2.3.2
 hypothesis==6.0.0
-importlib-resources==5.0;python_version<'3.9'
+importlib-resources==5.0; python_version < "3.9"
 isort==5.10
 jax[cpu]==0.4.13
 jinja2==3.0.0
@@ -78,7 +109,7 @@ sphinx==4.4
 sphinx_rtd_theme==1.0
 sympy==1.9
 tabulate==0.8.10
-tomli==2.0.1;python_version<'3.11'
+tomli==2.0.1; python_version < "3.11"
 tox==3.2.0
 types-all==1.0.0
 typing-extensions==4.2

min-requirements-test.txt

@@ -1,23 +1,52 @@
+#
+# Generated automatically by cog from pyproject.toml and requirements-dev.in
+# Run: 
+#       tox r -e requirements-common
+#
+
 ##[[[cog
-## import re, sys
+## import copy, sys
+## from packaging import requirements as reqs, specifiers as specs
 ## if sys.version_info >= (3, 11):
-##    import tomllib
+##     import tomllib
 ## else:
-##    import tomli as tomllib
+##     import tomli as tomllib
+##
+## def make_min_req(r: reqs.Requirement) -> reqs.Requirement:
+##     for s in r.specifier:
+##        if (ss := str(s)).startswith(">"):
+##            min_spec = f"=={ss[1:]}" if ss[1] != "=" else f"=={ss[2:]}"
+##            spec = specs.SpecifierSet(min_spec)
+##            break
+##     min_r = copy.deepcopy(r)
+##     min_r.specifier = spec
+##     return min_r
+##
 ## project = tomllib.loads(open("pyproject.toml").read())
-## requirements = set(
-##     rr
-##     for r in (
-##         project["project"]["dependencies"]
-##         + open("requirements-dev.in").readlines()
-##     )
-##     if (rr := r.strip()) and not rr.startswith('#')
-## )
-## for r in sorted(requirements):
-##     m = re.match("^([\w-][\w\d\[\]-]*)[=>~][=]([^,]+)", r)
-##     print(f"{m[1]}=={m[2].strip()}")
+## all_cpu_extra = project["project"]["optional-dependencies"]["all-cpu"]
+## assert len(all_cpu_extra) == 1 and all_cpu_extra[0].startswith("gt4py[")
+## opt_req_versions = {
+##     reqs.Requirement(r).name: reqs.Requirement(r)
+##     for e in reqs.Requirement(all_cpu_extra[0]).extras
+##     for r in project["project"]["optional-dependencies"][e]
+## }
+## requirements = [
+##     reqs.Requirement(rr)
+##     for r in (project["project"]["dependencies"] + open("requirements-dev.in").readlines())
+##     if (rr := (r[: r.find("#")] if "#" in r else r))
+## ]
+## processed = set()
+## result = []
+## for r in requirements:
+##     assert r.name not in processed
+##     processed.add(r.name)
+##     if not r.specifier:
+##         assert r.name in opt_req_versions, f"Missing contraints for '{r.name}'"
+##         r = opt_req_versions[r.name]
+##     result.append(str(make_min_req(r)))
+## print("\n".join(sorted(result)))
 ##]]]
-astunparse==1.6.3;python_version<'3.9'
+astunparse==1.6.3; python_version < "3.9"
 attrs==21.3
 black==22.3
 boltons==20.1
@@ -44,7 +73,7 @@ flake8==5.0.4
 frozendict==2.3
 gridtools-cpp==2.3.2
 hypothesis==6.0.0
-importlib-resources==5.0;python_version<'3.9'
+importlib-resources==5.0; python_version < "3.9"
 isort==5.10
 jinja2==3.0.0
 jupytext==1.14
@@ -73,7 +102,7 @@ setuptools==65.5.0
 sphinx==4.4
 sphinx_rtd_theme==1.0
 tabulate==0.8.10
-tomli==2.0.1;python_version<'3.11'
+tomli==2.0.1; python_version < "3.11"
 tox==3.2.0
 types-all==1.0.0
 typing-extensions==4.2