build: Update requirements versions and scripts to support 3.11

.pre-commit-config.yaml

@@ -67,6 +67,18 @@ repos:
   hooks:
   - id: black
 
+# - repo: https://github.com/charliermarsh/ruff-pre-commit
+#   ##[[[cog
+#   ## import re
+#   ## version = re.search('ruff==([0-9\.]*)', open("constraints.txt").read())[1] 
+#   ## print(f"#   rev: 'v{version}'  # version from constraints.txt")
+#   ##]]]
+#   rev: 'v0.2.0'  # version from constraints.txt
+#   ##[[[end]]]
+#   hooks:
+#   - id: ruff
+#     #  args: [ --fix, --exit-non-zero-on-fix ]
+
 - repo: https://github.com/PyCQA/isort
   ##[[[cog
   ## import re
@@ -153,9 +165,13 @@ repos:
   - id: mypy
     additional_dependencies:  # versions from constraints.txt
     ##[[[cog
-    ## import re, tomli
+    ## import re, sys
+    ## if sys.version_info >= (3, 11):
+    ##    import tomllib
+    ## else:
+    ##    import tomli as tomllib
     ## constraints = open("constraints.txt").read()
-    ## project = tomli.loads(open("pyproject.toml").read())
+    ## project = tomllib.loads(open("pyproject.toml").read())
     ## packages = [re.match('^([\w-][\w\d-]*)', r)[1] for r in project["project"]["dependencies"] if r.strip()]
     ## for pkg in packages:
     ##     print(f"- {pkg}==" + str(re.search(f'\n{pkg}==([0-9\.]*)', constraints)[1]))

constraints.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.8
 # by the following command:
 #
-#    "tox run -e requirements-common"
+#    "tox run -e requirements-base"
 #
 aenum==3.1.15             # via dace
 alabaster==0.7.13         # via sphinx
@@ -17,7 +17,7 @@ boltons==23.1.1           # via gt4py (pyproject.toml)
 build==1.0.3              # via pip-tools
 cached-property==1.5.2    # via gt4py (pyproject.toml)
 cachetools==5.3.2         # via tox
-certifi==2023.11.17       # via requests
+certifi==2024.2.2         # via requests
 cffi==1.16.0              # via cryptography
 cfgv==3.4.0               # via pre-commit
 chardet==5.2.0            # via tox
@@ -29,7 +29,7 @@ cogapp==3.3.0             # via -r requirements-dev.in
 colorama==0.4.6           # via tox
 comm==0.2.1               # via ipykernel
 contourpy==1.1.1          # via matplotlib
-coverage==7.4.1           # via -r requirements-dev.in, coverage, pytest-cov
+coverage==7.4.1           # via -r requirements-dev.in, pytest-cov
 cryptography==42.0.2      # via types-paramiko, types-pyopenssl, types-redis
 cycler==0.12.1            # via matplotlib
 cytoolz==0.12.3           # via gt4py (pyproject.toml)
@@ -47,7 +47,7 @@ exceptiongroup==1.2.0     # via hypothesis, pytest
 execnet==2.0.2            # via pytest-cache, pytest-xdist
 executing==2.0.1          # via devtools, stack-data
 factory-boy==3.3.0        # via -r requirements-dev.in, pytest-factoryboy
-faker==22.6.0             # via factory-boy
+faker==22.7.0             # via factory-boy
 fastjsonschema==2.19.1    # via nbformat
 filelock==3.13.1          # via tox, virtualenv
 flake8==7.0.0             # via -r requirements-dev.in, flake8-bugbear, flake8-builtins, flake8-debugger, flake8-docstrings, flake8-eradicate, flake8-mutable, flake8-pyproject, flake8-rst-docstrings
@@ -59,23 +59,25 @@ flake8-eradicate==1.5.0   # via -r requirements-dev.in
 flake8-mutable==1.2.0     # via -r requirements-dev.in
 flake8-pyproject==1.2.3   # via -r requirements-dev.in
 flake8-rst-docstrings==0.3.0  # via -r requirements-dev.in
-flask==3.0.1              # via dace
+flask==3.0.2              # via dace
 fonttools==4.47.2         # via matplotlib
 fparser==0.1.4            # via dace
 frozendict==2.4.0         # via gt4py (pyproject.toml)
 gridtools-cpp==2.3.2      # via gt4py (pyproject.toml)
-hypothesis==6.97.3        # via -r requirements-dev.in, gt4py (pyproject.toml)
+hypothesis==6.98.2        # via -r requirements-dev.in, gt4py (pyproject.toml)
 identify==2.5.33          # via pre-commit
 idna==3.6                 # via requests
 imagesize==1.4.1          # via sphinx
-importlib-metadata==7.0.1  # via build, flask, jupyter-client, sphinx
+importlib-metadata==7.0.1  # via build, flask, jax, jupyter-client, sphinx
 importlib-resources==6.1.1 ; python_version < "3.9"  # via gt4py (pyproject.toml), jsonschema, jsonschema-specifications, matplotlib
 inflection==0.5.1         # via pytest-factoryboy
 iniconfig==2.0.0          # via pytest
-ipykernel==6.29.0         # via nbmake
+ipykernel==6.29.1         # via nbmake
 ipython==8.12.3           # via ipykernel
 isort==5.13.2             # via -r requirements-dev.in
 itsdangerous==2.1.2       # via flask
+jax==0.4.13               # via gt4py (pyproject.toml)
+jaxlib==0.4.13            # via jax
 jedi==0.19.1              # via ipython
 jinja2==3.1.3             # via flask, gt4py (pyproject.toml), sphinx
 jsonschema==4.21.1        # via nbformat
@@ -87,12 +89,13 @@ kiwisolver==1.4.5         # via matplotlib
 lark==1.1.9               # via gt4py (pyproject.toml)
 mako==1.3.2               # via gt4py (pyproject.toml)
 markdown-it-py==3.0.0     # via jupytext, mdit-py-plugins
-markupsafe==2.1.4         # via jinja2, mako, werkzeug
+markupsafe==2.1.5         # via jinja2, mako, werkzeug
 matplotlib==3.7.4         # via -r requirements-dev.in
 matplotlib-inline==0.1.6  # via ipykernel, ipython
 mccabe==0.7.0             # via flake8
 mdit-py-plugins==0.4.0    # via jupytext
 mdurl==0.1.2              # via markdown-it-py
+ml-dtypes==0.2.0          # via jax, jaxlib
 mpmath==1.3.0             # via sympy
 mypy==1.8.0               # via -r requirements-dev.in
 mypy-extensions==1.0.0    # via black, mypy
@@ -104,7 +107,8 @@ nest-asyncio==1.6.0       # via ipykernel, nbclient
 networkx==3.1             # via dace
 ninja==1.11.1.1           # via gt4py (pyproject.toml)
 nodeenv==1.8.0            # via pre-commit
-numpy==1.24.4             # via contourpy, dace, gt4py (pyproject.toml), matplotlib, types-jack-client
+numpy==1.24.4             # via contourpy, dace, gt4py (pyproject.toml), jax, jaxlib, matplotlib, ml-dtypes, opt-einsum, scipy, types-jack-client
+opt-einsum==3.3.0         # via jax
 ordered-set==4.1.0        # via deepdiff
 packaging==23.2           # via black, build, gt4py (pyproject.toml), ipykernel, jupytext, matplotlib, pyproject-api, pytest, setuptools-scm, sphinx, tox
 parso==0.8.3              # via jedi
@@ -136,16 +140,17 @@ pytest==8.0.0             # via -r requirements-dev.in, gt4py (pyproject.toml),
 pytest-cache==1.0         # via -r requirements-dev.in
 pytest-cov==4.1.0         # via -r requirements-dev.in
 pytest-factoryboy==2.6.0  # via -r requirements-dev.in
-pytest-xdist==3.5.0       # via -r requirements-dev.in, pytest-xdist
+pytest-xdist==3.5.0       # via -r requirements-dev.in
 python-dateutil==2.8.2    # via faker, jupyter-client, matplotlib
-pytz==2023.4              # via babel
+pytz==2024.1              # via babel
 pyyaml==6.0.1             # via dace, jupytext, pre-commit
 pyzmq==25.1.2             # via ipykernel, jupyter-client
 referencing==0.33.0       # via jsonschema, jsonschema-specifications
 requests==2.31.0          # via dace, sphinx
 restructuredtext-lint==1.4.0  # via flake8-rst-docstrings
 rpds-py==0.17.1           # via jsonschema, referencing
-ruff==0.1.15              # via -r requirements-dev.in
+ruff==0.2.1               # via -r requirements-dev.in
+scipy==1.10.1             # via gt4py (pyproject.toml), jax, jaxlib
 setuptools-scm==8.0.4     # via fparser
 six==1.16.0               # via asttokens, astunparse, python-dateutil
 snowballstemmer==2.2.0    # via pydocstyle, sphinx
@@ -163,7 +168,7 @@ stack-data==0.6.3         # via ipython
 sympy==1.9                # via dace, gt4py (pyproject.toml)
 tabulate==0.9.0           # via gt4py (pyproject.toml)
 toml==0.10.2              # via jupytext
-tomli==2.0.1              # via -r requirements-dev.in, black, build, coverage, flake8-pyproject, mypy, pip-tools, pyproject-api, pyproject-hooks, pytest, setuptools-scm, tox
+tomli==2.0.1 ; python_version < "3.11"  # via -r requirements-dev.in, black, build, coverage, flake8-pyproject, mypy, pip-tools, pyproject-api, pyproject-hooks, pytest, setuptools-scm, tox
 toolz==0.12.1             # via cytoolz
 tornado==6.4              # via ipykernel, jupyter-client
 tox==4.12.1               # via -r requirements-dev.in
@@ -175,15 +180,15 @@ types-atomicwrites==1.4.5.1  # via types-all
 types-backports==0.1.3    # via types-all
 types-backports-abc==0.5.2  # via types-all
 types-bleach==6.1.0.1     # via types-all
-types-boto==2.49.18.9     # via types-all
+types-boto==2.49.18.20240205  # via types-all
 types-cachetools==5.3.0.7  # via types-all
 types-certifi==2021.10.8.3  # via types-all
 types-cffi==1.16.0.20240106  # via types-jack-client
 types-characteristic==14.3.7  # via types-all
 types-chardet==5.0.4.6    # via types-all
 types-click==7.1.8        # via types-all, types-flask
 types-click-spinner==0.1.13.20240106  # via types-all
-types-colorama==0.4.15.20240106  # via types-all
+types-colorama==0.4.15.20240205  # via types-all
 types-contextvars==2.4.7.3  # via types-all
 types-croniter==2.0.0.20240106  # via types-all
 types-cryptography==3.3.23.2  # via types-all, types-openssl-python, types-pyjwt
@@ -193,7 +198,7 @@ types-datetimerange==2.0.0.6  # via types-all
 types-decorator==5.1.8.20240106  # via types-all
 types-deprecated==1.2.9.20240106  # via types-all
 types-docopt==0.6.11.4    # via types-all
-types-docutils==0.20.0.20240126  # via types-all
+types-docutils==0.20.0.20240201  # via types-all
 types-emoji==2.1.0.3      # via types-all
 types-enum34==1.1.8       # via types-all
 types-fb303==1.0.0        # via types-all, types-scribe
@@ -217,9 +222,9 @@ types-mypy-extensions==1.0.0.5  # via types-all
 types-nmap==0.1.6         # via types-all
 types-openssl-python==0.1.3  # via types-all
 types-orjson==3.6.2       # via types-all
-types-paramiko==3.4.0.20240120  # via types-all, types-pysftp
+types-paramiko==3.4.0.20240205  # via types-all, types-pysftp
 types-pathlib2==2.3.0     # via types-all
-types-pillow==10.2.0.20240125  # via types-all
+types-pillow==10.2.0.20240206  # via types-all
 types-pkg-resources==0.1.3  # via types-all
 types-polib==1.2.0.20240115  # via types-all
 types-protobuf==4.24.0.20240129  # via types-all
@@ -235,7 +240,7 @@ types-pysftp==0.2.17.20240106  # via types-all
 types-python-dateutil==2.8.19.20240106  # via types-all, types-datetimerange
 types-python-gflags==3.1.7.3  # via types-all
 types-python-slugify==8.0.2.20240127  # via types-all
-types-pytz==2023.4.0.20240130  # via types-all, types-tzlocal
+types-pytz==2024.1.0.20240203  # via types-all, types-tzlocal
 types-pyvmomi==8.0.0.6    # via types-all
 types-pyyaml==6.0.12.12   # via types-all
 types-redis==4.6.0.20240106  # via types-all
@@ -268,5 +273,5 @@ xxhash==3.0.0             # via gt4py (pyproject.toml)
 zipp==3.17.0              # via importlib-metadata, importlib-resources
 
 # The following packages are considered to be unsafe in a requirements file:
-pip==23.3.2               # via pip-tools
+pip==24.0                 # via pip-tools
 setuptools==69.0.3        # via gt4py (pyproject.toml), nodeenv, pip-tools, setuptools-scm

docs/development/tools/requirements.md

@@ -5,7 +5,7 @@ The specification of required third-party packages is scattered and partially du
 The following files in this repository contain information about required third-party packages:
 
 - `pyproject.toml`: GT4Py [package configuration](https://peps.python.org/pep-0621/) used by the build backend (`setuptools`). Install dependencies are specified in the _project.dependencies_ and _project.optional-dependencies_ tables.
-- `requirements-dev.in`: [requirements file](https://pip.pypa.io/en/stable/reference/requirements-file-format/) used by **pip**. It contains a list of packages required for the development of GT4Py. Part of its content is generated automatically from `pyproject.toml` using **cog**.
+- `requirements-dev.in`: [requirements file](https://pip.pypa.io/en/stable/reference/requirements-file-format/) used by **pip**. It contains a list of packages required for the development of GT4Py.
 - `requirements-dev.txt`: requirements file used by **pip**. It contains a completely frozen list of all packages required for installing and developing GT4Py. It is used by **pip** and **tox** to initialize the standard development and testing environments. It is automatically generated automatically from `requirements-dev.in` by **pip-compile**, when running the **tox** environment to update requirements.
 - `constraints.txt`: [constraints file](https://pip.pypa.io/en/stable/user_guide/#constraints-files) used by **pip** and **tox** to initialize a subset of the standard development environment making sure that if other packages are installed, transitive dependencies are taken from the frozen package list. It is generated automatically from `requirements-dev.in` using **pip-compile**.
 - `min-requirements-test.txt`: requirements file used by **pip**. It contains the minimum list of requirements to run GT4Py tests with the oldest compatible versions of all dependencies. It is generated automatically from `pyproject.toml` using **cog**.
@@ -14,14 +14,14 @@ The following files in this repository contain information about required third-
 
 The expected workflow to update GT4Py requirements is as follows:
 
-1. For changes in the GT4Py package dependencies, update the relevant table in `pyproject.toml`. When modifying the _project.optional-dependencies_ tables, make sure the `full` extra table **always** contains the combined dependencies from all the other extra tables.
+1. For changes in the GT4Py package dependencies, update the relevant table in `pyproject.toml`. When adding new tables to the _project.optional-dependencies_ section, make sure to add the new table as a dependency of the `all-` extra tables when possible.
 
 2. For changes in the development tools, update the `requirements-dev.in` file.
 
-3. Run the **tox** _requirements-common_ environment to update all files automatically with **pip-compile** and **cog**. Note that **pip-compile** will most likely update the versions of some unrelated tools if new versions are available in PyPI.
+3. Run the **tox** _requirements-base_ environment to update all files automatically with **pip-compile** and **cog**. Note that **pip-compile** will most likely update the versions of some unrelated tools if new versions are available in PyPI.
 
    ```bash
-   tox r -e requirements-common
+   tox r -e requirements-base
    ```
 
 4. Check that the **mypy** mirror used by **pre-commit** (https://github.com/pre-commit/mirrors-mypy) in `.pre-commit-config.yaml` supports the same version as in `constraints.txt`, and manually update the `rev` version number.

min-extra-requirements-test.txt

@@ -1,20 +1,55 @@
+#
+# Generated automatically by cog from pyproject.toml and requirements-dev.in
+# Run: 
+#       tox r -e requirements-common
+#
+
 ##[[[cog
-## import re, tomli
-## project = tomli.loads(open("pyproject.toml").read())
-## requirements = set(
-##     rr
-##     for r in (
-##         project["project"]["dependencies"]
-##         + project["project"]["optional-dependencies"]["full"]
-##         + open("requirements-dev.in").readlines()
-##     )
-##     if (rr := r.strip()) and not rr.startswith('#')
-## )
-## for r in sorted(requirements):
-##     m = re.match("^([\w-][\w\d\[\]-]*)[=>~][=]([^,]+)", r)
-##     print(f"{m[1]}=={m[2].strip()}")
+## import copy, sys
+## from packaging import requirements as reqs, specifiers as specs
+## if sys.version_info >= (3, 11):
+##     import tomllib
+## else:
+##     import tomli as tomllib
+##
+## def make_min_req(r: reqs.Requirement) -> reqs.Requirement:
+##     for s in r.specifier:
+##        if (ss := str(s)).startswith(">"):
+##            assert ss.startswith(">="), f"'{r!s}' requires a '>=' constraint"
+##            min_spec = specs.SpecifierSet(f"=={ss[2:]}")
+##            break
+##     min_r = copy.deepcopy(r)
+##     min_r.specifier = min_spec
+##     return min_r
+##
+## project = tomllib.loads(open("pyproject.toml").read())
+## all_cpu_extra = project["project"]["optional-dependencies"]["all-cpu"]
+## assert len(all_cpu_extra) == 1 and all_cpu_extra[0].startswith("gt4py[")
+## opt_req_versions = {
+##     reqs.Requirement(r).name: reqs.Requirement(r)
+##     for e in reqs.Requirement(all_cpu_extra[0]).extras
+##     for r in project["project"]["optional-dependencies"][e]
+## }
+## requirements = [
+##     reqs.Requirement(rr)
+##     for r in (project["project"]["dependencies"] + open("requirements-dev.in").readlines())
+##     if (rr := (r[: r.find("#")] if "#" in r else r))
+## ]
+## processed = set()
+## result = []
+## for r in requirements:
+##     assert r.name not in processed
+##     processed.add(r.name)
+##     if not r.specifier:
+##         assert r.name in opt_req_versions, f"Missing contraints for '{r.name}'"
+##         r = opt_req_versions[r.name]
+##     result.append(str(make_min_req(r)))
+## for r_name, r in opt_req_versions.items():
+##     if r_name not in processed:
+##         result.append(str(make_min_req(r)))
+## print("\n".join(sorted(result)))
 ##]]]
-astunparse==1.6.3;python_version<'3.9'
+astunparse==1.6.3; python_version < "3.9"
 attrs==21.3
 black==22.3
 boltons==20.1
@@ -24,7 +59,7 @@ click==8.0.0
 cmake==3.22
 cogapp==3.3
 coverage[toml]==5.0
-cytoolz==0.12.0
+cytoolz==0.12.1
 dace==0.15.1
 darglint==1.6
 deepdiff==5.6.0
@@ -42,7 +77,7 @@ flake8==5.0.4
 frozendict==2.3
 gridtools-cpp==2.3.2
 hypothesis==6.0.0
-importlib-resources==5.0;python_version<'3.9'
+importlib-resources==5.0; python_version < "3.9"
 isort==5.10
 jax[cpu]==0.4.13
 jinja2==3.0.0
@@ -54,27 +89,27 @@ mypy==1.0
 nanobind==1.4.0
 nbmake==1.4.6
 ninja==1.10
-numpy==1.21.2
+numpy==1.23.3
 packaging==20.0
 pip-tools==6.10
 pipdeptree==2.3
 pre-commit==2.17
 psutil==5.0
-pybind11==2.5
+pybind11==2.10.1
 pygments==2.7.3
 pytest-cache==1.0
 pytest-cov==2.8
 pytest-factoryboy==2.0.3
 pytest-xdist[psutil]==2.4
 pytest==7.0
-ruff==0.0.265
-scipy==1.7.2
+ruff==0.2.0
+scipy==1.9.2
 setuptools==65.5.0
 sphinx==4.4
 sphinx_rtd_theme==1.0
 sympy==1.9
 tabulate==0.8.10
-tomli==2.0.1
+tomli==2.0.1; python_version < "3.11"
 tox==3.2.0
 types-all==1.0.0
 typing-extensions==4.2