From 85bd4a08711bbfe83ddd617e72b4548a8e4030b6 Mon Sep 17 00:00:00 2001 From: Johan Herland Date: Fri, 16 Dec 2022 18:32:57 +0100 Subject: [PATCH] Use Nix and Nox to get fast local CI Use shell.nix to bring in all Python versions that we want to support. Also bring in Nox via its own dependency group. This, together with noxfile.py defines a set of new "actions" that we can run locally (and maybe also in CI in a future commit). For now we define three actions: - tests (run on all supported Python versions) - format (run formatting actions) - lint (run linters) Examples of how to run this: - nox -s tests # Run test suite on all supported Python versions - nox -s tests-3.7 # Run test suite on v3.7 only - nox -s format # Run formatting action (black + isort) - nox -s lint # Run linters (mypy, pylint, isort, black) - nox # Run all of the above Some complications worth mentioning: We have organized our dependencies using Poetry dependency groups (see [1] for more information on those), and we would therefore like to use those groups when telling Nox which dependencies are needed for each of the Nox actions described above. However, we cannot use `poetry install ...` to install these dependencies, because Poetry insists on installing into its own virtualenv - i.e. NOT the virtualenv that Nox creates for each session. We could have used nox-poetry (https://github.com/cjolowicz/nox-poetry), but unfortunately it does not support Poetry dependency groups, yet[2]. The workaround/solution is to export the required dependency groups from Poetry into a requirements.txt file that we can then pass on to Nox's session.install(). This is implemented by the install_groups() helper function in our noxfile.py. On my work laptop, the nox command (i.e. running all of the actions) completes in ~50s for the initial run, and ~17s on subsequent runs (when Nox can reuse its per-session virtualenvs). [1]: https://python-poetry.org/docs/master/managing-dependencies/#dependency-groups [2]: see https://github.com/cjolowicz/nox-poetry/pull/895 or https://github.com/cjolowicz/nox-poetry/issues/977 for more discussion. --- .github/workflows/ci.yaml | 2 +- .gitignore | 4 +- noxfile.py | 74 ++++++++++++++++++++++++ poetry.lock | 119 ++++++++++++++++++++++++++++++++++++-- pyproject.toml | 6 ++ shell.nix | 8 ++- 6 files changed, 203 insertions(+), 10 deletions(-) create mode 100644 noxfile.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f67fe5ea6..b430349c2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -47,7 +47,7 @@ jobs: path: ~/.cache/pypoetry/virtualenvs key: ${{ runner.os }}-poetry-linters-${{ hashFiles('poetry.lock') }} - name: Install project - run: poetry install --no-interaction --sync --with=test,dev + run: poetry install --no-interaction --sync --with=test,dev,nox - name: Run type checker run: poetry run mypy - name: Check formatting with Black diff --git a/.gitignore b/.gitignore index d04a61c8a..2f93893af 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -*/__pycache__/ -*/*/__pycache__/ +__pycache__/ dist/ +.nox/ .vscode/ diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 000000000..537ec5393 --- /dev/null +++ b/noxfile.py @@ -0,0 +1,74 @@ +import hashlib +from pathlib import Path +from typing import Iterable + +import nox + + +def install_groups( + session: nox.Session, + *, + include: Iterable[str] = (), + exclude: Iterable[str] = (), + include_self: bool = True, +) -> None: + """Install Poetry dependency groups + + This function installs the given dependency groups into the session's + virtual environment. When 'include_self' is true (the default), the + function also installs this package (".") and its default dependencies. + + We cannot use `poetry install` directly here, because it ignores the + session's virtualenv and installs into Poetry's own virtualenv. Instead, we + use `poetry export` with suitable options to generate a requirements.txt + file which we can then pass to session.install(). + + Auto-skip the `poetry export` step if the poetry.lock file is unchanged + since the last time this session was run. + """ + lockdata = Path("poetry.lock").read_bytes() + digest = hashlib.blake2b(lockdata).hexdigest() + requirements_txt = Path(session.cache_dir, session.name, "reqs_from_poetry.txt") + hashfile = requirements_txt.with_suffix(".hash") + + if not hashfile.is_file() or hashfile.read_text() != digest: + requirements_txt.parent.mkdir(parents=True, exist_ok=True) + argv = [ + "poetry", + "export", + "--format=requirements.txt", + f"--output={requirements_txt}", + ] + if include: + option = "only" if not include_self else "with" + argv.append(f"--{option}={','.join(include)}") + if exclude: + argv.append(f"--without={','.join(exclude)}") + session.run_always(*argv, external=True) + hashfile.write_text(digest) + + session.install("-r", str(requirements_txt)) + if include_self: + session.install(".") + + +@nox.session(python=["3.7", "3.8", "3.9", "3.10", "3.11"], reuse_venv=True) +def tests(session): + install_groups(session, include=["test"]) + session.run("pytest", "-x", "--log-level=debug", *session.posargs) + + +@nox.session(reuse_venv=True) +def format(session): + install_groups(session, include=["dev"], include_self=False) + session.run("isort", "fawltydeps", "tests") + session.run("black", ".") + + +@nox.session(reuse_venv=True) +def lint(session): + install_groups(session, include=["dev", "test"], include_self=False) + session.run("mypy", "fawltydeps", "tests") + session.run("pylint", "fawltydeps", "tests") + session.run("isort", "fawltydeps", "tests", "--check-only") + session.run("black", "--check", ".") diff --git a/poetry.lock b/poetry.lock index 1aa6d961f..9ae4b3476 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,17 @@ +[[package]] +name = "argcomplete" +version = "2.0.0" +description = "Bash tab completion for argparse" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +importlib-metadata = {version = ">=0.23,<5", markers = "python_version == \"3.7\""} + +[package.extras] +test = ["coverage", "flake8", "pexpect", "wheel"] + [[package]] name = "astroid" version = "2.12.13" @@ -73,6 +87,20 @@ category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +[[package]] +name = "colorlog" +version = "6.7.0" +description = "Add colours to the output of Python's logging module." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} + +[package.extras] +development = ["black", "flake8", "mypy", "pytest", "types-colorama"] + [[package]] name = "dill" version = "0.3.6" @@ -84,6 +112,14 @@ python-versions = ">=3.7" [package.extras] graph = ["objgraph (>=1.7.2)"] +[[package]] +name = "distlib" +version = "0.3.6" +description = "Distribution utilities" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "exceptiongroup" version = "1.1.0" @@ -95,9 +131,21 @@ python-versions = ">=3.7" [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "filelock" +version = "3.9.0" +description = "A platform independent file lock." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo (>=2022.12.7)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] +testing = ["covdefaults (>=2.2.2)", "coverage (>=7.0.1)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] + [[package]] name = "importlib-metadata" -version = "6.0.0" +version = "4.13.0" description = "Read metadata from Python packages" category = "dev" optional = false @@ -108,7 +156,7 @@ typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} zipp = ">=0.5" [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] perf = ["ipython"] testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] @@ -178,6 +226,25 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "nox" +version = "2022.11.21" +description = "Flexible test automation." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +argcomplete = ">=1.9.4,<3.0" +colorlog = ">=2.6.1,<7.0.0" +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} +packaging = ">=20.9" +typing-extensions = {version = ">=3.7.4", markers = "python_version < \"3.8\""} +virtualenv = ">=14" + +[package.extras] +tox-to-nox = ["jinja2", "tox"] + [[package]] name = "packaging" version = "22.0" @@ -311,6 +378,24 @@ category = "dev" optional = false python-versions = ">=3.7" +[[package]] +name = "virtualenv" +version = "20.17.1" +description = "Virtual Python Environment builder" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +distlib = ">=0.3.6,<1" +filelock = ">=3.4.1,<4" +importlib-metadata = {version = ">=4.8.3", markers = "python_version < \"3.8\""} +platformdirs = ">=2.4,<3" + +[package.extras] +docs = ["proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-argparse (>=0.3.2)", "sphinx-rtd-theme (>=1)", "towncrier (>=22.8)"] +testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] + [[package]] name = "wrapt" version = "1.14.1" @@ -334,9 +419,13 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [metadata] lock-version = "1.1" python-versions = "^3.7.2" -content-hash = "881ad440316490adbd947d0209c4901c14c52fe4dce76707962943a61a407ef3" +content-hash = "a828cad9b3d21df5c890b10155e92df64e925bfafc5b67e03ca885b521b02505" [metadata.files] +argcomplete = [ + {file = "argcomplete-2.0.0-py2.py3-none-any.whl", hash = "sha256:cffa11ea77999bb0dd27bb25ff6dc142a6796142f68d45b1a26b11f58724561e"}, + {file = "argcomplete-2.0.0.tar.gz", hash = "sha256:6372ad78c89d662035101418ae253668445b391755cfe94ea52f1b9d22425b20"}, +] astroid = [ {file = "astroid-2.12.13-py3-none-any.whl", hash = "sha256:10e0ad5f7b79c435179d0d0f0df69998c4eef4597534aae44910db060baeb907"}, {file = "astroid-2.12.13.tar.gz", hash = "sha256:1493fe8bd3dfd73dc35bd53c9d5b6e49ead98497c47b2307662556a5692d29d7"}, @@ -367,17 +456,29 @@ colorama = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +colorlog = [ + {file = "colorlog-6.7.0-py2.py3-none-any.whl", hash = "sha256:0d33ca236784a1ba3ff9c532d4964126d8a2c44f1f0cb1d2b0728196f512f662"}, + {file = "colorlog-6.7.0.tar.gz", hash = "sha256:bd94bd21c1e13fac7bd3153f4bc3a7dc0eb0974b8bc2fdf1a989e474f6e582e5"}, +] dill = [ {file = "dill-0.3.6-py3-none-any.whl", hash = "sha256:a07ffd2351b8c678dfc4a856a3005f8067aea51d6ba6c700796a4d9e280f39f0"}, {file = "dill-0.3.6.tar.gz", hash = "sha256:e5db55f3687856d8fbdab002ed78544e1c4559a130302693d839dfe8f93f2373"}, ] +distlib = [ + {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, + {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, +] exceptiongroup = [ {file = "exceptiongroup-1.1.0-py3-none-any.whl", hash = "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e"}, {file = "exceptiongroup-1.1.0.tar.gz", hash = "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"}, ] +filelock = [ + {file = "filelock-3.9.0-py3-none-any.whl", hash = "sha256:f58d535af89bb9ad5cd4df046f741f8553a418c01a7856bf0d173bbc9f6bd16d"}, + {file = "filelock-3.9.0.tar.gz", hash = "sha256:7b319f24340b51f55a2bf7a12ac0755a9b03e718311dac567a0f4f7fabd2f5de"}, +] importlib-metadata = [ - {file = "importlib_metadata-6.0.0-py3-none-any.whl", hash = "sha256:7efb448ec9a5e313a57655d35aa54cd3e01b7e1fbcf72dce1bf06119420f5bad"}, - {file = "importlib_metadata-6.0.0.tar.gz", hash = "sha256:e354bedeb60efa6affdcc8ae121b73544a7aa74156d047311948f6d711cd378d"}, + {file = "importlib_metadata-4.13.0-py3-none-any.whl", hash = "sha256:8a8a81bcf996e74fee46f0d16bd3eaa382a7eb20fd82445c3ad11f4090334116"}, + {file = "importlib_metadata-4.13.0.tar.gz", hash = "sha256:dd0173e8f150d6815e098fd354f6414b0f079af4644ddfe90c71e2fc6174346d"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, @@ -448,6 +549,10 @@ mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, ] +nox = [ + {file = "nox-2022.11.21-py3-none-any.whl", hash = "sha256:0e41a990e290e274cb205a976c4c97ee3c5234441a8132c8c3fd9ea3c22149eb"}, + {file = "nox-2022.11.21.tar.gz", hash = "sha256:e21c31de0711d1274ca585a2c5fde36b1aa962005ba8e9322bf5eeed16dcd684"}, +] packaging = [ {file = "packaging-22.0-py3-none-any.whl", hash = "sha256:957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3"}, {file = "packaging-22.0.tar.gz", hash = "sha256:2198ec20bd4c017b8f9717e00f0c8714076fc2fd93816750ab48e2c41de2cfd3"}, @@ -514,6 +619,10 @@ typing-extensions = [ {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, ] +virtualenv = [ + {file = "virtualenv-20.17.1-py3-none-any.whl", hash = "sha256:ce3b1684d6e1a20a3e5ed36795a97dfc6af29bc3970ca8dab93e11ac6094b3c4"}, + {file = "virtualenv-20.17.1.tar.gz", hash = "sha256:f8b927684efc6f1cc206c9db297a570ab9ad0e51c16fa9e45487d36d1905c058"}, +] wrapt = [ {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"}, {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef"}, diff --git a/pyproject.toml b/pyproject.toml index 40b2e1048..751687fcf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,12 @@ mypy = "^0.991" pylint = "^2.15.8" types-setuptools = "^65.6.0.2" +[tool.poetry.group.nox] +optional = true + +[tool.poetry.group.nox.dependencies] +nox = "^2022.11.21" + [tool.black] target-version = ["py37"] diff --git a/shell.nix b/shell.nix index ddcf23579..8b5715c17 100644 --- a/shell.nix +++ b/shell.nix @@ -8,13 +8,17 @@ pkgs.mkShell { name = "fawltydeps-env"; buildInputs = with pkgs; [ + python37 + python38 + python39 (python310.withPackages (pypkgs: with pypkgs; [ poetry ])) + python311 ]; shellHook = '' - poetry env use "$(which python)" - poetry install --sync --with=test,dev + poetry env use "${pkgs.python310}/bin/python" + poetry install --sync --with=nox,test,dev source "$(poetry env info --path)/bin/activate" ''; }