Skip to content

Commit 5e9d20d

Browse files
Primer tests "à la mypy" (#5173)
* Add changelog and warning about unstable API in testutil * Add primer tests, (running pylint on external libs during tests) In order to anticipate crash/fatal messages, false positives are harder to anticipate. Follow-up will be #5359 and later on #5364 Add '__tracebackhide__ = True' so the traceback is manageable Co-authored-by: Daniël van Noord <[email protected]>
1 parent e8fa469 commit 5e9d20d

File tree

12 files changed

+335
-3
lines changed

12 files changed

+335
-3
lines changed

.github/workflows/ci.yaml

+34
Original file line numberDiff line numberDiff line change
@@ -509,3 +509,37 @@ jobs:
509509
. venv/bin/activate
510510
pip install -e .
511511
pytest -m primer_stdlib -n auto
512+
513+
pytest-primer-external:
514+
name: Run primer tests on external libs Python ${{ matrix.python-version }} (Linux)
515+
runs-on: ubuntu-latest
516+
needs: prepare-tests-linux
517+
strategy:
518+
matrix:
519+
python-version: [3.8, 3.9, "3.10"]
520+
steps:
521+
- name: Check out code from GitHub
522+
uses: actions/[email protected]
523+
- name: Set up Python ${{ matrix.python-version }}
524+
id: python
525+
uses: actions/[email protected]
526+
with:
527+
python-version: ${{ matrix.python-version }}
528+
- name: Restore Python virtual environment
529+
id: cache-venv
530+
uses: actions/[email protected]
531+
with:
532+
path: venv
533+
key:
534+
${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
535+
needs.prepare-tests-linux.outputs.python-key }}
536+
- name: Fail job if Python cache restore failed
537+
if: steps.cache-venv.outputs.cache-hit != 'true'
538+
run: |
539+
echo "Failed to restore Python venv from cache"
540+
exit 1
541+
- name: Run pytest
542+
run: |
543+
. venv/bin/activate
544+
pip install -e .
545+
pytest -m primer_external -n auto

.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,4 @@ build-stamp
2020
.pytest_cache/
2121
.mypy_cache/
2222
.benchmarks/
23+
.pylint_primer_tests/

ChangeLog

+5
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,11 @@ Release date: TBA
1515

1616
Closes #4982
1717

18+
* Introduced primer tests and a configuration tests framework. The helper classes available in
19+
``pylint/testutil/`` are still unstable and might be modified in the near future.
20+
21+
Closes #4412 #5287
22+
1823
* Fix ``install graphiz`` message which isn't needed for puml output format.
1924

2025
* Fix a crash in the ``check_elif`` extensions where an undetected if in a comprehension

doc/whatsnew/2.12.rst

+6-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ New checkers
3535

3636
* Fix ``useless-super-delegation`` false positive when default keyword argument is a variable.
3737

38-
* Added new checker `use-implicit-booleanness``: Emitted when using collection
38+
* Added new checker ``use-implicit-booleanness``: Emitted when using collection
3939
literals for boolean comparisons
4040

4141
* Added new checker ``use-implicit-booleaness-not-comparison``: Emitted when
@@ -204,3 +204,8 @@ Other Changes
204204
* Fix crash on ``open()`` calls when the ``mode`` argument is not a simple string.
205205

206206
Partially closes #5321
207+
208+
* Introduced primer tests and a configuration tests framework. The helper classes available in
209+
``pylint/testutil/`` are still unstable and might be modified in the near future.
210+
211+
Closes #4412 #5287

pylint/constants.py

+1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
from pylint.__pkginfo__ import __version__
1010

11+
PY37_PLUS = sys.version_info[:2] >= (3, 7)
1112
PY38_PLUS = sys.version_info[:2] >= (3, 8)
1213
PY39_PLUS = sys.version_info[:2] >= (3, 9)
1314

pylint/testutils/primer.py

+107
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
import logging
2+
from pathlib import Path
3+
from typing import Dict, List, Optional, Union
4+
5+
import git
6+
7+
PRIMER_DIRECTORY_PATH = Path(".pylint_primer_tests")
8+
9+
10+
class PackageToLint:
11+
"""Represents data about a package to be tested during primer tests"""
12+
13+
url: str
14+
"""URL of the repository to clone"""
15+
16+
branch: str
17+
"""Branch of the repository to clone"""
18+
19+
directories: List[str]
20+
"""Directories within the repository to run pylint over"""
21+
22+
commit: Optional[str]
23+
"""Commit hash to pin the repository on"""
24+
25+
pylint_additional_args: List[str]
26+
"""Arguments to give to pylint"""
27+
28+
pylintrc_relpath: Optional[str]
29+
"""Path relative to project's main directory to the pylintrc if it exists"""
30+
31+
def __init__(
32+
self,
33+
url: str,
34+
branch: str,
35+
directories: List[str],
36+
commit: Optional[str] = None,
37+
pylint_additional_args: Optional[List[str]] = None,
38+
pylintrc_relpath: Optional[str] = None,
39+
) -> None:
40+
self.url = url
41+
self.branch = branch
42+
self.directories = directories
43+
self.commit = commit
44+
self.pylint_additional_args = pylint_additional_args or []
45+
self.pylintrc_relpath = pylintrc_relpath
46+
47+
@property
48+
def pylintrc(self) -> Optional[Path]:
49+
if self.pylintrc_relpath is None:
50+
return None
51+
return self.clone_directory / self.pylintrc_relpath
52+
53+
@property
54+
def clone_directory(self) -> Path:
55+
"""Directory to clone repository into"""
56+
clone_name = "/".join(self.url.split("/")[-2:]).replace(".git", "")
57+
return PRIMER_DIRECTORY_PATH / clone_name
58+
59+
@property
60+
def paths_to_lint(self) -> List[str]:
61+
"""The paths we need to lint"""
62+
return [str(self.clone_directory / path) for path in self.directories]
63+
64+
@property
65+
def pylint_args(self) -> List[str]:
66+
options: List[str] = []
67+
if self.pylintrc is not None:
68+
# There is an error if rcfile is given but does not exists
69+
options += [f"--rcfile={self.pylintrc}"]
70+
return self.paths_to_lint + options + self.pylint_additional_args
71+
72+
def lazy_clone(self) -> None: # pragma: no cover
73+
"""Concatenates the target directory and clones the file
74+
75+
Not expected to be tested as the primer won't work if it doesn't.
76+
It's tested in the continuous integration primers, only the coverage
77+
is not calculated on everything. If lazy clone breaks for local use
78+
we'll probably notice because we'll have a fatal when launching the
79+
primer locally.
80+
"""
81+
logging.info("Lazy cloning %s", self.url)
82+
if not self.clone_directory.exists():
83+
options: Dict[str, Union[str, int]] = {
84+
"url": self.url,
85+
"to_path": str(self.clone_directory),
86+
"branch": self.branch,
87+
"depth": 1,
88+
}
89+
logging.info("Directory does not exists, cloning: %s", options)
90+
git.Repo.clone_from(**options)
91+
return
92+
93+
remote_sha1_commit = (
94+
git.cmd.Git().ls_remote(self.url, self.branch).split("\t")[0]
95+
)
96+
local_sha1_commit = git.Repo(self.clone_directory).head.object.hexsha
97+
if remote_sha1_commit != local_sha1_commit:
98+
logging.info(
99+
"Remote sha is '%s' while local sha is '%s': pulling new commits",
100+
remote_sha1_commit,
101+
local_sha1_commit,
102+
)
103+
repo = git.Repo(self.clone_directory)
104+
origin = repo.remotes.origin
105+
origin.pull()
106+
else:
107+
logging.info("Repository already up to date.")

requirements_test_min.txt

+1
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@
33
astroid==2.9.0 # Pinned to a specific version for tests
44
pytest~=6.2
55
pytest-benchmark~=3.4
6+
gitpython>3

setup.cfg

+5-1
Original file line numberDiff line numberDiff line change
@@ -69,9 +69,10 @@ test = pytest
6969
[tool:pytest]
7070
testpaths = tests
7171
python_files = *test_*.py
72-
addopts = -m "not primer_stdlib"
72+
addopts = --strict-markers -m "not primer_stdlib and not primer_external"
7373
markers =
7474
primer_stdlib: Checks for crashes and errors when running pylint on stdlib
75+
primer_external: Checks for crashes and errors when running pylint on external libs
7576
benchmark: Baseline of pylint performance, if this regress something serious happened
7677
7778
[isort]
@@ -118,3 +119,6 @@ ignore_missing_imports = True
118119
119120
[mypy-toml.decoder]
120121
ignore_missing_imports = True
122+
123+
[mypy-git.*]
124+
ignore_missing_imports = True

tests/primer/packages_to_lint.json

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
{
2+
"flask": {
3+
"url": "https://github.com/pallets/flask.git",
4+
"branch": "main",
5+
"directories": ["src/flask"]
6+
},
7+
"pytest": {
8+
"url": "https://github.com/pytest-dev/pytest.git",
9+
"branch": "main",
10+
"directories": ["src/_pytest"]
11+
},
12+
"psycopg": {
13+
"url": "https://github.com/psycopg/psycopg.git",
14+
"branch": "master",
15+
"directories": ["psycopg/psycopg"]
16+
},
17+
"keras": {
18+
"url": "https://github.com/keras-team/keras.git",
19+
"branch": "master",
20+
"directories": ["keras"]
21+
},
22+
"sentry": {
23+
"url": "https://github.com/getsentry/sentry.git",
24+
"branch": "master",
25+
"directories": ["src/sentry"]
26+
},
27+
"django": {
28+
"url": "https://github.com/django/django.git",
29+
"branch": "main",
30+
"directories": ["django"]
31+
},
32+
"pandas": {
33+
"url": "https://github.com/pandas-dev/pandas.git",
34+
"branch": "master",
35+
"directories": ["pandas"],
36+
"pylint_additional_args": ["--ignore-patterns=\"test_"]
37+
},
38+
"black": {
39+
"url": "https://github.com/psf/black.git",
40+
"branch": "main",
41+
"directories": ["src/black/", "src/blackd/", "src/black_primer/", "src/blib2to3/"]
42+
},
43+
"home-assistant": {
44+
"url": "https://github.com/home-assistant/core.git",
45+
"branch": "dev",
46+
"directories": ["homeassistant"]
47+
},
48+
"graph-explorer": {
49+
"url": "https://github.com/vimeo/graph-explorer.git",
50+
"branch": "master",
51+
"directories": ["graph_explorer"],
52+
"pylintrc_relpath": ".pylintrc"
53+
},
54+
"pygame": {
55+
"url": "https://github.com/pygame/pygame.git",
56+
"branch": "main",
57+
"directories": ["src_py"]
58+
}
59+
}

tests/primer/test_primer_external.py

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
2+
# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE
3+
import json
4+
import logging
5+
import subprocess
6+
from pathlib import Path
7+
from typing import Dict, Union
8+
9+
import pytest
10+
from pytest import LogCaptureFixture
11+
12+
from pylint.testutils.primer import PackageToLint
13+
14+
PRIMER_DIRECTORY = Path(".pylint_primer_tests/").resolve()
15+
16+
17+
def get_packages_to_lint_from_json(
18+
json_path: Union[Path, str]
19+
) -> Dict[str, PackageToLint]:
20+
result: Dict[str, PackageToLint] = {}
21+
with open(json_path, encoding="utf8") as f:
22+
for name, package_data in json.load(f).items():
23+
result[name] = PackageToLint(**package_data)
24+
return result
25+
26+
27+
PACKAGE_TO_LINT_JSON = Path(__file__).parent / "packages_to_lint.json"
28+
PACKAGES_TO_LINT = get_packages_to_lint_from_json(PACKAGE_TO_LINT_JSON)
29+
"""Dictionary of external packages used during the primer test"""
30+
31+
32+
class TestPrimer:
33+
@staticmethod
34+
@pytest.mark.primer_external
35+
@pytest.mark.parametrize("package", PACKAGES_TO_LINT.values(), ids=PACKAGES_TO_LINT)
36+
def test_primer_external_packages_no_crash(
37+
package: PackageToLint,
38+
caplog: LogCaptureFixture,
39+
) -> None:
40+
__tracebackhide__ = True # pylint: disable=unused-variable
41+
TestPrimer._primer_test(package, caplog)
42+
43+
@staticmethod
44+
def _primer_test(package: PackageToLint, caplog: LogCaptureFixture) -> None:
45+
"""Runs pylint over external packages to check for crashes and fatal messages
46+
47+
We only check for crashes (bit-encoded exit code 32) and fatal messages
48+
(bit-encoded exit code 1). We assume that these external repositories do not
49+
have any fatal errors in their code so that any fatal errors are pylint false
50+
positives
51+
"""
52+
caplog.set_level(logging.INFO)
53+
package.lazy_clone()
54+
55+
try:
56+
# We want to test all the code we can
57+
enables = ["--enable-all-extensions", "--enable=all"]
58+
# Duplicate code takes too long and is relatively safe
59+
disables = ["--disable=duplicate-code"]
60+
command = ["pylint"] + enables + disables + package.pylint_args
61+
logging.info("Launching primer:\n%s", " ".join(command))
62+
subprocess.run(command, check=True)
63+
except subprocess.CalledProcessError as ex:
64+
msg = f"Encountered {{}} during primer test for {package}"
65+
assert ex.returncode != 32, msg.format("a crash")
66+
assert ex.returncode % 2 == 0, msg.format("a message of category 'fatal'")

tests/primer/test_primer_stdlib.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,11 @@ def _patch_stdout(out):
4343
@pytest.mark.parametrize(
4444
("test_module_location", "test_module_name"), MODULES_TO_CHECK, ids=MODULES_NAMES
4545
)
46-
def test_lib_module_no_crash(
46+
def test_primer_stdlib_no_crash(
4747
test_module_location: str, test_module_name: str, capsys: CaptureFixture
4848
) -> None:
4949
"""Test that pylint does not produces any crashes or fatal errors on stdlib modules"""
50+
__tracebackhide__ = True # pylint: disable=unused-variable
5051
os.chdir(test_module_location)
5152
with _patch_stdout(io.StringIO()):
5253
try:

0 commit comments

Comments
 (0)