Skip to content

Commit

Permalink
[CI] Update flake8, ruff, pyfmt linters in CI but with very broad ign…
Browse files Browse the repository at this point in the history 
…ore lists (#6164)

Enables flake8, ruff, pyfmt in CI but with very broad ignore lists so
that I don't need to change those failures right now

Remove UFMT in favor of pyfmt since thats what pytorch/pytorch does

Removing ufmt and replacing it with pyfmt does decrease our coverage in
CI, but I'll put up another PR to add them back into the list of linted
files and apply the new changes that the linter wants

Changes --take to --skip since there are more linters being used than
being not used

Updates scripts and arguments for flake8, ruff, pyfmt based on pytorch

Updates some things in pyproject.toml
  • Loading branch information
@clee2000
clee2000 authored Jan 13, 2025
1 parent bbd33bb commit 80c1fb9
Show file tree
Hide file tree
Showing 7 changed files with 326 additions and 200 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/lint.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ jobs:
- name: Run lintrunner on all files - Linux
run: |
set +e
if ! lintrunner -v --force-color --all-files --tee-json=lint.json --take ACTIONLINT,MYPY,RUSTFMT,COPYRIGHT,LINTRUNNER_VERSION,UFMT,NEWLINE,TABS,SPACES; then
if ! lintrunner -v --force-color --all-files --tee-json=lint.json --skip TYPEIGNORE,NOQA,PYPIDEP,EXEC; then
echo ""
echo -e "\e[1m\e[36mYou can reproduce these results locally by using \`lintrunner -m main\`.\e[0m"
exit 1
Expand Down
42 changes: 28 additions & 14 deletions .lintrunner.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ code = 'FLAKE8'
include_patterns = ['**/*.py']
exclude_patterns = [
'.git/**',
'.github/scripts/**',
'aws/lambda/**',
's3_management/**',
'tools/**',
]
command = [
'python3',
Expand All @@ -16,16 +20,17 @@ init_command = [
'python3',
'tools/linter/adapters/pip_init.py',
'--dry-run={{DRYRUN}}',
'flake8==6.0.0',
'flake8==6.1.0',
'flake8-bugbear==23.3.23',
'flake8-comprehensions==3.12.0',
'flake8-comprehensions==3.15.0',
'flake8-executable==2.1.3',
'flake8-logging-format==0.9.0',
'flake8-pyi==23.3.1',
'flake8-simplify==0.19.3',
'mccabe==0.7.0',
'pycodestyle==2.10.0',
'pyflakes==3.0.1',
'pycodestyle==2.11.1',
'pyflakes==3.1.0',
'torchfix==0.4.0 ; python_version >= "3.9" and python_version < "3.13"',
]

[[linter]]
Expand Down Expand Up @@ -254,17 +259,21 @@ init_command = [
'--output-name=actionlint',
]

# Black + usort
# usort + ruff-format
[[linter]]
code = 'UFMT'
code = 'PYFMT'
include_patterns = [
'tools/torchci/**/*.py',
'.github/scripts/*.py',
'aws/lambda/whl_metadata_upload_pep658/**/*.py',
'**/*.py',
]
exclude_patterns = [
'.github/scripts/**',
'aws/lambda/**',
's3_management/**',
'tools/**',
]
command = [
'python3',
'tools/linter/adapters/ufmt_linter.py',
'tools/linter/adapters/pyfmt_linter.py',
'--',
'@{{PATHSFILE}}'
]
Expand All @@ -273,9 +282,10 @@ init_command = [
'tools/linter/adapters/pip_init.py',
'--dry-run={{DRYRUN}}',
'--no-black-binary',
'black==23.3.0',
'ufmt==2.1.0',
'usort==1.0.6',
'black==23.12.1',
'usort==1.0.8.post1',
'isort==5.13.2',
'ruff==0.8.4', # sync with RUFF
]
is_formatter = true

Expand Down Expand Up @@ -309,6 +319,10 @@ command = [
code = 'RUFF'
include_patterns = ['**/*.py']
exclude_patterns = [
'.github/scripts/**',
'aws/lambda/**',
's3_management/**',
'tools/**',
]
command = [
'python3',
Expand All @@ -322,7 +336,7 @@ init_command = [
'python3',
'tools/linter/adapters/pip_init.py',
'--dry-run={{DRYRUN}}',
'ruff==0.0.269',
'ruff==0.8.4', # sync with PYFMT
]
is_formatter = true

Expand Down
6 changes: 3 additions & 3 deletions pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,11 @@ build-backend = "setuptools.build_meta:__legacy__"
# line-length = 120
target-version = ["py38", "py39", "py310", "py311"]


[tool.ruff]
target-version = "py38"
line-length = 88

[tool.ruff.lint]
# NOTE: Synchoronize the ignores with .flake8
ignore = [
# these ignores are from flake8-bugbear; please fix!
Expand Down Expand Up @@ -55,7 +56,6 @@ ignore = [
"SIM117",
"SIM118",
]
line-length = 120
select = [
"B",
"C4",
Expand All @@ -66,5 +66,5 @@ select = [
"W",
# Not included in flake8
"PLE",
"TRY302",
"TRY203",
]
57 changes: 28 additions & 29 deletions tools/linter/adapters/flake8_linter.py
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
from __future__ import annotations

import argparse
import json
import logging
Expand All @@ -7,7 +9,7 @@
import sys
import time
from enum import Enum
from typing import Any, Dict, List, NamedTuple, Optional, Pattern, Set
from typing import Any, NamedTuple


IS_WINDOWS: bool = os.name == "nt"
Expand All @@ -25,15 +27,15 @@ class LintSeverity(str, Enum):


class LintMessage(NamedTuple):
path: Optional[str]
line: Optional[int]
char: Optional[int]
path: str | None
line: int | None
char: int | None
code: str
severity: LintSeverity
name: str
original: Optional[str]
replacement: Optional[str]
description: Optional[str]
original: str | None
replacement: str | None
description: str | None


def as_posix(name: str) -> str:
Expand All @@ -42,7 +44,7 @@ def as_posix(name: str) -> str:

# fmt: off
# https://www.flake8rules.com/
DOCUMENTED_IN_FLAKE8RULES: Set[str] = {
DOCUMENTED_IN_FLAKE8RULES: set[str] = {
"E101", "E111", "E112", "E113", "E114", "E115", "E116", "E117",
"E121", "E122", "E123", "E124", "E125", "E126", "E127", "E128", "E129",
"E131", "E133",
Expand Down Expand Up @@ -78,14 +80,14 @@ def as_posix(name: str) -> str:
}

# https://pypi.org/project/flake8-comprehensions/#rules
DOCUMENTED_IN_FLAKE8COMPREHENSIONS: Set[str] = {
DOCUMENTED_IN_FLAKE8COMPREHENSIONS: set[str] = {
"C400", "C401", "C402", "C403", "C404", "C405", "C406", "C407", "C408", "C409",
"C410",
"C411", "C412", "C413", "C413", "C414", "C415", "C416",
"C411", "C412", "C413", "C414", "C415", "C416",
}

# https://github.com/PyCQA/flake8-bugbear#list-of-warnings
DOCUMENTED_IN_BUGBEAR: Set[str] = {
DOCUMENTED_IN_BUGBEAR: set[str] = {
"B001", "B002", "B003", "B004", "B005", "B006", "B007", "B008", "B009", "B010",
"B011", "B012", "B013", "B014", "B015",
"B301", "B302", "B303", "B304", "B305", "B306",
Expand All @@ -98,7 +100,7 @@ def as_posix(name: str) -> str:
# stdin:3:6: T484 Name 'foo' is not defined
# stdin:3:-100: W605 invalid escape sequence '\/'
# stdin:3:1: E302 expected 2 blank lines, found 1
RESULTS_RE: Pattern[str] = re.compile(
RESULTS_RE: re.Pattern[str] = re.compile(
r"""(?mx)
^
(?P<file>.*?):
Expand All @@ -113,7 +115,8 @@ def as_posix(name: str) -> str:

def _test_results_re() -> None:
"""
>>> def t(s): return RESULTS_RE.search(s).groupdict()
>>> def t(s):
... return RESULTS_RE.search(s).groupdict()
>>> t(r"file.py:80:1: E302 expected 2 blank lines, found 1")
... # doctest: +NORMALIZE_WHITESPACE
Expand All @@ -130,14 +133,13 @@ def _test_results_re() -> None:
{'file': 'file.py', 'line': '8', 'column': '-10', 'code': 'W605',
'message': "invalid escape sequence '/'"}
"""
pass


def _run_command(
args: List[str],
args: list[str],
*,
extra_env: Optional[Dict[str, str]],
) -> "subprocess.CompletedProcess[str]":
extra_env: dict[str, str] | None,
) -> subprocess.CompletedProcess[str]:
logging.debug(
"$ %s",
" ".join(
Expand All @@ -158,11 +160,11 @@ def _run_command(


def run_command(
args: List[str],
args: list[str],
*,
extra_env: Optional[Dict[str, str]],
extra_env: dict[str, str] | None,
retries: int,
) -> "subprocess.CompletedProcess[str]":
) -> subprocess.CompletedProcess[str]:
remaining_retries = retries
while True:
try:
Expand Down Expand Up @@ -243,11 +245,11 @@ def get_issue_documentation_url(code: str) -> str:


def check_files(
filenames: List[str],
flake8_plugins_path: Optional[str],
severities: Dict[str, LintSeverity],
filenames: list[str],
flake8_plugins_path: str | None,
severities: dict[str, LintSeverity],
retries: int,
) -> List[LintMessage]:
) -> list[LintMessage]:
try:
proc = run_command(
[sys.executable, "-mflake8", "--exit-zero"] + filenames,
Expand Down Expand Up @@ -289,10 +291,7 @@ def check_files(
LintMessage(
path=match["file"],
name=match["code"],
description="{}\nSee {}".format(
match["message"],
get_issue_documentation_url(match["code"]),
),
description=f"{match['message']}\nSee {get_issue_documentation_url(match['code'])}",
line=int(match["line"]),
char=int(match["column"])
if match["column"] is not None and not match["column"].startswith("-")
Expand Down Expand Up @@ -354,7 +353,7 @@ def main() -> None:
else os.path.realpath(args.flake8_plugins_path)
)

severities: Dict[str, LintSeverity] = {}
severities: dict[str, LintSeverity] = {}
if args.severity:
for severity in args.severity:
parts = severity.split(":", 1)
Expand Down
Loading

0 comments on commit 80c1fb9

Please sign in to comment.