From 78ab98bc809473f9dce4313e26945f7844556302 Mon Sep 17 00:00:00 2001 From: Chi-Sheng Liu Date: Wed, 25 Dec 2024 16:34:29 +0800 Subject: [PATCH] [CI][Linter] Migrate to ruff Closes: ray-project/ray#48508 Signed-off-by: Chi-Sheng Liu --- .flake8 | 53 -------------------- .pre-commit-config.yaml | 25 +++------- ci/lint/format.sh | 59 ----------------------- pyproject.toml | 40 +++++++++++++++ python/ray/tune/requirements-dev.txt | 2 - python/requirements/lint-requirements.txt | 6 +-- 6 files changed, 48 insertions(+), 137 deletions(-) delete mode 100644 .flake8 create mode 100644 pyproject.toml diff --git a/.flake8 b/.flake8 deleted file mode 100644 index d8cea10fe7f5..000000000000 --- a/.flake8 +++ /dev/null @@ -1,53 +0,0 @@ -[flake8] -exclude = - python/ray/core/generated/ - doc/source/conf.py - python/ray/cloudpickle/ - python/ray/thirdparty_files/ - python/ray/_private/runtime_env/agent/thirdparty_files/ - python/build/ - python/.eggs/ - python/ray/_private/thirdparty/* - python/ray/workflow/tests/mock_server.py - python/ray/serve/tests/test_config_files/syntax_error.py - python/ray/serve/_private/benchmarks/streaming/_grpc/test_server_pb2_grpc.py -max-line-length = 88 -inline-quotes = " -ignore = - B028 - C408 - C417 - E121 - E123 - E126 - E203 - E226 - E24 - E704 - W503 - W504 - W605 - I - N - B001 - B002 - B003 - B004 - B005 - B007 - B008 - B009 - B010 - B011 - B012 - B013 - B014 - B015 - B016 - B017 -avoid-escape = no -# Error E731 is ignored because of the migration from YAPF to Black. -# See https://github.com/ray-project/ray/issues/21315 for more information. -per-file-ignores = - rllib/evaluation/worker_set.py:E731 - rllib/evaluation/sampler.py:E731 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 927052e35105..be21a6367068 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,6 +29,13 @@ repos: ) - id: check-toml + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.8.4 + hooks: + - id: ruff + args: [ --fix, --exit-non-zero-on-fix ] +# - id: ruff-format + - repo: https://github.com/psf/black rev: 22.10.0 hooks: @@ -46,17 +53,6 @@ repos: ) types_or: [python] - - repo: https://github.com/pycqa/flake8 - rev: 3.9.1 - hooks: - - id: flake8 - additional_dependencies: - [ - flake8-comprehensions==3.10.1, - flake8-quotes==2.0.0, - flake8-bugbear==21.9.2, - ] - - repo: https://github.com/pre-commit/mirrors-prettier rev: v3.0.3 hooks: @@ -83,13 +79,6 @@ repos: types-PyYAML==6.0.12.2, ] - - repo: https://github.com/pycqa/isort - rev: 5.12.0 - hooks: - - id: isort - name: isort (python) - types_or: [python] - - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.10.0 hooks: diff --git a/ci/lint/format.sh b/ci/lint/format.sh index bb3923918286..0f6c55d38e18 100755 --- a/ci/lint/format.sh +++ b/ci/lint/format.sh @@ -5,11 +5,9 @@ # Cause the script to exit if a single command fails set -euo pipefail -FLAKE8_VERSION_REQUIRED="3.9.1" BLACK_VERSION_REQUIRED="22.10.0" SHELLCHECK_VERSION_REQUIRED="0.7.1" MYPY_VERSION_REQUIRED="1.7.0" -ISORT_VERSION_REQUIRED="5.10.1" check_python_command_exist() { VERSION="" @@ -17,15 +15,9 @@ check_python_command_exist() { black) VERSION=$BLACK_VERSION_REQUIRED ;; - flake8) - VERSION=$FLAKE8_VERSION_REQUIRED - ;; mypy) VERSION=$MYPY_VERSION_REQUIRED ;; - isort) - VERSION=$ISORT_VERSION_REQUIRED - ;; *) echo "$1 is not a required dependency" exit 1 @@ -53,9 +45,7 @@ check_docstyle() { # TODO(can): add shellcheck, clang-format, and google-java-format to this check check_python_command_exist black -check_python_command_exist flake8 check_python_command_exist mypy -check_python_command_exist isort # this stops git rev-parse from failing if we run this from the .git directory builtin cd "$(dirname "${BASH_SOURCE:-$0}")" @@ -77,9 +67,7 @@ then else BLACK_VERSION=$(echo "$BLACK_VERSION_STR" | head -n 1 | awk '{print $3}') fi -FLAKE8_VERSION=$(flake8 --version | head -n 1 | awk '{print $1}') MYPY_VERSION=$(mypy --version | awk '{print $2}') -ISORT_VERSION=$(isort --version | grep VERSION | awk '{print $2}') GOOGLE_JAVA_FORMAT_JAR=/tmp/google-java-format-1.7-all-deps.jar # params: tool name, tool version, required version @@ -89,10 +77,8 @@ tool_version_check() { fi } -tool_version_check "flake8" "$FLAKE8_VERSION" "$FLAKE8_VERSION_REQUIRED" tool_version_check "black" "$BLACK_VERSION" "$BLACK_VERSION_REQUIRED" tool_version_check "mypy" "$MYPY_VERSION" "$MYPY_VERSION_REQUIRED" -tool_version_check "isort" "$ISORT_VERSION" "$ISORT_VERSION_REQUIRED" if command -v shellcheck >/dev/null; then SHELLCHECK_VERSION=$(shellcheck --version | awk '/^version:/ {print $2}') @@ -117,14 +103,6 @@ else echo "WARNING:java is not installed, skip format java files!" fi -if [[ $(flake8 --version) != *"flake8_quotes"* ]]; then - echo "WARNING: Ray uses flake8 with flake8_quotes. Might error without it. Install with: pip install flake8-quotes" -fi - -if [[ $(flake8 --version) != *"flake8-bugbear"* ]]; then - echo "WARNING: Ray uses flake8 with flake8-bugbear. Might error without it. Install with: pip install flake8-bugbear" -fi - SHELLCHECK_FLAGS=( "--exclude=1090" # "Can't follow non-constant source. Use a directive to specify location." "--exclude=1091" # "Not following {file} due to some error" @@ -180,11 +158,6 @@ for f in "${JAVA_EXCLUDES[@]}"; do done JAVA_EXCLUDES_REGEX=${JAVA_EXCLUDES_REGEX#|} -# TODO(barakmich): This should be cleaned up. I've at least excised the copies -# of these arguments to this location, but the long-term answer is to actually -# make a flake8 config file -FLAKE8_PYX_IGNORES="--ignore=C408,E121,E123,E126,E211,E225,E226,E227,E24,E704,E999,W503,W504,W605" - shellcheck_scripts() { shellcheck "${SHELLCHECK_FLAGS[@]}" "$@" } @@ -247,7 +220,6 @@ format_files() { done if [ 0 -lt "${#python_files[@]}" ]; then - isort "${python_files[@]}" black "${python_files[@]}" fi @@ -266,26 +238,11 @@ format_files() { } format_all_scripts() { - command -v flake8 &> /dev/null; - HAS_FLAKE8=$? - - # Run isort before black to fix imports and let black deal with file format. - echo "$(date)" "isort...." - git ls-files -- '*.py' "${GIT_LS_EXCLUDES[@]}" | xargs -P 10 \ - isort echo "$(date)" "Black...." git ls-files -- '*.py' "${GIT_LS_EXCLUDES[@]}" | xargs -P 10 \ black "${BLACK_EXCLUDES[@]}" echo "$(date)" "MYPY...." mypy_on_each "${MYPY_FILES[@]}" - if [ $HAS_FLAKE8 ]; then - echo "$(date)" "Flake8...." - git ls-files -- '*.py' "${GIT_LS_EXCLUDES[@]}" | xargs -P 5 \ - flake8 --config=.flake8 - - git ls-files -- '*.pyx' '*.pxd' '*.pxi' "${GIT_LS_EXCLUDES[@]}" | xargs -P 5 \ - flake8 --config=.flake8 "$FLAKE8_PYX_IGNORES" - fi if command -v shellcheck >/dev/null; then local shell_files bin_like_files @@ -330,25 +287,9 @@ format_changed() { # exist on both branches. MERGEBASE="$(git merge-base upstream/master HEAD)" - if ! git diff --diff-filter=ACRM --quiet --exit-code "$MERGEBASE" -- '*.py' &>/dev/null; then - git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- '*.py' | xargs -P 5 \ - isort - fi - if ! git diff --diff-filter=ACRM --quiet --exit-code "$MERGEBASE" -- '*.py' &>/dev/null; then git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- '*.py' | xargs -P 5 \ black "${BLACK_EXCLUDES[@]}" - if command -v flake8 >/dev/null; then - git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- '*.py' | xargs -P 5 \ - flake8 --config=.flake8 - fi - fi - - if ! git diff --diff-filter=ACRM --quiet --exit-code "$MERGEBASE" -- '*.pyx' '*.pxd' '*.pxi' &>/dev/null; then - if command -v flake8 >/dev/null; then - git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- '*.pyx' '*.pxd' '*.pxi' | xargs -P 5 \ - flake8 --config=.flake8 "$FLAKE8_PYX_IGNORES" - fi fi if command -v clang-format >/dev/null; then diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000000..72f4b1df0613 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,40 @@ +[project] +requires-python = ">=3.9" + +[tool.ruff.lint] +extend-select = ["I", "B", "Q", "C4" ] +ignore = [ + "E402", + "E711", + "E721", + "E741", + "F401", + "F403", + "F405", + "F602", + "F811", + "F821", + "F841", + "B003", + "B005", + "B007", + "B008", + "B011", + "B012", + "B015", + "B017", + "B018", + "B023", + "B024", + "B026", + "B027", + "B028", + "B035", + "B039", + "B904", + "C408", + "C416", + "C417", + "C418", + "C419", +] diff --git a/python/ray/tune/requirements-dev.txt b/python/ray/tune/requirements-dev.txt index e4432a5471c6..7b35f4ef7670 100644 --- a/python/ray/tune/requirements-dev.txt +++ b/python/ray/tune/requirements-dev.txt @@ -1,5 +1,3 @@ -flake8==3.9.1 -flake8-quotes gym>=0.21.0,<0.24.0 scikit-image pandas diff --git a/python/requirements/lint-requirements.txt b/python/requirements/lint-requirements.txt index 3a0889a28853..530e7141318d 100644 --- a/python/requirements/lint-requirements.txt +++ b/python/requirements/lint-requirements.txt @@ -1,14 +1,10 @@ clang-format==12.0.1 docutils -flake8==3.9.1 -flake8-comprehensions==3.10.1 -flake8-quotes==2.0.0 -flake8-bugbear==21.9.2 +ruff==0.8.4 mypy==1.7.0 mypy-extensions==1.0.0 types-PyYAML==6.0.12.2 black==22.10.0 -isort==5.10.1 semgrep==1.32.0 shellcheck-py==0.7.1.1 yq