From e5cdf7f8362fc5b8ca08ebfa5bb995a26d735c06 Mon Sep 17 00:00:00 2001 From: Kaya Unalmis Date: Fri, 21 Jul 2023 23:32:47 -0500 Subject: [PATCH 1/2] Add instructions for pre-commit, remove old pylint --- devtools/README.rst | 30 +- devtools/dev-requirements.txt | 1 + devtools/dev-requirements_conda.yml | 1 + devtools/pylintrc | 423 ---------------------------- devtools/run_all_linting.sh | 4 - devtools/run_pylint.sh | 8 - docs/installation.rst | 6 +- 7 files changed, 13 insertions(+), 460 deletions(-) delete mode 100644 devtools/pylintrc delete mode 100644 devtools/run_all_linting.sh delete mode 100644 devtools/run_pylint.sh diff --git a/devtools/README.rst b/devtools/README.rst index 3471f38669..93d89b92f0 100644 --- a/devtools/README.rst +++ b/devtools/README.rst @@ -1,20 +1,21 @@ -########################################## -Scripts for Linting with Pylint and Flake8 -########################################## +############################### +Scripts for Linting with Flake8 +############################### Usage ====== -To use, run either the run_pylint_linting.sh or run_flake8_linting.sh scripts +It is recommended to run ``pre-commit install`` in a terminal under the main ``DESC`` directory. -These will generate a pylint.output file, or flake8_errors.ouput and flake8_summary.output files. +Then anytime you commit with ``git commit`` the linting suite will run automatically. -Currently, black is still ran through GitHub actions. +To manually generate a linting report you need to execute the shell script ``run_flake8_linting.sh`` by typing ``sh run_flake8_linting.sh`` in a terminal from the ``devtools`` directory. -Most formatting-level errors are being suppressed; errors that touch the code logic are primarily the ones being raised. +This will generate the ``flake8_errors.output`` and ``flake8_summary.output`` files. Currently, black is still ran through GitHub actions. +Most formatting-level errors are being suppressed; errors that touch the code logic are primarily the ones being raised. Outputs @@ -25,15 +26,6 @@ Flake8 flake8_errors.output will contain a line-by-line listing of errors flake8_summary.output will total these up into categories in a quick summary -Pylint ------- -Pylint.output currently will contain, in the following order : - 1. a series of line-specific reports, then - 2. show duplicated code blocks - 3. a dependency tree - 4. an error summary - - Configuration ============= @@ -43,9 +35,3 @@ Flake8 Currently, error messages about whitespace and indenting that black does not care about have been suppressed. These will be fixed in a future cleanup branch. More errors can be added to the [flake8] section of settings.cfg after "ignore=", separated by commas. - -Pylint: -------- -Currently several classes of error are being suppressed, mostly to do with preferring encapsulating behavior into simpler classes and modules. -To disable certain classes of error message, go to the [MESSAGES] section of pylinrc, and add the error to "disable=" separated by commas. -Additionally, several very minor errors are being suppressed to be fixed in a future cleanup branch. diff --git a/devtools/dev-requirements.txt b/devtools/dev-requirements.txt index 1dffd01e21..f75ab443e4 100644 --- a/devtools/dev-requirements.txt +++ b/devtools/dev-requirements.txt @@ -15,6 +15,7 @@ flake8-docstrings >= 1.0.0, <=2.0.0 flake8-eradicate >= 1.0.0, <=2.0.0 flake8-isort >=5.0.0, <= 6.0.0 + # testing and benchmarking markupsafe == 2.0.1 nbmake diff --git a/devtools/dev-requirements_conda.yml b/devtools/dev-requirements_conda.yml index ea82c08d51..d9efd4cdcf 100644 --- a/devtools/dev-requirements_conda.yml +++ b/devtools/dev-requirements_conda.yml @@ -33,6 +33,7 @@ dependencies: - flake8-docstrings >= 1.0.0, <=2.0.0 - flake8-eradicate >= 1.0.0, <=2.0.0 - flake8-isort >=5.0.0, <= 6.0.0 + - pre-commit # testing and benchmarking - markupsafe = 2.0.1 diff --git a/devtools/pylintrc b/devtools/pylintrc deleted file mode 100644 index 6de7a1f40e..0000000000 --- a/devtools/pylintrc +++ /dev/null @@ -1,423 +0,0 @@ -[MASTER] - -# Specify a configuration file. -#rcfile= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=third_party - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns=object_detection_grpc_client.py,prediction_pb2.py,prediction_pb2_grpc.py - -# Pickle collected data for later comparisons. -persistent=no - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - -# Use multiple processes to speed up Pylint. -jobs=4 - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -#enable= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -# -# Kubeflow disables string-interpolation because we are starting to use f -# style strings -disable= - #Settings that prefer simpler code practices and logic - too-many-locals, - too-many-arguments, - too-many-instance-attributes, - too-many-statements, - too-many-branches, - too-many-lines, - too-many-public-methods, - too-many-nested-blocks, - too-many-function-args, - consider-using-with, - consider-using-dict-items, - consider-using-enumerate, - consider-using-generator, - consider-using-f-string, - consider-using-set-comprehension, - consider-iterating-dictionary, - - #Settings about imports- will clean up in a future branch - wrong-import-position, - ungrouped-imports, - import-outside-toplevel, - unused-import, - wrong-import-order, - - #Settings that dislike how we handle args and variable names- some might be valid - invalid-name, - arguments-differ, - arguments-renamed, - - #Settings with minor fixes that will be cleaned up in future branch - singleton-comparison, - simplifiable-if-statement, - pointless-string-statement, - trailing-whitespace, - consider-swap-variables, - -enable= - - -[REPORTS] - -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html. You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". This option is deprecated -# and it will be removed in Pylint 2.0. -files-output=no - -# Tells whether to display a full report or only the messages -reports=yes - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - - -[BASIC] - -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_,l - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -property-classes=abc.abstractproperty - -# Regular expression matching correct function names -function-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for function names -function-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct variable names -variable-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for variable names -variable-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct constant names -const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Naming hint for constant names -const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Regular expression matching correct attribute names -attr-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for attribute names -attr-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct argument names -argument-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for argument names -argument-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct class attribute names -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Regular expression matching correct inline iteration names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ - -# Regular expression matching correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression matching correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression matching correct method names -method-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for method names -method-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - - -[ELIF] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - - -[TYPECHECK] - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - - -[FORMAT] - -# Maximum number of characters on a single line. -max-line-length=100 - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma,dict-separator - -# Maximum number of lines in a module -max-module-lines=1000 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -# Use 2 spaces consistent with TensorFlow style. -indent-string=' ' - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO - - -[VARIABLES] - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=(_+[a-zA-Z0-9]*?$)|dummy - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,future.builtins - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging - - -[SIMILARITIES] - -# Minimum lines number of a similarity. -min-similarity-lines=4 - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - - -[IMPORTS] - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,TERMIOS,Bastion,rexec - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analyzed. -analyse-fallback-blocks=no - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=7 - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.* - -# Maximum number of locals for function / method body -max-locals=15 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of branch for function / method body -max-branches=12 - -# Maximum number of statements in function / method body -max-statements=50 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=0 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict,_fields,_replace,_source,_make - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception diff --git a/devtools/run_all_linting.sh b/devtools/run_all_linting.sh deleted file mode 100644 index cdea973aa1..0000000000 --- a/devtools/run_all_linting.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -sh 'run_flake8_linting.sh' -sh 'run_pylint.sh' diff --git a/devtools/run_pylint.sh b/devtools/run_pylint.sh deleted file mode 100644 index 64f9082915..0000000000 --- a/devtools/run_pylint.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -echo 'Removing old output files...' -rm --force 'pylint.output' - -echo 'Running Pylint on settings defined in pylintrc...' -pylint '../desc' '../tests' --output 'pylint.output' 1>/dev/null -echo 'Done!' diff --git a/docs/installation.rst b/docs/installation.rst index 949a079f51..215eba698d 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -24,8 +24,6 @@ On Your Local Machine **Or from GitHub (for development builds)** -`Option 1 tested to work on M1 Macbook on 5-3-23` - First download the repository from GitHub. .. code-block:: sh @@ -37,6 +35,8 @@ Now pick one of the installation options below. Option 1: Using pip to install packages (this will only install DESC + JAX with CPU capabilities, NOT GPU) +`Option 1 tested to work on M1 Macbook on 5-3-23` + .. code-block:: sh conda create --name desc-env 'python>=3.8' @@ -179,7 +179,7 @@ On Clusters with IBM Power Architecture If pre-built JAX binaries are not available, you will first need to build JAX from source. More info can be found here: https://jax.readthedocs.io/en/latest/developer.html -The following are instructions tested to work on the Traverse supercomputer at Princeton: +These instructions were tested and confirmed to work on the Traverse supercomputer at Princeton as of 7-10-2023. .. code-block:: sh From 5f00a73afdef789769b5afdb6a6cb315a94b806d Mon Sep 17 00:00:00 2001 From: Kaya Unalmis Date: Fri, 21 Jul 2023 23:35:05 -0500 Subject: [PATCH 2/2] Add pre-commit to dev requirements --- devtools/dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devtools/dev-requirements.txt b/devtools/dev-requirements.txt index f75ab443e4..581e687e30 100644 --- a/devtools/dev-requirements.txt +++ b/devtools/dev-requirements.txt @@ -14,7 +14,7 @@ flake8 >= 5.0.0, <=6.0.0 flake8-docstrings >= 1.0.0, <=2.0.0 flake8-eradicate >= 1.0.0, <=2.0.0 flake8-isort >=5.0.0, <= 6.0.0 - +pre-commit # testing and benchmarking markupsafe == 2.0.1