Upgrade edx-lint BOM-1298 (#23227)

The last time we tried this upgrade we encountered timeouts on the quality job, which it now appears were due to the worker running pylint common running out of memory and killing the Jenkins process. Switching to a different worker type with double the RAM (8 GB vs. 4 GB) seems to have fixed this; about 5.5 GB was used. Upstream is aware of the high memory usage on large projects, it's apparently due primarily to a cache of parsed modules: pylint-dev/pylint#1495 .

Even after disabling some of the new checks that have been added, the new version of pylint found about twice as much to complain about. Just bumping the threshold for now to unblock the Django upgrade, we can try automated utilities like pyupgrade to fix some of these later.

pylintrc

@@ -356,6 +356,11 @@ disable =
 	xrange-builtin,
 	zip-builtin-not-iterating,
 	native-string,
+	import-outside-toplevel,
+	inconsistent-return-statements,
+	no-else-break,
+	no-else-continue,
+	useless-object-inheritance,
 
 [REPORTS]
 output-format = text
@@ -455,4 +460,4 @@ int-import-graph =
 [EXCEPTIONS]
 overgeneral-exceptions = Exception
 
-# 40eae2f3b3491a12ea2f2c98c632d7cb43904d56
+# e89c3d2722e0389b5f94fdaba1c1501a3f855af9

pylintrc_tweaks

@@ -4,9 +4,15 @@ ignore+ = ,.git,.tox,migrations,node_modules,.pycharm_helpers
 init-hook="import sys; sys.path.extend(['lms/djangoapps', 'cms/djangoapps', 'common/djangoapps'])"
 
 [MESSAGES CONTROL]
-# Disable unicode-format-string until we can agree to turn it on.
 disable+ =
+    # Disable unicode-format-string until we can agree to turn it on.
     native-string,
+    # Disable new refactoring suggestions for now, we have more important things to fix first
+    import-outside-toplevel,
+    inconsistent-return-statements,
+    no-else-break,
+    no-else-continue,
+    useless-object-inheritance,
 
 [BASIC]
 attr-rgx = [a-z_][a-z0-9_]{2,40}$

requirements/constraints.txt

@@ -77,5 +77,8 @@ oauthlib==2.1.0
 #   https://github.com/transifex/transifex-client/issues/252
 transifex-client==0.13.4
 
+# Constraint from astroid 2.3.3
+wrapt==1.11.*
+
 # zipp 2.0.0 requires Python >= 3.6
 zipp==1.0.0

requirements/edx/base.txt

@@ -244,7 +244,7 @@ watchdog==0.10.2          # via -r requirements/edx/paver.txt
 web-fragments==0.3.1      # via -r requirements/edx/base.in, staff-graded-xblock, xblock, xblock-utils
 webencodings==0.5.1       # via bleach, html5lib
 webob==1.8.6              # via xblock, xmodule
-wrapt==1.12.0             # via -r requirements/edx/paver.txt
+wrapt==1.11.2             # via -c requirements/edx/../constraints.txt, -r requirements/edx/paver.txt
 git+https://github.com/edx-solutions/[email protected]#egg=xblock-drag-and-drop-v2==2.2.8  # via -r requirements/edx/github.in
 git+https://github.com/open-craft/xblock-poll@3c7dcaf6c933d914188f0740a60711603f948d26#egg=xblock-poll==1.9.1  # via -r requirements/edx/github.in
 xblock-utils==1.2.4       # via -r requirements/edx/base.in, edx-sga, lti-consumer-xblock, staff-graded-xblock, xblock-drag-and-drop-v2, xblock-google-drive

requirements/edx/development.txt

@@ -28,7 +28,7 @@ aniso8601==8.0.0          # via -r requirements/edx/testing.txt, edx-tincan-py35
 anyjson==0.3.3            # via -r requirements/edx/testing.txt, kombu
 apipkg==1.5               # via -r requirements/edx/testing.txt, execnet
 appdirs==1.4.3            # via -r requirements/edx/testing.txt, fs, virtualenv
-astroid==1.5.3            # via -r requirements/edx/testing.txt, pylint, pylint-celery
+astroid==2.3.3            # via -r requirements/edx/testing.txt, pylint, pylint-celery
 attrs==19.3.0             # via -r requirements/edx/testing.txt, edx-ace, jsonschema, pytest
 babel==2.8.0              # via -r requirements/edx/testing.txt, django-babel, django-babel-underscore, sphinx
 beautifulsoup4==4.8.2     # via -r requirements/edx/testing.txt, pynliner
@@ -118,7 +118,7 @@ edx-django-utils==3.0     # via -r requirements/edx/testing.txt, django-config-m
 edx-drf-extensions==2.4.6  # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, edx-completion, edx-enterprise, edx-organizations, edx-proctoring, edx-rbac, edx-when, edxval
 edx-enterprise==2.4.1     # via -r requirements/edx/testing.txt
 edx-i18n-tools==0.5.0     # via -r requirements/edx/testing.txt, ora2
-edx-lint==1.3.0           # via -r requirements/edx/testing.txt
+edx-lint==1.4.1           # via -r requirements/edx/testing.txt
 edx-milestones==0.2.6     # via -r requirements/edx/testing.txt
 edx-oauth2-provider==1.3.1  # via -r requirements/edx/testing.txt
 edx-opaque-keys[django]==2.0.1  # via -r requirements/edx/testing.txt, edx-bulk-grades, edx-ccx-keys, edx-completion, edx-drf-extensions, edx-enterprise, edx-milestones, edx-organizations, edx-proctoring, edx-user-state-client, edx-when, xmodule
@@ -234,9 +234,9 @@ pyinotify==0.9.6          # via -r requirements/edx/development.in
 pyjwkest==1.3.2           # via -r requirements/edx/testing.txt, edx-drf-extensions
 pyjwt==1.5.2              # via -r requirements/edx/testing.txt, djangorestframework-jwt, edx-oauth2-provider, edx-rest-api-client, social-auth-core
 pylint-celery==0.3        # via -r requirements/edx/testing.txt, edx-lint
-pylint-django==0.7.2      # via -r requirements/edx/testing.txt, edx-lint
+pylint-django==2.0.11     # via -r requirements/edx/testing.txt, edx-lint
 pylint-plugin-utils==0.6  # via -r requirements/edx/testing.txt, pylint-celery, pylint-django
-pylint==1.7.6             # via -r requirements/edx/testing.txt, edx-lint, pylint-celery, pylint-django, pylint-plugin-utils
+pylint==2.4.2             # via -r requirements/edx/testing.txt, edx-lint, pylint-celery, pylint-django, pylint-plugin-utils
 pymongo==3.9.0            # via -r requirements/edx/testing.txt, edx-opaque-keys, event-tracking, mongodbproxy, mongoengine
 pynliner==0.8.0           # via -r requirements/edx/testing.txt
 pyparsing==2.2.0          # via -r requirements/edx/testing.txt, calc, chem, packaging, pycontracts
@@ -282,7 +282,7 @@ shapely==1.7.0            # via -r requirements/edx/testing.txt
 shortuuid==0.5.0          # via -r requirements/edx/testing.txt, edx-django-oauth2-provider
 simplejson==3.17.0        # via -r requirements/edx/testing.txt, sailthru-client, super-csv, xblock-utils
 singledispatch==3.4.0.3   # via -r requirements/edx/testing.txt
-six==1.14.0               # via -r requirements/edx/pip-tools.txt, -r requirements/edx/testing.txt, analytics-python, astroid, bleach, bok-choy, calc, cryptography, diff-cover, django-appconf, django-classy-tags, django-countries, django-pyfs, django-sekizai, django-simple-history, django-statici18n, drf-yasg, edx-ace, edx-ccx-keys, edx-django-release-util, edx-drf-extensions, edx-enterprise, edx-i18n-tools, edx-lint, edx-milestones, edx-opaque-keys, edx-rbac, edx-search, edx-sphinx-theme, event-tracking, freezegun, fs, fs-s3fs, help-tokens, html5lib, httpretty, isodate, jsonschema, libsass, mando, mock, nltk, packaging, pathlib2, paver, pip-tools, pycontracts, pyjwkest, pylint, pytest-xdist, python-dateutil, python-memcached, python-swiftclient, singledispatch, social-auth-app-django, social-auth-core, sphinxcontrib-httpdomain, stevedore, tox, transifex-client, virtualenv, xblock
+six==1.14.0               # via -r requirements/edx/pip-tools.txt, -r requirements/edx/testing.txt, analytics-python, astroid, bleach, bok-choy, calc, cryptography, diff-cover, django-appconf, django-classy-tags, django-countries, django-pyfs, django-sekizai, django-simple-history, django-statici18n, drf-yasg, edx-ace, edx-ccx-keys, edx-django-release-util, edx-drf-extensions, edx-enterprise, edx-i18n-tools, edx-lint, edx-milestones, edx-opaque-keys, edx-rbac, edx-search, edx-sphinx-theme, event-tracking, freezegun, fs, fs-s3fs, help-tokens, html5lib, httpretty, isodate, jsonschema, libsass, mando, mock, nltk, packaging, pathlib2, paver, pip-tools, pycontracts, pyjwkest, pytest-xdist, python-dateutil, python-memcached, python-swiftclient, singledispatch, social-auth-app-django, social-auth-core, sphinxcontrib-httpdomain, stevedore, tox, transifex-client, virtualenv, xblock
 slumber==0.7.1            # via -r requirements/edx/testing.txt, edx-bulk-grades, edx-enterprise, edx-rest-api-client
 snowballstemmer==2.0.0    # via sphinx
 social-auth-core==3.2.0   # via -r requirements/edx/testing.txt, social-auth-app-django
@@ -309,6 +309,7 @@ toml==0.10.0              # via -r requirements/edx/testing.txt, tox
 tox-battery==0.5.2        # via -r requirements/edx/testing.txt
 tox==3.14.5               # via -r requirements/edx/testing.txt, tox-battery
 transifex-client==0.13.4  # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt
+typed-ast==1.4.1          # via -r requirements/edx/testing.txt, astroid
 unicodecsv==0.14.1        # via -r requirements/edx/testing.txt, edx-enterprise
 unidiff==0.5.5            # via -r requirements/edx/testing.txt, coverage-pytest-plugin
 uritemplate==3.0.1        # via -r requirements/edx/testing.txt, coreapi, drf-yasg
@@ -322,7 +323,7 @@ wcwidth==0.1.8            # via -r requirements/edx/testing.txt, pytest
 web-fragments==0.3.1      # via -r requirements/edx/testing.txt, staff-graded-xblock, xblock, xblock-utils
 webencodings==0.5.1       # via -r requirements/edx/testing.txt, bleach, html5lib
 webob==1.8.6              # via -r requirements/edx/testing.txt, xblock, xmodule
-wrapt==1.12.0             # via -r requirements/edx/testing.txt, astroid
+wrapt==1.11.2             # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, astroid
 git+https://github.com/edx-solutions/[email protected]#egg=xblock-drag-and-drop-v2==2.2.8  # via -r requirements/edx/testing.txt
 git+https://github.com/open-craft/xblock-poll@3c7dcaf6c933d914188f0740a60711603f948d26#egg=xblock-poll==1.9.1  # via -r requirements/edx/testing.txt
 xblock-utils==1.2.4       # via -r requirements/edx/testing.txt, edx-sga, lti-consumer-xblock, staff-graded-xblock, xblock-drag-and-drop-v2, xblock-google-drive

requirements/edx/paver.txt

@@ -26,5 +26,5 @@ six==1.14.0               # via edx-opaque-keys, libsass, mock, paver, python-me
 stevedore==1.32.0         # via -r requirements/edx/paver.in, edx-opaque-keys
 urllib3==1.25.8           # via requests
 watchdog==0.10.2          # via -r requirements/edx/paver.in
-wrapt==1.12.0             # via -r requirements/edx/paver.in
+wrapt==1.11.2             # via -c requirements/edx/../constraints.txt, -r requirements/edx/paver.in
 zipp==1.0.0               # via -c requirements/edx/../constraints.txt, importlib-metadata

requirements/edx/testing.in

@@ -23,7 +23,7 @@ code-annotations          # Perform code annotation checking, such as for PII an
 cssselect                 # Used to extract HTML fragments via CSS selectors in 2 test cases and pyquery
 ddt                       # Run a test case multiple times with different input; used in many, many of our tests
 edx-i18n-tools>=0.4.6     # Commands for developers and translators to extract, compile and validate translations
-edx-lint==1.3.0           # pylint extensions for Open edX repositories
+edx-lint                  # pylint extensions for Open edX repositories
 factory-boy               # Library for creating test fixtures, used in many tests
 # Pinning the freezegun version because 0.3.13 is causing failures which have also been reported on the git repo by public.
 freezegun                 # Allows tests to mock the output of assorted datetime module functions

requirements/edx/testing.txt

@@ -27,7 +27,7 @@ aniso8601==8.0.0          # via -r requirements/edx/base.txt, edx-tincan-py35
 anyjson==0.3.3            # via -r requirements/edx/base.txt, kombu
 apipkg==1.5               # via execnet
 appdirs==1.4.3            # via -r requirements/edx/base.txt, fs, virtualenv
-astroid==1.5.3            # via pylint, pylint-celery
+astroid==2.3.3            # via pylint, pylint-celery
 attrs==19.3.0             # via -r requirements/edx/base.txt, edx-ace, pytest
 babel==2.8.0              # via -r requirements/edx/base.txt, django-babel, django-babel-underscore
 beautifulsoup4==4.8.2     # via -r requirements/edx/base.txt, -r requirements/edx/testing.in, pynliner
@@ -114,7 +114,7 @@ edx-django-utils==3.0     # via -r requirements/edx/base.txt, django-config-mode
 edx-drf-extensions==2.4.6  # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, edx-completion, edx-enterprise, edx-organizations, edx-proctoring, edx-rbac, edx-when, edxval
 edx-enterprise==2.4.1     # via -r requirements/edx/base.txt
 edx-i18n-tools==0.5.0     # via -r requirements/edx/base.txt, -r requirements/edx/testing.in, ora2
-edx-lint==1.3.0           # via -r requirements/edx/testing.in
+edx-lint==1.4.1           # via -r requirements/edx/testing.in
 edx-milestones==0.2.6     # via -r requirements/edx/base.txt
 edx-oauth2-provider==1.3.1  # via -r requirements/edx/base.txt
 edx-opaque-keys[django]==2.0.1  # via -r requirements/edx/base.txt, edx-bulk-grades, edx-ccx-keys, edx-completion, edx-drf-extensions, edx-enterprise, edx-milestones, edx-organizations, edx-proctoring, edx-user-state-client, edx-when, xmodule
@@ -223,9 +223,9 @@ pygments==2.5.2           # via -r requirements/edx/base.txt, -r requirements/ed
 pyjwkest==1.3.2           # via -r requirements/edx/base.txt, edx-drf-extensions
 pyjwt==1.5.2              # via -r requirements/edx/base.txt, djangorestframework-jwt, edx-oauth2-provider, edx-rest-api-client, social-auth-core
 pylint-celery==0.3        # via edx-lint
-pylint-django==0.7.2      # via edx-lint
+pylint-django==2.0.11     # via edx-lint
 pylint-plugin-utils==0.6  # via pylint-celery, pylint-django
-pylint==1.7.6             # via edx-lint, pylint-celery, pylint-django, pylint-plugin-utils
+pylint==2.4.2             # via edx-lint, pylint-celery, pylint-django, pylint-plugin-utils
 pymongo==3.9.0            # via -r requirements/edx/base.txt, edx-opaque-keys, event-tracking, mongodbproxy, mongoengine
 pynliner==0.8.0           # via -r requirements/edx/base.txt
 pyparsing==2.2.0          # via -r requirements/edx/base.txt, calc, chem, packaging, pycontracts
@@ -270,7 +270,7 @@ shapely==1.7.0            # via -r requirements/edx/base.txt
 shortuuid==0.5.0          # via -r requirements/edx/base.txt, edx-django-oauth2-provider
 simplejson==3.17.0        # via -r requirements/edx/base.txt, sailthru-client, super-csv, xblock-utils
 singledispatch==3.4.0.3   # via -r requirements/edx/testing.in
-six==1.14.0               # via -r requirements/edx/base.txt, -r requirements/edx/coverage.txt, analytics-python, astroid, bleach, bok-choy, calc, cryptography, diff-cover, django-appconf, django-classy-tags, django-countries, django-pyfs, django-sekizai, django-simple-history, django-statici18n, drf-yasg, edx-ace, edx-ccx-keys, edx-django-release-util, edx-drf-extensions, edx-enterprise, edx-i18n-tools, edx-lint, edx-milestones, edx-opaque-keys, edx-rbac, edx-search, event-tracking, freezegun, fs, fs-s3fs, help-tokens, html5lib, httpretty, isodate, libsass, mando, mock, nltk, packaging, pathlib2, paver, pycontracts, pyjwkest, pylint, pytest-xdist, python-dateutil, python-memcached, python-swiftclient, singledispatch, social-auth-app-django, social-auth-core, stevedore, tox, transifex-client, virtualenv, xblock
+six==1.14.0               # via -r requirements/edx/base.txt, -r requirements/edx/coverage.txt, analytics-python, astroid, bleach, bok-choy, calc, cryptography, diff-cover, django-appconf, django-classy-tags, django-countries, django-pyfs, django-sekizai, django-simple-history, django-statici18n, drf-yasg, edx-ace, edx-ccx-keys, edx-django-release-util, edx-drf-extensions, edx-enterprise, edx-i18n-tools, edx-lint, edx-milestones, edx-opaque-keys, edx-rbac, edx-search, event-tracking, freezegun, fs, fs-s3fs, help-tokens, html5lib, httpretty, isodate, libsass, mando, mock, nltk, packaging, pathlib2, paver, pycontracts, pyjwkest, pytest-xdist, python-dateutil, python-memcached, python-swiftclient, singledispatch, social-auth-app-django, social-auth-core, stevedore, tox, transifex-client, virtualenv, xblock
 slumber==0.7.1            # via -r requirements/edx/base.txt, edx-bulk-grades, edx-enterprise, edx-rest-api-client
 social-auth-core==3.2.0   # via -r requirements/edx/base.txt, social-auth-app-django
 git+https://github.com/jazzband/sorl-thumbnail.git@13bedfb7d2970809eda597e3ef79318a6fa80ac2#egg=sorl-thumbnail  # via -r requirements/edx/base.txt
@@ -287,6 +287,7 @@ toml==0.10.0              # via tox
 tox-battery==0.5.2        # via -r requirements/edx/testing.in
 tox==3.14.5               # via -r requirements/edx/testing.in, tox-battery
 transifex-client==0.13.4  # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.in
+typed-ast==1.4.1          # via astroid
 unicodecsv==0.14.1        # via -r requirements/edx/base.txt, edx-enterprise
 unidiff==0.5.5            # via -r requirements/edx/testing.in, coverage-pytest-plugin
 uritemplate==3.0.1        # via -r requirements/edx/base.txt, coreapi, drf-yasg
@@ -299,7 +300,7 @@ wcwidth==0.1.8            # via pytest
 web-fragments==0.3.1      # via -r requirements/edx/base.txt, staff-graded-xblock, xblock, xblock-utils
 webencodings==0.5.1       # via -r requirements/edx/base.txt, bleach, html5lib
 webob==1.8.6              # via -r requirements/edx/base.txt, xblock, xmodule
-wrapt==1.12.0             # via -r requirements/edx/base.txt, astroid
+wrapt==1.11.2             # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, astroid
 git+https://github.com/edx-solutions/[email protected]#egg=xblock-drag-and-drop-v2==2.2.8  # via -r requirements/edx/base.txt
 git+https://github.com/open-craft/xblock-poll@3c7dcaf6c933d914188f0740a60711603f948d26#egg=xblock-poll==1.9.1  # via -r requirements/edx/base.txt
 xblock-utils==1.2.4       # via -r requirements/edx/base.txt, edx-sga, lti-consumer-xblock, staff-graded-xblock, xblock-drag-and-drop-v2, xblock-google-drive

scripts/Jenkinsfiles/quality

@@ -74,7 +74,8 @@ pipeline {
         stage('Run Tests') {
             parallel {
                 stage("commonlib pylint") {
-                    agent { label "jenkins-worker" }
+                    // "pylint common" requires 5.5 GB of RAM, so use js-worker (8 GB) instead of jenkins-worker (4 GB)
+                    agent { label "js-worker" }
                     environment {
                         TEST_SUITE = "quality"
                         SHARD = 1

scripts/thresholds.sh

@@ -2,6 +2,6 @@
 set -e
 
 export LOWER_PYLINT_THRESHOLD=1000
-export UPPER_PYLINT_THRESHOLD=1962
+export UPPER_PYLINT_THRESHOLD=4050
 export ESLINT_THRESHOLD=5530
 export STYLELINT_THRESHOLD=880