From 45644a351118e5469da48f4a173c6ddb080cac13 Mon Sep 17 00:00:00 2001 From: Jeremy Bowman Date: Mon, 2 Mar 2020 08:27:19 -0500 Subject: [PATCH] Upgrade edx-lint BOM-1298 (#23227) The last time we tried this upgrade we encountered timeouts on the quality job, which it now appears were due to the worker running pylint common running out of memory and killing the Jenkins process. Switching to a different worker type with double the RAM (8 GB vs. 4 GB) seems to have fixed this; about 5.5 GB was used. Upstream is aware of the high memory usage on large projects, it's apparently due primarily to a cache of parsed modules: https://github.com/PyCQA/pylint/issues/1495 . Even after disabling some of the new checks that have been added, the new version of pylint found about twice as much to complain about. Just bumping the threshold for now to unblock the Django upgrade, we can try automated utilities like pyupgrade to fix some of these later. --- pylintrc | 7 ++++++- pylintrc_tweaks | 8 +++++++- requirements/constraints.txt | 3 +++ requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 13 +++++++------ requirements/edx/paver.txt | 2 +- requirements/edx/testing.in | 2 +- requirements/edx/testing.txt | 13 +++++++------ scripts/Jenkinsfiles/quality | 3 ++- scripts/thresholds.sh | 2 +- 10 files changed, 36 insertions(+), 19 deletions(-) diff --git a/pylintrc b/pylintrc index f9643b0ed9..b94a6d1965 100644 --- a/pylintrc +++ b/pylintrc @@ -356,6 +356,11 @@ disable = xrange-builtin, zip-builtin-not-iterating, native-string, + import-outside-toplevel, + inconsistent-return-statements, + no-else-break, + no-else-continue, + useless-object-inheritance, [REPORTS] output-format = text @@ -455,4 +460,4 @@ int-import-graph = [EXCEPTIONS] overgeneral-exceptions = Exception -# 40eae2f3b3491a12ea2f2c98c632d7cb43904d56 +# e89c3d2722e0389b5f94fdaba1c1501a3f855af9 diff --git a/pylintrc_tweaks b/pylintrc_tweaks index 00799bdfe5..0032009f15 100644 --- a/pylintrc_tweaks +++ b/pylintrc_tweaks @@ -4,9 +4,15 @@ ignore+ = ,.git,.tox,migrations,node_modules,.pycharm_helpers init-hook="import sys; sys.path.extend(['lms/djangoapps', 'cms/djangoapps', 'common/djangoapps'])" [MESSAGES CONTROL] -# Disable unicode-format-string until we can agree to turn it on. disable+ = + # Disable unicode-format-string until we can agree to turn it on. native-string, + # Disable new refactoring suggestions for now, we have more important things to fix first + import-outside-toplevel, + inconsistent-return-statements, + no-else-break, + no-else-continue, + useless-object-inheritance, [BASIC] attr-rgx = [a-z_][a-z0-9_]{2,40}$ diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 183316bb06..082ce26666 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -77,5 +77,8 @@ oauthlib==2.1.0 # https://github.com/transifex/transifex-client/issues/252 transifex-client==0.13.4 +# Constraint from astroid 2.3.3 +wrapt==1.11.* + # zipp 2.0.0 requires Python >= 3.6 zipp==1.0.0 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 991ac80720..69a3b4ab0c 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -244,7 +244,7 @@ watchdog==0.10.2 # via -r requirements/edx/paver.txt web-fragments==0.3.1 # via -r requirements/edx/base.in, staff-graded-xblock, xblock, xblock-utils webencodings==0.5.1 # via bleach, html5lib webob==1.8.6 # via xblock, xmodule -wrapt==1.12.0 # via -r requirements/edx/paver.txt +wrapt==1.11.2 # via -c requirements/edx/../constraints.txt, -r requirements/edx/paver.txt git+https://github.com/edx-solutions/xblock-drag-and-drop-v2@v2.2.8#egg=xblock-drag-and-drop-v2==2.2.8 # via -r requirements/edx/github.in git+https://github.com/open-craft/xblock-poll@3c7dcaf6c933d914188f0740a60711603f948d26#egg=xblock-poll==1.9.1 # via -r requirements/edx/github.in xblock-utils==1.2.4 # via -r requirements/edx/base.in, edx-sga, lti-consumer-xblock, staff-graded-xblock, xblock-drag-and-drop-v2, xblock-google-drive diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 775cf7c073..78f4c3ca3e 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -28,7 +28,7 @@ aniso8601==8.0.0 # via -r requirements/edx/testing.txt, edx-tincan-py35 anyjson==0.3.3 # via -r requirements/edx/testing.txt, kombu apipkg==1.5 # via -r requirements/edx/testing.txt, execnet appdirs==1.4.3 # via -r requirements/edx/testing.txt, fs, virtualenv -astroid==1.5.3 # via -r requirements/edx/testing.txt, pylint, pylint-celery +astroid==2.3.3 # via -r requirements/edx/testing.txt, pylint, pylint-celery attrs==19.3.0 # via -r requirements/edx/testing.txt, edx-ace, jsonschema, pytest babel==2.8.0 # via -r requirements/edx/testing.txt, django-babel, django-babel-underscore, sphinx beautifulsoup4==4.8.2 # via -r requirements/edx/testing.txt, pynliner @@ -118,7 +118,7 @@ edx-django-utils==3.0 # via -r requirements/edx/testing.txt, django-config-m edx-drf-extensions==2.4.6 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, edx-completion, edx-enterprise, edx-organizations, edx-proctoring, edx-rbac, edx-when, edxval edx-enterprise==2.4.1 # via -r requirements/edx/testing.txt edx-i18n-tools==0.5.0 # via -r requirements/edx/testing.txt, ora2 -edx-lint==1.3.0 # via -r requirements/edx/testing.txt +edx-lint==1.4.1 # via -r requirements/edx/testing.txt edx-milestones==0.2.6 # via -r requirements/edx/testing.txt edx-oauth2-provider==1.3.1 # via -r requirements/edx/testing.txt edx-opaque-keys[django]==2.0.1 # via -r requirements/edx/testing.txt, edx-bulk-grades, edx-ccx-keys, edx-completion, edx-drf-extensions, edx-enterprise, edx-milestones, edx-organizations, edx-proctoring, edx-user-state-client, edx-when, xmodule @@ -234,9 +234,9 @@ pyinotify==0.9.6 # via -r requirements/edx/development.in pyjwkest==1.3.2 # via -r requirements/edx/testing.txt, edx-drf-extensions pyjwt==1.5.2 # via -r requirements/edx/testing.txt, djangorestframework-jwt, edx-oauth2-provider, edx-rest-api-client, social-auth-core pylint-celery==0.3 # via -r requirements/edx/testing.txt, edx-lint -pylint-django==0.7.2 # via -r requirements/edx/testing.txt, edx-lint +pylint-django==2.0.11 # via -r requirements/edx/testing.txt, edx-lint pylint-plugin-utils==0.6 # via -r requirements/edx/testing.txt, pylint-celery, pylint-django -pylint==1.7.6 # via -r requirements/edx/testing.txt, edx-lint, pylint-celery, pylint-django, pylint-plugin-utils +pylint==2.4.2 # via -r requirements/edx/testing.txt, edx-lint, pylint-celery, pylint-django, pylint-plugin-utils pymongo==3.9.0 # via -r requirements/edx/testing.txt, edx-opaque-keys, event-tracking, mongodbproxy, mongoengine pynliner==0.8.0 # via -r requirements/edx/testing.txt pyparsing==2.2.0 # via -r requirements/edx/testing.txt, calc, chem, packaging, pycontracts @@ -282,7 +282,7 @@ shapely==1.7.0 # via -r requirements/edx/testing.txt shortuuid==0.5.0 # via -r requirements/edx/testing.txt, edx-django-oauth2-provider simplejson==3.17.0 # via -r requirements/edx/testing.txt, sailthru-client, super-csv, xblock-utils singledispatch==3.4.0.3 # via -r requirements/edx/testing.txt -six==1.14.0 # via -r requirements/edx/pip-tools.txt, -r requirements/edx/testing.txt, analytics-python, astroid, bleach, bok-choy, calc, cryptography, diff-cover, django-appconf, django-classy-tags, django-countries, django-pyfs, django-sekizai, django-simple-history, django-statici18n, drf-yasg, edx-ace, edx-ccx-keys, edx-django-release-util, edx-drf-extensions, edx-enterprise, edx-i18n-tools, edx-lint, edx-milestones, edx-opaque-keys, edx-rbac, edx-search, edx-sphinx-theme, event-tracking, freezegun, fs, fs-s3fs, help-tokens, html5lib, httpretty, isodate, jsonschema, libsass, mando, mock, nltk, packaging, pathlib2, paver, pip-tools, pycontracts, pyjwkest, pylint, pytest-xdist, python-dateutil, python-memcached, python-swiftclient, singledispatch, social-auth-app-django, social-auth-core, sphinxcontrib-httpdomain, stevedore, tox, transifex-client, virtualenv, xblock +six==1.14.0 # via -r requirements/edx/pip-tools.txt, -r requirements/edx/testing.txt, analytics-python, astroid, bleach, bok-choy, calc, cryptography, diff-cover, django-appconf, django-classy-tags, django-countries, django-pyfs, django-sekizai, django-simple-history, django-statici18n, drf-yasg, edx-ace, edx-ccx-keys, edx-django-release-util, edx-drf-extensions, edx-enterprise, edx-i18n-tools, edx-lint, edx-milestones, edx-opaque-keys, edx-rbac, edx-search, edx-sphinx-theme, event-tracking, freezegun, fs, fs-s3fs, help-tokens, html5lib, httpretty, isodate, jsonschema, libsass, mando, mock, nltk, packaging, pathlib2, paver, pip-tools, pycontracts, pyjwkest, pytest-xdist, python-dateutil, python-memcached, python-swiftclient, singledispatch, social-auth-app-django, social-auth-core, sphinxcontrib-httpdomain, stevedore, tox, transifex-client, virtualenv, xblock slumber==0.7.1 # via -r requirements/edx/testing.txt, edx-bulk-grades, edx-enterprise, edx-rest-api-client snowballstemmer==2.0.0 # via sphinx social-auth-core==3.2.0 # via -r requirements/edx/testing.txt, social-auth-app-django @@ -309,6 +309,7 @@ toml==0.10.0 # via -r requirements/edx/testing.txt, tox tox-battery==0.5.2 # via -r requirements/edx/testing.txt tox==3.14.5 # via -r requirements/edx/testing.txt, tox-battery transifex-client==0.13.4 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt +typed-ast==1.4.1 # via -r requirements/edx/testing.txt, astroid unicodecsv==0.14.1 # via -r requirements/edx/testing.txt, edx-enterprise unidiff==0.5.5 # via -r requirements/edx/testing.txt, coverage-pytest-plugin uritemplate==3.0.1 # via -r requirements/edx/testing.txt, coreapi, drf-yasg @@ -322,7 +323,7 @@ wcwidth==0.1.8 # via -r requirements/edx/testing.txt, pytest web-fragments==0.3.1 # via -r requirements/edx/testing.txt, staff-graded-xblock, xblock, xblock-utils webencodings==0.5.1 # via -r requirements/edx/testing.txt, bleach, html5lib webob==1.8.6 # via -r requirements/edx/testing.txt, xblock, xmodule -wrapt==1.12.0 # via -r requirements/edx/testing.txt, astroid +wrapt==1.11.2 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, astroid git+https://github.com/edx-solutions/xblock-drag-and-drop-v2@v2.2.8#egg=xblock-drag-and-drop-v2==2.2.8 # via -r requirements/edx/testing.txt git+https://github.com/open-craft/xblock-poll@3c7dcaf6c933d914188f0740a60711603f948d26#egg=xblock-poll==1.9.1 # via -r requirements/edx/testing.txt xblock-utils==1.2.4 # via -r requirements/edx/testing.txt, edx-sga, lti-consumer-xblock, staff-graded-xblock, xblock-drag-and-drop-v2, xblock-google-drive diff --git a/requirements/edx/paver.txt b/requirements/edx/paver.txt index 0594be7f77..4a4d8fe20c 100644 --- a/requirements/edx/paver.txt +++ b/requirements/edx/paver.txt @@ -26,5 +26,5 @@ six==1.14.0 # via edx-opaque-keys, libsass, mock, paver, python-me stevedore==1.32.0 # via -r requirements/edx/paver.in, edx-opaque-keys urllib3==1.25.8 # via requests watchdog==0.10.2 # via -r requirements/edx/paver.in -wrapt==1.12.0 # via -r requirements/edx/paver.in +wrapt==1.11.2 # via -c requirements/edx/../constraints.txt, -r requirements/edx/paver.in zipp==1.0.0 # via -c requirements/edx/../constraints.txt, importlib-metadata diff --git a/requirements/edx/testing.in b/requirements/edx/testing.in index 90af6ec548..1a5b5c279d 100644 --- a/requirements/edx/testing.in +++ b/requirements/edx/testing.in @@ -23,7 +23,7 @@ code-annotations # Perform code annotation checking, such as for PII an cssselect # Used to extract HTML fragments via CSS selectors in 2 test cases and pyquery ddt # Run a test case multiple times with different input; used in many, many of our tests edx-i18n-tools>=0.4.6 # Commands for developers and translators to extract, compile and validate translations -edx-lint==1.3.0 # pylint extensions for Open edX repositories +edx-lint # pylint extensions for Open edX repositories factory-boy # Library for creating test fixtures, used in many tests # Pinning the freezegun version because 0.3.13 is causing failures which have also been reported on the git repo by public. freezegun # Allows tests to mock the output of assorted datetime module functions diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 318cfd32e0..a1279b41e7 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -27,7 +27,7 @@ aniso8601==8.0.0 # via -r requirements/edx/base.txt, edx-tincan-py35 anyjson==0.3.3 # via -r requirements/edx/base.txt, kombu apipkg==1.5 # via execnet appdirs==1.4.3 # via -r requirements/edx/base.txt, fs, virtualenv -astroid==1.5.3 # via pylint, pylint-celery +astroid==2.3.3 # via pylint, pylint-celery attrs==19.3.0 # via -r requirements/edx/base.txt, edx-ace, pytest babel==2.8.0 # via -r requirements/edx/base.txt, django-babel, django-babel-underscore beautifulsoup4==4.8.2 # via -r requirements/edx/base.txt, -r requirements/edx/testing.in, pynliner @@ -114,7 +114,7 @@ edx-django-utils==3.0 # via -r requirements/edx/base.txt, django-config-mode edx-drf-extensions==2.4.6 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, edx-completion, edx-enterprise, edx-organizations, edx-proctoring, edx-rbac, edx-when, edxval edx-enterprise==2.4.1 # via -r requirements/edx/base.txt edx-i18n-tools==0.5.0 # via -r requirements/edx/base.txt, -r requirements/edx/testing.in, ora2 -edx-lint==1.3.0 # via -r requirements/edx/testing.in +edx-lint==1.4.1 # via -r requirements/edx/testing.in edx-milestones==0.2.6 # via -r requirements/edx/base.txt edx-oauth2-provider==1.3.1 # via -r requirements/edx/base.txt edx-opaque-keys[django]==2.0.1 # via -r requirements/edx/base.txt, edx-bulk-grades, edx-ccx-keys, edx-completion, edx-drf-extensions, edx-enterprise, edx-milestones, edx-organizations, edx-proctoring, edx-user-state-client, edx-when, xmodule @@ -223,9 +223,9 @@ pygments==2.5.2 # via -r requirements/edx/base.txt, -r requirements/ed pyjwkest==1.3.2 # via -r requirements/edx/base.txt, edx-drf-extensions pyjwt==1.5.2 # via -r requirements/edx/base.txt, djangorestframework-jwt, edx-oauth2-provider, edx-rest-api-client, social-auth-core pylint-celery==0.3 # via edx-lint -pylint-django==0.7.2 # via edx-lint +pylint-django==2.0.11 # via edx-lint pylint-plugin-utils==0.6 # via pylint-celery, pylint-django -pylint==1.7.6 # via edx-lint, pylint-celery, pylint-django, pylint-plugin-utils +pylint==2.4.2 # via edx-lint, pylint-celery, pylint-django, pylint-plugin-utils pymongo==3.9.0 # via -r requirements/edx/base.txt, edx-opaque-keys, event-tracking, mongodbproxy, mongoengine pynliner==0.8.0 # via -r requirements/edx/base.txt pyparsing==2.2.0 # via -r requirements/edx/base.txt, calc, chem, packaging, pycontracts @@ -270,7 +270,7 @@ shapely==1.7.0 # via -r requirements/edx/base.txt shortuuid==0.5.0 # via -r requirements/edx/base.txt, edx-django-oauth2-provider simplejson==3.17.0 # via -r requirements/edx/base.txt, sailthru-client, super-csv, xblock-utils singledispatch==3.4.0.3 # via -r requirements/edx/testing.in -six==1.14.0 # via -r requirements/edx/base.txt, -r requirements/edx/coverage.txt, analytics-python, astroid, bleach, bok-choy, calc, cryptography, diff-cover, django-appconf, django-classy-tags, django-countries, django-pyfs, django-sekizai, django-simple-history, django-statici18n, drf-yasg, edx-ace, edx-ccx-keys, edx-django-release-util, edx-drf-extensions, edx-enterprise, edx-i18n-tools, edx-lint, edx-milestones, edx-opaque-keys, edx-rbac, edx-search, event-tracking, freezegun, fs, fs-s3fs, help-tokens, html5lib, httpretty, isodate, libsass, mando, mock, nltk, packaging, pathlib2, paver, pycontracts, pyjwkest, pylint, pytest-xdist, python-dateutil, python-memcached, python-swiftclient, singledispatch, social-auth-app-django, social-auth-core, stevedore, tox, transifex-client, virtualenv, xblock +six==1.14.0 # via -r requirements/edx/base.txt, -r requirements/edx/coverage.txt, analytics-python, astroid, bleach, bok-choy, calc, cryptography, diff-cover, django-appconf, django-classy-tags, django-countries, django-pyfs, django-sekizai, django-simple-history, django-statici18n, drf-yasg, edx-ace, edx-ccx-keys, edx-django-release-util, edx-drf-extensions, edx-enterprise, edx-i18n-tools, edx-lint, edx-milestones, edx-opaque-keys, edx-rbac, edx-search, event-tracking, freezegun, fs, fs-s3fs, help-tokens, html5lib, httpretty, isodate, libsass, mando, mock, nltk, packaging, pathlib2, paver, pycontracts, pyjwkest, pytest-xdist, python-dateutil, python-memcached, python-swiftclient, singledispatch, social-auth-app-django, social-auth-core, stevedore, tox, transifex-client, virtualenv, xblock slumber==0.7.1 # via -r requirements/edx/base.txt, edx-bulk-grades, edx-enterprise, edx-rest-api-client social-auth-core==3.2.0 # via -r requirements/edx/base.txt, social-auth-app-django git+https://github.com/jazzband/sorl-thumbnail.git@13bedfb7d2970809eda597e3ef79318a6fa80ac2#egg=sorl-thumbnail # via -r requirements/edx/base.txt @@ -287,6 +287,7 @@ toml==0.10.0 # via tox tox-battery==0.5.2 # via -r requirements/edx/testing.in tox==3.14.5 # via -r requirements/edx/testing.in, tox-battery transifex-client==0.13.4 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.in +typed-ast==1.4.1 # via astroid unicodecsv==0.14.1 # via -r requirements/edx/base.txt, edx-enterprise unidiff==0.5.5 # via -r requirements/edx/testing.in, coverage-pytest-plugin uritemplate==3.0.1 # via -r requirements/edx/base.txt, coreapi, drf-yasg @@ -299,7 +300,7 @@ wcwidth==0.1.8 # via pytest web-fragments==0.3.1 # via -r requirements/edx/base.txt, staff-graded-xblock, xblock, xblock-utils webencodings==0.5.1 # via -r requirements/edx/base.txt, bleach, html5lib webob==1.8.6 # via -r requirements/edx/base.txt, xblock, xmodule -wrapt==1.12.0 # via -r requirements/edx/base.txt, astroid +wrapt==1.11.2 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, astroid git+https://github.com/edx-solutions/xblock-drag-and-drop-v2@v2.2.8#egg=xblock-drag-and-drop-v2==2.2.8 # via -r requirements/edx/base.txt git+https://github.com/open-craft/xblock-poll@3c7dcaf6c933d914188f0740a60711603f948d26#egg=xblock-poll==1.9.1 # via -r requirements/edx/base.txt xblock-utils==1.2.4 # via -r requirements/edx/base.txt, edx-sga, lti-consumer-xblock, staff-graded-xblock, xblock-drag-and-drop-v2, xblock-google-drive diff --git a/scripts/Jenkinsfiles/quality b/scripts/Jenkinsfiles/quality index 483020d242..8ed91ec29e 100644 --- a/scripts/Jenkinsfiles/quality +++ b/scripts/Jenkinsfiles/quality @@ -74,7 +74,8 @@ pipeline { stage('Run Tests') { parallel { stage("commonlib pylint") { - agent { label "jenkins-worker" } + // "pylint common" requires 5.5 GB of RAM, so use js-worker (8 GB) instead of jenkins-worker (4 GB) + agent { label "js-worker" } environment { TEST_SUITE = "quality" SHARD = 1 diff --git a/scripts/thresholds.sh b/scripts/thresholds.sh index 381f107f7a..eb18c1c821 100755 --- a/scripts/thresholds.sh +++ b/scripts/thresholds.sh @@ -2,6 +2,6 @@ set -e export LOWER_PYLINT_THRESHOLD=1000 -export UPPER_PYLINT_THRESHOLD=1962 +export UPPER_PYLINT_THRESHOLD=4050 export ESLINT_THRESHOLD=5530 export STYLELINT_THRESHOLD=880