Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Upgrade edx-lint BOM-1298 (#23227)

Browse Source 
The last time we tried this upgrade we encountered timeouts on the quality job, which it now appears were due to the worker running pylint common running out of memory and killing the Jenkins process. Switching to a different worker type with double the RAM (8 GB vs. 4 GB) seems to have fixed this; about 5.5 GB was used. Upstream is aware of the high memory usage on large projects, it's apparently due primarily to a cache of parsed modules: https://github.com/PyCQA/pylint/issues/1495 .

Even after disabling some of the new checks that have been added, the new version of pylint found about twice as much to complain about. Just bumping the threshold for now to unblock the Django upgrade, we can try automated utilities like pyupgrade to fix some of these later.
This commit is contained in:
Jeremy Bowman
2020-03-02 08:27:19 -05:00
committed by GitHub
parent 6330dd1d54
commit 45644a3511
10 changed files with 36 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
pylintrc
View File

@@ -356,6 +356,11 @@ disable =
xrange-builtin,
zip-builtin-not-iterating,
native-string,
import-outside-toplevel,
inconsistent-return-statements,
no-else-break,
no-else-continue,
useless-object-inheritance,
[REPORTS]
output-format = text
@@ -455,4 +460,4 @@ int-import-graph =
[EXCEPTIONS]
overgeneral-exceptions = Exception
# 40eae2f3b3491a12ea2f2c98c632d7cb43904d56
# e89c3d2722e0389b5f94fdaba1c1501a3f855af9

8
pylintrc_tweaks
View File

@@ -4,9 +4,15 @@ ignore+ = ,.git,.tox,migrations,node_modules,.pycharm_helpers
init-hook="import sys; sys.path.extend(['lms/djangoapps', 'cms/djangoapps', 'common/djangoapps'])"
[MESSAGES CONTROL]
# Disable unicode-format-string until we can agree to turn it on.
disable+ =
# Disable unicode-format-string until we can agree to turn it on.
native-string,
# Disable new refactoring suggestions for now, we have more important things to fix first
import-outside-toplevel,
inconsistent-return-statements,
no-else-break,
no-else-continue,
useless-object-inheritance,
[BASIC]
attr-rgx = [a-z_][a-z0-9_]{2,40}$

3
requirements/constraints.txt
View File

@@ -77,5 +77,8 @@ oauthlib==2.1.0
# https://github.com/transifex/transifex-client/issues/252
transifex-client==0.13.4
# Constraint from astroid 2.3.3
wrapt==1.11.*
# zipp 2.0.0 requires Python >= 3.6
zipp==1.0.0

2
requirements/edx/base.txt
View File

@@ -244,7 +244,7 @@ watchdog==0.10.2 # via -r requirements/edx/paver.txt
web-fragments==0.3.1 # via -r requirements/edx/base.in, staff-graded-xblock, xblock, xblock-utils
webencodings==0.5.1 # via bleach, html5lib
webob==1.8.6 # via xblock, xmodule
wrapt==1.12.0 # via -r requirements/edx/paver.txt
wrapt==1.11.2 # via -c requirements/edx/../constraints.txt, -r requirements/edx/paver.txt
git+https://github.com/edx-solutions/xblock-drag-and-drop-v2@v2.2.8#egg=xblock-drag-and-drop-v2==2.2.8 # via -r requirements/edx/github.in
git+https://github.com/open-craft/xblock-poll@3c7dcaf6c933d914188f0740a60711603f948d26#egg=xblock-poll==1.9.1 # via -r requirements/edx/github.in
xblock-utils==1.2.4 # via -r requirements/edx/base.in, edx-sga, lti-consumer-xblock, staff-graded-xblock, xblock-drag-and-drop-v2, xblock-google-drive

13
requirements/edx/development.txt
View File

@@ -28,7 +28,7 @@ aniso8601==8.0.0 # via -r requirements/edx/testing.txt, edx-tincan-py35
anyjson==0.3.3 # via -r requirements/edx/testing.txt, kombu
apipkg==1.5 # via -r requirements/edx/testing.txt, execnet
appdirs==1.4.3 # via -r requirements/edx/testing.txt, fs, virtualenv
astroid==1.5.3 # via -r requirements/edx/testing.txt, pylint, pylint-celery
astroid==2.3.3 # via -r requirements/edx/testing.txt, pylint, pylint-celery
attrs==19.3.0 # via -r requirements/edx/testing.txt, edx-ace, jsonschema, pytest
babel==2.8.0 # via -r requirements/edx/testing.txt, django-babel, django-babel-underscore, sphinx
beautifulsoup4==4.8.2 # via -r requirements/edx/testing.txt, pynliner
@@ -118,7 +118,7 @@ edx-django-utils==3.0 # via -r requirements/edx/testing.txt, django-config-m
edx-drf-extensions==2.4.6 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, edx-completion, edx-enterprise, edx-organizations, edx-proctoring, edx-rbac, edx-when, edxval
edx-enterprise==2.4.1 # via -r requirements/edx/testing.txt
edx-i18n-tools==0.5.0 # via -r requirements/edx/testing.txt, ora2
edx-lint==1.3.0 # via -r requirements/edx/testing.txt
edx-lint==1.4.1 # via -r requirements/edx/testing.txt
edx-milestones==0.2.6 # via -r requirements/edx/testing.txt
edx-oauth2-provider==1.3.1 # via -r requirements/edx/testing.txt
edx-opaque-keys[django]==2.0.1 # via -r requirements/edx/testing.txt, edx-bulk-grades, edx-ccx-keys, edx-completion, edx-drf-extensions, edx-enterprise, edx-milestones, edx-organizations, edx-proctoring, edx-user-state-client, edx-when, xmodule
@@ -234,9 +234,9 @@ pyinotify==0.9.6 # via -r requirements/edx/development.in
pyjwkest==1.3.2 # via -r requirements/edx/testing.txt, edx-drf-extensions
pyjwt==1.5.2 # via -r requirements/edx/testing.txt, djangorestframework-jwt, edx-oauth2-provider, edx-rest-api-client, social-auth-core
pylint-celery==0.3 # via -r requirements/edx/testing.txt, edx-lint
pylint-django==0.7.2 # via -r requirements/edx/testing.txt, edx-lint
pylint-django==2.0.11 # via -r requirements/edx/testing.txt, edx-lint
pylint-plugin-utils==0.6 # via -r requirements/edx/testing.txt, pylint-celery, pylint-django
pylint==1.7.6 # via -r requirements/edx/testing.txt, edx-lint, pylint-celery, pylint-django, pylint-plugin-utils
pylint==2.4.2 # via -r requirements/edx/testing.txt, edx-lint, pylint-celery, pylint-django, pylint-plugin-utils
pymongo==3.9.0 # via -r requirements/edx/testing.txt, edx-opaque-keys, event-tracking, mongodbproxy, mongoengine
pynliner==0.8.0 # via -r requirements/edx/testing.txt
pyparsing==2.2.0 # via -r requirements/edx/testing.txt, calc, chem, packaging, pycontracts
@@ -282,7 +282,7 @@ shapely==1.7.0 # via -r requirements/edx/testing.txt
shortuuid==0.5.0 # via -r requirements/edx/testing.txt, edx-django-oauth2-provider
simplejson==3.17.0 # via -r requirements/edx/testing.txt, sailthru-client, super-csv, xblock-utils
singledispatch==3.4.0.3 # via -r requirements/edx/testing.txt
six==1.14.0 # via -r requirements/edx/pip-tools.txt, -r requirements/edx/testing.txt, analytics-python, astroid, bleach, bok-choy, calc, cryptography, diff-cover, django-appconf, django-classy-tags, django-countries, django-pyfs, django-sekizai, django-simple-history, django-statici18n, drf-yasg, edx-ace, edx-ccx-keys, edx-django-release-util, edx-drf-extensions, edx-enterprise, edx-i18n-tools, edx-lint, edx-milestones, edx-opaque-keys, edx-rbac, edx-search, edx-sphinx-theme, event-tracking, freezegun, fs, fs-s3fs, help-tokens, html5lib, httpretty, isodate, jsonschema, libsass, mando, mock, nltk, packaging, pathlib2, paver, pip-tools, pycontracts, pyjwkest, pylint, pytest-xdist, python-dateutil, python-memcached, python-swiftclient, singledispatch, social-auth-app-django, social-auth-core, sphinxcontrib-httpdomain, stevedore, tox, transifex-client, virtualenv, xblock
six==1.14.0 # via -r requirements/edx/pip-tools.txt, -r requirements/edx/testing.txt, analytics-python, astroid, bleach, bok-choy, calc, cryptography, diff-cover, django-appconf, django-classy-tags, django-countries, django-pyfs, django-sekizai, django-simple-history, django-statici18n, drf-yasg, edx-ace, edx-ccx-keys, edx-django-release-util, edx-drf-extensions, edx-enterprise, edx-i18n-tools, edx-lint, edx-milestones, edx-opaque-keys, edx-rbac, edx-search, edx-sphinx-theme, event-tracking, freezegun, fs, fs-s3fs, help-tokens, html5lib, httpretty, isodate, jsonschema, libsass, mando, mock, nltk, packaging, pathlib2, paver, pip-tools, pycontracts, pyjwkest, pytest-xdist, python-dateutil, python-memcached, python-swiftclient, singledispatch, social-auth-app-django, social-auth-core, sphinxcontrib-httpdomain, stevedore, tox, transifex-client, virtualenv, xblock
slumber==0.7.1 # via -r requirements/edx/testing.txt, edx-bulk-grades, edx-enterprise, edx-rest-api-client
snowballstemmer==2.0.0 # via sphinx
social-auth-core==3.2.0 # via -r requirements/edx/testing.txt, social-auth-app-django
@@ -309,6 +309,7 @@ toml==0.10.0 # via -r requirements/edx/testing.txt, tox
tox-battery==0.5.2 # via -r requirements/edx/testing.txt
tox==3.14.5 # via -r requirements/edx/testing.txt, tox-battery
transifex-client==0.13.4 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt
typed-ast==1.4.1 # via -r requirements/edx/testing.txt, astroid
unicodecsv==0.14.1 # via -r requirements/edx/testing.txt, edx-enterprise
unidiff==0.5.5 # via -r requirements/edx/testing.txt, coverage-pytest-plugin
uritemplate==3.0.1 # via -r requirements/edx/testing.txt, coreapi, drf-yasg
@@ -322,7 +323,7 @@ wcwidth==0.1.8 # via -r requirements/edx/testing.txt, pytest
web-fragments==0.3.1 # via -r requirements/edx/testing.txt, staff-graded-xblock, xblock, xblock-utils
webencodings==0.5.1 # via -r requirements/edx/testing.txt, bleach, html5lib
webob==1.8.6 # via -r requirements/edx/testing.txt, xblock, xmodule
wrapt==1.12.0 # via -r requirements/edx/testing.txt, astroid
wrapt==1.11.2 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, astroid
git+https://github.com/edx-solutions/xblock-drag-and-drop-v2@v2.2.8#egg=xblock-drag-and-drop-v2==2.2.8 # via -r requirements/edx/testing.txt
git+https://github.com/open-craft/xblock-poll@3c7dcaf6c933d914188f0740a60711603f948d26#egg=xblock-poll==1.9.1 # via -r requirements/edx/testing.txt
xblock-utils==1.2.4 # via -r requirements/edx/testing.txt, edx-sga, lti-consumer-xblock, staff-graded-xblock, xblock-drag-and-drop-v2, xblock-google-drive

2
requirements/edx/paver.txt
View File

@@ -26,5 +26,5 @@ six==1.14.0 # via edx-opaque-keys, libsass, mock, paver, python-me
stevedore==1.32.0 # via -r requirements/edx/paver.in, edx-opaque-keys
urllib3==1.25.8 # via requests
watchdog==0.10.2 # via -r requirements/edx/paver.in
wrapt==1.12.0 # via -r requirements/edx/paver.in
wrapt==1.11.2 # via -c requirements/edx/../constraints.txt, -r requirements/edx/paver.in
zipp==1.0.0 # via -c requirements/edx/../constraints.txt, importlib-metadata

2
requirements/edx/testing.in
View File

@@ -23,7 +23,7 @@ code-annotations # Perform code annotation checking, such as for PII an
cssselect # Used to extract HTML fragments via CSS selectors in 2 test cases and pyquery
ddt # Run a test case multiple times with different input; used in many, many of our tests
edx-i18n-tools>=0.4.6 # Commands for developers and translators to extract, compile and validate translations
edx-lint==1.3.0 # pylint extensions for Open edX repositories
edx-lint # pylint extensions for Open edX repositories
factory-boy # Library for creating test fixtures, used in many tests
# Pinning the freezegun version because 0.3.13 is causing failures which have also been reported on the git repo by public.
freezegun # Allows tests to mock the output of assorted datetime module functions

13
requirements/edx/testing.txt
View File

@@ -27,7 +27,7 @@ aniso8601==8.0.0 # via -r requirements/edx/base.txt, edx-tincan-py35
anyjson==0.3.3 # via -r requirements/edx/base.txt, kombu
apipkg==1.5 # via execnet
appdirs==1.4.3 # via -r requirements/edx/base.txt, fs, virtualenv
astroid==1.5.3 # via pylint, pylint-celery
astroid==2.3.3 # via pylint, pylint-celery
attrs==19.3.0 # via -r requirements/edx/base.txt, edx-ace, pytest
babel==2.8.0 # via -r requirements/edx/base.txt, django-babel, django-babel-underscore
beautifulsoup4==4.8.2 # via -r requirements/edx/base.txt, -r requirements/edx/testing.in, pynliner
@@ -114,7 +114,7 @@ edx-django-utils==3.0 # via -r requirements/edx/base.txt, django-config-mode
edx-drf-extensions==2.4.6 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, edx-completion, edx-enterprise, edx-organizations, edx-proctoring, edx-rbac, edx-when, edxval
edx-enterprise==2.4.1 # via -r requirements/edx/base.txt
edx-i18n-tools==0.5.0 # via -r requirements/edx/base.txt, -r requirements/edx/testing.in, ora2
edx-lint==1.3.0 # via -r requirements/edx/testing.in
edx-lint==1.4.1 # via -r requirements/edx/testing.in
edx-milestones==0.2.6 # via -r requirements/edx/base.txt
edx-oauth2-provider==1.3.1 # via -r requirements/edx/base.txt
edx-opaque-keys[django]==2.0.1 # via -r requirements/edx/base.txt, edx-bulk-grades, edx-ccx-keys, edx-completion, edx-drf-extensions, edx-enterprise, edx-milestones, edx-organizations, edx-proctoring, edx-user-state-client, edx-when, xmodule
@@ -223,9 +223,9 @@ pygments==2.5.2 # via -r requirements/edx/base.txt, -r requirements/ed
pyjwkest==1.3.2 # via -r requirements/edx/base.txt, edx-drf-extensions
pyjwt==1.5.2 # via -r requirements/edx/base.txt, djangorestframework-jwt, edx-oauth2-provider, edx-rest-api-client, social-auth-core
pylint-celery==0.3 # via edx-lint
pylint-django==0.7.2 # via edx-lint
pylint-django==2.0.11 # via edx-lint
pylint-plugin-utils==0.6 # via pylint-celery, pylint-django
pylint==1.7.6 # via edx-lint, pylint-celery, pylint-django, pylint-plugin-utils
pylint==2.4.2 # via edx-lint, pylint-celery, pylint-django, pylint-plugin-utils
pymongo==3.9.0 # via -r requirements/edx/base.txt, edx-opaque-keys, event-tracking, mongodbproxy, mongoengine
pynliner==0.8.0 # via -r requirements/edx/base.txt
pyparsing==2.2.0 # via -r requirements/edx/base.txt, calc, chem, packaging, pycontracts
@@ -270,7 +270,7 @@ shapely==1.7.0 # via -r requirements/edx/base.txt
shortuuid==0.5.0 # via -r requirements/edx/base.txt, edx-django-oauth2-provider
simplejson==3.17.0 # via -r requirements/edx/base.txt, sailthru-client, super-csv, xblock-utils
singledispatch==3.4.0.3 # via -r requirements/edx/testing.in
six==1.14.0 # via -r requirements/edx/base.txt, -r requirements/edx/coverage.txt, analytics-python, astroid, bleach, bok-choy, calc, cryptography, diff-cover, django-appconf, django-classy-tags, django-countries, django-pyfs, django-sekizai, django-simple-history, django-statici18n, drf-yasg, edx-ace, edx-ccx-keys, edx-django-release-util, edx-drf-extensions, edx-enterprise, edx-i18n-tools, edx-lint, edx-milestones, edx-opaque-keys, edx-rbac, edx-search, event-tracking, freezegun, fs, fs-s3fs, help-tokens, html5lib, httpretty, isodate, libsass, mando, mock, nltk, packaging, pathlib2, paver, pycontracts, pyjwkest, pylint, pytest-xdist, python-dateutil, python-memcached, python-swiftclient, singledispatch, social-auth-app-django, social-auth-core, stevedore, tox, transifex-client, virtualenv, xblock
six==1.14.0 # via -r requirements/edx/base.txt, -r requirements/edx/coverage.txt, analytics-python, astroid, bleach, bok-choy, calc, cryptography, diff-cover, django-appconf, django-classy-tags, django-countries, django-pyfs, django-sekizai, django-simple-history, django-statici18n, drf-yasg, edx-ace, edx-ccx-keys, edx-django-release-util, edx-drf-extensions, edx-enterprise, edx-i18n-tools, edx-lint, edx-milestones, edx-opaque-keys, edx-rbac, edx-search, event-tracking, freezegun, fs, fs-s3fs, help-tokens, html5lib, httpretty, isodate, libsass, mando, mock, nltk, packaging, pathlib2, paver, pycontracts, pyjwkest, pytest-xdist, python-dateutil, python-memcached, python-swiftclient, singledispatch, social-auth-app-django, social-auth-core, stevedore, tox, transifex-client, virtualenv, xblock
slumber==0.7.1 # via -r requirements/edx/base.txt, edx-bulk-grades, edx-enterprise, edx-rest-api-client
social-auth-core==3.2.0 # via -r requirements/edx/base.txt, social-auth-app-django
git+https://github.com/jazzband/sorl-thumbnail.git@13bedfb7d2970809eda597e3ef79318a6fa80ac2#egg=sorl-thumbnail # via -r requirements/edx/base.txt
@@ -287,6 +287,7 @@ toml==0.10.0 # via tox
tox-battery==0.5.2 # via -r requirements/edx/testing.in
tox==3.14.5 # via -r requirements/edx/testing.in, tox-battery
transifex-client==0.13.4 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.in
typed-ast==1.4.1 # via astroid
unicodecsv==0.14.1 # via -r requirements/edx/base.txt, edx-enterprise
unidiff==0.5.5 # via -r requirements/edx/testing.in, coverage-pytest-plugin
uritemplate==3.0.1 # via -r requirements/edx/base.txt, coreapi, drf-yasg
@@ -299,7 +300,7 @@ wcwidth==0.1.8 # via pytest
web-fragments==0.3.1 # via -r requirements/edx/base.txt, staff-graded-xblock, xblock, xblock-utils
webencodings==0.5.1 # via -r requirements/edx/base.txt, bleach, html5lib
webob==1.8.6 # via -r requirements/edx/base.txt, xblock, xmodule
wrapt==1.12.0 # via -r requirements/edx/base.txt, astroid
wrapt==1.11.2 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, astroid
git+https://github.com/edx-solutions/xblock-drag-and-drop-v2@v2.2.8#egg=xblock-drag-and-drop-v2==2.2.8 # via -r requirements/edx/base.txt
git+https://github.com/open-craft/xblock-poll@3c7dcaf6c933d914188f0740a60711603f948d26#egg=xblock-poll==1.9.1 # via -r requirements/edx/base.txt
xblock-utils==1.2.4 # via -r requirements/edx/base.txt, edx-sga, lti-consumer-xblock, staff-graded-xblock, xblock-drag-and-drop-v2, xblock-google-drive

3
scripts/Jenkinsfiles/quality
View File

@@ -74,7 +74,8 @@ pipeline {
stage('Run Tests') {
parallel {
stage("commonlib pylint") {
agent { label "jenkins-worker" }
// "pylint common" requires 5.5 GB of RAM, so use js-worker (8 GB) instead of jenkins-worker (4 GB)
agent { label "js-worker" }
environment {
TEST_SUITE = "quality"
SHARD = 1

2
scripts/thresholds.sh
View File

@@ -2,6 +2,6 @@
set -e
export LOWER_PYLINT_THRESHOLD=1000
export UPPER_PYLINT_THRESHOLD=1962
export UPPER_PYLINT_THRESHOLD=4050
export ESLINT_THRESHOLD=5530
export STYLELINT_THRESHOLD=880