From 0cb7539999dbe6804aa2d8cbd366723dc7e0be11 Mon Sep 17 00:00:00 2001 From: Jeremy Bowman Date: Fri, 27 Dec 2019 11:00:46 -0500 Subject: [PATCH] Python dependencies cleanup (#22612) --- pylintrc | 32 +++++++++---------- pylintrc_tweaks | 1 - requirements/constraints.txt | 49 +++++++++++++++-------------- requirements/edx-sandbox/base.txt | 6 +++- requirements/edx-sandbox/py35.in | 1 + requirements/edx-sandbox/py35.txt | 2 +- requirements/edx-sandbox/shared.in | 1 - requirements/edx-sandbox/shared.txt | 2 -- requirements/edx/base.in | 1 - requirements/edx/base.txt | 4 +-- requirements/edx/coverage.txt | 2 +- requirements/edx/development.in | 4 +-- requirements/edx/development.txt | 14 +++------ requirements/edx/testing.in | 1 - requirements/edx/testing.txt | 8 +---- 15 files changed, 55 insertions(+), 73 deletions(-) diff --git a/pylintrc b/pylintrc index 6111309db6..f9643b0ed9 100644 --- a/pylintrc +++ b/pylintrc @@ -66,15 +66,14 @@ [MASTER] ignore = ,.git,.tox,migrations,node_modules,.pycharm_helpers persistent = yes -load-plugins = edx_lint.pylint,pylint_django,pylint_celery,caniusepython3.pylint_checker +load-plugins = edx_lint.pylint,pylint_django,pylint_celery init-hook = "import sys; sys.path.extend(['lms/djangoapps', 'cms/djangoapps', 'common/djangoapps'])" [MESSAGES CONTROL] enable = - # These are controlled by explicit choices in the pylintrc files blacklisted-name, line-too-long, - # These affect the correctness of the code + syntax-error, init-is-generator, return-in-init, @@ -197,7 +196,7 @@ enable = anomalous-unicode-escape-in-string, bad-open-mode, boolean-datetime, - # Checking failed for some reason + fatal, astroid-error, parse-error, @@ -205,21 +204,20 @@ enable = django-not-available, raw-checker-failed, django-not-available-placeholder, - # Documentation is important + empty-docstring, invalid-characters-in-docstring, missing-docstring, wrong-spelling-in-comment, wrong-spelling-in-docstring, - # Unused code should be deleted + unused-import, unused-variable, unused-argument, - # These are dangerous! + exec-used, eval-used, - # These represent idiomatic python. Not adhering to them - # will raise red flags with future readers. + bad-classmethod-argument, bad-mcs-classmethod-argument, bad-mcs-method-argument, @@ -254,33 +252,31 @@ enable = model-has-unicode, model-no-explicit-unicode, protected-access, - # Don't use things that are deprecated + deprecated-module, deprecated-method, - # These help manage code complexity + too-many-nested-blocks, too-many-statements, too-many-boolean-expressions, - # Consistent import order makes finding where code is - # imported from easier + wrong-import-order, wrong-import-position, wildcard-import, - # These should be auto-fixed by any competent editor + missing-final-newline, mixed-line-endings, trailing-newlines, trailing-whitespace, unexpected-line-ending-format, mixed-indentation, - # These attempt to limit pylint line-noise + bad-option-value, unrecognized-inline-option, useless-suppression, bad-inline-option, deprecated-pragma, disable = - # These should be left to the discretion of the reviewer bad-continuation, invalid-name, misplaced-comparison-constant, @@ -290,7 +286,7 @@ disable = unused-wildcard-import, global-statement, no-else-return, - # These are disabled by pylint by default + apply-builtin, backtick, basestring-builtin, @@ -459,4 +455,4 @@ int-import-graph = [EXCEPTIONS] overgeneral-exceptions = Exception -# c9c6a6da98d5eb4ede060038e6b0f1892f32f83b +# 40eae2f3b3491a12ea2f2c98c632d7cb43904d56 diff --git a/pylintrc_tweaks b/pylintrc_tweaks index ad6b502096..00799bdfe5 100644 --- a/pylintrc_tweaks +++ b/pylintrc_tweaks @@ -2,7 +2,6 @@ [MASTER] ignore+ = ,.git,.tox,migrations,node_modules,.pycharm_helpers init-hook="import sys; sys.path.extend(['lms/djangoapps', 'cms/djangoapps', 'common/djangoapps'])" -load-plugins+=,caniusepython3.pylint_checker [MESSAGES CONTROL] # Disable unicode-format-string until we can agree to turn it on. diff --git a/requirements/constraints.txt b/requirements/constraints.txt index d0cd60c25e..375ed47084 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -8,9 +8,32 @@ # pin when possible. Writing an issue against the offending project and # linking to it here is good. +# Constraining this since the newer versions no longer work with the deprecated MIDDLEWARE_CLASSES setting +django-debug-toolbar<2.0 + # Version 4.0.0 dropped support for Django < 2.0.1 django-model-utils<4.0.0 +# 1.16.1 requires djangorestframework>=3.8 +drf-yasg==1.16 + +# 1.2.3 breaks unittest in +# lms.djangoapps.course_api.tests.test_views.CourseListSearchViewTest.test_list_all_with_search_term +# acceptance.tests.lms.test_lms_course_discovery.CourseDiscoveryTest.test_search +edx-search==1.2.2 + +# 4.0.0 dropped support for Python 3.5 +inflect<4.0.0 + +# Pinned by moto to this but since moto is only a test requirement, we don't +# see the constraint when generating base.txt +# +# There are incompatible versions in the resolved dependencies: +# jsondiff (from edx-enterprise==1.8.6->-r requirements/edx/base.txt (line 108)) +# jsondiff==1.1.1 (from moto==1.2.0->-r requirements/edx/testing.in (line 33)) +# jsondiff==1.2.0 (from -r requirements/edx/base.txt (line 146)) +jsondiff==1.1.1 + # Convert text markup to HTML; used in capa problems, forums, and course wikis; pin Markdown version as tests failed for its upgrade to the latest release Markdown==2.6.11 @@ -20,10 +43,6 @@ matplotlib<3.1 # mysqlclient 1.5 is scheduled to change internal APIs used by Django 1.11 mysqlclient<1.5 -# transifex-client 0.13.5 and 0.13.6 pin six and urllib3 to old versions needlessly -# https://github.com/transifex/transifex-client/issues/252 -transifex-client==0.13.4 - # Bumping requests-oauthlib to 1.2 updates oauthlib to 3.0.0, which changes a response code in certain cases # This causes a test to fail. Before fixing, we would need to make sure mobile apps are functional with this change # See https://github.com/oauthlib/oauthlib/issues/264 for more context @@ -37,25 +56,9 @@ python3-saml==1.5.0 # xmodule.tests.test_resource_templates.ResourceTemplatesTests See TE-2391 for details. pytest<4.6.0 -# 1.2.3 breaks unittest in -# lms.djangoapps.course_api.tests.test_views.CourseListSearchViewTest.test_list_all_with_search_term -# acceptance.tests.lms.test_lms_course_discovery.CourseDiscoveryTest.test_search -edx-search==1.2.2 - -# 1.16.1 requires djangorestframework>=3.8 -drf-yasg==1.16 - # 2.0.0 is a dummy package, because faulthandler has been incorporated into pytest 5.0 pytest-faulthandler<2.0.0 -# Pinned by moto to this but since moto is only a test requirement, we don't -# see the constraint when generating base.txt -# -# There are incompatible versions in the resolved dependencies: -# jsondiff (from edx-enterprise==1.8.6->-r requirements/edx/base.txt (line 108)) -# jsondiff==1.1.1 (from moto==1.2.0->-r requirements/edx/testing.in (line 33)) -# jsondiff==1.2.0 (from -r requirements/edx/base.txt (line 146)) -jsondiff==1.1.1 - -# Constraining this since the newer versions no longer work with the deprecated MIDDLEWARE_CLASSES setting -django-debug-toolbar<2.0 +# transifex-client 0.13.5 and 0.13.6 pin six and urllib3 to old versions needlessly +# https://github.com/transifex/transifex-client/issues/252 +transifex-client==0.13.4 diff --git a/requirements/edx-sandbox/base.txt b/requirements/edx-sandbox/base.txt index e9a3a1e25a..f3ebfe4835 100644 --- a/requirements/edx-sandbox/base.txt +++ b/requirements/edx-sandbox/base.txt @@ -1,7 +1,11 @@ # # This file has been frozen for the benefit of Python 2.7 sandboxes, and # "make upgrade" no longer updates it. It can be removed once that -# version of the sandbox is no longer run in production. +# version of the sandbox is no longer run in production. Before removal, +# the configuration repository needs to be updated to use the new file: +# https://github.com/edx/configuration/blob/master/playbooks/roles/edxapp/defaults/main.yml#L1628 +# Also, this change needs to be announced and communicated to partners +# before implementation. # common/lib/sandbox-packages common/lib/symmath diff --git a/requirements/edx-sandbox/py35.in b/requirements/edx-sandbox/py35.in index 24c7e94717..045d546aec 100644 --- a/requirements/edx-sandbox/py35.in +++ b/requirements/edx-sandbox/py35.in @@ -11,6 +11,7 @@ -r shared.txt # Dependencies in common with LMS and Studio matplotlib==2.2.4 # 2D plotting library +networkx==2.2 # Utilities for creating, manipulating, and studying network graphs numpy==1.16.5 # Numeric array processing utilities; used by scipy pyparsing==2.2.0 # Python Parsing module random2 # Implementation of random module that works identically under Python 2 and 3 diff --git a/requirements/edx-sandbox/py35.txt b/requirements/edx-sandbox/py35.txt index 1f607d1cf8..d8efd5effd 100644 --- a/requirements/edx-sandbox/py35.txt +++ b/requirements/edx-sandbox/py35.txt @@ -11,7 +11,7 @@ cffi==1.13.2 git+https://github.com/edx/openedx-chem.git@ff4e3a03d3c7610e47a9af08eb648d8aabe2eb18#egg=chem==1.0.0 cryptography==2.8 cycler==0.10.0 # via matplotlib -decorator==4.4.1 +decorator==4.4.1 # via networkx kiwisolver==1.1.0 # via matplotlib lxml==4.4.1 markupsafe==1.1.1 diff --git a/requirements/edx-sandbox/shared.in b/requirements/edx-sandbox/shared.in index b18a669a0d..9b60704afb 100644 --- a/requirements/edx-sandbox/shared.in +++ b/requirements/edx-sandbox/shared.in @@ -11,5 +11,4 @@ cryptography # Implementations of assorted cryptography algorithms lxml==4.4.1 # XML parser -networkx==2.2 # Utilities for creating, manipulating, and studying network graphs nltk # Natural language processing; used by the chem package diff --git a/requirements/edx-sandbox/shared.txt b/requirements/edx-sandbox/shared.txt index 7001854da2..df2e18088c 100644 --- a/requirements/edx-sandbox/shared.txt +++ b/requirements/edx-sandbox/shared.txt @@ -6,9 +6,7 @@ # cffi==1.13.2 # via cryptography cryptography==2.8 -decorator==4.4.1 # via networkx lxml==4.4.1 -networkx==2.2 nltk==3.4.5 pycparser==2.19 # via cffi six==1.13.0 # via cryptography, nltk diff --git a/requirements/edx/base.in b/requirements/edx/base.in index 460d5aa566..396347817f 100644 --- a/requirements/edx/base.in +++ b/requirements/edx/base.in @@ -118,7 +118,6 @@ PyContracts==1.7.1 pycountry pycryptodomex pygments # Used to support colors in paver command output -pygraphviz # No longer in use? Optional dependency of networkx, from edx-sandbox/shared.in pyjwkest==1.3.2 # TODO Replace PyJWT usage with pyjwkest PyJWT==1.5.2 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 18013d5113..e51646c7cb 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -49,7 +49,7 @@ git+https://github.com/edx/crowdsourcehinter.git@a7ffc85b134b7d8909bf1fefd23dbdb cryptography==2.8 cssutils==1.0.2 # via pynliner ddt==1.2.2 -decorator==4.4.1 +decorator==4.4.1 # via pycontracts defusedxml==0.5.0 django-appconf==1.0.3 # via django-statici18n django-babel-underscore==0.5.2 @@ -161,7 +161,6 @@ git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752 mongoengine==0.10.0 mpmath==1.1.0 # via sympy mysqlclient==1.4.6 -networkx==2.2 newrelic==5.4.1.134 nltk==3.4.5 nodeenv==1.3.3 @@ -186,7 +185,6 @@ pycparser==2.19 pycryptodome==3.9.4 # via pdfminer.six pycryptodomex==3.9.4 pygments==2.5.2 -pygraphviz==1.5 pyjwkest==1.3.2 pyjwt==1.5.2 pymongo==3.9.0 diff --git a/requirements/edx/coverage.txt b/requirements/edx/coverage.txt index ccd7e9ee07..02cfed5b68 100644 --- a/requirements/edx/coverage.txt +++ b/requirements/edx/coverage.txt @@ -6,7 +6,7 @@ # coverage==5.0.1 diff-cover==2.5.0 -importlib-metadata==1.3.0 # via inflect, pluggy +importlib-metadata==1.3.0 # via pluggy inflect==3.0.2 # via jinja2-pluralize jinja2-pluralize==0.3.0 # via diff-cover jinja2==2.10.3 # via diff-cover, jinja2-pluralize diff --git a/requirements/edx/development.in b/requirements/edx/development.in index f95dd2fdba..6425d57a48 100644 --- a/requirements/edx/development.in +++ b/requirements/edx/development.in @@ -17,7 +17,5 @@ click # Used for perf_tests utilities in modulesto django-debug-toolbar # A set of panels that display debug information about the current request/response edx-sphinx-theme # Documentation theme pyinotify # More efficient checking for runserver reload trigger events +sphinxcontrib-openapi[markdown] # OpenAPI (fka Swagger) spec renderer for Sphinx vulture # Detects possible dead/unused code, used in scripts/find-dead-code.sh - -# Using a github hash because the "include" feature wasn't in the latest release (0.5.0): -git+https://github.com/sphinx-contrib/openapi.git@9306435601ae05138edea54419dd83ce9e729ad8#egg=sphinxcontrib-openapi[markdown]==0.0 diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 6235f4db0a..1a55f32fd8 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -27,13 +27,11 @@ anyjson==0.3.3 apipkg==1.5 appdirs==1.4.3 argh==0.26.2 -argparse==1.4.0 astroid==1.5.3 atomicwrites==1.3.0 attrs==19.3.0 aws-xray-sdk==0.95 babel==1.3 -backports.functools-lru-cache==1.6.1 beautifulsoup4==4.8.2 billiard==3.3.0.23 bleach==2.1.4 @@ -43,7 +41,6 @@ boto==2.39.0 botocore==1.8.17 git+https://github.com/edx/bridgekeeper.git@4e34894e4ac5d0467ed1901811a81fd87ee01937#egg=bridgekeeper==0.0 git+https://github.com/edx/openedx-calc.git@e9b698c85ad1152002bc0868f475f153dce88952#egg=calc==0.4 -caniusepython3==7.2.0 celery==3.1.25 certifi==2019.11.28 cffi==1.13.2 @@ -67,7 +64,6 @@ ddt==1.2.2 decorator==4.4.1 defusedxml==0.5.0 diff-cover==2.5.0 -distlib==0.3.0 django-appconf==1.0.3 django-babel-underscore==0.5.2 django-babel==0.6.2 @@ -178,7 +174,7 @@ jmespath==0.9.4 jsondiff==1.1.1 jsonfield==2.0.2 jsonpickle==1.2 -jsonschema==3.2.0 +jsonschema==3.2.0 # via sphinxcontrib-openapi kombu==3.0.37 laboratory==1.0.2 lazy-object-proxy==1.4.3 @@ -188,7 +184,7 @@ libsass==0.10.0 loremipsum==1.0.5 git+https://github.com/edx/xblock-lti-consumer.git@v1.2.3#egg=lti_consumer-xblock==1.2.3 lxml==4.4.1 -m2r==0.2.1 +m2r==0.2.1 # via sphinxcontrib-openapi mailsnake==1.6.4 mako==1.0.2 mando==0.6.4 @@ -205,7 +201,6 @@ more-itertools==8.0.2 moto==1.3.1 mpmath==1.1.0 mysqlclient==1.4.6 -networkx==2.2 newrelic==5.4.1.134 nltk==3.4.5 nodeenv==1.3.3 @@ -238,7 +233,6 @@ pycryptodome==3.9.4 pycryptodomex==3.9.4 pyflakes==2.1.1 pygments==2.5.2 -pygraphviz==1.5 pyinotify==0.9.6 pyjwkest==1.3.2 pyjwt==1.5.2 @@ -302,9 +296,9 @@ sphinx==2.3.1 # via edx-sphinx-theme, sphinxcontrib-httpdomain sphinxcontrib-applehelp==1.0.1 # via sphinx sphinxcontrib-devhelp==1.0.1 # via sphinx sphinxcontrib-htmlhelp==1.0.2 # via sphinx -sphinxcontrib-httpdomain==1.7.0 +sphinxcontrib-httpdomain==1.7.0 # via sphinxcontrib-openapi sphinxcontrib-jsmath==1.0.1 # via sphinx -git+https://github.com/sphinx-contrib/openapi.git@9306435601ae05138edea54419dd83ce9e729ad8#egg=sphinxcontrib-openapi[markdown]==0.0 +sphinxcontrib-openapi[markdown]==0.6.0 sphinxcontrib-qthelp==1.0.2 # via sphinx sphinxcontrib-serializinghtml==1.1.3 # via sphinx sqlparse==0.3.0 diff --git a/requirements/edx/testing.in b/requirements/edx/testing.in index c7ad81d7ff..eaadfc0b3f 100644 --- a/requirements/edx/testing.in +++ b/requirements/edx/testing.in @@ -19,7 +19,6 @@ beautifulsoup4 # Library for extracting data from HTML and XML files bok-choy # Framework for browser automation tests, based on selenium -caniusepython3 # Library for checking the ability to upgrade to python3 code-annotations # Perform code annotation checking, such as for PII annotations cssselect # Used to extract HTML fragments via CSS selectors in 2 test cases and pyquery ddt # Run a test case multiple times with different input; used in many, many of our tests diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index e578160a82..a7678c1c26 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -26,13 +26,11 @@ anyjson==0.3.3 apipkg==1.5 # via execnet appdirs==1.4.3 argh==0.26.2 -argparse==1.4.0 # via caniusepython3 astroid==1.5.3 # via pylint, pylint-celery atomicwrites==1.3.0 # via pytest attrs==19.3.0 aws-xray-sdk==0.95 # via moto babel==1.3 -backports.functools-lru-cache==1.6.1 # via caniusepython3 beautifulsoup4==4.8.2 billiard==3.3.0.23 bleach==2.1.4 @@ -42,7 +40,6 @@ boto==2.39.0 botocore==1.8.17 git+https://github.com/edx/bridgekeeper.git@4e34894e4ac5d0467ed1901811a81fd87ee01937#egg=bridgekeeper==0.0 git+https://github.com/edx/openedx-calc.git@e9b698c85ad1152002bc0868f475f153dce88952#egg=calc==0.4 -caniusepython3==7.2.0 celery==3.1.25 certifi==2019.11.28 cffi==1.13.2 @@ -66,7 +63,6 @@ ddt==1.2.2 decorator==4.4.1 defusedxml==0.5.0 diff-cover==2.5.0 -distlib==0.3.0 # via caniusepython3 django-appconf==1.0.3 django-babel-underscore==0.5.2 django-babel==0.6.2 @@ -197,7 +193,6 @@ more-itertools==8.0.2 moto==1.3.1 mpmath==1.1.0 mysqlclient==1.4.6 -networkx==2.2 newrelic==5.4.1.134 nltk==3.4.5 nodeenv==1.3.3 @@ -205,7 +200,7 @@ numpy==1.18.0 git+https://github.com/joestump/python-oauth2.git@b94f69b1ad195513547924e380d9265133e995fa#egg=oauth2 oauthlib==2.1.0 git+https://github.com/edx/edx-ora2.git@2.5.3#egg=ora2==2.5.3 -packaging==19.2 # via caniusepython3, tox +packaging==19.2 # via tox path.py==8.2.1 pathlib2==2.3.5 # via pytest pathtools==0.1.2 @@ -229,7 +224,6 @@ pycryptodome==3.9.4 pycryptodomex==3.9.4 pyflakes==2.1.1 # via flake8 pygments==2.5.2 -pygraphviz==1.5 pyjwkest==1.3.2 pyjwt==1.5.2 pylint-celery==0.3 # via edx-lint