From b2046f6674e4d15ee632cf21fbe61eb73c822817 Mon Sep 17 00:00:00 2001 From: Jeremy Bowman Date: Thu, 26 Dec 2019 15:13:02 -0500 Subject: [PATCH] Switch make upgrade to Python 3.5 (#22609) * Switch make upgrade to Python 3.5 * bok-choy fixes, constrain django-debug-toolbar * Remove flaky bok-choy test --- Makefile | 2 +- .../tests/lms/test_account_settings.py | 10 ++-- .../tests/studio/test_studio_settings.py | 6 +-- .../tests/video/test_studio_video_editor.py | 2 +- .../video/test_studio_video_transcript.py | 48 ----------------- requirements/constraints.txt | 51 +++--------------- requirements/edx-sandbox/base.txt | 7 ++- requirements/edx-sandbox/py35.in | 26 ++++++++++ requirements/edx-sandbox/py35.txt | 33 ++++++++++++ requirements/edx-sandbox/shared.in | 4 +- requirements/edx-sandbox/shared.txt | 10 ++-- requirements/edx/base.in | 1 - requirements/edx/base.txt | 29 +++++------ requirements/edx/coverage.txt | 8 +-- requirements/edx/development.in | 2 - requirements/edx/development.txt | 52 ++++++++----------- requirements/edx/paver.txt | 1 - requirements/edx/testing.in | 1 - requirements/edx/testing.txt | 46 ++++++---------- scripts/post-pip-compile.sh | 2 +- scripts/upgrade_pysqlite.sh | 21 -------- tox.ini | 5 +- 22 files changed, 142 insertions(+), 225 deletions(-) create mode 100644 requirements/edx-sandbox/py35.in create mode 100644 requirements/edx-sandbox/py35.txt delete mode 100755 scripts/upgrade_pysqlite.sh diff --git a/Makefile b/Makefile index c3be7025ce..7e6bb596e4 100644 --- a/Makefile +++ b/Makefile @@ -74,7 +74,7 @@ REQ_FILES = \ requirements/edx/coverage \ requirements/edx/paver \ requirements/edx-sandbox/shared \ - requirements/edx-sandbox/base \ + requirements/edx-sandbox/py35 \ requirements/edx/base \ requirements/edx/testing \ requirements/edx/development \ diff --git a/common/test/acceptance/tests/lms/test_account_settings.py b/common/test/acceptance/tests/lms/test_account_settings.py index a0a361e460..ac56e21b35 100644 --- a/common/test/acceptance/tests/lms/test_account_settings.py +++ b/common/test/acceptance/tests/lms/test_account_settings.py @@ -410,15 +410,17 @@ class AccountSettingsPageTest(AccountSettingsTestMixin, AcceptanceTest): """ first_social_media_link = self.account_settings_page.get_social_first_element() - valid_value = six.u('https://www.twitter.com/edX') + valid_value = 'https://www.twitter.com/edX' if 'face' in first_social_media_link.lower(): - valid_value = six.u('https://www.facebook.com/edX') + valid_value = 'https://www.facebook.com/edX' + elif 'linked' in first_social_media_link.lower(): + valid_value = 'https://www.linkedin.com/in/edX' self._test_text_field( - six.u('social_links'), + 'social_links', first_social_media_link, self.social_link, - six.u('www.google.com/invalidlink)'), + 'www.google.com/invalidlink)', [valid_value, self.social_link], ) diff --git a/common/test/acceptance/tests/studio/test_studio_settings.py b/common/test/acceptance/tests/studio/test_studio_settings.py index 7c71ae66a5..49f632f4c8 100644 --- a/common/test/acceptance/tests/studio/test_studio_settings.py +++ b/common/test/acceptance/tests/studio/test_studio_settings.py @@ -468,9 +468,9 @@ class AdvancedSettingsValidationTest(StudioCourseTest): } json_input = json.dumps(inputs) self.advanced_settings.set('Discussion Topic Mapping', json_input) - self.assertEqual( - self.advanced_settings.get('Discussion Topic Mapping'), - '{\n "key": "value",\n "key_2": "value_2"\n}' + assert self.advanced_settings.get('Discussion Topic Mapping') in ( + '{\n "key": "value",\n "key_2": "value_2"\n}', + '{\n "key_2": "value_2",\n "key": "value"\n}' ) def test_automatic_quoting_of_non_json_value(self): diff --git a/common/test/acceptance/tests/video/test_studio_video_editor.py b/common/test/acceptance/tests/video/test_studio_video_editor.py index bad5862cd5..6012852554 100644 --- a/common/test/acceptance/tests/video/test_studio_video_editor.py +++ b/common/test/acceptance/tests/video/test_studio_video_editor.py @@ -110,7 +110,7 @@ class VideoEditorTest(CMSVideoBaseTest): self.save_unit_settings() self.assertTrue(self.video.is_captions_visible()) self.assertIn(unicode_text, self.video.captions_text) - self.assertEqual(list(self.video.caption_languages.keys()), ['zh', 'uk']) + self.assertEqual(set(self.video.caption_languages.keys()), {'zh', 'uk'}) def test_save_language_upload_no_transcript(self): """ diff --git a/common/test/acceptance/tests/video/test_studio_video_transcript.py b/common/test/acceptance/tests/video/test_studio_video_transcript.py index f119f65daf..7485ec49e0 100644 --- a/common/test/acceptance/tests/video/test_studio_video_transcript.py +++ b/common/test/acceptance/tests/video/test_studio_video_transcript.py @@ -419,54 +419,6 @@ class VideoTranscriptTest(CMSVideoBaseTest): self.video.set_url_field('uk_transcripts.webm', 3) self.assertEqual(self.video.message('status'), 'Timed Transcript Found') - def test_two_html5_sources_w_transcripts(self): - """ - Scenario: Enter 2 HTML5 sources with transcripts, they are not the same, choose - Given I have created a Video component and subtitles "t_not_exist" and "t__eq_exist" - are present in contentstore. - - And I enter a "t_not_exist.mp4" source to field number 1 - Then I see status message "Timed Transcript Found" - `download_to_edit` and `upload_new_timed_transcripts` buttons are shown - - And I enter a "t__eq_exist.webm" source to field number 2 - Then I see status message "Timed Transcript Conflict" - `Timed Transcript from t_not_exist.mp4` and `Timed Transcript from t__eq_exist.webm` buttons are shown - And I click transcript button "Timed Transcript from t_not_exist.mp4" - Then I see status message "Timed Transcript Found" And I save video component - Then I see that the captions are visible. - """ - # Setup a course, navigate to the unit containing - # video component and edit the video component. - self.assets.append('subs_t_not_exist.srt.sjson') - self.assets.append('subs_t__eq_exist.srt.sjson') - self.navigate_to_course_unit() - self.edit_component() - - self.video.set_url_field('t_not_exist.mp4', 1) - self.assertEqual(self.video.message('status'), 'Timed Transcript Found') - self.assertTrue(self.video.is_transcript_button_visible('download_to_edit')) - self.assertTrue(self.video.is_transcript_button_visible('upload_new_timed_transcripts')) - - self.video.set_url_field('t__eq_exist.webm', 2) - self.assertEqual(self.video.message('status'), 'Timed Transcript Conflict') - - self.assertTrue(self.video.is_transcript_button_visible( - 'choose', - button_text='Timed Transcript from t__eq_exist.webm' - )) - self.assertTrue(self.video.is_transcript_button_visible( - 'choose', - index=1, - button_text='Timed Transcript from t_not_exist.mp4' - )) - - self.video.click_button('choose', index=1) - self.assertEqual(self.video.message('status'), 'Timed Transcript Found') - self.save_unit_settings() - - self.assertTrue(self.video.is_captions_visible()) - def test_one_field_only(self): """ Scenario: Work with 1 field only: Enter HTML5 source with transcripts - save -> change it to another one diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 5cf048abe2..d0cd60c25e 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -8,22 +8,18 @@ # pin when possible. Writing an issue against the offending project and # linking to it here is good. -# These packages are backports which can only be installed on Python 2.7 -functools32 ; python_version == "2.7" -futures ; python_version == "2.7" -python-openid ; python_version == "2.7" -pysqlite ; python_version == "2.7" +# Version 4.0.0 dropped support for Django < 2.0.1 +django-model-utils<4.0.0 -# Convert text markup to HTML; used in capa problems, forums, and course wikis; pin Markdown version as tests failed for its upgrade to latest release +# Convert text markup to HTML; used in capa problems, forums, and course wikis; pin Markdown version as tests failed for its upgrade to the latest release Markdown==2.6.11 +# Matplotlib 3.1 requires Python 3.6 +matplotlib<3.1 + # mysqlclient 1.5 is scheduled to change internal APIs used by Django 1.11 mysqlclient<1.5 -# required by edx-lint and pinned explicitly here because newer versions don't guarantee python 2 support. -# Can be removed when we get to Python 3. -pylint-plugin-utils==0.3 - # transifex-client 0.13.5 and 0.13.6 pin six and urllib3 to old versions needlessly # https://github.com/transifex/transifex-client/issues/252 transifex-client==0.13.4 @@ -34,15 +30,9 @@ transifex-client==0.13.4 requests-oauthlib==1.1.0 oauthlib==2.1.0 -# pytest-randomly dropped support for Python 2 with 2.0.0, and only supports Python 3.4+ -pytest-randomly<2.0.0 - # python3-saml 1.6.0 breaks unittests in common/djangoapps/third_party_auth/tests/test_views.py::SAMLMetadataTest python3-saml==1.5.0 -# version 1.3.0 breaks make upgrade, as it requires py3.5 or above -scipy<=1.2.1 - # Unless we constrain pytest to <4.6.0, it may break all tests inside of # xmodule.tests.test_resource_templates.ResourceTemplatesTests See TE-2391 for details. pytest<4.6.0 @@ -52,22 +42,12 @@ pytest<4.6.0 # acceptance.tests.lms.test_lms_course_discovery.CourseDiscoveryTest.test_search edx-search==1.2.2 -# This ruamel.ordereddict will not be supported in python3 but is a dependency of ruamel.yaml -# install it only on python 2.7 -ruamel.ordereddict; python_version == "2.7" - # 1.16.1 requires djangorestframework>=3.8 drf-yasg==1.16 # 2.0.0 is a dummy package, because faulthandler has been incorporated into pytest 5.0 pytest-faulthandler<2.0.0 -# faulthandler is part of core python staring with python 3.3 -faulthandler; python_version == "2.7" - -# Numpy 1.17.0 only supports python >= 3.5 -numpy<1.17.0 - # Pinned by moto to this but since moto is only a test requirement, we don't # see the constraint when generating base.txt # @@ -77,20 +57,5 @@ numpy<1.17.0 # jsondiff==1.2.0 (from -r requirements/edx/base.txt (line 146)) jsondiff==1.1.1 -# Constraining this since the newer versions require Python 3 -more-itertools==5.0.0 - -# Constraining this Gunicorn requires Python 3.x >= 3.4. -gunicorn<20.0.0 - - -# Upgrading to django-object-actions <=1.1.2 requires support for Python 3. In newer versions Python 2 support is -# dropped. This constraint can be removed once we have Python 3. -django-object-actions==1.1.2 - - -# Constraining this since the newer versions require Python 3 -django-debug-toolbar<2.0 # A set of panels that display debug information about the current request/response - -# Constraining this since the newer versions require Python 3 -django-model-utils<4.0.0 \ No newline at end of file +# Constraining this since the newer versions no longer work with the deprecated MIDDLEWARE_CLASSES setting +django-debug-toolbar<2.0 diff --git a/requirements/edx-sandbox/base.txt b/requirements/edx-sandbox/base.txt index d1105fbd50..e9a3a1e25a 100644 --- a/requirements/edx-sandbox/base.txt +++ b/requirements/edx-sandbox/base.txt @@ -1,8 +1,7 @@ # -# This file is autogenerated by pip-compile -# To update, run: -# -# make upgrade +# This file has been frozen for the benefit of Python 2.7 sandboxes, and +# "make upgrade" no longer updates it. It can be removed once that +# version of the sandbox is no longer run in production. # common/lib/sandbox-packages common/lib/symmath diff --git a/requirements/edx-sandbox/py35.in b/requirements/edx-sandbox/py35.in new file mode 100644 index 0000000000..24c7e94717 --- /dev/null +++ b/requirements/edx-sandbox/py35.in @@ -0,0 +1,26 @@ +# Packages to install in the Python sandbox for secured execution of code provided by course authors. +# +# DON'T JUST ADD NEW DEPENDENCIES!!! +# +# If you open a pull request that adds a new dependency, you should: +# * verify that the dependency has a license compatible with AGPLv3 +# * confirm that it has no system requirements beyond what we already install +# * run "make upgrade" to update the detailed requirements files + +-c ../constraints.txt + +-r shared.txt # Dependencies in common with LMS and Studio +matplotlib==2.2.4 # 2D plotting library +numpy==1.16.5 # Numeric array processing utilities; used by scipy +pyparsing==2.2.0 # Python Parsing module +random2 # Implementation of random module that works identically under Python 2 and 3 +scipy==1.2.1 # Math, science, and engineering library +sympy==1.4 # Symbolic math library +git+https://github.com/edx/openedx-calc.git@e9b698c85ad1152002bc0868f475f153dce88952#egg=calc==0.4 +git+https://github.com/edx/openedx-chem.git@ff4e3a03d3c7610e47a9af08eb648d8aabe2eb18#egg=chem==1.0.0 + +# Install these packages from the edx-platform working tree +# NOTE: if you change code in these packages, you MUST change the version +# number in its setup.py or the code WILL NOT be installed during deploy. +-e common/lib/sandbox-packages +-e common/lib/symmath diff --git a/requirements/edx-sandbox/py35.txt b/requirements/edx-sandbox/py35.txt new file mode 100644 index 0000000000..1f607d1cf8 --- /dev/null +++ b/requirements/edx-sandbox/py35.txt @@ -0,0 +1,33 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# make upgrade +# +common/lib/sandbox-packages +common/lib/symmath +git+https://github.com/edx/openedx-calc.git@e9b698c85ad1152002bc0868f475f153dce88952#egg=calc==0.4 +cffi==1.13.2 +git+https://github.com/edx/openedx-chem.git@ff4e3a03d3c7610e47a9af08eb648d8aabe2eb18#egg=chem==1.0.0 +cryptography==2.8 +cycler==0.10.0 # via matplotlib +decorator==4.4.1 +kiwisolver==1.1.0 # via matplotlib +lxml==4.4.1 +markupsafe==1.1.1 +matplotlib==2.2.4 +mpmath==1.1.0 # via sympy +networkx==2.2 +nltk==3.4.5 +numpy==1.16.5 +pycparser==2.19 +pyparsing==2.2.0 +python-dateutil==2.8.1 # via matplotlib +pytz==2019.3 # via matplotlib +random2==1.0.1 +scipy==1.2.1 +six==1.13.0 +sympy==1.4 + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/edx-sandbox/shared.in b/requirements/edx-sandbox/shared.in index 936226ea39..b18a669a0d 100644 --- a/requirements/edx-sandbox/shared.in +++ b/requirements/edx-sandbox/shared.in @@ -10,6 +10,6 @@ -c ../constraints.txt cryptography # Implementations of assorted cryptography algorithms -lxml==3.8.0 # XML parser -networkx==1.7 # Utilities for creating, manipulating, and studying network graphs +lxml==4.4.1 # XML parser +networkx==2.2 # Utilities for creating, manipulating, and studying network graphs nltk # Natural language processing; used by the chem package diff --git a/requirements/edx-sandbox/shared.txt b/requirements/edx-sandbox/shared.txt index 7f5a70ac73..7001854da2 100644 --- a/requirements/edx-sandbox/shared.txt +++ b/requirements/edx-sandbox/shared.txt @@ -6,11 +6,9 @@ # cffi==1.13.2 # via cryptography cryptography==2.8 -enum34==1.1.6 # via cryptography -ipaddress==1.0.23 # via cryptography -lxml==3.8.0 -networkx==1.7 +decorator==4.4.1 # via networkx +lxml==4.4.1 +networkx==2.2 nltk==3.4.5 pycparser==2.19 # via cffi -singledispatch==3.4.0.3 # via nltk -six==1.13.0 # via cryptography, nltk, singledispatch +six==1.13.0 # via cryptography, nltk diff --git a/requirements/edx/base.in b/requirements/edx/base.in index 0c388bd5f6..460d5aa566 100644 --- a/requirements/edx/base.in +++ b/requirements/edx/base.in @@ -91,7 +91,6 @@ edx-submissions edx-user-state-client edx-when edxval -enum34==1.1.6 # Backport of Enum from Python 3.4+ event-tracking feedparser==5.1.3 fs==2.0.18 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index da4633ff4a..35b0ff2e1e 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -27,8 +27,7 @@ appdirs==1.4.3 # via fs argh==0.26.2 attrs==19.3.0 babel==1.3 -backports.functools-lru-cache==1.6.1 # via soupsieve -beautifulsoup4==4.8.1 # via pynliner +beautifulsoup4==4.8.2 # via pynliner billiard==3.3.0.23 # via celery bleach==2.1.4 boto3==1.4.8 @@ -50,7 +49,7 @@ git+https://github.com/edx/crowdsourcehinter.git@a7ffc85b134b7d8909bf1fefd23dbdb cryptography==2.8 cssutils==1.0.2 # via pynliner ddt==1.2.2 -decorator==4.4.1 # via pycontracts +decorator==4.4.1 defusedxml==0.5.0 django-appconf==1.0.3 # via django-statici18n django-babel-underscore==0.5.2 @@ -71,7 +70,7 @@ django-multi-email-field==0.5.1 # via edx-enterprise django-mysql==2.4.1 git+https://github.com/edx/django-oauth-plus.git@b9b64a3ac24fd11f471763c88462bbf3c53e46e6#egg=django-oauth-plus==2.2.9.edx-4 django-oauth-toolkit==1.1.3 -django-object-actions==1.1.2 # via edx-enterprise +django-object-actions==2.0.0 # via edx-enterprise django-pipeline==1.6.14 django-pyfs==2.1 django-ratelimit-backend==1.1.1 @@ -105,7 +104,7 @@ edx-django-release-util==0.3.2 edx-django-sites-extensions==2.4.2 edx-django-utils==2.0.2 edx-drf-extensions==2.4.5 -edx-enterprise==2.0.32 +edx-enterprise==2.0.34 edx-i18n-tools==0.5.0 edx-milestones==0.2.6 edx-oauth2-provider==1.3.1 @@ -122,17 +121,15 @@ edx-user-state-client==1.1.2 edx-when==0.5.2 edxval==1.1.33 elasticsearch==1.9.0 # via edx-search -enum34==1.1.6 +enum34==1.1.6 # via edxval event-tracking==0.3.0 feedparser==5.1.3 fs-s3fs==0.1.8 fs==2.0.18 -funcsigs==1.0.2 future==0.18.2 # via django-ses, edx-celeryutils, edx-enterprise, pyjwkest -futures==3.3.0 ; python_version == "2.7" # via django-pipeline, python-swiftclient, s3transfer, xblock-utils geoip2==2.9.0 glob2==0.7 -gunicorn==19.10.0 +gunicorn==20.0.4 help-tokens==1.0.5 html5lib==1.0.1 httplib2==0.15.0 @@ -152,7 +149,7 @@ lepl==5.1.3 # via rfc6266-parser libsass==0.10.0 loremipsum==1.0.5 git+https://github.com/edx/xblock-lti-consumer.git@v1.2.3#egg=lti_consumer-xblock==1.2.3 -lxml==3.8.0 +lxml==4.4.1 mailsnake==1.6.4 mako==1.0.2 markdown==2.6.11 @@ -164,11 +161,11 @@ git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752 mongoengine==0.10.0 mpmath==1.1.0 # via sympy mysqlclient==1.4.6 -networkx==1.7 +networkx==2.2 newrelic==5.4.1.134 nltk==3.4.5 nodeenv==1.3.3 -numpy==1.16.5 +numpy==1.18.0 # via scipy git+https://github.com/joestump/python-oauth2.git@b94f69b1ad195513547924e380d9265133e995fa#egg=oauth2 oauthlib==2.1.0 git+https://github.com/edx/edx-ora2.git@2.5.3#egg=ora2==2.5.3 @@ -199,9 +196,9 @@ pysrt==1.1.1 python-dateutil==2.4.0 python-levenshtein==0.12.0 python-memcached==1.59 -python-openid==2.2.5 ; python_version == "2.7" # via social-auth-core python-slugify==4.0.0 # via code-annotations python-swiftclient==3.8.1 +python3-openid==3.1.0 ; python_version >= "3" python3-saml==1.5.0 pytz==2019.3 pyuca==1.1 @@ -212,19 +209,17 @@ redis==2.10.6 requests-oauthlib==1.1.0 requests==2.22.0 rest-condition==1.0.3 -rfc6266-parser==0.0.5.post2 -ruamel.ordereddict==0.4.14 ; python_version == "2.7" # via ruamel.yaml +rfc6266-parser==0.0.6 ruamel.yaml.clib==0.2.0 # via ruamel.yaml ruamel.yaml==0.16.5 # via drf-yasg rules==2.1 s3transfer==0.1.13 # via boto3 sailthru-client==2.2.3 -scipy==1.2.1 +scipy==1.4.1 semantic-version==2.8.4 # via edx-drf-extensions shapely==1.6.4.post2 shortuuid==0.5.0 # via edx-django-oauth2-provider simplejson==3.17.0 -singledispatch==3.4.0.3 six==1.13.0 slumber==0.7.1 # via edx-bulk-grades, edx-enterprise, edx-rest-api-client social-auth-app-django==3.1.0 diff --git a/requirements/edx/coverage.txt b/requirements/edx/coverage.txt index 9d8dd57559..ccd7e9ee07 100644 --- a/requirements/edx/coverage.txt +++ b/requirements/edx/coverage.txt @@ -4,8 +4,6 @@ # # make upgrade # -configparser==4.0.2 # via importlib-metadata -contextlib2==0.6.0.post1 # via importlib-metadata coverage==5.0.1 diff-cover==2.5.0 importlib-metadata==1.3.0 # via inflect, pluggy @@ -13,10 +11,8 @@ inflect==3.0.2 # via jinja2-pluralize jinja2-pluralize==0.3.0 # via diff-cover jinja2==2.10.3 # via diff-cover, jinja2-pluralize markupsafe==1.1.1 # via jinja2 -more-itertools==5.0.0 # via zipp -pathlib2==2.3.5 # via importlib-metadata +more-itertools==8.0.2 # via zipp pluggy==0.13.1 # via diff-cover pygments==2.5.2 # via diff-cover -scandir==1.10.0 # via pathlib2 -six==1.13.0 # via diff-cover, pathlib2 +six==1.13.0 # via diff-cover zipp==0.6.0 # via importlib-metadata diff --git a/requirements/edx/development.in b/requirements/edx/development.in index 90858c79c5..f95dd2fdba 100644 --- a/requirements/edx/development.in +++ b/requirements/edx/development.in @@ -17,9 +17,7 @@ click # Used for perf_tests utilities in modulesto django-debug-toolbar # A set of panels that display debug information about the current request/response edx-sphinx-theme # Documentation theme pyinotify # More efficient checking for runserver reload trigger events -sphinx==1.8.5 # Pinned because 2.0.0 release requires Python '>=3.5' but current Python is 2.7.12 vulture # Detects possible dead/unused code, used in scripts/find-dead-code.sh -modernize # Used to make Python 2 code more modern with the intention of eventually porting it over to Python 3. # Using a github hash because the "include" feature wasn't in the latest release (0.5.0): git+https://github.com/sphinx-contrib/openapi.git@9306435601ae05138edea54419dd83ce9e729ad8#egg=sphinxcontrib-openapi[markdown]==0.0 diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 0007a3ba9d..3d7640378d 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -34,10 +34,7 @@ attrs==19.3.0 aws-xray-sdk==0.95 babel==1.3 backports.functools-lru-cache==1.6.1 -backports.ssl-match-hostname==3.7.0.1 -backports.tempfile==1.0 -backports.weakref==1.0.post1 -beautifulsoup4==4.8.1 +beautifulsoup4==4.8.2 billiard==3.3.0.23 bleach==2.1.4 bok-choy==1.0.1 @@ -56,7 +53,6 @@ click-log==0.3.2 click==7.0 code-annotations==0.3.3 colorama==0.4.3 -configparser==4.0.2 contextlib2==0.6.0.post1 cookies==2.2.1 coreapi==2.3.3 @@ -92,7 +88,7 @@ django-multi-email-field==0.5.1 django-mysql==2.4.1 git+https://github.com/edx/django-oauth-plus.git@b9b64a3ac24fd11f471763c88462bbf3c53e46e6#egg=django-oauth-plus==2.2.9.edx-4 django-oauth-toolkit==1.1.3 -django-object-actions==1.1.2 +django-object-actions==2.0.0 django-pipeline==1.6.14 django-pyfs==2.1 django-ratelimit-backend==1.1.1 @@ -127,7 +123,7 @@ edx-django-release-util==0.3.2 edx-django-sites-extensions==2.4.2 edx-django-utils==2.0.2 edx-drf-extensions==2.4.5 -edx-enterprise==2.0.32 +edx-enterprise==2.0.34 edx-i18n-tools==0.5.0 edx-lint==1.3.0 edx-milestones==0.2.6 @@ -152,7 +148,6 @@ event-tracking==0.3.0 execnet==1.7.1 factory_boy==2.8.1 faker==3.0.0 -faulthandler==3.1 ; python_version == "2.7" feedparser==5.1.3 filelock==3.0.12 flake8-polyfill==1.0.2 @@ -160,19 +155,16 @@ flake8==3.7.9 freezegun==0.3.12 fs-s3fs==0.1.8 fs==2.0.18 -funcsigs==1.0.2 -functools32==3.2.3.post2 ; python_version == "2.7" future==0.18.2 -futures==3.3.0 ; python_version == "2.7" geoip2==2.9.0 glob2==0.7 -gunicorn==19.10.0 +gunicorn==20.0.4 help-tokens==1.0.5 html5lib==1.0.1 httplib2==0.15.0 httpretty==0.9.7 idna==2.8 -imagesize==1.1.0 # via sphinx +imagesize==1.2.0 # via sphinx importlib-metadata==1.3.0 inflect==3.0.2 inflection==0.3.1 @@ -195,7 +187,7 @@ lepl==5.1.3 libsass==0.10.0 loremipsum==1.0.5 git+https://github.com/edx/xblock-lti-consumer.git@v1.2.3#egg=lti_consumer-xblock==1.2.3 -lxml==3.8.0 +lxml==4.4.1 m2r==0.2.1 mailsnake==1.6.4 mako==1.0.2 @@ -207,18 +199,17 @@ maxminddb==1.5.2 mccabe==0.6.1 mistune==0.8.4 # via m2r mock==3.0.5 -modernize==0.7 git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a#egg=MongoDBProxy==0.1.0+edx.2 mongoengine==0.10.0 -more-itertools==5.0.0 +more-itertools==8.0.2 moto==1.3.1 mpmath==1.1.0 mysqlclient==1.4.6 -networkx==1.7 +networkx==2.2 newrelic==5.4.1.134 nltk==3.4.5 nodeenv==1.3.3 -numpy==1.16.5 +numpy==1.18.0 git+https://github.com/joestump/python-oauth2.git@b94f69b1ad195513547924e380d9265133e995fa#egg=oauth2 oauthlib==2.1.0 git+https://github.com/edx/edx-ora2.git@2.5.3#egg=ora2==2.5.3 @@ -253,29 +244,28 @@ pyjwkest==1.3.2 pyjwt==1.5.2 pylint-celery==0.3 pylint-django==0.7.2 -pylint-plugin-utils==0.3 +pylint-plugin-utils==0.6 pylint==1.7.6 pymongo==3.9.0 pynliner==0.8.0 pyparsing==2.2.0 pyquery==1.4.1 pyrsistent==0.15.6 # via jsonschema -pysqlite==2.8.3 ; python_version == "2.7" pysrt==1.1.1 pytest-attrib==0.1.3 git+https://github.com/nedbat/pytest-cov.git@nedbat/cov5-combine#egg=pytest-cov==0.0 pytest-django==3.7.0 pytest-faulthandler==1.6.0 pytest-forked==1.1.3 -pytest-randomly==1.2.3 +pytest-randomly==3.2.0 pytest-xdist==1.31.0 pytest==4.5.0 python-dateutil==2.4.0 python-levenshtein==0.12.0 python-memcached==1.59 -python-openid==2.2.5 ; python_version == "2.7" python-slugify==4.0.0 python-swiftclient==3.8.1 +python3-openid==3.1.0 ; python_version >= "3" python3-saml==1.5.0 pytz==2019.3 pyuca==1.1 @@ -287,15 +277,13 @@ redis==2.10.6 requests-oauthlib==1.1.0 requests==2.22.0 rest-condition==1.0.3 -rfc6266-parser==0.0.5.post2 -ruamel.ordereddict==0.4.14 ; python_version == "2.7" +rfc6266-parser==0.0.6 ruamel.yaml.clib==0.2.0 ruamel.yaml==0.16.5 rules==2.1 s3transfer==0.1.13 sailthru-client==2.2.3 -scandir==1.10.0 -scipy==1.2.1 +scipy==1.4.1 selenium==3.141.0 semantic-version==2.8.4 shapely==1.6.4.post2 @@ -310,10 +298,15 @@ social-auth-core==3.2.0 sorl-thumbnail==12.3 sortedcontainers==2.1.0 soupsieve==1.9.5 -sphinx==1.8.5 +sphinx==2.3.1 # via edx-sphinx-theme, sphinxcontrib-httpdomain +sphinxcontrib-applehelp==1.0.1 # via sphinx +sphinxcontrib-devhelp==1.0.1 # via sphinx +sphinxcontrib-htmlhelp==1.0.2 # via sphinx sphinxcontrib-httpdomain==1.7.0 +sphinxcontrib-jsmath==1.0.1 # via sphinx git+https://github.com/sphinx-contrib/openapi.git@9306435601ae05138edea54419dd83ce9e729ad8#egg=sphinxcontrib-openapi[markdown]==0.0 -sphinxcontrib-websupport==1.1.2 # via sphinx +sphinxcontrib-qthelp==1.0.2 # via sphinx +sphinxcontrib-serializinghtml==1.1.3 # via sphinx sqlparse==0.3.0 staff-graded-xblock==0.6 stevedore==1.31.0 @@ -326,7 +319,6 @@ toml==0.10.0 tox-battery==0.5.1 tox==3.14.2 transifex-client==0.13.4 -typing==3.7.4.1 unicodecsv==0.14.1 unidiff==0.5.5 uritemplate==3.0.1 @@ -340,7 +332,7 @@ wcwidth==0.1.7 web-fragments==0.3.1 webencodings==0.5.1 webob==1.8.5 -websocket-client==0.56.0 +websocket-client==0.57.0 werkzeug==0.16.0 wrapt==1.10.5 git+https://github.com/edx-solutions/xblock-drag-and-drop-v2@v2.2.6#egg=xblock-drag-and-drop-v2==2.2.6 diff --git a/requirements/edx/paver.txt b/requirements/edx/paver.txt index c02248a221..4e4a7ac9dd 100644 --- a/requirements/edx/paver.txt +++ b/requirements/edx/paver.txt @@ -8,7 +8,6 @@ argh==0.26.2 # via watchdog certifi==2019.11.28 # via requests chardet==3.0.4 # via requests edx-opaque-keys==2.0.1 -funcsigs==1.0.2 # via mock idna==2.8 # via requests lazy==1.1 libsass==0.10.0 diff --git a/requirements/edx/testing.in b/requirements/edx/testing.in index 269bcdaf4b..c7ad81d7ff 100644 --- a/requirements/edx/testing.in +++ b/requirements/edx/testing.in @@ -33,7 +33,6 @@ moto==1.3.1 # Lets tests mock AWS access via the boto library pycodestyle # Checker for compliance with the Python style guide (PEP 8) polib # Library for manipulating gettext translation files, used to test paver i18n commands pyquery # jQuery-like API for retrieving fragments of HTML and XML files in tests -pysqlite # DB-API 2.0 interface for SQLite 3.x (used as the relational database for most tests) pytest # Testing framework pytest-attrib # Select tests based on attributes git+https://github.com/nedbat/pytest-cov.git@nedbat/cov5-combine#egg=pytest-cov==0.0 diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 3a8290e09e..10720703bf 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -32,11 +32,8 @@ atomicwrites==1.3.0 # via pytest attrs==19.3.0 aws-xray-sdk==0.95 # via moto babel==1.3 -backports.functools-lru-cache==1.6.1 -backports.ssl-match-hostname==3.7.0.1 # via docker -backports.tempfile==1.0 # via moto -backports.weakref==1.0.post1 # via backports.tempfile -beautifulsoup4==4.8.1 +backports.functools-lru-cache==1.6.1 # via caniusepython3 +beautifulsoup4==4.8.2 billiard==3.3.0.23 bleach==2.1.4 bok-choy==1.0.1 @@ -55,7 +52,6 @@ click-log==0.3.2 # via edx-lint click==7.0 code-annotations==0.3.3 colorama==0.4.3 # via radon -configparser==4.0.2 contextlib2==0.6.0.post1 cookies==2.2.1 # via moto coreapi==2.3.3 @@ -90,7 +86,7 @@ django-multi-email-field==0.5.1 django-mysql==2.4.1 git+https://github.com/edx/django-oauth-plus.git@b9b64a3ac24fd11f471763c88462bbf3c53e46e6#egg=django-oauth-plus==2.2.9.edx-4 django-oauth-toolkit==1.1.3 -django-object-actions==1.1.2 +django-object-actions==2.0.0 django-pipeline==1.6.14 django-pyfs==2.1 django-ratelimit-backend==1.1.1 @@ -124,7 +120,7 @@ edx-django-release-util==0.3.2 edx-django-sites-extensions==2.4.2 edx-django-utils==2.0.2 edx-drf-extensions==2.4.5 -edx-enterprise==2.0.32 +edx-enterprise==2.0.34 edx-i18n-tools==0.5.0 edx-lint==1.3.0 edx-milestones==0.2.6 @@ -148,7 +144,6 @@ event-tracking==0.3.0 execnet==1.7.1 # via pytest-xdist factory_boy==2.8.1 faker==3.0.0 # via factory-boy -faulthandler==3.1 ; python_version == "2.7" # via pytest-faulthandler feedparser==5.1.3 filelock==3.0.12 # via tox flake8-polyfill==1.0.2 # via radon @@ -156,13 +151,10 @@ flake8==3.7.9 # via flake8-polyfill freezegun==0.3.12 fs-s3fs==0.1.8 fs==2.0.18 -funcsigs==1.0.2 -functools32==3.2.3.post2 ; python_version == "2.7" # via flake8 future==0.18.2 -futures==3.3.0 ; python_version == "2.7" geoip2==2.9.0 glob2==0.7 -gunicorn==19.10.0 +gunicorn==20.0.4 help-tokens==1.0.5 html5lib==1.0.1 httplib2==0.15.0 @@ -189,7 +181,7 @@ lepl==5.1.3 libsass==0.10.0 loremipsum==1.0.5 git+https://github.com/edx/xblock-lti-consumer.git@v1.2.3#egg=lti_consumer-xblock==1.2.3 -lxml==3.8.0 +lxml==4.4.1 mailsnake==1.6.4 mako==1.0.2 mando==0.6.4 # via radon @@ -201,21 +193,21 @@ mccabe==0.6.1 # via flake8, pylint mock==3.0.5 git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a#egg=MongoDBProxy==0.1.0+edx.2 mongoengine==0.10.0 -more-itertools==5.0.0 +more-itertools==8.0.2 moto==1.3.1 mpmath==1.1.0 mysqlclient==1.4.6 -networkx==1.7 +networkx==2.2 newrelic==5.4.1.134 nltk==3.4.5 nodeenv==1.3.3 -numpy==1.16.5 +numpy==1.18.0 git+https://github.com/joestump/python-oauth2.git@b94f69b1ad195513547924e380d9265133e995fa#egg=oauth2 oauthlib==2.1.0 git+https://github.com/edx/edx-ora2.git@2.5.3#egg=ora2==2.5.3 packaging==19.2 # via caniusepython3, tox path.py==8.2.1 -pathlib2==2.3.5 +pathlib2==2.3.5 # via pytest pathtools==0.1.2 paver==1.3.4 pbr==5.4.4 @@ -242,28 +234,27 @@ pyjwkest==1.3.2 pyjwt==1.5.2 pylint-celery==0.3 # via edx-lint pylint-django==0.7.2 # via edx-lint -pylint-plugin-utils==0.3 # via pylint-celery, pylint-django -pylint==1.7.6 # via edx-lint, pylint-celery, pylint-django +pylint-plugin-utils==0.6 # via pylint-celery, pylint-django +pylint==1.7.6 # via edx-lint, pylint-celery, pylint-django, pylint-plugin-utils pymongo==3.9.0 pynliner==0.8.0 pyparsing==2.2.0 pyquery==1.4.1 -pysqlite==2.8.3 ; python_version == "2.7" pysrt==1.1.1 pytest-attrib==0.1.3 git+https://github.com/nedbat/pytest-cov.git@nedbat/cov5-combine#egg=pytest-cov==0.0 pytest-django==3.7.0 pytest-faulthandler==1.6.0 pytest-forked==1.1.3 # via pytest-xdist -pytest-randomly==1.2.3 +pytest-randomly==3.2.0 pytest-xdist==1.31.0 pytest==4.5.0 python-dateutil==2.4.0 python-levenshtein==0.12.0 python-memcached==1.59 -python-openid==2.2.5 ; python_version == "2.7" python-slugify==4.0.0 python-swiftclient==3.8.1 +python3-openid==3.1.0 ; python_version >= "3" python3-saml==1.5.0 pytz==2019.3 pyuca==1.1 @@ -275,15 +266,13 @@ redis==2.10.6 requests-oauthlib==1.1.0 requests==2.22.0 rest-condition==1.0.3 -rfc6266-parser==0.0.5.post2 -ruamel.ordereddict==0.4.14 ; python_version == "2.7" +rfc6266-parser==0.0.6 ruamel.yaml.clib==0.2.0 ruamel.yaml==0.16.5 rules==2.1 s3transfer==0.1.13 sailthru-client==2.2.3 -scandir==1.10.0 -scipy==1.2.1 +scipy==1.4.1 selenium==3.141.0 semantic-version==2.8.4 shapely==1.6.4.post2 @@ -309,7 +298,6 @@ toml==0.10.0 # via tox tox-battery==0.5.1 tox==3.14.2 transifex-client==0.13.4 -typing==3.7.4.1 # via flake8 unicodecsv==0.14.1 unidiff==0.5.5 uritemplate==3.0.1 @@ -322,7 +310,7 @@ wcwidth==0.1.7 # via pytest web-fragments==0.3.1 webencodings==0.5.1 webob==1.8.5 -websocket-client==0.56.0 # via docker +websocket-client==0.57.0 # via docker werkzeug==0.16.0 # via moto wrapt==1.10.5 git+https://github.com/edx-solutions/xblock-drag-and-drop-v2@v2.2.6#egg=xblock-drag-and-drop-v2==2.2.6 diff --git a/scripts/post-pip-compile.sh b/scripts/post-pip-compile.sh index 5f4b18bad8..5860261391 100755 --- a/scripts/post-pip-compile.sh +++ b/scripts/post-pip-compile.sh @@ -28,7 +28,7 @@ function clean_file { # Code sandbox local package installs must be non-editable due to file # permissions issues. edxapp ones must stay editable until assorted # packaging bugs are fixed. - if [[ "${FILE_PATH}" == "requirements/edx-sandbox/base.txt" ]]; then + if [[ "${FILE_PATH}" == "requirements/edx-sandbox/py35.txt" ]]; then sed "s|-e common/lib/|common/lib/|" ${FILE_PATH} > ${TEMP_FILE} mv ${TEMP_FILE} ${FILE_PATH} fi diff --git a/scripts/upgrade_pysqlite.sh b/scripts/upgrade_pysqlite.sh deleted file mode 100755 index 91e8c031df..0000000000 --- a/scripts/upgrade_pysqlite.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# First, check to see if the pysqlite version has already been upgraded. If so, no installation is needed. -# The pip-installed version of pysqlite has a sqlite_version of "3.11.0". -sqlite_version=`python -c "from pysqlite2._sqlite import sqlite_version; print sqlite_version"` -if [ $sqlite_version = "3.14.1" ]; then - exit 0 -fi - -# Upgrade the version of pysqlite/sqlite to avoid crashes during testing. -# Ideally, this code would just install a pre-built wheel - change it to this -# once edX has its own artifact server. -pip uninstall -y pysqlite -rm -rf tmp_pysqlite_upgrade && mkdir -p tmp_pysqlite_upgrade && cd tmp_pysqlite_upgrade -curl -o 2.8.3.tar.gz https://codeload.github.com/ghaering/pysqlite/tar.gz/2.8.3 -curl -o sqlite-autoconf-3140100.tar.gz https://www.sqlite.org/2016/sqlite-autoconf-3140100.tar.gz -tar -xzvf sqlite-autoconf-3140100.tar.gz -tar -xzvf 2.8.3.tar.gz -cp -av sqlite-autoconf-3140100/. pysqlite-2.8.3/ -cd ./pysqlite-2.8.3 && python setup.py build_static install -cd ../.. && rm -rf tmp_pysqlite_upgrade diff --git a/tox.ini b/tox.ini index 9b0752d9b9..4360f57f32 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{27,35,36,37}-django{111,20,21} +envlist = py{35,36,37}-django{111,20,21} # This is needed to prevent the lms, cms, and openedx packages inside the "Open # edX" package (defined in setup.py) from getting installed into site-packages @@ -77,7 +77,4 @@ whitelist_externals = /usr/bin/curl /bin/tar commands = - # Upgrade sqlite to fix crashes during testing. - py27: bash scripts/upgrade_pysqlite.sh - # Now perform testing. {posargs}