From d1728b3d6a91c1c8ef7250b4c7142a395ed65da8 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 12 Sep 2019 08:40:03 -0400 Subject: [PATCH 1/3] Fixes for bokchoy running in python3. --- common/lib/xmodule/xmodule/static_content.py | 6 ++++++ tox.ini | 2 ++ 2 files changed, 8 insertions(+) diff --git a/common/lib/xmodule/xmodule/static_content.py b/common/lib/xmodule/xmodule/static_content.py index 6d276fd670..37adc9a46d 100755 --- a/common/lib/xmodule/xmodule/static_content.py +++ b/common/lib/xmodule/xmodule/static_content.py @@ -230,6 +230,12 @@ def _write_files(output_root, contents, generated_suffix_map=None): not_file = not output_file.isfile() + # Sometimes content is already unicode and sometimes it's not + # so we add this conditional here to make sure that below we're + # always working with streams of bytes. + if not isinstance(file_content, six.binary_type): + file_content = file_content.encode('utf-8') + # not_file is included to short-circuit this check, because # read_md5 depends on the file already existing write_file = not_file or output_file.read_md5() != hashlib.md5(file_content).digest() diff --git a/tox.ini b/tox.ini index 388b9e0ae7..628cfa44f4 100644 --- a/tox.ini +++ b/tox.ini @@ -61,6 +61,8 @@ passenv = XDIST_WORKER_SECURITY_GROUP XDIST_WORKER_SUBNET PYTHON_VERSION + LMS_CFG + STUDIO_CFG deps = django111: -r requirements/edx/django.txt From 9acad23fc4c9f8138dc76c13847b341e164bd420 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 12 Sep 2019 09:41:35 -0400 Subject: [PATCH 2/3] Remove snakefood and our usage of it. Unfortunately snakefood is not supported in python3 so I'm removing it and the tools built on top of it for now as they are not in active use. If you're looking at this wishing it still existed, Check out https://github.com/bwohlberg/jonga or https://pypi.org/project/findimports/ --- requirements/edx/development.in | 1 - requirements/edx/development.txt | 1 - scripts/dependencies/development.py | 47 ----------------- scripts/dependencies/enumerate.sh | 37 -------------- scripts/dependencies/from_package.py | 38 -------------- scripts/dependencies/on_package.py | 35 ------------- scripts/dependencies/testing.py | 76 ---------------------------- 7 files changed, 235 deletions(-) delete mode 100755 scripts/dependencies/development.py delete mode 100755 scripts/dependencies/enumerate.sh delete mode 100755 scripts/dependencies/from_package.py delete mode 100755 scripts/dependencies/on_package.py delete mode 100755 scripts/dependencies/testing.py diff --git a/requirements/edx/development.in b/requirements/edx/development.in index 1b5cdae5f9..7cbf550c8d 100644 --- a/requirements/edx/development.in +++ b/requirements/edx/development.in @@ -17,7 +17,6 @@ click # Used for perf_tests utilities in modulesto django-debug-toolbar==1.8 # A set of panels that display debug information about the current request/response edx-sphinx-theme # Documentation theme pyinotify # More efficient checking for runserver reload trigger events -snakefood # Lists dependencies between Python modules, used in scripts/dependencies/* sphinx==1.8.5 # Pinned because 2.0.0 release requires Python '>=3.5' but current Python is 2.7.12 vulture # Detects possible dead/unused code, used in scripts/find-dead-code.sh modernize # Used to make Python 2 code more modern with the intention of eventually porting it over to Python 3. diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 22abbc87f9..ef279b1822 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -303,7 +303,6 @@ simplejson==3.16.0 singledispatch==3.4.0.3 six==1.12.0 slumber==0.7.1 -snakefood==1.4 snowballstemmer==1.9.1 # via sphinx social-auth-app-django==2.1.0 social-auth-core==1.7.0 diff --git a/scripts/dependencies/development.py b/scripts/dependencies/development.py deleted file mode 100755 index 506d7c34d8..0000000000 --- a/scripts/dependencies/development.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python -""" -List any dependencies on development utilities in edx-platform from -non-development modules. Generally, there shouldn't be any; such a dependency -could result in ImportErrors in production or tests where development packages -aren't installed. - -This script counts on scripts/dependencies/enumerate.sh having already -been run in order to generate a dependency data file to work from. -""" -from __future__ import absolute_import, print_function - -import os -import re -import sys - -# Enumerate all the Python modules that should only be imported from development utilities -pattern_fragments = [ - # Development utility modules within edx-platform - r'^xmodule/modulestore/perf_tests' # modulestore performance tests - - # Development-only package dependencies - r'^debug_toolbar', # django-debug-toolbar -] - -dev_pattern = re.compile('|'.join(pattern_fragments)) - -data_path = 'reports/dependencies/dependencies.txt' -if not os.path.exists(data_path): - print('The dependencies data file is unavailable; run scripts/dependencies/enumerate.sh first.') - sys.exit(1) -exit_status = 0 -with open(data_path, 'r') as f: - for dep in map(eval, f): - (from_root, from_name), (to_root, to_name) = dep - if to_name is None: - continue - if dev_pattern.search(to_name) and not dev_pattern.search(from_name): - # We usually don't care about dependencies between modules in site-packages - if from_root.endswith(u'site-packages') and to_root.endswith(u'site-packages'): - continue - # The django-debug-toolbar URL imports are safely behind conditions on INSTALLED_APPS - if from_name in {u'cms/urls.py', u'lms/urls.py'} and to_name == u'debug_toolbar': - continue - print(dep) - exit_status = 1 -sys.exit(exit_status) diff --git a/scripts/dependencies/enumerate.sh b/scripts/dependencies/enumerate.sh deleted file mode 100755 index 1ea2ae970e..0000000000 --- a/scripts/dependencies/enumerate.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -set -e - -############################################################################ -# -# enumerate.sh -# -# Enumerates all dependencies (imports) from Python modules in -# edx-platform. The resulting data file generated at -# reports/dependencies/dependencies.txt can then be used by other scripts -# to detect inappropriate imports, such as: -# -# * Imports of test modules or testing packages from core application code -# * Imports of development-only packages from core or test code -# * Imports from a package we want to stop using as a dependency -# * Imports of other edx-platform modules from a module we want to move to -# a separate package in its own repository -# -# This script can take a while to run (a few minutes), so it should be run -# independently of the other scripts which use this data. -# -# While running, a number of warnings such as "Could not import module -# 'assert_equal'" may be generated. This is normal; the snakefood utility -# can't really distinguish between the import of a module and the import of -# an object within a module, so it prints a warning on all instances of the -# latter just in case it actually was an attempt to import a module which -# it couldn't find in the current PYTHONPATH. If you do see some modules -# listed which you think should be findable, you may need to run -# "make requirements" or update the ROOTS variable in this script. -# -############################################################################ - -OUTPUT_DIR="reports/dependencies" -mkdir -p ${OUTPUT_DIR} -DEPENDENCIES=${OUTPUT_DIR}/dependencies.txt -ROOTS=cms/djangoapps:common/djangoapps:lms/djangoapps:scripts/xsslint -PYTHONPATH=${ROOTS} sfood cms common lms openedx pavelib scripts manage.py pavement.py > ${DEPENDENCIES} diff --git a/scripts/dependencies/from_package.py b/scripts/dependencies/from_package.py deleted file mode 100755 index 614fa49bd9..0000000000 --- a/scripts/dependencies/from_package.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python -""" -List any modules that are imported from the given package. This can be used -to determine what needs to be refactored to allow a package to be broken out -into a separately installed package. The package argument to the script -should be formatted as shown in these examples: - -* scripts/dependencies/from_package.py xmodule -* scripts/dependencies/from_package.py openedx/features/course_experience -* scripts/dependencies/from_package.py cms/djangoapps/verify_student - -This script counts on scripts/dependencies/enumerate.sh having already -been run in order to generate a dependency data file to work from. -""" -from __future__ import absolute_import, print_function - -import os -import re -import sys - -pattern = re.compile(u'^{}'.format(sys.argv[1])) - -data_path = 'reports/dependencies/dependencies.txt' -if not os.path.exists(data_path): - print('The dependencies data file is unavailable; run scripts/dependencies/enumerate.sh first.') -with open(data_path, 'r') as f: - for dep in map(eval, f): - (from_root, from_name), (to_root, to_name) = dep - if to_name is None: - continue - if pattern.search(from_name) and not pattern.search(to_name): - # We usually don't care about dependencies between modules in site-packages - if from_root.endswith(u'site-packages') and to_root.endswith(u'site-packages'): - continue - # We don't really care about dependencies on the standard library - if to_root.startswith('/usr/lib/python') or to_root.endswith('lib/python2.7'): - continue - print(dep) diff --git a/scripts/dependencies/on_package.py b/scripts/dependencies/on_package.py deleted file mode 100755 index 77ad4e9c1f..0000000000 --- a/scripts/dependencies/on_package.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python -""" -List any modules that import code from the given package. This can be used -to determine if the package can be safely removed, or just to understand -what context it's used in. The package argument to the script should be -formatted as shown in these examples: - -* scripts/dependencies/on_package.py nose -* scripts/dependencies/on_package.py third_parth_auth -* scripts/dependencies/on_package.py cms/djangoapps/verify_student - -This script counts on scripts/dependencies/enumerate.sh having already -been run in order to generate a dependency data file to work from. -""" -from __future__ import absolute_import, print_function - -import os -import re -import sys - -pattern = re.compile(u'^{}'.format(sys.argv[1])) - -data_path = 'reports/dependencies/dependencies.txt' -if not os.path.exists(data_path): - print('The dependencies data file is unavailable; run scripts/dependencies/enumerate.sh first.') -with open(data_path, 'r') as f: - for dep in map(eval, f): - (from_root, from_name), (to_root, to_name) = dep - if to_name is None: - continue - if pattern.search(to_name) and not pattern.search(from_name): - # We usually don't care about dependencies between modules in site-packages - if from_root.endswith(u'site-packages') and to_root.endswith(u'site-packages'): - continue - print(dep) diff --git a/scripts/dependencies/testing.py b/scripts/dependencies/testing.py deleted file mode 100755 index dfd4409d92..0000000000 --- a/scripts/dependencies/testing.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python -""" -List any dependencies on test modules in edx-platform from non-test modules. -Generally, there shouldn't be any; such a dependency could result in -ImportErrors in production where testing packages aren't installed. - -This script counts on scripts/dependencies/enumerate.sh having already -been run in order to generate a dependency data file to work from. -""" -from __future__ import absolute_import, print_function - -import os -import re -import sys - -# Enumerate all the Python modules that should only be imported during test runs -pattern_fragments = [ - # Test modules within edx-platform - r'/tests?\.py', # test.py, tests.py - r'/tests?_[^/]*\.py', # test_*.py, tests_*.py - r'/[^/]*_tests\.py', # *_tests.py - r'/tests?/', # */test/*, */tests/* - r'[cl]ms/.*/features/', # cms/*/features/*, lms/*/features/* - r'/testing\.py', # testing.py - r'/testutils\.py', # testutils.py - r'/tests$', # tests/__init__.py - r'conftest\.py', # conftest.py - r'/factories\.py', # factories.py - r'^terrain', # terrain/* - r'/setup_models_to_send_test_emails\.py', # setup_models_to_send_test_emails management command - - # Testing-only package dependencies - r'^bs4', # beautifulsoup4 - r'^before_after$', # before_after - r'^bok_choy', # bok-choy - r'^cssselect', # cssselect - r'^factory', # factory_boy - r'^freezegun', # freezegun - r'^httpretty', # httpretty - r'^moto', # moto - r'^nose', # nose - r'^pyquery', # pyquery - r'^pytest.py$', # pytest - r'^selenium', # selenium - r'^singledispatch', # singledispatch - r'^testfixtures', # testfixtures -] - -test_pattern = re.compile('|'.join(pattern_fragments)) - -data_path = 'reports/dependencies/dependencies.txt' -if not os.path.exists(data_path): - print('The dependencies data file is unavailable; run scripts/dependencies/enumerate.sh first.') - sys.exit(1) -exit_status = 0 -with open(data_path, 'r') as f: - for dep in map(eval, f): - (from_root, from_name), (to_root, to_name) = dep - if to_name is None: - continue - if test_pattern.search(to_name) and not test_pattern.search(from_name): - # snakefood sometimes picks a weird place to split the root path and filename - if from_root.endswith('/tests'): - continue - # We usually don't care about dependencies between modules in site-packages - if from_root.endswith(u'site-packages') and to_root.endswith(u'site-packages'): - continue - # Dependencies on django.test and waffle.testutils are ok - if to_name.startswith(u'django/test') or to_name == u'waffle/testutils.py': - continue - # Dependencies within pavelib are ok - if from_name.startswith(u'pavelib') and to_name.startswith(u'pavelib'): - continue - print(dep) - exit_status = 1 -sys.exit(exit_status) From dfe77a968eefe0c766a1de918dc9502abe27973c Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 12 Sep 2019 09:54:18 -0400 Subject: [PATCH 3/3] Remove the mongo plugin for the djnago-debug-toolbar. We are currently running this plugin from a has directly off of master to get support for a newer version of DJDT. However even on master there is not yet support for python 3. We're running into this when running runserver and when collecting static assets. A quick survey of the development team at edX shows that people are no longer actively using this debug panel.(14 responders all saying no to the question "In the last year have you used the mongo portion of the django-debug-toolbar in edx-platform?") This makes sense since the module store and courseware related developement is not very active right now. Given all this, I'm removing this toolbar for now and if we need it again we can survey what's available at that time to choose the best tool. --- cms/envs/devstack.py | 7 +------ lms/envs/devstack.py | 3 +-- requirements/edx/development.in | 3 --- requirements/edx/development.txt | 1 - 4 files changed, 2 insertions(+), 12 deletions(-) diff --git a/cms/envs/devstack.py b/cms/envs/devstack.py index 6229c13721..67cde0116d 100644 --- a/cms/envs/devstack.py +++ b/cms/envs/devstack.py @@ -71,7 +71,7 @@ CELERY_ALWAYS_EAGER = True ################################ DEBUG TOOLBAR ################################ -INSTALLED_APPS += ['debug_toolbar', 'debug_toolbar_mongo'] +INSTALLED_APPS += ['debug_toolbar'] MIDDLEWARE_CLASSES.append('debug_toolbar.middleware.DebugToolbarMiddleware') INTERNAL_IPS = ('127.0.0.1',) @@ -105,11 +105,6 @@ def should_show_debug_toolbar(request): return True -# To see stacktraces for MongoDB queries, set this to True. -# Stacktraces slow down page loads drastically (for pages with lots of queries). -DEBUG_TOOLBAR_MONGO_STACKTRACES = False - - ########################### API DOCS ################################# FEATURES['ENABLE_API_DOCS'] = True diff --git a/lms/envs/devstack.py b/lms/envs/devstack.py index f27969d765..4f8a9a7c53 100644 --- a/lms/envs/devstack.py +++ b/lms/envs/devstack.py @@ -66,7 +66,7 @@ DJFS = { ################################ DEBUG TOOLBAR ################################ -INSTALLED_APPS += ['debug_toolbar', 'debug_toolbar_mongo'] +INSTALLED_APPS += ['debug_toolbar'] MIDDLEWARE_CLASSES += [ 'lms.djangoapps.discussion.django_comment_client.utils.QueryCountDebugMiddleware', 'debug_toolbar.middleware.DebugToolbarMiddleware', @@ -83,7 +83,6 @@ DEBUG_TOOLBAR_PANELS = ( 'debug_toolbar.panels.sql.SQLPanel', 'debug_toolbar.panels.signals.SignalsPanel', 'debug_toolbar.panels.logging.LoggingPanel', - 'debug_toolbar_mongo.panel.MongoDebugPanel', # ProfilingPanel has been intentionally removed for default devstack.py # runtimes for performance reasons. If you wish to re-enable it in your # local development environment, please create a new settings file diff --git a/requirements/edx/development.in b/requirements/edx/development.in index 7cbf550c8d..3a199d85e2 100644 --- a/requirements/edx/development.in +++ b/requirements/edx/development.in @@ -20,6 +20,3 @@ pyinotify # More efficient checking for runserver relo sphinx==1.8.5 # Pinned because 2.0.0 release requires Python '>=3.5' but current Python is 2.7.12 vulture # Detects possible dead/unused code, used in scripts/find-dead-code.sh modernize # Used to make Python 2 code more modern with the intention of eventually porting it over to Python 3. - -# The officially released version of django-debug-toolbar-mongo doesn't support DJDT 1.x. This commit does. --e git+https://github.com/hmarr/django-debug-toolbar-mongo.git@b0686a76f1ce3532088c4aee6e76b9abe61cc808#egg=django-debug-toolbar-mongo==0.1.10 diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index ef279b1822..cebfb414fe 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -7,7 +7,6 @@ -e git+https://github.com/edx/acid-block.git@e46f9cda8a03e121a00c7e347084d142d22ebfb7#egg=acid-xblock -e common/lib/capa -e git+https://github.com/edx/codejail.git@ed3d36c27913254a23273da95ad627a1bbbffa44#egg=codejail --e git+https://github.com/hmarr/django-debug-toolbar-mongo.git@b0686a76f1ce3532088c4aee6e76b9abe61cc808#egg=django-debug-toolbar-mongo==0.1.10 -e git+https://github.com/edx/django-wiki.git@v0.0.23#egg=django-wiki -e git+https://github.com/edx/DoneXBlock.git@01a14f3bd80ae47dd08cdbbe2f88f3eb88d00fba#egg=done-xblock -e git+https://github.com/jazkarta/edx-jsme.git@690dbf75441fa91c7c4899df0b83d77f7deb5458#egg=edx-jsme