From 8eb5945e3902ab3bc2f3b7663c2aa52be499e337 Mon Sep 17 00:00:00 2001 From: John Eskew Date: Tue, 12 Sep 2017 15:57:48 -0400 Subject: [PATCH 1/7] Move Django requirement into separate file to enable tox testing. --- circle.yml | 1 + pavelib/prereqs.py | 1 + requirements/edx/base.txt | 1 - requirements/edx/django.txt | 1 + requirements/monitoring/requirements.txt | 1 + 5 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 requirements/edx/django.txt diff --git a/circle.yml b/circle.yml index b970851c3a..09fefa5a74 100644 --- a/circle.yml +++ b/circle.yml @@ -27,6 +27,7 @@ dependencies: # dependency on a version range of pbr. # Install a version which falls within that range. - pip install --exists-action w pbr==0.9.0 + - pip install --exists-action w -r requirements/edx/django.txt - pip install --exists-action w -r requirements/edx/base.txt - pip install --exists-action w -r requirements/edx/paver.txt - pip install --exists-action w -r requirements/edx/testing.txt diff --git a/pavelib/prereqs.py b/pavelib/prereqs.py index 9571e6dd9e..8d123d857f 100644 --- a/pavelib/prereqs.py +++ b/pavelib/prereqs.py @@ -26,6 +26,7 @@ PYTHON_REQ_FILES = [ 'requirements/edx/pre.txt', 'requirements/edx/github.txt', 'requirements/edx/local.txt', + 'requirements/edx/django.txt', 'requirements/edx/base.txt', 'requirements/edx/paver.txt', 'requirements/edx/development.txt', diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 82ee0d1c69..5371d24d0f 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -34,7 +34,6 @@ django-statici18n==1.4.0 django-storages==1.4.1 django-method-override==0.1.0 django-user-tasks==0.1.5 -django==1.8.18 django-waffle==0.12.0 djangorestframework-jwt==1.11.0 enum34==1.1.6 diff --git a/requirements/edx/django.txt b/requirements/edx/django.txt new file mode 100644 index 0000000000..1dc7e115af --- /dev/null +++ b/requirements/edx/django.txt @@ -0,0 +1 @@ +Django==1.8.18 diff --git a/requirements/monitoring/requirements.txt b/requirements/monitoring/requirements.txt index 5971db5ac6..28cbb66860 100644 --- a/requirements/monitoring/requirements.txt +++ b/requirements/monitoring/requirements.txt @@ -9,6 +9,7 @@ # Requirements for the web application -r ../edx/pre.txt +-r ../edx/django.txt -r ../edx/base.txt -r ../edx/local.txt -r ../edx/github.txt From 9d93c7508902cea4f8f63b1a35ea82d89b14c0e7 Mon Sep 17 00:00:00 2001 From: John Eskew Date: Tue, 12 Sep 2017 15:58:24 -0400 Subject: [PATCH 2/7] Remove unused EASE requirement. --- requirements/edx/github.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/edx/github.txt b/requirements/edx/github.txt index 5ec2053613..8e0adca171 100644 --- a/requirements/edx/github.txt +++ b/requirements/edx/github.txt @@ -86,7 +86,6 @@ git+https://github.com/edx/django-celery.git@f87c6f914a1410463f54aebf68458c0653b -e git+https://github.com/edx/django-splash.git@v0.2#egg=django-splash==0.2 -e git+https://github.com/edx/acid-block.git@e46f9cda8a03e121a00c7e347084d142d22ebfb7#egg=acid-xblock git+https://github.com/edx/edx-ora2.git@2.1.1#egg=ora2==2.1.1 -git+https://github.com/edx/ease.git@release-2015-07-14#egg=ease==0.1.3 git+https://github.com/edx/RecommenderXBlock.git@0e744b393cf1f8b886fe77bc697e7d9d78d65cd6#egg=recommender-xblock==1.2 git+https://github.com/solashirai/crowdsourcehinter.git@518605f0a95190949fe77bd39158450639e2e1dc#egg=crowdsourcehinter-xblock==0.1 -e git+https://github.com/edx/RateXBlock.git@367e19c0f6eac8a5f002fd0f1559555f8e74bfff#egg=rate-xblock From 2194d390c8cdd59e454b791e0c5591d2dc729207 Mon Sep 17 00:00:00 2001 From: John Eskew Date: Tue, 12 Sep 2017 16:01:20 -0400 Subject: [PATCH 3/7] Remove memory dump signal until meliae packaging issues are resolved. --- cms/wsgi.py | 3 --- lms/wsgi.py | 3 --- openedx/core/operations.py | 40 -------------------------------------- requirements/edx/base.txt | 3 --- requirements/edx/pre.txt | 3 --- 5 files changed, 52 deletions(-) delete mode 100644 openedx/core/operations.py diff --git a/cms/wsgi.py b/cms/wsgi.py index aa760d0782..b46d824c46 100644 --- a/cms/wsgi.py +++ b/cms/wsgi.py @@ -6,9 +6,6 @@ defuse_xml_libs() import contracts contracts.disable_all() -import openedx.core.operations -openedx.core.operations.install_memory_dumper() - import os os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cms.envs.aws") diff --git a/lms/wsgi.py b/lms/wsgi.py index 9ac8309669..d71a2c26eb 100644 --- a/lms/wsgi.py +++ b/lms/wsgi.py @@ -16,9 +16,6 @@ defuse_xml_libs() import contracts contracts.disable_all() -import openedx.core.operations -openedx.core.operations.install_memory_dumper() - import os os.environ.setdefault("DJANGO_SETTINGS_MODULE", "lms.envs.aws") diff --git a/openedx/core/operations.py b/openedx/core/operations.py deleted file mode 100644 index 87721c843a..0000000000 --- a/openedx/core/operations.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -Workflows useful for reporting on runtime characteristics of the system -""" -import gc -import os -import signal -import tempfile -from datetime import datetime - -from meliae import scanner - - -def dump_memory(signum, frame): - """ - Dump memory stats for the current process to a temp directory. - Uses the meliae output format. - """ - - timestamp = datetime.now().isoformat() - format_str = '{}/meliae.{}.{}.{{}}.dump'.format( - tempfile.gettempdir(), - timestamp, - os.getpid(), - ) - - scanner.dump_all_objects(format_str.format('pre-gc')) - - # force garbarge collection - for gen in xrange(3): - gc.collect(gen) - scanner.dump_all_objects( - format_str.format("gc-gen-{}".format(gen)) - ) - - -def install_memory_dumper(dump_signal=signal.SIGPROF): - """ - Install a signal handler on `signal` to dump memory stats for the current process. - """ - signal.signal(dump_signal, dump_memory) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 5371d24d0f..66fdaa88e3 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -68,9 +68,6 @@ httpretty==0.8.3 lazy==1.1 mako==1.0.2 Markdown>=2.6,<2.7 ---allow-external meliae ---allow-unverified meliae -meliae==0.4.0 mongoengine==0.10.0 MySQL-python==1.2.5 networkx==1.7 diff --git a/requirements/edx/pre.txt b/requirements/edx/pre.txt index 2005ea12b8..2dd4a0f168 100644 --- a/requirements/edx/pre.txt +++ b/requirements/edx/pre.txt @@ -12,6 +12,3 @@ pip==9.0.1 # Numpy and scipy can't be installed in the same pip run. # Install numpy before other things to help resolve the problem. numpy==1.6.2 - -# Needed for meliae -Cython==0.21.2 From 550c1e0725567fa1e8a58ccda5fd77db22481b61 Mon Sep 17 00:00:00 2001 From: John Eskew Date: Thu, 14 Sep 2017 10:45:11 -0400 Subject: [PATCH 4/7] Use simpler contract spec to fix parsing error. --- common/lib/xmodule/xmodule/modulestore/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/lib/xmodule/xmodule/modulestore/__init__.py b/common/lib/xmodule/xmodule/modulestore/__init__.py index e3502d494f..6fd1d96437 100644 --- a/common/lib/xmodule/xmodule/modulestore/__init__.py +++ b/common/lib/xmodule/xmodule/modulestore/__init__.py @@ -602,7 +602,7 @@ class ModuleStoreAssetBase(object): @contract( course_key='CourseKey', asset_type='None | basestring', - start='int | None', maxresults='int | None', sort='tuple(str,(int,>=1,<=2))|None' + start='int | None', maxresults='int | None', sort='tuple(str,int) | None' ) def get_all_asset_metadata(self, course_key, asset_type, start=0, maxresults=-1, sort=None, **kwargs): """ From 617085b5d116b09539053f2ecc65a21b8f924981 Mon Sep 17 00:00:00 2001 From: John Eskew Date: Thu, 14 Sep 2017 11:13:22 -0400 Subject: [PATCH 5/7] Move psutil req from paver to base - used in non-test code. --- requirements/edx/base.txt | 1 + requirements/edx/paver.txt | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 66fdaa88e3..040c429681 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -77,6 +77,7 @@ path.py==8.2.1 piexif==1.0.2 Pillow==3.4 polib==1.0.3 +psutil==1.2.1 pycrypto>=2.6 pygments==2.2.0 pygraphviz==1.1 diff --git a/requirements/edx/paver.txt b/requirements/edx/paver.txt index 287dfd09cf..d0d74d58fd 100644 --- a/requirements/edx/paver.txt +++ b/requirements/edx/paver.txt @@ -1,6 +1,5 @@ -# Requirements to run Paver +# Requirements to run and test Paver Paver==1.2.4 -psutil==1.2.1 lazy==1.1 path.py==8.2.1 watchdog==0.8.3 From 4de2ad4c25a74ef8234a67487c6dcd8cd5dd904a Mon Sep 17 00:00:00 2001 From: John Eskew Date: Thu, 14 Sep 2017 11:14:33 -0400 Subject: [PATCH 6/7] Upgrade lxml to use pre-built pkg - no longer requires Cython. --- common/lib/capa/capa/responsetypes.py | 3 +-- requirements/edx/base.txt | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index f1d2403dc3..da7bd34899 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -2468,8 +2468,7 @@ class CustomResponse(LoncapaResponse): msg = msg.replace('<', '<') # Use etree to prettify the HTML - msg = etree.tostring(fromstring_bs(msg, convertEntities=None), - pretty_print=True) + msg = etree.tostring(fromstring_bs(msg), pretty_print=True) msg = msg.replace(' ', '') diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 040c429681..f32a9bbd8a 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -115,8 +115,7 @@ pyuca==1.1 wrapt==1.10.5 zendesk==1.1.1 -# This needs to be installed *after* Cython, which is in pre.txt -lxml==3.4.4 +lxml==3.8.0 # Used for shopping cart's pdf invoice/receipt generation reportlab==3.1.44 From be14aef51c271e309d0d854238bca1df5229a335 Mon Sep 17 00:00:00 2001 From: John Eskew Date: Tue, 12 Sep 2017 16:09:32 -0400 Subject: [PATCH 7/7] Add tox.ini to enable tox testing. --- .gitignore | 1 + tox.ini | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index c3e36ba796..f1096ef85a 100644 --- a/.gitignore +++ b/.gitignore @@ -81,6 +81,7 @@ test_root/export_course_repos/ test_root/paver_logs/ test_root/uploads/ django-pyfs +.tox/ ### Installation artifacts *.egg-info diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000000..728c8ec163 --- /dev/null +++ b/tox.ini @@ -0,0 +1,22 @@ +[tox] +envlist = py27-django{18,111} + +[testenv] +setenv = + PYTHONHASHSEED = 0 +passenv = + EDX_PLATFORM_SETTINGS + EDXAPP_TEST_MONGO_HOST +deps = + django18: Django>=1.8,<1.9 + django111: Django>=1.11,<2 + -rrequirements/edx/pre.txt + -rrequirements/edx/github.txt + -rrequirements/edx/local.txt + -rrequirements/edx/base.txt + -rrequirements/edx/development.txt + -rrequirements/edx/testing.txt + -rrequirements/edx/post.txt + +commands = + pytest {posargs}