From e4af7287b6f204dc759f1e9a349bf29a6864a025 Mon Sep 17 00:00:00 2001 From: JonahStanley Date: Tue, 18 Jun 2013 14:04:43 -0400 Subject: [PATCH 1/9] Initial testing for parallelization --- cms/djangoapps/contentstore/tests/test_contentstore.py | 6 ++++++ cms/envs/test.py | 2 +- .../lib/xmodule/xmodule/modulestore/tests/django_utils.py | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cms/djangoapps/contentstore/tests/test_contentstore.py b/cms/djangoapps/contentstore/tests/test_contentstore.py index d24deacecf..86699ef479 100644 --- a/cms/djangoapps/contentstore/tests/test_contentstore.py +++ b/cms/djangoapps/contentstore/tests/test_contentstore.py @@ -43,10 +43,13 @@ from django_comment_common.utils import are_permissions_roles_seeded from xmodule.exceptions import InvalidVersionError import datetime from pytz import UTC +#from uuid import uuid4 TEST_DATA_MODULESTORE = copy.deepcopy(settings.MODULESTORE) TEST_DATA_MODULESTORE['default']['OPTIONS']['fs_root'] = path('common/test/data') TEST_DATA_MODULESTORE['direct']['OPTIONS']['fs_root'] = path('common/test/data') +TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE) +TEST_DATA_CONTENTSTORE['OPTIONS']['db'] = 'test_xcontent_%s' % 4 #uuid4().hex class MongoCollectionFindWrapper(object): @@ -60,6 +63,7 @@ class MongoCollectionFindWrapper(object): @override_settings(MODULESTORE=TEST_DATA_MODULESTORE) +@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE) class ContentStoreToyCourseTest(ModuleStoreTestCase): """ Tests that rely on the toy courses. @@ -83,6 +87,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): self.client = Client() self.client.login(username=uname, password=password) + def check_components_on_page(self, component_types, expected_types): """ Ensure that the right types end up on the page. @@ -809,6 +814,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): export_to_xml(module_store, content_store, location, root_dir, 'test_export') +@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE) class ContentStoreTest(ModuleStoreTestCase): """ Tests for the CMS ContentStore application. diff --git a/cms/envs/test.py b/cms/envs/test.py index 954a553e10..89813dd937 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -70,7 +70,7 @@ CONTENTSTORE = { 'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore', 'OPTIONS': { 'host': 'localhost', - 'db': 'test_xmodule', + 'db': 'test_xcontent', }, # allow for additional options that can be keyed on a name, e.g. 'trashcan' 'ADDITIONAL_OPTIONS': { diff --git a/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py b/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py index 04e79ce521..e0e5c1a46f 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py @@ -27,6 +27,7 @@ class ModuleStoreTestCase(TestCase): # Remove everything except templates modulestore.collection.remove(query) + modulestore.collection.drop() @staticmethod def load_templates_if_necessary(): From f90ed69cd792091c9f2d57bce1cbafe0efd51094 Mon Sep 17 00:00:00 2001 From: Jay Zoldak Date: Tue, 18 Jun 2013 15:09:53 -0400 Subject: [PATCH 2/9] move override of MODULESTORE settings into ModuleStore test case class --- cms/djangoapps/contentstore/tests/test_contentstore.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cms/djangoapps/contentstore/tests/test_contentstore.py b/cms/djangoapps/contentstore/tests/test_contentstore.py index 86699ef479..9c3ec2e3ba 100644 --- a/cms/djangoapps/contentstore/tests/test_contentstore.py +++ b/cms/djangoapps/contentstore/tests/test_contentstore.py @@ -45,9 +45,7 @@ import datetime from pytz import UTC #from uuid import uuid4 -TEST_DATA_MODULESTORE = copy.deepcopy(settings.MODULESTORE) -TEST_DATA_MODULESTORE['default']['OPTIONS']['fs_root'] = path('common/test/data') -TEST_DATA_MODULESTORE['direct']['OPTIONS']['fs_root'] = path('common/test/data') + TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE) TEST_DATA_CONTENTSTORE['OPTIONS']['db'] = 'test_xcontent_%s' % 4 #uuid4().hex @@ -62,7 +60,7 @@ class MongoCollectionFindWrapper(object): return self.original(query, *args, **kwargs) -@override_settings(MODULESTORE=TEST_DATA_MODULESTORE) +#@override_settings(MODULESTORE=TEST_DATA_MODULESTORE) @override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE) class ContentStoreToyCourseTest(ModuleStoreTestCase): """ @@ -70,6 +68,9 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): TODO: refactor using CourseFactory so they do not. """ def setUp(self): + + settings.MODULESTORE['default']['OPTIONS']['fs_root'] = path('common/test/data') + settings.MODULESTORE['direct']['OPTIONS']['fs_root'] = path('common/test/data') uname = 'testuser' email = 'test+courses@edx.org' password = 'foo' @@ -88,6 +89,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): self.client.login(username=uname, password=password) + def check_components_on_page(self, component_types, expected_types): """ Ensure that the right types end up on the page. From 51f8c0cfebedb7807b04ef849cfb806a0dcdba0e Mon Sep 17 00:00:00 2001 From: JonahStanley Date: Wed, 19 Jun 2013 11:27:22 -0400 Subject: [PATCH 3/9] Added the beginnings of self cleanup --- .../contentstore/tests/test_contentstore.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/cms/djangoapps/contentstore/tests/test_contentstore.py b/cms/djangoapps/contentstore/tests/test_contentstore.py index 9c3ec2e3ba..46d6a069ce 100644 --- a/cms/djangoapps/contentstore/tests/test_contentstore.py +++ b/cms/djangoapps/contentstore/tests/test_contentstore.py @@ -43,11 +43,12 @@ from django_comment_common.utils import are_permissions_roles_seeded from xmodule.exceptions import InvalidVersionError import datetime from pytz import UTC -#from uuid import uuid4 +from uuid import uuid4 +import pymongo TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE) -TEST_DATA_CONTENTSTORE['OPTIONS']['db'] = 'test_xcontent_%s' % 4 #uuid4().hex +TEST_DATA_CONTENTSTORE['OPTIONS']['db'] = 'test_xcontent_%s' % uuid4().hex class MongoCollectionFindWrapper(object): @@ -88,7 +89,10 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): self.client = Client() self.client.login(username=uname, password=password) - + def tearDown(self): + m = pymongo.MongoClient() + m.drop_database(TEST_DATA_CONTENTSTORE['OPTIONS']['db']) + #contentstore().fs_files.drop() def check_components_on_page(self, component_types, expected_types): """ @@ -449,7 +453,6 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): content_store = contentstore() trash_store = contentstore('trashcan') module_store = modulestore('direct') - import_from_xml(module_store, 'common/test/data/', ['full'], static_content_store=content_store) # look up original (and thumbnail) in content store, should be there after import @@ -853,6 +856,11 @@ class ContentStoreTest(ModuleStoreTestCase): 'display_name': 'Robot Super Course', } + def tearDown(self): + m = pymongo.MongoClient() + m.drop_database(TEST_DATA_CONTENTSTORE['OPTIONS']['db']) + #contentstore().fs_files.drop() + def test_create_course(self): """Test new course creation - happy path""" resp = self.client.post(reverse('create_new_course'), self.course_data) From fa18b48f6eaec45bc65f16f9585fa2555462ad55 Mon Sep 17 00:00:00 2001 From: JonahStanley Date: Thu, 20 Jun 2013 09:05:35 -0400 Subject: [PATCH 4/9] Contentstore singleton is now cleared during teardown --- cms/djangoapps/contentstore/tests/test_contentstore.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cms/djangoapps/contentstore/tests/test_contentstore.py b/cms/djangoapps/contentstore/tests/test_contentstore.py index 46d6a069ce..b0cbcee032 100644 --- a/cms/djangoapps/contentstore/tests/test_contentstore.py +++ b/cms/djangoapps/contentstore/tests/test_contentstore.py @@ -41,6 +41,7 @@ from xmodule.exceptions import NotFoundError from django_comment_common.utils import are_permissions_roles_seeded from xmodule.exceptions import InvalidVersionError +import xmodule.contentstore.django import datetime from pytz import UTC from uuid import uuid4 @@ -92,7 +93,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): def tearDown(self): m = pymongo.MongoClient() m.drop_database(TEST_DATA_CONTENTSTORE['OPTIONS']['db']) - #contentstore().fs_files.drop() + xmodule.contentstore.django._CONTENTSTORE.clear() def check_components_on_page(self, component_types, expected_types): """ @@ -859,7 +860,7 @@ class ContentStoreTest(ModuleStoreTestCase): def tearDown(self): m = pymongo.MongoClient() m.drop_database(TEST_DATA_CONTENTSTORE['OPTIONS']['db']) - #contentstore().fs_files.drop() + xmodule.contentstore.django._CONTENTSTORE.clear() def test_create_course(self): """Test new course creation - happy path""" From cb04f9f0b82dfe46777ceb0584fadb656f7b6780 Mon Sep 17 00:00:00 2001 From: JonahStanley Date: Thu, 20 Jun 2013 17:10:36 -0400 Subject: [PATCH 5/9] Moved port range to rake file --- jenkins/test.sh | 3 --- rakelib/tests.rake | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/jenkins/test.sh b/jenkins/test.sh index e5ac4f6f71..2ff32a9911 100755 --- a/jenkins/test.sh +++ b/jenkins/test.sh @@ -60,9 +60,6 @@ fi export PIP_DOWNLOAD_CACHE=/mnt/pip-cache -# Allow django liveserver tests to use a range of ports -export DJANGO_LIVE_TEST_SERVER_ADDRESS=${DJANGO_LIVE_TEST_SERVER_ADDRESS-localhost:8000-9000} - source /mnt/virtualenvs/"$JOB_NAME"/bin/activate bundle install diff --git a/rakelib/tests.rake b/rakelib/tests.rake index f169d28256..c0592cca7a 100644 --- a/rakelib/tests.rake +++ b/rakelib/tests.rake @@ -16,7 +16,7 @@ def run_tests(system, report_dir, test_id=nil, stop_on_failure=true) ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml") dirs = Dir["common/djangoapps/*"] + Dir["#{system}/djangoapps/*"] test_id = dirs.join(' ') if test_id.nil? or test_id == '' - cmd = django_admin(system, :test, 'test', '--logging-clear-handlers', test_id) + cmd = django_admin(system, :test, 'test', '--logging-clear-handlers', '--liveserver=localhost:8000-9000', test_id) test_sh(run_under_coverage(cmd, system)) end From 58fe6d4e8367c570648068d3307cc3105a88edf5 Mon Sep 17 00:00:00 2001 From: JonahStanley Date: Fri, 21 Jun 2013 16:17:33 -0400 Subject: [PATCH 6/9] Cleaned up import and comment --- cms/djangoapps/contentstore/tests/test_contentstore.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cms/djangoapps/contentstore/tests/test_contentstore.py b/cms/djangoapps/contentstore/tests/test_contentstore.py index b0cbcee032..6d2055d459 100644 --- a/cms/djangoapps/contentstore/tests/test_contentstore.py +++ b/cms/djangoapps/contentstore/tests/test_contentstore.py @@ -45,7 +45,7 @@ import xmodule.contentstore.django import datetime from pytz import UTC from uuid import uuid4 -import pymongo +from pymongo import MongoClient TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE) @@ -62,7 +62,6 @@ class MongoCollectionFindWrapper(object): return self.original(query, *args, **kwargs) -#@override_settings(MODULESTORE=TEST_DATA_MODULESTORE) @override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE) class ContentStoreToyCourseTest(ModuleStoreTestCase): """ @@ -91,7 +90,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): self.client.login(username=uname, password=password) def tearDown(self): - m = pymongo.MongoClient() + m = MongoClient() m.drop_database(TEST_DATA_CONTENTSTORE['OPTIONS']['db']) xmodule.contentstore.django._CONTENTSTORE.clear() @@ -858,7 +857,7 @@ class ContentStoreTest(ModuleStoreTestCase): } def tearDown(self): - m = pymongo.MongoClient() + m = MongoClient() m.drop_database(TEST_DATA_CONTENTSTORE['OPTIONS']['db']) xmodule.contentstore.django._CONTENTSTORE.clear() From 5e6de488abaa45f765b5aef48a1b36851a673be1 Mon Sep 17 00:00:00 2001 From: JonahStanley Date: Fri, 21 Jun 2013 16:28:32 -0400 Subject: [PATCH 7/9] Fixed pylint/pep8 violations --- .../contentstore/tests/test_contentstore.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/cms/djangoapps/contentstore/tests/test_contentstore.py b/cms/djangoapps/contentstore/tests/test_contentstore.py index 6d2055d459..514b631521 100644 --- a/cms/djangoapps/contentstore/tests/test_contentstore.py +++ b/cms/djangoapps/contentstore/tests/test_contentstore.py @@ -90,8 +90,8 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): self.client.login(username=uname, password=password) def tearDown(self): - m = MongoClient() - m.drop_database(TEST_DATA_CONTENTSTORE['OPTIONS']['db']) + mongo = MongoClient() + mongo.drop_database(TEST_DATA_CONTENTSTORE['OPTIONS']['db']) xmodule.contentstore.django._CONTENTSTORE.clear() def check_components_on_page(self, component_types, expected_types): @@ -414,7 +414,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): self.assertGreater(len(all_assets), 0) # make sure we have some thumbnails in our contentstore - all_thumbnails = content_store.get_all_content_thumbnails_for_course(course_location) + content_store.get_all_content_thumbnails_for_course(course_location) # # cdodge: temporarily comment out assertion on thumbnails because many environments @@ -543,7 +543,6 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): all_assets = trash_store.get_all_content_for_course(course_location) self.assertEqual(len(all_assets), 0) - all_thumbnails = trash_store.get_all_content_thumbnails_for_course(course_location) self.assertEqual(len(all_thumbnails), 0) @@ -608,7 +607,6 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): self.assertRaises(InvalidVersionError, draft_store.unpublish, location) - def test_bad_contentstore_request(self): resp = self.client.get('http://localhost:8001/c4x/CDX/123123/asset/&images_circuits_Lab7Solution2.png') self.assertEqual(resp.status_code, 400) @@ -857,8 +855,8 @@ class ContentStoreTest(ModuleStoreTestCase): } def tearDown(self): - m = MongoClient() - m.drop_database(TEST_DATA_CONTENTSTORE['OPTIONS']['db']) + mongo = MongoClient() + mongo.drop_database(TEST_DATA_CONTENTSTORE['OPTIONS']['db']) xmodule.contentstore.django._CONTENTSTORE.clear() def test_create_course(self): From 3f9a72e6ce805a63d091cc387b44021d079d46c4 Mon Sep 17 00:00:00 2001 From: JonahStanley Date: Fri, 21 Jun 2013 16:32:13 -0400 Subject: [PATCH 8/9] Consolidated imports --- cms/djangoapps/contentstore/tests/test_contentstore.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cms/djangoapps/contentstore/tests/test_contentstore.py b/cms/djangoapps/contentstore/tests/test_contentstore.py index 514b631521..66fead562e 100644 --- a/cms/djangoapps/contentstore/tests/test_contentstore.py +++ b/cms/djangoapps/contentstore/tests/test_contentstore.py @@ -23,7 +23,7 @@ from xmodule.modulestore import Location from xmodule.modulestore.store_utilities import clone_course from xmodule.modulestore.store_utilities import delete_course from xmodule.modulestore.django import modulestore -from xmodule.contentstore.django import contentstore +from xmodule.contentstore.django import contentstore, _CONTENTSTORE from xmodule.templates import update_templates from xmodule.modulestore.xml_exporter import export_to_xml from xmodule.modulestore.xml_importer import import_from_xml, perform_xlint @@ -41,7 +41,6 @@ from xmodule.exceptions import NotFoundError from django_comment_common.utils import are_permissions_roles_seeded from xmodule.exceptions import InvalidVersionError -import xmodule.contentstore.django import datetime from pytz import UTC from uuid import uuid4 @@ -92,7 +91,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): def tearDown(self): mongo = MongoClient() mongo.drop_database(TEST_DATA_CONTENTSTORE['OPTIONS']['db']) - xmodule.contentstore.django._CONTENTSTORE.clear() + _CONTENTSTORE.clear() def check_components_on_page(self, component_types, expected_types): """ @@ -857,7 +856,7 @@ class ContentStoreTest(ModuleStoreTestCase): def tearDown(self): mongo = MongoClient() mongo.drop_database(TEST_DATA_CONTENTSTORE['OPTIONS']['db']) - xmodule.contentstore.django._CONTENTSTORE.clear() + _CONTENTSTORE.clear() def test_create_course(self): """Test new course creation - happy path""" From 8a9125f121a1b983b33d4c7f8cd16deeee8335cf Mon Sep 17 00:00:00 2001 From: JonahStanley Date: Tue, 25 Jun 2013 11:33:46 -0400 Subject: [PATCH 9/9] Test Mongo database is now unique and destroyed in teardown --- common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py index c5ef0d751a..44e69fb0ed 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py @@ -13,11 +13,12 @@ from xmodule.templates import update_templates from .test_modulestore import check_path_to_location from . import DATA_DIR +from uuid import uuid4 HOST = 'localhost' PORT = 27017 -DB = 'test' +DB = 'test_mongo_%s' % uuid4().hex COLLECTION = 'modulestore' FS_ROOT = DATA_DIR # TODO (vshnayder): will need a real fs_root for testing load_item DEFAULT_CLASS = 'xmodule.raw_module.RawDescriptor' @@ -39,7 +40,8 @@ class TestMongoModuleStore(object): @classmethod def teardownClass(cls): - pass + cls.connection = pymongo.connection.Connection(HOST, PORT) + cls.connection.drop_database(DB) @staticmethod def initdb():