From 701f5fc1c6dec4652e8de26d205d53451c239207 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Thu, 23 Jul 2015 10:06:44 -0400 Subject: [PATCH] Optimize test course creation with bulk_operations. Use bulk operations to both minimize the number of writes to MongoDB, as well as to suppress signals which could trigger things like search indexing or course structure caching. --- .../xmodule/modulestore/tests/django_utils.py | 100 +++++++++--------- .../xmodule/modulestore/tests/factories.py | 21 ++-- .../xmodule/modulestore/tests/utils.py | 5 +- 3 files changed, 67 insertions(+), 59 deletions(-) diff --git a/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py b/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py index 2a05094abc..971e784d9f 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py @@ -371,54 +371,54 @@ class ModuleStoreTestCase(TestCase): """ Create an equivalent to the toy xml course """ -# with self.store.bulk_operations(self.store.make_course_key(org, course, run)): - self.toy_loc = self.create_sample_course( # pylint: disable=attribute-defined-outside-init - org, course, run, TOY_BLOCK_INFO_TREE, - { - "textbooks": [["Textbook", "https://s3.amazonaws.com/edx-textbooks/guttag_computation_v3/"]], - "wiki_slug": "toy", - "display_name": "Toy Course", - "graded": True, - "discussion_topics": {"General": {"id": "i4x-edX-toy-course-2012_Fall"}}, - "graceperiod": datetime.timedelta(days=2, seconds=21599), - "start": datetime.datetime(2015, 07, 17, 12, tzinfo=pytz.utc), - "xml_attributes": {"filename": ["course/2012_Fall.xml", "course/2012_Fall.xml"]}, - "pdf_textbooks": [ - { - "tab_title": "Sample Multi Chapter Textbook", - "id": "MyTextbook", - "chapters": [ - {"url": "/static/Chapter1.pdf", "title": "Chapter 1"}, - {"url": "/static/Chapter2.pdf", "title": "Chapter 2"} - ] - } - ], - "course_image": "just_a_test.jpg", - } - ) - with self.store.branch_setting(ModuleStoreEnum.Branch.draft_preferred, self.toy_loc): - self.store.create_item( - self.user.id, self.toy_loc, "about", block_id="short_description", - fields={"data": "A course about toys."} + with self.store.bulk_operations(self.store.make_course_key(org, course, run), emit_signals=False): + self.toy_loc = self.create_sample_course( # pylint: disable=attribute-defined-outside-init + org, course, run, TOY_BLOCK_INFO_TREE, + { + "textbooks": [["Textbook", "https://s3.amazonaws.com/edx-textbooks/guttag_computation_v3/"]], + "wiki_slug": "toy", + "display_name": "Toy Course", + "graded": True, + "discussion_topics": {"General": {"id": "i4x-edX-toy-course-2012_Fall"}}, + "graceperiod": datetime.timedelta(days=2, seconds=21599), + "start": datetime.datetime(2015, 07, 17, 12, tzinfo=pytz.utc), + "xml_attributes": {"filename": ["course/2012_Fall.xml", "course/2012_Fall.xml"]}, + "pdf_textbooks": [ + { + "tab_title": "Sample Multi Chapter Textbook", + "id": "MyTextbook", + "chapters": [ + {"url": "/static/Chapter1.pdf", "title": "Chapter 1"}, + {"url": "/static/Chapter2.pdf", "title": "Chapter 2"} + ] + } + ], + "course_image": "just_a_test.jpg", + } ) - self.store.create_item( - self.user.id, self.toy_loc, "about", block_id="effort", - fields={"data": "6 hours"} - ) - self.store.create_item( - self.user.id, self.toy_loc, "about", block_id="end_date", - fields={"data": "TBD"} - ) - self.store.create_item( - self.user.id, self.toy_loc, "course_info", "handouts", - fields={"data": "Sample"} - ) - self.store.create_item( - self.user.id, self.toy_loc, "static_tab", "resources", - fields={"display_name": "Resources"}, - ) - self.store.create_item( - self.user.id, self.toy_loc, "static_tab", "syllabus", - fields={"display_name": "Syllabus"}, - ) - return self.toy_loc + with self.store.branch_setting(ModuleStoreEnum.Branch.draft_preferred, self.toy_loc): + self.store.create_item( + self.user.id, self.toy_loc, "about", block_id="short_description", + fields={"data": "A course about toys."} + ) + self.store.create_item( + self.user.id, self.toy_loc, "about", block_id="effort", + fields={"data": "6 hours"} + ) + self.store.create_item( + self.user.id, self.toy_loc, "about", block_id="end_date", + fields={"data": "TBD"} + ) + self.store.create_item( + self.user.id, self.toy_loc, "course_info", "handouts", + fields={"data": "Sample"} + ) + self.store.create_item( + self.user.id, self.toy_loc, "static_tab", "resources", + fields={"display_name": "Resources"}, + ) + self.store.create_item( + self.user.id, self.toy_loc, "static_tab", "syllabus", + fields={"display_name": "Syllabus"}, + ) + return self.toy_loc diff --git a/common/lib/xmodule/xmodule/modulestore/tests/factories.py b/common/lib/xmodule/xmodule/modulestore/tests/factories.py index df5de468b0..8748591c55 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/factories.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/factories.py @@ -99,7 +99,11 @@ class CourseFactory(XModuleFactory): # pylint: disable=unused-argument @classmethod def _create(cls, target_class, **kwargs): - + """ + Create and return a new course. For performance reasons, we do not emit + signals during this process, but if you need signals to run, you can + pass `emit_signals=True` to this method. + """ # All class attributes (from this class and base classes) are # passed in via **kwargs. However, some of those aren't actual field values, # so pop those off for use separately @@ -110,20 +114,23 @@ class CourseFactory(XModuleFactory): name = kwargs.get('name', kwargs.get('run', Location.clean(kwargs.get('display_name')))) run = kwargs.pop('run', name) user_id = kwargs.pop('user_id', ModuleStoreEnum.UserID.test) + emit_signals = kwargs.get('emit_signals', False) # Pass the metadata just as field=value pairs kwargs.update(kwargs.pop('metadata', {})) default_store_override = kwargs.pop('default_store', None) with store.branch_setting(ModuleStoreEnum.Branch.draft_preferred): - if default_store_override is not None: - with store.default_store(default_store_override): + course_key = store.make_course_key(org, number, run) + with store.bulk_operations(course_key, emit_signals=emit_signals): + if default_store_override is not None: + with store.default_store(default_store_override): + new_course = store.create_course(org, number, run, user_id, fields=kwargs) + else: new_course = store.create_course(org, number, run, user_id, fields=kwargs) - else: - new_course = store.create_course(org, number, run, user_id, fields=kwargs) - last_course.loc = new_course.location - return new_course + last_course.loc = new_course.location + return new_course class LibraryFactory(XModuleFactory): diff --git a/common/lib/xmodule/xmodule/modulestore/tests/utils.py b/common/lib/xmodule/xmodule/modulestore/tests/utils.py index 60994e4993..aa8832ba10 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/utils.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/utils.py @@ -149,7 +149,7 @@ class ProceduralCourseTestMixin(object): """ Contains methods for testing courses generated procedurally """ - def populate_course(self, branching=2): + def populate_course(self, branching=2, emit_signals=False): """ Add k chapters, k^2 sections, k^3 verticals, k^4 problems to self.course (where k = branching) """ @@ -172,4 +172,5 @@ class ProceduralCourseTestMixin(object): ) descend(child, stack[1:]) - descend(self.course, ['chapter', 'sequential', 'vertical', 'problem']) + with self.store.bulk_operations(self.course.id, emit_signals=emit_signals): + descend(self.course, ['chapter', 'sequential', 'vertical', 'problem'])