From d4943c5337a26ceaf5adb238db13ab81fde077f7 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 25 Jul 2017 10:38:50 -0400 Subject: [PATCH] Use bulk_operations when loading items This has been causing a huge memory leak, wrapping the call in this context manager properly cleans up the 'records' entries. EDUCATOR-274 --- cms/djangoapps/contentstore/tests/test_course_listing.py | 2 +- common/lib/xmodule/xmodule/modulestore/split_mongo/split.py | 3 ++- .../xmodule/modulestore/tests/test_mixed_modulestore.py | 4 ++-- lms/djangoapps/django_comment_client/base/tests.py | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cms/djangoapps/contentstore/tests/test_course_listing.py b/cms/djangoapps/contentstore/tests/test_course_listing.py index 296b53bb61..8d66a64ff6 100644 --- a/cms/djangoapps/contentstore/tests/test_course_listing.py +++ b/cms/djangoapps/contentstore/tests/test_course_listing.py @@ -318,7 +318,7 @@ class TestCourseListing(ModuleStoreTestCase, XssTestMixin): ) @ddt.data( - (ModuleStoreEnum.Type.split, 3, 13), + (ModuleStoreEnum.Type.split, 4, 23), (ModuleStoreEnum.Type.mongo, USER_COURSES_COUNT, 2) ) @ddt.unpack diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py index 3e1a393f52..16c238ec4b 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py @@ -788,7 +788,8 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): if should_cache_items: self.cache_items(runtime, block_keys, course_entry.course_key, depth, lazy) - return [runtime.load_item(block_key, course_entry, **kwargs) for block_key in block_keys] + with self.bulk_operations(course_entry.course_key, emit_signals=False): + return [runtime.load_item(block_key, course_entry, **kwargs) for block_key in block_keys] def _get_cache(self, course_version_guid): """ diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py b/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py index 59917387f5..b25a07415b 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py @@ -418,7 +418,7 @@ class TestMixedModuleStore(CommonMixedModuleStoreSetup): # wildcard query, 6! load pertinent items for inheritance calls, load parents, course root fetch (why) # Split: # active_versions (with regex), structure, and spurious active_versions refetch - @ddt.data((ModuleStoreEnum.Type.mongo, 14, 0), (ModuleStoreEnum.Type.split, 3, 0)) + @ddt.data((ModuleStoreEnum.Type.mongo, 14, 0), (ModuleStoreEnum.Type.split, 4, 0)) @ddt.unpack def test_get_items(self, default_ms, max_find, max_send): self.initdb(default_ms) @@ -1043,7 +1043,7 @@ class TestMixedModuleStore(CommonMixedModuleStoreSetup): # 1) wildcard split search, # 2-4) active_versions, structure, definition (s/b lazy; so, unnecessary) # 5) wildcard draft mongo which has none - @ddt.data((ModuleStoreEnum.Type.mongo, 3, 0), (ModuleStoreEnum.Type.split, 5, 0)) + @ddt.data((ModuleStoreEnum.Type.mongo, 3, 0), (ModuleStoreEnum.Type.split, 6, 0)) @ddt.unpack def test_get_courses(self, default_ms, max_find, max_send): self.initdb(default_ms) diff --git a/lms/djangoapps/django_comment_client/base/tests.py b/lms/djangoapps/django_comment_client/base/tests.py index 35964b9524..23ccb09113 100644 --- a/lms/djangoapps/django_comment_client/base/tests.py +++ b/lms/djangoapps/django_comment_client/base/tests.py @@ -405,7 +405,7 @@ class ViewsQueryCountTestCase( @ddt.data( (ModuleStoreEnum.Type.mongo, 3, 4, 31), - (ModuleStoreEnum.Type.split, 3, 12, 31), + (ModuleStoreEnum.Type.split, 3, 13, 31), ) @ddt.unpack @count_queries @@ -414,7 +414,7 @@ class ViewsQueryCountTestCase( @ddt.data( (ModuleStoreEnum.Type.mongo, 3, 3, 27), - (ModuleStoreEnum.Type.split, 3, 9, 27), + (ModuleStoreEnum.Type.split, 3, 10, 27), ) @ddt.unpack @count_queries