Merge pull request #15658 from edx/efischer/fix_leaks

CMS memory leak fix

cms/djangoapps/contentstore/tests/test_course_listing.py

@@ -318,7 +318,7 @@ class TestCourseListing(ModuleStoreTestCase, XssTestMixin):
         )
 
     @ddt.data(
-        (ModuleStoreEnum.Type.split, 3, 13),
+        (ModuleStoreEnum.Type.split, 4, 23),
         (ModuleStoreEnum.Type.mongo, USER_COURSES_COUNT, 2)
     )
     @ddt.unpack

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -7,7 +7,6 @@ import logging
 import re
 import json
 import datetime
-import traceback
 
 from pytz import UTC
 from collections import defaultdict
@@ -158,19 +157,7 @@ class ActiveBulkThread(threading.local):
     """
     def __init__(self, bulk_ops_record_type, **kwargs):
         super(ActiveBulkThread, self).__init__(**kwargs)
-        self._records = defaultdict(bulk_ops_record_type)
-        self.CMS_LEAK_DEBUG_GLOBAL = True  # only log once per process
-
-    @property
-    def records(self):
-        if self.CMS_LEAK_DEBUG_GLOBAL and len(self._records) > 2000:  # arbitrary limit, we peak around ~2750 on edx.org
-            log.info(
-                "EDUCATOR-768: The memory leak issue may be in progress. How we got here:\n{}".format(
-                    "".join(traceback.format_stack())
-                )
-            )
-            self.CMS_LEAK_DEBUG_GLOBAL = False
-        return self._records
+        self.records = defaultdict(bulk_ops_record_type)
 
 
 class BulkOperationsMixin(object):

common/lib/xmodule/xmodule/modulestore/split_mongo/split.py

@@ -788,7 +788,8 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
         if should_cache_items:
             self.cache_items(runtime, block_keys, course_entry.course_key, depth, lazy)
 
-        return [runtime.load_item(block_key, course_entry, **kwargs) for block_key in block_keys]
+        with self.bulk_operations(course_entry.course_key, emit_signals=False):
+            return [runtime.load_item(block_key, course_entry, **kwargs) for block_key in block_keys]
 
     def _get_cache(self, course_version_guid):
         """

common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py

@@ -418,7 +418,7 @@ class TestMixedModuleStore(CommonMixedModuleStoreSetup):
     #    wildcard query, 6! load pertinent items for inheritance calls, load parents, course root fetch (why)
     # Split:
     #    active_versions (with regex), structure, and spurious active_versions refetch
-    @ddt.data((ModuleStoreEnum.Type.mongo, 14, 0), (ModuleStoreEnum.Type.split, 3, 0))
+    @ddt.data((ModuleStoreEnum.Type.mongo, 14, 0), (ModuleStoreEnum.Type.split, 4, 0))
     @ddt.unpack
     def test_get_items(self, default_ms, max_find, max_send):
         self.initdb(default_ms)
@@ -1043,7 +1043,7 @@ class TestMixedModuleStore(CommonMixedModuleStoreSetup):
     #   1) wildcard split search,
     #   2-4) active_versions, structure, definition (s/b lazy; so, unnecessary)
     #   5) wildcard draft mongo which has none
-    @ddt.data((ModuleStoreEnum.Type.mongo, 3, 0), (ModuleStoreEnum.Type.split, 5, 0))
+    @ddt.data((ModuleStoreEnum.Type.mongo, 3, 0), (ModuleStoreEnum.Type.split, 6, 0))
     @ddt.unpack
     def test_get_courses(self, default_ms, max_find, max_send):
         self.initdb(default_ms)

lms/djangoapps/django_comment_client/base/tests.py

@@ -405,7 +405,7 @@ class ViewsQueryCountTestCase(
 
     @ddt.data(
         (ModuleStoreEnum.Type.mongo, 3, 4, 31),
-        (ModuleStoreEnum.Type.split, 3, 12, 31),
+        (ModuleStoreEnum.Type.split, 3, 13, 31),
     )
     @ddt.unpack
     @count_queries
@@ -414,7 +414,7 @@ class ViewsQueryCountTestCase(
 
     @ddt.data(
         (ModuleStoreEnum.Type.mongo, 3, 3, 27),
-        (ModuleStoreEnum.Type.split, 3, 9, 27),
+        (ModuleStoreEnum.Type.split, 3, 10, 27),
     )
     @ddt.unpack
     @count_queries