Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: skip data caching for content more than or equal to 2MB (#36359)

Browse Source 
Co-authored-by: M Umar Khan <umar.khan@A006-01609.local>
This commit is contained in:
Muhammad Umar Khan
2025-03-12 17:46:31 +05:00
committed by GitHub
parent f5fde97ae8
commit c443acddd8
2 changed files with 17 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
xmodule/modulestore/split_mongo/mongo_connection.py
View File

@@ -248,17 +248,16 @@ class CourseStructureCache:
# We rely on the course structure cache default timeout, which should be
# high by default (~ a few days).
try:
total_bytes_in_one_mb = 1024 * 1024
if data_size < total_bytes_in_one_mb * 2: # Only data with a size smaller than 2MB will be cached
self.cache.set(key, compressed_pickled_data)
except Exception: # pylint: disable=broad-except
total_bytes_in_one_mb = 1024 * 1024
else:
chunk_size_in_mbs = round(data_size / total_bytes_in_one_mb, 2)
# .. custom_attribute_name: split_mongo_compressed_size
# .. custom_attribute_description: contains the data chunk size in MBs. The size on which
# the memcached client failed to store value in course structure cache.
monitoring.set_custom_attribute('split_mongo_compressed_size', chunk_size_in_mbs)
log.info('Data caching (course structure) failed on chunk size: {} MB'.format(chunk_size_in_mbs))
class MongoPersistenceBackend:

33
xmodule/modulestore/tests/test_split_modulestore.py
View File

@@ -16,7 +16,6 @@ import ddt
from ccx_keys.locator import CCXBlockUsageLocator
from django.core.cache import InvalidCacheBackendError, caches
from opaque_keys.edx.locator import BlockUsageLocator, CourseKey, CourseLocator, LocalId
from testfixtures import LogCapture
from xblock.fields import Reference, ReferenceList, ReferenceValueDict
from openedx.core.djangolib.testing.utils import CacheIsolationMixin
@@ -842,42 +841,38 @@ class TestCourseStructureCache(CacheIsolationMixin, SplitModuleTest):
# now make sure that you get the same structure
assert cached_structure == not_cached_structure
@patch('xmodule.modulestore.split_mongo.mongo_connection.monitoring.set_custom_attribute')
@patch('django.core.cache.cache.set')
@patch('xmodule.modulestore.split_mongo.mongo_connection.get_cache')
def test_course_structure_cache_with_data_chunk_greater_than_one_mb(self, mock_get_cache, mock_set_cache):
def test_course_structure_cache_with_data_chunk_greater_than_two_mb(self, mock_get_cache, mock_set_cache,
mock_set_custom_attribute):
enabled_cache = caches['default']
mock_get_cache.return_value = enabled_cache
mock_set_cache.side_effect = Exception
course_cache = CourseStructureCache()
size = 300000000
size = 3000000000
# this data_chunk will be compressed before being cached
data_chunk = b'\x00' * size
logger_name = 'xmodule.modulestore.split_mongo.mongo_connection'
expected_message = 'Data caching (course structure) failed on chunk size: 1.25 MB'
with LogCapture(logger_name) as capture:
course_cache.set('my_data_chunk', data_chunk)
self.assertEqual(capture.records[0].name, logger_name)
self.assertEqual(capture.records[0].msg, expected_message)
self.assertEqual(capture.records[0].levelname, 'INFO')
course_cache.set('my_data_chunk', data_chunk)
mock_set_cache.assert_not_called()
mock_set_custom_attribute.assert_called()
@patch('xmodule.modulestore.split_mongo.mongo_connection.monitoring.set_custom_attribute')
@patch('django.core.cache.cache.set')
@patch('xmodule.modulestore.split_mongo.mongo_connection.get_cache')
def test_course_structure_cache_with_data_chunk_lesser_than_one_mb(self, mock_get_cache):
def test_course_structure_cache_with_data_chunk_lesser_than_two_mb(self, mock_get_cache, mock_set_cache,
mock_set_custom_attribute):
enabled_cache = caches['default']
mock_get_cache.return_value = enabled_cache
course_cache = CourseStructureCache()
size = 30000
data_chunk = b'\x00' * size
logger_name = 'xmodule.modulestore.split_mongo.mongo_connection'
with LogCapture(logger_name) as capture:
course_cache.set('my_data_chunk', data_chunk)
# data chunk was less than 1MB so no logs were added.
self.assertEqual(len(capture.records), 0)
course_cache.set('my_data_chunk', data_chunk)
mock_set_cache.assert_called()
mock_set_custom_attribute.assert_not_called()
def _get_structure(self, course):
"""