From 004a0f5c75b1e257af5aaf0c5cb5cb5cd48b56f1 Mon Sep 17 00:00:00 2001 From: Muhammad Umar Khan <42294172+mumarkhan999@users.noreply.github.com> Date: Tue, 25 Jul 2023 14:49:43 +0500 Subject: [PATCH] chore: add monitoring on course strcuture cache failure (#32831) --- .../split_mongo/mongo_connection.py | 20 +++++++++++++------ .../tests/test_split_modulestore.py | 7 +++++-- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/xmodule/modulestore/split_mongo/mongo_connection.py b/xmodule/modulestore/split_mongo/mongo_connection.py index 6469a20270..76821cd400 100644 --- a/xmodule/modulestore/split_mongo/mongo_connection.py +++ b/xmodule/modulestore/split_mongo/mongo_connection.py @@ -20,6 +20,7 @@ import pytz from mongodb_proxy import autoretry_read # Import this just to export it from pymongo.errors import DuplicateKeyError # pylint: disable=unused-import +from edx_django_utils import monitoring from edx_django_utils.cache import RequestCache from common.djangoapps.split_modulestore_django.models import SplitModulestoreCourseIndex @@ -245,12 +246,19 @@ class CourseStructureCache: data_size = len(compressed_pickled_data) tagger.measure('compressed_size', data_size) - total_bytes_in_one_mb = 1024 * 1024 - # only print logs when data size is greater than or equal to 1MB - if data_size >= total_bytes_in_one_mb: - log.info('Data to be cached is: {:.2f} MB'.format(data_size / total_bytes_in_one_mb)) - # Stuctures are immutable, so we set a timeout of "never" - self.cache.set(key, compressed_pickled_data, None) + # Structures are immutable, so we set a timeout of "never" + try: + self.cache.set(key, compressed_pickled_data, None) + except Exception: # pylint: disable=broad-except + total_bytes_in_one_mb = 1024 * 1024 + chunk_size_in_mbs = round(data_size / total_bytes_in_one_mb, 2) + + # .. custom_attribute_name: split_mongo_compressed_size + # .. custom_attribute_description: contains the data chunk size in MBs. The size on which + # the memcached client failed to store value in course structure cache. + monitoring.set_custom_attribute('split_mongo_compressed_size', chunk_size_in_mbs) + monitoring.record_exception() + log.info('Data caching (course structure) failed on chunk size: {} MB'.format(chunk_size_in_mbs)) class MongoPersistenceBackend: diff --git a/xmodule/modulestore/tests/test_split_modulestore.py b/xmodule/modulestore/tests/test_split_modulestore.py index fb151915cf..bee57733ac 100644 --- a/xmodule/modulestore/tests/test_split_modulestore.py +++ b/xmodule/modulestore/tests/test_split_modulestore.py @@ -842,10 +842,13 @@ class TestCourseStructureCache(CacheIsolationMixin, SplitModuleTest): # now make sure that you get the same structure assert cached_structure == not_cached_structure + @patch('django.core.cache.cache.set') @patch('xmodule.modulestore.split_mongo.mongo_connection.get_cache') - def test_course_structure_cache_with_data_chunk_greater_than_one_mb(self, mock_get_cache): + def test_course_structure_cache_with_data_chunk_greater_than_one_mb(self, mock_get_cache, mock_set_cache): enabled_cache = caches['default'] mock_get_cache.return_value = enabled_cache + mock_set_cache.side_effect = Exception + course_cache = CourseStructureCache() size = 300000000 @@ -853,7 +856,7 @@ class TestCourseStructureCache(CacheIsolationMixin, SplitModuleTest): data_chunk = b'\x00' * size logger_name = 'xmodule.modulestore.split_mongo.mongo_connection' - expected_message = 'Data to be cached is: 1.25 MB' + expected_message = 'Data caching (course structure) failed on chunk size: 1.25 MB' with LogCapture(logger_name) as capture: course_cache.set('my_data_chunk', data_chunk)