Merge pull request #8708 from edx/benmcmorran/discussion-caching-2

TNL-2458 Cache discussion id mapping on course publish

lms/djangoapps/django_comment_client/base/views.py

@@ -29,6 +29,7 @@ from django_comment_client.utils import (
     get_group_id_for_comments_service,
     get_discussion_categories_ids,
     get_discussion_id_map,
+    get_cached_discussion_id_map,
 )
 from django_comment_client.permissions import check_permissions_by_view, has_permission
 from eventtracking import tracker
@@ -78,10 +79,11 @@ def track_forum_event(request, event_name, course, obj, data, id_map=None):
     """
     user = request.user
     data['id'] = obj.id
-    if id_map is None:
-        id_map = get_discussion_id_map(course, user)
-
     commentable_id = data['commentable_id']
+
+    if id_map is None:
+        id_map = get_cached_discussion_id_map(course, commentable_id, user)
+
     if commentable_id in id_map:
         data['category_name'] = id_map[commentable_id]["title"]
         data['category_id'] = commentable_id

lms/djangoapps/django_comment_client/tests/test_utils.py

@@ -18,6 +18,7 @@ from courseware.tests.factories import InstructorFactory
 from courseware.tabs import get_course_tab_list
 from openedx.core.djangoapps.course_groups.cohorts import set_course_cohort_settings
 from student.tests.factories import UserFactory, AdminFactory, CourseEnrollmentFactory
+from openedx.core.djangoapps.content.course_structures.models import CourseStructure
 from openedx.core.djangoapps.util.testing import ContentGroupTestCase
 from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
@@ -154,6 +155,94 @@ class CoursewareContextTestCase(ModuleStoreTestCase):
         assertThreadCorrect(threads[1], self.discussion2, "Subsection / Discussion 2")
 
 
+class CachedDiscussionIdMapTestCase(ModuleStoreTestCase):
+    """
+    Tests that using the cache of discussion id mappings has the same behavior as searching through the course.
+    """
+    def setUp(self):
+        super(CachedDiscussionIdMapTestCase, self).setUp(create_user=True)
+
+        self.course = CourseFactory.create(org='TestX', number='101', display_name='Test Course')
+        self.discussion = ItemFactory.create(
+            parent_location=self.course.location,
+            category='discussion',
+            discussion_id='test_discussion_id',
+            discussion_category='Chapter',
+            discussion_target='Discussion 1'
+        )
+        self.private_discussion = ItemFactory.create(
+            parent_location=self.course.location,
+            category='discussion',
+            discussion_id='private_discussion_id',
+            discussion_category='Chapter 3',
+            discussion_target='Beta Testing',
+            visible_to_staff_only=True
+        )
+        self.bad_discussion = ItemFactory.create(
+            parent_location=self.course.location,
+            category='discussion',
+            discussion_id='bad_discussion_id',
+            discussion_category=None,
+            discussion_target=None
+        )
+
+    def test_cache_returns_correct_key(self):
+        usage_key = utils.get_cached_discussion_key(self.course, 'test_discussion_id')
+        self.assertEqual(usage_key, self.discussion.location)
+
+    def test_cache_returns_none_if_id_is_not_present(self):
+        usage_key = utils.get_cached_discussion_key(self.course, 'bogus_id')
+        self.assertIsNone(usage_key)
+
+    def test_cache_raises_exception_if_course_structure_not_cached(self):
+        CourseStructure.objects.all().delete()
+        with self.assertRaises(utils.DiscussionIdMapIsNotCached):
+            utils.get_cached_discussion_key(self.course, 'test_discussion_id')
+
+    def test_cache_raises_exception_if_discussion_id_not_cached(self):
+        cache = CourseStructure.objects.get(course_id=self.course.id)
+        cache.discussion_id_map_json = None
+        cache.save()
+
+        with self.assertRaises(utils.DiscussionIdMapIsNotCached):
+            utils.get_cached_discussion_key(self.course, 'test_discussion_id')
+
+    def test_module_does_not_have_required_keys(self):
+        self.assertTrue(utils.has_required_keys(self.discussion))
+        self.assertFalse(utils.has_required_keys(self.bad_discussion))
+
+    def verify_discussion_metadata(self):
+        """Retrieves the metadata for self.discussion and verifies that it is correct"""
+        metadata = utils.get_cached_discussion_id_map(self.course, 'test_discussion_id', self.user)
+        metadata = metadata[self.discussion.discussion_id]
+        self.assertEqual(metadata['location'], self.discussion.location)
+        self.assertEqual(metadata['title'], 'Chapter / Discussion 1')
+
+    def test_get_discussion_id_map_from_cache(self):
+        self.verify_discussion_metadata()
+
+    def test_get_discussion_id_map_without_cache(self):
+        CourseStructure.objects.all().delete()
+        self.verify_discussion_metadata()
+
+    def test_get_missing_discussion_id_map_from_cache(self):
+        metadata = utils.get_cached_discussion_id_map(self.course, 'bogus_id', self.user)
+        self.assertEqual(metadata, {})
+
+    def test_get_discussion_id_map_from_cache_without_access(self):
+        user = UserFactory.create()
+
+        metadata = utils.get_cached_discussion_id_map(self.course, 'private_discussion_id', self.user)
+        self.assertEqual(metadata['private_discussion_id']['title'], 'Chapter 3 / Beta Testing')
+
+        metadata = utils.get_cached_discussion_id_map(self.course, 'private_discussion_id', user)
+        self.assertEqual(metadata, {})
+
+    def test_get_bad_discussion_id(self):
+        metadata = utils.get_cached_discussion_id_map(self.course, 'bad_discussion_id', self.user)
+        self.assertEqual(metadata, {})
+
+
 class CategoryMapTestMixin(object):
     """
     Provides functionality for classes that test

lms/djangoapps/django_comment_client/utils.py

@@ -20,6 +20,7 @@ from django_comment_client.settings import MAX_COMMENT_DEPTH
 from edxmako import lookup_template
 
 from courseware.access import has_access
+from openedx.core.djangoapps.content.course_structures.models import CourseStructure
 from openedx.core.djangoapps.course_groups.cohorts import (
     get_course_cohort_settings, get_cohort_by_id, get_cohort_id, is_commentable_cohorted, is_course_cohorted
 )
@@ -62,6 +63,15 @@ def has_forum_access(uname, course_id, rolename):
     return role.users.filter(username=uname).exists()
 
 
+def has_required_keys(module):
+    """Returns True iff module has the proper attributes for generating metadata with get_discussion_id_map_entry()"""
+    for key in ('discussion_id', 'discussion_category', 'discussion_target'):
+        if getattr(module, key, None) is None:
+            log.debug("Required key '%s' not in discussion %s, leaving out of category map", key, module.location)
+            return False
+    return True
+
+
 def get_accessible_discussion_modules(course, user, include_all=False):  # pylint: disable=invalid-name
     """
     Return a list of all valid discussion modules in this course that
@@ -69,31 +79,68 @@ def get_accessible_discussion_modules(course, user, include_all=False):  # pylin
     """
     all_modules = modulestore().get_items(course.id, qualifiers={'category': 'discussion'})
 
-    def has_required_keys(module):
-        for key in ('discussion_id', 'discussion_category', 'discussion_target'):
-            if getattr(module, key, None) is None:
-                log.warning("Required key '%s' not in discussion %s, leaving out of category map" % (key, module.location))
-                return False
-        return True
-
     return [
         module for module in all_modules
         if has_required_keys(module) and (include_all or has_access(user, 'load', module, course.id))
     ]
 
 
+def get_discussion_id_map_entry(module):
+    """
+    Returns a tuple of (discussion_id, metadata) suitable for inclusion in the results of get_discussion_id_map().
+    """
+    return (
+        module.discussion_id,
+        {
+            "location": module.location,
+            "title": module.discussion_category.split("/")[-1].strip() + " / " + module.discussion_target
+        }
+    )
+
+
+class DiscussionIdMapIsNotCached(Exception):
+    """Thrown when the discussion id map is not cached for this course, but an attempt was made to access it."""
+    pass
+
+
+def get_cached_discussion_key(course, discussion_id):
+    """
+    Returns the usage key of the discussion module associated with discussion_id if it is cached. If the discussion id
+    map is cached but does not contain discussion_id, returns None. If the discussion id map is not cached for course,
+    raises a DiscussionIdMapIsNotCached exception.
+    """
+    try:
+        cached_mapping = CourseStructure.objects.get(course_id=course.id).discussion_id_map
+        if not cached_mapping:
+            raise DiscussionIdMapIsNotCached()
+        return cached_mapping.get(discussion_id)
+    except CourseStructure.DoesNotExist:
+        raise DiscussionIdMapIsNotCached()
+
+
+def get_cached_discussion_id_map(course, discussion_id, user):
+    """
+    Returns a dict mapping discussion_id to discussion module metadata if it is cached and visible to the user.
+    If not, returns the result of get_discussion_id_map
+    """
+    try:
+        key = get_cached_discussion_key(course, discussion_id)
+        if not key:
+            return {}
+        module = modulestore().get_item(key)
+        if not (has_required_keys(module) and has_access(user, 'load', module, course.id)):
+            return {}
+        return dict([get_discussion_id_map_entry(module)])
+    except DiscussionIdMapIsNotCached:
+        return get_discussion_id_map(course, user)
+
+
 def get_discussion_id_map(course, user):
     """
     Transform the list of this course's discussion modules (visible to a given user) into a dictionary of metadata keyed
     by discussion_id.
     """
-    def get_entry(module):  # pylint: disable=missing-docstring
-        discussion_id = module.discussion_id
-        title = module.discussion_target
-        last_category = module.discussion_category.split("/")[-1].strip()
-        return (discussion_id, {"location": module.location, "title": last_category + " / " + title})
-
-    return dict(map(get_entry, get_accessible_discussion_modules(course, user)))
+    return dict(map(get_discussion_id_map_entry, get_accessible_discussion_modules(course, user)))
 
 
 def _filter_unstarted_categories(category_map):