Bulk-reads and Request caching in Course Grade Report

This reverts commit 5388d5d1fc.

common/djangoapps/course_modes/models.py

@@ -9,8 +9,11 @@ from config_models.models import ConfigurationModel
 from django.core.exceptions import ValidationError
 from django.db import models
 from django.db.models import Q
+from django.dispatch import receiver
 from django.utils.translation import ugettext_lazy as _
 from openedx.core.djangoapps.xmodule_django.models import CourseKeyField
+from request_cache.middleware import ns_request_cached, RequestCache
+
 
 Mode = namedtuple('Mode',
                   [
@@ -141,6 +144,8 @@ class CourseMode(models.Model):
     DEFAULT_SHOPPINGCART_MODE_SLUG = HONOR
     DEFAULT_SHOPPINGCART_MODE = Mode(HONOR, _('Honor'), 0, '', 'usd', None, None, None, None)
 
+    CACHE_NAMESPACE = u"course_modes.CourseMode.cache."
+
     class Meta(object):
         unique_together = ('course_id', 'mode_slug', 'currency')
 
@@ -265,6 +270,7 @@ class CourseMode(models.Model):
         return [mode.to_tuple() for mode in found_course_modes]
 
     @classmethod
+    @ns_request_cached(CACHE_NAMESPACE)
     def modes_for_course(cls, course_id, include_expired=False, only_selectable=True):
         """
         Returns a list of the non-expired modes for a given course id
@@ -666,6 +672,13 @@ class CourseMode(models.Model):
         )
 
 
+@receiver(models.signals.post_save, sender=CourseMode)
+@receiver(models.signals.post_delete, sender=CourseMode)
+def invalidate_course_mode_cache(sender, **kwargs):   # pylint: disable=unused-argument
+    """Invalidate the cache of course modes. """
+    RequestCache.clear_request_cache(name=CourseMode.CACHE_NAMESPACE)
+
+
 class CourseModesArchive(models.Model):
     """
     Store the past values of course_mode that a course had in the past. We decided on having

common/djangoapps/course_modes/tests/test_models.py

@@ -16,7 +16,7 @@ from opaque_keys.edx.locator import CourseLocator
 import pytz
 
 from course_modes.helpers import enrollment_mode_display
-from course_modes.models import CourseMode, Mode
+from course_modes.models import CourseMode, Mode, invalidate_course_mode_cache
 from course_modes.tests.factories import CourseModeFactory
 
 
@@ -31,6 +31,9 @@ class CourseModeModelTest(TestCase):
         self.course_key = SlashSeparatedCourseKey('Test', 'TestCourse', 'TestCourseRun')
         CourseMode.objects.all().delete()
 
+    def tearDown(self):
+        invalidate_course_mode_cache(sender=None)
+
     def create_mode(
             self,
             mode_slug,

common/djangoapps/request_cache/__init__.py

@@ -41,6 +41,16 @@ def get_cache(name):
     return middleware.RequestCache.get_request_cache(name)
 
 
+def clear_cache(name):
+    """
+    Clears the request cache named ``name``.
+
+    Arguments:
+        name (str): The name of the request cache to clear
+    """
+    return middleware.RequestCache.clear_request_cache(name)
+
+
 def get_request():
     """
     Return the current request.

common/djangoapps/request_cache/middleware.py

@@ -39,11 +39,14 @@ class RequestCache(object):
         return crum.get_current_request()
 
     @classmethod
-    def clear_request_cache(cls):
+    def clear_request_cache(cls, name=None):
         """
         Empty the request cache.
         """
-        REQUEST_CACHE.data = {}
+        if name is None:
+            REQUEST_CACHE.data = {}
+        elif REQUEST_CACHE.data.get(name):
+            REQUEST_CACHE.data[name] = {}
 
     def process_request(self, request):
         self.clear_request_cache()
@@ -82,25 +85,43 @@ def request_cached(f):
               cache the value it returns, and return that cached value for subsequent calls with the
               same args/kwargs within a single request
     """
-    def wrapper(*args, **kwargs):
+    return ns_request_cached()(f)
+
+
+def ns_request_cached(namespace=None):
+    """
+    Same as request_cached above, except an optional namespace can be passed in to compartmentalize the cache.
+
+    Arguments:
+        namespace (string): An optional namespace to use for the cache.  Useful if the caller wants to manage
+            their own sub-cache by, for example, calling RequestCache.clear_request_cache for their own namespace.
+    """
+    def outer_wrapper(f):
         """
-        Wrapper function to decorate with.
+        Outer wrapper that decorates the given function
+
+        Arguments:
+            f (func): the function to wrap
         """
-        # Check to see if we have a result in cache.  If not, invoke our wrapped
-        # function.  Cache and return the result to the caller.
-        rcache = RequestCache.get_request_cache()
-        cache_key = func_call_cache_key(f, *args, **kwargs)
+        def inner_wrapper(*args, **kwargs):
+            """
+            Wrapper function to decorate with.
+            """
+            # Check to see if we have a result in cache.  If not, invoke our wrapped
+            # function.  Cache and return the result to the caller.
+            rcache = RequestCache.get_request_cache(namespace)
+            rcache = rcache.data if namespace is None else rcache
+            cache_key = func_call_cache_key(f, *args, **kwargs)
 
-        if cache_key in rcache.data:
-            return rcache.data.get(cache_key)
-        else:
-            result = f(*args, **kwargs)
-            rcache.data[cache_key] = result
+            if cache_key in rcache:
+                return rcache.get(cache_key)
+            else:
+                result = f(*args, **kwargs)
+                rcache[cache_key] = result
+                return result
 
-            return result
-
-    wrapper.request_cached_contained_func = f
-    return wrapper
+        return inner_wrapper
+    return outer_wrapper
 
 
 def func_call_cache_key(func, *args, **kwargs):

common/djangoapps/student/models.py

@@ -998,7 +998,9 @@ class CourseEnrollment(models.Model):
     history = HistoricalRecords()
 
     # cache key format e.g enrollment.<username>.<course_key>.mode = 'honor'
-    COURSE_ENROLLMENT_CACHE_KEY = u"enrollment.{}.{}.mode"
+    COURSE_ENROLLMENT_CACHE_KEY = u"enrollment.{}.{}.mode"  # TODO Can this be removed?  It doesn't seem to be used.
+
+    MODE_CACHE_NAMESPACE = u'CourseEnrollment.mode_and_active'
 
     class Meta(object):
         unique_together = (('user', 'course_id'),)
@@ -1697,12 +1699,28 @@ class CourseEnrollment(models.Model):
             cls._update_enrollment_in_request_cache(user, course_key, enrollment_state)
         return enrollment_state
 
+    @classmethod
+    def bulk_fetch_enrollment_states(cls, users, course_key):
+        """
+        Bulk pre-fetches the enrollment states for the given users
+        for the given course.
+        """
+        # before populating the cache with another bulk set of data,
+        # remove previously cached entries to keep memory usage low.
+        request_cache.clear_cache(cls.MODE_CACHE_NAMESPACE)
+
+        records = cls.objects.filter(user__in=users, course_id=course_key).select_related('user__id')
+        cache = cls._get_mode_active_request_cache()
+        for record in records:
+            enrollment_state = CourseEnrollmentState(record.mode, record.is_active)
+            cls._update_enrollment(cache, record.user.id, course_key, enrollment_state)
+
     @classmethod
     def _get_mode_active_request_cache(cls):
         """
         Returns the request-specific cache for CourseEnrollment
         """
-        return request_cache.get_cache('CourseEnrollment.mode_and_active')
+        return request_cache.get_cache(cls.MODE_CACHE_NAMESPACE)
 
     @classmethod
     def _get_enrollment_in_request_cache(cls, user, course_key):
@@ -1718,7 +1736,15 @@ class CourseEnrollment(models.Model):
         Updates the cached value for the user's enrollment in the
         request cache.
         """
-        cls._get_mode_active_request_cache()[(user.id, course_key)] = enrollment_state
+        cls._update_enrollment(cls._get_mode_active_request_cache(), user.id, course_key, enrollment_state)
+
+    @classmethod
+    def _update_enrollment(cls, cache, user_id, course_key, enrollment_state):
+        """
+        Updates the cached value for the user's enrollment in the
+        given cache.
+        """
+        cache[(user_id, course_key)] = enrollment_state
 
 
 @receiver(models.signals.post_save, sender=CourseEnrollment)

common/djangoapps/student/roles.py

@@ -4,10 +4,12 @@ adding users, removing users, and listing members
 """
 
 from abc import ABCMeta, abstractmethod
+from collections import defaultdict
 
 from django.contrib.auth.models import User
 import logging
 
+from request_cache import get_cache
 from student.models import CourseAccessRole
 from openedx.core.djangoapps.xmodule_django.models import CourseKeyField
 
@@ -34,14 +36,38 @@ def register_access_role(cls):
     return cls
 
 
+class BulkRoleCache(object):
+    CACHE_NAMESPACE = u"student.roles.BulkRoleCache"
+    CACHE_KEY = u'roles_by_user'
+
+    @classmethod
+    def prefetch(cls, users):
+        roles_by_user = defaultdict(set)
+        get_cache(cls.CACHE_NAMESPACE)[cls.CACHE_KEY] = roles_by_user
+
+        for role in CourseAccessRole.objects.filter(user__in=users).select_related('user__id'):
+            roles_by_user[role.user.id].add(role)
+
+        users_without_roles = filter(lambda u: u.id not in roles_by_user, users)
+        for user in users_without_roles:
+            roles_by_user[user.id] = set()
+
+    @classmethod
+    def get_user_roles(cls, user):
+        return get_cache(cls.CACHE_NAMESPACE)[cls.CACHE_KEY][user.id]
+
+
 class RoleCache(object):
     """
     A cache of the CourseAccessRoles held by a particular user
     """
     def __init__(self, user):
-        self._roles = set(
-            CourseAccessRole.objects.filter(user=user).all()
-        )
+        try:
+            self._roles = BulkRoleCache.get_user_roles(user)
+        except KeyError:
+            self._roles = set(
+                CourseAccessRole.objects.filter(user=user).all()
+            )
 
     def has_role(self, role, course_id, org):
         """