Merge pull request #65 from MITx/safe-cache-key

Use a key function that is safe for use w/ memcache

This is needed for the asset-pipeline. The pipeline has to map a requested asset name ('css/application.css') to a string that encodes the file contents ('css/application.12323423abcf.css'). It caches these associations in memcache. We have the potential for filenames with spaces in them, which memcache can't handle, so we need to escape the cache keys in such a way that memcache can handle them.

It puts these in a cache named 'staticfiles' (http://django-staticfiles.readthedocs.org/en/latest/helpers/#cachedstaticfilesstorage). In the future, it may make sense to put that cache onto the filesystem, rather than memcache, but we should do that only after performance testing.

djangoapps/courseware/content_parser.py

@@ -7,7 +7,6 @@ Does some caching (to be explained).
 
 '''
 
-import hashlib
 import logging
 import os
 import re
@@ -16,6 +15,7 @@ import urllib
 
 from datetime import timedelta
 from lxml import etree
+from util.memcache import fasthash
 
 try: # This lets us do __name__ == ='__main__'
     from django.conf import settings
@@ -57,10 +57,6 @@ def parse_timedelta(time_str):
             time_params[name] = int(param)
     return timedelta(**time_params)
 
-def fasthash(string):
-    m = hashlib.new("md4")
-    m.update(string)
-    return "id"+m.hexdigest()
 
 def xpath(xml, query_string, **args):
     ''' Safe xpath query into an xml tree:
@@ -122,7 +118,7 @@ def id_tag(course):
             new_id = default_ids[elem.tag] + new_id
             elem.set('id', new_id)
         else:
-            elem.set('id', fasthash(etree.tostring(elem)))
+            elem.set('id', "id"+fasthash(etree.tostring(elem)))
             
 def propogate_downward_tag(element, attribute_name, parent_attribute = None):
     ''' This call is to pass down an attribute to all children. If an element
@@ -160,11 +156,11 @@ def user_groups(user):
     cache_expiration = 60 * 60 # one hour
     
     # Kill caching on dev machines -- we switch groups a lot
-    group_names = cache.get(fasthash(key))
+    group_names = cache.get(key)
  
     if group_names is None:
         group_names = [u.name for u in UserTestGroup.objects.filter(users=user)]
-        cache.set(fasthash(key), group_names, cache_expiration)
+        cache.set(key, group_names, cache_expiration)
 
     return group_names
 
@@ -203,7 +199,7 @@ def course_file(user,coursename=None):
     
     cache_key = filename + "_processed?dev_content:" + str(options['dev_content']) + "&groups:" + str(sorted(groups))
     if "dev" not in settings.DEFAULT_GROUPS:
-        tree_string = cache.get(fasthash(cache_key))
+        tree_string = cache.get(cache_key)
     else: 
         tree_string = None
 
@@ -211,7 +207,7 @@ def course_file(user,coursename=None):
         tree = course_xml_process(etree.XML(render_to_string(filename, options, namespace = 'course')))
         tree_string = etree.tostring(tree)
         
-        cache.set(fasthash(cache_key), tree_string, 60)
+        cache.set(cache_key, tree_string, 60)
     else:
         tree = etree.XML(tree_string)
 

envs/dev.py

@@ -31,7 +31,8 @@ CACHES = {
     # In staging/prod envs, the sessions also live here.
     'default': {
         'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
-        'LOCATION': 'mitx_loc_mem_cache'
+        'LOCATION': 'mitx_loc_mem_cache',
+        'KEY_FUNCTION': 'util.memcache.safe_key',
     },
 
     # The general cache is what you get if you use our util.cache. It's used for
@@ -43,6 +44,7 @@ CACHES = {
         'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
         'KEY_PREFIX': 'general',
         'VERSION': 4,
+        'KEY_FUNCTION': 'util.cache.memcache_safe_key',
     }
 }
 

envs/devplus.py

@@ -30,12 +30,14 @@ CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
        'LOCATION': '127.0.0.1:11211',
+       'KEY_FUNCTION': 'util.memcache.safe_key',
    },
    'general': {
        'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
        'LOCATION': '127.0.0.1:11211',
        'KEY_PREFIX' : 'general',
        'VERSION' : 5,
+       'KEY_FUNCTION': 'util.memcache.safe_key',
    }
 }
 

envs/static.py

@@ -30,7 +30,8 @@ CACHES = {
     # In staging/prod envs, the sessions also live here.
     'default': {
         'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
-        'LOCATION': 'mitx_loc_mem_cache'
+        'LOCATION': 'mitx_loc_mem_cache',
+        'KEY_FUNCTION': 'util.memcache.safe_key',
     },
 
     # The general cache is what you get if you use our util.cache. It's used for
@@ -42,6 +43,7 @@ CACHES = {
         'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
         'KEY_PREFIX': 'general',
         'VERSION': 4,
+        'KEY_FUNCTION': 'util.memcache.safe_key',
     }
 }
 

envs/test.py

@@ -54,7 +54,8 @@ CACHES = {
     # In staging/prod envs, the sessions also live here.
     'default': {
         'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
-        'LOCATION': 'mitx_loc_mem_cache'
+        'LOCATION': 'mitx_loc_mem_cache',
+        'KEY_FUNCTION': 'util.memcache.safe_key',
     },
 
     # The general cache is what you get if you use our util.cache. It's used for
@@ -66,6 +67,7 @@ CACHES = {
         'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
         'KEY_PREFIX': 'general',
         'VERSION': 4,
+        'KEY_FUNCTION': 'util.memcache.safe_key',
     }
 }
 

lib/util/memcache.py

@@ -0,0 +1,20 @@
+"""
+This module provides a KEY_FUNCTION suitable for use with a memcache backend
+so that we can cache any keys, not just ones that memcache would ordinarily accept
+"""
+from django.utils.encoding import smart_str
+import hashlib
+import urllib
+
+def fasthash(string):
+    m = hashlib.new("md4")
+    m.update(string)
+    return m.hexdigest()
+
+def safe_key(key, key_prefix, version):
+    safe_key = urllib.quote_plus(smart_str(key))
+
+    if len(safe_key) > 250:
+        safe_key = fasthash(safe_key)
+
+    return ":".join([key_prefix, str(version), safe_key])