Merge pull request #780 from MITx/feature/cdodge/cms-static-content-management

Feature/cdodge/cms static content management

cms/djangoapps/contentstore/views.py

@@ -1,15 +1,22 @@
 from util.json_request import expect_json
 import json
+import os
 import logging
 import sys
+import mimetypes
+import StringIO
 from collections import defaultdict
 
-from django.http import HttpResponse, Http404
+# to install PIL on MacOSX: 'easy_install http://dist.repoze.org/PIL-1.1.6.tar.gz'
+from PIL import Image
+
+from django.http import HttpResponse, Http404, HttpResponseBadRequest, HttpResponseForbidden
 from django.contrib.auth.decorators import login_required
 from django.core.context_processors import csrf
 from django_future.csrf import ensure_csrf_cookie
 from django.core.urlresolvers import reverse
 from django.conf import settings
+from django import forms
 
 from xmodule.modulestore import Location
 from xmodule.x_module import ModuleSystem
@@ -26,6 +33,13 @@ from functools import partial
 from itertools import groupby
 from operator import attrgetter
 
+from xmodule.contentstore.django import contentstore
+from xmodule.contentstore.content import StaticContent
+
+#from django.core.cache import cache
+
+from cache_toolbox.core import set_cached_content, get_cached_content, del_cached_content
+
 log = logging.getLogger(__name__)
 
 
@@ -89,9 +103,26 @@ def course_index(request, org, course, name):
         raise Http404  # TODO (vshnayder): better error
 
     # TODO (cpennington): These need to be read in from the active user
-    course = modulestore().get_item(location)
-    weeks = course.get_children()
-    return render_to_response('course_index.html', {'weeks': weeks})
+    _course = modulestore().get_item(location)
+    weeks = _course.get_children()
+
+    #upload_asset_callback_url = "/{org}/{course}/course/{name}/upload_asset".format(
+    #    org = org, 
+    #    course = course,
+    #    name = name
+    #    )
+
+    upload_asset_callback_url = reverse('upload_asset', kwargs = {
+            'org' : org,
+            'course' : course,
+            'coursename' : name
+            })
+    logging.debug(upload_asset_callback_url)
+
+    return render_to_response('course_index.html', {
+            'weeks': weeks, 
+            'upload_asset_callback_url': upload_asset_callback_url
+            })
 
 
 @login_required
@@ -115,12 +146,13 @@ def edit_item(request):
         lms_link = "{lms_base}/courses/{course_id}/jump_to/{location}".format(
             lms_base=settings.LMS_BASE,
             # TODO: These will need to be changed to point to the particular instance of this problem in the particular course
-            course_id=modulestore().get_containing_courses(item.location)[0].id,
+            course_id= modulestore().get_containing_courses(item.location)[0].id,
             location=item.location,
         )
     else:
         lms_link = None
 
+    
     return render_to_response('unit.html', {
         'contents': item.get_html(),
         'js_module': item.js_module_name,
@@ -333,7 +365,7 @@ def save_item(request):
     if request.POST['data']:
         data = request.POST['data']
         modulestore().update_item(item_location, data)
-
+        
     if request.POST['children']:
         children = request.POST['children']
         modulestore().update_children(item_location, children)
@@ -390,3 +422,94 @@ def clone_item(request):
     modulestore().update_children(parent_location, parent.definition.get('children', []) + [new_item.location.url()])
 
     return HttpResponse()
+
+'''
+cdodge: this method allows for POST uploading of files into the course asset library, which will
+be supported by GridFS in MongoDB.
+'''
+#@login_required
+#@ensure_csrf_cookie
+def upload_asset(request, org, course, coursename):
+
+    if request.method != 'POST':
+        # (cdodge) @todo: Is there a way to do a - say - 'raise Http400'?
+        return HttpResponseBadRequest()
+
+    # construct a location from the passed in path
+    location = ['i4x', org, course, 'course', coursename]
+    if not has_access(request.user, location):
+        return HttpResponseForbidden()
+    
+    # Does the course actually exist?!?
+    
+    try:
+        item = modulestore().get_item(location)
+    except:
+        # no return it as a Bad Request response
+        logging.error('Could not find course' + location)
+        return HttpResponseBadRequest()
+
+    # compute a 'filename' which is similar to the location formatting, we're using the 'filename'
+    # nomenclature since we're using a FileSystem paradigm here. We're just imposing
+    # the Location string formatting expectations to keep things a bit more consistent
+
+    name = request.FILES['file'].name
+    mime_type = request.FILES['file'].content_type
+    filedata = request.FILES['file'].read()
+
+    file_location = StaticContent.compute_location_filename(org, course, name)
+
+    content = StaticContent(file_location, name, mime_type, filedata)
+
+    # first commit to the DB
+    contentstore().save(content)
+
+    # then remove the cache so we're not serving up stale content
+    # NOTE: we're not re-populating the cache here as the DB owns the last-modified timestamp
+    # which is used when serving up static content. This integrity is needed for
+    # browser-side caching support. We *could* re-fetch the saved content so that we have the
+    # timestamp populated, but we might as well wait for the first real request to come in
+    # to re-populate the cache.
+    del_cached_content(file_location)
+
+    # if we're uploading an image, then let's generate a thumbnail so that we can
+    # serve it up when needed without having to rescale on the fly
+    if mime_type.split('/')[0] == 'image':
+        try:
+            # not sure if this is necessary, but let's rewind the stream just in case
+            request.FILES['file'].seek(0)
+
+            # use PIL to do the thumbnail generation (http://www.pythonware.com/products/pil/)
+            # My understanding is that PIL will maintain aspect ratios while restricting
+            # the max-height/width to be whatever you pass in as 'size'
+            # @todo: move the thumbnail size to a configuration setting?!?
+            im = Image.open(request.FILES['file'])
+
+            # I've seen some exceptions from the PIL library when trying to save palletted 
+            # PNG files to JPEG. Per the google-universe, they suggest converting to RGB first.
+            im = im.convert('RGB')
+            size = 128, 128
+            im.thumbnail(size, Image.ANTIALIAS)
+            thumbnail_file = StringIO.StringIO()
+            im.save(thumbnail_file, 'JPEG')
+            thumbnail_file.seek(0)
+        
+            # use a naming convention to associate originals with the thumbnail
+            #   <name_without_extention>.thumbnail.jpg
+            thumbnail_name = os.path.splitext(name)[0] + '.thumbnail.jpg'
+            # then just store this thumbnail as any other piece of content
+            thumbnail_file_location = StaticContent.compute_location_filename(org, course, 
+                                                                              thumbnail_name)
+            thumbnail_content = StaticContent(thumbnail_file_location, thumbnail_name, 
+                                              'image/jpeg', thumbnail_file)
+            contentstore().save(thumbnail_content)
+
+            # remove any cached content at this location, as thumbnails are treated just like any
+            # other bit of static content
+            del_cached_content(thumbnail_file_location)
+        except:
+            # catch, log, and continue as thumbnails are not a hard requirement
+            logging.error('Failed to generate thumbnail for {0}. Continuing...'.format(name))
+
+    return HttpResponse('Upload completed')
+

cms/envs/common.py

@@ -118,6 +118,7 @@ TEMPLATE_LOADERS = (
 )
 
 MIDDLEWARE_CLASSES = (
+    'contentserver.middleware.StaticContentServer',
     'django.middleware.cache.UpdateCacheMiddleware',
     'django.middleware.common.CommonMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
@@ -130,7 +131,7 @@ MIDDLEWARE_CLASSES = (
     'track.middleware.TrackMiddleware',
     'mitxmako.middleware.MakoMiddleware',
 
-    'django.middleware.transaction.TransactionMiddleware',
+    'django.middleware.transaction.TransactionMiddleware'
 )
 
 ############################ SIGNAL HANDLERS ################################

cms/envs/dev.py

@@ -28,6 +28,17 @@ MODULESTORE = {
     }
 }
 
+# cdodge: This is the specifier for the MongoDB (using GridFS) backed static content store
+# This is for static content for courseware, not system static content (e.g. javascript, css, edX branding, etc)
+CONTENTSTORE = {
+    'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',
+    'OPTIONS': {
+        'host': 'localhost',
+        'db' : 'xcontent',
+    }
+}
+
+
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',

cms/templates/course_index.html

@@ -10,5 +10,7 @@
   <section class="main-content">
   </section>
 
+  <%include file="widgets/upload_assets.html"/>
+
 </section>
 </%block>

cms/templates/widgets/upload_assets.html

@@ -0,0 +1,44 @@
+<section>
+  <div class="assset-upload">
+    You can upload file assets (such as images) to reference in your courseware
+    <form action="${upload_asset_callback_url}" method="post" enctype="multipart/form-data">
+      <input type="file" name="file">
+      <input type="submit" value="Upload File">
+    </form>
+    <div class="progress" style="position:relative; width:400px; border: 1px solid #ddd; padding: 1px; border-radius: 3px;">
+      <div class="bar" style="background-color: #B4F5B4; width:0%; height:20px; border-radius: 3px;"></div>
+      <div class="percent">0%</div>
+    </div>
+
+    <div id="status"></div>
+  </div>
+
+</section>
+
+<script src="http://malsup.github.com/jquery.form.js"></script>
+<script>
+(function() {
+    
+var bar = $('.bar');
+var percent = $('.percent');
+var status = $('#status');
+   
+$('form').ajaxForm({
+    beforeSend: function() {
+        status.empty();
+        var percentVal = '0%';
+        bar.width(percentVal)
+        percent.html(percentVal);
+    },
+    uploadProgress: function(event, position, total, percentComplete) {
+        var percentVal = percentComplete + '%';
+        bar.width(percentVal)
+        percent.html(percentVal);
+    },
+complete: function(xhr) {
+status.html(xhr.responseText);
+}
+}); 
+
+})();       
+</script>

cms/urls.py

@@ -17,7 +17,9 @@ urlpatterns = ('',
         'contentstore.views.course_index', name='course_index'),
     url(r'^github_service_hook$', 'github_sync.views.github_post_receive'),
     url(r'^preview/modx/(?P<preview_id>[^/]*)/(?P<location>.*?)/(?P<dispatch>[^/]*)$',
-        'contentstore.views.preview_dispatch', name='preview_dispatch')
+        'contentstore.views.preview_dispatch', name='preview_dispatch'),
+    url(r'^(?P<org>[^/]+)/(?P<course>[^/]+)/course/(?P<coursename>[^/]+)/upload_asset$', 
+        'contentstore.views.upload_asset', name='upload_asset')
 )
 
 # User creation and updating views

common/djangoapps/cache_toolbox/core.py

@@ -107,3 +107,15 @@ def instance_key(model, instance_or_pk):
         model._meta.module_name,
         getattr(instance_or_pk, 'pk', instance_or_pk),
     )
+
+def content_key(filename):
+    return 'content:%s' % (filename)
+
+def set_cached_content(content):
+    cache.set(content_key(content.filename), content)
+
+def get_cached_content(filename):
+    return cache.get(content_key(filename))
+
+def del_cached_content(filename):
+    cache.delete(content_key(filename))

common/djangoapps/contentserver/middleware.py

@@ -0,0 +1,49 @@
+import logging
+import time
+
+from django.http import HttpResponse, Http404, HttpResponseNotModified
+
+from xmodule.contentstore.django import contentstore
+from xmodule.contentstore.content import StaticContent, XASSET_LOCATION_TAG
+from cache_toolbox.core import get_cached_content, set_cached_content
+from xmodule.exceptions import NotFoundError
+
+
+class StaticContentServer(object):
+    def process_request(self, request):
+        # look to see if the request is prefixed with 'c4x' tag
+        if request.path.startswith('/' + XASSET_LOCATION_TAG):
+
+            # first look in our cache so we don't have to round-trip to the DB
+            content = get_cached_content(request.path)
+            if content is None:
+                # nope, not in cache, let's fetch from DB
+                try:
+                    content = contentstore().find(request.path)
+                except NotFoundError:
+                    raise Http404
+
+                # since we fetched it from DB, let's cache it going forward
+                set_cached_content(content)
+            else:
+                # @todo: we probably want to have 'cache hit' counters so we can
+                # measure the efficacy of our caches
+                pass
+
+            # see if the last-modified at hasn't changed, if not return a 302 (Not Modified)
+
+            # convert over the DB persistent last modified timestamp to a HTTP compatible
+            # timestamp, so we can simply compare the strings
+            last_modified_at_str = content.last_modified_at.strftime("%a, %d-%b-%Y %H:%M:%S GMT")
+
+            # see if the client has cached this content, if so then compare the
+            # timestamps, if they are the same then just return a 304 (Not Modified)
+            if 'HTTP_IF_MODIFIED_SINCE' in request.META:
+                if_modified_since = request.META['HTTP_IF_MODIFIED_SINCE']
+                if if_modified_since == last_modified_at_str:
+                    return HttpResponseNotModified()
+
+            response = HttpResponse(content.data, content_type=content.content_type)
+            response['Last-Modified'] = last_modified_at_str
+
+            return response

common/lib/xmodule/xmodule/capa_module.py

@@ -10,6 +10,7 @@ import sys
 
 from datetime import timedelta
 from lxml import etree
+from lxml.html import rewrite_links
 from pkg_resources import resource_string
 
 from capa.capa_problem import LoncapaProblem
@@ -332,6 +333,15 @@ class CapaModule(XModule):
             html = '<div id="problem_{id}" class="problem" data-url="{ajax_url}">'.format(
                 id=self.location.html_id(), ajax_url=self.system.ajax_url) + html + "</div>"
 
+        # cdodge: OK, we have to do two rounds of url reference subsitutions
+        # one which uses the 'asset library' that is served by the contentstore and the
+        # more global /static/ filesystem based static content.
+        # NOTE: rewrite_content_links is defined in XModule
+        # This is a bit unfortunate and I'm sure we'll try to considate this into
+        # a one step process.
+        html = rewrite_links(html, self.rewrite_content_links)
+
+        # now do the substitutions which are filesystem based, e.g. '/static/' prefixes
         return self.system.replace_urls(html, self.metadata['data_dir'])
 
     def handle_ajax(self, dispatch, get):

common/lib/xmodule/xmodule/contentstore/content.py

@@ -0,0 +1,26 @@
+XASSET_LOCATION_TAG = 'c4x'
+XASSET_SRCREF_PREFIX = 'xasset:'
+
+class StaticContent(object):
+    def __init__(self, filename, name, content_type, data, last_modified_at=None):
+        self.filename = filename
+        self.name = name
+        self.content_type = content_type
+        self.data = data
+        self.last_modified_at = last_modified_at
+
+    @staticmethod
+    def compute_location_filename(org, course, name):
+        return '/{0}/{1}/{2}/asset/{3}'.format(XASSET_LOCATION_TAG, org, course, name)
+
+'''
+Abstraction for all ContentStore providers (e.g. MongoDB)
+'''
+class ContentStore(object):
+    def save(self, content):
+        raise NotImplementedError
+
+    def find(self, filename):
+        raise NotImplementedError
+
+            

common/lib/xmodule/xmodule/contentstore/django.py

@@ -0,0 +1,29 @@
+from __future__ import absolute_import
+from importlib import import_module
+from os import environ
+
+from django.conf import settings
+
+_CONTENTSTORE = None
+
+def load_function(path):
+    """
+    Load a function by name.
+
+    path is a string of the form "path.to.module.function"
+    returns the imported python object `function` from `path.to.module`
+    """
+    module_path, _, name = path.rpartition('.')
+    return getattr(import_module(module_path), name)
+
+
+def contentstore():
+    global _CONTENTSTORE
+
+    if _CONTENTSTORE is None:
+        class_ = load_function(settings.CONTENTSTORE['ENGINE'])
+        options = {}
+        options.update(settings.CONTENTSTORE['OPTIONS'])
+        _CONTENTSTORE = class_(**options)
+
+    return _CONTENTSTORE

common/lib/xmodule/xmodule/contentstore/mongo.py

@@ -0,0 +1,32 @@
+from pymongo import Connection
+import gridfs
+from gridfs.errors import NoFile
+
+import sys
+import logging
+
+from .content import StaticContent, ContentStore
+from xmodule.exceptions import NotFoundError
+
+
+class MongoContentStore(ContentStore):
+    def __init__(self, host, db, port=27017):
+        logging.debug( 'Using MongoDB for static content serving at host={0} db={1}'.format(host,db))
+        _db = Connection(host=host, port=port)[db]
+        self.fs = gridfs.GridFS(_db)
+
+    def save(self, content):
+        with self.fs.new_file(filename=content.filename, content_type=content.content_type, displayname=content.name) as fp:
+            fp.write(content.data)
+            return content
+        
+    
+    def find(self, filename):
+        try:
+            with self.fs.get_last_version(filename) as fp:
+                return StaticContent(fp.filename, fp.displayname, fp.content_type, fp.read(), fp.uploadDate)
+        except NoFile:
+            raise NotFoundError()
+
+
+        

common/lib/xmodule/xmodule/html_module.py

@@ -4,6 +4,7 @@ import logging
 import os
 import sys
 from lxml import etree
+from lxml.html import rewrite_links
 from path import path
 
 from .x_module import XModule
@@ -12,6 +13,9 @@ from .xml_module import XmlDescriptor, name_to_pathname
 from .editing_module import EditingDescriptor
 from .stringify import stringify_children
 from .html_checker import check_html
+from xmodule.modulestore import Location
+
+from xmodule.contentstore.content import XASSET_SRCREF_PREFIX, StaticContent
 
 log = logging.getLogger("mitx.courseware")
 
@@ -21,7 +25,8 @@ class HtmlModule(XModule):
     js_module_name = "HTMLModule"
     
     def get_html(self):
-        return self.html
+        # cdodge: perform link substitutions for any references to course static content (e.g. images)
+        return rewrite_links(self.html, self.rewrite_content_links)
 
     def __init__(self, system, location, definition, descriptor,
                  instance_state=None, shared_state=None, **kwargs):
@@ -30,6 +35,7 @@ class HtmlModule(XModule):
         self.html = self.definition['data']
 
 
+
 class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
     """
     Module for putting raw html in a course

common/lib/xmodule/xmodule/x_module.py

@@ -12,6 +12,8 @@ from pkg_resources import resource_listdir, resource_string, resource_isdir
 from xmodule.modulestore import Location
 from xmodule.timeparse import parse_time
 
+from xmodule.contentstore.content import StaticContent, XASSET_SRCREF_PREFIX
+
 log = logging.getLogger('mitx.' + __name__)
 
 
@@ -317,6 +319,20 @@ class XModule(HTMLSnippet):
             get is a dictionary-like object '''
         return ""
 
+    # cdodge: added to support dynamic substitutions of 
+    # links for courseware assets (e.g. images). <link> is passed through from lxml.html parser
+    def rewrite_content_links(self, link):
+        # see if we start with our format, e.g. 'xasset:<filename>'
+        if link.startswith(XASSET_SRCREF_PREFIX):
+            # yes, then parse out the name
+            name = link[len(XASSET_SRCREF_PREFIX):]
+            loc = Location(self.location)
+            # resolve the reference to our internal 'filepath' which
+            link = StaticContent.compute_location_filename(loc.org, loc.course, name)
+
+        return link
+
+
 
 def policy_key(location):
     """

requirements.txt

@@ -48,3 +48,4 @@ sorl-thumbnail
 networkx
 pygraphviz
 -r repo-requirements.txt
+pil