diff --git a/cms/djangoapps/contentstore/views.py b/cms/djangoapps/contentstore/views.py index da6611f248..3223bfda29 100644 --- a/cms/djangoapps/contentstore/views.py +++ b/cms/djangoapps/contentstore/views.py @@ -1,15 +1,22 @@ from util.json_request import expect_json import json +import os import logging import sys +import mimetypes +import StringIO from collections import defaultdict -from django.http import HttpResponse, Http404 +# to install PIL on MacOSX: 'easy_install http://dist.repoze.org/PIL-1.1.6.tar.gz' +from PIL import Image + +from django.http import HttpResponse, Http404, HttpResponseBadRequest, HttpResponseForbidden from django.contrib.auth.decorators import login_required from django.core.context_processors import csrf from django_future.csrf import ensure_csrf_cookie from django.core.urlresolvers import reverse from django.conf import settings +from django import forms from xmodule.modulestore import Location from xmodule.x_module import ModuleSystem @@ -26,6 +33,13 @@ from functools import partial from itertools import groupby from operator import attrgetter +from xmodule.contentstore.django import contentstore +from xmodule.contentstore.content import StaticContent + +#from django.core.cache import cache + +from cache_toolbox.core import set_cached_content, get_cached_content, del_cached_content + log = logging.getLogger(__name__) @@ -89,9 +103,26 @@ def course_index(request, org, course, name): raise Http404 # TODO (vshnayder): better error # TODO (cpennington): These need to be read in from the active user - course = modulestore().get_item(location) - weeks = course.get_children() - return render_to_response('course_index.html', {'weeks': weeks}) + _course = modulestore().get_item(location) + weeks = _course.get_children() + + #upload_asset_callback_url = "/{org}/{course}/course/{name}/upload_asset".format( + # org = org, + # course = course, + # name = name + # ) + + upload_asset_callback_url = reverse('upload_asset', kwargs = { + 'org' : org, + 'course' : course, + 'coursename' : name + }) + logging.debug(upload_asset_callback_url) + + return render_to_response('course_index.html', { + 'weeks': weeks, + 'upload_asset_callback_url': upload_asset_callback_url + }) @login_required @@ -115,12 +146,13 @@ def edit_item(request): lms_link = "{lms_base}/courses/{course_id}/jump_to/{location}".format( lms_base=settings.LMS_BASE, # TODO: These will need to be changed to point to the particular instance of this problem in the particular course - course_id=modulestore().get_containing_courses(item.location)[0].id, + course_id= modulestore().get_containing_courses(item.location)[0].id, location=item.location, ) else: lms_link = None + return render_to_response('unit.html', { 'contents': item.get_html(), 'js_module': item.js_module_name, @@ -333,7 +365,7 @@ def save_item(request): if request.POST['data']: data = request.POST['data'] modulestore().update_item(item_location, data) - + if request.POST['children']: children = request.POST['children'] modulestore().update_children(item_location, children) @@ -390,3 +422,94 @@ def clone_item(request): modulestore().update_children(parent_location, parent.definition.get('children', []) + [new_item.location.url()]) return HttpResponse() + +''' +cdodge: this method allows for POST uploading of files into the course asset library, which will +be supported by GridFS in MongoDB. +''' +#@login_required +#@ensure_csrf_cookie +def upload_asset(request, org, course, coursename): + + if request.method != 'POST': + # (cdodge) @todo: Is there a way to do a - say - 'raise Http400'? + return HttpResponseBadRequest() + + # construct a location from the passed in path + location = ['i4x', org, course, 'course', coursename] + if not has_access(request.user, location): + return HttpResponseForbidden() + + # Does the course actually exist?!? + + try: + item = modulestore().get_item(location) + except: + # no return it as a Bad Request response + logging.error('Could not find course' + location) + return HttpResponseBadRequest() + + # compute a 'filename' which is similar to the location formatting, we're using the 'filename' + # nomenclature since we're using a FileSystem paradigm here. We're just imposing + # the Location string formatting expectations to keep things a bit more consistent + + name = request.FILES['file'].name + mime_type = request.FILES['file'].content_type + filedata = request.FILES['file'].read() + + file_location = StaticContent.compute_location_filename(org, course, name) + + content = StaticContent(file_location, name, mime_type, filedata) + + # first commit to the DB + contentstore().save(content) + + # then remove the cache so we're not serving up stale content + # NOTE: we're not re-populating the cache here as the DB owns the last-modified timestamp + # which is used when serving up static content. This integrity is needed for + # browser-side caching support. We *could* re-fetch the saved content so that we have the + # timestamp populated, but we might as well wait for the first real request to come in + # to re-populate the cache. + del_cached_content(file_location) + + # if we're uploading an image, then let's generate a thumbnail so that we can + # serve it up when needed without having to rescale on the fly + if mime_type.split('/')[0] == 'image': + try: + # not sure if this is necessary, but let's rewind the stream just in case + request.FILES['file'].seek(0) + + # use PIL to do the thumbnail generation (http://www.pythonware.com/products/pil/) + # My understanding is that PIL will maintain aspect ratios while restricting + # the max-height/width to be whatever you pass in as 'size' + # @todo: move the thumbnail size to a configuration setting?!? + im = Image.open(request.FILES['file']) + + # I've seen some exceptions from the PIL library when trying to save palletted + # PNG files to JPEG. Per the google-universe, they suggest converting to RGB first. + im = im.convert('RGB') + size = 128, 128 + im.thumbnail(size, Image.ANTIALIAS) + thumbnail_file = StringIO.StringIO() + im.save(thumbnail_file, 'JPEG') + thumbnail_file.seek(0) + + # use a naming convention to associate originals with the thumbnail + # .thumbnail.jpg + thumbnail_name = os.path.splitext(name)[0] + '.thumbnail.jpg' + # then just store this thumbnail as any other piece of content + thumbnail_file_location = StaticContent.compute_location_filename(org, course, + thumbnail_name) + thumbnail_content = StaticContent(thumbnail_file_location, thumbnail_name, + 'image/jpeg', thumbnail_file) + contentstore().save(thumbnail_content) + + # remove any cached content at this location, as thumbnails are treated just like any + # other bit of static content + del_cached_content(thumbnail_file_location) + except: + # catch, log, and continue as thumbnails are not a hard requirement + logging.error('Failed to generate thumbnail for {0}. Continuing...'.format(name)) + + return HttpResponse('Upload completed') + diff --git a/cms/envs/common.py b/cms/envs/common.py index dc82af85af..7190ba9e51 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -118,6 +118,7 @@ TEMPLATE_LOADERS = ( ) MIDDLEWARE_CLASSES = ( + 'contentserver.middleware.StaticContentServer', 'django.middleware.cache.UpdateCacheMiddleware', 'django.middleware.common.CommonMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', @@ -130,7 +131,7 @@ MIDDLEWARE_CLASSES = ( 'track.middleware.TrackMiddleware', 'mitxmako.middleware.MakoMiddleware', - 'django.middleware.transaction.TransactionMiddleware', + 'django.middleware.transaction.TransactionMiddleware' ) ############################ SIGNAL HANDLERS ################################ diff --git a/cms/envs/dev.py b/cms/envs/dev.py index fc2a5a5684..dd0e0337f6 100644 --- a/cms/envs/dev.py +++ b/cms/envs/dev.py @@ -28,6 +28,17 @@ MODULESTORE = { } } +# cdodge: This is the specifier for the MongoDB (using GridFS) backed static content store +# This is for static content for courseware, not system static content (e.g. javascript, css, edX branding, etc) +CONTENTSTORE = { + 'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore', + 'OPTIONS': { + 'host': 'localhost', + 'db' : 'xcontent', + } +} + + DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', diff --git a/cms/templates/course_index.html b/cms/templates/course_index.html index e490ad7817..37b5a8b371 100644 --- a/cms/templates/course_index.html +++ b/cms/templates/course_index.html @@ -10,5 +10,7 @@
+ <%include file="widgets/upload_assets.html"/> + diff --git a/cms/templates/widgets/upload_assets.html b/cms/templates/widgets/upload_assets.html new file mode 100644 index 0000000000..510313c905 --- /dev/null +++ b/cms/templates/widgets/upload_assets.html @@ -0,0 +1,44 @@ +
+
+ You can upload file assets (such as images) to reference in your courseware +
+ + +
+
+
+
0%
+
+ +
+
+ +
+ + + diff --git a/cms/urls.py b/cms/urls.py index e51ae59b08..ddd54adc65 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -17,7 +17,9 @@ urlpatterns = ('', 'contentstore.views.course_index', name='course_index'), url(r'^github_service_hook$', 'github_sync.views.github_post_receive'), url(r'^preview/modx/(?P[^/]*)/(?P.*?)/(?P[^/]*)$', - 'contentstore.views.preview_dispatch', name='preview_dispatch') + 'contentstore.views.preview_dispatch', name='preview_dispatch'), + url(r'^(?P[^/]+)/(?P[^/]+)/course/(?P[^/]+)/upload_asset$', + 'contentstore.views.upload_asset', name='upload_asset') ) # User creation and updating views diff --git a/common/djangoapps/cache_toolbox/core.py b/common/djangoapps/cache_toolbox/core.py index 208be34a73..a7f0c0819f 100644 --- a/common/djangoapps/cache_toolbox/core.py +++ b/common/djangoapps/cache_toolbox/core.py @@ -107,3 +107,15 @@ def instance_key(model, instance_or_pk): model._meta.module_name, getattr(instance_or_pk, 'pk', instance_or_pk), ) + +def content_key(filename): + return 'content:%s' % (filename) + +def set_cached_content(content): + cache.set(content_key(content.filename), content) + +def get_cached_content(filename): + return cache.get(content_key(filename)) + +def del_cached_content(filename): + cache.delete(content_key(filename)) diff --git a/common/djangoapps/contentserver/__init__.py b/common/djangoapps/contentserver/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/common/djangoapps/contentserver/middleware.py b/common/djangoapps/contentserver/middleware.py new file mode 100644 index 0000000000..56d4ed8d1c --- /dev/null +++ b/common/djangoapps/contentserver/middleware.py @@ -0,0 +1,49 @@ +import logging +import time + +from django.http import HttpResponse, Http404, HttpResponseNotModified + +from xmodule.contentstore.django import contentstore +from xmodule.contentstore.content import StaticContent, XASSET_LOCATION_TAG +from cache_toolbox.core import get_cached_content, set_cached_content +from xmodule.exceptions import NotFoundError + + +class StaticContentServer(object): + def process_request(self, request): + # look to see if the request is prefixed with 'c4x' tag + if request.path.startswith('/' + XASSET_LOCATION_TAG): + + # first look in our cache so we don't have to round-trip to the DB + content = get_cached_content(request.path) + if content is None: + # nope, not in cache, let's fetch from DB + try: + content = contentstore().find(request.path) + except NotFoundError: + raise Http404 + + # since we fetched it from DB, let's cache it going forward + set_cached_content(content) + else: + # @todo: we probably want to have 'cache hit' counters so we can + # measure the efficacy of our caches + pass + + # see if the last-modified at hasn't changed, if not return a 302 (Not Modified) + + # convert over the DB persistent last modified timestamp to a HTTP compatible + # timestamp, so we can simply compare the strings + last_modified_at_str = content.last_modified_at.strftime("%a, %d-%b-%Y %H:%M:%S GMT") + + # see if the client has cached this content, if so then compare the + # timestamps, if they are the same then just return a 304 (Not Modified) + if 'HTTP_IF_MODIFIED_SINCE' in request.META: + if_modified_since = request.META['HTTP_IF_MODIFIED_SINCE'] + if if_modified_since == last_modified_at_str: + return HttpResponseNotModified() + + response = HttpResponse(content.data, content_type=content.content_type) + response['Last-Modified'] = last_modified_at_str + + return response diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py index 4211434619..a821ec9c12 100644 --- a/common/lib/xmodule/xmodule/capa_module.py +++ b/common/lib/xmodule/xmodule/capa_module.py @@ -10,6 +10,7 @@ import sys from datetime import timedelta from lxml import etree +from lxml.html import rewrite_links from pkg_resources import resource_string from capa.capa_problem import LoncapaProblem @@ -332,6 +333,15 @@ class CapaModule(XModule): html = '
'.format( id=self.location.html_id(), ajax_url=self.system.ajax_url) + html + "
" + # cdodge: OK, we have to do two rounds of url reference subsitutions + # one which uses the 'asset library' that is served by the contentstore and the + # more global /static/ filesystem based static content. + # NOTE: rewrite_content_links is defined in XModule + # This is a bit unfortunate and I'm sure we'll try to considate this into + # a one step process. + html = rewrite_links(html, self.rewrite_content_links) + + # now do the substitutions which are filesystem based, e.g. '/static/' prefixes return self.system.replace_urls(html, self.metadata['data_dir']) def handle_ajax(self, dispatch, get): diff --git a/common/lib/xmodule/xmodule/contentstore/__init__.py b/common/lib/xmodule/xmodule/contentstore/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/common/lib/xmodule/xmodule/contentstore/content.py b/common/lib/xmodule/xmodule/contentstore/content.py new file mode 100644 index 0000000000..712c5e7851 --- /dev/null +++ b/common/lib/xmodule/xmodule/contentstore/content.py @@ -0,0 +1,26 @@ +XASSET_LOCATION_TAG = 'c4x' +XASSET_SRCREF_PREFIX = 'xasset:' + +class StaticContent(object): + def __init__(self, filename, name, content_type, data, last_modified_at=None): + self.filename = filename + self.name = name + self.content_type = content_type + self.data = data + self.last_modified_at = last_modified_at + + @staticmethod + def compute_location_filename(org, course, name): + return '/{0}/{1}/{2}/asset/{3}'.format(XASSET_LOCATION_TAG, org, course, name) + +''' +Abstraction for all ContentStore providers (e.g. MongoDB) +''' +class ContentStore(object): + def save(self, content): + raise NotImplementedError + + def find(self, filename): + raise NotImplementedError + + diff --git a/common/lib/xmodule/xmodule/contentstore/django.py b/common/lib/xmodule/xmodule/contentstore/django.py new file mode 100644 index 0000000000..d8b3084135 --- /dev/null +++ b/common/lib/xmodule/xmodule/contentstore/django.py @@ -0,0 +1,29 @@ +from __future__ import absolute_import +from importlib import import_module +from os import environ + +from django.conf import settings + +_CONTENTSTORE = None + +def load_function(path): + """ + Load a function by name. + + path is a string of the form "path.to.module.function" + returns the imported python object `function` from `path.to.module` + """ + module_path, _, name = path.rpartition('.') + return getattr(import_module(module_path), name) + + +def contentstore(): + global _CONTENTSTORE + + if _CONTENTSTORE is None: + class_ = load_function(settings.CONTENTSTORE['ENGINE']) + options = {} + options.update(settings.CONTENTSTORE['OPTIONS']) + _CONTENTSTORE = class_(**options) + + return _CONTENTSTORE diff --git a/common/lib/xmodule/xmodule/contentstore/mongo.py b/common/lib/xmodule/xmodule/contentstore/mongo.py new file mode 100644 index 0000000000..7903a77cb6 --- /dev/null +++ b/common/lib/xmodule/xmodule/contentstore/mongo.py @@ -0,0 +1,32 @@ +from pymongo import Connection +import gridfs +from gridfs.errors import NoFile + +import sys +import logging + +from .content import StaticContent, ContentStore +from xmodule.exceptions import NotFoundError + + +class MongoContentStore(ContentStore): + def __init__(self, host, db, port=27017): + logging.debug( 'Using MongoDB for static content serving at host={0} db={1}'.format(host,db)) + _db = Connection(host=host, port=port)[db] + self.fs = gridfs.GridFS(_db) + + def save(self, content): + with self.fs.new_file(filename=content.filename, content_type=content.content_type, displayname=content.name) as fp: + fp.write(content.data) + return content + + + def find(self, filename): + try: + with self.fs.get_last_version(filename) as fp: + return StaticContent(fp.filename, fp.displayname, fp.content_type, fp.read(), fp.uploadDate) + except NoFile: + raise NotFoundError() + + + diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py index 216b63a12e..0ed64897e2 100644 --- a/common/lib/xmodule/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -4,6 +4,7 @@ import logging import os import sys from lxml import etree +from lxml.html import rewrite_links from path import path from .x_module import XModule @@ -12,6 +13,9 @@ from .xml_module import XmlDescriptor, name_to_pathname from .editing_module import EditingDescriptor from .stringify import stringify_children from .html_checker import check_html +from xmodule.modulestore import Location + +from xmodule.contentstore.content import XASSET_SRCREF_PREFIX, StaticContent log = logging.getLogger("mitx.courseware") @@ -21,7 +25,8 @@ class HtmlModule(XModule): js_module_name = "HTMLModule" def get_html(self): - return self.html + # cdodge: perform link substitutions for any references to course static content (e.g. images) + return rewrite_links(self.html, self.rewrite_content_links) def __init__(self, system, location, definition, descriptor, instance_state=None, shared_state=None, **kwargs): @@ -30,6 +35,7 @@ class HtmlModule(XModule): self.html = self.definition['data'] + class HtmlDescriptor(XmlDescriptor, EditingDescriptor): """ Module for putting raw html in a course diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py index cf08a0b7b2..dd0df2125a 100644 --- a/common/lib/xmodule/xmodule/x_module.py +++ b/common/lib/xmodule/xmodule/x_module.py @@ -12,6 +12,8 @@ from pkg_resources import resource_listdir, resource_string, resource_isdir from xmodule.modulestore import Location from xmodule.timeparse import parse_time +from xmodule.contentstore.content import StaticContent, XASSET_SRCREF_PREFIX + log = logging.getLogger('mitx.' + __name__) @@ -317,6 +319,20 @@ class XModule(HTMLSnippet): get is a dictionary-like object ''' return "" + # cdodge: added to support dynamic substitutions of + # links for courseware assets (e.g. images). is passed through from lxml.html parser + def rewrite_content_links(self, link): + # see if we start with our format, e.g. 'xasset:' + if link.startswith(XASSET_SRCREF_PREFIX): + # yes, then parse out the name + name = link[len(XASSET_SRCREF_PREFIX):] + loc = Location(self.location) + # resolve the reference to our internal 'filepath' which + link = StaticContent.compute_location_filename(loc.org, loc.course, name) + + return link + + def policy_key(location): """ diff --git a/requirements.txt b/requirements.txt index 3376fc1a1d..c3322c5b7c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,3 +48,4 @@ sorl-thumbnail networkx pygraphviz -r repo-requirements.txt +pil