From 85e6c233860e1ed0e34e7c15dfb10b8e4d10e2b0 Mon Sep 17 00:00:00 2001 From: Chris Dodge Date: Wed, 26 Sep 2012 15:24:21 -0400 Subject: [PATCH 1/8] work-in-flight for uploading/serving of static content for courses --- cms/djangoapps/contentstore/views.py | 82 +++++++++++++++++++++++-- cms/envs/common.py | 3 +- cms/envs/dev.py | 11 ++++ cms/templates/course_index.html | 3 + cms/urls.py | 4 +- common/djangoapps/cache_toolbox/core.py | 17 +++++ 6 files changed, 113 insertions(+), 7 deletions(-) diff --git a/cms/djangoapps/contentstore/views.py b/cms/djangoapps/contentstore/views.py index da6611f248..6692192590 100644 --- a/cms/djangoapps/contentstore/views.py +++ b/cms/djangoapps/contentstore/views.py @@ -2,14 +2,16 @@ from util.json_request import expect_json import json import logging import sys +import mimetypes from collections import defaultdict -from django.http import HttpResponse, Http404 +from django.http import HttpResponse, Http404, HttpResponseBadRequest, HttpResponseForbidden from django.contrib.auth.decorators import login_required from django.core.context_processors import csrf from django_future.csrf import ensure_csrf_cookie from django.core.urlresolvers import reverse from django.conf import settings +from django import forms from xmodule.modulestore import Location from xmodule.x_module import ModuleSystem @@ -26,6 +28,13 @@ from functools import partial from itertools import groupby from operator import attrgetter +from xmodule.contentstore.django import contentstore +from xmodule.contentstore import StaticContent + +#from django.core.cache import cache + +from cache_toolbox.core import set_cached_content, get_cached_content + log = logging.getLogger(__name__) @@ -89,9 +98,19 @@ def course_index(request, org, course, name): raise Http404 # TODO (vshnayder): better error # TODO (cpennington): These need to be read in from the active user - course = modulestore().get_item(location) - weeks = course.get_children() - return render_to_response('course_index.html', {'weeks': weeks}) + _course = modulestore().get_item(location) + weeks = _course.get_children() + + upload_asset_callback_url = "/{org}/{course}/course/{name}/upload_asset".format( + org = org, + course = course, + name = name + ) + + return render_to_response('course_index.html', { + 'weeks': weeks, + 'upload_asset_callback_url': upload_asset_callback_url + }) @login_required @@ -115,12 +134,13 @@ def edit_item(request): lms_link = "{lms_base}/courses/{course_id}/jump_to/{location}".format( lms_base=settings.LMS_BASE, # TODO: These will need to be changed to point to the particular instance of this problem in the particular course - course_id=modulestore().get_containing_courses(item.location)[0].id, + course_id= modulestore().get_containing_courses(item.location)[0].id, location=item.location, ) else: lms_link = None + return render_to_response('unit.html', { 'contents': item.get_html(), 'js_module': item.js_module_name, @@ -390,3 +410,55 @@ def clone_item(request): modulestore().update_children(parent_location, parent.definition.get('children', []) + [new_item.location.url()]) return HttpResponse() + +''' +cdodge: this method allows for POST uploading of files into the course asset library, which will +be supported by GridFS in MongoDB. +''' +#@login_required +#@ensure_csrf_cookie +def upload_asset(request, org, course, coursename): + + if request.method != 'POST': + # (cdodge) @todo: Is there a way to do a - say - 'raise Http400'? + return HttpResponseBadRequest() + + # construct a location from the passed in path + location = ['i4x', org, course, 'course', coursename] + if not has_access(request.user, location): + return HttpResponseForbidden() + + # Does the course actually exist?!? + + try: + item = modulestore().get_item(location) + except: + # no return it as a Bad Request response + logging.error('Could not find course' + location) + return HttpResponseBadRequest() + + # compute a 'filename' which is similar to the location formatting, we're using the 'filename' + # nomenclature since we're using a FileSystem paradigm here. We're just imposing + # the Location string formatting expectations to keep things a bit more consistent + + name = request.FILES['file'].name + mime_type = request.FILES['file'].content_type + filedata = request.FILES['file'].read() + + file_location = StaticContent.compute_location_filename(org, course, name) + + content = StaticContent(file_location, name, mime_type, filedata) + + # first commit to the DB + contentstore().update(content) + + # then update the cache so we're not serving up stale content + set_cached_content(content) + + return HttpResponse('Upload completed') + + + +class UploadFileForm(forms.Form): + title = forms.CharField(max_length=50) + file = forms.FileField() diff --git a/cms/envs/common.py b/cms/envs/common.py index dc82af85af..7190ba9e51 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -118,6 +118,7 @@ TEMPLATE_LOADERS = ( ) MIDDLEWARE_CLASSES = ( + 'contentserver.middleware.StaticContentServer', 'django.middleware.cache.UpdateCacheMiddleware', 'django.middleware.common.CommonMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', @@ -130,7 +131,7 @@ MIDDLEWARE_CLASSES = ( 'track.middleware.TrackMiddleware', 'mitxmako.middleware.MakoMiddleware', - 'django.middleware.transaction.TransactionMiddleware', + 'django.middleware.transaction.TransactionMiddleware' ) ############################ SIGNAL HANDLERS ################################ diff --git a/cms/envs/dev.py b/cms/envs/dev.py index fc2a5a5684..dd0e0337f6 100644 --- a/cms/envs/dev.py +++ b/cms/envs/dev.py @@ -28,6 +28,17 @@ MODULESTORE = { } } +# cdodge: This is the specifier for the MongoDB (using GridFS) backed static content store +# This is for static content for courseware, not system static content (e.g. javascript, css, edX branding, etc) +CONTENTSTORE = { + 'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore', + 'OPTIONS': { + 'host': 'localhost', + 'db' : 'xcontent', + } +} + + DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', diff --git a/cms/templates/course_index.html b/cms/templates/course_index.html index e490ad7817..347e93ef4a 100644 --- a/cms/templates/course_index.html +++ b/cms/templates/course_index.html @@ -7,8 +7,11 @@ <%include file="widgets/navigation.html"/> + <%include file="widgets/upload_assets.html"/> +
+ diff --git a/cms/urls.py b/cms/urls.py index e51ae59b08..ddd54adc65 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -17,7 +17,9 @@ urlpatterns = ('', 'contentstore.views.course_index', name='course_index'), url(r'^github_service_hook$', 'github_sync.views.github_post_receive'), url(r'^preview/modx/(?P[^/]*)/(?P.*?)/(?P[^/]*)$', - 'contentstore.views.preview_dispatch', name='preview_dispatch') + 'contentstore.views.preview_dispatch', name='preview_dispatch'), + url(r'^(?P[^/]+)/(?P[^/]+)/course/(?P[^/]+)/upload_asset$', + 'contentstore.views.upload_asset', name='upload_asset') ) # User creation and updating views diff --git a/common/djangoapps/cache_toolbox/core.py b/common/djangoapps/cache_toolbox/core.py index 208be34a73..85e1c8a246 100644 --- a/common/djangoapps/cache_toolbox/core.py +++ b/common/djangoapps/cache_toolbox/core.py @@ -10,6 +10,7 @@ Core methods from django.core.cache import cache from django.db import DEFAULT_DB_ALIAS +from xmodule.contentstore import StaticContent from . import app_settings @@ -107,3 +108,19 @@ def instance_key(model, instance_or_pk): model._meta.module_name, getattr(instance_or_pk, 'pk', instance_or_pk), ) + +def content_key(filename): + return 'content:%s' % (filename) + +def set_cached_content(content): + cache.set(content_key(content.filename), content) + +def get_cached_content(filename): + return cache.get(content_key(filename)) + + +#def set_cached_content(filename, content_type, data): +# cache.set(content_key(filename), (filename, content_type, data)) + +#def get_cached_content(filename): +# return cache.get(content_key(filename)) From 7512b78eb2e81ac4615e194cd496b4ca773d55f0 Mon Sep 17 00:00:00 2001 From: Chris Dodge Date: Wed, 26 Sep 2012 15:26:13 -0400 Subject: [PATCH 2/8] work-in-flight for uploading/serving of static content for courses --- cms/templates/widgets/upload_assets.html | 44 +++++++++++++++++++ common/djangoapps/contentserver/__init__.py | 0 common/djangoapps/contentserver/middleware.py | 36 +++++++++++++++ .../xmodule/xmodule/contentstore/__init__.py | 15 +++++++ .../xmodule/xmodule/contentstore/django.py | 29 ++++++++++++ .../lib/xmodule/xmodule/contentstore/mongo.py | 32 ++++++++++++++ 6 files changed, 156 insertions(+) create mode 100644 cms/templates/widgets/upload_assets.html create mode 100644 common/djangoapps/contentserver/__init__.py create mode 100644 common/djangoapps/contentserver/middleware.py create mode 100644 common/lib/xmodule/xmodule/contentstore/__init__.py create mode 100644 common/lib/xmodule/xmodule/contentstore/django.py create mode 100644 common/lib/xmodule/xmodule/contentstore/mongo.py diff --git a/cms/templates/widgets/upload_assets.html b/cms/templates/widgets/upload_assets.html new file mode 100644 index 0000000000..876166fdf7 --- /dev/null +++ b/cms/templates/widgets/upload_assets.html @@ -0,0 +1,44 @@ +
+
+ You can upload file assets to reference in your courseware +
+ + +
+
+
+
+
0%
+
+ +
+ +
+ + + diff --git a/common/djangoapps/contentserver/__init__.py b/common/djangoapps/contentserver/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/common/djangoapps/contentserver/middleware.py b/common/djangoapps/contentserver/middleware.py new file mode 100644 index 0000000000..48b760a8d3 --- /dev/null +++ b/common/djangoapps/contentserver/middleware.py @@ -0,0 +1,36 @@ +import logging + +from django.http import HttpResponse, Http404 + +from xmodule.contentstore.django import contentstore +from xmodule.contentstore import StaticContent +from cache_toolbox.core import get_cached_content, set_cached_content +from xmodule.exceptions import NotFoundError + + +class StaticContentServer(object): + def __init__(self): + self.match_tag = StaticContent.get_location_tag() + + def process_request(self, request): + # look to see if the request is prefixed with 'c4x' tag + if request.path.startswith('/' + self.match_tag): + + # first look in our cache so we don't have to round-trip to the DB + content = get_cached_content(request.path) + if content is None: + # nope, not in cache, let's fetch from DB + try: + content = contentstore().find(request.path) + except NotFoundError: + raise Http404 + + # since we fetched it from DB, let's cache it going forward + set_cached_content(content) + else: + logging.debug('cache hit on {0}'.format(content.filename)) + + response = HttpResponse(content.data, content_type=content.content_type) + response['Content-Disposition'] = 'attachment; filename={0}'.format(content.name) + + return response diff --git a/common/lib/xmodule/xmodule/contentstore/__init__.py b/common/lib/xmodule/xmodule/contentstore/__init__.py new file mode 100644 index 0000000000..08658ea721 --- /dev/null +++ b/common/lib/xmodule/xmodule/contentstore/__init__.py @@ -0,0 +1,15 @@ +class StaticContent(object): + def __init__(self, filename, name, content_type, data): + self.filename = filename + self.name = name + self.content_type = content_type + self.data = data + + @staticmethod + def get_location_tag(): + return 'c4x' + + @staticmethod + def compute_location_filename(org, course, name): + return '/{0}/{1}/{2}/asset/{3}'.format(StaticContent.get_location_tag(), org, course, name) + diff --git a/common/lib/xmodule/xmodule/contentstore/django.py b/common/lib/xmodule/xmodule/contentstore/django.py new file mode 100644 index 0000000000..d8b3084135 --- /dev/null +++ b/common/lib/xmodule/xmodule/contentstore/django.py @@ -0,0 +1,29 @@ +from __future__ import absolute_import +from importlib import import_module +from os import environ + +from django.conf import settings + +_CONTENTSTORE = None + +def load_function(path): + """ + Load a function by name. + + path is a string of the form "path.to.module.function" + returns the imported python object `function` from `path.to.module` + """ + module_path, _, name = path.rpartition('.') + return getattr(import_module(module_path), name) + + +def contentstore(): + global _CONTENTSTORE + + if _CONTENTSTORE is None: + class_ = load_function(settings.CONTENTSTORE['ENGINE']) + options = {} + options.update(settings.CONTENTSTORE['OPTIONS']) + _CONTENTSTORE = class_(**options) + + return _CONTENTSTORE diff --git a/common/lib/xmodule/xmodule/contentstore/mongo.py b/common/lib/xmodule/xmodule/contentstore/mongo.py new file mode 100644 index 0000000000..eddb162539 --- /dev/null +++ b/common/lib/xmodule/xmodule/contentstore/mongo.py @@ -0,0 +1,32 @@ +from pymongo import Connection +import gridfs +from gridfs.errors import NoFile + +import sys +import logging + +from . import StaticContent +from xmodule.exceptions import NotFoundError + + +class MongoContentStore(object): + def __init__(self, host, db, port=27017): + logging.debug( 'Using MongoDB for static content serving at host={0} db={1}'.format(host,db)) + _db = Connection(host=host, port=port)[db] + self.fs = gridfs.GridFS(_db) + + def update(self, content): + with self.fs.new_file(filename=content.filename, content_type=content.content_type, displayname=content.name) as fp: + fp.write(content.data) + return content + + def find(self, filename): + try: + with self.fs.get_last_version(filename) as fp: + logging.debug('fetched {0}'.format(fp.name)) + return StaticContent(fp.filename, fp.displayname, fp.content_type, fp.read()) + except NoFile: + raise NotFoundError() + + + From 7bf7eab616cc2d17941ee715c6885755f1ad3483 Mon Sep 17 00:00:00 2001 From: Chris Dodge Date: Wed, 26 Sep 2012 16:39:02 -0400 Subject: [PATCH 3/8] support client-side caching through 'Last-Modified' response headers --- common/djangoapps/contentserver/middleware.py | 22 ++++++++++++++++--- .../xmodule/xmodule/contentstore/__init__.py | 3 ++- .../lib/xmodule/xmodule/contentstore/mongo.py | 3 +-- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/common/djangoapps/contentserver/middleware.py b/common/djangoapps/contentserver/middleware.py index 48b760a8d3..f0734dd202 100644 --- a/common/djangoapps/contentserver/middleware.py +++ b/common/djangoapps/contentserver/middleware.py @@ -1,6 +1,7 @@ import logging +import time -from django.http import HttpResponse, Http404 +from django.http import HttpResponse, Http404, HttpResponseNotModified from xmodule.contentstore.django import contentstore from xmodule.contentstore import StaticContent @@ -30,7 +31,22 @@ class StaticContentServer(object): else: logging.debug('cache hit on {0}'.format(content.filename)) + # see if the last-modified at hasn't changed, if not return a 302 (Not Modified) + + logging.debug(request.META) + + # convert over the DB persistent last modified timestamp to a HTTP compatible + # timestamp + last_modified_at_str = content.last_modified_at.strftime("%a, %d-%b-%Y %H:%M:%S GMT") + + # see if the client has cached this content, if so then compare the + # timestamps, if they are the same then just return a 304 (Not Modified) + if 'HTTP_IF_MODIFIED_SINCE' in request.META: + if_modified_since = request.META['HTTP_IF_MODIFIED_SINCE'] + if if_modified_since == last_modified_at_str: + return HttpResponseNotModified() + response = HttpResponse(content.data, content_type=content.content_type) - response['Content-Disposition'] = 'attachment; filename={0}'.format(content.name) - + response['Last-Modified'] = last_modified_at_str + return response diff --git a/common/lib/xmodule/xmodule/contentstore/__init__.py b/common/lib/xmodule/xmodule/contentstore/__init__.py index 08658ea721..e48c2d5303 100644 --- a/common/lib/xmodule/xmodule/contentstore/__init__.py +++ b/common/lib/xmodule/xmodule/contentstore/__init__.py @@ -1,9 +1,10 @@ class StaticContent(object): - def __init__(self, filename, name, content_type, data): + def __init__(self, filename, name, content_type, data, last_modified_at=None): self.filename = filename self.name = name self.content_type = content_type self.data = data + self.last_modified_at = last_modified_at @staticmethod def get_location_tag(): diff --git a/common/lib/xmodule/xmodule/contentstore/mongo.py b/common/lib/xmodule/xmodule/contentstore/mongo.py index eddb162539..12d783366d 100644 --- a/common/lib/xmodule/xmodule/contentstore/mongo.py +++ b/common/lib/xmodule/xmodule/contentstore/mongo.py @@ -23,8 +23,7 @@ class MongoContentStore(object): def find(self, filename): try: with self.fs.get_last_version(filename) as fp: - logging.debug('fetched {0}'.format(fp.name)) - return StaticContent(fp.filename, fp.displayname, fp.content_type, fp.read()) + return StaticContent(fp.filename, fp.displayname, fp.content_type, fp.read(), fp.uploadDate) except NoFile: raise NotFoundError() From 39e64c1e5658b17adc416f2c5f2b3773d8efb2b8 Mon Sep 17 00:00:00 2001 From: Chris Dodge Date: Thu, 27 Sep 2012 09:55:30 -0400 Subject: [PATCH 4/8] work in progress. Need to commit before rebasing to master --- cms/djangoapps/contentstore/views.py | 7 +------ cms/templates/widgets/upload_assets.html | 4 ++-- common/djangoapps/contentserver/middleware.py | 4 +--- common/lib/xmodule/xmodule/html_module.py | 10 +++++++++- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/cms/djangoapps/contentstore/views.py b/cms/djangoapps/contentstore/views.py index 6692192590..fcc77a02c0 100644 --- a/cms/djangoapps/contentstore/views.py +++ b/cms/djangoapps/contentstore/views.py @@ -353,7 +353,7 @@ def save_item(request): if request.POST['data']: data = request.POST['data'] modulestore().update_item(item_location, data) - + if request.POST['children']: children = request.POST['children'] modulestore().update_children(item_location, children) @@ -457,8 +457,3 @@ def upload_asset(request, org, course, coursename): return HttpResponse('Upload completed') - - -class UploadFileForm(forms.Form): - title = forms.CharField(max_length=50) - file = forms.FileField() diff --git a/cms/templates/widgets/upload_assets.html b/cms/templates/widgets/upload_assets.html index 876166fdf7..75414709d1 100644 --- a/cms/templates/widgets/upload_assets.html +++ b/cms/templates/widgets/upload_assets.html @@ -6,8 +6,8 @@ -
-
+
+
0%
diff --git a/common/djangoapps/contentserver/middleware.py b/common/djangoapps/contentserver/middleware.py index f0734dd202..d30a3f7900 100644 --- a/common/djangoapps/contentserver/middleware.py +++ b/common/djangoapps/contentserver/middleware.py @@ -33,10 +33,8 @@ class StaticContentServer(object): # see if the last-modified at hasn't changed, if not return a 302 (Not Modified) - logging.debug(request.META) - # convert over the DB persistent last modified timestamp to a HTTP compatible - # timestamp + # timestamp, so we can simply compare the strings last_modified_at_str = content.last_modified_at.strftime("%a, %d-%b-%Y %H:%M:%S GMT") # see if the client has cached this content, if so then compare the diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py index f5672ca1b8..471a1ea7fd 100644 --- a/common/lib/xmodule/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -4,6 +4,7 @@ import logging import os import sys from lxml import etree +from lxml.html import rewrite_links from path import path from .x_module import XModule @@ -18,7 +19,9 @@ log = logging.getLogger("mitx.courseware") class HtmlModule(XModule): def get_html(self): - return self.html + # cdodge: perform link substitutions for any references to course static content (e.g. images) + return rewrite_links(self.html, self.rewrite_content_links, self) + #return self.html def __init__(self, system, location, definition, descriptor, instance_state=None, shared_state=None, **kwargs): @@ -26,6 +29,11 @@ class HtmlModule(XModule): instance_state, shared_state, **kwargs) self.html = self.definition['data'] + def rewrite_content_links(link, self): + if link.startswith('xasset:'): + logging.debug('found link: {0}'.format(link)) + return link + class HtmlDescriptor(XmlDescriptor, EditingDescriptor): """ From adad3c550b343dc361ca5f42ac90743ca7d2dec5 Mon Sep 17 00:00:00 2001 From: Chris Dodge Date: Thu, 27 Sep 2012 12:31:45 -0400 Subject: [PATCH 5/8] Support asset library references in CapaModule. Also some code refactoring/cleanup. Preparing for review and submit --- cms/djangoapps/contentstore/views.py | 15 +++++++---- cms/templates/widgets/upload_assets.html | 2 +- common/djangoapps/cache_toolbox/core.py | 9 ++----- common/djangoapps/contentserver/middleware.py | 7 ++--- common/lib/xmodule/xmodule/capa_module.py | 10 +++++++ .../xmodule/xmodule/contentstore/__init__.py | 16 ------------ .../xmodule/xmodule/contentstore/content.py | 26 +++++++++++++++++++ .../lib/xmodule/xmodule/contentstore/mongo.py | 7 ++--- common/lib/xmodule/xmodule/html_module.py | 10 +++---- common/lib/xmodule/xmodule/x_module.py | 16 ++++++++++++ 10 files changed, 75 insertions(+), 43 deletions(-) create mode 100644 common/lib/xmodule/xmodule/contentstore/content.py diff --git a/cms/djangoapps/contentstore/views.py b/cms/djangoapps/contentstore/views.py index fcc77a02c0..d8cbe0c7a9 100644 --- a/cms/djangoapps/contentstore/views.py +++ b/cms/djangoapps/contentstore/views.py @@ -29,11 +29,11 @@ from itertools import groupby from operator import attrgetter from xmodule.contentstore.django import contentstore -from xmodule.contentstore import StaticContent +from xmodule.contentstore.content import StaticContent #from django.core.cache import cache -from cache_toolbox.core import set_cached_content, get_cached_content +from cache_toolbox.core import set_cached_content, get_cached_content, del_cached_content log = logging.getLogger(__name__) @@ -450,10 +450,15 @@ def upload_asset(request, org, course, coursename): content = StaticContent(file_location, name, mime_type, filedata) # first commit to the DB - contentstore().update(content) + contentstore().save(content) - # then update the cache so we're not serving up stale content - set_cached_content(content) + # then remove the cache so we're not serving up stale content + # NOTE: we're not re-populating the cache here as the DB owns the last-modified timestamp + # which is used when serving up static content. This integrity is needed for + # browser-side caching support. We *could* re-fetch the saved content so that we have the + # timestamp populated, but we might as well wait for the first real request to come in + # to re-populate the cache. + del_cached_content(file_location) return HttpResponse('Upload completed') diff --git a/cms/templates/widgets/upload_assets.html b/cms/templates/widgets/upload_assets.html index 75414709d1..5072adedab 100644 --- a/cms/templates/widgets/upload_assets.html +++ b/cms/templates/widgets/upload_assets.html @@ -6,7 +6,7 @@
-
+
0%
diff --git a/common/djangoapps/cache_toolbox/core.py b/common/djangoapps/cache_toolbox/core.py index 85e1c8a246..a7f0c0819f 100644 --- a/common/djangoapps/cache_toolbox/core.py +++ b/common/djangoapps/cache_toolbox/core.py @@ -10,7 +10,6 @@ Core methods from django.core.cache import cache from django.db import DEFAULT_DB_ALIAS -from xmodule.contentstore import StaticContent from . import app_settings @@ -118,9 +117,5 @@ def set_cached_content(content): def get_cached_content(filename): return cache.get(content_key(filename)) - -#def set_cached_content(filename, content_type, data): -# cache.set(content_key(filename), (filename, content_type, data)) - -#def get_cached_content(filename): -# return cache.get(content_key(filename)) +def del_cached_content(filename): + cache.delete(content_key(filename)) diff --git a/common/djangoapps/contentserver/middleware.py b/common/djangoapps/contentserver/middleware.py index d30a3f7900..1f0cdbba2a 100644 --- a/common/djangoapps/contentserver/middleware.py +++ b/common/djangoapps/contentserver/middleware.py @@ -4,18 +4,15 @@ import time from django.http import HttpResponse, Http404, HttpResponseNotModified from xmodule.contentstore.django import contentstore -from xmodule.contentstore import StaticContent +from xmodule.contentstore.content import StaticContent, XASSET_LOCATION_TAG from cache_toolbox.core import get_cached_content, set_cached_content from xmodule.exceptions import NotFoundError class StaticContentServer(object): - def __init__(self): - self.match_tag = StaticContent.get_location_tag() - def process_request(self, request): # look to see if the request is prefixed with 'c4x' tag - if request.path.startswith('/' + self.match_tag): + if request.path.startswith('/' + XASSET_LOCATION_TAG): # first look in our cache so we don't have to round-trip to the DB content = get_cached_content(request.path) diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py index ae7ebc3b78..261ab5d53d 100644 --- a/common/lib/xmodule/xmodule/capa_module.py +++ b/common/lib/xmodule/xmodule/capa_module.py @@ -10,6 +10,7 @@ import sys from datetime import timedelta from lxml import etree +from lxml.html import rewrite_links from pkg_resources import resource_string from capa.capa_problem import LoncapaProblem @@ -332,6 +333,15 @@ class CapaModule(XModule): html = '
'.format( id=self.location.html_id(), ajax_url=self.system.ajax_url) + html + "
" + # cdodge: OK, we have to do two rounds of url reference subsitutions + # one which uses the 'asset library' that is served by the contentstore and the + # more global /static/ filesystem based static content. + # NOTE: rewrite_content_links is defined in XModule + # This is a bit unfortunate and I'm sure we'll try to considate this into + # a one step process. + html = rewrite_links(html, self.rewrite_content_links) + + # now do the substitutions which are filesystem based, e.g. '/static/' prefixes return self.system.replace_urls(html, self.metadata['data_dir']) def handle_ajax(self, dispatch, get): diff --git a/common/lib/xmodule/xmodule/contentstore/__init__.py b/common/lib/xmodule/xmodule/contentstore/__init__.py index e48c2d5303..e69de29bb2 100644 --- a/common/lib/xmodule/xmodule/contentstore/__init__.py +++ b/common/lib/xmodule/xmodule/contentstore/__init__.py @@ -1,16 +0,0 @@ -class StaticContent(object): - def __init__(self, filename, name, content_type, data, last_modified_at=None): - self.filename = filename - self.name = name - self.content_type = content_type - self.data = data - self.last_modified_at = last_modified_at - - @staticmethod - def get_location_tag(): - return 'c4x' - - @staticmethod - def compute_location_filename(org, course, name): - return '/{0}/{1}/{2}/asset/{3}'.format(StaticContent.get_location_tag(), org, course, name) - diff --git a/common/lib/xmodule/xmodule/contentstore/content.py b/common/lib/xmodule/xmodule/contentstore/content.py new file mode 100644 index 0000000000..712c5e7851 --- /dev/null +++ b/common/lib/xmodule/xmodule/contentstore/content.py @@ -0,0 +1,26 @@ +XASSET_LOCATION_TAG = 'c4x' +XASSET_SRCREF_PREFIX = 'xasset:' + +class StaticContent(object): + def __init__(self, filename, name, content_type, data, last_modified_at=None): + self.filename = filename + self.name = name + self.content_type = content_type + self.data = data + self.last_modified_at = last_modified_at + + @staticmethod + def compute_location_filename(org, course, name): + return '/{0}/{1}/{2}/asset/{3}'.format(XASSET_LOCATION_TAG, org, course, name) + +''' +Abstraction for all ContentStore providers (e.g. MongoDB) +''' +class ContentStore(object): + def save(self, content): + raise NotImplementedError + + def find(self, filename): + raise NotImplementedError + + diff --git a/common/lib/xmodule/xmodule/contentstore/mongo.py b/common/lib/xmodule/xmodule/contentstore/mongo.py index 12d783366d..7903a77cb6 100644 --- a/common/lib/xmodule/xmodule/contentstore/mongo.py +++ b/common/lib/xmodule/xmodule/contentstore/mongo.py @@ -5,20 +5,21 @@ from gridfs.errors import NoFile import sys import logging -from . import StaticContent +from .content import StaticContent, ContentStore from xmodule.exceptions import NotFoundError -class MongoContentStore(object): +class MongoContentStore(ContentStore): def __init__(self, host, db, port=27017): logging.debug( 'Using MongoDB for static content serving at host={0} db={1}'.format(host,db)) _db = Connection(host=host, port=port)[db] self.fs = gridfs.GridFS(_db) - def update(self, content): + def save(self, content): with self.fs.new_file(filename=content.filename, content_type=content.content_type, displayname=content.name) as fp: fp.write(content.data) return content + def find(self, filename): try: diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py index 471a1ea7fd..f9098b4e0e 100644 --- a/common/lib/xmodule/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -13,6 +13,9 @@ from .xml_module import XmlDescriptor, name_to_pathname from .editing_module import EditingDescriptor from .stringify import stringify_children from .html_checker import check_html +from xmodule.modulestore import Location + +from xmodule.contentstore.content import XASSET_SRCREF_PREFIX, StaticContent log = logging.getLogger("mitx.courseware") @@ -20,8 +23,7 @@ log = logging.getLogger("mitx.courseware") class HtmlModule(XModule): def get_html(self): # cdodge: perform link substitutions for any references to course static content (e.g. images) - return rewrite_links(self.html, self.rewrite_content_links, self) - #return self.html + return rewrite_links(self.html, self.rewrite_content_links) def __init__(self, system, location, definition, descriptor, instance_state=None, shared_state=None, **kwargs): @@ -29,10 +31,6 @@ class HtmlModule(XModule): instance_state, shared_state, **kwargs) self.html = self.definition['data'] - def rewrite_content_links(link, self): - if link.startswith('xasset:'): - logging.debug('found link: {0}'.format(link)) - return link class HtmlDescriptor(XmlDescriptor, EditingDescriptor): diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py index cf08a0b7b2..dd0df2125a 100644 --- a/common/lib/xmodule/xmodule/x_module.py +++ b/common/lib/xmodule/xmodule/x_module.py @@ -12,6 +12,8 @@ from pkg_resources import resource_listdir, resource_string, resource_isdir from xmodule.modulestore import Location from xmodule.timeparse import parse_time +from xmodule.contentstore.content import StaticContent, XASSET_SRCREF_PREFIX + log = logging.getLogger('mitx.' + __name__) @@ -317,6 +319,20 @@ class XModule(HTMLSnippet): get is a dictionary-like object ''' return "" + # cdodge: added to support dynamic substitutions of + # links for courseware assets (e.g. images). is passed through from lxml.html parser + def rewrite_content_links(self, link): + # see if we start with our format, e.g. 'xasset:' + if link.startswith(XASSET_SRCREF_PREFIX): + # yes, then parse out the name + name = link[len(XASSET_SRCREF_PREFIX):] + loc = Location(self.location) + # resolve the reference to our internal 'filepath' which + link = StaticContent.compute_location_filename(loc.org, loc.course, name) + + return link + + def policy_key(location): """ From 1ee7646cfceb07354fb0a805737dd576e0aad944 Mon Sep 17 00:00:00 2001 From: Chris Dodge Date: Fri, 28 Sep 2012 10:20:57 -0400 Subject: [PATCH 6/8] per feedback, use reverse() when generating callback urls. Also generate thumbnails when ingesting static images. We'll need these handy for the CMS UI. --- cms/djangoapps/contentstore/views.py | 61 +++++++++++++++++-- common/djangoapps/contentserver/middleware.py | 4 +- 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/cms/djangoapps/contentstore/views.py b/cms/djangoapps/contentstore/views.py index d8cbe0c7a9..3223bfda29 100644 --- a/cms/djangoapps/contentstore/views.py +++ b/cms/djangoapps/contentstore/views.py @@ -1,10 +1,15 @@ from util.json_request import expect_json import json +import os import logging import sys import mimetypes +import StringIO from collections import defaultdict +# to install PIL on MacOSX: 'easy_install http://dist.repoze.org/PIL-1.1.6.tar.gz' +from PIL import Image + from django.http import HttpResponse, Http404, HttpResponseBadRequest, HttpResponseForbidden from django.contrib.auth.decorators import login_required from django.core.context_processors import csrf @@ -101,11 +106,18 @@ def course_index(request, org, course, name): _course = modulestore().get_item(location) weeks = _course.get_children() - upload_asset_callback_url = "/{org}/{course}/course/{name}/upload_asset".format( - org = org, - course = course, - name = name - ) + #upload_asset_callback_url = "/{org}/{course}/course/{name}/upload_asset".format( + # org = org, + # course = course, + # name = name + # ) + + upload_asset_callback_url = reverse('upload_asset', kwargs = { + 'org' : org, + 'course' : course, + 'coursename' : name + }) + logging.debug(upload_asset_callback_url) return render_to_response('course_index.html', { 'weeks': weeks, @@ -460,5 +472,44 @@ def upload_asset(request, org, course, coursename): # to re-populate the cache. del_cached_content(file_location) + # if we're uploading an image, then let's generate a thumbnail so that we can + # serve it up when needed without having to rescale on the fly + if mime_type.split('/')[0] == 'image': + try: + # not sure if this is necessary, but let's rewind the stream just in case + request.FILES['file'].seek(0) + + # use PIL to do the thumbnail generation (http://www.pythonware.com/products/pil/) + # My understanding is that PIL will maintain aspect ratios while restricting + # the max-height/width to be whatever you pass in as 'size' + # @todo: move the thumbnail size to a configuration setting?!? + im = Image.open(request.FILES['file']) + + # I've seen some exceptions from the PIL library when trying to save palletted + # PNG files to JPEG. Per the google-universe, they suggest converting to RGB first. + im = im.convert('RGB') + size = 128, 128 + im.thumbnail(size, Image.ANTIALIAS) + thumbnail_file = StringIO.StringIO() + im.save(thumbnail_file, 'JPEG') + thumbnail_file.seek(0) + + # use a naming convention to associate originals with the thumbnail + # .thumbnail.jpg + thumbnail_name = os.path.splitext(name)[0] + '.thumbnail.jpg' + # then just store this thumbnail as any other piece of content + thumbnail_file_location = StaticContent.compute_location_filename(org, course, + thumbnail_name) + thumbnail_content = StaticContent(thumbnail_file_location, thumbnail_name, + 'image/jpeg', thumbnail_file) + contentstore().save(thumbnail_content) + + # remove any cached content at this location, as thumbnails are treated just like any + # other bit of static content + del_cached_content(thumbnail_file_location) + except: + # catch, log, and continue as thumbnails are not a hard requirement + logging.error('Failed to generate thumbnail for {0}. Continuing...'.format(name)) + return HttpResponse('Upload completed') diff --git a/common/djangoapps/contentserver/middleware.py b/common/djangoapps/contentserver/middleware.py index 1f0cdbba2a..56d4ed8d1c 100644 --- a/common/djangoapps/contentserver/middleware.py +++ b/common/djangoapps/contentserver/middleware.py @@ -26,7 +26,9 @@ class StaticContentServer(object): # since we fetched it from DB, let's cache it going forward set_cached_content(content) else: - logging.debug('cache hit on {0}'.format(content.filename)) + # @todo: we probably want to have 'cache hit' counters so we can + # measure the efficacy of our caches + pass # see if the last-modified at hasn't changed, if not return a 302 (Not Modified) From a92a4df1bc8c0bc8ce051bc093289c59e7cd2f78 Mon Sep 17 00:00:00 2001 From: Chris Dodge Date: Fri, 28 Sep 2012 11:22:35 -0400 Subject: [PATCH 7/8] Add PIL (Python Image Library) to the requirements.txt file --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 3376fc1a1d..c3322c5b7c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,3 +48,4 @@ sorl-thumbnail networkx pygraphviz -r repo-requirements.txt +pil From 84a06037547fd6755d79d0b4f87e8142f3edb9e8 Mon Sep 17 00:00:00 2001 From: Chris Dodge Date: Fri, 28 Sep 2012 11:42:11 -0400 Subject: [PATCH 8/8] shimmy around the upload_asset widget to at least not mess up the edit unit rendering. Now it'll always be at the bottom --- cms/templates/course_index.html | 3 +-- cms/templates/widgets/upload_assets.html | 16 ++++++++-------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/cms/templates/course_index.html b/cms/templates/course_index.html index 347e93ef4a..37b5a8b371 100644 --- a/cms/templates/course_index.html +++ b/cms/templates/course_index.html @@ -7,11 +7,10 @@ <%include file="widgets/navigation.html"/> - <%include file="widgets/upload_assets.html"/> -
+ <%include file="widgets/upload_assets.html"/>
diff --git a/cms/templates/widgets/upload_assets.html b/cms/templates/widgets/upload_assets.html index 5072adedab..510313c905 100644 --- a/cms/templates/widgets/upload_assets.html +++ b/cms/templates/widgets/upload_assets.html @@ -1,19 +1,19 @@
- You can upload file assets to reference in your courseware + You can upload file assets (such as images) to reference in your courseware
-
-
-
-
0%
+
+
+
0%
+
+ +
-
- -
+