From a84ef1b24369ebc1734a73750e5cf6bc96bbfa33 Mon Sep 17 00:00:00 2001 From: Chris Dodge Date: Thu, 11 Oct 2012 12:10:28 -0400 Subject: [PATCH] wip: static asset import --- .../management/commands/import.py | 3 +- .../xmodule/xmodule/contentstore/content.py | 48 +++++++++++++++++++ .../xmodule/modulestore/xml_importer.py | 47 +++++++++++++++++- 3 files changed, 96 insertions(+), 2 deletions(-) diff --git a/cms/djangoapps/contentstore/management/commands/import.py b/cms/djangoapps/contentstore/management/commands/import.py index a8ec2c2685..d15abc53bd 100644 --- a/cms/djangoapps/contentstore/management/commands/import.py +++ b/cms/djangoapps/contentstore/management/commands/import.py @@ -5,6 +5,7 @@ from django.core.management.base import BaseCommand, CommandError from xmodule.modulestore.xml_importer import import_from_xml from xmodule.modulestore.django import modulestore +from xmodule.contentstore.django import contentstore unnamed_modules = 0 @@ -26,4 +27,4 @@ class Command(BaseCommand): print "Importing. Data_dir={data}, course_dirs={courses}".format( data=data_dir, courses=course_dirs) - import_from_xml(modulestore('direct'), data_dir, course_dirs, load_error_modules=False) + import_from_xml(modulestore('direct'), data_dir, course_dirs, load_error_modules=False,static_content_store=contentstore()) diff --git a/common/lib/xmodule/xmodule/contentstore/content.py b/common/lib/xmodule/xmodule/contentstore/content.py index 48ef078fd9..22b959cd59 100644 --- a/common/lib/xmodule/xmodule/contentstore/content.py +++ b/common/lib/xmodule/xmodule/contentstore/content.py @@ -5,7 +5,11 @@ XASSET_THUMBNAIL_TAIL_NAME = '.thumbnail.jpg' import os import logging +import StringIO + from xmodule.modulestore import Location +from .django import contentstore +from PIL import Image class StaticContent(object): def __init__(self, loc, name, content_type, data, last_modified_at=None): @@ -24,6 +28,10 @@ class StaticContent(object): @staticmethod def compute_location(org, course, name, revision=None): + # replace some illegal characters + # for example, when importing courseware static assets, typically the source repository has subdirectories. + # right now the content store does not support a hierarchy structure, so collapse those subpaths + name = name.replace('/', '_') return Location([XASSET_LOCATION_TAG, org, course, 'asset', name, revision]) def get_id(self): @@ -66,3 +74,43 @@ class ContentStore(object): def get_all_content_for_course(self, location): raise NotImplementedError + + def generate_thumbnail(self, content): + thumbnail_content = None + # if we're uploading an image, then let's generate a thumbnail so that we can + # serve it up when needed without having to rescale on the fly + if content.content_type is not None and content.content_type.split('/')[0] == 'image': + try: + # use PIL to do the thumbnail generation (http://www.pythonware.com/products/pil/) + # My understanding is that PIL will maintain aspect ratios while restricting + # the max-height/width to be whatever you pass in as 'size' + # @todo: move the thumbnail size to a configuration setting?!? + im = Image.open(StringIO.StringIO(content.data)) + + # I've seen some exceptions from the PIL library when trying to save palletted + # PNG files to JPEG. Per the google-universe, they suggest converting to RGB first. + im = im.convert('RGB') + size = 128, 128 + im.thumbnail(size, Image.ANTIALIAS) + thumbnail_file = StringIO.StringIO() + im.save(thumbnail_file, 'JPEG') + thumbnail_file.seek(0) + + # use a naming convention to associate originals with the thumbnail + thumbnail_name = content.generate_thumbnail_name() + + # then just store this thumbnail as any other piece of content + thumbnail_file_location = StaticContent.compute_location(content.location.org, content.location.course, + thumbnail_name) + thumbnail_content = StaticContent(thumbnail_file_location, thumbnail_name, + 'image/jpeg', thumbnail_file) + + contentstore().save(thumbnail_content) + except: + raise + + return thumbnail_content + + + + diff --git a/common/lib/xmodule/xmodule/modulestore/xml_importer.py b/common/lib/xmodule/xmodule/modulestore/xml_importer.py index 1522288896..0a8e986c14 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml_importer.py +++ b/common/lib/xmodule/xmodule/modulestore/xml_importer.py @@ -1,14 +1,18 @@ import logging +import os +import mimetypes from .xml import XMLModuleStore from .exceptions import DuplicateItemError +from xmodule.modulestore import Location +from xmodule.contentstore.content import StaticContent log = logging.getLogger(__name__) def import_from_xml(store, data_dir, course_dirs=None, default_class='xmodule.raw_module.RawDescriptor', - load_error_modules=True): + load_error_modules=True, static_content_store=None): """ Import the specified xml data_dir into the "store" modulestore, using org and course as the location org and course. @@ -23,9 +27,16 @@ def import_from_xml(store, data_dir, course_dirs=None, course_dirs=course_dirs, load_error_modules=load_error_modules, ) + for course_id in module_store.modules.keys(): + course_data_dir = None + course_loc = None + for module in module_store.modules[course_id].itervalues(): + if module.category == 'course': + course_loc = module.location + if 'data' in module.definition: store.update_item(module.location, module.definition['data']) if 'children' in module.definition: @@ -33,5 +44,39 @@ def import_from_xml(store, data_dir, course_dirs=None, # NOTE: It's important to use own_metadata here to avoid writing # inherited metadata everywhere. store.update_metadata(module.location, dict(module.own_metadata)) + course_data_dir = module.metadata['data_dir'] + + if static_content_store is not None: + ''' + now import all static assets + ''' + static_dir = '{0}/{1}/static/'.format(data_dir, course_data_dir) + + for dirname, dirnames, filenames in os.walk(static_dir): + for filename in filenames: + + try: + content_path = os.path.join(dirname, filename) + fullname_with_subpath = content_path.replace(static_dir, '') # strip away leading path from the name + content_loc = StaticContent.compute_location(course_loc.org, course_loc.course, fullname_with_subpath) + mime_type = mimetypes.guess_type(filename)[0] + + print 'importing static asset {0} of mime-type {1} from path {2}'.format(content_loc, + mime_type, content_path) + + f = open(content_path, 'rb') + data = f.read() + f.close() + + content = StaticContent(content_loc, filename, mime_type, data) + + static_content_store.save(content) + + # this will be a NOP if content is not an image + thumbnail_content = static_content_store.generate_thumbnail(content) + + except: + raise + return module_store