From b3ab9c23f7d9f3071168ed83e191fbac7e0084a5 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Wed, 11 Jul 2012 22:17:27 -0400 Subject: [PATCH 1/3] Allow importing of courses that don't have the expected path/filename structure for problems and html --- common/lib/xmodule/xmodule/capa_module.py | 7 +++++ common/lib/xmodule/xmodule/html_module.py | 10 +++++++ common/lib/xmodule/xmodule/xml_module.py | 33 +++++++++++++++++------ 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py index 67e3aac93d..f6f9bd52ec 100644 --- a/common/lib/xmodule/xmodule/capa_module.py +++ b/common/lib/xmodule/xmodule/capa_module.py @@ -503,3 +503,10 @@ class CapaDescriptor(RawDescriptor): """ module_class = CapaModule + + # TODO (cpennington): Delete this method once all fall 2012 course are being + # edited in the cms + @classmethod + def backcompat_path(cls, path): + if path.startswith('problem'): + return 'problems/' + path[7:] diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py index 337a833dc3..a5554a8403 100644 --- a/common/lib/xmodule/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -28,6 +28,16 @@ class HtmlDescriptor(RawDescriptor): js = {'coffee': [resource_string(__name__, 'js/module/html.coffee')]} js_module = 'HTML' + # TODO (cpennington): Delete this method once all fall 2012 course are being + # edited in the cms + @classmethod + def backcompat_path(cls, path): + if path.startswith('html/html'): + path = path[5:] + if path.endswith('.html.html'): + path = path[:-5] + return path + @classmethod def file_to_xml(cls, file_object): parser = etree.HTMLParser() diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index 9be9b0f900..96d7da3fb1 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -4,6 +4,8 @@ from lxml import etree import copy import logging from collections import namedtuple +from fs.errors import ResourceNotFoundError +import os log = logging.getLogger(__name__) @@ -154,13 +156,28 @@ class XmlDescriptor(XModuleDescriptor): definition_xml = copy.deepcopy(xml_object) else: filepath = cls._format_filepath(xml_object.tag, filename) - log.debug('filepath=%s, resources_fs=%s' % (filepath,system.resources_fs)) - with system.resources_fs.open(filepath) as file: - try: - definition_xml = cls.file_to_xml(file) - except: - log.exception("Failed to parse xml in file %s" % filepath) - raise + + # TODO (cpennington): If the file doesn't exist at the right path, + # give the class a chance to fix it up. The file will be written out again + # in the correct format. + # This should go away once the CMS is online and has imported all current (fall 2012) + # courses from xml + if not system.resources_fs.exists(filepath) and hasattr(cls, 'backcompat_path'): + new_filepath = cls.backcompat_path(filepath) + if new_filepath is not None and system.resources_fs.exists(new_filepath): + filepath = new_filepath + + log.debug('filepath=%s, resources_fs=%s' % (filepath, system.resources_fs)) + try: + with system.resources_fs.open(filepath) as file: + try: + definition_xml = cls.file_to_xml(file) + except: + log.exception("Failed to parse xml in file %s" % filepath) + raise + except ResourceNotFoundError: + log.exception('Unable to load file contents at path %s' % filepath) + return {'data': 'Error loading file contents at path %s' % filepath} cls.clean_metadata_from_xml(definition_xml) return cls.definition_from_xml(definition_xml, system) @@ -200,7 +217,7 @@ class XmlDescriptor(XModuleDescriptor): if len(list(xml_object.iter())) > 5: filepath = self.__class__._format_filepath(self.category, self.name) - resource_fs.makedir(self.category, allow_recreate=True) + resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True) with resource_fs.open(filepath, 'w') as file: file.write(etree.tostring(xml_object, pretty_print=True)) From 379c6d164d625ea59d8082b38a40d0be8e76b0eb Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Wed, 11 Jul 2012 22:17:53 -0400 Subject: [PATCH 2/3] Allow importing of courses that still have mako templating in them --- common/lib/xmodule/xmodule/modulestore/xml.py | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py index c70defaf64..4df2743e70 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml.py +++ b/common/lib/xmodule/xmodule/modulestore/xml.py @@ -5,7 +5,9 @@ from lxml import etree from path import path from xmodule.x_module import XModuleDescriptor, XMLParsingSystem from xmodule.mako_module import MakoDescriptorSystem +from cStringIO import StringIO import os +import re from . import ModuleStore, Location from .exceptions import ItemNotFoundError @@ -16,6 +18,13 @@ etree.set_default_parser(etree.XMLParser(dtd_validation=False, load_dtd=False, log = logging.getLogger('mitx.' + __name__) +# TODO (cpennington): Remove this once all fall 2012 courses have been imported into the cms from xml +def clean_out_mako_templating(xml_string): + xml_string = xml_string.replace('%include', 'include') + xml_string = re.sub("(?m)^\s*%.*$", '', xml_string) + return xml_string + + class XMLModuleStore(ModuleStore): """ An XML backed ModuleStore @@ -54,8 +63,11 @@ class XMLModuleStore(ModuleStore): if not os.path.exists(self.data_dir / course_dir / "course.xml"): continue - course_descriptor = self.load_course(course_dir) - self.courses[course_dir] = course_descriptor + try: + course_descriptor = self.load_course(course_dir) + self.courses[course_dir] = course_descriptor + except: + log.exception("Failed to load course %s" % course_dir) def load_course(self, course_dir): """ @@ -65,6 +77,9 @@ class XMLModuleStore(ModuleStore): with open(self.data_dir / course_dir / "course.xml") as course_file: + # TODO (cpennington): Remove this once all fall 2012 courses have been imported into the cms from xml + course_file = StringIO(clean_out_mako_templating(course_file.read())) + course_data = etree.parse(course_file).getroot() org = course_data.get('org') @@ -91,6 +106,8 @@ class XMLModuleStore(ModuleStore): def process_xml(xml): try: + # TODO (cpennington): Remove this once all fall 2012 courses have been imported into the cms from xml + xml = clean_out_mako_templating(xml) xml_data = etree.fromstring(xml) except: log.exception("Unable to parse xml: {xml}".format(xml=xml)) From b29649c49b6b7e3dfe398683d03c5f43df6035dc Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Wed, 11 Jul 2012 23:00:56 -0400 Subject: [PATCH 3/3] Allow for more flexible candidates for replacement filepaths --- common/lib/xmodule/xmodule/capa_module.py | 9 ++++++--- common/lib/xmodule/xmodule/html_module.py | 12 ++++++++---- common/lib/xmodule/xmodule/xml_module.py | 10 ++++++---- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py index f6f9bd52ec..527b0a6fa8 100644 --- a/common/lib/xmodule/xmodule/capa_module.py +++ b/common/lib/xmodule/xmodule/capa_module.py @@ -6,6 +6,7 @@ import logging import traceback import re import StringIO +import os from datetime import timedelta from lxml import etree @@ -507,6 +508,8 @@ class CapaDescriptor(RawDescriptor): # TODO (cpennington): Delete this method once all fall 2012 course are being # edited in the cms @classmethod - def backcompat_path(cls, path): - if path.startswith('problem'): - return 'problems/' + path[7:] + def backcompat_paths(cls, path): + return [ + 'problems/' + path[8:], + path[8:], + ] diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py index a5554a8403..1555b96f82 100644 --- a/common/lib/xmodule/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -1,4 +1,5 @@ import logging +import os from lxml import etree from xmodule.x_module import XModule @@ -31,12 +32,15 @@ class HtmlDescriptor(RawDescriptor): # TODO (cpennington): Delete this method once all fall 2012 course are being # edited in the cms @classmethod - def backcompat_path(cls, path): - if path.startswith('html/html'): - path = path[5:] + def backcompat_paths(cls, path): if path.endswith('.html.html'): path = path[:-5] - return path + candidates = [] + while os.sep in path: + candidates.append(path) + _, _, path = path.partition(os.sep) + + return candidates @classmethod def file_to_xml(cls, file_object): diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index 96d7da3fb1..b0bfaebab5 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -162,10 +162,12 @@ class XmlDescriptor(XModuleDescriptor): # in the correct format. # This should go away once the CMS is online and has imported all current (fall 2012) # courses from xml - if not system.resources_fs.exists(filepath) and hasattr(cls, 'backcompat_path'): - new_filepath = cls.backcompat_path(filepath) - if new_filepath is not None and system.resources_fs.exists(new_filepath): - filepath = new_filepath + if not system.resources_fs.exists(filepath) and hasattr(cls, 'backcompat_paths'): + candidates = cls.backcompat_paths(filepath) + for candidate in candidates: + if system.resources_fs.exists(candidate): + filepath = candidate + break log.debug('filepath=%s, resources_fs=%s' % (filepath, system.resources_fs)) try: