diff --git a/cms/djangoapps/contentstore/management/commands/xlint.py b/cms/djangoapps/contentstore/management/commands/xlint.py new file mode 100644 index 0000000000..6bc254a1ff --- /dev/null +++ b/cms/djangoapps/contentstore/management/commands/xlint.py @@ -0,0 +1,28 @@ +from django.core.management.base import BaseCommand, CommandError +from xmodule.modulestore.xml_importer import perform_xlint +from xmodule.modulestore.django import modulestore +from xmodule.contentstore.django import contentstore + + +unnamed_modules = 0 + + +class Command(BaseCommand): + help = \ + ''' + Verify the structure of courseware as to it's suitability for import + To run test: rake cms:xlint DATA_DIR=../data [COURSE_DIR=content-edx-101 (optional parameter)] + ''' + def handle(self, *args, **options): + if len(args) == 0: + raise CommandError("import requires at least one argument: [...]") + + data_dir = args[0] + if len(args) > 1: + course_dirs = args[1:] + else: + course_dirs = None + print "Importing. Data_dir={data}, course_dirs={courses}".format( + data=data_dir, + courses=course_dirs) + perform_xlint(data_dir, course_dirs, load_error_modules=False) diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py index 0fe29a3845..abd9c7d0eb 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml.py +++ b/common/lib/xmodule/xmodule/modulestore/xml.py @@ -303,7 +303,7 @@ class XMLModuleStore(ModuleStoreBase): try: course_descriptor = self.load_course(course_dir, errorlog.tracker) except Exception as e: - msg = "Failed to load course '{0}': {1}".format(course_dir, str(e)) + msg = "ERROR: Failed to load course '{0}': {1}".format(course_dir, str(e)) log.exception(msg) errorlog.tracker(msg) @@ -337,7 +337,7 @@ class XMLModuleStore(ModuleStoreBase): with open(policy_path) as f: return json.load(f) except (IOError, ValueError) as err: - msg = "Error loading course policy from {0}".format(policy_path) + msg = "ERROR: loading course policy from {0}".format(policy_path) tracker(msg) log.warning(msg + " " + str(err)) return {} @@ -465,7 +465,8 @@ class XMLModuleStore(ModuleStoreBase): module.metadata['data_dir'] = course_dir self.modules[course_descriptor.id][module.location] = module except Exception, e: - logging.exception("Failed to load {0}. Skipping... Exception: {1}".format(filepath, str(e))) + logging.exception("Failed to load {0}. Skipping... Exception: {1}".format(filepath, str(e))) + system.error_tracker("ERROR: " + str(e)) def get_instance(self, course_id, location, depth=0): """ diff --git a/common/lib/xmodule/xmodule/modulestore/xml_importer.py b/common/lib/xmodule/xmodule/modulestore/xml_importer.py index 02cbb7121f..12b495ab68 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml_importer.py +++ b/common/lib/xmodule/xmodule/modulestore/xml_importer.py @@ -88,7 +88,7 @@ def verify_content_links(module, base_dir, static_content_store, link, remap_dic def import_from_xml(store, data_dir, course_dirs=None, default_class='xmodule.raw_module.RawDescriptor', - load_error_modules=True, static_content_store=None, target_location_namespace = None): + load_error_modules=True, static_content_store=None, target_location_namespace=None): """ Import the specified xml data_dir into the "store" modulestore, using org and course as the location org and course. @@ -202,7 +202,6 @@ def import_from_xml(store, data_dir, course_dirs=None, store.update_item(module.location, module_data) - if 'children' in module.definition: store.update_children(module.location, module.definition['children']) @@ -210,6 +209,100 @@ def import_from_xml(store, data_dir, course_dirs=None, # inherited metadata everywhere. store.update_metadata(module.location, dict(module.own_metadata)) - - return module_store, course_items + + +def validate_category_hierarcy(module_store, course_id, parent_category, expected_child_category): + err_cnt = 0 + + parents = [] + # get all modules of parent_category + for module in module_store.modules[course_id].itervalues(): + if module.location.category == parent_category: + parents.append(module) + + for parent in parents: + for child_loc in [Location(child) for child in parent.definition.get('children', [])]: + if child_loc.category != expected_child_category: + err_cnt += 1 + print 'ERROR: child {0} of parent {1} was expected to be category of {2} but was {3}'.format( + child_loc, parent.location, expected_child_category, child_loc.category) + + return err_cnt + +def validate_data_source_path_existence(path, is_err = True, extra_msg = None): + _cnt = 0 + if not os.path.exists(path): + print ("{0}: Expected folder at {1}. {2}".format('ERROR' if is_err == True else 'WARNING', path, extra_msg if + extra_msg is not None else '')) + _cnt = 1 + return _cnt + +def validate_data_source_paths(data_dir, course_dir): + # check that there is a '/static/' directory + course_path = data_dir / course_dir + err_cnt = 0 + warn_cnt = 0 + err_cnt += validate_data_source_path_existence(course_path / 'static') + warn_cnt += validate_data_source_path_existence(course_path / 'static/subs', is_err = False, + extra_msg = 'Video captions (if they are used) will not work unless they are static/subs.') + return err_cnt, warn_cnt + + +def perform_xlint(data_dir, course_dirs, + default_class='xmodule.raw_module.RawDescriptor', + load_error_modules=True): + err_cnt = 0 + warn_cnt = 0 + + module_store = XMLModuleStore( + data_dir, + default_class=default_class, + course_dirs=course_dirs, + load_error_modules=load_error_modules + ) + + # check all data source path information + for course_dir in course_dirs: + _err_cnt, _warn_cnt = validate_data_source_paths(path(data_dir), course_dir) + err_cnt += _err_cnt + warn_cnt += _warn_cnt + + # first count all errors and warnings as part of the XMLModuleStore import + for err_log in module_store._location_errors.itervalues(): + for err_log_entry in err_log.errors: + msg = err_log_entry[0] + if msg.startswith('ERROR:'): + err_cnt+=1 + else: + warn_cnt+=1 + + # then count outright all courses that failed to load at all + for err_log in module_store.errored_courses.itervalues(): + for err_log_entry in err_log.errors: + msg = err_log_entry[0] + print msg + if msg.startswith('ERROR:'): + err_cnt+=1 + else: + warn_cnt+=1 + + for course_id in module_store.modules.keys(): + # constrain that courses only have 'chapter' children + err_cnt += validate_category_hierarcy(module_store, course_id, "course", "chapter") + # constrain that chapters only have 'sequentials' + err_cnt += validate_category_hierarcy(module_store, course_id, "chapter", "sequential") + # constrain that sequentials only have 'verticals' + err_cnt += validate_category_hierarcy(module_store, course_id, "sequential", "vertical") + + print "\n\n------------------------------------------\nVALIDATION SUMMARY: {0} Errors {1} Warnings\n".format(err_cnt, warn_cnt) + + if err_cnt > 0: + print "This course is not suitable for importing. Please fix courseware according to specifications before importing." + elif warn_cnt > 0: + print "This course can be imported, but some errors may occur during the run of the course. It is recommend that you fix your courseware before importing" + else: + print "This course can be imported successfully." + + + diff --git a/common/lib/xmodule/xmodule/seq_module.py b/common/lib/xmodule/xmodule/seq_module.py index 0ade3e0e7d..155ad99480 100644 --- a/common/lib/xmodule/xmodule/seq_module.py +++ b/common/lib/xmodule/xmodule/seq_module.py @@ -127,8 +127,10 @@ class SequenceDescriptor(MakoModuleDescriptor, XmlDescriptor): for child in xml_object: try: children.append(system.process_xml(etree.tostring(child)).location.url()) - except: + except Exception, e: log.exception("Unable to load child when parsing Sequence. Continuing...") + if system.error_tracker is not None: + system.error_tracker("ERROR: " + str(e)) continue return {'children': children} diff --git a/rakefile b/rakefile index 4317784159..fa8bb4f4ab 100644 --- a/rakefile +++ b/rakefile @@ -408,6 +408,20 @@ namespace :cms do end end +namespace :cms do + desc "Import course data within the given DATA_DIR variable" + task :xlint do + if ENV['DATA_DIR'] and ENV['COURSE_DIR'] + sh(django_admin(:cms, :dev, :xlint, ENV['DATA_DIR'], ENV['COURSE_DIR'])) + elsif ENV['DATA_DIR'] + sh(django_admin(:cms, :dev, :xlint, ENV['DATA_DIR'])) + else + raise "Please specify a DATA_DIR variable that point to your data directory.\n" + + "Example: \`rake cms:import DATA_DIR=../data\`" + end + end +end + desc "Build a properties file used to trigger autodeploy builds" task :autodeploy_properties do File.open("autodeploy.properties", "w") do |file|