Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1048 from MITx/feature/cdodge/xlint

Browse Source 
Feature/cdodge/xlint
This commit is contained in:
Calen Pennington
2012-11-26 08:11:34 -08:00
parent ee618d4223 fff3782884
commit 4e3da342d7
5 changed files with 146 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
cms/djangoapps/contentstore/management/commands/xlint.py Normal file
View File

@@ -0,0 +1,28 @@
from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.xml_importer import perform_xlint
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore
unnamed_modules = 0
class Command(BaseCommand):
help = \
'''
Verify the structure of courseware as to it's suitability for import
To run test: rake cms:xlint DATA_DIR=../data [COURSE_DIR=content-edx-101 (optional parameter)]
'''
def handle(self, *args, **options):
if len(args) == 0:
raise CommandError("import requires at least one argument: <data directory> [<course dir>...]")
data_dir = args[0]
if len(args) > 1:
course_dirs = args[1:]
else:
course_dirs = None
print "Importing. Data_dir={data}, course_dirs={courses}".format(
data=data_dir,
courses=course_dirs)
perform_xlint(data_dir, course_dirs, load_error_modules=False)

7
common/lib/xmodule/xmodule/modulestore/xml.py
View File

@@ -303,7 +303,7 @@ class XMLModuleStore(ModuleStoreBase):
try:
course_descriptor = self.load_course(course_dir, errorlog.tracker)
except Exception as e:
msg = "Failed to load course '{0}': {1}".format(course_dir, str(e))
msg = "ERROR: Failed to load course '{0}': {1}".format(course_dir, str(e))
log.exception(msg)
errorlog.tracker(msg)
@@ -337,7 +337,7 @@ class XMLModuleStore(ModuleStoreBase):
with open(policy_path) as f:
return json.load(f)
except (IOError, ValueError) as err:
msg = "Error loading course policy from {0}".format(policy_path)
msg = "ERROR: loading course policy from {0}".format(policy_path)
tracker(msg)
log.warning(msg + " " + str(err))
return {}
@@ -465,7 +465,8 @@ class XMLModuleStore(ModuleStoreBase):
module.metadata['data_dir'] = course_dir
self.modules[course_descriptor.id][module.location] = module
except Exception, e:
logging.exception("Failed to load {0}. Skipping... Exception: {1}".format(filepath, str(e)))
logging.exception("Failed to load {0}. Skipping... Exception: {1}".format(filepath, str(e)))
system.error_tracker("ERROR: " + str(e))
def get_instance(self, course_id, location, depth=0):
"""

101
common/lib/xmodule/xmodule/modulestore/xml_importer.py
View File

@@ -88,7 +88,7 @@ def verify_content_links(module, base_dir, static_content_store, link, remap_dic
def import_from_xml(store, data_dir, course_dirs=None,
default_class='xmodule.raw_module.RawDescriptor',
load_error_modules=True, static_content_store=None, target_location_namespace = None):
load_error_modules=True, static_content_store=None, target_location_namespace=None):
"""
Import the specified xml data_dir into the "store" modulestore,
using org and course as the location org and course.
@@ -202,7 +202,6 @@ def import_from_xml(store, data_dir, course_dirs=None,
store.update_item(module.location, module_data)
if 'children' in module.definition:
store.update_children(module.location, module.definition['children'])
@@ -210,6 +209,100 @@ def import_from_xml(store, data_dir, course_dirs=None,
# inherited metadata everywhere.
store.update_metadata(module.location, dict(module.own_metadata))
return module_store, course_items
def validate_category_hierarcy(module_store, course_id, parent_category, expected_child_category):
err_cnt = 0
parents = []
# get all modules of parent_category
for module in module_store.modules[course_id].itervalues():
if module.location.category == parent_category:
parents.append(module)
for parent in parents:
for child_loc in [Location(child) for child in parent.definition.get('children', [])]:
if child_loc.category != expected_child_category:
err_cnt += 1
print 'ERROR: child {0} of parent {1} was expected to be category of {2} but was {3}'.format(
child_loc, parent.location, expected_child_category, child_loc.category)
return err_cnt
def validate_data_source_path_existence(path, is_err = True, extra_msg = None):
_cnt = 0
if not os.path.exists(path):
print ("{0}: Expected folder at {1}. {2}".format('ERROR' if is_err == True else 'WARNING', path, extra_msg if
extra_msg is not None else ''))
_cnt = 1
return _cnt
def validate_data_source_paths(data_dir, course_dir):
# check that there is a '/static/' directory
course_path = data_dir / course_dir
err_cnt = 0
warn_cnt = 0
err_cnt += validate_data_source_path_existence(course_path / 'static')
warn_cnt += validate_data_source_path_existence(course_path / 'static/subs', is_err = False,
extra_msg = 'Video captions (if they are used) will not work unless they are static/subs.')
return err_cnt, warn_cnt
def perform_xlint(data_dir, course_dirs,
default_class='xmodule.raw_module.RawDescriptor',
load_error_modules=True):
err_cnt = 0
warn_cnt = 0
module_store = XMLModuleStore(
data_dir,
default_class=default_class,
course_dirs=course_dirs,
load_error_modules=load_error_modules
)
# check all data source path information
for course_dir in course_dirs:
_err_cnt, _warn_cnt = validate_data_source_paths(path(data_dir), course_dir)
err_cnt += _err_cnt
warn_cnt += _warn_cnt
# first count all errors and warnings as part of the XMLModuleStore import
for err_log in module_store._location_errors.itervalues():
for err_log_entry in err_log.errors:
msg = err_log_entry[0]
if msg.startswith('ERROR:'):
err_cnt+=1
else:
warn_cnt+=1
# then count outright all courses that failed to load at all
for err_log in module_store.errored_courses.itervalues():
for err_log_entry in err_log.errors:
msg = err_log_entry[0]
print msg
if msg.startswith('ERROR:'):
err_cnt+=1
else:
warn_cnt+=1
for course_id in module_store.modules.keys():
# constrain that courses only have 'chapter' children
err_cnt += validate_category_hierarcy(module_store, course_id, "course", "chapter")
# constrain that chapters only have 'sequentials'
err_cnt += validate_category_hierarcy(module_store, course_id, "chapter", "sequential")
# constrain that sequentials only have 'verticals'
err_cnt += validate_category_hierarcy(module_store, course_id, "sequential", "vertical")
print "\n\n------------------------------------------\nVALIDATION SUMMARY: {0} Errors {1} Warnings\n".format(err_cnt, warn_cnt)
if err_cnt > 0:
print "This course is not suitable for importing. Please fix courseware according to specifications before importing."
elif warn_cnt > 0:
print "This course can be imported, but some errors may occur during the run of the course. It is recommend that you fix your courseware before importing"
else:
print "This course can be imported successfully."

4
common/lib/xmodule/xmodule/seq_module.py
View File

@@ -127,8 +127,10 @@ class SequenceDescriptor(MakoModuleDescriptor, XmlDescriptor):
for child in xml_object:
try:
children.append(system.process_xml(etree.tostring(child)).location.url())
except:
except Exception, e:
log.exception("Unable to load child when parsing Sequence. Continuing...")
if system.error_tracker is not None:
system.error_tracker("ERROR: " + str(e))
continue
return {'children': children}

14
rakefile
View File

@@ -408,6 +408,20 @@ namespace :cms do
end
end
namespace :cms do
desc "Import course data within the given DATA_DIR variable"
task :xlint do
if ENV['DATA_DIR'] and ENV['COURSE_DIR']
sh(django_admin(:cms, :dev, :xlint, ENV['DATA_DIR'], ENV['COURSE_DIR']))
elsif ENV['DATA_DIR']
sh(django_admin(:cms, :dev, :xlint, ENV['DATA_DIR']))
else
raise "Please specify a DATA_DIR variable that point to your data directory.\n" +
"Example: \`rake cms:import DATA_DIR=../data\`"
end
end
end
desc "Build a properties file used to trigger autodeploy builds"
task :autodeploy_properties do
File.open("autodeploy.properties", "w") do |file|