diff --git a/cms/djangoapps/contentstore/management/commands/import.py b/cms/djangoapps/contentstore/management/commands/import.py index e24111dbb7..f97ac10d41 100644 --- a/cms/djangoapps/contentstore/management/commands/import.py +++ b/cms/djangoapps/contentstore/management/commands/import.py @@ -3,19 +3,15 @@ ### from django.core.management.base import BaseCommand, CommandError -from keystore.django import keystore -from lxml import etree -from keystore.xml import XMLModuleStore +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.xml import XMLModuleStore unnamed_modules = 0 -etree.set_default_parser(etree.XMLParser(dtd_validation=False, load_dtd=False, - remove_comments=True)) - class Command(BaseCommand): help = \ -'''Import the specified data directory into the default keystore''' +'''Import the specified data directory into the default ModuleStore''' def handle(self, *args, **options): if len(args) != 3: @@ -23,10 +19,11 @@ class Command(BaseCommand): org, course, data_dir = args - module_store = XMLModuleStore(org, course, data_dir, 'xmodule.raw_module.RawDescriptor') + module_store = XMLModuleStore(org, course, data_dir, 'xmodule.raw_module.RawDescriptor', eager=True) for module in module_store.modules.itervalues(): - keystore().create_item(module.location) + modulestore().create_item(module.location) if 'data' in module.definition: - keystore().update_item(module.location, module.definition['data']) + modulestore().update_item(module.location, module.definition['data']) if 'children' in module.definition: - keystore().update_children(module.location, module.definition['children']) + modulestore().update_children(module.location, module.definition['children']) + modulestore().update_metadata(module.location, dict(module.metadata)) diff --git a/cms/djangoapps/contentstore/views.py b/cms/djangoapps/contentstore/views.py index f7d5efe22a..76a904a403 100644 --- a/cms/djangoapps/contentstore/views.py +++ b/cms/djangoapps/contentstore/views.py @@ -1,9 +1,10 @@ from mitxmako.shortcuts import render_to_response -from keystore.django import keystore +from xmodule.modulestore.django import modulestore from django_future.csrf import ensure_csrf_cookie from django.http import HttpResponse import json +from fs.osfs import OSFS @ensure_csrf_cookie def index(request): @@ -11,14 +12,14 @@ def index(request): org = 'mit.edu' course = '6002xs12' name = '6.002_Spring_2012' - course = keystore().get_item(['i4x', org, course, 'course', name]) + course = modulestore().get_item(['i4x', org, course, 'course', name]) weeks = course.get_children() return render_to_response('index.html', {'weeks': weeks}) def edit_item(request): item_id = request.GET['id'] - item = keystore().get_item(item_id) + item = modulestore().get_item(item_id) return render_to_response('unit.html', { 'contents': item.get_html(), 'js_module': item.js_module_name(), @@ -30,5 +31,18 @@ def edit_item(request): def save_item(request): item_id = request.POST['id'] data = json.loads(request.POST['data']) - keystore().update_item(item_id, data) + modulestore().update_item(item_id, data) return HttpResponse(json.dumps({})) + + +def temp_force_export(request): + org = 'mit.edu' + course = '6002xs12' + name = '6.002_Spring_2012' + course = modulestore().get_item(['i4x', org, course, 'course', name]) + fs = OSFS('../data-export-test') + xml = course.export_to_xml(fs) + with fs.open('course.xml', 'w') as course_xml: + course_xml.write(xml) + + return HttpResponse('Done') diff --git a/cms/envs/dev.py b/cms/envs/dev.py index ce775d962a..b4bcbfa9ce 100644 --- a/cms/envs/dev.py +++ b/cms/envs/dev.py @@ -3,12 +3,16 @@ This config file runs the simplest dev environment""" from .common import * +import logging +import sys +logging.basicConfig(stream=sys.stdout, ) + DEBUG = True TEMPLATE_DEBUG = DEBUG -KEYSTORE = { +MODULESTORE = { 'default': { - 'ENGINE': 'keystore.mongo.MongoModuleStore', + 'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore', 'OPTIONS': { 'default_class': 'xmodule.raw_module.RawDescriptor', 'host': 'localhost', diff --git a/cms/envs/test.py b/cms/envs/test.py index 1a20d9e6f8..032de92953 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -17,7 +17,7 @@ for app in os.listdir(PROJECT_ROOT / 'djangoapps'): NOSE_ARGS += ['--cover-package', app] TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' -KEYSTORE = { +MODULESTORE = { 'host': 'localhost', 'db': 'mongo_base', 'collection': 'key_store', diff --git a/cms/templates/widgets/html-edit.html b/cms/templates/widgets/html-edit.html index 666aa1de81..f0f63ea905 100644 --- a/cms/templates/widgets/html-edit.html +++ b/cms/templates/widgets/html-edit.html @@ -33,8 +33,8 @@ - -
${module.definition['data']}
+ +
${data}
Save & Update diff --git a/cms/urls.py b/cms/urls.py index d7314aafae..9d827c3fe3 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -8,4 +8,5 @@ urlpatterns = patterns('', url(r'^$', 'contentstore.views.index', name='index'), url(r'^edit_item$', 'contentstore.views.edit_item', name='edit_item'), url(r'^save_item$', 'contentstore.views.save_item', name='save_item'), + url(r'^temp_force_export$', 'contentstore.views.temp_force_export') ) diff --git a/common/lib/capa/capa_problem.py b/common/lib/capa/capa_problem.py index a06ac1d7b6..2e3620021b 100644 --- a/common/lib/capa/capa_problem.py +++ b/common/lib/capa/capa_problem.py @@ -68,14 +68,13 @@ class LoncapaProblem(object): Main class for capa Problems. ''' - def __init__(self, fileobject, id, state=None, seed=None, system=None): + def __init__(self, problem_text, id, state=None, seed=None, system=None): ''' - Initializes capa Problem. The problem itself is defined by the XML file - pointed to by fileobject. + Initializes capa Problem. Arguments: - - filesobject : an OSFS instance: see fs.osfs + - problem_text : xml defining the problem - id : string used as the identifier for this problem; often a filename (no spaces) - state : student state (represented as a dict) - seed : random number generator seed (int) @@ -103,14 +102,11 @@ class LoncapaProblem(object): if not self.seed: self.seed = struct.unpack('i', os.urandom(4))[0] - self.fileobject = fileobject # save problem file object, so we can use for debugging information later - if getattr(system, 'DEBUG', False): # get the problem XML string from the problem file - log.info("[courseware.capa.capa_problem.lcp.init] fileobject = %s" % fileobject) - file_text = fileobject.read() - file_text = re.sub("startouttext\s*/", "text", file_text) # Convert startouttext and endouttext to proper - file_text = re.sub("endouttext\s*/", "/text", file_text) + problem_text = re.sub("startouttext\s*/", "text", problem_text) # Convert startouttext and endouttext to proper + problem_text = re.sub("endouttext\s*/", "/text", problem_text) + self.problem_text = problem_text - self.tree = etree.XML(file_text) # parse problem XML file into an element tree + self.tree = etree.XML(problem_text) # parse problem XML file into an element tree self._process_includes() # handle any tags # construct script processor context (eg for customresponse problems) @@ -130,7 +126,7 @@ class LoncapaProblem(object): self.done = False def __unicode__(self): - return u"LoncapaProblem ({0})".format(self.fileobject) + return u"LoncapaProblem ({0})".format(self.problem_text) def get_state(self): ''' Stored per-user session data neeeded to: @@ -272,7 +268,7 @@ class LoncapaProblem(object): parent = inc.getparent() # insert new XML into tree in place of inlcude parent.insert(parent.index(inc),incxml) parent.remove(inc) - log.debug('Included %s into %s' % (file,self.fileobject)) + log.debug('Included %s into %s' % (file, self.id)) def _extract_context(self, tree, seed=struct.unpack('i', os.urandom(4))[0]): # private ''' diff --git a/common/lib/keystore/django.py b/common/lib/keystore/django.py deleted file mode 100644 index 89aa9d07b0..0000000000 --- a/common/lib/keystore/django.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -Module that provides a connection to the keystore specified in the django settings. - -Passes settings.KEYSTORE as kwargs to MongoModuleStore -""" - -from __future__ import absolute_import - -from importlib import import_module - -from django.conf import settings - -_KEYSTORES = {} - - -def keystore(name='default'): - global _KEYSTORES - - if name not in _KEYSTORES: - class_path = settings.KEYSTORE[name]['ENGINE'] - module_path, _, class_name = class_path.rpartition('.') - class_ = getattr(import_module(module_path), class_name) - _KEYSTORES[name] = class_( - **settings.KEYSTORE[name]['OPTIONS']) - - return _KEYSTORES[name] diff --git a/common/lib/keystore/xml.py b/common/lib/keystore/xml.py deleted file mode 100644 index e7adb56ad6..0000000000 --- a/common/lib/keystore/xml.py +++ /dev/null @@ -1,96 +0,0 @@ -import logging -from fs.osfs import OSFS -from importlib import import_module -from lxml import etree -from path import path -from xmodule.x_module import XModuleDescriptor, XMLParsingSystem - -from . import ModuleStore, Location -from .exceptions import ItemNotFoundError - -etree.set_default_parser(etree.XMLParser(dtd_validation=False, load_dtd=False, - remove_comments=True)) - -log = logging.getLogger(__name__) - - -class XMLModuleStore(ModuleStore): - """ - An XML backed ModuleStore - """ - def __init__(self, org, course, data_dir, default_class=None): - self.data_dir = path(data_dir) - self.modules = {} - - module_path, _, class_name = default_class.rpartition('.') - class_ = getattr(import_module(module_path), class_name) - self.default_class = class_ - - with open(self.data_dir / "course.xml") as course_file: - class ImportSystem(XMLParsingSystem): - def __init__(self, modulestore): - """ - modulestore: the XMLModuleStore to store the loaded modules in - """ - self.unnamed_modules = 0 - - def process_xml(xml): - try: - xml_data = etree.fromstring(xml) - except: - log.exception("Unable to parse xml: {xml}".format(xml=xml)) - raise - if xml_data.get('name'): - xml_data.set('slug', Location.clean(xml_data.get('name'))) - else: - self.unnamed_modules += 1 - xml_data.set('slug', '{tag}_{count}'.format(tag=xml_data.tag, count=self.unnamed_modules)) - - module = XModuleDescriptor.load_from_xml(etree.tostring(xml_data), self, org, course, modulestore.default_class) - modulestore.modules[module.location] = module - return module - - XMLParsingSystem.__init__(self, modulestore.get_item, OSFS(data_dir), process_xml) - - ImportSystem(self).process_xml(course_file.read()) - - def get_item(self, location): - """ - Returns an XModuleDescriptor instance for the item at location. - If location.revision is None, returns the most item with the most - recent revision - - If any segment of the location is None except revision, raises - keystore.exceptions.InsufficientSpecificationError - If no object is found at that location, raises keystore.exceptions.ItemNotFoundError - - location: Something that can be passed to Location - """ - location = Location(location) - try: - return self.modules[location] - except KeyError: - raise ItemNotFoundError(location) - - def create_item(self, location): - raise NotImplementedError("XMLModuleStores are read-only") - - def update_item(self, location, data): - """ - Set the data in the item specified by the location to - data - - location: Something that can be passed to Location - data: A nested dictionary of problem data - """ - raise NotImplementedError("XMLModuleStores are read-only") - - def update_children(self, location, children): - """ - Set the children for the item specified by the location to - data - - location: Something that can be passed to Location - children: A list of child item identifiers - """ - raise NotImplementedError("XMLModuleStores are read-only") diff --git a/common/lib/xmodule/mako_module.py b/common/lib/xmodule/mako_module.py deleted file mode 100644 index 2260dddd92..0000000000 --- a/common/lib/xmodule/mako_module.py +++ /dev/null @@ -1,22 +0,0 @@ -from x_module import XModuleDescriptor -from mitxmako.shortcuts import render_to_string - - -class MakoModuleDescriptor(XModuleDescriptor): - """ - Module descriptor intended as a mixin that uses a mako template - to specify the module html. - - Expects the descriptor to have the `mako_template` attribute set - with the name of the template to render, and it will pass - the descriptor as the `module` parameter to that template - """ - - def get_context(self): - """ - Return the context to render the mako template with - """ - return {'module': self} - - def get_html(self): - return render_to_string(self.mako_template, self.get_context()) diff --git a/common/lib/xmodule/setup.py b/common/lib/xmodule/setup.py index e45e6654c2..77b0838ff2 100644 --- a/common/lib/xmodule/setup.py +++ b/common/lib/xmodule/setup.py @@ -3,10 +3,10 @@ from setuptools import setup, find_packages setup( name="XModule", version="0.1", - packages=find_packages(), + packages=find_packages(exclude=["tests"]), install_requires=['distribute'], package_data={ - '': ['js/*'] + 'xmodule': ['js/module/*'] }, # See http://guide.python-distribute.org/creation.html#entry-points diff --git a/common/lib/xmodule/tests.py b/common/lib/xmodule/tests/__init__.py similarity index 96% rename from common/lib/xmodule/tests.py rename to common/lib/xmodule/tests/__init__.py index 90187abc2a..4fb270df13 100644 --- a/common/lib/xmodule/tests.py +++ b/common/lib/xmodule/tests/__init__.py @@ -43,12 +43,10 @@ class ModelsTest(unittest.TestCase): def setUp(self): pass - def test_get_module_class(self): - vc = xmodule.get_module_class('video') - vc_str = "" + def test_load_class(self): + vc = xmodule.x_module.XModuleDescriptor.load_class('video') + vc_str = "" self.assertEqual(str(vc), vc_str) - video_id = xmodule.get_default_ids()['video'] - self.assertEqual(video_id, 'youtube') def test_calc(self): variables={'R1':2.0, 'R3':4.0} @@ -98,7 +96,7 @@ class ModelsTest(unittest.TestCase): class MultiChoiceTest(unittest.TestCase): def test_MC_grade(self): multichoice_file = os.path.dirname(__file__)+"/test_files/multichoice.xml" - test_lcp = lcp.LoncapaProblem(open(multichoice_file), '1', system=i4xs) + test_lcp = lcp.LoncapaProblem(open(multichoice_file).read(), '1', system=i4xs) correct_answers = {'1_2_1':'choice_foil3'} self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct') false_answers = {'1_2_1':'choice_foil2'} @@ -106,7 +104,7 @@ class MultiChoiceTest(unittest.TestCase): def test_MC_bare_grades(self): multichoice_file = os.path.dirname(__file__)+"/test_files/multi_bare.xml" - test_lcp = lcp.LoncapaProblem(open(multichoice_file), '1', system=i4xs) + test_lcp = lcp.LoncapaProblem(open(multichoice_file).read(), '1', system=i4xs) correct_answers = {'1_2_1':'choice_2'} self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct') false_answers = {'1_2_1':'choice_1'} @@ -114,7 +112,7 @@ class MultiChoiceTest(unittest.TestCase): def test_TF_grade(self): truefalse_file = os.path.dirname(__file__)+"/test_files/truefalse.xml" - test_lcp = lcp.LoncapaProblem(open(truefalse_file), '1', system=i4xs) + test_lcp = lcp.LoncapaProblem(open(truefalse_file).read(), '1', system=i4xs) correct_answers = {'1_2_1':['choice_foil2', 'choice_foil1']} self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct') false_answers = {'1_2_1':['choice_foil1']} @@ -129,7 +127,7 @@ class MultiChoiceTest(unittest.TestCase): class ImageResponseTest(unittest.TestCase): def test_ir_grade(self): imageresponse_file = os.path.dirname(__file__)+"/test_files/imageresponse.xml" - test_lcp = lcp.LoncapaProblem(open(imageresponse_file), '1', system=i4xs) + test_lcp = lcp.LoncapaProblem(open(imageresponse_file).read(), '1', system=i4xs) correct_answers = {'1_2_1':'(490,11)-(556,98)', '1_2_2':'(242,202)-(296,276)'} test_answers = {'1_2_1':'[500,20]', @@ -142,7 +140,7 @@ class SymbolicResponseTest(unittest.TestCase): def test_sr_grade(self): raise SkipTest() # This test fails due to dependencies on a local copy of snuggletex-webapp. Until we have figured that out, we'll just skip this test symbolicresponse_file = os.path.dirname(__file__)+"/test_files/symbolicresponse.xml" - test_lcp = lcp.LoncapaProblem(open(symbolicresponse_file), '1', system=i4xs) + test_lcp = lcp.LoncapaProblem(open(symbolicresponse_file).read(), '1', system=i4xs) correct_answers = {'1_2_1':'cos(theta)*[[1,0],[0,1]] + i*sin(theta)*[[0,1],[1,0]]', '1_2_1_dynamath': ''' @@ -235,7 +233,7 @@ class OptionResponseTest(unittest.TestCase): ''' def test_or_grade(self): optionresponse_file = os.path.dirname(__file__)+"/test_files/optionresponse.xml" - test_lcp = lcp.LoncapaProblem(open(optionresponse_file), '1', system=i4xs) + test_lcp = lcp.LoncapaProblem(open(optionresponse_file).read(), '1', system=i4xs) correct_answers = {'1_2_1':'True', '1_2_2':'False'} test_answers = {'1_2_1':'True', @@ -251,7 +249,7 @@ class FormulaResponseWithHintTest(unittest.TestCase): ''' def test_or_grade(self): problem_file = os.path.dirname(__file__)+"/test_files/formularesponse_with_hint.xml" - test_lcp = lcp.LoncapaProblem(open(problem_file), '1', system=i4xs) + test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=i4xs) correct_answers = {'1_2_1':'2.5*x-5.0'} test_answers = {'1_2_1':'0.4*x-5.0'} self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct') @@ -265,7 +263,7 @@ class StringResponseWithHintTest(unittest.TestCase): ''' def test_or_grade(self): problem_file = os.path.dirname(__file__)+"/test_files/stringresponse_with_hint.xml" - test_lcp = lcp.LoncapaProblem(open(problem_file), '1', system=i4xs) + test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=i4xs) correct_answers = {'1_2_1':'Michigan'} test_answers = {'1_2_1':'Minnesota'} self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct') @@ -618,7 +616,6 @@ class ModuleProgressTest(unittest.TestCase): ''' def test_xmodule_default(self): '''Make sure default get_progress exists, returns None''' - xm = x_module.XModule(i4xs, "", "dummy") + xm = x_module.XModule(i4xs, 'a://b/c/d/e', {}) p = xm.get_progress() self.assertEqual(p, None) - diff --git a/common/lib/xmodule/tests/test_export.py b/common/lib/xmodule/tests/test_export.py new file mode 100644 index 0000000000..97da2c4fe5 --- /dev/null +++ b/common/lib/xmodule/tests/test_export.py @@ -0,0 +1,28 @@ +from xmodule.modulestore.xml import XMLModuleStore +from nose.tools import assert_equals +from tempfile import mkdtemp +from fs.osfs import OSFS + + +def check_export_roundtrip(data_dir): + print "Starting import" + initial_import = XMLModuleStore('org', 'course', data_dir, eager=True) + initial_course = initial_import.course + + print "Starting export" + export_dir = mkdtemp() + fs = OSFS(export_dir) + xml = initial_course.export_to_xml(fs) + with fs.open('course.xml', 'w') as course_xml: + course_xml.write(xml) + + print "Starting second import" + second_import = XMLModuleStore('org', 'course', export_dir, eager=True) + + print "Checking key equality" + assert_equals(initial_import.modules.keys(), second_import.modules.keys()) + + print "Checking module equality" + for location in initial_import.modules.keys(): + print "Checking", location + assert_equals(initial_import.modules[location], second_import.modules[location]) diff --git a/common/lib/xmodule/test_files/formularesponse_with_hint.xml b/common/lib/xmodule/tests/test_files/formularesponse_with_hint.xml similarity index 100% rename from common/lib/xmodule/test_files/formularesponse_with_hint.xml rename to common/lib/xmodule/tests/test_files/formularesponse_with_hint.xml diff --git a/common/lib/xmodule/test_files/imageresponse.xml b/common/lib/xmodule/tests/test_files/imageresponse.xml similarity index 100% rename from common/lib/xmodule/test_files/imageresponse.xml rename to common/lib/xmodule/tests/test_files/imageresponse.xml diff --git a/common/lib/xmodule/test_files/multi_bare.xml b/common/lib/xmodule/tests/test_files/multi_bare.xml similarity index 100% rename from common/lib/xmodule/test_files/multi_bare.xml rename to common/lib/xmodule/tests/test_files/multi_bare.xml diff --git a/common/lib/xmodule/test_files/multichoice.xml b/common/lib/xmodule/tests/test_files/multichoice.xml similarity index 100% rename from common/lib/xmodule/test_files/multichoice.xml rename to common/lib/xmodule/tests/test_files/multichoice.xml diff --git a/common/lib/xmodule/test_files/optionresponse.xml b/common/lib/xmodule/tests/test_files/optionresponse.xml similarity index 100% rename from common/lib/xmodule/test_files/optionresponse.xml rename to common/lib/xmodule/tests/test_files/optionresponse.xml diff --git a/common/lib/xmodule/test_files/stringresponse_with_hint.xml b/common/lib/xmodule/tests/test_files/stringresponse_with_hint.xml similarity index 100% rename from common/lib/xmodule/test_files/stringresponse_with_hint.xml rename to common/lib/xmodule/tests/test_files/stringresponse_with_hint.xml diff --git a/common/lib/xmodule/test_files/symbolicresponse.xml b/common/lib/xmodule/tests/test_files/symbolicresponse.xml similarity index 100% rename from common/lib/xmodule/test_files/symbolicresponse.xml rename to common/lib/xmodule/tests/test_files/symbolicresponse.xml diff --git a/common/lib/xmodule/test_files/truefalse.xml b/common/lib/xmodule/tests/test_files/truefalse.xml similarity index 100% rename from common/lib/xmodule/test_files/truefalse.xml rename to common/lib/xmodule/tests/test_files/truefalse.xml diff --git a/common/lib/xmodule/xml_module.py b/common/lib/xmodule/xml_module.py deleted file mode 100644 index b167e52e88..0000000000 --- a/common/lib/xmodule/xml_module.py +++ /dev/null @@ -1,53 +0,0 @@ -from xmodule.x_module import XModuleDescriptor -from lxml import etree - - -class XmlDescriptor(XModuleDescriptor): - """ - Mixin class for standardized parsing of from xml - """ - - @classmethod - def definition_from_xml(cls, xml_object, system): - """ - Return the definition to be passed to the newly created descriptor - during from_xml - """ - raise NotImplementedError("%s does not implement definition_from_xml" % cls.__class__.__name__) - - @classmethod - def from_xml(cls, xml_data, system, org=None, course=None): - """ - Creates an instance of this descriptor from the supplied xml_data. - This may be overridden by subclasses - - xml_data: A string of xml that will be translated into data and children for - this module - system: An XModuleSystem for interacting with external resources - org and course are optional strings that will be used in the generated modules - url identifiers - """ - xml_object = etree.fromstring(xml_data) - - metadata = {} - for attr in ('format', 'graceperiod', 'showanswer', 'rerandomize', 'due'): - from_xml = xml_object.get(attr) - if from_xml is not None: - metadata[attr] = from_xml - - if xml_object.get('graded') is not None: - metadata['graded'] = xml_object.get('graded') == 'true' - - if xml_object.get('name') is not None: - metadata['display_name'] = xml_object.get('name') - - return cls( - system, - cls.definition_from_xml(xml_object, system), - location=['i4x', - org, - course, - xml_object.tag, - xml_object.get('slug')], - metadata=metadata, - ) diff --git a/common/lib/xmodule/__init__.py b/common/lib/xmodule/xmodule/__init__.py similarity index 100% rename from common/lib/xmodule/__init__.py rename to common/lib/xmodule/xmodule/__init__.py diff --git a/common/lib/xmodule/abtest_module.py b/common/lib/xmodule/xmodule/abtest_module.py similarity index 70% rename from common/lib/xmodule/abtest_module.py rename to common/lib/xmodule/xmodule/abtest_module.py index beaeb4ad1c..c3e32732f3 100644 --- a/common/lib/xmodule/abtest_module.py +++ b/common/lib/xmodule/xmodule/abtest_module.py @@ -7,6 +7,8 @@ from xmodule.raw_module import RawDescriptor from xmodule.xml_module import XmlDescriptor from xmodule.exceptions import InvalidDefinitionError +DEFAULT = "_DEFAULT_GROUP" + def group_from_value(groups, v): ''' Given group: (('a',0.3),('b',0.4),('c',0.3)) And random value @@ -39,30 +41,17 @@ class ABTestModule(XModule): def __init__(self, system, location, definition, instance_state=None, shared_state=None, **kwargs): XModule.__init__(self, system, location, definition, instance_state, shared_state, **kwargs) - target_groups = self.definition['data'].keys() if shared_state is None: self.group = group_from_value( - self.definition['data']['group_portions'], + self.definition['data']['group_portions'].items(), random.uniform(0, 1) ) else: shared_state = json.loads(shared_state) - - # TODO (cpennington): Remove this once we aren't passing in - # groups from django groups - if 'groups' in shared_state: - self.group = None - target_names = [elem.get('name') for elem in target_groups] - for group in shared_state['groups']: - if group in target_names: - self.group = group - break - else: - self.group = shared_state['group'] + self.group = shared_state['group'] def get_shared_state(self): - print self.group return json.dumps({'group': self.group}) def displayable_items(self): @@ -88,18 +77,16 @@ class ABTestDescriptor(RawDescriptor, XmlDescriptor): definition = { 'data': { 'experiment': experiment, - 'group_portions': [], - 'group_content': {None: []}, + 'group_portions': {}, + 'group_content': {DEFAULT: []}, }, 'children': []} for group in xml_object: if group.tag == 'default': - name = None + name = DEFAULT else: name = group.get('name') - definition['data']['group_portions'].append( - (name, float(group.get('portion', 0))) - ) + definition['data']['group_portions'][name] = float(group.get('portion', 0)) child_content_urls = [ system.process_xml(etree.tostring(child)).location.url() @@ -109,10 +96,29 @@ class ABTestDescriptor(RawDescriptor, XmlDescriptor): definition['data']['group_content'][name] = child_content_urls definition['children'].extend(child_content_urls) - default_portion = 1 - sum(portion for (name, portion) in definition['data']['group_portions']) + default_portion = 1 - sum(portion for (name, portion) in definition['data']['group_portions'].items()) if default_portion < 0: raise InvalidDefinitionError("ABTest portions must add up to less than or equal to 1") - definition['data']['group_portions'].append((None, default_portion)) + definition['data']['group_portions'][DEFAULT] = default_portion + definition['children'].sort() return definition + + def definition_to_xml(self, resource_fs): + xml_object = etree.Element('abtest') + xml_object.set('experiment', self.definition['data']['experiment']) + for name, group in self.definition['data']['group_content'].items(): + if name == DEFAULT: + group_elem = etree.SubElement(xml_object, 'default') + else: + group_elem = etree.SubElement(xml_object, 'group', attrib={ + 'portion': str(self.definition['data']['group_portions'][name]), + 'name': name, + }) + + for child_loc in group: + child = self.system.load_item(child_loc) + group_elem.append(etree.fromstring(child.export_to_xml(resource_fs))) + + return xml_object diff --git a/common/lib/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py similarity index 93% rename from common/lib/xmodule/capa_module.py rename to common/lib/xmodule/xmodule/capa_module.py index 6a95789417..b7f8e68b5e 100644 --- a/common/lib/xmodule/capa_module.py +++ b/common/lib/xmodule/xmodule/capa_module.py @@ -117,8 +117,6 @@ class CapaModule(XModule): if instance_state != None and 'attempts' in instance_state: self.attempts = instance_state['attempts'] - # TODO: Should be: self.filename=only_one(dom2.xpath('/problem/@filename')) - self.filename = "problems/" + only_one(dom2.xpath('/problem/@filename')) + ".xml" self.name = only_one(dom2.xpath('/problem/@name')) weight_string = only_one(dom2.xpath('/problem/@weight')) @@ -133,28 +131,18 @@ class CapaModule(XModule): seed = system.id else: seed = None + try: - fp = self.system.filestore.open(self.filename) + self.lcp = LoncapaProblem(self.definition['data'], self.location.html_id(), instance_state, seed=seed, system=self.system) except Exception: - log.exception('cannot open file %s' % self.filename) - if self.system.DEBUG: - # create a dummy problem instead of failing - fp = StringIO.StringIO('Problem file %s is missing' % self.filename) - fp.name = "StringIO" - else: - raise - try: - self.lcp = LoncapaProblem(fp, self.location.html_id(), instance_state, seed=seed, system=self.system) - except Exception: - msg = 'cannot create LoncapaProblem %s' % self.filename + msg = 'cannot create LoncapaProblem %s' % self.url log.exception(msg) if self.system.DEBUG: msg = '

%s

' % msg.replace('<', '<') msg += '

%s

' % traceback.format_exc().replace('<', '<') # create a dummy problem with error message instead of failing - fp = StringIO.StringIO('Problem file %s has an error:%s' % (self.filename, msg)) - fp.name = "StringIO" - self.lcp = LoncapaProblem(fp, self.location.html_id(), instance_state, seed=seed, system=self.system) + problem_text = 'Problem file %s has an error:%s' % (self.filename, msg) + self.lcp = LoncapaProblem(problem_text, self.location.html_id(), instance_state, seed=seed, system=self.system) else: raise @@ -406,13 +394,13 @@ class CapaModule(XModule): correct_map = self.lcp.grade_answers(answers) except StudentInputError as inst: # TODO (vshnayder): why is this line here? - self.lcp = LoncapaProblem(self.system.filestore.open(self.filename), + self.lcp = LoncapaProblem(self.system.filestore.open(self.filename).read(), id=lcp_id, state=old_state, system=self.system) traceback.print_exc() return {'success': inst.message} except: # TODO: why is this line here? - self.lcp = LoncapaProblem(self.system.filestore.open(self.filename), + self.lcp = LoncapaProblem(self.system.filestore.open(self.filename).read(), id=lcp_id, state=old_state, system=self.system) traceback.print_exc() raise Exception("error in capa_module") @@ -497,7 +485,7 @@ class CapaModule(XModule): # reset random number generator seed (note the self.lcp.get_state() in next line) self.lcp.seed = None - self.lcp = LoncapaProblem(self.system.filestore.open(self.filename), + self.lcp = LoncapaProblem(self.system.filestore.open(self.filename).read(), self.location.html_id(), self.lcp.get_state(), system=self.system) event_info['new_state'] = self.lcp.get_state() diff --git a/common/lib/xmodule/exceptions.py b/common/lib/xmodule/xmodule/exceptions.py similarity index 100% rename from common/lib/xmodule/exceptions.py rename to common/lib/xmodule/xmodule/exceptions.py diff --git a/common/lib/xmodule/graders.py b/common/lib/xmodule/xmodule/graders.py similarity index 100% rename from common/lib/xmodule/graders.py rename to common/lib/xmodule/xmodule/graders.py diff --git a/common/lib/xmodule/hidden_module.py b/common/lib/xmodule/xmodule/hidden_module.py similarity index 100% rename from common/lib/xmodule/hidden_module.py rename to common/lib/xmodule/xmodule/hidden_module.py diff --git a/common/lib/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py similarity index 63% rename from common/lib/xmodule/html_module.py rename to common/lib/xmodule/xmodule/html_module.py index 32963600cd..08fe4bbecc 100644 --- a/common/lib/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -1,10 +1,7 @@ -import json import logging from xmodule.x_module import XModule -from xmodule.mako_module import MakoModuleDescriptor -from xmodule.xml_module import XmlDescriptor -from lxml import etree +from xmodule.raw_module import RawDescriptor from pkg_resources import resource_string log = logging.getLogger("mitx.courseware") @@ -16,19 +13,16 @@ class HtmlModule(XModule): def __init__(self, system, location, definition, instance_state=None, shared_state=None, **kwargs): XModule.__init__(self, system, location, definition, instance_state, shared_state, **kwargs) - self.html = self.definition['data']['text'] + self.html = self.definition['data'] -class HtmlDescriptor(MakoModuleDescriptor, XmlDescriptor): +class HtmlDescriptor(RawDescriptor): """ Module for putting raw html in a course """ mako_template = "widgets/html-edit.html" module_class = HtmlModule + filename_extension = "html" js = {'coffee': [resource_string(__name__, 'js/module/html.coffee')]} js_module = 'HTML' - - @classmethod - def definition_from_xml(cls, xml_object, system): - return {'data': {'text': etree.tostring(xml_object)}} diff --git a/common/lib/xmodule/js/module/html.coffee b/common/lib/xmodule/xmodule/js/module/html.coffee similarity index 100% rename from common/lib/xmodule/js/module/html.coffee rename to common/lib/xmodule/xmodule/js/module/html.coffee diff --git a/common/lib/xmodule/js/module/raw.coffee b/common/lib/xmodule/xmodule/js/module/raw.coffee similarity index 100% rename from common/lib/xmodule/js/module/raw.coffee rename to common/lib/xmodule/xmodule/js/module/raw.coffee diff --git a/common/lib/xmodule/xmodule/mako_module.py b/common/lib/xmodule/xmodule/mako_module.py new file mode 100644 index 0000000000..9a90afb896 --- /dev/null +++ b/common/lib/xmodule/xmodule/mako_module.py @@ -0,0 +1,32 @@ +from x_module import XModuleDescriptor, DescriptorSystem + + +class MakoDescriptorSystem(DescriptorSystem): + def __init__(self, render_template, *args, **kwargs): + self.render_template = render_template + super(MakoDescriptorSystem, self).__init__(*args, **kwargs) + + +class MakoModuleDescriptor(XModuleDescriptor): + """ + Module descriptor intended as a mixin that uses a mako template + to specify the module html. + + Expects the descriptor to have the `mako_template` attribute set + with the name of the template to render, and it will pass + the descriptor as the `module` parameter to that template + """ + + def __init__(self, system, definition=None, **kwargs): + if getattr(system, 'render_template', None) is None: + raise TypeError('{system} must have a render_template function in order to use a MakoDescriptor'.format(system=system)) + super(MakoModuleDescriptor, self).__init__(system, definition, **kwargs) + + def get_context(self): + """ + Return the context to render the mako template with + """ + return {'module': self} + + def get_html(self): + return self.system.render_template(self.mako_template, self.get_context()) diff --git a/common/lib/keystore/__init__.py b/common/lib/xmodule/xmodule/modulestore/__init__.py similarity index 90% rename from common/lib/keystore/__init__.py rename to common/lib/xmodule/xmodule/modulestore/__init__.py index 0671e7e568..00b3b13bb0 100644 --- a/common/lib/keystore/__init__.py +++ b/common/lib/xmodule/xmodule/modulestore/__init__.py @@ -119,7 +119,7 @@ class Location(_LocationBase): """ Return a string with a version of the location that is safe for use in html id attributes """ - return "-".join(str(v) for v in self if v is not None) + return "-".join(str(v) for v in self.list() if v is not None).replace('.', '_') def dict(self): return self.__dict__ @@ -145,8 +145,8 @@ class ModuleStore(object): recent revision If any segment of the location is None except revision, raises - keystore.exceptions.InsufficientSpecificationError - If no object is found at that location, raises keystore.exceptions.ItemNotFoundError + xmodule.modulestore.exceptions.InsufficientSpecificationError + If no object is found at that location, raises xmodule.modulestore.exceptions.ItemNotFoundError location: Something that can be passed to Location default_class: An XModuleDescriptor subclass to use if no plugin matching the @@ -171,9 +171,19 @@ class ModuleStore(object): def update_children(self, location, children): """ Set the children for the item specified by the location to - data + children location: Something that can be passed to Location children: A list of child item identifiers """ raise NotImplementedError + + def update_metadata(self, location, metadata): + """ + Set the metadata for the item specified by the location to + metadata + + location: Something that can be passed to Location + metadata: A nested dictionary of module metadata + """ + raise NotImplementedError diff --git a/common/lib/xmodule/xmodule/modulestore/django.py b/common/lib/xmodule/xmodule/modulestore/django.py new file mode 100644 index 0000000000..546aaf30c8 --- /dev/null +++ b/common/lib/xmodule/xmodule/modulestore/django.py @@ -0,0 +1,26 @@ +""" +Module that provides a connection to the ModuleStore specified in the django settings. + +Passes settings.MODULESTORE as kwargs to MongoModuleStore +""" + +from __future__ import absolute_import + +from importlib import import_module + +from django.conf import settings + +_MODULESTORES = {} + + +def modulestore(name='default'): + global _MODULESTORES + + if name not in _MODULESTORES: + class_path = settings.MODULESTORE[name]['ENGINE'] + module_path, _, class_name = class_path.rpartition('.') + class_ = getattr(import_module(module_path), class_name) + _MODULESTORES[name] = class_( + **settings.MODULESTORE[name]['OPTIONS']) + + return _MODULESTORES[name] diff --git a/common/lib/keystore/exceptions.py b/common/lib/xmodule/xmodule/modulestore/exceptions.py similarity index 100% rename from common/lib/keystore/exceptions.py rename to common/lib/xmodule/xmodule/modulestore/exceptions.py diff --git a/common/lib/keystore/mongo.py b/common/lib/xmodule/xmodule/modulestore/mongo.py similarity index 74% rename from common/lib/keystore/mongo.py rename to common/lib/xmodule/xmodule/modulestore/mongo.py index 20c4ffde1a..cc731c929c 100644 --- a/common/lib/keystore/mongo.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo.py @@ -1,6 +1,8 @@ import pymongo from importlib import import_module -from xmodule.x_module import XModuleDescriptor, DescriptorSystem +from xmodule.x_module import XModuleDescriptor +from xmodule.mako_module import MakoDescriptorSystem +from mitxmako.shortcuts import render_to_string from . import ModuleStore, Location from .exceptions import ItemNotFoundError, InsufficientSpecificationError @@ -15,6 +17,7 @@ class MongoModuleStore(ModuleStore): host=host, port=port )[db][collection] + self.collection.ensure_index('location') # Force mongo to report errors, at the expense of performance self.collection.safe = True @@ -30,8 +33,8 @@ class MongoModuleStore(ModuleStore): recent revision If any segment of the location is None except revision, raises - keystore.exceptions.InsufficientSpecificationError - If no object is found at that location, raises keystore.exceptions.ItemNotFoundError + xmodule.modulestore.exceptions.InsufficientSpecificationError + If no object is found at that location, raises xmodule.modulestore.exceptions.ItemNotFoundError location: Something that can be passed to Location """ @@ -53,7 +56,7 @@ class MongoModuleStore(ModuleStore): # TODO (cpennington): Pass a proper resources_fs to the system return XModuleDescriptor.load_from_json( - item, DescriptorSystem(self.get_item, None), self.default_class) + item, MakoDescriptorSystem(load_item=self.get_item, resources_fs=None, render_template=render_to_string), self.default_class) def create_item(self, location): """ @@ -84,7 +87,7 @@ class MongoModuleStore(ModuleStore): def update_children(self, location, children): """ Set the children for the item specified by the location to - data + children location: Something that can be passed to Location children: A list of child item identifiers @@ -96,3 +99,19 @@ class MongoModuleStore(ModuleStore): {'location': Location(location).dict()}, {'$set': {'definition.children': children}} ) + + def update_metadata(self, location, metadata): + """ + Set the children for the item specified by the location to + metadata + + location: Something that can be passed to Location + metadata: A nested dictionary of module metadata + """ + + # See http://www.mongodb.org/display/DOCS/Updating for + # atomic update syntax + self.collection.update( + {'location': Location(location).dict()}, + {'$set': {'metadata': metadata}} + ) diff --git a/common/lib/keystore/tests/test_location.py b/common/lib/xmodule/xmodule/modulestore/tests/test_location.py similarity index 95% rename from common/lib/keystore/tests/test_location.py rename to common/lib/xmodule/xmodule/modulestore/tests/test_location.py index 01d36d946b..d598d8ae6d 100644 --- a/common/lib/keystore/tests/test_location.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_location.py @@ -1,6 +1,6 @@ from nose.tools import assert_equals, assert_raises, assert_not_equals -from keystore import Location -from keystore.exceptions import InvalidLocationError +from xmodule.modulestore import Location +from xmodule.modulestore.exceptions import InvalidLocationError def check_string_roundtrip(url): diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py new file mode 100644 index 0000000000..a5db17054b --- /dev/null +++ b/common/lib/xmodule/xmodule/modulestore/xml.py @@ -0,0 +1,137 @@ +import logging +from fs.osfs import OSFS +from importlib import import_module +from lxml import etree +from path import path +from xmodule.x_module import XModuleDescriptor, XMLParsingSystem +from xmodule.mako_module import MakoDescriptorSystem + +from . import ModuleStore, Location +from .exceptions import ItemNotFoundError + +etree.set_default_parser(etree.XMLParser(dtd_validation=False, load_dtd=False, + remove_comments=True, remove_blank_text=True)) + +log = logging.getLogger(__name__) + + +class XMLModuleStore(ModuleStore): + """ + An XML backed ModuleStore + """ + def __init__(self, org, course, data_dir, default_class=None, eager=False): + """ + Initialize an XMLModuleStore from data_dir + + org, course: Strings to be used in module keys + data_dir: path to data directory containing course.xml + default_class: dot-separated string defining the default descriptor class to use if non is specified in entry_points + eager: If true, load the modules children immediately to force the entire course tree to be parsed + """ + self.data_dir = path(data_dir) + self.modules = {} + + if default_class is None: + self.default_class = None + else: + module_path, _, class_name = default_class.rpartition('.') + class_ = getattr(import_module(module_path), class_name) + self.default_class = class_ + + with open(self.data_dir / "course.xml") as course_file: + class ImportSystem(XMLParsingSystem, MakoDescriptorSystem): + def __init__(self, modulestore): + """ + modulestore: the XMLModuleStore to store the loaded modules in + """ + self.unnamed_modules = 0 + self.used_slugs = set() + + def process_xml(xml): + try: + xml_data = etree.fromstring(xml) + except: + log.exception("Unable to parse xml: {xml}".format(xml=xml)) + raise + if xml_data.get('slug') is None: + if xml_data.get('name'): + slug = Location.clean(xml_data.get('name')) + else: + self.unnamed_modules += 1 + slug = '{tag}_{count}'.format(tag=xml_data.tag, count=self.unnamed_modules) + + if slug in self.used_slugs: + self.unnamed_modules += 1 + slug = '{slug}_{count}'.format(slug=slug, count=self.unnamed_modules) + + self.used_slugs.add(slug) + xml_data.set('slug', slug) + + module = XModuleDescriptor.load_from_xml(etree.tostring(xml_data), self, org, course, modulestore.default_class) + modulestore.modules[module.location] = module + + if eager: + module.get_children() + return module + + system_kwargs = dict( + render_template=lambda: '', + load_item=modulestore.get_item, + resources_fs=OSFS(data_dir), + process_xml=process_xml + ) + MakoDescriptorSystem.__init__(self, **system_kwargs) + XMLParsingSystem.__init__(self, **system_kwargs) + + self.course = ImportSystem(self).process_xml(course_file.read()) + + def get_item(self, location): + """ + Returns an XModuleDescriptor instance for the item at location. + If location.revision is None, returns the most item with the most + recent revision + + If any segment of the location is None except revision, raises + xmodule.modulestore.exceptions.InsufficientSpecificationError + If no object is found at that location, raises xmodule.modulestore.exceptions.ItemNotFoundError + + location: Something that can be passed to Location + """ + location = Location(location) + try: + return self.modules[location] + except KeyError: + raise ItemNotFoundError(location) + + def create_item(self, location): + raise NotImplementedError("XMLModuleStores are read-only") + + def update_item(self, location, data): + """ + Set the data in the item specified by the location to + data + + location: Something that can be passed to Location + data: A nested dictionary of problem data + """ + raise NotImplementedError("XMLModuleStores are read-only") + + def update_children(self, location, children): + """ + Set the children for the item specified by the location to + data + + location: Something that can be passed to Location + children: A list of child item identifiers + """ + raise NotImplementedError("XMLModuleStores are read-only") + + def update_metadata(self, location, metadata): + """ + Set the metadata for the item specified by the location to + metadata + + location: Something that can be passed to Location + metadata: A nested dictionary of module metadata + """ + raise NotImplementedError("XMLModuleStores are read-only") diff --git a/common/lib/xmodule/progress.py b/common/lib/xmodule/xmodule/progress.py similarity index 100% rename from common/lib/xmodule/progress.py rename to common/lib/xmodule/xmodule/progress.py diff --git a/common/lib/xmodule/raw_module.py b/common/lib/xmodule/xmodule/raw_module.py similarity index 86% rename from common/lib/xmodule/raw_module.py rename to common/lib/xmodule/xmodule/raw_module.py index 43a92303ad..9fe9a9198b 100644 --- a/common/lib/xmodule/raw_module.py +++ b/common/lib/xmodule/xmodule/raw_module.py @@ -3,6 +3,7 @@ from lxml import etree from xmodule.mako_module import MakoModuleDescriptor from xmodule.xml_module import XmlDescriptor + class RawDescriptor(MakoModuleDescriptor, XmlDescriptor): """ Module that provides a raw editing view of it's data and children @@ -21,3 +22,6 @@ class RawDescriptor(MakoModuleDescriptor, XmlDescriptor): @classmethod def definition_from_xml(cls, xml_object, system): return {'data': etree.tostring(xml_object)} + + def definition_to_xml(self, resource_fs): + return etree.fromstring(self.definition['data']) diff --git a/common/lib/xmodule/schematic_module.py b/common/lib/xmodule/xmodule/schematic_module.py similarity index 100% rename from common/lib/xmodule/schematic_module.py rename to common/lib/xmodule/xmodule/schematic_module.py diff --git a/common/lib/xmodule/seq_module.py b/common/lib/xmodule/xmodule/seq_module.py similarity index 94% rename from common/lib/xmodule/seq_module.py rename to common/lib/xmodule/xmodule/seq_module.py index 6d493b96ad..9f00c3be87 100644 --- a/common/lib/xmodule/seq_module.py +++ b/common/lib/xmodule/xmodule/seq_module.py @@ -108,3 +108,9 @@ class SequenceDescriptor(MakoModuleDescriptor, XmlDescriptor): system.process_xml(etree.tostring(child_module)).location.url() for child_module in xml_object ]} + + def definition_to_xml(self, resource_fs): + xml_object = etree.Element('sequential') + for child in self.get_children(): + xml_object.append(etree.fromstring(child.export_to_xml(resource_fs))) + return xml_object diff --git a/common/lib/xmodule/template_module.py b/common/lib/xmodule/xmodule/template_module.py similarity index 99% rename from common/lib/xmodule/template_module.py rename to common/lib/xmodule/xmodule/template_module.py index 1057fc2a25..064d48f431 100644 --- a/common/lib/xmodule/template_module.py +++ b/common/lib/xmodule/xmodule/template_module.py @@ -37,5 +37,6 @@ class CustomTagModule(XModule): def get_html(self): return self.html + class CustomTagDescriptor(RawDescriptor): module_class = CustomTagModule diff --git a/common/lib/xmodule/translation_module.py b/common/lib/xmodule/xmodule/translation_module.py similarity index 100% rename from common/lib/xmodule/translation_module.py rename to common/lib/xmodule/xmodule/translation_module.py diff --git a/common/lib/xmodule/vertical_module.py b/common/lib/xmodule/xmodule/vertical_module.py similarity index 100% rename from common/lib/xmodule/vertical_module.py rename to common/lib/xmodule/xmodule/vertical_module.py diff --git a/common/lib/xmodule/video_module.py b/common/lib/xmodule/xmodule/video_module.py similarity index 100% rename from common/lib/xmodule/video_module.py rename to common/lib/xmodule/xmodule/video_module.py diff --git a/common/lib/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py similarity index 80% rename from common/lib/xmodule/x_module.py rename to common/lib/xmodule/xmodule/x_module.py index 191cda6b06..8bfbb5f91a 100644 --- a/common/lib/xmodule/x_module.py +++ b/common/lib/xmodule/xmodule/x_module.py @@ -2,7 +2,7 @@ from lxml import etree import pkg_resources import logging -from keystore import Location +from xmodule.modulestore import Location from functools import partial log = logging.getLogger('mitx.' + __name__) @@ -23,30 +23,38 @@ class Plugin(object): entry_point: The name of the entry point to load plugins from """ + + _plugin_cache = None + @classmethod def load_class(cls, identifier, default=None): """ - Loads a single class intance specified by identifier. If identifier + Loads a single class instance specified by identifier. If identifier specifies more than a single class, then logs a warning and returns the first class identified. If default is not None, will return default if no entry_point matching identifier is found. Otherwise, will raise a ModuleMissingError """ - identifier = identifier.lower() - classes = list(pkg_resources.iter_entry_points(cls.entry_point, name=identifier)) - if len(classes) > 1: - log.warning("Found multiple classes for {entry_point} with identifier {id}: {classes}. Returning the first one.".format( - entry_point=cls.entry_point, - id=identifier, - classes=", ".join(class_.module_name for class_ in classes))) + if cls._plugin_cache is None: + cls._plugin_cache = {} - if len(classes) == 0: - if default is not None: - return default - raise ModuleMissingError(identifier) + if identifier not in cls._plugin_cache: + identifier = identifier.lower() + classes = list(pkg_resources.iter_entry_points(cls.entry_point, name=identifier)) + if len(classes) > 1: + log.warning("Found multiple classes for {entry_point} with identifier {id}: {classes}. Returning the first one.".format( + entry_point=cls.entry_point, + id=identifier, + classes=", ".join(class_.module_name for class_ in classes))) - return classes[0].load() + if len(classes) == 0: + if default is not None: + return default + raise ModuleMissingError(identifier) + + cls._plugin_cache[identifier] = classes[0].load() + return cls._plugin_cache[identifier] @classmethod def load_classes(cls): @@ -205,6 +213,11 @@ class XModuleDescriptor(Plugin): # A list of metadata that this module can inherit from its parent module inheritable_metadata = ('graded', 'due', 'graceperiod', 'showanswer', 'rerandomize') + # A list of descriptor attributes that must be equal for the discriptors to be + # equal + equality_attributes = ('definition', 'metadata', 'location', 'shared_state_key', '_inherited_metadata') + + # ============================= STRUCTURAL MANIPULATION =========================== def __init__(self, system, definition=None, @@ -222,7 +235,7 @@ class XModuleDescriptor(Plugin): definition: A dict containing `data` and `children` representing the problem definition Current arguments passed in kwargs: - location: A keystore.Location object indicating the name and ownership of this problem + location: A xmodule.modulestore.Location object indicating the name and ownership of this problem shared_state_key: The key to use for sharing StudentModules with other modules of this type metadata: A dictionary containing the following optional keys: @@ -244,7 +257,46 @@ class XModuleDescriptor(Plugin): self.shared_state_key = kwargs.get('shared_state_key') self._child_instances = None + self._inherited_metadata = set() + def inherit_metadata(self, metadata): + """ + Updates this module with metadata inherited from a containing module. + Only metadata specified in self.inheritable_metadata will + be inherited + """ + # Set all inheritable metadata from kwargs that are + # in self.inheritable_metadata and aren't already set in metadata + for attr in self.inheritable_metadata: + if attr not in self.metadata and attr in metadata: + self._inherited_metadata.add(attr) + self.metadata[attr] = metadata[attr] + + def get_children(self): + """Returns a list of XModuleDescriptor instances for the children of this module""" + if self._child_instances is None: + self._child_instances = [] + for child_loc in self.definition.get('children', []): + child = self.system.load_item(child_loc) + child.inherit_metadata(self.metadata) + self._child_instances.append(child) + + return self._child_instances + + def xmodule_constructor(self, system): + """ + Returns a constructor for an XModule. This constructor takes two arguments: + instance_state and shared_state, and returns a fully nstantiated XModule + """ + return partial( + self.module_class, + system, + self.location, + self.definition, + metadata=self.metadata + ) + + # ================================= JSON PARSING =================================== @staticmethod def load_from_json(json_data, system, default_class=None): """ @@ -272,6 +324,7 @@ class XModuleDescriptor(Plugin): """ return cls(system=system, **json_data) + # ================================= XML PARSING ==================================== @staticmethod def load_from_xml(xml_data, system, @@ -307,6 +360,20 @@ class XModuleDescriptor(Plugin): """ raise NotImplementedError('Modules must implement from_xml to be parsable from xml') + def export_to_xml(self, resource_fs): + """ + Returns an xml string representing this module, and all modules underneath it. + May also write required resources out to resource_fs + + Assumes that modules have single parantage (that no module appears twice in the same course), + and that it is thus safe to nest modules as xml children as appropriate. + + The returned XML should be able to be parsed back into an identical XModuleDescriptor + using the from_xml method with the same system, org, and course + """ + raise NotImplementedError('Modules must implement export_to_xml to enable xml export') + + # ================================== HTML INTERFACE DEFINITIONS ====================== @classmethod def get_javascript(cls): """ @@ -326,52 +393,36 @@ class XModuleDescriptor(Plugin): """ return self.js_module - - def inherit_metadata(self, metadata): - """ - Updates this module with metadata inherited from a containing module. - Only metadata specified in self.inheritable_metadata will - be inherited - """ - # Set all inheritable metadata from kwargs that are - # in self.inheritable_metadata and aren't already set in metadata - for attr in self.inheritable_metadata: - if attr not in self.metadata and attr in metadata: - self.metadata[attr] = metadata[attr] - - def get_children(self): - """Returns a list of XModuleDescriptor instances for the children of this module""" - if self._child_instances is None: - self._child_instances = [] - for child_loc in self.definition.get('children', []): - child = self.system.load_item(child_loc) - child.inherit_metadata(self.metadata) - self._child_instances.append(child) - - return self._child_instances - def get_html(self): """ Return the html used to edit this module """ raise NotImplementedError("get_html() must be provided by specific modules") - def xmodule_constructor(self, system): - """ - Returns a constructor for an XModule. This constructor takes two arguments: - instance_state and shared_state, and returns a fully nstantiated XModule - """ - return partial( - self.module_class, - system, - self.location, - self.definition, + # =============================== BUILTIN METHODS =========================== + def __eq__(self, other): + eq = (self.__class__ == other.__class__ and + all(getattr(self, attr, None) == getattr(other, attr, None) + for attr in self.equality_attributes)) + + if not eq: + for attr in self.equality_attributes: + print getattr(self, attr, None), getattr(other, attr, None), getattr(self, attr, None) == getattr(other, attr, None) + + return eq + + def __repr__(self): + return "{class_}({system!r}, {definition!r}, location={location!r}, metadata={metadata!r})".format( + class_=self.__class__.__name__, + system=self.system, + definition=self.definition, + location=self.location, metadata=self.metadata ) class DescriptorSystem(object): - def __init__(self, load_item, resources_fs): + def __init__(self, load_item, resources_fs, **kwargs): """ load_item: Takes a Location and returns an XModuleDescriptor resources_fs: A Filesystem object that contains all of the @@ -383,7 +434,7 @@ class DescriptorSystem(object): class XMLParsingSystem(DescriptorSystem): - def __init__(self, load_item, resources_fs, process_xml): + def __init__(self, load_item, resources_fs, process_xml, **kwargs): """ process_xml: Takes an xml string, and returns the the XModuleDescriptor created from that xml """ diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py new file mode 100644 index 0000000000..aebb024a59 --- /dev/null +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -0,0 +1,199 @@ +from collections import MutableMapping +from xmodule.x_module import XModuleDescriptor +from lxml import etree +import copy +import logging + +log = logging.getLogger(__name__) + +class LazyLoadingDict(MutableMapping): + """ + A dictionary object that lazily loads it's contents from a provided + function on reads (of members that haven't already been set) + """ + + def __init__(self, loader): + self._contents = {} + self._loaded = False + self._loader = loader + self._deleted = set() + + def __getitem__(self, name): + if not (self._loaded or name in self._contents or name in self._deleted): + self.load() + + return self._contents[name] + + def __setitem__(self, name, value): + self._contents[name] = value + self._deleted.discard(name) + + def __delitem__(self, name): + del self._contents[name] + self._deleted.add(name) + + def __contains__(self, name): + self.load() + return name in self._contents + + def __len__(self): + self.load() + return len(self._contents) + + def __iter__(self): + self.load() + return iter(self._contents) + + def __repr__(self): + self.load() + return repr(self._contents) + + def load(self): + if self._loaded: + return + + loaded_contents = self._loader() + loaded_contents.update(self._contents) + self._contents = loaded_contents + self._loaded = True + + +class XmlDescriptor(XModuleDescriptor): + """ + Mixin class for standardized parsing of from xml + """ + + # Extension to append to filename paths + filename_extension = 'xml' + + # The attributes will be removed from the definition xml passed + # to definition_from_xml, and from the xml returned by definition_to_xml + metadata_attributes = ('format', 'graceperiod', 'showanswer', 'rerandomize', + 'due', 'graded', 'name', 'slug') + + @classmethod + def definition_from_xml(cls, xml_object, system): + """ + Return the definition to be passed to the newly created descriptor + during from_xml + + xml_object: An etree Element + """ + raise NotImplementedError("%s does not implement definition_from_xml" % cls.__name__) + + @classmethod + def clean_metadata_from_xml(cls, xml_object): + """ + Remove any attribute named in self.metadata_attributes from the supplied xml_object + """ + for attr in cls.metadata_attributes: + if xml_object.get(attr) is not None: + del xml_object.attrib[attr] + + @classmethod + def from_xml(cls, xml_data, system, org=None, course=None): + """ + Creates an instance of this descriptor from the supplied xml_data. + This may be overridden by subclasses + + xml_data: A string of xml that will be translated into data and children for + this module + system: An XModuleSystem for interacting with external resources + org and course are optional strings that will be used in the generated modules + url identifiers + """ + xml_object = etree.fromstring(xml_data) + + def metadata_loader(): + metadata = {} + for attr in ('format', 'graceperiod', 'showanswer', 'rerandomize', 'due'): + from_xml = xml_object.get(attr) + if from_xml is not None: + metadata[attr] = from_xml + + if xml_object.get('graded') is not None: + metadata['graded'] = xml_object.get('graded') == 'true' + + if xml_object.get('name') is not None: + metadata['display_name'] = xml_object.get('name') + + return metadata + + def definition_loader(): + filename = xml_object.get('filename') + if filename is None: + definition_xml = copy.deepcopy(xml_object) + else: + filepath = cls._format_filepath(xml_object.tag, filename) + with system.resources_fs.open(filepath) as file: + try: + definition_xml = etree.parse(file).getroot() + except: + log.exception("Failed to parse xml in file %s" % filepath) + raise + + cls.clean_metadata_from_xml(definition_xml) + return cls.definition_from_xml(definition_xml, system) + + return cls( + system, + LazyLoadingDict(definition_loader), + location=['i4x', + org, + course, + xml_object.tag, + xml_object.get('slug')], + metadata=LazyLoadingDict(metadata_loader), + ) + + @classmethod + def _format_filepath(cls, type, name): + return '{type}/{name}.{ext}'.format(type=type, name=name, ext=cls.filename_extension) + + def export_to_xml(self, resource_fs): + """ + Returns an xml string representing this module, and all modules underneath it. + May also write required resources out to resource_fs + + Assumes that modules have single parantage (that no module appears twice in the same course), + and that it is thus safe to nest modules as xml children as appropriate. + + The returned XML should be able to be parsed back into an identical XModuleDescriptor + using the from_xml method with the same system, org, and course + """ + xml_object = self.definition_to_xml(resource_fs) + self.__class__.clean_metadata_from_xml(xml_object) + + # Put content in a separate file if it's large (has more than 5 descendent tags) + if len(list(xml_object.iter())) > 5: + + filepath = self.__class__._format_filepath(self.category, self.name) + resource_fs.makedir(self.category, allow_recreate=True) + with resource_fs.open(filepath, 'w') as file: + file.write(etree.tostring(xml_object, pretty_print=True)) + + for child in xml_object: + xml_object.remove(child) + + xml_object.set('filename', self.name) + + xml_object.set('slug', self.name) + xml_object.tag = self.category + + for attr in ('format', 'graceperiod', 'showanswer', 'rerandomize', 'due'): + if attr in self.metadata and attr not in self._inherited_metadata: + xml_object.set(attr, self.metadata[attr]) + + if 'graded' in self.metadata and 'graded' not in self._inherited_metadata: + xml_object.set('graded', str(self.metadata['graded']).lower()) + + if 'display_name' in self.metadata: + xml_object.set('name', self.metadata['display_name']) + + return etree.tostring(xml_object, pretty_print=True) + + def definition_to_xml(self, resource_fs): + """ + Return a new etree Element object created from this modules definition. + """ + raise NotImplementedError("%s does not implement definition_to_xml" % self.__class__.__name__) diff --git a/lms/djangoapps/courseware/management/commands/check_course.py b/lms/djangoapps/courseware/management/commands/check_course.py index afc7e47857..6ccd6d5fe7 100644 --- a/lms/djangoapps/courseware/management/commands/check_course.py +++ b/lms/djangoapps/courseware/management/commands/check_course.py @@ -10,7 +10,7 @@ import xmodule import mitxmako.middleware as middleware middleware.MakoMiddleware() -from keystore.django import keystore +from xmodule.modulestore.django import modulestore from courseware.models import StudentModuleCache from courseware.module_render import get_module @@ -78,7 +78,7 @@ class Command(BaseCommand): # TODO (cpennington): Get coursename in a legitimate way course_location = 'i4x://edx/6002xs12/course/6.002_Spring_2012' - student_module_cache = StudentModuleCache(sample_user, keystore().get_item(course_location)) + student_module_cache = StudentModuleCache(sample_user, modulestore().get_item(course_location)) (course, _, _, _) = get_module(sample_user, None, course_location, student_module_cache) to_run = [ diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 5119cc2910..679084f28c 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -6,7 +6,7 @@ from django.http import Http404 from django.http import HttpResponse from lxml import etree -from keystore.django import keystore +from xmodule.modulestore.django import modulestore from mitxmako.shortcuts import render_to_string from models import StudentModule, StudentModuleCache @@ -129,7 +129,7 @@ def toc_for_course(user, request, course_location, active_chapter, active_sectio chapters with name 'hidden' are skipped. ''' - student_module_cache = StudentModuleCache(user, keystore().get_item(course_location), depth=2) + student_module_cache = StudentModuleCache(user, modulestore().get_item(course_location), depth=2) (course, _, _, _) = get_module(user, request, course_location, student_module_cache) chapters = list() @@ -161,7 +161,7 @@ def get_section(course, chapter, section): section: Section name """ try: - course_module = keystore().get_item(course) + course_module = modulestore().get_item(course) except: log.exception("Unable to load course_module") return None @@ -205,7 +205,7 @@ def get_module(user, request, location, student_module_cache, position=None): instance_module is a StudentModule specific to this module for this student shared_module is a StudentModule specific to all modules with the same 'shared_state_key' attribute, or None if the module doesn't elect to share state ''' - descriptor = keystore().get_item(location) + descriptor = modulestore().get_item(location) instance_module = student_module_cache.lookup(descriptor.category, descriptor.location.url()) shared_state_key = getattr(descriptor, 'shared_state_key', None) @@ -273,8 +273,11 @@ def add_histogram(module): module_id = module.id histogram = grade_histogram(module_id) render_histogram = len(histogram) > 0 - staff_context = {'definition': json.dumps(module.definition, indent=4), - 'metadata': json.dumps(module.metadata, indent=4), + + # Cast module.definition and module.metadata to dicts so that json can dump them + # even though they are lazily loaded + staff_context = {'definition': json.dumps(dict(module.definition), indent=4), + 'metadata': json.dumps(dict(module.metadata), indent=4), 'element_id': module.location.html_id(), 'histogram': json.dumps(histogram), 'render_histogram': render_histogram, @@ -301,7 +304,7 @@ def modx_dispatch(request, dispatch=None, id=None): # If there are arguments, get rid of them dispatch, _, _ = dispatch.partition('?') - student_module_cache = StudentModuleCache(request.user, keystore().get_item(id)) + student_module_cache = StudentModuleCache(request.user, modulestore().get_item(id)) instance, instance_module, shared_module, module_type = get_module(request.user, request, id, student_module_cache) if instance_module is None: diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py index 48e9bcc795..8b723ca980 100644 --- a/lms/djangoapps/courseware/views.py +++ b/lms/djangoapps/courseware/views.py @@ -12,13 +12,11 @@ from mitxmako.shortcuts import render_to_response, render_to_string from django_future.csrf import ensure_csrf_cookie from django.views.decorators.cache import cache_control -from lxml import etree - from module_render import toc_for_course, get_module, get_section from models import StudentModuleCache from student.models import UserProfile from multicourse import multicourse_settings -from keystore.django import keystore +from xmodule.modulestore.django import modulestore from util.cache import cache from student.models import UserTestGroup @@ -26,9 +24,6 @@ from courseware import grades log = logging.getLogger("mitx.courseware") -etree.set_default_parser(etree.XMLParser(dtd_validation=False, load_dtd=False, - remove_comments=True)) - template_imports = {'urllib': urllib} @@ -68,7 +63,7 @@ def gradebook(request): course_location = multicourse_settings.get_course_location(coursename) for student in student_objects: - student_module_cache = StudentModuleCache(student, keystore().get_item(course_location)) + student_module_cache = StudentModuleCache(student, modulestore().get_item(course_location)) course, _, _, _ = get_module(request.user, request, course_location, student_module_cache) student_info.append({ 'username': student.username, @@ -98,7 +93,7 @@ def profile(request, student_id=None): coursename = multicourse_settings.get_coursename_from_request(request) course_location = multicourse_settings.get_course_location(coursename) - student_module_cache = StudentModuleCache(request.user, keystore().get_item(course_location)) + student_module_cache = StudentModuleCache(request.user, modulestore().get_item(course_location)) course, _, _, _ = get_module(request.user, request, course_location, student_module_cache) context = {'name': user_info.name, diff --git a/lms/envs/common.py b/lms/envs/common.py index d1faf00f62..4c3cdc2dda 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -138,9 +138,9 @@ COURSE_SETTINGS = {'6.002_Spring_2012': {'number' : '6.002x', ############################### XModule Store ################################## -KEYSTORE = { +MODULESTORE = { 'default': { - 'ENGINE': 'keystore.xml.XMLModuleStore', + 'ENGINE': 'xmodule.modulestore.xml.XMLModuleStore', 'OPTIONS': { 'org': 'edx', 'course': '6002xs12', diff --git a/rakefile b/rakefile index e76e200777..f5e32d8110 100644 --- a/rakefile +++ b/rakefile @@ -84,6 +84,14 @@ default_options = { args.with_defaults(:env => 'dev', :options => default_options[system]) sh(django_admin(system, args.env, 'runserver', args.options)) end + + Dir["#{system}/envs/*.py"].each do |env_file| + env = File.basename(env_file).gsub(/\.py/, '') + desc "Attempt to import the settings file #{system}.envs.#{env} and report any errors" + task "#{system}:check_settings:#{env}" do + sh("echo 'import #{system}.envs.#{env}' | #{django_admin(system, env, 'shell')}") + end + end end Dir["common/lib/*"].each do |lib|