Merge remote-tracking branch 'origin/master' into dormsbee/multicourse

Conflicts:
	cms/djangoapps/contentstore/management/commands/import.py
	cms/djangoapps/contentstore/views.py
	cms/envs/common.py
	cms/envs/dev.py
	cms/envs/test.py
	cms/static/sass/README.txt
	cms/static/sass/_base.scss
	cms/static/sass/_calendar.scss
	cms/static/sass/base-style.scss
	cms/templates/base.html
	cms/templates/index.html
	cms/templates/widgets/header.html
	cms/templates/widgets/module-dropdown.html
	cms/templates/widgets/navigation.html
	cms/templates/widgets/problem-edit.html
	cms/urls.py
	common/djangoapps/pipeline_mako/__init__.py
	common/djangoapps/util/views.py
	common/lib/capa/capa_problem.py
	common/lib/capa/templates/textinput_dynamath.html
	common/lib/mitxmako/middleware.py
	common/lib/mitxmako/shortcuts.py
	common/lib/mitxmako/template.py
	common/lib/xmodule/capa_module.py
	common/lib/xmodule/seq_module.py
	common/lib/xmodule/setup.py
	common/lib/xmodule/test_files/symbolicresponse.xml
	common/lib/xmodule/test_files/test_files/symbolicresponse.xml
	common/lib/xmodule/tests/__init__.py
	common/lib/xmodule/tests/test_files/symbolicresponse.xml
	common/lib/xmodule/vertical_module.py
	common/lib/xmodule/video_module.py
	common/lib/xmodule/x_module.py
	lms/djangoapps/courseware/content_parser.py
	lms/djangoapps/courseware/grades.py
	lms/djangoapps/courseware/module_render.py
	lms/djangoapps/courseware/views.py
	lms/static/coffee/spec/helper.coffee
	lms/static/coffee/spec/modules/video/video_player_spec.coffee
	lms/static/coffee/spec/modules/video/video_volume_control_spec.coffee
	lms/static/coffee/src/modules/problem.coffee
	lms/static/coffee/src/modules/sequence.coffee
	lms/static/coffee/src/modules/video/video_player.coffee
	lms/static/coffee/src/modules/video/video_volume_control.coffee
	lms/static/js/vendor/jquery-1.6.2.min.js
	lms/static/js/vendor/jquery-ui-1.8.16.custom.min.js
	lms/static/sass/application.scss
	lms/static/sass/courseware/_sequence-nav.scss
	lms/static/sass/courseware/_video.scss
	lms/templates/main.html
	lms/urls.py
	rakefile
	requirements.txt

README

@@ -1,2 +1 @@
-This branch (re-)adds dynamic math and symbolicresponse.
-Test cases included.
+see doc/ for documentation.

cms/djangoapps/contentstore/management/commands/import.py

@@ -1,167 +1,29 @@
-### 
+###
 ### One-off script for importing courseware form XML format
 ###
 
+from django.core.management.base import BaseCommand, CommandError
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.xml import XMLModuleStore
 
-#import mitxmako.middleware
-#from courseware import content_parser
-#from django.contrib.auth.models import User
-import os.path
-from StringIO import StringIO
-from mako.template import Template
-from mako.lookup import TemplateLookup
+unnamed_modules = 0
 
-from django.core.management.base import BaseCommand
-from keystore.django import keystore
-
-from lxml import etree
 
 class Command(BaseCommand):
     help = \
-''' Run FTP server.'''
+'''Import the specified data directory into the default ModuleStore'''
+
     def handle(self, *args, **options):
-        print args
-        data_dir = args[0]
- 
-        parser = etree.XMLParser(remove_comments = True)
+        if len(args) != 3:
+            raise CommandError("import requires 3 arguments: <org> <course> <data directory>")
 
-        lookup = TemplateLookup(directories=[data_dir])
-        template = lookup.get_template("course.xml")
-        course_string = template.render(groups=[])
-        course = etree.parse(StringIO(course_string), parser=parser)
+        org, course, data_dir = args
 
-        elements = list(course.iter())
-
-        tag_to_category = {
-            # Custom tags
-            'videodev': 'Custom',
-            'slides': 'Custom',
-            'book': 'Custom',
-            'image': 'Custom',
-            'discuss': 'Custom',
-            # Simple lists
-            'chapter': 'Week',
-            'course': 'Course',
-            'sequential': 'LectureSequence',
-            'vertical': 'ProblemSet',
-            'section': {
-                'Lab': 'Lab',
-                'Lecture Sequence': 'LectureSequence',
-                'Homework': 'Homework',
-                'Tutorial Index': 'TutorialIndex',
-                'Video': 'VideoSegment',
-                'Midterm': 'Exam',
-                'Final': 'Exam',
-                None: 'Section',
-            },
-            # True types
-            'video': 'VideoSegment',
-            'html': 'HTML',
-            'problem': 'Problem',
-            }
-
-        name_index = 0
-        for e in elements:
-            name = e.attrib.get('name', None)
-            for f in elements:
-                if f != e and f.attrib.get('name', None) == name:
-                    name = None
-            if not name:
-                name = "{tag}_{index}".format(tag=e.tag, index=name_index)
-                name_index = name_index + 1
-            if e.tag in tag_to_category:
-                category = tag_to_category[e.tag]
-                if isinstance(category, dict):
-                    category = category[e.get('format')]
-                category = category.replace('/', '-')
-                name = name.replace('/', '-')
-                e.set('url', 'i4x://mit.edu/6002xs12/{category}/{name}'.format(
-                    category=category,
-                    name=name))
-
-
-        def handle_skip(e):
-            print "Skipping ", e
-
-        results = {}
-
-        def handle_custom(e):
-            data = {'type':'i4x://mit.edu/6002xs12/tag/{tag}'.format(tag=e.tag),
-                    'attrib':dict(e.attrib)}
-            results[e.attrib['url']] = {'data':data}
-
-        def handle_list(e):
-            if e.attrib.get("class", None) == "tutorials":
-                return
-            children = [le.attrib['url'] for le in e.getchildren()]
-            results[e.attrib['url']] = {'children':children}
-
-        def handle_video(e):
-            url = e.attrib['url']
-            clip_url = url.replace('VideoSegment', 'VideoClip')
-            # Take: 0.75:izygArpw-Qo,1.0:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8
-            # Make: [(0.75, 'izygArpw-Qo'), (1.0, 'p2Q6BrNhdh8'), (1.25, '1EeWXzPdhSA'), (1.5, 'rABDYkeK0x8')]
-            youtube_str = e.attrib['youtube'] 
-            youtube_list =  [(float(x), y) for x,y in map(lambda x:x.split(':'), youtube_str.split(','))]
-            clip_infos = [{ "status": "ready", 
-                            "format": "youtube", 
-                            "sane": True, 
-                            "location": "youtube", 
-                            "speed": speed, 
-                            "id": youtube_id, 
-                            "size": None} \
-                          for (speed, youtube_id) \
-                              in youtube_list]
-            results[clip_url] = {'data':{'clip_infos':clip_infos}}
-            results[url] = {'children' : [{'url':clip_url}]}
-
-        def handle_html(e):
-            if 'src' in e.attrib:
-                text = open(data_dir+'html/'+e.attrib['src']).read()
-            else:
-                textlist=[e.text]+[etree.tostring(elem) for elem in e]+[e.tail]
-                textlist=[i for i in textlist if type(i)==str]
-                text = "".join(textlist)
-
-            results[e.attrib['url']] = {'data':{'text':text}}
-
-        def handle_problem(e):
-            data = open(os.path.join(data_dir, 'problems', e.attrib['filename']+'.xml')).read()
-            results[e.attrib['url']] = {'data':{'statement':data}}
-
-        element_actions = {# Inside HTML ==> Skip these
-            'a': handle_skip,
-            'h1': handle_skip,
-            'h2': handle_skip,
-            'hr': handle_skip,
-            'strong': handle_skip,
-            'ul': handle_skip,
-            'li': handle_skip,
-            'p': handle_skip,
-            # Custom tags
-            'videodev': handle_custom,
-            'slides': handle_custom,
-            'book': handle_custom,
-            'image': handle_custom,
-            'discuss': handle_custom,
-            # Simple lists
-            'chapter': handle_list,
-            'course': handle_list,
-            'sequential': handle_list,
-            'vertical': handle_list,
-            'section': handle_list,
-            # True types
-            'video': handle_video,
-            'html': handle_html,
-            'problem': handle_problem,
-            }
-
-        for e in elements:
-            element_actions[e.tag](e)
-
-        for k in results:
-            keystore().create_item(k, 'Piotr Mitros')
-            if 'data' in results[k]:
-                keystore().update_item(k, results[k]['data'])
-            if 'children' in results[k]:
-                keystore().update_children(k, results[k]['children'])
+        module_store = XMLModuleStore(org, course, data_dir, 'xmodule.raw_module.RawDescriptor', eager=True)
+        for module in module_store.modules.itervalues():
+            modulestore().create_item(module.location)
+            if 'data' in module.definition:
+                modulestore().update_item(module.location, module.definition['data'])
+            if 'children' in module.definition:
+                modulestore().update_children(module.location, module.definition['children'])
+            modulestore().update_metadata(module.location, dict(module.metadata))

cms/djangoapps/contentstore/views.py

@@ -1,13 +1,48 @@
 from mitxmako.shortcuts import render_to_response
-from keystore.django import keystore
-from django.contrib.auth.decorators import login_required
+from xmodule.modulestore.django import modulestore
+from django_future.csrf import ensure_csrf_cookie
+from django.http import HttpResponse
+import json
 
+from fs.osfs import OSFS
 
+@ensure_csrf_cookie
 def index(request):
     # TODO (cpennington): These need to be read in from the active user
     org = 'mit.edu'
     course = '6002xs12'
-    name = '6.002 Spring 2012'
-    course = keystore().get_item(['i4x', org, course, 'Course', name])
+    name = '6.002_Spring_2012'
+    course = modulestore().get_item(['i4x', org, course, 'course', name])
     weeks = course.get_children()
     return render_to_response('index.html', {'weeks': weeks})
+
+
+def edit_item(request):
+    item_id = request.GET['id']
+    item = modulestore().get_item(item_id)
+    return render_to_response('unit.html', {
+        'contents': item.get_html(),
+        'js_module': item.js_module_name(),
+        'category': item.category,
+        'name': item.name,
+    })
+
+
+def save_item(request):
+    item_id = request.POST['id']
+    data = json.loads(request.POST['data'])
+    modulestore().update_item(item_id, data)
+    return HttpResponse(json.dumps({}))
+
+
+def temp_force_export(request):
+    org = 'mit.edu'
+    course = '6002xs12'
+    name = '6.002_Spring_2012'
+    course = modulestore().get_item(['i4x', org, course, 'course', name])
+    fs = OSFS('../data-export-test')
+    xml = course.export_to_xml(fs)
+    with fs.open('course.xml', 'w') as course_xml:
+        course_xml.write(xml)
+
+    return HttpResponse('Done')

cms/envs/common.py

@@ -21,6 +21,9 @@ Longer TODO:
 
 import sys
 import tempfile
+import os.path
+import os
+import errno
 from path import path
 
 ############################ FEATURE CONFIGURATION #############################
@@ -154,7 +157,39 @@ PIPELINE_CSS = {
 
 PIPELINE_ALWAYS_RECOMPILE = ['sass/base-style.scss']
 
+from xmodule.x_module import XModuleDescriptor
+from xmodule.raw_module import RawDescriptor
+js_file_dir = PROJECT_ROOT / "static" / "coffee" / "module"
+try:
+    os.makedirs(js_file_dir)
+except OSError as exc:
+    if exc.errno == errno.EEXIST:
+        pass
+    else:
+        raise
+
+module_js_sources = []
+for xmodule in XModuleDescriptor.load_classes() + [RawDescriptor]:
+    js = xmodule.get_javascript()
+    for filetype in ('coffee', 'js'):
+        for idx, fragment in enumerate(js.get(filetype, [])):
+            path = os.path.join(js_file_dir, "{name}.{idx}.{type}".format(
+                name=xmodule.__name__,
+                idx=idx,
+                type=filetype))
+            with open(path, 'w') as js_file:
+                js_file.write(fragment)
+            module_js_sources.append(path.replace(PROJECT_ROOT / "static/", ""))
+
 PIPELINE_JS = {
+    'main': {
+        'source_filenames': ['coffee/main.coffee', 'coffee/unit.coffee'],
+        'output_filename': 'js/main.js',
+    },
+    'module-js': {
+        'source_filenames': module_js_sources,
+        'output_filename': 'js/modules.js',
+    }
 }
 
 PIPELINE_COMPILERS = [

cms/envs/dev.py

@@ -3,14 +3,22 @@ This config file runs the simplest dev environment"""
 
 from .common import *
 
+import logging
+import sys
+logging.basicConfig(stream=sys.stdout, )
+
 DEBUG = True
 TEMPLATE_DEBUG = DEBUG
 
-KEYSTORE = {
+MODULESTORE = {
     'default': {
-        'host': 'localhost',
-        'db': 'mongo_base',
-        'collection': 'key_store',
+        'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
+        'OPTIONS': {
+            'default_class': 'xmodule.raw_module.RawDescriptor',
+            'host': 'localhost',
+            'db': 'mongo_base',
+            'collection': 'key_store',
+        }
     }
 }
 

cms/envs/test.py

@@ -17,7 +17,7 @@ for app in os.listdir(PROJECT_ROOT / 'djangoapps'):
     NOSE_ARGS += ['--cover-package', app]
 TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
 
-KEYSTORE = {
+MODULESTORE = {
     'host': 'localhost',
     'db': 'mongo_base',
     'collection': 'key_store',

cms/static/coffee/.gitignore

@@ -0,0 +1,2 @@
+*.js
+module

cms/static/coffee/main.coffee

@@ -0,0 +1,90 @@
+class @CMS
+    @setHeight = =>
+        windowHeight = $(this).height()
+        @contentHeight = windowHeight - 29
+
+    @bind = =>
+        $('a.module-edit').click ->
+            CMS.edit_item($(this).attr('id'))
+            return false
+        $(window).bind('resize', CMS.setHeight)
+
+    @edit_item = (id) =>
+        $.get('/edit_item', {id: id}, (data) =>
+            $('#module-html').empty().append(data)
+            CMS.bind()
+            $('body.content .cal').css('height', @contentHeight)
+            $('body').addClass('content')
+            $('section.edit-pane').show()
+            new Unit('unit-wrapper', id)
+        )
+
+$ ->
+    $.ajaxSetup
+        headers : { 'X-CSRFToken': $.cookie 'csrftoken' }
+    $('section.main-content').children().hide()
+    $('.editable').inlineEdit()
+    $('.editable-textarea').inlineEdit({control: 'textarea'})
+
+    heighest = 0
+    $('.cal ol > li').each ->
+        heighest = if $(this).height() > heighest then $(this).height() else heighest
+
+    $('.cal ol > li').css('height',heighest + 'px')
+
+    $('.add-new-section').click -> return false
+
+    $('.new-week .close').click ->
+        $(this).parents('.new-week').hide()
+        $('p.add-new-week').show()
+        return false
+
+    $('.save-update').click ->
+        $(this).parent().parent().hide()
+        return false
+
+    # $('html').keypress ->
+    #   $('.wip').css('visibility', 'visible')
+
+    setHeight = ->
+        windowHeight = $(this).height()
+        contentHeight = windowHeight - 29
+
+        $('section.main-content > section').css('min-height', contentHeight)
+        $('body.content .cal').css('height', contentHeight)
+
+        $('.edit-week').click ->
+            $('body').addClass('content')
+            $('body.content .cal').css('height', contentHeight)
+            $('section.edit-pane').show()
+            return false
+
+        $('a.week-edit').click ->
+            $('body').addClass('content')
+            $('body.content .cal').css('height', contentHeight)
+            $('section.edit-pane').show()
+            return false
+
+        $('a.sequence-edit').click ->
+            $('body').addClass('content')
+            $('body.content .cal').css('height', contentHeight)
+            $('section.edit-pane').show()
+            return false
+
+        $('a.module-edit').click ->
+          $('body.content .cal').css('height', contentHeight)
+
+    $(document).ready(setHeight)
+    $(window).bind('resize', setHeight)
+
+    $('.video-new a').click ->
+        $('section.edit-pane').show()
+        return false
+
+    $('.problem-new a').click ->
+        $('section.edit-pane').show()
+        return false
+    
+    CMS.setHeight()
+    CMS.bind()
+

cms/static/coffee/unit.coffee

@@ -0,0 +1,15 @@
+class @Unit
+    constructor: (@element_id, @module_id) ->
+        @module = new window[$("##{@element_id}").attr('class')] 'module-html'
+
+        $("##{@element_id} .save-update").click (event) =>
+            event.preventDefault()
+            $.post("save_item", {
+                id: @module_id
+                data: JSON.stringify(@module.save())
+            })
+
+        $("##{@element_id} .cancel").click (event) =>
+            event.preventDefault()
+            CMS.edit_item(@module_id)
+

cms/static/js/jquery.cookie.js

@@ -0,0 +1,47 @@
+/*!
+ * jQuery Cookie Plugin
+ * https://github.com/carhartl/jquery-cookie
+ *
+ * Copyright 2011, Klaus Hartl
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://www.opensource.org/licenses/mit-license.php
+ * http://www.opensource.org/licenses/GPL-2.0
+ */
+(function($) {
+    $.cookie = function(key, value, options) {
+
+        // key and at least value given, set cookie...
+        if (arguments.length > 1 && (!/Object/.test(Object.prototype.toString.call(value)) || value === null || value === undefined)) {
+            options = $.extend({}, options);
+
+            if (value === null || value === undefined) {
+                options.expires = -1;
+            }
+
+            if (typeof options.expires === 'number') {
+                var days = options.expires, t = options.expires = new Date();
+                t.setDate(t.getDate() + days);
+            }
+
+            value = String(value);
+
+            return (document.cookie = [
+                encodeURIComponent(key), '=', options.raw ? value : encodeURIComponent(value),
+                options.expires ? '; expires=' + options.expires.toUTCString() : '', // use expires attribute, max-age is not supported by IE
+                options.path    ? '; path=' + options.path : '',
+                options.domain  ? '; domain=' + options.domain : '',
+                options.secure  ? '; secure' : ''
+            ].join(''));
+        }
+
+        // key and possibly options given, get cookie...
+        options = value || {};
+        var decode = options.raw ? function(s) { return s; } : decodeURIComponent;
+
+        var pairs = document.cookie.split('; ');
+        for (var i = 0, pair; pair = pairs[i] && pairs[i].split('='); i++) {
+            if (decode(pair[0]) === key) return decode(pair[1] || ''); // IE saves cookies with empty string as "c; ", e.g. without "=" as opposed to EOMB, thus pair[1] may be undefined
+        }
+        return null;
+    };
+})(jQuery);

cms/static/sass/README.txt

@@ -1,3 +1,3 @@
 Sass Watch:
 
-sass --watch cms/static/sass:cms/static/css -r ./cms/static/sass/bourbon/lib/bourbon.rb
+sass --watch cms/static/sass:cms/static/sass -r ./cms/static/sass/bourbon/lib/bourbon.rb

cms/static/sass/_base.scss

@@ -1,82 +1,65 @@
 $fg-column: 70px;
 $fg-gutter: 26px;
 $fg-max-columns: 12;
-$body-font-family: "Lucida Grande", "Lucida Sans Unicode", "Lucida Sans", Geneva, Verdana, sans-serif;
+$body-font-family: "Open Sans", "Lucida Grande", "Lucida Sans Unicode", "Lucida Sans", Geneva, Verdana, sans-serif;
+$body-font-size: 14px;
+$body-line-height: 20px;
 
+$light-blue: #f0f8fa;
+$dark-blue: #50545c;
+$bright-blue: #3c8ebf;
+$orange: #f96e5b;
+$yellow: #fff8af;
+
+// Base html styles
 html {
   height: 100%;
 }
 
-body {
-  @include clearfix();
-  height: 100%;
-  font: 14px $body-font-family;
-
-  > section {
-    display: table;
-    width: 100%;
-  }
-
-  > header {
-    background: #000;
-    color: #fff;
-    display: block;
-    float: none;
-    padding: 6px 20px;
-    width: 100%;
-    @include box-sizing(border-box);
-
-    nav {
-      @include clearfix;
-
-      h2 {
-        font-size: 14px;
-        text-transform: uppercase;
-        float: left;
-      }
-
-      ul {
-        float: left;
-
-        &.user-nav {
-          float: right;
-        }
-
-        li {
-          @include inline-block();
-          margin-left: 15px;
-        }
-      }
-    }
-  }
-
-  &.content {
-    section.main-content {
-      border-left: 2px solid #000;
-      @include box-sizing(border-box);
-      width: flex-grid(9);
-      float: left;
-      @include box-shadow( -2px 0 3px #ddd );
-    }
-  }
-}
-
 a {
   text-decoration: none;
   color: #888;
+  @include transition;
+
+  &:hover {
+    color: #ccc;
+  }
 }
 
 input {
   font-family: $body-font-family;
 }
 
-input[type="submit"], .button {
-  border: 1px solid #ccc;
-  background: #efefef;
-  @include border-radius(3px);
-  padding: 6px;
+button, input[type="submit"], .button {
+  background-color: $orange;
+  color: #fff;
+  -webkit-font-smoothing: antialiased;
+  padding: 8px 10px;
+  border: 0;
+  font-weight: bold;
+
+  &:hover {
+    background-color: shade($orange, 10%);
+  }
 }
 
+#{$all-text-inputs}, textarea {
+  border: 1px solid $dark-blue;
+  font: $body-font-size $body-font-family;
+  padding: 4px 6px;
+  @include box-shadow(inset 0 3px 6px $light-blue);
+}
+
+textarea {
+  @include box-sizing(border-box);
+  display: block;
+  line-height: lh();
+  padding: 15px;
+  width: 100%;
+
+}
+
+// Extends
 .new-module {
   position: relative;
 
@@ -111,3 +94,26 @@ input[type="submit"], .button {
   display: block;
   float: right;
 }
+
+.editable {
+  &:hover {
+    background: lighten($yellow, 10%);
+  }
+  button {
+    padding: 4px 10px;
+  }
+}
+
+.wip {
+  outline: 1px solid #f00 !important;
+  position: relative;
+
+  &:after {
+   content: "WIP";
+   font-size: 8px;
+   padding: 2px;
+   background: #f00;
+   color: #fff;
+   @include position(absolute, 0px 0px 0 0);
+  }
+}

cms/static/sass/_calendar.scss

@@ -6,8 +6,12 @@ section.cal {
   > header {
     @include clearfix;
     margin-bottom: 10px;
-    background: #efefef;
-    border: 1px solid #ddd;
+    opacity: .4;
+    @include transition;
+
+    &:hover {
+      opacity: 1;
+    }
 
     h2 {
       @include inline-block();
@@ -15,7 +19,6 @@ section.cal {
       letter-spacing: 1px;
       font-size: 14px;
       padding: 6px;
-      margin-left: 6px;
       font-size: 12px;
     }
 
@@ -25,9 +28,8 @@ section.cal {
       li  {
         @include inline-block;
         margin-left: 6px;
-        padding-left: 6px;
         border-left: 1px solid #ddd;
-        padding: 6px;
+        padding: 0 6px;
 
         a {
           @include inline-block();
@@ -49,32 +51,35 @@ section.cal {
   ol {
     list-style: none;
     @include clearfix;
-    @include box-sizing(border-box);
-    border-left: 1px solid #333;
-    border-top: 1px solid #333;
+    border-left: 1px solid lighten($dark-blue, 40%);
+    border-top: 1px solid lighten($dark-blue, 40%);
     width: 100%;
+    @include box-sizing(border-box);
 
     > li {
-      border-right: 1px solid #333;
-      border-bottom: 1px solid;
+      border-right: 1px solid lighten($dark-blue, 40%);
+      border-bottom: 1px solid lighten($dark-blue, 40%);
       @include box-sizing(border-box);
       float: left;
       width: flex-grid(3) + ((flex-gutter() * 3) / 4);
+      background-color: lighten($light-blue, 2%);
+
 
       header {
-        border-bottom: 1px solid #000;
-        @include box-shadow(0 1px 2px #aaa);
+        border-bottom: 1px solid lighten($dark-blue, 40%);
+        @include box-shadow(0 2px 2px $light-blue);
         display: block;
         margin-bottom: 2px;
+        background: #FFF;
 
         h1 {
           font-size: 14px;
           text-transform: uppercase;
-          border-bottom: 1px solid #ccc;
+          border-bottom: 1px solid lighten($dark-blue, 60%);
           padding: 6px;
 
           a {
-            color: #000;
+            color: $bright-blue;
             display: block;
           }
         }
@@ -85,6 +90,10 @@ section.cal {
             color: #888;
             border-bottom: 0;
             font-size: 12px;
+
+            &:hover {
+              background: #fff;
+            }
           }
         }
       }
@@ -94,10 +103,17 @@ section.cal {
         margin-bottom: 1px;
 
         li {
-          background: #efefef;
-          border-bottom: 1px solid #666;
+          border-bottom: 1px solid darken($light-blue, 8%);
           padding: 6px;
 
+          &:hover {
+            background: lighten($yellow, 10%);
+          }
+
+          a {
+            color: lighten($dark-blue, 10%);
+          }
+
           &.create-module {
             position: relative;
 

cms/static/sass/_fonts.scss

@@ -0,0 +1,36 @@
+@font-face {
+  font-family: 'Open Sans';
+  font-style: normal;
+  font-weight: 700;
+  src: local('Open Sans Bold'), local('OpenSans-Bold'), url(http://themes.googleusercontent.com/static/fonts/opensans/v6/k3k702ZOKiLJc3WVjuplzKRDOzjiPcYnFooOUGCOsRk.woff) format('woff');
+}
+@font-face {
+  font-family: 'Open Sans';
+  font-style: normal;
+  font-weight: 300;
+  src: local('Open Sans Light'), local('OpenSans-Light'), url(http://themes.googleusercontent.com/static/fonts/opensans/v6/DXI1ORHCpsQm3Vp6mXoaTaRDOzjiPcYnFooOUGCOsRk.woff) format('woff');
+}
+@font-face {
+  font-family: 'Open Sans';
+  font-style: italic;
+  font-weight: 700;
+  src: local('Open Sans Bold Italic'), local('OpenSans-BoldItalic'), url(http://themes.googleusercontent.com/static/fonts/opensans/v6/PRmiXeptR36kaC0GEAetxhbnBKKEOwRKgsHDreGcocg.woff) format('woff');
+}
+@font-face {
+  font-family: 'Open Sans';
+  font-style: italic;
+  font-weight: 300;
+  src: local('Open Sans Light Italic'), local('OpenSansLight-Italic'), url(http://themes.googleusercontent.com/static/fonts/opensans/v6/PRmiXeptR36kaC0GEAetxvR_54zmj3SbGZQh3vCOwvY.woff) format('woff');
+}
+@font-face {
+  font-family: 'Open Sans';
+  font-style: italic;
+  font-weight: 400;
+  src: local('Open Sans Italic'), local('OpenSans-Italic'), url(http://themes.googleusercontent.com/static/fonts/opensans/v6/xjAJXh38I15wypJXxuGMBrrIa-7acMAeDBVuclsi6Gc.woff) format('woff');
+}
+@font-face {
+  font-family: 'Open Sans';
+  font-style: normal;
+  font-weight: 400;
+  src: local('Open Sans'), local('OpenSans'), url(http://themes.googleusercontent.com/static/fonts/opensans/v6/cJZKeOuBrn4kERxqtaUH3bO3LdcAZYWl9Si6vvxL-qU.woff) format('woff');
+}

cms/static/sass/_layout.scss

@@ -0,0 +1,80 @@
+body {
+  @include clearfix();
+  height: 100%;
+  font: 14px $body-font-family;
+
+  > section {
+    display: table;
+    width: 100%;
+  }
+
+  > header {
+    background: $dark-blue;
+    color: #fff;
+    display: block;
+    float: none;
+    padding: 8px 25px;
+    width: 100%;
+    @include box-sizing(border-box);
+    -webkit-font-smoothing: antialiased;
+
+    nav {
+      @include clearfix;
+
+      h2 {
+        font-size: 14px;
+        text-transform: uppercase;
+        float: left;
+        margin-right: 15px;
+
+        a {
+          color: #fff;
+
+          &:hover {
+            color: rgba(#fff, .6);
+          }
+        }
+      }
+
+      a {
+        color: rgba(#fff, .8);
+
+        &:hover {
+          color: rgba(#fff, .6);
+        }
+      }
+
+      ul {
+        float: left;
+
+        &.user-nav {
+          float: right;
+        }
+
+        li {
+          @include inline-block();
+
+          a {
+            padding: 8px 10px;
+            display: block;
+            margin: -8px 0;
+
+            &:hover {
+              background-color: darken($dark-blue, 15%);
+            }
+          }
+        }
+      }
+    }
+  }
+
+  &.content {
+    section.main-content {
+      border-left: 2px solid $dark-blue;
+      @include box-sizing(border-box);
+      width: flex-grid(9);
+      float: left;
+      @include box-shadow( -2px 0 0 darken($light-blue, 3%));
+    }
+  }
+}

cms/static/sass/_section.scss

@@ -0,0 +1,187 @@
+section#unit-wrapper {
+  section.filters {
+    @include clearfix;
+    margin-bottom: 10px;
+    opacity: .4;
+    @include transition;
+
+    &:hover {
+      opacity: 1;
+    }
+
+    ul {
+      @include clearfix();
+      list-style: none;
+      padding: 6px;
+
+      li {
+        @include inline-block();
+
+        &.advanced {
+          float: right;
+        }
+      }
+    }
+  }
+
+  div.content {
+    display: table;
+    border: 1px solid lighten($dark-blue, 40%);
+    width: 100%;
+
+    section {
+      header {
+        background: #fff;
+        padding: 6px;
+        border-bottom: 1px solid lighten($dark-blue, 60%);
+        border-top: 1px solid lighten($dark-blue, 60%);
+        margin-top: -1px;
+        @include clearfix;
+
+        h2 {
+          text-transform: uppercase;
+          letter-spacing: 1px;
+          font-size: 12px;
+          float: left;
+          color: $bright-blue;
+        }
+      }
+
+      &.modules {
+        @include box-sizing(border-box);
+        display: table-cell;
+        width: flex-grid(6, 9);
+        border-right: 1px solid lighten($dark-blue, 40%);
+
+        &.empty {
+          text-align: center;
+          vertical-align: middle;
+
+          a {
+            @extend .button;
+            @include inline-block();
+            margin-top: 10px;
+          }
+        }
+
+        ol {
+          list-style: none;
+
+          li {
+            border-bottom: 1px solid lighten($dark-blue, 60%);
+
+            a {
+              color: #000;
+            }
+
+            ol {
+              list-style: none;
+
+              li {
+                padding: 6px;
+
+                &:last-child {
+                  border-bottom: 0;
+                }
+
+                &:hover {
+                  a.draggable {
+                    opacity: 1;
+                  }
+                }
+
+                a.draggable {
+                  float: right;
+                  opacity: .5;
+                }
+
+                &.group {
+                  padding: 0;
+
+                  header {
+                    padding: 6px;
+                    background: none;
+
+                    h3 {
+                      font-size: 14px;
+                    }
+                  }
+
+
+                  ol {
+                    border-left: 4px solid #999;
+                    border-bottom: 0;
+
+                    li {
+                      &:last-child {
+                        border-bottom: 0;
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+
+      &.scratch-pad {
+        @include box-sizing(border-box);
+        display: table-cell;
+        width: flex-grid(3, 9) + flex-gutter(9);
+        vertical-align: top;
+
+        ol {
+          list-style: none;
+
+          li {
+            border-bottom: 1px solid darken($light-blue, 8%);
+            background: lighten($light-blue, 2%);
+
+            &:last-child {
+              border-bottom: 0;
+            }
+
+            ul {
+              list-style: none;
+
+              li {
+                padding: 6px;
+
+                &:last-child {
+                  border-bottom: 0;
+                }
+
+                &:hover {
+                  a.draggable {
+                    opacity: 1;
+                  }
+                }
+
+                &.empty {
+                  padding: 12px;
+
+                  a {
+                    @extend .button;
+                    display: block;
+                    text-align: center;
+                  }
+                }
+
+                a.draggable {
+                  float: right;
+                  opacity: .3;
+                }
+
+                a {
+                  color: #000;
+                }
+              }
+            }
+
+          }
+        }
+      }
+    }
+  }
+}

cms/static/sass/_unit.scss

@@ -0,0 +1,225 @@
+section#unit-wrapper {
+  > header {
+    border-bottom: 2px solid $dark-blue;
+    @include clearfix();
+    @include box-shadow( 0 2px 0 darken($light-blue, 3%));
+    padding: 6px 20px;
+
+    section {
+      float: left;
+
+      h1 {
+        font-size: 16px;
+        text-transform: uppercase;
+        letter-spacing: 1px;
+        @include inline-block();
+        color: $bright-blue;
+      }
+
+      p {
+        @include inline-block();
+        margin-left: 10px;
+        color: #999;
+        font-size: 12px;
+        font-style: italic;
+      }
+    }
+
+    div {
+      float: right;
+      color: #666;
+
+      a {
+        display: block;
+        @include inline-block;
+
+       &.cancel {
+         margin-right: 20px;
+         font-style: italic;
+         font-size: 12px;
+       }
+
+       &.save-update {
+         @extend .button;
+         margin: -6px -25px -6px 0;
+       }
+      }
+    }
+  }
+
+  > section {
+    padding: 20px;
+
+    section.meta {
+      section {
+        &.status-settings {
+          float: left;
+          margin-bottom: 10px;
+          color: $dark-blue;
+
+          ul {
+            list-style: none;
+            border: 1px solid lighten($dark-blue, 40%);
+            @include inline-block();
+
+            li {
+              @include inline-block();
+              border-right: 1px solid lighten($dark-blue, 40%);
+              padding: 6px;
+
+              &:last-child {
+                border-right: 0;
+              }
+
+              &.current {
+                background: #eee;
+              }
+
+              a {
+                color: $dark-blue;
+              }
+            }
+          }
+
+          a.settings {
+            @include inline-block();
+            margin: 0 20px;
+            padding: 6px;
+            border: 1px solid lighten($dark-blue, 40%);
+            color: $dark-blue;
+          }
+
+          select {
+            float: right;
+          }
+        }
+
+        &.author {
+          float: right;
+          color: lighten($dark-blue, 6%);
+
+          dl {
+            dt {
+              font-weight: bold;
+            }
+
+            dd, dt {
+              @include inline-block();
+            }
+          }
+        }
+
+        &.tags {
+          background: $light-blue;
+          color: lighten($dark-blue, 6%);
+          padding: 10px;
+          margin: 0 0 20px;
+          @include clearfix();
+          clear: both;
+
+          div {
+            float: left;
+            margin-right: 20px;
+
+            h2 {
+              font-size: 14px;
+              @include inline-block();
+            }
+
+            p {
+              @include inline-block();
+            }
+          }
+        }
+      }
+    }
+
+    //general styles for main content
+    .preview {
+      @include box-sizing(border-box);
+      border: 1px solid  lighten($dark-blue, 40%);
+      min-height: 40px;
+      padding: 10px;
+      width: 100%;
+      margin-top: 10px;
+
+      h1 {
+        font-size: 24px;
+        margin-bottom: 1em;
+      }
+
+      h2 {
+        font-size: 20px;
+        margin-bottom: 1em;
+      }
+
+      h3 {
+        font-size: 18;
+        margin-bottom: 1em;
+      }
+
+      ul {
+        padding-left: 20px;
+        margin-bottom: 1em;
+      }
+
+      p {
+        margin-bottom: 1em;
+      }
+    }
+
+    //notes
+    section.notes {
+      margin-top: 40px;
+      padding: 40px 0 0;
+      border-top: 2px solid lighten( $dark-blue, 60% );
+
+      h2 {
+        margin-bottom: 6px;
+        font-size: $body-font-size;
+        text-transform: uppercase;
+        color: $bright-blue;
+        letter-spacing: 1px;
+      }
+
+      form {
+        margin-bottom: 20px;
+
+        input[type="submit"]{
+          margin-top: 10px;
+        }
+      }
+
+      ul {
+        list-style: none;
+
+        li {
+          margin-bottom: 20px;
+
+          p {
+            margin-bottom: 10px;
+
+            &.author {
+              font-style: italic;
+              color: #999;
+            }
+          }
+        }
+      }
+    }
+
+    div.actions {
+      a.save-update {
+        @extend .button;
+        @include inline-block();
+        margin-top: 20px;
+      }
+
+      a.cancel {
+        float: right;
+        font-style: italic;
+        margin-top: 20px;
+      }
+    }
+  }
+}

cms/static/sass/base-style.scss

@@ -1,6 +1,6 @@
 @import 'bourbon/bourbon';
 @import 'reset';
 
-@import 'base';
+@import 'base', 'layout';
 @import 'calendar';
-@import 'week', 'video', 'problem', 'module-header';
+@import 'section', 'unit';

cms/templates/base.html

@@ -23,11 +23,18 @@
 
       <%block name="content"></%block>
 
-    <script src="//ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js"></script>
+    <script type="text/javascript" src="${ STATIC_URL}/js/jquery.min.js"></script>
     <script type="text/javascript" src="${ STATIC_URL }/js/markitup/jquery.markitup.js"></script>
     <script type="text/javascript" src="${ STATIC_URL }/js/markitup/sets/wiki/set.js"></script>
+    % if settings.MITX_FEATURES['USE_DJANGO_PIPELINE']:
+    <%static:js group='main'/>
+    % else:
     <script src="${ STATIC_URL }/js/main.js"></script>
+    % endif
+
+    <%static:js group='module-js'/>
     <script src="${ STATIC_URL }/js/jquery.inlineedit.js"></script>
+    <script src="${ STATIC_URL }/js/jquery.cookie.js"></script>
     <script src="${ STATIC_URL }/js/jquery.leanModal.min.js"></script>
     <script src="${ STATIC_URL }/js/jquery.tablednd.js"></script>
   </body>

cms/templates/index.html

@@ -7,13 +7,9 @@
   <%include file="widgets/navigation.html"/>
 
   <section class="main-content">
-    <%include file="widgets/week-edit.html"/>
-    <%include file="widgets/week-new.html"/>
-    <%include file="widgets/sequnce-edit.html"/>
-    <%include file="widgets/video-edit.html"/>
-    <%include file="widgets/video-new.html"/>
-    <%include file="widgets/problem-edit.html"/>
-    <%include file="widgets/problem-new.html"/>
+    <section class="edit-pane">
+      <div id="module-html"/>
+    </section>
   </section>
 
 </section>

cms/templates/unit.html

@@ -0,0 +1,17 @@
+<section id="unit-wrapper" class="${js_module}">
+  <header>
+    <section>
+      <h1 class="editable">${name}</h1>
+      <p>${category}</p>
+    </section>
+
+    <div class="actions">
+      <a href="#" class="cancel">Cancel</a>
+      <a href="" class="save-update">Save &amp; Update</a>
+    </div>
+  </header>
+
+  <section>
+    ${contents}
+  </section>
+</section>

cms/templates/widgets/header.html

@@ -5,9 +5,6 @@
       <li>
         <a href="#" class="new-module">New Section</a>
       </li>
-      <li>
-        <a href="#" class="new-module">New Module</a>
-      </li>
       <li>
         <a href="#" class="new-module">New Unit</a>
       </li>

cms/templates/widgets/html-edit.html

@@ -0,0 +1,45 @@
+<section class="html-edit">
+  <section class="meta wip">
+
+    <section class="status-settings">
+      <ul>
+        <li><a href="#" class="current">Scrap</a></li>
+        <li><a href="#">Draft</a></li>
+        <li><a href="#">Proofed</a></li>
+        <li><a href="#">Published</a></li>
+      </ul>
+      <a href="#" class="settings">Settings</a>
+    </section>
+
+    <section class="author">
+      <dl>
+        <dt>Last modified:</dt>
+        <dd>mm/dd/yy</dd>
+        <dt>By</dt>
+        <dd>Anant Agarwal</dd>
+      </dl>
+    </section>
+
+    <section class="tags">
+      <div>
+        <h2>Tags:</h2>
+        <p class="editable">Click to edit</p>
+      </div>
+
+      <div>
+        <h2>Goal</h2>
+        <p class="editable">Click to edit</p>
+      </div>
+    </section>
+  </section>
+
+  <textarea name="" class="edit-box" rows="8" cols="40">${data}</textarea>
+  <div class="preview">${data}</div>
+
+  <div class="actions wip">
+    <a href="" class="save-update">Save &amp; Update</a>
+    <a href="#" class="cancel">Cancel</a>
+  </div>
+
+  <%include file="notes.html"/>
+</section>

cms/templates/widgets/module-dropdown.html

@@ -1,4 +1,4 @@
-<li class="create-module">
+<li class="create-module wip">
   <a href="#" class="new-module">
     + Add new module
   </a>

cms/templates/widgets/navigation.html

@@ -1,5 +1,5 @@
 <section class="cal">
-  <header>
+  <header class="wip">
     <h2>Filter content:</h2>
     <ul>
       <li>
@@ -38,10 +38,10 @@
     % for week in weeks:
     <li>
       <header>
-        <h1><a href="#">${week.name}</a></h1>
+        <h1><a href="#" class="module-edit" id="${week.location.url()}">${week.name}</a></h1>
         <ul>
-          % if week.goals:
-            % for goal in week.goals:
+          % if 'goals' in week.metadata:
+            % for goal in week.metadata['goals']:
             <li class="goal editable">${goal}</li>
             % endfor
           % else:
@@ -52,8 +52,8 @@
 
       <ul>
         % for module in week.get_children():
-        <li class="${module.type}">
-            <a href="#" class="${module.type}-edit">${module.name}</a>
+        <li class="${module.category}">
+            <a href="#" class="module-edit" id="${module.location.url()}">${module.name}</a>
             <a href="#" class="draggable">handle</a>
         </li>
         % endfor

cms/templates/widgets/notes.html

@@ -0,0 +1,21 @@
+<section class="notes wip">
+  <h2>Notes</h2>
+
+  <ul>
+    <li>
+      <p>Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt.</p>
+      <p class="author">Anant Agarwal</p>
+    </li>
+    <li>
+      <p>Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt.</p>
+      <p class="author">Anant Agarwal</p>
+    </li>
+  </ul>
+
+  <form>
+    <h2>Add a note</h2>
+    <textarea name="" id= rows="8" cols="40"></textarea>
+    <input type="submit" name="" id="" value="Post" />
+  </form>
+</section>
+

cms/templates/widgets/problem-edit.html

@@ -1,73 +1,48 @@
 <section class="problem-edit">
-  <header>
-    <a href="#" class="cancel">Cancel</a>
-    <a href="#" class="save-update">Save &amp; Update</a>
-  </header>
-
-  <section>
-    <header>
-      <h1 class="editable">New Problem</h1>
-      <section class="author">
-        <div>
-          <h2>Last modified:</h2>
-          <p>mm/dd/yy</p>
-        </div>
-
-        <div>
-          <h2>By</h2>
-          <p>Anant Agarwal</p>
-        </div>
-      </section>
-
-      <section class="status-settings">
-        <ul>
-          <li><a href="#" class="current">Scrap</a></li>
-          <li><a href="#">Draft</a></li>
-          <li><a href="#">Proofed</a></li>
-          <li><a href="#">Published</a></li>
-        </ul>
-        <a href="#" class="settings">Settings</a>
-
-        <select name="" id="">
-          <option>Global</option>
-        </select>
-      </section>
-      <section class="meta">
-        <div>
-          <h2>Tags:</h2>
-          <p class="editable">Click to edit</p>
-        </div>
-
-        <div>
-          <h2>Goal</h2>
-          <p class="editable">Click to edit</p>
-        </div>
-      </section>
-    </header>
-
-    <section>
-      <textarea name="" id= rows="8" cols="40">Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt.</textarea>
-      <div class="preview">
-        Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt.
-      </div>
+  <section class="meta">
+    <section class="status-settings">
+      <ul>
+        <li><a href="#" class="current">Scrap</a></li>
+        <li><a href="#">Draft</a></li>
+        <li><a href="#">Proofed</a></li>
+        <li><a href="#">Published</a></li>
+      </ul>
+      <a href="#" class="settings">Settings</a>
     </section>
 
-    <section class="notes">
-      <h2>Add notes</h2>
-      <textarea name="" id= rows="8" cols="40"></textarea>
-      <input type="submit" name="" id="" value="post" />
+    <section class="author">
+      <dl>
+        <dt>Last modified:</dt>
+        <dd>mm/dd/yy</dd>
+        <dt>By</dt>
+        <dd>Anant Agarwal</dd>
+      </dl>
+    </section>
 
-      <ul>
-        <li>
-          <p>Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt.</p>
-          <p class="author">Anant Agarwal</p>
-        </li>
-        <li>
-          <p>Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt.</p>
-          <p class="author">Anant Agarwal</p>
-        </li>
-      </ul>    </section>
+    <section class="tags">
+      <div>
+        <h2>Tags:</h2>
+        <p class="editable">Click to edit</p>
+      </div>
 
-    <a href="" class="save-update">Save &amp; Update</a>
+      <div>
+        <h2>Goal</h2>
+        <p class="editable">Click to edit</p>
+      </div>
+    </section>
   </section>
+
+  <section>
+    <textarea name="" id= rows="8" cols="40">Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt.</textarea>
+    <div class="preview">
+      Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt.
+    </div>
+
+    <div class="actions">
+      <a href="#" class="cancel">Cancel</a>
+      <a href="" class="save-update">Save &amp; Update</a>
+    </div>
+  </section>
+
+  <%include file="notes.html"/>
 </section>

cms/templates/widgets/raw-edit.html

@@ -0,0 +1,45 @@
+<section class="raw-edit">
+  <section class="meta wip">
+
+    <section class="status-settings">
+      <ul>
+        <li><a href="#" class="current">Scrap</a></li>
+        <li><a href="#">Draft</a></li>
+        <li><a href="#">Proofed</a></li>
+        <li><a href="#">Published</a></li>
+      </ul>
+      <a href="#" class="settings">Settings</a>
+    </section>
+
+    <section class="author">
+      <dl>
+        <dt>Last modified:</dt>
+        <dd>mm/dd/yy</dd>
+        <dt>By</dt>
+        <dd>Anant Agarwal</dd>
+      </dl>
+    </section>
+
+    <section class="tags">
+      <div>
+        <h2>Tags:</h2>
+        <p class="editable">Click to edit</p>
+      </div>
+
+      <div>
+        <h2>Goal</h2>
+        <p class="editable">Click to edit</p>
+      </div>
+    </section>
+  </section>
+
+  <textarea name="" class="edit-box" rows="8" cols="40">${data}</textarea>
+  <pre class="preview">${data | h}</pre>
+
+  <div class="actions wip">
+    <a href="" class="save-update">Save &amp; Update</a>
+    <a href="#" class="cancel">Cancel</a>
+  </div>
+
+  <%include file="notes.html"/>
+</section>

cms/templates/widgets/sequence-edit.html

@@ -0,0 +1,106 @@
+<section class="sequence-edit">
+  <section class="filters">
+    <ul>
+      <li>
+        <label for="">Sort by</label>
+        <select>
+          <option value="">Recently Modified</option>
+        </select>
+      </li>
+
+      <li>
+        <label for="">Display</label>
+        <select>
+          <option value="">All content</option>
+        </select>
+      </li>
+      <li>
+        <select>
+          <option value="">Internal Only</option>
+        </select>
+      </li>
+
+      <li class="advanced">
+        <a href="#">Advanced filters</a>
+      </li>
+
+      <li>
+        <input type="search" name="" id="" value="" />
+      </li>
+    </ul>
+  </section>
+
+  <div class="content">
+    <section class="modules">
+      <ol>
+        <li>
+          <ol>
+            % for child in module.get_children():
+            <li>
+              <a href="#" class="module-edit" id="${child.location.url()}">${child.name}</a>
+              <a href="#" class="draggable">handle</a>
+            </li>
+            %endfor
+          </ol>
+        </li>
+
+      </ol>
+    </section>
+
+    <section class="scratch-pad">
+      <ol>
+        <li class="new-module">
+          <%include file="new-module.html"/>
+        </li>
+        <li>
+          <header>
+            <h2>Section Scratch</h2>
+          </header>
+          <ul>
+            <li>
+              <a href="#" class="problem-edit">Problem title 11</a>
+              <a href="#" class="draggable">handle</a>
+            </li>
+            <li>
+              <a href="#" class="problem-edit">Problem title 13 </a>
+              <a href="#" class="draggable">handle</a>
+            </li>
+            <li>
+              <a href="#" class="problem-edit"> Problem title 14</a>
+              <a href="#" class="draggable">handle</a>
+            </li>
+            <li>
+              <a href="" class="video-edit">Video 3</a>
+              <a href="#" class="draggable">handle</a>
+            </li>
+          </ul>
+        </li>
+        <li>
+          <header>
+            <h2>Course Scratch</h2>
+          </header>
+
+          <ul>
+            <li>
+              <a href="#" class="problem-edit">Problem title 11</a>
+              <a href="#" class="draggable">handle</a>
+            </li>
+            <li>
+              <a href="#" class="problem-edit">Problem title 13 </a>
+              <a href="#" class="draggable">handle</a>
+            </li>
+            <li>
+              <a href="#" class="problem-edit"> Problem title 14</a>
+              <a href="#" class="draggable">handle</a>
+            </li>
+            <li>
+              <a href="" class="video-edit">Video 3</a>
+              <a href="#" class="draggable">handle</a>
+            </li>
+          </ul>
+        </li>
+      </ol>
+    </section>
+  </div>
+</section>
+

cms/urls.py

@@ -6,4 +6,7 @@ from django.conf.urls.defaults import patterns, url
 
 urlpatterns = patterns('',
     url(r'^$', 'contentstore.views.index', name='index'),
+    url(r'^edit_item$', 'contentstore.views.edit_item', name='edit_item'),
+    url(r'^save_item$', 'contentstore.views.save_item', name='save_item'),
+    url(r'^temp_force_export$', 'contentstore.views.temp_force_export')
 )

common/djangoapps/util/views.py

@@ -57,23 +57,9 @@ def send_feedback(request):
               )
     return HttpResponse(json.dumps({'success':True}))
 
-def info(request, course_id=None):
+def info(request):
     ''' Info page (link from main header) '''
-    try:
-        course = settings.COURSES_BY_ID[course_id]
-    except KeyError:
-        raise Http404("Course not found")
-
-    # We're bypassing the templating system for this part. We should cache
-    # this.
-    sections = ["updates", "handouts", "guest_updates", "guest_handouts"]
-    sections_to_content = { 'course': course }
-    for section in sections:
-        filename = section + ".html"
-        with open(course.path / "info" / filename) as f:
-            sections_to_content[section] = f.read()
-
-    return render_to_response("info.html", sections_to_content)
+    return render_to_response("info.html", {})
 
 # From http://djangosnippets.org/snippets/1042/
 def parse_accept_header(accept):

common/lib/capa/capa_problem.py

@@ -68,14 +68,13 @@ class LoncapaProblem(object):
     Main class for capa Problems.
     '''
 
-    def __init__(self, fileobject, id, state=None, seed=None, system=None):
+    def __init__(self, problem_text, id, state=None, seed=None, system=None):
         '''
-        Initializes capa Problem.  The problem itself is defined by the XML file
-        pointed to by fileobject.
+        Initializes capa Problem.
 
         Arguments:
 
-         - filesobject  : an OSFS instance: see fs.osfs
+         - problem_text : xml defining the problem
          - id           : string used as the identifier for this problem; often a filename (no spaces)
          - state        : student state (represented as a dict)
          - seed         : random number generator seed (int)
@@ -103,14 +102,11 @@ class LoncapaProblem(object):
         if not self.seed:
             self.seed = struct.unpack('i', os.urandom(4))[0]
 
-        self.fileobject = fileobject    	# save problem file object, so we can use for debugging information later
-        if getattr(system, 'DEBUG', False):	# get the problem XML string from the problem file
-            log.info("[courseware.capa.capa_problem.lcp.init]  fileobject = %s" % fileobject)
-        file_text = fileobject.read()
-        file_text = re.sub("startouttext\s*/", "text", file_text)   # Convert startouttext and endouttext to proper <text></text>
-        file_text = re.sub("endouttext\s*/", "/text", file_text)
+        problem_text = re.sub("startouttext\s*/", "text", problem_text)   # Convert startouttext and endouttext to proper <text></text>
+        problem_text = re.sub("endouttext\s*/", "/text", problem_text)
+        self.problem_text = problem_text
 
-        self.tree = etree.XML(file_text)	# parse problem XML file into an element tree
+        self.tree = etree.XML(problem_text)  # parse problem XML file into an element tree
         self._process_includes()		# handle any <include file="foo"> tags
 
         # construct script processor context (eg for customresponse problems)
@@ -130,7 +126,7 @@ class LoncapaProblem(object):
         self.done = False
 
     def __unicode__(self):
-        return u"LoncapaProblem ({0})".format(self.fileobject)
+        return u"LoncapaProblem ({0})".format(self.problem_text)
 
     def get_state(self):
         ''' Stored per-user session data neeeded to:
@@ -272,7 +268,7 @@ class LoncapaProblem(object):
                 parent = inc.getparent()			# insert  new XML into tree in place of inlcude
                 parent.insert(parent.index(inc),incxml)
                 parent.remove(inc)
-                log.debug('Included %s into %s' % (file,self.fileobject))
+                log.debug('Included %s into %s' % (file, self.id))
 
     def _extract_context(self, tree, seed=struct.unpack('i', os.urandom(4))[0]):  # private
         '''

common/lib/capa/templates/textinput.html

@@ -1,22 +1,36 @@
 <section id="textinput_${id}" class="textinput">
+    % if state == 'unsubmitted':
+    <div class="unanswered" id="status_${id}">
+    % elif state == 'correct':
+    <div class="correct" id="status_${id}">
+    % elif state == 'incorrect':
+    <div class="incorrect" id="status_${id}">
+    % elif state == 'incomplete':
+    <div class="incorrect" id="status_${id}">
+    % endif
+
   <input type="text" name="input_${id}" id="input_${id}" value="${value}"
    % if size:
    size="${size}"
    % endif
    />
 
-  <span id="answer_${id}"></span>
+      <p class="status">
+        % if state == 'unsubmitted':
+        unanswered
+        % elif state == 'correct':
+        correct
+        % elif state == 'incorrect':
+        incorrect
+        % elif state == 'incomplete':
+        incomplete
+        % endif
+      </p>
+
+  <p id="answer_${id}" class="answer"></p>
 
-  % if state == 'unsubmitted':
-  <span class="unanswered" style="display:inline-block;" id="status_${id}"></span>
-  % elif state == 'correct':
-  <span class="correct" id="status_${id}"></span>
-  % elif state == 'incorrect':
-  <span class="incorrect" id="status_${id}"></span>
-  % elif state == 'incomplete':
-  <span class="incorrect" id="status_${id}"></span>
-  % endif
   % if msg:
       <span class="message">${msg|n}</span>
   % endif
+</div>
 </section>

common/lib/capa/templates/textinput_dynamath.html

@@ -2,35 +2,37 @@
 ### version of textline.html which does dynammic math
 ###
 <section class="text-input-dynamath">
-  <table style="display:inline; vertical-align:middle;">
-    <tr>
-      <td>
-        <input type="text" name="input_${id}" id="input_${id}" value="${value}" class="math" size="${size if size else ''}" />
-      </td>
-      <td>
-        <span id="answer_${id}"></span>
+    % if state == 'unsubmitted':
+    <div class="unanswered" id="status_${id}">
+    % elif state == 'correct':
+    <div class="correct" id="status_${id}">
+    % elif state == 'incorrect':
+    <div class="incorrect" id="status_${id}">
+    % elif state == 'incomplete':
+    <div class="incorrect" id="status_${id}">
+    % endif
 
+      <input type="text" name="input_${id}" id="input_${id}" value="${value}" class="math" size="${size if size else ''}" />
+
+      <p class="status">
         % if state == 'unsubmitted':
-        <span class="unanswered" style="display:inline-block;" id="status_${id}"></span>
+        unanswered
         % elif state == 'correct':
-        <span class="correct" id="status_${id}"></span>
+        correct
         % elif state == 'incorrect':
-        <span class="incorrect" id="status_${id}"></span>
+        incorrect
         % elif state == 'incomplete':
-        <span class="incorrect" id="status_${id}"></span>
+        incomplete
         % endif
-      </td>
-    </tr>
-    <tr>
-      <td>
-        <span id="display_${id}">`{::}`</span>
-      </td>
-      <td>
-        <textarea style="display:none" id="input_${id}_dynamath" name="input_${id}_dynamath">  </textarea>
-      </td>
-    </tr>
-  </table>
-  % if msg:
-      <span class="message">${msg|n}</span>
-  % endif
+      </p>
+
+      <p id="answer_${id}" class="answer"></p>
+
+      <div id="display_${id}" class="equation">`{::}`</div>
+
+    </div>
+    <textarea style="display:none" id="input_${id}_dynamath" name="input_${id}_dynamath">  </textarea>
+    % if msg:
+    <span class="message">${msg|n}</span>
+    % endif
 </section>

common/lib/mitxmako/shortcuts.py

@@ -12,13 +12,16 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
+import logging
+
+log = logging.getLogger("mitx." + __name__)
+
 from django.template import Context
 from django.http import HttpResponse
 
 from . import middleware
 from django.conf import settings
 
-
 def render_to_string(template_name, dictionary, context=None, namespace='main'):
     context_instance = Context(dictionary)
     # add dictionary to context_instance
@@ -27,8 +30,11 @@ def render_to_string(template_name, dictionary, context=None, namespace='main'):
     context_dictionary = {}
     context_instance['settings'] = settings
     context_instance['MITX_ROOT_URL'] = settings.MITX_ROOT_URL
-    for d in middleware.requestcontext:
-        context_dictionary.update(d)
+
+    # In various testing contexts, there might not be a current request context.
+    if middleware.requestcontext is not None:
+        for d in middleware.requestcontext:
+            context_dictionary.update(d)
     for d in context_instance:
         context_dictionary.update(d)
     if context:

common/lib/xmodule/__init__.py

@@ -1,62 +0,0 @@
-import capa_module
-import html_module
-import schematic_module
-import seq_module
-import template_module
-import vertical_module
-import video_module
-
-# Import all files in modules directory, excluding backups (# and . in name) 
-# and __init__
-#
-# Stick them in a list
-# modx_module_list = []
-
-# for f in os.listdir(os.path.dirname(__file__)):
-#     if f!='__init__.py' and \
-#             f[-3:] == ".py" and \
-#             "." not in f[:-3] \
-#             and '#' not in f:
-#         mod_path = 'courseware.modules.'+f[:-3]
-#         mod = __import__(mod_path, fromlist = "courseware.modules")
-#         if 'Module' in mod.__dict__:
-#             modx_module_list.append(mod)
-
-#print modx_module_list
-modx_module_list = [capa_module,  html_module,  schematic_module,  seq_module,  template_module,  vertical_module,  video_module]
-#print modx_module_list
-
-modx_modules = {}
-
-# Convert list to a dictionary for lookup by tag
-def update_modules():
-    global modx_modules
-    modx_modules = dict()
-    for module in modx_module_list:
-        for tag in module.Module.get_xml_tags():
-            modx_modules[tag] = module.Module
-
-update_modules()
-
-def get_module_class(tag):
-    ''' Given an XML tag (e.g. 'video'), return 
-    the associated module (e.g. video_module.Module). 
-    '''
-    if tag not in modx_modules:
-        update_modules()
-    return modx_modules[tag]
-
-def get_module_id(tag):
-    ''' Given an XML tag (e.g. 'video'), return 
-    the default ID for that module (e.g. 'youtube_id')
-    '''
-    return modx_modules[tag].id_attribute
-
-def get_valid_tags():
-    return modx_modules.keys()
-
-def get_default_ids():
-    tags = get_valid_tags()
-    ids = map(get_module_id, tags)
-    return dict(zip(tags, ids))
-

common/lib/xmodule/html_module.py

@@ -1,43 +0,0 @@
-import json
-import logging
-
-from x_module import XModule, XModuleDescriptor
-from lxml import etree
-
-log = logging.getLogger("mitx.courseware")
-
-#-----------------------------------------------------------------------------
-class ModuleDescriptor(XModuleDescriptor):
-    pass
-
-class Module(XModule):
-    id_attribute = 'filename'
-
-    def get_state(self):
-        return json.dumps({ })
-
-    @classmethod
-    def get_xml_tags(c):
-        return ["html"]
-        
-    def get_html(self):
-        if self.filename==None:
-            xmltree=etree.fromstring(self.xml)
-            textlist=[xmltree.text]+[etree.tostring(i) for i in xmltree]+[xmltree.tail]
-            textlist=[i for i in textlist if type(i)==str]
-            return "".join(textlist)
-        try: 
-            filename="html/"+self.filename
-            return self.filestore.open(filename).read()
-        except: # For backwards compatibility. TODO: Remove
-            if self.DEBUG:
-                log.info('[courseware.modules.html_module] filename=%s' % self.filename)
-            return self.system.render_template(self.filename, {'id': self.item_id}, namespace='course')
-
-    def __init__(self, system, xml, item_id, state=None):
-        XModule.__init__(self, system, xml, item_id, state)
-        xmltree=etree.fromstring(xml)
-        self.filename = None
-        filename_l=xmltree.xpath("/html/@filename")
-        if len(filename_l)>0:
-            self.filename=str(filename_l[0])

common/lib/xmodule/seq_module.py

@@ -1,121 +0,0 @@
-import json
-import logging
-
-from lxml import etree
-
-from x_module import XModule, XModuleDescriptor
-from xmodule.progress import Progress
-
-log = logging.getLogger("mitx.common.lib.seq_module")
-
-# HACK: This shouldn't be hard-coded to two types
-# OBSOLETE: This obsoletes 'type'
-class_priority = ['video', 'problem']
-
-class ModuleDescriptor(XModuleDescriptor):
-    pass
-
-class Module(XModule):
-    ''' Layout module which lays out content in a temporal sequence
-    '''
-    id_attribute = 'id'
-
-    def get_state(self):
-        return json.dumps({ 'position':self.position })
-
-    @classmethod
-    def get_xml_tags(c):
-        obsolete_tags = ["sequential", 'tab']
-        modern_tags = ["videosequence"]
-        return obsolete_tags + modern_tags
-        
-    def get_html(self):
-        self.render()
-        return self.content
-
-    def get_init_js(self):
-        self.render()
-        return self.init_js
-
-    def get_destroy_js(self):
-        self.render()
-        return self.destroy_js
-
-    def get_progress(self):
-        ''' Return the total progress, adding total done and total available.
-        (assumes that each submodule uses the same "units" for progress.)
-        '''
-        # TODO: Cache progress or children array?
-        children = self.get_children()
-        progresses = [child.get_progress() for child in children]
-        progress = reduce(Progress.add_counts, progresses)
-        return progress
-
-    def handle_ajax(self, dispatch, get):		# TODO: bounds checking
-        ''' get = request.POST instance '''
-        if dispatch=='goto_position':
-            self.position = int(get['position'])
-            return json.dumps({'success':True})
-        raise self.system.exception404
-
-    def render(self):
-        if self.rendered:
-            return
-        ## Returns a set of all types of all sub-children
-        child_classes = [set([i.tag for i in e.iter()]) for e in self.xmltree]
-
-        titles = ["\n".join([i.get("name").strip() for i in e.iter() if i.get("name") is not None]) \
-                       for e in self.xmltree]
-
-        children = self.get_children()
-        progresses = [child.get_progress() for child in children]
-        
-        self.contents = self.rendered_children()
-
-        for contents, title, progress in zip(self.contents, titles, progresses):
-            contents['title'] = title
-            contents['progress_status'] = Progress.to_js_status_str(progress)
-            contents['progress_detail'] = Progress.to_js_detail_str(progress)
-
-        for (content, element_class) in zip(self.contents, child_classes):
-            new_class = 'other'
-            for c in class_priority:
-                if c in element_class:
-                    new_class = c
-            content['type'] = new_class
-
-        # Split </script> tags -- browsers handle this as end
-        # of script, even if it occurs mid-string. Do this after json.dumps()ing
-        # so that we can be sure of the quotations being used
-        params={'items': json.dumps(self.contents).replace('</script>', '<"+"/script>'),
-                'id': self.item_id,
-                'position': self.position,
-                'titles': titles,
-                'tag': self.xmltree.tag}
-
-        if self.xmltree.tag in ['sequential', 'videosequence']:
-            self.content = self.system.render_template('seq_module.html', params)
-        if self.xmltree.tag == 'tab':
-            self.content = self.system.render_template('tab_module.html', params)
-        log.debug("rendered content: %s", content)
-        self.rendered = True
-
-    def __init__(self, system, xml, item_id, state=None):
-        XModule.__init__(self, system, xml, item_id, state)
-        self.xmltree = etree.fromstring(xml)
-
-        self.position = 1
-
-        if state is not None:
-            state = json.loads(state)
-            if 'position' in state: self.position = int(state['position'])
-
-        # if position is specified in system, then use that instead
-        if system.get('position'):
-            self.position = int(system.get('position'))
-
-        self.rendered = False
-
-
-class SectionDescriptor(XModuleDescriptor):
-    pass

common/lib/xmodule/setup.py

@@ -3,22 +3,33 @@ from setuptools import setup, find_packages
 setup(
     name="XModule",
     version="0.1",
-    packages=find_packages(),
+    packages=find_packages(exclude=["tests"]),
     install_requires=['distribute'],
+    package_data={
+        'xmodule': ['js/module/*']
+    },
 
     # See http://guide.python-distribute.org/creation.html#entry-points
     # for a description of entry_points
     entry_points={
         'xmodule.v1': [
-            "Course = seq_module:SectionDescriptor",
-            "Week = seq_module:SectionDescriptor",
-            "Section = seq_module:SectionDescriptor",
-            "LectureSequence = seq_module:SectionDescriptor",
-            "Lab = seq_module:SectionDescriptor",
-            "Homework = seq_module:SectionDescriptor",
-            "TutorialIndex = seq_module:SectionDescriptor",
-            "Exam = seq_module:SectionDescriptor",
-            "VideoSegment = video_module:VideoSegmentDescriptor",
+            "abtest = xmodule.abtest_module:ABTestDescriptor",
+            "book = xmodule.translation_module:TranslateCustomTagDescriptor",
+            "chapter = xmodule.seq_module:SequenceDescriptor",
+            "course = xmodule.seq_module:SequenceDescriptor",
+            "customtag = xmodule.template_module:CustomTagDescriptor",
+            "discuss = xmodule.translation_module:TranslateCustomTagDescriptor",
+            "html = xmodule.html_module:HtmlDescriptor",
+            "image = xmodule.translation_module:TranslateCustomTagDescriptor",
+            "problem = xmodule.capa_module:CapaDescriptor",
+            "problemset = xmodule.vertical_module:VerticalDescriptor",
+            "section = xmodule.translation_module:SemanticSectionDescriptor",
+            "sequential = xmodule.seq_module:SequenceDescriptor",
+            "slides = xmodule.translation_module:TranslateCustomTagDescriptor",
+            "vertical = xmodule.vertical_module:VerticalDescriptor",
+            "video = xmodule.video_module:VideoDescriptor",
+            "videodev = xmodule.translation_module:TranslateCustomTagDescriptor",
+            "videosequence = xmodule.seq_module:SequenceDescriptor",
         ]
     }
 )

common/lib/xmodule/tests/__init__.py

@@ -43,12 +43,10 @@ class ModelsTest(unittest.TestCase):
     def setUp(self):
         pass
 
-    def test_get_module_class(self):
-        vc = xmodule.get_module_class('video')
-        vc_str = "<class 'xmodule.video_module.Module'>"
+    def test_load_class(self):
+        vc = xmodule.x_module.XModuleDescriptor.load_class('video')
+        vc_str = "<class 'xmodule.video_module.VideoDescriptor'>"
         self.assertEqual(str(vc), vc_str)
-        video_id = xmodule.get_default_ids()['video']
-        self.assertEqual(video_id, 'youtube')
 
     def test_calc(self):
         variables={'R1':2.0, 'R3':4.0}
@@ -98,7 +96,7 @@ class ModelsTest(unittest.TestCase):
 class MultiChoiceTest(unittest.TestCase):
     def test_MC_grade(self):
         multichoice_file = os.path.dirname(__file__)+"/test_files/multichoice.xml"
-        test_lcp = lcp.LoncapaProblem(open(multichoice_file), '1', system=i4xs)
+        test_lcp = lcp.LoncapaProblem(open(multichoice_file).read(), '1', system=i4xs)
         correct_answers = {'1_2_1':'choice_foil3'}
         self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct')
         false_answers = {'1_2_1':'choice_foil2'}
@@ -106,7 +104,7 @@ class MultiChoiceTest(unittest.TestCase):
 
     def test_MC_bare_grades(self):
         multichoice_file = os.path.dirname(__file__)+"/test_files/multi_bare.xml"
-        test_lcp = lcp.LoncapaProblem(open(multichoice_file), '1', system=i4xs)
+        test_lcp = lcp.LoncapaProblem(open(multichoice_file).read(), '1', system=i4xs)
         correct_answers = {'1_2_1':'choice_2'}
         self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct')
         false_answers = {'1_2_1':'choice_1'}
@@ -114,7 +112,7 @@ class MultiChoiceTest(unittest.TestCase):
         
     def test_TF_grade(self):
         truefalse_file =  os.path.dirname(__file__)+"/test_files/truefalse.xml"
-        test_lcp = lcp.LoncapaProblem(open(truefalse_file), '1', system=i4xs)
+        test_lcp = lcp.LoncapaProblem(open(truefalse_file).read(), '1', system=i4xs)
         correct_answers = {'1_2_1':['choice_foil2', 'choice_foil1']}
         self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct')
         false_answers = {'1_2_1':['choice_foil1']}
@@ -129,7 +127,7 @@ class MultiChoiceTest(unittest.TestCase):
 class ImageResponseTest(unittest.TestCase):
     def test_ir_grade(self):
         imageresponse_file = os.path.dirname(__file__)+"/test_files/imageresponse.xml"
-        test_lcp = lcp.LoncapaProblem(open(imageresponse_file), '1', system=i4xs)
+        test_lcp = lcp.LoncapaProblem(open(imageresponse_file).read(), '1', system=i4xs)
         correct_answers = {'1_2_1':'(490,11)-(556,98)',
                            '1_2_2':'(242,202)-(296,276)'}
         test_answers = {'1_2_1':'[500,20]',
@@ -142,7 +140,7 @@ class SymbolicResponseTest(unittest.TestCase):
     def test_sr_grade(self):
         raise SkipTest() # This test fails due to dependencies on a local copy of snuggletex-webapp. Until we have figured that out, we'll just skip this test
         symbolicresponse_file = os.path.dirname(__file__)+"/test_files/symbolicresponse.xml"
-        test_lcp = lcp.LoncapaProblem(open(symbolicresponse_file), '1', system=i4xs)
+        test_lcp = lcp.LoncapaProblem(open(symbolicresponse_file).read(), '1', system=i4xs)
         correct_answers = {'1_2_1':'cos(theta)*[[1,0],[0,1]] + i*sin(theta)*[[0,1],[1,0]]',
                            '1_2_1_dynamath': '''
 <math xmlns="http://www.w3.org/1998/Math/MathML">
@@ -235,7 +233,7 @@ class OptionResponseTest(unittest.TestCase):
     '''
     def test_or_grade(self):
         optionresponse_file = os.path.dirname(__file__)+"/test_files/optionresponse.xml"
-        test_lcp = lcp.LoncapaProblem(open(optionresponse_file), '1', system=i4xs)
+        test_lcp = lcp.LoncapaProblem(open(optionresponse_file).read(), '1', system=i4xs)
         correct_answers = {'1_2_1':'True',
                            '1_2_2':'False'}
         test_answers = {'1_2_1':'True',
@@ -251,7 +249,7 @@ class FormulaResponseWithHintTest(unittest.TestCase):
     '''
     def test_or_grade(self):
         problem_file = os.path.dirname(__file__)+"/test_files/formularesponse_with_hint.xml"
-        test_lcp = lcp.LoncapaProblem(open(problem_file), '1', system=i4xs)
+        test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=i4xs)
         correct_answers = {'1_2_1':'2.5*x-5.0'}
         test_answers = {'1_2_1':'0.4*x-5.0'}
         self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct')
@@ -265,7 +263,7 @@ class StringResponseWithHintTest(unittest.TestCase):
     '''
     def test_or_grade(self):
         problem_file = os.path.dirname(__file__)+"/test_files/stringresponse_with_hint.xml"
-        test_lcp = lcp.LoncapaProblem(open(problem_file), '1', system=i4xs)
+        test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=i4xs)
         correct_answers = {'1_2_1':'Michigan'}
         test_answers = {'1_2_1':'Minnesota'}
         self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct')
@@ -413,212 +411,211 @@ class GraderTest(unittest.TestCase):
         self.assertAlmostEqual( graded['percent'], 0.7688095238095238 )
         self.assertEqual( len(graded['section_breakdown']), (12 + 1) + (7+1) + 1 )
         self.assertEqual( len(graded['grade_breakdown']), 3 )
-        
-        graded = zeroWeightsGrader.grade(self.test_gradesheet)
-        self.assertAlmostEqual( graded['percent'], 0.2525 )
-        self.assertEqual( len(graded['section_breakdown']), (12 + 1) + (7+1) + 1 )
-        self.assertEqual( len(graded['grade_breakdown']), 3 )
-        
-        
-        graded = allZeroWeightsGrader.grade(self.test_gradesheet)
+    
+    graded = zeroWeightsGrader.grade(self.test_gradesheet)
+    self.assertAlmostEqual( graded['percent'], 0.2525 )
+    self.assertEqual( len(graded['section_breakdown']), (12 + 1) + (7+1) + 1 )
+    self.assertEqual( len(graded['grade_breakdown']), 3 )
+    
+    
+    graded = allZeroWeightsGrader.grade(self.test_gradesheet)
+    self.assertAlmostEqual( graded['percent'], 0.0 )
+    self.assertEqual( len(graded['section_breakdown']), (12 + 1) + (7+1) + 1 )
+    self.assertEqual( len(graded['grade_breakdown']), 3 )
+    
+    for graded in [ weightedGrader.grade(self.empty_gradesheet), 
+                    weightedGrader.grade(self.incomplete_gradesheet),
+                    zeroWeightsGrader.grade(self.empty_gradesheet),
+                    allZeroWeightsGrader.grade(self.empty_gradesheet)]:
         self.assertAlmostEqual( graded['percent'], 0.0 )
         self.assertEqual( len(graded['section_breakdown']), (12 + 1) + (7+1) + 1 )
         self.assertEqual( len(graded['grade_breakdown']), 3 )
         
-        for graded in [ weightedGrader.grade(self.empty_gradesheet), 
-                        weightedGrader.grade(self.incomplete_gradesheet),
-                        zeroWeightsGrader.grade(self.empty_gradesheet),
-                        allZeroWeightsGrader.grade(self.empty_gradesheet)]:
-            self.assertAlmostEqual( graded['percent'], 0.0 )
-            self.assertEqual( len(graded['section_breakdown']), (12 + 1) + (7+1) + 1 )
-            self.assertEqual( len(graded['grade_breakdown']), 3 )
-            
-            
-        graded = emptyGrader.grade(self.test_gradesheet)
-        self.assertAlmostEqual( graded['percent'], 0.0 )
-        self.assertEqual( len(graded['section_breakdown']), 0 )
-        self.assertEqual( len(graded['grade_breakdown']), 0 )
         
+    graded = emptyGrader.grade(self.test_gradesheet)
+    self.assertAlmostEqual( graded['percent'], 0.0 )
+    self.assertEqual( len(graded['section_breakdown']), 0 )
+    self.assertEqual( len(graded['grade_breakdown']), 0 )
     
 
-    def test_graderFromConf(self):
-        
-        #Confs always produce a graders.WeightedSubsectionsGrader, so we test this by repeating the test
-        #in test_graders.WeightedSubsectionsGrader, but generate the graders with confs.
-        
-        weightedGrader = graders.grader_from_conf([
-            {
-                'type' : "Homework",
-                'min_count' : 12,
-                'drop_count' : 2,
-                'short_label' : "HW",
-                'weight' : 0.25,
-            },
-            {
-                'type' : "Lab",
-                'min_count' : 7,
-                'drop_count' : 3,
-                'category' : "Labs",
-                'weight' : 0.25
-            },
-            {
-                'type' : "Midterm",
-                'name' : "Midterm Exam",
-                'short_label' : "Midterm",
-                'weight' : 0.5,
-            },
-        ])
-        
-        emptyGrader = graders.grader_from_conf([])
-        
-        graded = weightedGrader.grade(self.test_gradesheet)
-        self.assertAlmostEqual( graded['percent'], 0.5106547619047619 )
-        self.assertEqual( len(graded['section_breakdown']), (12 + 1) + (7+1) + 1 )
-        self.assertEqual( len(graded['grade_breakdown']), 3 )
-        
-        graded = emptyGrader.grade(self.test_gradesheet)
-        self.assertAlmostEqual( graded['percent'], 0.0 )
-        self.assertEqual( len(graded['section_breakdown']), 0 )
-        self.assertEqual( len(graded['grade_breakdown']), 0 )
-        
-        #Test that graders can also be used instead of lists of dictionaries
-        homeworkGrader = graders.AssignmentFormatGrader("Homework", 12, 2)
-        homeworkGrader2 = graders.grader_from_conf(homeworkGrader)
-        
-        graded = homeworkGrader2.grade(self.test_gradesheet)
-        self.assertAlmostEqual( graded['percent'], 0.11 )
-        self.assertEqual( len(graded['section_breakdown']), 12 + 1 )
-        
-        #TODO: How do we test failure cases? The parser only logs an error when it can't parse something. Maybe it should throw exceptions?
+
+def test_graderFromConf(self):
+    
+    #Confs always produce a graders.WeightedSubsectionsGrader, so we test this by repeating the test
+    #in test_graders.WeightedSubsectionsGrader, but generate the graders with confs.
+    
+    weightedGrader = graders.grader_from_conf([
+        {
+            'type' : "Homework",
+            'min_count' : 12,
+            'drop_count' : 2,
+            'short_label' : "HW",
+            'weight' : 0.25,
+        },
+        {
+            'type' : "Lab",
+            'min_count' : 7,
+            'drop_count' : 3,
+            'category' : "Labs",
+            'weight' : 0.25
+        },
+        {
+            'type' : "Midterm",
+            'name' : "Midterm Exam",
+            'short_label' : "Midterm",
+            'weight' : 0.5,
+        },
+    ])
+    
+    emptyGrader = graders.grader_from_conf([])
+    
+    graded = weightedGrader.grade(self.test_gradesheet)
+    self.assertAlmostEqual( graded['percent'], 0.5106547619047619 )
+    self.assertEqual( len(graded['section_breakdown']), (12 + 1) + (7+1) + 1 )
+    self.assertEqual( len(graded['grade_breakdown']), 3 )
+    
+    graded = emptyGrader.grade(self.test_gradesheet)
+    self.assertAlmostEqual( graded['percent'], 0.0 )
+    self.assertEqual( len(graded['section_breakdown']), 0 )
+    self.assertEqual( len(graded['grade_breakdown']), 0 )
+    
+    #Test that graders can also be used instead of lists of dictionaries
+    homeworkGrader = graders.AssignmentFormatGrader("Homework", 12, 2)
+    homeworkGrader2 = graders.grader_from_conf(homeworkGrader)
+    
+    graded = homeworkGrader2.grade(self.test_gradesheet)
+    self.assertAlmostEqual( graded['percent'], 0.11 )
+    self.assertEqual( len(graded['section_breakdown']), 12 + 1 )
+    
+    #TODO: How do we test failure cases? The parser only logs an error when it can't parse something. Maybe it should throw exceptions?
 
 # --------------------------------------------------------------------------
 # Module progress tests
-        
-class ProgressTest(unittest.TestCase):
-    ''' Test that basic Progress objects work.  A Progress represents a
-    fraction between 0 and 1.
-    '''
-    not_started = Progress(0, 17)
-    part_done = Progress(2, 6)
-    half_done = Progress(3, 6)
-    also_half_done = Progress(1, 2)
-    done = Progress(7, 7)
     
-    def test_create_object(self):
-        # These should work:
-        p = Progress(0, 2)
-        p = Progress(1, 2)
-        p = Progress(2, 2)
+class ProgressTest(unittest.TestCase):
+''' Test that basic Progress objects work.  A Progress represents a
+fraction between 0 and 1.
+'''
+not_started = Progress(0, 17)
+part_done = Progress(2, 6)
+half_done = Progress(3, 6)
+also_half_done = Progress(1, 2)
+done = Progress(7, 7)
 
-        p = Progress(2.5, 5.0)
-        p = Progress(3.7, 12.3333)
-        
-        # These shouldn't
-        self.assertRaises(ValueError, Progress, 0, 0)
-        self.assertRaises(ValueError, Progress, 2, 0)
-        self.assertRaises(ValueError, Progress, 1, -2)
-        self.assertRaises(ValueError, Progress, 3, 2)
-        self.assertRaises(ValueError, Progress, -2, 5)
+def test_create_object(self):
+    # These should work:
+    p = Progress(0, 2)
+    p = Progress(1, 2)
+    p = Progress(2, 2)
 
-        self.assertRaises(TypeError, Progress, 0, "all")
-        # check complex numbers just for the heck of it :)
-        self.assertRaises(TypeError, Progress, 2j, 3)
+    p = Progress(2.5, 5.0)
+    p = Progress(3.7, 12.3333)
+    
+    # These shouldn't
+    self.assertRaises(ValueError, Progress, 0, 0)
+    self.assertRaises(ValueError, Progress, 2, 0)
+    self.assertRaises(ValueError, Progress, 1, -2)
+    self.assertRaises(ValueError, Progress, 3, 2)
+    self.assertRaises(ValueError, Progress, -2, 5)
 
-    def test_frac(self):
-        p = Progress(1, 2)
-        (a, b) = p.frac()
-        self.assertEqual(a, 1)
-        self.assertEqual(b, 2)
+    self.assertRaises(TypeError, Progress, 0, "all")
+    # check complex numbers just for the heck of it :)
+    self.assertRaises(TypeError, Progress, 2j, 3)
 
-    def test_percent(self):
-        self.assertEqual(self.not_started.percent(), 0)
-        self.assertAlmostEqual(self.part_done.percent(), 33.33333333333333)
-        self.assertEqual(self.half_done.percent(), 50)
-        self.assertEqual(self.done.percent(), 100)
+def test_frac(self):
+    p = Progress(1, 2)
+    (a, b) = p.frac()
+    self.assertEqual(a, 1)
+    self.assertEqual(b, 2)
 
-        self.assertEqual(self.half_done.percent(), self.also_half_done.percent())
+def test_percent(self):
+    self.assertEqual(self.not_started.percent(), 0)
+    self.assertAlmostEqual(self.part_done.percent(), 33.33333333333333)
+    self.assertEqual(self.half_done.percent(), 50)
+    self.assertEqual(self.done.percent(), 100)
 
-    def test_started(self):
-        self.assertFalse(self.not_started.started())
+    self.assertEqual(self.half_done.percent(), self.also_half_done.percent())
 
-        self.assertTrue(self.part_done.started())
-        self.assertTrue(self.half_done.started())
-        self.assertTrue(self.done.started())
+def test_started(self):
+    self.assertFalse(self.not_started.started())
 
-    def test_inprogress(self):
-        # only true if working on it
-        self.assertFalse(self.done.inprogress())
-        self.assertFalse(self.not_started.inprogress())
+    self.assertTrue(self.part_done.started())
+    self.assertTrue(self.half_done.started())
+    self.assertTrue(self.done.started())
 
-        self.assertTrue(self.part_done.inprogress())
-        self.assertTrue(self.half_done.inprogress())
+def test_inprogress(self):
+    # only true if working on it
+    self.assertFalse(self.done.inprogress())
+    self.assertFalse(self.not_started.inprogress())
 
-    def test_done(self):
-        self.assertTrue(self.done.done())
-        self.assertFalse(self.half_done.done())
-        self.assertFalse(self.not_started.done())
-        
-    def test_str(self):
-        self.assertEqual(str(self.not_started), "0/17")
-        self.assertEqual(str(self.part_done), "2/6")
-        self.assertEqual(str(self.done), "7/7")
+    self.assertTrue(self.part_done.inprogress())
+    self.assertTrue(self.half_done.inprogress())
 
-    def test_ternary_str(self):
-        self.assertEqual(self.not_started.ternary_str(), "none")
-        self.assertEqual(self.half_done.ternary_str(), "in_progress")
-        self.assertEqual(self.done.ternary_str(), "done")
+def test_done(self):
+    self.assertTrue(self.done.done())
+    self.assertFalse(self.half_done.done())
+    self.assertFalse(self.not_started.done())
+    
+def test_str(self):
+    self.assertEqual(str(self.not_started), "0/17")
+    self.assertEqual(str(self.part_done), "2/6")
+    self.assertEqual(str(self.done), "7/7")
 
-    def test_to_js_status(self):
-        '''Test the Progress.to_js_status_str() method'''
-        
-        self.assertEqual(Progress.to_js_status_str(self.not_started), "none")
-        self.assertEqual(Progress.to_js_status_str(self.half_done), "in_progress")
-        self.assertEqual(Progress.to_js_status_str(self.done), "done")
-        self.assertEqual(Progress.to_js_status_str(None), "NA")
+def test_ternary_str(self):
+    self.assertEqual(self.not_started.ternary_str(), "none")
+    self.assertEqual(self.half_done.ternary_str(), "in_progress")
+    self.assertEqual(self.done.ternary_str(), "done")
 
-    def test_to_js_detail_str(self):
-        '''Test the Progress.to_js_detail_str() method'''
-        f = Progress.to_js_detail_str
-        for p in (self.not_started, self.half_done, self.done):
-            self.assertEqual(f(p), str(p))
-        # But None should be encoded as NA
-        self.assertEqual(f(None), "NA")
+def test_to_js_status(self):
+    '''Test the Progress.to_js_status_str() method'''
+    
+    self.assertEqual(Progress.to_js_status_str(self.not_started), "none")
+    self.assertEqual(Progress.to_js_status_str(self.half_done), "in_progress")
+    self.assertEqual(Progress.to_js_status_str(self.done), "done")
+    self.assertEqual(Progress.to_js_status_str(None), "NA")
 
-    def test_add(self):
-        '''Test the Progress.add_counts() method'''
-        p = Progress(0, 2)
-        p2 = Progress(1, 3)
-        p3 = Progress(2, 5)
-        pNone = None
-        add = lambda a, b: Progress.add_counts(a, b).frac()
+def test_to_js_detail_str(self):
+    '''Test the Progress.to_js_detail_str() method'''
+    f = Progress.to_js_detail_str
+    for p in (self.not_started, self.half_done, self.done):
+        self.assertEqual(f(p), str(p))
+    # But None should be encoded as NA
+    self.assertEqual(f(None), "NA")
 
-        self.assertEqual(add(p, p), (0, 4))
-        self.assertEqual(add(p, p2), (1, 5))
-        self.assertEqual(add(p2, p3), (3, 8))
-        
-        self.assertEqual(add(p2, pNone), p2.frac())
-        self.assertEqual(add(pNone, p2), p2.frac())
+def test_add(self):
+    '''Test the Progress.add_counts() method'''
+    p = Progress(0, 2)
+    p2 = Progress(1, 3)
+    p3 = Progress(2, 5)
+    pNone = None
+    add = lambda a, b: Progress.add_counts(a, b).frac()
 
-    def test_equality(self):
-        '''Test that comparing Progress objects for equality
-        works correctly.'''
-        p = Progress(1, 2)
-        p2 = Progress(2, 4)
-        p3 = Progress(1, 2)
-        self.assertTrue(p == p3)
-        self.assertFalse(p == p2)
+    self.assertEqual(add(p, p), (0, 4))
+    self.assertEqual(add(p, p2), (1, 5))
+    self.assertEqual(add(p2, p3), (3, 8))
+    
+    self.assertEqual(add(p2, pNone), p2.frac())
+    self.assertEqual(add(pNone, p2), p2.frac())
 
-        # Check != while we're at it
-        self.assertTrue(p != p2)
-        self.assertFalse(p != p3)
+def test_equality(self):
+    '''Test that comparing Progress objects for equality
+    works correctly.'''
+    p = Progress(1, 2)
+    p2 = Progress(2, 4)
+    p3 = Progress(1, 2)
+    self.assertTrue(p == p3)
+    self.assertFalse(p == p2)
+
+    # Check != while we're at it
+    self.assertTrue(p != p2)
+    self.assertFalse(p != p3)
 
 
 class ModuleProgressTest(unittest.TestCase):
-    ''' Test that get_progress() does the right thing for the different modules
-    '''
-    def test_xmodule_default(self):
-        '''Make sure default get_progress exists, returns None'''
-        xm = x_module.XModule(i4xs, "<dummy />", "dummy")
+''' Test that get_progress() does the right thing for the different modules
+'''
+def test_xmodule_default(self):
+    '''Make sure default get_progress exists, returns None'''
+        xm = x_module.XModule(i4xs, 'a://b/c/d/e', {})
         p = xm.get_progress()
         self.assertEqual(p, None)
-        

common/lib/xmodule/tests/test_export.py

@@ -0,0 +1,28 @@
+from xmodule.modulestore.xml import XMLModuleStore
+from nose.tools import assert_equals
+from tempfile import mkdtemp
+from fs.osfs import OSFS
+
+
+def check_export_roundtrip(data_dir):
+    print "Starting import"
+    initial_import = XMLModuleStore('org', 'course', data_dir, eager=True)
+    initial_course = initial_import.course
+
+    print "Starting export"
+    export_dir = mkdtemp()
+    fs = OSFS(export_dir)
+    xml = initial_course.export_to_xml(fs)
+    with fs.open('course.xml', 'w') as course_xml:
+        course_xml.write(xml)
+
+    print "Starting second import"
+    second_import = XMLModuleStore('org', 'course', export_dir, eager=True)
+
+    print "Checking key equality"
+    assert_equals(initial_import.modules.keys(), second_import.modules.keys())
+
+    print "Checking module equality"
+    for location in initial_import.modules.keys():
+        print "Checking", location
+        assert_equals(initial_import.modules[location], second_import.modules[location])

common/lib/xmodule/tests/test_files/formularesponse_with_hint.xml

@@ -0,0 +1,45 @@
+<problem>
+<script type="loncapa/python">
+# from loncapa import *
+x1 = 4 # lc_random(2,4,1)
+y1 = 5 # lc_random(3,7,1)
+
+x2 = 10 # lc_random(x1+1,9,1)
+y2 = 20 # lc_random(y1+1,15,1)
+
+m = (y2-y1)/(x2-x1)
+b = y1 - m*x1
+answer = "%s*x+%s" % (m,b)
+answer = answer.replace('+-','-')
+
+inverted_m = (x2-x1)/(y2-y1)
+inverted_b = b
+wrongans = "%s*x+%s" % (inverted_m,inverted_b)
+wrongans = wrongans.replace('+-','-')
+</script>
+
+<text>
+<p>Hints can be provided to students, based on the last response given, as well as the history of responses given.  Here is an example of a hint produced by a Formula Response problem.</p>
+
+<p>
+What is the equation of the line which passess through ($x1,$y1) and
+($x2,$y2)?</p>
+
+<p>The correct answer is <tt>$answer</tt>. A common error is to invert the equation for the slope.  Enter <tt>
+$wrongans</tt> to see a hint.</p>
+
+</text>
+
+<formularesponse samples="x@-5:5#11" id="11" answer="$answer">
+ <responseparam description="Numerical Tolerance" type="tolerance" default="0.001" name="tol" /> 
+ <text>y = <textline size="25" /></text>
+  <hintgroup>
+   <formulahint samples="x@-5:5#11" answer="$wrongans" name="inversegrad">
+   </formulahint>
+   <hintpart on="inversegrad">
+    <text>You have inverted the slope in the question.</text>
+   </hintpart>
+  </hintgroup>     
+</formularesponse>
+</problem>
+

common/lib/xmodule/tests/test_files/stringresponse_with_hint.xml

@@ -0,0 +1,25 @@
+<problem >
+ <text><h2>Example: String Response Problem</h2>
+ <br/>
+ </text>
+ 
+ <text>Which US state has Lansing as its capital?</text>
+  <stringresponse answer="Michigan" type="ci">
+  <textline size="20" />
+  <hintgroup>
+   <stringhint answer="wisconsin" type="cs" name="wisc">
+   </stringhint>
+   <stringhint answer="minnesota" type="cs" name="minn">
+   </stringhint>
+   <hintpart on="wisc">
+    <text>The state capital of Wisconsin is Madison.</text>
+   </hintpart>
+   <hintpart on="minn">
+    <text>The state capital of Minnesota is St. Paul.</text>
+   </hintpart>
+   <hintpart on="default">
+    <text>The state you are looking for is also known as the 'Great Lakes State'</text>
+   </hintpart>     
+  </hintgroup>
+ </stringresponse>
+</problem>

common/lib/xmodule/tests/test_files/symbolicresponse.xml

@@ -0,0 +1,29 @@
+<problem>
+<text>
+<h2>Example: Symbolic Math Response Problem</h2>
+
+<p>
+A symbolic math response problem presents one or more symbolic math
+input fields for input.  Correctness of input is evaluated based on
+the symbolic properties of the expression entered.  The student enters
+text, but sees a proper symbolic rendition of the entered formula, in
+real time, next to the input box.
+</p>
+
+<p>This is a correct answer which may be entered below: </p>
+<p><tt>cos(theta)*[[1,0],[0,1]] + i*sin(theta)*[[0,1],[1,0]]</tt></p>
+
+<script>
+from symmath import *
+</script>
+  <text>Compute [mathjax] U = \exp\left( i \theta \left[ \begin{matrix} 0 &amp; 1 \\ 1 &amp; 0 \end{matrix} \right] \right) [/mathjax]
+      and give the resulting \(2 \times 2\) matrix. <br/>
+      Your input should be typed in as a list of lists, eg <tt>[[1,2],[3,4]]</tt>. <br/>
+       [mathjax]U=[/mathjax] <symbolicresponse cfn="symmath_check" answer="[[cos(theta),I*sin(theta)],[I*sin(theta),cos(theta)]]" options="matrix,imaginaryi" id="filenamedogi0VpEBOWedxsymmathresponse_1" state="unsubmitted">
+          <textline size="80" math="1" response_id="2" answer_id="1" id="filenamedogi0VpEBOWedxsymmathresponse_2_1"/>
+        </symbolicresponse>
+      <br/>
+  </text>
+
+</text>
+</problem>

common/lib/xmodule/vertical_module.py

@@ -1,37 +0,0 @@
-import json
-
-from x_module import XModule, XModuleDescriptor
-from xmodule.progress import Progress
-from lxml import etree
-
-class ModuleDescriptor(XModuleDescriptor):
-    pass
-
-class Module(XModule):
-    ''' Layout module for laying out submodules vertically.'''
-    id_attribute = 'id'
-
-    def get_state(self):
-        return json.dumps({ })
-
-    @classmethod
-    def get_xml_tags(c):
-        return ["vertical", "problemset"]
-        
-    def get_html(self):
-        return self.system.render_template('vert_module.html', {
-            'items': self.contents
-        })
-
-    def get_progress(self):
-        # TODO: Cache progress or children array?
-        children = self.get_children()
-        progresses = [child.get_progress() for child in children]
-        progress = reduce(Progress.add_counts, progresses)
-        return progress
-
-    def __init__(self, system, xml, item_id, state=None):
-        XModule.__init__(self, system, xml, item_id, state)
-        xmltree=etree.fromstring(xml)
-        self.contents=[(e.get("name"),self.render_function(e)) \
-                      for e in xmltree]

common/lib/xmodule/x_module.py

@@ -1,257 +0,0 @@
-from lxml import etree
-import pkg_resources
-import logging
-
-from keystore import Location
-from progress import Progress
-
-log = logging.getLogger('mitx.' + __name__)
-
-def dummy_track(event_type, event):
-    pass
-
-
-class ModuleMissingError(Exception):
-    pass
-
-
-class Plugin(object):
-    @classmethod
-    def load_class(cls, identifier):
-        classes = list(pkg_resources.iter_entry_points(cls.entry_point, name=identifier))
-        if len(classes) > 1:
-            log.warning("Found multiple classes for {entry_point} with identifier {id}: {classes}. Returning the first one.".format(
-                entry_point=cls.entry_point,
-                id=identifier,
-                classes=", ".join(class_.module_name for class_ in classes)))
-
-        if len(classes) == 0:
-            raise ModuleMissingError(identifier)
-
-        return classes[0].load()
-
-
-class XModule(object):
-    ''' Implements a generic learning module. 
-        Initialized on access with __init__, first time with state=None, and
-        then with state
-
-        See the HTML module for a simple example
-    '''
-    id_attribute='id' # An attribute guaranteed to be unique
-
-    @classmethod
-    def get_xml_tags(c):
-        ''' Tags in the courseware file guaranteed to correspond to the module '''
-        return []
-        
-    @classmethod
-    def get_usage_tags(c):
-        ''' We should convert to a real module system
-            For now, this tells us whether we use this as an xmodule, a CAPA response type
-            or a CAPA input type '''
-        return ['xmodule']
-
-    def get_name(self):
-        name = self.__xmltree.get('name')
-        if name: 
-            return name
-        else: 
-            raise "We should iterate through children and find a default name"
-
-    def get_children(self):
-        '''
-        Return module instances for all the children of this module.
-        '''
-        children = [self.module_from_xml(e) for e in self.__xmltree]
-        return children            
-
-    def rendered_children(self):
-        '''
-        Render all children. 
-        This really ought to return a list of xmodules, instead of dictionaries
-        '''
-        children = [self.render_function(e) for e in self.__xmltree]
-        return children            
-
-    def __init__(self, system = None, xml = None, item_id = None, 
-                 json = None, track_url=None, state=None):
-        ''' In most cases, you must pass state or xml'''
-        if not item_id: 
-            raise ValueError("Missing Index")
-        if not xml and not json:
-            raise ValueError("xml or json required")
-        if not system:
-            raise ValueError("System context required")
-
-        self.xml = xml
-        self.json = json
-        self.item_id = item_id
-        self.state = state
-        self.DEBUG = False
-        
-        self.__xmltree = etree.fromstring(xml) # PRIVATE
-
-        if system: 
-            ## These are temporary; we really should go 
-            ## through self.system. 
-            self.ajax_url = system.ajax_url
-            self.tracker = system.track_function
-            self.filestore = system.filestore
-            self.render_function = system.render_function
-            self.module_from_xml = system.module_from_xml
-            self.DEBUG = system.DEBUG
-        self.system = system
-
-    ### Functions used in the LMS
-
-    def get_state(self):
-        ''' State of the object, as stored in the database 
-        '''
-        return ""
-
-    def get_score(self):
-        ''' Score the student received on the problem. 
-        '''
-        return None
-
-    def max_score(self):
-        ''' Maximum score. Two notes: 
-            * This is generic; in abstract, a problem could be 3/5 points on one randomization, and 5/7 on another
-            * In practice, this is a Very Bad Idea, and (a) will break some code in place (although that code
-              should get fixed), and (b) break some analytics we plan to put in place. 
-        ''' 
-        return None
-
-    def get_html(self):
-        ''' HTML, as shown in the browser. This is the only method that must be implemented
-        '''
-        return "Unimplemented"
-
-    def get_progress(self):
-        ''' Return a progress.Progress object that represents how far the student has gone
-        in this module.  Must be implemented to get correct progress tracking behavior in
-        nesting modules like sequence and vertical.
-
-        If this module has no notion of progress, return None.
-        '''
-        return None
-
-    def handle_ajax(self, dispatch, get):
-        ''' dispatch is last part of the URL. 
-            get is a dictionary-like object ''' 
-        return ""
-
-
-class XModuleDescriptor(Plugin):
-    """
-    An XModuleDescriptor is a specification for an element of a course. This could
-    be a problem, an organizational element (a group of content), or a segment of video,
-    for example.
-
-    XModuleDescriptors are independent and agnostic to the current student state on a
-    problem. They handle the editing interface used by instructors to create a problem,
-    and can generate XModules (which do know about student state).
-    """
-    entry_point = "xmodule.v1"
-
-    @staticmethod
-    def load_from_json(json_data, system):
-        """
-        This method instantiates the correct subclass of XModuleDescriptor based
-        on the contents of json_data.
-
-        json_data must contain a 'location' element, and must be suitable to be
-        passed into the subclasses `from_json` method.
-        """
-        class_ = XModuleDescriptor.load_class(json_data['location']['category'])
-        return class_.from_json(json_data, system)
-
-    @classmethod
-    def from_json(cls, json_data, system):
-        """
-        Creates an instance of this descriptor from the supplied json_data.
-        This may be overridden by subclasses
-
-        json_data: Json data specifying the data, children, and metadata for the descriptor
-        system: An XModuleSystem for interacting with external resources
-        """
-        return cls(system=system, **json_data)
-
-    def __init__(self,
-                 system,
-                 definition=None,
-                 **kwargs):
-        """
-        Construct a new XModuleDescriptor. The only required arguments are the
-        system, used for interaction with external resources, and the definition,
-        which specifies all the data needed to edit and display the problem (but none
-        of the associated metadata that handles recordkeeping around the problem).
-
-        This allows for maximal flexibility to add to the interface while preserving
-        backwards compatibility.
-
-        system: An XModuleSystem for interacting with external resources
-        definition: A dict containing `data` and `children` representing the problem definition
-
-        Current arguments passed in kwargs:
-            location: A keystore.Location object indicating the name and ownership of this problem
-            goals: A list of strings of learning goals associated with this module
-        """
-        self.system = system
-        self.definition = definition if definition is not None else {}
-        self.name = Location(kwargs.get('location')).name
-        self.type = Location(kwargs.get('location')).category
-
-        # For now, we represent goals as a list of strings, but this
-        # is one of the things that we are going to be iterating on heavily
-        # to find the best teaching method
-        self.goals = kwargs.get('goals', [])
-
-        self._child_instances = None
-
-    def get_children(self, categories=None):
-        """Returns a list of XModuleDescriptor instances for the children of this module"""
-        if self._child_instances is None:
-            self._child_instances = [self.system.load_item(child) for child in self.definition['children']]
-
-        if categories is None:
-            return self._child_instances
-        else:
-            return [child for child in self._child_instances if child.type in categories]
-
-    def get_xml(self):
-        ''' For conversions between JSON and legacy XML representations.
-        '''
-        if self.xml: 
-            return self.xml
-        else: 
-            raise NotImplementedError("JSON->XML Translation not implemented")
-
-    def get_json(self):
-        ''' For conversions between JSON and legacy XML representations.
-        '''
-        if self.json: 
-            raise NotImplementedError
-            return self.json # TODO: Return context as well -- files, etc. 
-        else: 
-            raise NotImplementedError("XML->JSON Translation not implemented")
-
-    #def handle_cms_json(self):
-    #    raise NotImplementedError
-
-    #def render(self, size):
-    #    ''' Size: [thumbnail, small, full] 
-    #    Small ==> what we drag around
-    #    Full ==> what we edit
-    #    '''
-    #    raise NotImplementedError
-
-
-class DescriptorSystem(object):
-    def __init__(self, load_item):
-        """
-        load_item: Takes a Location and returns and XModuleDescriptor
-        """
-
-        self.load_item = load_item

common/lib/xmodule/xmodule/abtest_module.py

@@ -0,0 +1,124 @@
+import json
+import random
+from lxml import etree
+
+from xmodule.x_module import XModule
+from xmodule.raw_module import RawDescriptor
+from xmodule.xml_module import XmlDescriptor
+from xmodule.exceptions import InvalidDefinitionError
+
+DEFAULT = "_DEFAULT_GROUP"
+
+
+def group_from_value(groups, v):
+    ''' Given group: (('a',0.3),('b',0.4),('c',0.3)) And random value
+    in [0,1], return the associated group (in the above case, return
+    'a' if v<0.3, 'b' if 0.3<=v<0.7, and 'c' if v>0.7
+'''
+    sum = 0
+    for (g, p) in groups:
+        sum = sum + p
+        if sum > v:
+            return g
+
+    # Round off errors might cause us to run to the end of the list
+    # If the do, return the last element
+    return g
+
+
+class ABTestModule(XModule):
+    """
+    Implements an A/B test with an aribtrary number of competing groups
+
+    Format:
+    <abtest>
+        <group name="a" portion=".1"><contenta/></group>
+        <group name="b" portion=".2"><contentb/></group>
+        <default><contentdefault/></default>
+    </abtest>
+    """
+
+    def __init__(self, system, location, definition, instance_state=None, shared_state=None, **kwargs):
+        XModule.__init__(self, system, location, definition, instance_state, shared_state, **kwargs)
+
+        if shared_state is None:
+
+            self.group = group_from_value(
+                self.definition['data']['group_portions'].items(),
+                random.uniform(0, 1)
+            )
+        else:
+            shared_state = json.loads(shared_state)
+            self.group = shared_state['group']
+
+    def get_shared_state(self):
+        return json.dumps({'group': self.group})
+
+    def displayable_items(self):
+        return [self.system.get_module(child)
+                for child
+                in self.definition['data']['group_content'][self.group]]
+
+
+class ABTestDescriptor(RawDescriptor, XmlDescriptor):
+    module_class = ABTestModule
+
+    def __init__(self, system, definition=None, **kwargs):
+        kwargs['shared_state_key'] = definition['data']['experiment']
+        RawDescriptor.__init__(self, system, definition, **kwargs)
+
+    @classmethod
+    def definition_from_xml(cls, xml_object, system):
+        experiment = xml_object.get('experiment')
+
+        if experiment is None:
+            raise InvalidDefinitionError("ABTests must specify an experiment. Not found in:\n{xml}".format(xml=etree.tostring(xml_object, pretty_print=True)))
+
+        definition = {
+            'data': {
+                'experiment': experiment,
+                'group_portions': {},
+                'group_content': {DEFAULT: []},
+            },
+            'children': []}
+        for group in xml_object:
+            if group.tag == 'default':
+                name = DEFAULT
+            else:
+                name = group.get('name')
+                definition['data']['group_portions'][name] = float(group.get('portion', 0))
+
+            child_content_urls = [
+                system.process_xml(etree.tostring(child)).location.url()
+                for child in group
+            ]
+
+            definition['data']['group_content'][name] = child_content_urls
+            definition['children'].extend(child_content_urls)
+
+        default_portion = 1 - sum(portion for (name, portion) in definition['data']['group_portions'].items())
+        if default_portion < 0:
+            raise InvalidDefinitionError("ABTest portions must add up to less than or equal to 1")
+
+        definition['data']['group_portions'][DEFAULT] = default_portion
+        definition['children'].sort()
+
+        return definition
+
+    def definition_to_xml(self, resource_fs):
+        xml_object = etree.Element('abtest')
+        xml_object.set('experiment', self.definition['data']['experiment'])
+        for name, group in self.definition['data']['group_content'].items():
+            if name == DEFAULT:
+                group_elem = etree.SubElement(xml_object, 'default')
+            else:
+                group_elem = etree.SubElement(xml_object, 'group', attrib={
+                    'portion': str(self.definition['data']['group_portions'][name]),
+                    'name': name,
+                })
+
+            for child_loc in group:
+                child = self.system.load_item(child_loc)
+                group_elem.append(etree.fromstring(child.export_to_xml(resource_fs)))
+
+        return xml_object

common/lib/xmodule/xmodule/capa_module.py

@@ -10,7 +10,8 @@ import StringIO
 from datetime import timedelta
 from lxml import etree
 
-from x_module import XModule, XModuleDescriptor
+from xmodule.x_module import XModule
+from xmodule.raw_module import RawDescriptor
 from progress import Progress
 from capa.capa_problem import LoncapaProblem
 from capa.responsetypes import StudentInputError
@@ -63,38 +64,114 @@ class ComplexEncoder(json.JSONEncoder):
         return json.JSONEncoder.default(self, obj)
 
 
-class ModuleDescriptor(XModuleDescriptor):
-    pass
-
-
-class Module(XModule):
+class CapaModule(XModule):
     ''' Interface between capa_problem and x_module. Originally a hack
     meant to be refactored out, but it seems to be serving a useful
     prupose now. We can e.g .destroy and create the capa_problem on a
     reset.
     '''
+    icon_class = 'problem'
 
-    id_attribute = "filename"
+    def __init__(self, system, location, definition, instance_state=None, shared_state=None, **kwargs):
+        XModule.__init__(self, system, location, definition, instance_state, shared_state, **kwargs)
 
-    @classmethod
-    def get_xml_tags(c):
-        return ["problem"]
+        self.attempts = 0
+        self.max_attempts = None
 
+        dom2 = etree.fromstring(definition['data'])
 
-    def get_state(self):
+        self.explanation = "problems/" + only_one(dom2.xpath('/problem/@explain'),
+                                                  default="closed")
+        # TODO: Should be converted to: self.explanation=only_one(dom2.xpath('/problem/@explain'), default="closed")
+        self.explain_available = only_one(dom2.xpath('/problem/@explain_available'))
+
+        display_due_date_string = self.metadata.get('due', None)
+        if display_due_date_string is not None:
+            self.display_due_date = dateutil.parser.parse(display_due_date_string)
+            #log.debug("Parsed " + display_due_date_string + " to " + str(self.display_due_date))
+        else:
+            self.display_due_date = None
+
+        grace_period_string = self.metadata.get('graceperiod', None)
+        if grace_period_string is not None and self.display_due_date:
+            self.grace_period = parse_timedelta(grace_period_string)
+            self.close_date = self.display_due_date + self.grace_period
+            #log.debug("Then parsed " + grace_period_string + " to closing date" + str(self.close_date))
+        else:
+            self.grace_period = None
+            self.close_date = self.display_due_date
+
+        self.max_attempts = only_one(dom2.xpath('/problem/@attempts'))
+        if len(self.max_attempts) > 0:
+            self.max_attempts = int(self.max_attempts)
+        else:
+            self.max_attempts = None
+
+        self.show_answer = self.metadata.get('showanwser', 'closed')
+
+        if self.show_answer == "":
+            self.show_answer = "closed"
+
+        if instance_state != None:
+            instance_state = json.loads(instance_state)
+        if instance_state != None and 'attempts' in instance_state:
+            self.attempts = instance_state['attempts']
+
+        self.name = only_one(dom2.xpath('/problem/@name'))
+
+        weight_string = only_one(dom2.xpath('/problem/@weight'))
+        if weight_string:
+            self.weight = float(weight_string)
+        else:
+            self.weight = 1
+
+        if self.rerandomize == 'never':
+            seed = 1
+        elif self.rerandomize == "per_student" and hasattr(system, 'id'):
+            seed = system.id
+        else:
+            seed = None
+
+        try:
+            self.lcp = LoncapaProblem(self.definition['data'], self.location.html_id(), instance_state, seed=seed, system=self.system)
+        except Exception:
+            msg = 'cannot create LoncapaProblem %s' % self.url
+            log.exception(msg)
+            if self.system.DEBUG:
+                msg = '<p>%s</p>' % msg.replace('<', '&lt;')
+                msg += '<p><pre>%s</pre></p>' % traceback.format_exc().replace('<', '&lt;')
+                # create a dummy problem with error message instead of failing
+                problem_text = '<problem><text><font color="red" size="+2">Problem %s has an error:</font>%s</text></problem>' % (self.location.url(), msg)
+                self.lcp = LoncapaProblem(problem_text, self.location.html_id(), instance_state, seed=seed, system=self.system)
+            else:
+                raise
+
+    @property
+    def rerandomize(self):
+        """
+        Property accessor that returns self.metadata['rerandomize'] in a canonical form
+        """
+        rerandomize = self.metadata.get('rerandomize', 'always')
+        if rerandomize in ("", "always", "true"):
+            return "always"
+        elif rerandomize in ("false", "per_student"):
+            return "per_student"
+        elif rerandomize == "never":
+            return "never"
+        else:
+            raise Exception("Invalid rerandomize attribute " + rerandomize)
+
+    def get_instance_state(self):
         state = self.lcp.get_state()
         state['attempts'] = self.attempts
         return json.dumps(state)
 
-
     def get_score(self):
         return self.lcp.get_score()
 
-
     def max_score(self):
         return self.lcp.get_max_score()
 
-
     def get_progress(self):
         ''' For now, just return score / max_score
         '''
@@ -105,20 +182,19 @@ class Module(XModule):
             return Progress(score, total)
         return None
 
-
     def get_html(self):
         return self.system.render_template('problem_ajax.html', {
-            'id': self.item_id,
-            'ajax_url': self.ajax_url,
+            'element_id': self.location.html_id(),
+            'id': self.id,
+            'ajax_url': self.system.ajax_url,
         })
 
-
     def get_problem_html(self, encapsulate=True):
         '''Return html for the problem.  Adds check, reset, save buttons
         as necessary based on the problem config and state.'''
 
         html = self.lcp.get_html()
-        content = {'name': self.name,
+        content = {'name': self.metadata['display_name'],
                    'html': html,
                    'weight': self.weight,
                   }
@@ -165,12 +241,12 @@ class Module(XModule):
             explain = False
 
         context = {'problem': content,
-                   'id': self.item_id,
+                   'id': self.id,
                    'check_button': check_button,
                    'reset_button': reset_button,
                    'save_button': save_button,
                    'answer_available': self.answer_available(),
-                   'ajax_url': self.ajax_url,
+                   'ajax_url': self.system.ajax_url,
                    'attempts_used': self.attempts,
                    'attempts_allowed': self.max_attempts,
                    'explain': explain,
@@ -180,100 +256,10 @@ class Module(XModule):
         html = self.system.render_template('problem.html', context)
         if encapsulate:
             html = '<div id="problem_{id}" class="problem" data-url="{ajax_url}">'.format(
-                id=self.item_id, ajax_url=self.ajax_url) + html + "</div>"
+                id=self.location.html_id(), ajax_url=self.system.ajax_url) + html + "</div>"
 
         return html
 
-    def __init__(self, system, xml, item_id, state=None):
-        XModule.__init__(self, system, xml, item_id, state)
-
-        self.attempts = 0
-        self.max_attempts = None
-
-        dom2 = etree.fromstring(xml)
-
-        self.explanation = "problems/" + only_one(dom2.xpath('/problem/@explain'),
-                                                  default="closed")
-        # TODO: Should be converted to: self.explanation=only_one(dom2.xpath('/problem/@explain'), default="closed")
-        self.explain_available = only_one(dom2.xpath('/problem/@explain_available'))
-
-        display_due_date_string = only_one(dom2.xpath('/problem/@due'))
-        if len(display_due_date_string) > 0:
-            self.display_due_date = dateutil.parser.parse(display_due_date_string)
-            #log.debug("Parsed " + display_due_date_string + " to " + str(self.display_due_date))
-        else:
-            self.display_due_date = None
-
-        grace_period_string = only_one(dom2.xpath('/problem/@graceperiod'))
-        if len(grace_period_string) >0 and self.display_due_date:
-            self.grace_period = parse_timedelta(grace_period_string)
-            self.close_date = self.display_due_date + self.grace_period
-            #log.debug("Then parsed " + grace_period_string + " to closing date" + str(self.close_date))
-        else:
-            self.grace_period = None
-            self.close_date = self.display_due_date
-
-        self.max_attempts = only_one(dom2.xpath('/problem/@attempts'))
-        if len(self.max_attempts) > 0:
-            self.max_attempts = int(self.max_attempts)
-        else:
-            self.max_attempts = None
-
-        self.show_answer = only_one(dom2.xpath('/problem/@showanswer'))
-
-        if self.show_answer == "":
-            self.show_answer = "closed"
-
-        self.rerandomize = only_one(dom2.xpath('/problem/@rerandomize'))
-        if self.rerandomize == "" or self.rerandomize=="always" or self.rerandomize=="true":
-            self.rerandomize="always"
-        elif self.rerandomize=="false" or self.rerandomize=="per_student":
-            self.rerandomize="per_student"
-        elif self.rerandomize=="never":
-            self.rerandomize="never"
-        else:
-            raise Exception("Invalid rerandomize attribute "+self.rerandomize)
-
-        if state!=None:
-            state=json.loads(state)
-        if state!=None and 'attempts' in state:
-            self.attempts=state['attempts']
-
-        # TODO: Should be: self.filename=only_one(dom2.xpath('/problem/@filename'))
-        self.filename= "problems/"+only_one(dom2.xpath('/problem/@filename'))+".xml"
-        self.name=only_one(dom2.xpath('/problem/@name'))
-        self.weight=only_one(dom2.xpath('/problem/@weight'))
-        if self.rerandomize == 'never':
-            seed = 1
-        elif self.rerandomize == "per_student" and hasattr(system, 'id'):
-            seed = system.id
-        else:
-            seed = None
-        try:
-            fp = self.filestore.open(self.filename)
-        except Exception,err:
-            log.exception('[courseware.capa.capa_module.Module.init] error %s: cannot open file %s' % (err,self.filename))
-            if self.DEBUG:
-                # create a dummy problem instead of failing
-                fp = StringIO.StringIO('<problem><text><font color="red" size="+2">Problem file %s is missing</font></text></problem>' % self.filename)
-                fp.name = "StringIO"
-            else:
-                raise
-        try:
-            self.lcp=LoncapaProblem(fp, self.item_id, state, seed = seed, system=self.system)
-        except Exception,err:
-            msg = '[courseware.capa.capa_module.Module.init] error %s: cannot create LoncapaProblem %s' % (err,self.filename)
-            log.exception(msg)
-            if self.DEBUG:
-                msg = '<p>%s</p>' % msg.replace('<','&lt;')
-                msg += '<p><pre>%s</pre></p>' % traceback.format_exc().replace('<','&lt;')
-                # create a dummy problem with error message instead of failing
-                fp = StringIO.StringIO('<problem><text><font color="red" size="+2">Problem file %s has an error:</font>%s</text></problem>' % (self.filename,msg))
-                fp.name = "StringIO"
-                self.lcp=LoncapaProblem(fp, self.item_id, state, seed = seed, system=self.system)
-            else:
-                raise
-
     def handle_ajax(self, dispatch, get):
         '''
         This is called by courseware.module_render, to handle an AJAX call.
@@ -299,8 +285,8 @@ class Module(XModule):
         d = handlers[dispatch](get)
         after = self.get_progress()
         d.update({
-            'progress_changed' : after != before,
-            'progress_status' : Progress.to_js_status_str(after),
+            'progress_changed': after != before,
+            'progress_status': Progress.to_js_status_str(after),
             })
         return json.dumps(d, cls=ComplexEncoder)
 
@@ -313,7 +299,6 @@ class Module(XModule):
 
         return False
 
-
     def answer_available(self):
         ''' Is the user allowed to see an answer?
         '''
@@ -334,7 +319,8 @@ class Module(XModule):
 
         if self.show_answer == 'always':
             return True
-        raise self.system.exception404 #TODO: Not 404
+        #TODO: Not 404
+        raise self.system.exception404
 
     def get_answer(self, get):
         '''
@@ -348,8 +334,7 @@ class Module(XModule):
             raise self.system.exception404
         else:
             answers = self.lcp.get_question_answers()
-            return {'answers' : answers}
-
+            return {'answers': answers}
 
     # Figure out if we should move these to capa_problem?
     def get_problem(self, get):
@@ -358,8 +343,8 @@ class Module(XModule):
 
             Used if we want to reconfirm we have the right thing e.g. after
             several AJAX calls.
-        ''' 
-        return {'html' : self.get_problem_html(encapsulate=False)}
+        '''
+        return {'html': self.get_problem_html(encapsulate=False)}
 
     @staticmethod
     def make_dict_of_responses(get):
@@ -383,7 +368,7 @@ class Module(XModule):
             '''
         event_info = dict()
         event_info['state'] = self.lcp.get_state()
-        event_info['filename'] = self.filename
+        event_info['problem_id'] = self.location.url()
 
         answers = self.make_dict_of_responses(get)
 
@@ -392,7 +377,7 @@ class Module(XModule):
         # Too late. Cannot submit
         if self.closed():
             event_info['failure'] = 'closed'
-            self.tracker('save_problem_check_fail', event_info)
+            self.system.track_function('save_problem_check_fail', event_info)
             # TODO (vshnayder): probably not 404?
             raise self.system.exception404
 
@@ -400,7 +385,7 @@ class Module(XModule):
         # again.
         if self.lcp.done and self.rerandomize == "always":
             event_info['failure'] = 'unreset'
-            self.tracker('save_problem_check_fail', event_info)
+            self.system.track_function('save_problem_check_fail', event_info)
             raise self.system.exception404
 
         try:
@@ -409,18 +394,16 @@ class Module(XModule):
             correct_map = self.lcp.grade_answers(answers)
         except StudentInputError as inst:
             # TODO (vshnayder): why is this line here?
-            self.lcp = LoncapaProblem(self.filestore.open(self.filename),
+            self.lcp = LoncapaProblem(self.definition['data'],
                                       id=lcp_id, state=old_state, system=self.system)
             traceback.print_exc()
             return {'success': inst.message}
         except:
             # TODO: why is this line here?
-            self.lcp = LoncapaProblem(self.filestore.open(self.filename),
+            self.lcp = LoncapaProblem(self.definition['data'],
                                       id=lcp_id, state=old_state, system=self.system)
             traceback.print_exc()
-            raise Exception,"error in capa_module"
-            # TODO: Dead code...  is this a bug, or just old?
-            return {'success':'Unknown Error'}
+            raise Exception("error in capa_module")
 
         self.attempts = self.attempts + 1
         self.lcp.done = True
@@ -431,21 +414,18 @@ class Module(XModule):
             if not correct_map.is_correct(answer_id):
                 success = 'incorrect'
 
-        event_info['correct_map'] = correct_map.get_dict()	# log this in the tracker
+        # log this in the track_function
+        event_info['correct_map'] = correct_map.get_dict()
         event_info['success'] = success
-        self.tracker('save_problem_check', event_info)
+        self.system.track_function('save_problem_check', event_info)
 
-        try:
-            html = self.get_problem_html(encapsulate=False)	# render problem into HTML
-        except Exception,err:
-            log.error('failed to generate html')
-            raise
+        # render problem into HTML
+        html = self.get_problem_html(encapsulate=False)
 
         return {'success': success,
                 'contents': html,
                 }
 
-
     def save_problem(self, get):
         '''
         Save the passed in answers.
@@ -454,7 +434,7 @@ class Module(XModule):
         '''
         event_info = dict()
         event_info['state'] = self.lcp.get_state()
-        event_info['filename'] = self.filename
+        event_info['problem_id'] = self.location.url()
 
         answers = self.make_dict_of_responses(get)
         event_info['answers'] = answers
@@ -462,7 +442,7 @@ class Module(XModule):
         # Too late. Cannot submit
         if self.closed():
             event_info['failure'] = 'closed'
-            self.tracker('save_problem_fail', event_info)
+            self.system.track_function('save_problem_fail', event_info)
             return {'success': False,
                     'error': "Problem is closed"}
 
@@ -470,14 +450,14 @@ class Module(XModule):
         # again.
         if self.lcp.done and self.rerandomize == "always":
             event_info['failure'] = 'done'
-            self.tracker('save_problem_fail', event_info)
-            return {'success' : False,
-                    'error' : "Problem needs to be reset prior to save."}
+            self.system.track_function('save_problem_fail', event_info)
+            return {'success': False,
+                    'error': "Problem needs to be reset prior to save."}
 
         self.lcp.student_answers = answers
 
         # TODO: should this be save_problem_fail?  Looks like success to me...
-        self.tracker('save_problem_fail', event_info)
+        self.system.track_function('save_problem_fail', event_info)
         return {'success': True}
 
     def reset_problem(self, get):
@@ -485,30 +465,39 @@ class Module(XModule):
             and causes problem to rerender itself.
 
             Returns problem html as { 'html' : html-string }.
-        ''' 
+        '''
         event_info = dict()
         event_info['old_state'] = self.lcp.get_state()
-        event_info['filename'] = self.filename
+        event_info['problem_id'] = self.location.url()
 
         if self.closed():
             event_info['failure'] = 'closed'
-            self.tracker('reset_problem_fail', event_info)
+            self.system.track_function('reset_problem_fail', event_info)
             return "Problem is closed"
 
         if not self.lcp.done:
             event_info['failure'] = 'not_done'
-            self.tracker('reset_problem_fail', event_info)
+            self.system.track_function('reset_problem_fail', event_info)
             return "Refresh the page and make an attempt before resetting."
 
         self.lcp.do_reset()
         if self.rerandomize == "always":
             # reset random number generator seed (note the self.lcp.get_state() in next line)
-            self.lcp.seed=None
-            
-        self.lcp = LoncapaProblem(self.filestore.open(self.filename),
-                                  self.item_id, self.lcp.get_state(), system=self.system)
+            self.lcp.seed = None
+
+        self.lcp = LoncapaProblem(self.definition['data'],
+                                  self.location.html_id(), self.lcp.get_state(), system=self.system)
 
         event_info['new_state'] = self.lcp.get_state()
-        self.tracker('reset_problem', event_info)
+        self.system.track_function('reset_problem', event_info)
 
-        return {'html' : self.get_problem_html(encapsulate=False)}
+        return {'html': self.get_problem_html(encapsulate=False)}
+
+
+class CapaDescriptor(RawDescriptor):
+    """
+    Module implementing problems in the LON-CAPA format,
+    as implemented by capa.capa_problem
+    """
+
+    module_class = CapaModule

common/lib/xmodule/xmodule/exceptions.py

@@ -0,0 +1,2 @@
+class InvalidDefinitionError(Exception):
+    pass

common/lib/xmodule/xmodule/hidden_module.py

@@ -0,0 +1,10 @@
+from xmodule.x_module import XModule
+from xmodule.raw_module import RawDescriptor
+
+
+class HiddenModule(XModule):
+    pass
+
+
+class HiddenDescriptor(RawDescriptor):
+    module_class = HiddenModule

common/lib/xmodule/xmodule/html_module.py

@@ -0,0 +1,28 @@
+import logging
+
+from xmodule.x_module import XModule
+from xmodule.raw_module import RawDescriptor
+from pkg_resources import resource_string
+
+log = logging.getLogger("mitx.courseware")
+
+
+class HtmlModule(XModule):
+    def get_html(self):
+        return self.html
+
+    def __init__(self, system, location, definition, instance_state=None, shared_state=None, **kwargs):
+        XModule.__init__(self, system, location, definition, instance_state, shared_state, **kwargs)
+        self.html = self.definition['data']
+
+
+class HtmlDescriptor(RawDescriptor):
+    """
+    Module for putting raw html in a course
+    """
+    mako_template = "widgets/html-edit.html"
+    module_class = HtmlModule
+    filename_extension = "html"
+
+    js = {'coffee': [resource_string(__name__, 'js/module/html.coffee')]}
+    js_module = 'HTML'

common/lib/xmodule/xmodule/js/module/html.coffee

@@ -0,0 +1,9 @@
+class @HTML
+    constructor: (@id) ->
+        @edit_box = $("##{@id} .edit-box")
+        @preview = $("##{@id} .preview")
+        @edit_box.on('input', =>
+            @preview.empty().append(@edit_box.val())
+        )
+
+    save: -> {text: @edit_box.val()}

common/lib/xmodule/xmodule/js/module/raw.coffee

@@ -0,0 +1,9 @@
+class @Raw
+    constructor: (@id) ->
+        @edit_box = $("##{@id} .edit-box")
+        @preview = $("##{@id} .preview")
+        @edit_box.on('input', =>
+            @preview.empty().text(@edit_box.val())
+        )
+
+    save: -> @edit_box.val()

common/lib/xmodule/xmodule/mako_module.py

@@ -0,0 +1,32 @@
+from x_module import XModuleDescriptor, DescriptorSystem
+
+
+class MakoDescriptorSystem(DescriptorSystem):
+    def __init__(self, render_template, *args, **kwargs):
+        self.render_template = render_template
+        super(MakoDescriptorSystem, self).__init__(*args, **kwargs)
+
+
+class MakoModuleDescriptor(XModuleDescriptor):
+    """
+    Module descriptor intended as a mixin that uses a mako template
+    to specify the module html.
+
+    Expects the descriptor to have the `mako_template` attribute set
+    with the name of the template to render, and it will pass
+    the descriptor as the `module` parameter to that template
+    """
+
+    def __init__(self, system, definition=None, **kwargs):
+        if getattr(system, 'render_template', None) is None:
+            raise TypeError('{system} must have a render_template function in order to use a MakoDescriptor'.format(system=system))
+        super(MakoModuleDescriptor, self).__init__(system, definition, **kwargs)
+
+    def get_context(self):
+        """
+        Return the context to render the mako template with
+        """
+        return {'module': self}
+
+    def get_html(self):
+        return self.system.render_template(self.mako_template, self.get_context())

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -0,0 +1,189 @@
+"""
+This module provides an abstraction for working with XModuleDescriptors
+that are stored in a database an accessible using their Location as an identifier
+"""
+
+import re
+from collections import namedtuple
+from .exceptions import InvalidLocationError
+
+URL_RE = re.compile("""
+    (?P<tag>[^:]+)://
+    (?P<org>[^/]+)/
+    (?P<course>[^/]+)/
+    (?P<category>[^/]+)/
+    (?P<name>[^/]+)
+    (/(?P<revision>[^/]+))?
+    """, re.VERBOSE)
+
+INVALID_CHARS = re.compile(r"[^\w.-]")
+
+_LocationBase = namedtuple('LocationBase', 'tag org course category name revision')
+class Location(_LocationBase):
+    '''
+    Encodes a location.
+
+    Locations representations of URLs of the
+    form {tag}://{org}/{course}/{category}/{name}[/{revision}]
+
+    However, they can also be represented a dictionaries (specifying each component),
+    tuples or list (specified in order), or as strings of the url
+    '''
+    __slots__ = ()
+
+    @classmethod
+    def clean(cls, value):
+        """
+        Return value, made into a form legal for locations
+        """
+        return re.sub('_+', '_', INVALID_CHARS.sub('_', value))
+
+    def __new__(_cls, loc_or_tag, org=None, course=None, category=None, name=None, revision=None):
+        """
+        Create a new location that is a clone of the specifed one.
+
+        location - Can be any of the following types:
+            string: should be of the form {tag}://{org}/{course}/{category}/{name}[/{revision}]
+            list: should be of the form [tag, org, course, category, name, revision]
+            dict: should be of the form {
+                'tag': tag,
+                'org': org,
+                'course': course,
+                'category': category,
+                'name': name,
+                'revision': revision,
+            }
+            Location: another Location object
+
+        In both the dict and list forms, the revision is optional, and can be ommitted.
+
+        Components must be composed of alphanumeric characters, or the characters '_', '-', and '.'
+
+        Components may be set to None, which may be interpreted by some contexts to mean
+        wildcard selection
+        """
+
+        if org is None and course is None and category is None and name is None and revision is None:
+            location = loc_or_tag
+        else:
+            location = (loc_or_tag, org, course, category, name, revision)
+
+        def check_dict(dict_):
+            check_list(dict_.values())
+
+        def check_list(list_):
+            for val in list_:
+                if val is not None and INVALID_CHARS.search(val) is not None:
+                    raise InvalidLocationError(location)
+
+        if isinstance(location, basestring):
+            match = URL_RE.match(location)
+            if match is None:
+                raise InvalidLocationError(location)
+            else:
+                groups = match.groupdict()
+                check_dict(groups)
+                return _LocationBase.__new__(_cls, **groups)
+        elif isinstance(location, (list, tuple)):
+            if len(location) not in (5, 6):
+                raise InvalidLocationError(location)
+
+            if len(location) == 5:
+                args = tuple(location) + (None, )
+            else:
+                args = tuple(location)
+
+            check_list(args)
+            return _LocationBase.__new__(_cls, *args)
+        elif isinstance(location, dict):
+            kwargs = dict(location)
+            kwargs.setdefault('revision', None)
+
+            check_dict(kwargs)
+            return _LocationBase.__new__(_cls, **kwargs)
+        elif isinstance(location, Location):
+            return _LocationBase.__new__(_cls, location)
+        else:
+            raise InvalidLocationError(location)
+
+    def url(self):
+        """
+        Return a string containing the URL for this location
+        """
+        url = "{tag}://{org}/{course}/{category}/{name}".format(**self.dict())
+        if self.revision:
+            url += "/" + self.revision
+        return url
+
+    def html_id(self):
+        """
+        Return a string with a version of the location that is safe for use in html id attributes
+        """
+        return "-".join(str(v) for v in self.list() if v is not None).replace('.', '_')
+
+    def dict(self):
+        return self._asdict()
+
+    def list(self):
+        return list(self)
+
+    def __str__(self):
+        return self.url()
+
+    def __repr__(self):
+        return "Location%s" % repr(tuple(self))
+
+
+class ModuleStore(object):
+    """
+    An abstract interface for a database backend that stores XModuleDescriptor instances
+    """
+    def get_item(self, location, default_class=None):
+        """
+        Returns an XModuleDescriptor instance for the item at location.
+        If location.revision is None, returns the item with the most
+        recent revision
+
+        If any segment of the location is None except revision, raises
+            xmodule.modulestore.exceptions.InsufficientSpecificationError
+        If no object is found at that location, raises xmodule.modulestore.exceptions.ItemNotFoundError
+
+        location: Something that can be passed to Location
+        default_class: An XModuleDescriptor subclass to use if no plugin matching the
+            location is found
+        """
+        raise NotImplementedError
+
+    # TODO (cpennington): Replace with clone_item
+    def create_item(self, location, editor):
+        raise NotImplementedError
+
+    def update_item(self, location, data):
+        """
+        Set the data in the item specified by the location to
+        data
+
+        location: Something that can be passed to Location
+        data: A nested dictionary of problem data
+        """
+        raise NotImplementedError
+
+    def update_children(self, location, children):
+        """
+        Set the children for the item specified by the location to
+        children
+
+        location: Something that can be passed to Location
+        children: A list of child item identifiers
+        """
+        raise NotImplementedError
+
+    def update_metadata(self, location, metadata):
+        """
+        Set the metadata for the item specified by the location to
+        metadata
+
+        location: Something that can be passed to Location
+        metadata: A nested dictionary of module metadata
+        """
+        raise NotImplementedError

common/lib/xmodule/xmodule/modulestore/django.py

@@ -0,0 +1,26 @@
+"""
+Module that provides a connection to the ModuleStore specified in the django settings.
+
+Passes settings.MODULESTORE as kwargs to MongoModuleStore
+"""
+
+from __future__ import absolute_import
+
+from importlib import import_module
+
+from django.conf import settings
+
+_MODULESTORES = {}
+
+
+def modulestore(name='default'):
+    global _MODULESTORES
+
+    if name not in _MODULESTORES:
+        class_path = settings.MODULESTORE[name]['ENGINE']
+        module_path, _, class_name = class_path.rpartition('.')
+        class_ = getattr(import_module(module_path), class_name)
+        _MODULESTORES[name] = class_(
+            **settings.MODULESTORE[name]['OPTIONS'])
+
+    return _MODULESTORES[name]

common/lib/xmodule/xmodule/modulestore/exceptions.py

@@ -0,0 +1,14 @@
+"""
+Exceptions thrown by KeyStore objects
+"""
+
+
+class ItemNotFoundError(Exception):
+    pass
+
+
+class InsufficientSpecificationError(Exception):
+    pass
+
+class InvalidLocationError(Exception):
+    pass

common/lib/xmodule/xmodule/modulestore/mongo.py

@@ -0,0 +1,117 @@
+import pymongo
+from importlib import import_module
+from xmodule.x_module import XModuleDescriptor
+from xmodule.mako_module import MakoDescriptorSystem
+from mitxmako.shortcuts import render_to_string
+
+from . import ModuleStore, Location
+from .exceptions import ItemNotFoundError, InsufficientSpecificationError
+
+
+class MongoModuleStore(ModuleStore):
+    """
+    A Mongodb backed ModuleStore
+    """
+    def __init__(self, host, db, collection, port=27017, default_class=None):
+        self.collection = pymongo.connection.Connection(
+            host=host,
+            port=port
+        )[db][collection]
+        self.collection.ensure_index('location')
+
+        # Force mongo to report errors, at the expense of performance
+        self.collection.safe = True
+
+        module_path, _, class_name = default_class.rpartition('.')
+        class_ = getattr(import_module(module_path), class_name)
+        self.default_class = class_
+
+    def get_item(self, location):
+        """
+        Returns an XModuleDescriptor instance for the item at location.
+        If location.revision is None, returns the most item with the most
+        recent revision
+
+        If any segment of the location is None except revision, raises
+            xmodule.modulestore.exceptions.InsufficientSpecificationError
+        If no object is found at that location, raises xmodule.modulestore.exceptions.ItemNotFoundError
+
+        location: Something that can be passed to Location
+        """
+
+        query = {}
+        for key, val in Location(location).dict().iteritems():
+            if key != 'revision' and val is None:
+                raise InsufficientSpecificationError(location)
+
+            if val is not None:
+                query['location.{key}'.format(key=key)] = val
+
+        item = self.collection.find_one(
+            query,
+            sort=[('revision', pymongo.ASCENDING)],
+        )
+        if item is None:
+            raise ItemNotFoundError(location)
+
+        # TODO (cpennington): Pass a proper resources_fs to the system
+        return XModuleDescriptor.load_from_json(
+            item, MakoDescriptorSystem(load_item=self.get_item, resources_fs=None, render_template=render_to_string), self.default_class)
+
+    def create_item(self, location):
+        """
+        Create an empty item at the specified location with the supplied editor
+
+        location: Something that can be passed to Location
+        """
+        self.collection.insert({
+            'location': Location(location).dict(),
+        })
+
+    def update_item(self, location, data):
+        """
+        Set the data in the item specified by the location to
+        data
+
+        location: Something that can be passed to Location
+        data: A nested dictionary of problem data
+        """
+
+        # See http://www.mongodb.org/display/DOCS/Updating for
+        # atomic update syntax
+        self.collection.update(
+            {'location': Location(location).dict()},
+            {'$set': {'definition.data': data}}
+        )
+
+    def update_children(self, location, children):
+        """
+        Set the children for the item specified by the location to
+        children
+
+        location: Something that can be passed to Location
+        children: A list of child item identifiers
+        """
+
+        # See http://www.mongodb.org/display/DOCS/Updating for
+        # atomic update syntax
+        self.collection.update(
+            {'location': Location(location).dict()},
+            {'$set': {'definition.children': children}}
+        )
+
+    def update_metadata(self, location, metadata):
+        """
+        Set the children for the item specified by the location to
+        metadata
+
+        location: Something that can be passed to Location
+        metadata: A nested dictionary of module metadata
+        """
+
+        # See http://www.mongodb.org/display/DOCS/Updating for
+        # atomic update syntax
+        self.collection.update(
+            {'location': Location(location).dict()},
+            {'$set': {'metadata': metadata}}
+        )

common/lib/xmodule/xmodule/modulestore/tests/test_location.py

@@ -0,0 +1,63 @@
+from nose.tools import assert_equals, assert_raises, assert_not_equals
+from xmodule.modulestore import Location
+from xmodule.modulestore.exceptions import InvalidLocationError
+
+
+def check_string_roundtrip(url):
+    assert_equals(url, Location(url).url())
+    assert_equals(url, str(Location(url)))
+
+
+def test_string_roundtrip():
+    check_string_roundtrip("tag://org/course/category/name")
+    check_string_roundtrip("tag://org/course/category/name/revision")
+
+
+def test_dict():
+    input_dict = {
+        'tag': 'tag',
+        'course': 'course',
+        'category': 'category',
+        'name': 'name',
+        'org': 'org'
+    }
+    assert_equals("tag://org/course/category/name", Location(input_dict).url())
+    assert_equals(dict(revision=None, **input_dict), Location(input_dict).dict())
+
+    input_dict['revision'] = 'revision'
+    assert_equals("tag://org/course/category/name/revision", Location(input_dict).url())
+    assert_equals(input_dict, Location(input_dict).dict())
+
+
+def test_list():
+    input_list = ['tag', 'org', 'course', 'category', 'name']
+    assert_equals("tag://org/course/category/name", Location(input_list).url())
+    assert_equals(input_list + [None], Location(input_list).list())
+
+    input_list.append('revision')
+    assert_equals("tag://org/course/category/name/revision", Location(input_list).url())
+    assert_equals(input_list, Location(input_list).list())
+
+
+def test_location():
+    input_list = ['tag', 'org', 'course', 'category', 'name']
+    assert_equals("tag://org/course/category/name", Location(Location(input_list)).url())
+
+
+def test_invalid_locations():
+    assert_raises(InvalidLocationError, Location, "foo")
+    assert_raises(InvalidLocationError, Location, ["foo", "bar"])
+    assert_raises(InvalidLocationError, Location, ["foo", "bar", "baz", "blat", "foo/bar"])
+    assert_raises(InvalidLocationError, Location, None)
+    assert_raises(InvalidLocationError, Location, "tag://org/course/category/name with spaces/revision")
+
+def test_equality():
+    assert_equals(
+        Location('tag', 'org', 'course', 'category', 'name'),
+        Location('tag', 'org', 'course', 'category', 'name')
+    )
+
+    assert_not_equals(
+        Location('tag', 'org', 'course', 'category', 'name1'),
+        Location('tag', 'org', 'course', 'category', 'name')
+    )

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -0,0 +1,137 @@
+import logging
+from fs.osfs import OSFS
+from importlib import import_module
+from lxml import etree
+from path import path
+from xmodule.x_module import XModuleDescriptor, XMLParsingSystem
+from xmodule.mako_module import MakoDescriptorSystem
+
+from . import ModuleStore, Location
+from .exceptions import ItemNotFoundError
+
+etree.set_default_parser(etree.XMLParser(dtd_validation=False, load_dtd=False,
+                                         remove_comments=True, remove_blank_text=True))
+
+log = logging.getLogger(__name__)
+
+
+class XMLModuleStore(ModuleStore):
+    """
+    An XML backed ModuleStore
+    """
+    def __init__(self, org, course, data_dir, default_class=None, eager=False):
+        """
+        Initialize an XMLModuleStore from data_dir
+
+        org, course: Strings to be used in module keys
+        data_dir: path to data directory containing course.xml
+        default_class: dot-separated string defining the default descriptor class to use if non is specified in entry_points
+        eager: If true, load the modules children immediately to force the entire course tree to be parsed
+        """
+        self.data_dir = path(data_dir)
+        self.modules = {}
+
+        if default_class is None:
+            self.default_class = None
+        else:
+            module_path, _, class_name = default_class.rpartition('.')
+            class_ = getattr(import_module(module_path), class_name)
+            self.default_class = class_
+
+        with open(self.data_dir / "course.xml") as course_file:
+            class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
+                def __init__(self, modulestore):
+                    """
+                    modulestore: the XMLModuleStore to store the loaded modules in
+                    """
+                    self.unnamed_modules = 0
+                    self.used_slugs = set()
+
+                    def process_xml(xml):
+                        try:
+                            xml_data = etree.fromstring(xml)
+                        except:
+                            log.exception("Unable to parse xml: {xml}".format(xml=xml))
+                            raise
+                        if xml_data.get('slug') is None:
+                            if xml_data.get('name'):
+                                slug = Location.clean(xml_data.get('name'))
+                            else:
+                                self.unnamed_modules += 1
+                                slug = '{tag}_{count}'.format(tag=xml_data.tag, count=self.unnamed_modules)
+
+                            if slug in self.used_slugs:
+                                self.unnamed_modules += 1
+                                slug = '{slug}_{count}'.format(slug=slug, count=self.unnamed_modules)
+
+                            self.used_slugs.add(slug)
+                            xml_data.set('slug', slug)
+
+                        module = XModuleDescriptor.load_from_xml(etree.tostring(xml_data), self, org, course, modulestore.default_class)
+                        modulestore.modules[module.location] = module
+
+                        if eager:
+                            module.get_children()
+                        return module
+
+                    system_kwargs = dict(
+                        render_template=lambda: '',
+                        load_item=modulestore.get_item,
+                        resources_fs=OSFS(data_dir),
+                        process_xml=process_xml
+                    )
+                    MakoDescriptorSystem.__init__(self, **system_kwargs)
+                    XMLParsingSystem.__init__(self, **system_kwargs)
+
+            self.course = ImportSystem(self).process_xml(course_file.read())
+
+    def get_item(self, location):
+        """
+        Returns an XModuleDescriptor instance for the item at location.
+        If location.revision is None, returns the most item with the most
+        recent revision
+
+        If any segment of the location is None except revision, raises
+            xmodule.modulestore.exceptions.InsufficientSpecificationError
+        If no object is found at that location, raises xmodule.modulestore.exceptions.ItemNotFoundError
+
+        location: Something that can be passed to Location
+        """
+        location = Location(location)
+        try:
+            return self.modules[location]
+        except KeyError:
+            raise ItemNotFoundError(location)
+
+    def create_item(self, location):
+        raise NotImplementedError("XMLModuleStores are read-only")
+
+    def update_item(self, location, data):
+        """
+        Set the data in the item specified by the location to
+        data
+
+        location: Something that can be passed to Location
+        data: A nested dictionary of problem data
+        """
+        raise NotImplementedError("XMLModuleStores are read-only")
+
+    def update_children(self, location, children):
+        """
+        Set the children for the item specified by the location to
+        data
+
+        location: Something that can be passed to Location
+        children: A list of child item identifiers
+        """
+        raise NotImplementedError("XMLModuleStores are read-only")
+
+    def update_metadata(self, location, metadata):
+        """
+        Set the metadata for the item specified by the location to
+        metadata
+
+        location: Something that can be passed to Location
+        metadata: A nested dictionary of module metadata
+        """
+        raise NotImplementedError("XMLModuleStores are read-only")

common/lib/xmodule/xmodule/progress.py

@@ -0,0 +1,160 @@
+'''
+Progress class for modules.  Represents where a student is in a module.
+
+Useful things to know:
+ - Use Progress.to_js_status_str() to convert a progress into a simple
+   status string to pass to js.  
+ - Use Progress.to_js_detail_str() to convert a progress into a more detailed
+   string to pass to js.
+
+In particular, these functions have a canonical handing of None.
+
+For most subclassing needs, you should only need to reimplement
+frac() and __str__().
+''' 
+
+from collections import namedtuple
+import numbers
+
+class Progress(object):
+    '''Represents a progress of a/b (a out of b done)
+
+    a and b must be numeric, but not necessarily integer, with
+    0 <= a <= b and b > 0.
+
+    Progress can only represent Progress for modules where that makes sense.  Other
+    modules (e.g. html) should return None from get_progress().
+
+    TODO: add tag for module type?  Would allow for smarter merging.
+    '''
+
+    def __init__(self, a, b):
+        '''Construct a Progress object.  a and b must be numbers, and must have
+              0 <= a <= b and b > 0
+        '''
+
+        # Want to do all checking at construction time, so explicitly check types
+        if not (isinstance(a, numbers.Number) and
+                isinstance(b, numbers.Number)):
+            raise TypeError('a and b must be numbers.  Passed {0}/{1}'.format(a, b))
+        
+        if not (0 <= a <= b and b > 0):
+            raise ValueError(
+                'fraction a/b = {0}/{1} must have 0 <= a <= b and b > 0'.format(a, b))
+
+        self._a = a
+        self._b = b
+
+    def frac(self):
+        ''' Return tuple (a,b) representing progress of a/b'''
+        return (self._a, self._b)
+
+    def percent(self):
+        ''' Returns a percentage progress as a float between 0 and 100.
+
+        subclassing note: implemented in terms of frac(), assumes sanity
+        checking is done at construction time.
+        '''
+        (a, b) = self.frac()
+        return 100.0 * a / b
+
+    def started(self):
+        ''' Returns True if fractional progress is greater than 0.
+
+        subclassing note: implemented in terms of frac(), assumes sanity
+        checking is done at construction time.
+        '''
+        return self.frac()[0] > 0
+
+
+    def inprogress(self):
+        ''' Returns True if fractional progress is strictly between 0 and 1.
+
+        subclassing note: implemented in terms of frac(), assumes sanity
+        checking is done at construction time.
+        ''' 
+        (a, b) = self.frac()
+        return a > 0 and a < b
+
+    def done(self):
+        ''' Return True if this represents done.
+
+        subclassing note: implemented in terms of frac(), assumes sanity
+        checking is done at construction time.
+        '''
+        (a, b) = self.frac()
+        return a==b
+
+
+    def ternary_str(self):
+        ''' Return a string version of this progress: either
+           "none", "in_progress", or "done".
+
+        subclassing note: implemented in terms of frac()
+        ''' 
+        (a, b) = self.frac()
+        if a == 0:
+            return "none"
+        if a < b:
+            return "in_progress"
+        return "done"
+
+    def __eq__(self, other):
+        ''' Two Progress objects are equal if they have identical values.
+        Implemented in terms of frac()'''
+        if not isinstance(other, Progress):
+            return False
+        (a, b) = self.frac()
+        (a2, b2) = other.frac()
+        return a == a2 and b == b2
+
+    def __ne__(self, other):
+        ''' The opposite of equal'''
+        return not self.__eq__(other)
+        
+    
+    def __str__(self):
+        ''' Return a string representation of this string.
+
+        subclassing note: implemented in terms of frac().
+        '''
+        (a, b) = self.frac()
+        return "{0}/{1}".format(a, b)
+
+    @staticmethod
+    def add_counts(a, b):
+        '''Add two progress indicators, assuming that each represents items done:
+        (a / b) + (c / d) = (a + c) / (b + d).
+        If either is None, returns the other.
+        '''
+        if a is None:
+            return b
+        if b is None:
+            return a
+        # get numerators + denominators
+        (n, d) = a.frac()
+        (n2, d2) = b.frac()
+        return Progress(n + n2, d + d2)
+
+    @staticmethod
+    def to_js_status_str(progress):
+        '''
+        Return the "status string" version of the passed Progress
+        object that should be passed to js.  Use this function when
+        sending Progress objects to js to limit dependencies.
+        '''
+        if progress is None:
+            return "NA"
+        return progress.ternary_str()
+
+
+    @staticmethod
+    def to_js_detail_str(progress):
+        '''
+        Return the "detail string" version of the passed Progress
+        object that should be passed to js.  Use this function when
+        passing Progress objects to js to limit dependencies.
+        '''
+        if progress is None:
+            return "NA"
+        return str(progress)

common/lib/xmodule/xmodule/raw_module.py

@@ -0,0 +1,27 @@
+from pkg_resources import resource_string
+from lxml import etree
+from xmodule.mako_module import MakoModuleDescriptor
+from xmodule.xml_module import XmlDescriptor
+
+
+class RawDescriptor(MakoModuleDescriptor, XmlDescriptor):
+    """
+    Module that provides a raw editing view of it's data and children
+    """
+    mako_template = "widgets/raw-edit.html"
+
+    js = {'coffee': [resource_string(__name__, 'js/module/raw.coffee')]}
+    js_module = 'Raw'
+
+    def get_context(self):
+        return {
+            'module': self,
+            'data': self.definition['data'],
+        }
+
+    @classmethod
+    def definition_from_xml(cls, xml_object, system):
+        return {'data': etree.tostring(xml_object)}
+
+    def definition_to_xml(self, resource_fs):
+        return etree.fromstring(self.definition['data'])

common/lib/xmodule/xmodule/schematic_module.py

@@ -6,18 +6,5 @@ class ModuleDescriptor(XModuleDescriptor):
     pass
 
 class Module(XModule):
-    id_attribute = 'id'
-
-    def get_state(self):
-        return json.dumps({ })
-
-    @classmethod
-    def get_xml_tags(c):
-        return ["schematic"]
-        
     def get_html(self):
         return '<input type="hidden" class="schematic" name="{item_id}" height="480" width="640">'.format(item_id=self.item_id)
-
-    def __init__(self, system, xml, item_id, state=None):
-        XModule.__init__(self, system, xml, item_id, state)
-

common/lib/xmodule/xmodule/seq_module.py

@@ -0,0 +1,116 @@
+import json
+import logging
+
+from lxml import etree
+
+from xmodule.mako_module import MakoModuleDescriptor
+from xmodule.xml_module import XmlDescriptor
+from xmodule.x_module import XModule
+from xmodule.progress import Progress
+
+log = logging.getLogger("mitx.common.lib.seq_module")
+
+# HACK: This shouldn't be hard-coded to two types
+# OBSOLETE: This obsoletes 'type'
+class_priority = ['video', 'problem']
+
+
+class SequenceModule(XModule):
+    ''' Layout module which lays out content in a temporal sequence
+    '''
+
+    def __init__(self, system, location, definition, instance_state=None, shared_state=None, **kwargs):
+        XModule.__init__(self, system, location, definition, instance_state, shared_state, **kwargs)
+        self.position = 1
+
+        if instance_state is not None:
+            state = json.loads(instance_state)
+            if 'position' in state:
+                self.position = int(state['position'])
+
+        # if position is specified in system, then use that instead
+        if system.get('position'):
+            self.position = int(system.get('position'))
+
+        self.rendered = False
+
+    def get_instance_state(self):
+        return json.dumps({'position': self.position})
+
+    def get_html(self):
+        self.render()
+        return self.content
+
+    def get_progress(self):
+        ''' Return the total progress, adding total done and total available.
+        (assumes that each submodule uses the same "units" for progress.)
+        '''
+        # TODO: Cache progress or children array?
+        children = self.get_children()
+        progresses = [child.get_progress() for child in children]
+        progress = reduce(Progress.add_counts, progresses)
+        return progress
+
+    def handle_ajax(self, dispatch, get):		# TODO: bounds checking
+        ''' get = request.POST instance '''
+        if dispatch=='goto_position':
+            self.position = int(get['position'])
+            return json.dumps({'success':True})
+        raise self.system.exception404
+
+    def render(self):
+        if self.rendered:
+            return
+        ## Returns a set of all types of all sub-children
+        contents = []
+        for child in self.get_display_items():
+            progress = child.get_progress()
+            contents.append({
+                'content': child.get_html(),
+                'title': "\n".join(
+                    grand_child.metadata['display_name'].strip()
+                    for grand_child in child.get_children()
+                    if 'display_name'  in grand_child.metadata
+                ),
+                'progress_status': Progress.to_js_status_str(progress),
+                'progress_detail': Progress.to_js_detail_str(progress),
+                'type': child.get_icon_class(),
+            })
+
+        # Split </script> tags -- browsers handle this as end
+        # of script, even if it occurs mid-string. Do this after json.dumps()ing
+        # so that we can be sure of the quotations being used
+        params = {'items': json.dumps(contents).replace('</script>', '<"+"/script>'),
+                  'element_id': self.location.html_id(),
+                  'item_id': self.id,
+                  'position': self.position,
+                  'tag': self.location.category}
+
+        self.content = self.system.render_template('seq_module.html', params)
+        self.rendered = True
+
+    def get_icon_class(self):
+        child_classes = set(child.get_icon_class() for child in self.get_children())
+        new_class = 'other'
+        for c in class_priority:
+            if c in child_classes:
+                new_class = c
+        return new_class
+
+
+class SequenceDescriptor(MakoModuleDescriptor, XmlDescriptor):
+    mako_template = 'widgets/sequence-edit.html'
+    module_class = SequenceModule
+
+    @classmethod
+    def definition_from_xml(cls, xml_object, system):
+        return {'children': [
+            system.process_xml(etree.tostring(child_module)).location.url()
+            for child_module in xml_object
+        ]}
+
+    def definition_to_xml(self, resource_fs):
+        xml_object = etree.Element('sequential')
+        for child in self.get_children():
+            xml_object.append(etree.fromstring(child.export_to_xml(resource_fs)))
+        return xml_object

common/lib/xmodule/xmodule/template_module.py

@@ -1,14 +1,9 @@
-import json
-
-from x_module import XModule, XModuleDescriptor
+from xmodule.x_module import XModule
+from xmodule.raw_module import RawDescriptor
 from lxml import etree
 
 
-class ModuleDescriptor(XModuleDescriptor):
-    pass
-
-
-class Module(XModule):
+class CustomTagModule(XModule):
     """
     This module supports tags of the form
     <customtag option="val" option2="val2">
@@ -31,19 +26,17 @@ class Module(XModule):
     Renders to::
         More information given in <a href="/book/234">the text</a>
     """
-    def get_state(self):
-        return json.dumps({})
 
-    @classmethod
-    def get_xml_tags(c):
-        return ['customtag']
+    def __init__(self, system, location, definition, instance_state=None, shared_state=None, **kwargs):
+        XModule.__init__(self, system, location, definition, instance_state, shared_state, **kwargs)
+        xmltree = etree.fromstring(self.definition['data'])
+        filename = xmltree.find('impl').text
+        params = dict(xmltree.items())
+        self.html = self.system.render_template(filename, params, namespace='custom_tags')
 
     def get_html(self):
         return self.html
 
-    def __init__(self, system, xml, item_id, state=None):
-        XModule.__init__(self, system, xml, item_id, state)
-        xmltree = etree.fromstring(xml)
-        filename = xmltree.find('impl').text
-        params = dict(xmltree.items())
-        self.html = self.system.render_template(filename, params, namespace='custom_tags')
+
+class CustomTagDescriptor(RawDescriptor):
+    module_class = CustomTagModule

common/lib/xmodule/xmodule/translation_module.py

@@ -0,0 +1,82 @@
+"""
+These modules exist to translate old format XML into newer, semantic forms
+"""
+from x_module import XModuleDescriptor
+from lxml import etree
+from functools import wraps
+import logging
+
+log = logging.getLogger(__name__)
+
+
+def process_includes(fn):
+    """
+    Wraps a XModuleDescriptor.from_xml method, and modifies xml_data to replace
+    any immediate child <include> items with the contents of the file that they are
+    supposed to include
+    """
+    @wraps(fn)
+    def from_xml(cls, xml_data, system, org=None, course=None):
+        xml_object = etree.fromstring(xml_data)
+        next_include = xml_object.find('include')
+        while next_include is not None:
+            file = next_include.get('file')
+            if file is not None:
+                try:
+                    ifp = system.resources_fs.open(file)
+                except Exception:
+                    log.exception('Error in problem xml include: %s' % (etree.tostring(next_include, pretty_print=True)))
+                    log.exception('Cannot find file %s in %s' % (file, dir))
+                    raise
+                try:
+                    # read in and convert to XML
+                    incxml = etree.XML(ifp.read())
+                except Exception:
+                    log.exception('Error in problem xml include: %s' % (etree.tostring(next_include, pretty_print=True)))
+                    log.exception('Cannot parse XML in %s' % (file))
+                    raise
+                # insert  new XML into tree in place of inlcude
+                parent = next_include.getparent()
+                parent.insert(parent.index(next_include), incxml)
+            parent.remove(next_include)
+
+            next_include = xml_object.find('include')
+        return fn(cls, etree.tostring(xml_object), system, org, course)
+    return from_xml
+
+
+class SemanticSectionDescriptor(XModuleDescriptor):
+    @classmethod
+    @process_includes
+    def from_xml(cls, xml_data, system, org=None, course=None):
+        """
+        Removes sections single child elements in favor of just embedding the child element
+
+        """
+        xml_object = etree.fromstring(xml_data)
+
+        if len(xml_object) == 1:
+            for (key, val) in xml_object.items():
+                xml_object[0].set(key, val)
+
+            return system.process_xml(etree.tostring(xml_object[0]))
+        else:
+            xml_object.tag = 'sequence'
+            return system.process_xml(etree.tostring(xml_object))
+
+
+class TranslateCustomTagDescriptor(XModuleDescriptor):
+    @classmethod
+    def from_xml(cls, xml_data, system, org=None, course=None):
+        """
+        Transforms the xml_data from <$custom_tag attr="" attr=""/> to
+        <customtag attr="" attr=""><impl>$custom_tag</impl></customtag>
+        """
+
+        xml_object = etree.fromstring(xml_data)
+        tag = xml_object.tag
+        xml_object.tag = 'customtag'
+        impl = etree.SubElement(xml_object, 'impl')
+        impl.text = tag
+
+        return system.process_xml(etree.tostring(xml_object))

common/lib/xmodule/xmodule/vertical_module.py

@@ -0,0 +1,42 @@
+from xmodule.x_module import XModule
+from xmodule.seq_module import SequenceDescriptor
+from xmodule.progress import Progress
+
+# HACK: This shouldn't be hard-coded to two types
+# OBSOLETE: This obsoletes 'type'
+class_priority = ['video', 'problem']
+
+
+class VerticalModule(XModule):
+    ''' Layout module for laying out submodules vertically.'''
+
+    def __init__(self, system, location, definition, instance_state=None, shared_state=None, **kwargs):
+        XModule.__init__(self, system, location, definition, instance_state, shared_state, **kwargs)
+        self.contents = None
+
+    def get_html(self):
+        if self.contents is None:
+            self.contents = [child.get_html() for child in self.get_display_items()]
+
+        return self.system.render_template('vert_module.html', {
+            'items': self.contents
+        })
+
+    def get_progress(self):
+        # TODO: Cache progress or children array?
+        children = self.get_children()
+        progresses = [child.get_progress() for child in children]
+        progress = reduce(Progress.add_counts, progresses)
+        return progress
+
+    def get_icon_class(self):
+        child_classes = set(child.get_icon_class() for child in self.get_children())
+        new_class = 'other'
+        for c in class_priority:
+            if c in child_classes:
+                new_class = c
+        return new_class
+
+
+class VerticalDescriptor(SequenceDescriptor):
+    module_class = VerticalModule

common/lib/xmodule/xmodule/video_module.py

@@ -3,17 +3,27 @@ import logging
 
 from lxml import etree
 
-from x_module import XModule, XModuleDescriptor
-from progress import Progress
+from xmodule.x_module import XModule
+from xmodule.raw_module import RawDescriptor
 
-log = logging.getLogger("mitx.courseware.modules")
+log = logging.getLogger(__name__)
 
-class ModuleDescriptor(XModuleDescriptor):
-    pass
 
-class Module(XModule):
-    id_attribute = 'youtube'
+class VideoModule(XModule):
     video_time = 0
+    icon_class = 'video'
+
+    def __init__(self, system, location, definition, instance_state=None, shared_state=None, **kwargs):
+        XModule.__init__(self, system, location, definition, instance_state, shared_state, **kwargs)
+        xmltree = etree.fromstring(self.definition['data'])
+        self.youtube = xmltree.get('youtube')
+        self.name = xmltree.get('name')
+        self.position = 0
+
+        if instance_state is not None:
+            state = json.loads(instance_state)
+            if 'position' in state:
+                self.position = int(float(state['position']))
 
     def handle_ajax(self, dispatch, get):
         '''
@@ -39,14 +49,9 @@ class Module(XModule):
         '''
         return None
 
-    def get_state(self):
+    def get_instance_state(self):
         log.debug(u"STATE POSITION {0}".format(self.position))
-        return json.dumps({ 'position': self.position })
-
-    @classmethod
-    def get_xml_tags(c):
-        '''Tags in the courseware file guaranteed to correspond to the module'''
-        return ["video"]
+        return json.dumps({'position': self.position})
 
     def video_list(self):
         return self.youtube
@@ -54,27 +59,11 @@ class Module(XModule):
     def get_html(self):
         return self.system.render_template('video.html', {
             'streams': self.video_list(),
-            'id': self.item_id,
+            'id': self.location.html_id(),
             'position': self.position,
             'name': self.name,
-            'annotations': self.annotations,
         })
 
-    def __init__(self, system, xml, item_id, state=None):
-        XModule.__init__(self, system, xml, item_id, state)
-        xmltree = etree.fromstring(xml)
-        self.youtube = xmltree.get('youtube')
-        self.name = xmltree.get('name')
-        self.position = 0
 
-        if state is not None:
-            state = json.loads(state)
-            if 'position' in state:
-                self.position = int(float(state['position']))
-
-        self.annotations=[(e.get("name"),self.render_function(e)) \
-                      for e in xmltree]
-
-
-class VideoSegmentDescriptor(XModuleDescriptor):
-    pass
+class VideoDescriptor(RawDescriptor):
+    module_class = VideoModule

common/lib/xmodule/xmodule/x_module.py

@@ -0,0 +1,442 @@
+from lxml import etree
+import pkg_resources
+import logging
+
+from xmodule.modulestore import Location
+from functools import partial
+
+log = logging.getLogger('mitx.' + __name__)
+
+def dummy_track(event_type, event):
+    pass
+
+
+class ModuleMissingError(Exception):
+    pass
+
+
+class Plugin(object):
+    """
+    Base class for a system that uses entry_points to load plugins.
+
+    Implementing classes are expected to have the following attributes:
+
+        entry_point: The name of the entry point to load plugins from
+    """
+
+    _plugin_cache = None
+
+    @classmethod
+    def load_class(cls, identifier, default=None):
+        """
+        Loads a single class instance specified by identifier. If identifier
+        specifies more than a single class, then logs a warning and returns the first
+        class identified.
+
+        If default is not None, will return default if no entry_point matching identifier
+        is found. Otherwise, will raise a ModuleMissingError
+        """
+        if cls._plugin_cache is None:
+            cls._plugin_cache = {}
+
+        if identifier not in cls._plugin_cache:
+            identifier = identifier.lower()
+            classes = list(pkg_resources.iter_entry_points(cls.entry_point, name=identifier))
+            if len(classes) > 1:
+                log.warning("Found multiple classes for {entry_point} with identifier {id}: {classes}. Returning the first one.".format(
+                    entry_point=cls.entry_point,
+                    id=identifier,
+                    classes=", ".join(class_.module_name for class_ in classes)))
+
+            if len(classes) == 0:
+                if default is not None:
+                    return default
+                raise ModuleMissingError(identifier)
+
+            cls._plugin_cache[identifier] = classes[0].load()
+        return cls._plugin_cache[identifier]
+
+    @classmethod
+    def load_classes(cls):
+        return [class_.load()
+                for class_
+                in pkg_resources.iter_entry_points(cls.entry_point)]
+
+
+class XModule(object):
+    ''' Implements a generic learning module.
+
+        Subclasses must at a minimum provide a definition for get_html in order to be displayed to users.
+
+        See the HTML module for a simple example.
+    '''
+
+    # The default implementation of get_icon_class returns the icon_class attribute of the class
+    # This attribute can be overridden by subclasses, and the function can also be overridden
+    # if the icon class depends on the data in the module
+    icon_class = 'other'
+
+    def __init__(self, system, location, definition, instance_state=None, shared_state=None, **kwargs):
+        '''
+        Construct a new xmodule
+
+        system: An I4xSystem allowing access to external resources
+        location: Something Location-like that identifies this xmodule
+        definition: A dictionary containing 'data' and 'children'. Both are optional
+            'data': is JSON-like (string, dictionary, list, bool, or None, optionally nested).
+                This defines all of the data necessary for a problem to display that is intrinsic to the problem.
+                It should not include any data that would vary between two courses using the same problem
+                (due dates, grading policy, randomization, etc.)
+            'children': is a list of Location-like values for child modules that this module depends on
+        instance_state: A string of serialized json that contains the state of this module for
+            current student accessing the system, or None if no state has been saved
+        shared_state: A string of serialized json that contains the state that is shared between
+            this module and any modules of the same type with the same shared_state_key. This
+            state is only shared per-student, not across different students
+        kwargs: Optional arguments. Subclasses should always accept kwargs and pass them
+            to the parent class constructor.
+            Current known uses of kwargs:
+                metadata: A dictionary containing data that specifies information that is particular
+                    to a problem in the context of a course
+        '''
+        self.system = system
+        self.location = Location(location)
+        self.definition = definition
+        self.instance_state = instance_state
+        self.shared_state = shared_state
+        self.id = self.location.url()
+        self.name = self.location.name
+        self.category = self.location.category
+        self.metadata = kwargs.get('metadata', {})
+        self._loaded_children = None
+
+    def get_name(self):
+        name = self.__xmltree.get('name')
+        if name:
+            return name
+        else:
+            raise "We should iterate through children and find a default name"
+
+    def get_children(self):
+        '''
+        Return module instances for all the children of this module.
+        '''
+        if self._loaded_children is None:
+            self._loaded_children = [self.system.get_module(child) for child in self.definition.get('children', [])]
+        return self._loaded_children
+
+    def get_display_items(self):
+        '''
+        Returns a list of descendent module instances that will display immediately
+        inside this module
+        '''
+        items = []
+        for child in self.get_children():
+            items.extend(child.displayable_items())
+
+        return items
+
+    def displayable_items(self):
+        '''
+        Returns list of displayable modules contained by this module. If this module
+        is visible, should return [self]
+        '''
+        return [self]
+
+    def get_icon_class(self):
+        '''
+        Return a css class identifying this module in the context of an icon
+        '''
+        return self.icon_class
+
+    ### Functions used in the LMS
+
+    def get_instance_state(self):
+        ''' State of the object, as stored in the database
+        '''
+        return '{}'
+
+    def get_shared_state(self):
+        '''
+        Get state that should be shared with other instances
+        using the same 'shared_state_key' attribute.
+        '''
+        return '{}'
+
+    def get_score(self):
+        ''' Score the student received on the problem.
+        '''
+        return None
+
+    def max_score(self):
+        ''' Maximum score. Two notes: 
+            * This is generic; in abstract, a problem could be 3/5 points on one randomization, and 5/7 on another
+            * In practice, this is a Very Bad Idea, and (a) will break some code in place (although that code
+              should get fixed), and (b) break some analytics we plan to put in place. 
+        ''' 
+        return None
+
+    def get_html(self):
+        ''' HTML, as shown in the browser. This is the only method that must be implemented
+        '''
+        raise NotImplementedError("get_html must be defined for all XModules that appear on the screen. Not defined in %s" % self.__class__.__name__)
+
+    def get_progress(self):
+        ''' Return a progress.Progress object that represents how far the student has gone
+        in this module.  Must be implemented to get correct progress tracking behavior in
+        nesting modules like sequence and vertical.
+
+        If this module has no notion of progress, return None.
+        '''
+        return None
+
+    def handle_ajax(self, dispatch, get):
+        ''' dispatch is last part of the URL. 
+            get is a dictionary-like object ''' 
+        return ""
+
+
+class XModuleDescriptor(Plugin):
+    """
+    An XModuleDescriptor is a specification for an element of a course. This could
+    be a problem, an organizational element (a group of content), or a segment of video,
+    for example.
+
+    XModuleDescriptors are independent and agnostic to the current student state on a
+    problem. They handle the editing interface used by instructors to create a problem,
+    and can generate XModules (which do know about student state).
+    """
+    entry_point = "xmodule.v1"
+    js = {}
+    js_module = None
+
+    # A list of metadata that this module can inherit from its parent module
+    inheritable_metadata = ('graded', 'due', 'graceperiod', 'showanswer', 'rerandomize')
+
+    # A list of descriptor attributes that must be equal for the discriptors to be
+    # equal
+    equality_attributes = ('definition', 'metadata', 'location', 'shared_state_key', '_inherited_metadata')
+
+    # ============================= STRUCTURAL MANIPULATION ===========================
+    def __init__(self,
+                 system,
+                 definition=None,
+                 **kwargs):
+        """
+        Construct a new XModuleDescriptor. The only required arguments are the
+        system, used for interaction with external resources, and the definition,
+        which specifies all the data needed to edit and display the problem (but none
+        of the associated metadata that handles recordkeeping around the problem).
+
+        This allows for maximal flexibility to add to the interface while preserving
+        backwards compatibility.
+
+        system: An XModuleSystem for interacting with external resources
+        definition: A dict containing `data` and `children` representing the problem definition
+
+        Current arguments passed in kwargs:
+            location: A xmodule.modulestore.Location object indicating the name and ownership of this problem
+            shared_state_key: The key to use for sharing StudentModules with other
+                modules of this type
+            metadata: A dictionary containing the following optional keys:
+                goals: A list of strings of learning goals associated with this module
+                display_name: The name to use for displaying this module to the user
+                format: The format of this module ('Homework', 'Lab', etc)
+                graded (bool): Whether this module is should be graded or not
+                due (string): The due date for this module
+                graceperiod (string): The amount of grace period to allow when enforcing the due date
+                showanswer (string): When to show answers for this module
+                rerandomize (string): When to generate a newly randomized instance of the module data
+        """
+        self.system = system
+        self.definition = definition if definition is not None else {}
+        self.name = Location(kwargs.get('location')).name
+        self.category = Location(kwargs.get('location')).category
+        self.location = Location(kwargs.get('location'))
+        self.metadata = kwargs.get('metadata', {})
+        self.shared_state_key = kwargs.get('shared_state_key')
+
+        self._child_instances = None
+        self._inherited_metadata = set()
+
+    def inherit_metadata(self, metadata):
+        """
+        Updates this module with metadata inherited from a containing module.
+        Only metadata specified in self.inheritable_metadata will
+        be inherited
+        """
+        # Set all inheritable metadata from kwargs that are
+        # in self.inheritable_metadata and aren't already set in metadata
+        for attr in self.inheritable_metadata:
+            if attr not in self.metadata and attr in metadata:
+                self._inherited_metadata.add(attr)
+                self.metadata[attr] = metadata[attr]
+
+    def get_children(self):
+        """Returns a list of XModuleDescriptor instances for the children of this module"""
+        if self._child_instances is None:
+            self._child_instances = []
+            for child_loc in self.definition.get('children', []):
+                child = self.system.load_item(child_loc)
+                child.inherit_metadata(self.metadata)
+                self._child_instances.append(child)
+
+        return self._child_instances
+
+    def xmodule_constructor(self, system):
+        """
+        Returns a constructor for an XModule. This constructor takes two arguments:
+        instance_state and shared_state, and returns a fully nstantiated XModule
+        """
+        return partial(
+            self.module_class,
+            system,
+            self.location,
+            self.definition,
+            metadata=self.metadata
+        )
+
+    # ================================= JSON PARSING ===================================
+    @staticmethod
+    def load_from_json(json_data, system, default_class=None):
+        """
+        This method instantiates the correct subclass of XModuleDescriptor based
+        on the contents of json_data.
+
+        json_data must contain a 'location' element, and must be suitable to be
+        passed into the subclasses `from_json` method.
+        """
+        class_ = XModuleDescriptor.load_class(
+            json_data['location']['category'],
+            default_class
+        )
+        return class_.from_json(json_data, system)
+
+    @classmethod
+    def from_json(cls, json_data, system):
+        """
+        Creates an instance of this descriptor from the supplied json_data.
+        This may be overridden by subclasses
+
+        json_data: A json object specifying the definition and any optional keyword arguments for
+            the XModuleDescriptor
+        system: An XModuleSystem for interacting with external resources
+        """
+        return cls(system=system, **json_data)
+
+    # ================================= XML PARSING ====================================
+    @staticmethod
+    def load_from_xml(xml_data,
+            system,
+            org=None,
+            course=None,
+            default_class=None):
+        """
+        This method instantiates the correct subclass of XModuleDescriptor based
+        on the contents of xml_data.
+
+        xml_data must be a string containing valid xml
+        system is an XMLParsingSystem
+        org and course are optional strings that will be used in the generated modules
+            url identifiers
+        """
+        class_ = XModuleDescriptor.load_class(
+            etree.fromstring(xml_data).tag,
+            default_class
+        )
+        return class_.from_xml(xml_data, system, org, course)
+
+    @classmethod
+    def from_xml(cls, xml_data, system, org=None, course=None):
+        """
+        Creates an instance of this descriptor from the supplied xml_data.
+        This may be overridden by subclasses
+
+        xml_data: A string of xml that will be translated into data and children for
+            this module
+        system is an XMLParsingSystem
+        org and course are optional strings that will be used in the generated modules
+            url identifiers
+        """
+        raise NotImplementedError('Modules must implement from_xml to be parsable from xml')
+
+    def export_to_xml(self, resource_fs):
+        """
+        Returns an xml string representing this module, and all modules underneath it.
+        May also write required resources out to resource_fs
+
+        Assumes that modules have single parantage (that no module appears twice in the same course),
+        and that it is thus safe to nest modules as xml children as appropriate.
+
+        The returned XML should be able to be parsed back into an identical XModuleDescriptor
+        using the from_xml method with the same system, org, and course
+        """
+        raise NotImplementedError('Modules must implement export_to_xml to enable xml export')
+
+    # ================================== HTML INTERFACE DEFINITIONS ======================
+    @classmethod
+    def get_javascript(cls):
+        """
+        Return a dictionary containing some of the following keys:
+            coffee: A list of coffeescript fragments that should be compiled and
+                    placed on the page
+            js: A list of javascript fragments that should be included on the page
+
+        All of these will be loaded onto the page in the CMS
+        """
+        return cls.js
+
+    def js_module_name(self):
+        """
+        Return the name of the javascript class to instantiate when
+        this module descriptor is loaded for editing
+        """
+        return self.js_module
+
+    def get_html(self):
+        """
+        Return the html used to edit this module
+        """
+        raise NotImplementedError("get_html() must be provided by specific modules")
+
+    # =============================== BUILTIN METHODS ===========================
+    def __eq__(self, other):
+        eq = (self.__class__ == other.__class__ and
+                all(getattr(self, attr, None) == getattr(other, attr, None)
+                    for attr in self.equality_attributes))
+
+        if not eq:
+            for attr in self.equality_attributes:
+                print getattr(self, attr, None), getattr(other, attr, None), getattr(self, attr, None) == getattr(other, attr, None)
+
+        return eq
+
+    def __repr__(self):
+        return "{class_}({system!r}, {definition!r}, location={location!r}, metadata={metadata!r})".format(
+            class_=self.__class__.__name__,
+            system=self.system,
+            definition=self.definition,
+            location=self.location,
+            metadata=self.metadata
+        )
+
+
+class DescriptorSystem(object):
+    def __init__(self, load_item, resources_fs, **kwargs):
+        """
+        load_item: Takes a Location and returns an XModuleDescriptor
+        resources_fs: A Filesystem object that contains all of the
+            resources needed for the course
+        """
+
+        self.load_item = load_item
+        self.resources_fs = resources_fs
+
+
+class XMLParsingSystem(DescriptorSystem):
+    def __init__(self, load_item, resources_fs, process_xml, **kwargs):
+        """
+        process_xml: Takes an xml string, and returns the the XModuleDescriptor created from that xml
+        """
+        DescriptorSystem.__init__(self, load_item, resources_fs)
+        self.process_xml = process_xml

common/lib/xmodule/xmodule/xml_module.py

@@ -0,0 +1,199 @@
+from collections import MutableMapping
+from xmodule.x_module import XModuleDescriptor
+from lxml import etree
+import copy
+import logging
+
+log = logging.getLogger(__name__)
+
+class LazyLoadingDict(MutableMapping):
+    """
+    A dictionary object that lazily loads it's contents from a provided
+    function on reads (of members that haven't already been set)
+    """
+
+    def __init__(self, loader):
+        self._contents = {}
+        self._loaded = False
+        self._loader = loader
+        self._deleted = set()
+
+    def __getitem__(self, name):
+        if not (self._loaded or name in self._contents or name in self._deleted):
+            self.load()
+
+        return self._contents[name]
+
+    def __setitem__(self, name, value):
+        self._contents[name] = value
+        self._deleted.discard(name)
+
+    def __delitem__(self, name):
+        del self._contents[name]
+        self._deleted.add(name)
+
+    def __contains__(self, name):
+        self.load()
+        return name in self._contents
+
+    def __len__(self):
+        self.load()
+        return len(self._contents)
+
+    def __iter__(self):
+        self.load()
+        return iter(self._contents)
+
+    def __repr__(self):
+        self.load()
+        return repr(self._contents)
+
+    def load(self):
+        if self._loaded:
+            return
+
+        loaded_contents = self._loader()
+        loaded_contents.update(self._contents)
+        self._contents = loaded_contents
+        self._loaded = True
+
+
+class XmlDescriptor(XModuleDescriptor):
+    """
+    Mixin class for standardized parsing of from xml
+    """
+
+    # Extension to append to filename paths
+    filename_extension = 'xml'
+
+    # The attributes will be removed from the definition xml passed
+    # to definition_from_xml, and from the xml returned by definition_to_xml
+    metadata_attributes = ('format', 'graceperiod', 'showanswer', 'rerandomize',
+        'due', 'graded', 'name', 'slug')
+
+    @classmethod
+    def definition_from_xml(cls, xml_object, system):
+        """
+        Return the definition to be passed to the newly created descriptor
+        during from_xml
+
+        xml_object: An etree Element
+        """
+        raise NotImplementedError("%s does not implement definition_from_xml" % cls.__name__)
+
+    @classmethod
+    def clean_metadata_from_xml(cls, xml_object):
+        """
+        Remove any attribute named in self.metadata_attributes from the supplied xml_object
+        """
+        for attr in cls.metadata_attributes:
+            if xml_object.get(attr) is not None:
+                del xml_object.attrib[attr]
+
+    @classmethod
+    def from_xml(cls, xml_data, system, org=None, course=None):
+        """
+        Creates an instance of this descriptor from the supplied xml_data.
+        This may be overridden by subclasses
+
+        xml_data: A string of xml that will be translated into data and children for
+            this module
+        system: An XModuleSystem for interacting with external resources
+        org and course are optional strings that will be used in the generated modules
+            url identifiers
+        """
+        xml_object = etree.fromstring(xml_data)
+
+        def metadata_loader():
+            metadata = {}
+            for attr in ('format', 'graceperiod', 'showanswer', 'rerandomize', 'due'):
+                from_xml = xml_object.get(attr)
+                if from_xml is not None:
+                    metadata[attr] = from_xml
+
+            if xml_object.get('graded') is not None:
+                metadata['graded'] = xml_object.get('graded') == 'true'
+
+            if xml_object.get('name') is not None:
+                metadata['display_name'] = xml_object.get('name')
+
+            return metadata
+
+        def definition_loader():
+            filename = xml_object.get('filename')
+            if filename is None:
+                definition_xml = copy.deepcopy(xml_object)
+            else:
+                filepath = cls._format_filepath(xml_object.tag, filename)
+                with system.resources_fs.open(filepath) as file:
+                    try:
+                        definition_xml = etree.parse(file).getroot()
+                    except:
+                        log.exception("Failed to parse xml in file %s" % filepath)
+                        raise
+
+            cls.clean_metadata_from_xml(definition_xml)
+            return cls.definition_from_xml(definition_xml, system)
+
+        return cls(
+            system,
+            LazyLoadingDict(definition_loader),
+            location=['i4x',
+                      org,
+                      course,
+                      xml_object.tag,
+                      xml_object.get('slug')],
+            metadata=LazyLoadingDict(metadata_loader),
+        )
+
+    @classmethod
+    def _format_filepath(cls, type, name):
+        return '{type}/{name}.{ext}'.format(type=type, name=name, ext=cls.filename_extension)
+
+    def export_to_xml(self, resource_fs):
+        """
+        Returns an xml string representing this module, and all modules underneath it.
+        May also write required resources out to resource_fs
+
+        Assumes that modules have single parantage (that no module appears twice in the same course),
+        and that it is thus safe to nest modules as xml children as appropriate.
+
+        The returned XML should be able to be parsed back into an identical XModuleDescriptor
+        using the from_xml method with the same system, org, and course
+        """
+        xml_object = self.definition_to_xml(resource_fs)
+        self.__class__.clean_metadata_from_xml(xml_object)
+
+        # Put content in a separate file if it's large (has more than 5 descendent tags)
+        if len(list(xml_object.iter())) > 5:
+
+            filepath = self.__class__._format_filepath(self.category, self.name)
+            resource_fs.makedir(self.category, allow_recreate=True)
+            with resource_fs.open(filepath, 'w') as file:
+                file.write(etree.tostring(xml_object, pretty_print=True))
+
+            for child in xml_object:
+                xml_object.remove(child)
+
+            xml_object.set('filename', self.name)
+
+        xml_object.set('slug', self.name)
+        xml_object.tag = self.category
+
+        for attr in ('format', 'graceperiod', 'showanswer', 'rerandomize', 'due'):
+            if attr in self.metadata and attr not in self._inherited_metadata:
+                xml_object.set(attr, self.metadata[attr])
+
+        if 'graded' in self.metadata and 'graded' not in self._inherited_metadata:
+            xml_object.set('graded', str(self.metadata['graded']).lower())
+
+        if 'display_name' in self.metadata:
+            xml_object.set('name', self.metadata['display_name'])
+
+        return etree.tostring(xml_object, pretty_print=True)
+
+    def definition_to_xml(self, resource_fs):
+        """
+        Return a new etree Element object created from this modules definition.
+        """
+        raise NotImplementedError("%s does not implement definition_to_xml" % self.__class__.__name__)

common/test/README.md

@@ -0,0 +1,6 @@
+Common test infrastructure for LMS + CMS
+===========================
+
+data/ has some test course data.  
+
+Once the course validation is separated from django, we should have scripts here that checks that a course consists only of xml that we understand.

common/test/data/full/README.md

@@ -0,0 +1 @@
+This is a realistic course, with many different module types and a lot of structure.  It is based on 6.002x.

common/test/data/simple/README.md

@@ -0,0 +1,2 @@
+This is a simple, but non-trivial, course using multiple module types and some nested structure.
+ 

common/test/data/toy/README.md

@@ -0,0 +1 @@
+This is a very very simple course, useful for initial debugging of processing code.

common/test/data/toy/toy_course.xml

@@ -0,0 +1,11 @@
+<course name="Toy Course" graceperiod="1 day 5 hours 59 minutes 59 seconds" showanswer="always" rerandomize="never">
+  <chapter name="Overview">
+    <section format="Video" name="Welcome">
+      <video youtube="0.75:izygArpw-Qo,1.0:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8"/>
+    </section>
+    <section format="Lecture Sequence" name="System Usage Sequence">
+      <html id="Lab2A" filename="Lab2A.html"/>
+      <video name="S0V1: Video Resources" youtube="0.75:EuzkdzfR0i8,1.0:1bK-WdDi6Qw,1.25:0v1VzoDVUTM,1.50:Bxk_-ZJb240"/>
+    </section>
+  </chapter>
+</course>

lms/djangoapps/courseware/content_parser.py

@@ -1,292 +0,0 @@
-'''
-courseware/content_parser.py
-
-This file interfaces between all courseware modules and the top-level course.xml file for a course.
-
-Does some caching (to be explained).
-
-'''
-
-import logging
-import os
-import sys
-import urllib
-
-from lxml import etree
-from util.memcache import fasthash
-
-from django.conf import settings
-
-from student.models import UserProfile
-from student.models import UserTestGroup
-from mitxmako.shortcuts import render_to_string
-from util.cache import cache
-from multicourse import multicourse_settings
-import xmodule
-
-''' This file will eventually form an abstraction layer between the
-course XML file and the rest of the system. 
-
-TODO: Shift everything from xml.dom.minidom to XPath (or XQuery)
-'''
-
-class ContentException(Exception):
-    pass
-
-log = logging.getLogger("mitx.courseware")
-
-def format_url_params(params):
-    return [ urllib.quote(string.replace(' ','_')) for string in params ]
-
-def xpath(xml, query_string, **args):
-    ''' Safe xpath query into an xml tree:
-        * xml is the tree.
-        * query_string is the query
-        * args are the parameters. Substitute for {params}. 
-        We should remove this with the move to lxml. 
-        We should also use lxml argument passing. '''
-    doc = etree.fromstring(xml)
-    #print type(doc)
-    def escape(x):
-        # TODO: This should escape the string. For now, we just assume it's made of valid characters. 
-        # Couldn't figure out how to escape for lxml in a few quick Googles
-        valid_chars="".join(map(chr, range(ord('a'),ord('z')+1)+range(ord('A'),ord('Z')+1)+range(ord('0'), ord('9')+1)))+"_ "
-        for e in x:
-            if e not in valid_chars:
-                raise Exception("Invalid char in xpath expression. TODO: Escape")
-        return x
-
-    args=dict( ((k, escape(args[k])) for k in args) )
-    #print args
-    results = doc.xpath(query_string.format(**args))
-    return results
-
-def xpath_remove(tree, path):
-    ''' Remove all items matching path from lxml tree.  Works in
-        place.'''
-    items = tree.xpath(path)
-    for item in items: 
-        item.getparent().remove(item)
-    return tree
-
-if __name__=='__main__':
-    print xpath('<html><problem name="Bob"></problem></html>', '/{search}/problem[@name="{name}"]', 
-                search='html', name="Bob")
-
-def id_tag(course):
-    ''' Tag all course elements with unique IDs '''
-    default_ids = xmodule.get_default_ids()
-
-    # Tag elements with unique IDs
-    elements = course.xpath("|".join(['//'+c for c in default_ids]))
-    for elem in elements:
-        if elem.get('id'):
-            pass
-        elif elem.get(default_ids[elem.tag]):
-            new_id = elem.get(default_ids[elem.tag]) 
-            new_id = "".join([a for a in new_id if a.isalnum()]) # Convert to alphanumeric
-            # Without this, a conflict may occur between an hmtl or youtube id
-            new_id = default_ids[elem.tag] + new_id
-            elem.set('id', new_id)
-        else:
-            elem.set('id', "id"+fasthash(etree.tostring(elem)))
-            
-def propogate_downward_tag(element, attribute_name, parent_attribute = None):
-    ''' This call is to pass down an attribute to all children. If an element
-    has this attribute, it will be "inherited" by all of its children. If a
-    child (A) already has that attribute, A will keep the same attribute and
-    all of A's children will inherit A's attribute. This is a recursive call.'''
-    
-    if (parent_attribute is None): #This is the entry call. Select all elements with this attribute
-        all_attributed_elements = element.xpath("//*[@" + attribute_name +"]")
-        for attributed_element in all_attributed_elements:
-            attribute_value = attributed_element.get(attribute_name)
-            for child_element in attributed_element:
-                propogate_downward_tag(child_element, attribute_name, attribute_value)
-    else:
-        '''The hack below is because we would get _ContentOnlyELements from the
-        iterator that can't have attributes set. We can't find API for it. If we
-        ever have an element which subclasses BaseElement, we will not tag it'''
-        if not element.get(attribute_name) and type(element) == etree._Element:
-            element.set(attribute_name, parent_attribute)
-            
-            for child_element in element:
-                propogate_downward_tag(child_element, attribute_name, parent_attribute)
-        else:
-            #This element would have already been found by Xpath, so we return
-            #for now and trust that this element will get its turn to propogate
-            #to its children later.
-            return
-
-def user_groups(user):
-    if not user.is_authenticated():
-        return []
-
-    # TODO: Rewrite in Django
-    key = 'user_group_names_{user.id}'.format(user=user)
-    cache_expiration = 60 * 60 # one hour
-    
-    # Kill caching on dev machines -- we switch groups a lot
-    group_names = cache.get(key)
- 
-    if group_names is None:
-        group_names = [u.name for u in UserTestGroup.objects.filter(users=user)]
-        cache.set(key, group_names, cache_expiration)
-
-    return group_names
-
-    # return [u.name for u in UserTestGroup.objects.raw("select * from auth_user, student_usertestgroup, student_usertestgroup_users where auth_user.id = student_usertestgroup_users.user_id and student_usertestgroup_users.usertestgroup_id = student_usertestgroup.id and auth_user.id = %s", [user.id])]
-
-def replace_custom_tags(course, tree):
-    try:
-        tags = os.listdir(course.path+'/custom_tags')
-        for tag in tags:
-            for element in tree.iter(tag):
-                element.tag = 'customtag'
-                impl = etree.SubElement(element, 'impl')
-                impl.text = tag
-    except os.error:
-        # The directory must not exist. This is okay, as it is optional. If it is empty, git has trouble tracking it
-        pass
-
-def course_xml_process(course, tree):
-    ''' Do basic pre-processing of an XML tree. Assign IDs to all
-    items without. Propagate due dates, grace periods, etc. to child
-    items. 
-    '''
-    replace_custom_tags(course, tree)
-    id_tag(tree)
-    propogate_downward_tag(tree, "due")
-    propogate_downward_tag(tree, "graded")
-    propogate_downward_tag(tree, "graceperiod")
-    propogate_downward_tag(tree, "showanswer")
-    propogate_downward_tag(tree, "rerandomize")
-    return tree
-
-def course_file(user,course=None):
-    ''' Given a user, return course.xml'''
-
-    if user.is_authenticated():
-        filename = os.path.basename(course.path)+"/"+UserProfile.objects.get(user=user).courseware # user.profile_cache.courseware 
-    else:
-        filename = 'guest_course.xml'
-
-    # if a specific course is specified, then use multicourse to get the right path to the course XML directory
-    # if coursename and settings.ENABLE_MULTICOURSE:
-    #     xp = multicourse_settings.get_course_xmlpath(coursename)
-    #     filename = xp + filename	# prefix the filename with the path
-
-    groups = user_groups(user)
-    options = {'dev_content':settings.DEV_CONTENT, 
-               'groups' : groups}
-
-    
-    cache_key = filename + "_processed?dev_content:" + str(options['dev_content']) + "&groups:" + str(sorted(groups))
-    if "dev" not in settings.DEFAULT_GROUPS:
-        tree_string = cache.get(cache_key)
-    else: 
-        tree_string = None
-
-    if settings.DEBUG:
-        log.info('[courseware.content_parser.course_file] filename=%s, cache_key=%s' % (filename,cache_key))
-        # print '[courseware.content_parser.course_file] tree_string = ',tree_string
-
-    if not tree_string:
-        tree = course_xml_process(course, etree.XML(render_to_string(filename, options, namespace = 'course')))
-        tree_string = etree.tostring(tree)
-        
-        cache.set(cache_key, tree_string, 60)
-    else:
-        tree = etree.XML(tree_string)
-
-    return tree
-
-def section_file(user, section, course=None, dironly=False):
-    '''
-    Given a user and the name of a section, return that section.
-    This is done specific to each course.
-    If dironly=True then return the sections directory.
-    TODO: This is a bit weird; dironly should be scrapped. 
-    '''
-    filename = section+".xml"
-
-    # if a specific course is specified, then use multicourse to get the right path to the course XML directory
-    xp = ''
-    if coursename and settings.ENABLE_MULTICOURSE: xp = multicourse_settings.get_course_xmlpath(coursename)
-
-    dirname = settings.DATA_DIR + xp + '/sections/'
-
-    if dironly: return dirname
-
-    if filename not in os.listdir(dirname):
-        log.error(filename+" not in "+str(os.listdir(dirname)))
-        return None
-
-    options = {'dev_content':settings.DEV_CONTENT, 
-               'groups' : user_groups(user)}
-
-    tree = course_xml_process(course, etree.XML(render_to_string(filename, options, namespace = 'sections')))
-    return tree
-
-
-def module_xml(user, module, id_tag, module_id, coursename=None):
-    ''' Get XML for a module based on module and module_id. Assumes
-        module occurs once in courseware XML file or hidden section. '''
-    # Sanitize input
-    if not module.isalnum():
-        raise Exception("Module is not alphanumeric")
-    if not module_id.isalnum():
-        raise Exception("Module ID is not alphanumeric")
-    # Generate search
-    xpath_search='//{module}[(@{id_tag} = "{id}") or (@id = "{id}")]'.format(module=module, 
-                                                                             id_tag=id_tag,
-                                                                             id=module_id)
-    #result_set=doc.xpathEval(xpath_search)
-    doc = course_file(user,coursename)
-    sdirname = section_file(user,'',coursename,True)	# get directory where sections information is stored
-    section_list = (s[:-4] for s in os.listdir(sdirname) if s[-4:]=='.xml')
-
-    result_set=doc.xpath(xpath_search)
-    if len(result_set)<1:
-        for section in section_list:
-            try: 
-                s = section_file(user, section, coursename)
-            except etree.XMLSyntaxError: 
-                ex= sys.exc_info()
-                raise ContentException("Malformed XML in " + section+ "("+str(ex[1].msg)+")")
-            result_set = s.xpath(xpath_search)
-            if len(result_set) != 0: 
-                break
-
-    if len(result_set)>1:
-        log.error("WARNING: Potentially malformed course file", module, module_id)
-    if len(result_set)==0:
-        if settings.DEBUG:
-            log.error('[courseware.content_parser.module_xml] cannot find %s in course.xml tree' % xpath_search)
-            log.error('tree = %s' % etree.tostring(doc,pretty_print=True))
-        return None
-    if settings.DEBUG:
-        log.info('[courseware.content_parser.module_xml] found %s' % result_set)
-    return etree.tostring(result_set[0])
-    #return result_set[0].serialize()
-
-def toc_from_xml(dom, active_chapter, active_section):
-    name = dom.xpath('//course/@name')[0]
-
-    chapters = dom.xpath('//course[@name=$name]/chapter', name=name)
-    ch=list()
-    for c in chapters:
-        if c.get('name') == 'hidden':
-            continue
-        sections=list()
-        for s in dom.xpath('//course[@name=$name]/chapter[@name=$chname]/section', name=name, chname=c.get('name')): 
-            sections.append({'name':s.get("name") or "", 
-                             'format':s.get("subtitle") if s.get("subtitle") else s.get("format") or "", 
-                             'due':s.get("due") or "",
-                             'active':(c.get("name")==active_chapter and \
-                                           s.get("name")==active_section)})
-        ch.append({'name':c.get("name"), 
-                   'sections':sections,
-                   'active':(c.get("name")==active_chapter)})
-    return ch
-

lms/djangoapps/courseware/course_settings.py

@@ -0,0 +1,92 @@
+"""
+Course settings module. All settings in the global_settings are
+first applied, and then any settings in the settings.DATA_DIR/course_settings.json
+are applied. A setting must be in ALL_CAPS. 
+    
+Settings are used by calling
+
+from courseware.course_settings import course_settings
+
+Note that courseware.course_settings.course_settings is not a module -- it's an object. So 
+importing individual settings is not possible:
+
+from courseware.course_settings.course_settings import GRADER  # This won't work.
+
+"""
+import json
+import logging
+
+from django.conf import settings
+
+from xmodule import graders
+
+log = logging.getLogger("mitx.courseware")
+
+global_settings_json = """
+{
+	"GRADER" : [
+	    {
+	        "type" : "Homework",
+	        "min_count" : 12,
+	        "drop_count" : 2,
+	        "short_label" : "HW",
+	        "weight" : 0.15
+	    },
+	    {
+	        "type" : "Lab",
+	        "min_count" : 12,
+	        "drop_count" : 2,
+	        "category" : "Labs",
+	        "weight" : 0.15
+	    },
+	    {
+	        "type" : "Midterm",
+	        "name" : "Midterm Exam",
+	        "short_label" : "Midterm",
+	        "weight" : 0.3
+	    },
+	    {
+	        "type" : "Final",
+	        "name" : "Final Exam",
+	        "short_label" : "Final",
+	        "weight" : 0.4
+	    }
+	],
+	"GRADE_CUTOFFS" : {
+		"A" : 0.87, 
+		"B" : 0.7, 
+		"C" : 0.6
+	}
+}
+""" 
+
+
+class Settings(object):
+    def __init__(self):
+        
+        # Load the global settings as a dictionary
+        global_settings = json.loads(global_settings_json)
+        
+        
+        # Load the course settings as a dictionary
+        course_settings = {}
+        try:
+            with open( settings.DATA_DIR + "/course_settings.json") as course_settings_file:
+                course_settings_string = course_settings_file.read()
+            course_settings = json.loads(course_settings_string)
+        except IOError:
+            log.warning("Unable to load course settings file from " + str(settings.DATA_DIR) + "/course_settings.json")
+        
+        
+        # Override any global settings with the course settings
+        global_settings.update(course_settings)
+        
+        # Now, set the properties from the course settings on ourselves
+        for setting in global_settings:
+            setting_value = global_settings[setting]
+            setattr(self, setting, setting_value)
+                
+        # Here is where we should parse any configurations, so that we can fail early
+        self.GRADER = graders.grader_from_conf(self.GRADER)
+
+course_settings = Settings()

lms/djangoapps/courseware/global_course_settings.py

@@ -1,28 +0,0 @@
-GRADER = [
-    {
-        'type' : "Homework",
-        'min_count' : 12,
-        'drop_count' : 2,
-        'short_label' : "HW",
-        'weight' : 0.15,
-    },
-    {
-        'type' : "Lab",
-        'min_count' : 12,
-        'drop_count' : 2,
-        'category' : "Labs",
-        'weight' : 0.15
-    },
-    {
-        'type' : "Midterm",
-        'name' : "Midterm Exam",
-        'short_label' : "Midterm",
-        'weight' : 0.3,
-    },
-    {
-        'type' : "Final",
-        'name' : "Final Exam",
-        'short_label' : "Final",
-        'weight' : 0.4,
-    }
-]

lms/djangoapps/courseware/grades.py

@@ -1,192 +1,125 @@
-"""
-Course settings module. The settings are based of django.conf. All settings in
-courseware.global_course_settings are first applied, and then any settings
-in the settings.DATA_DIR/course_settings.py are applied. A setting must be
-in ALL_CAPS.
-    
-Settings are used by calling
-    
-from courseware import course_settings
-
-Note that courseware.course_settings is not a module -- it's an object. So 
-importing individual settings is not possible:
-
-from courseware.course_settings import GRADER  # This won't work.
-
-"""
-
-from lxml import etree
 import random
-import imp
 import logging
-import sys
-import types
 
 from django.conf import settings
 
-from courseware import global_course_settings
+from courseware.course_settings import course_settings
 from xmodule import graders
 from xmodule.graders import Score
 from models import StudentModule
-import courseware.content_parser as content_parser
-import xmodule
 
 _log = logging.getLogger("mitx.courseware")
 
-class Settings(object):
-    def __init__(self):
-        # update this dict from global settings (but only for ALL_CAPS settings)
-        for setting in dir(global_course_settings):
-            if setting == setting.upper():
-                setattr(self, setting, getattr(global_course_settings, setting))
-        
-        
-        data_dir = settings.DATA_DIR
-        
-        fp = None
-        try:
-            fp, pathname, description = imp.find_module("course_settings", [data_dir])
-            mod = imp.load_module("course_settings", fp, pathname, description)
-        except Exception as e:
-            _log.exception("Unable to import course settings file from " + data_dir + ". Error: " + str(e))
-            mod = types.ModuleType('course_settings')
-        finally:
-            if fp:
-                fp.close()
-                
-        for setting in dir(mod):
-            if setting == setting.upper():
-                setting_value = getattr(mod, setting)
-                setattr(self, setting, setting_value)
-                
-        # Here is where we should parse any configurations, so that we can fail early
-        self.GRADER = graders.grader_from_conf(self.GRADER)
 
-course_settings = Settings()
-
-
-
-
-def grade_sheet(student,course=None):
+def grade_sheet(student, course, student_module_cache):
     """
     This pulls a summary of all problems in the course. It returns a dictionary with two datastructures:
-    
+
     - courseware_summary is a summary of all sections with problems in the course. It is organized as an array of chapters,
     each containing an array of sections, each containing an array of scores. This contains information for graded and ungraded
     problems, and is good for displaying a course summary with due dates, etc.
-    
+
     - grade_summary is the output from the course grader. More information on the format is in the docstring for CourseGrader.
+
+    Arguments:
+        student: A User object for the student to grade
+        course: An XModule containing the course to grade
+        student_module_cache: A StudentModuleCache initialized with all instance_modules for the student
     """
-    dom=content_parser.course_file(student,course)
-    dom_course = dom.xpath('//course/@name')[0]
-    xmlChapters = dom.xpath('//course[@name=$course]/chapter', course=dom_course)
-
-    responses=StudentModule.objects.filter(student=student)
-    response_by_id = {}
-    for response in responses:
-        response_by_id[response.module_id] = response
-    
-    
     totaled_scores = {}
-    chapters=[]
-    for c in xmlChapters:
+    chapters = []
+    for c in course.get_children():
         sections = []
-        chname=c.get('name')
-        
-        
-        for s in dom.xpath('//course[@name=$course]/chapter[@name=$chname]/section', 
-                           course=dom_course, chname=chname):
-            problems=dom.xpath('//course[@name=$course]/chapter[@name=$chname]/section[@name=$section]//problem', 
-                           course=dom_course, chname=chname, section=s.get('name'))
+        for s in c.get_children():
+            def yield_descendents(module):
+                yield module
+                for child in module.get_display_items():
+                    for module in yield_descendents(child):
+                        yield module
 
-            graded = True if s.get('graded') == "true" else False
-            scores=[]
-            if len(problems)>0:
-                for p in problems:
-                    (correct,total) = get_score(student, p, response_by_id, course=course)
-                    
-                    if settings.GENERATE_PROFILE_SCORES:
-                        if total > 1:
-                            correct = random.randrange( max(total-2, 1) , total + 1 )
-                        else:
-                            correct = total
-                    
-                    if not total > 0:
-                        #We simply cannot grade a problem that is 12/0, because we might need it as a percentage
-                        graded = False 
-                    scores.append( Score(correct,total, graded, p.get("name")) )
+            graded = s.metadata.get('graded', False)
+            scores = []
+            for module in yield_descendents(s):
+                (correct, total) = get_score(student, module, student_module_cache)
 
-                section_total, graded_total = graders.aggregate_scores(scores, s.get("name"))
-                #Add the graded total to totaled_scores
-                format = s.get('format', "")
-                subtitle = s.get('subtitle', format)
-                if format and graded_total[1] > 0:
-                    format_scores = totaled_scores.get(format, [])
-                    format_scores.append( graded_total )
-                    totaled_scores[ format ] = format_scores
+                if correct is None and total is None:
+                    continue
 
-                section_score={'section':s.get("name"),
-                       'scores':scores,
-                       'section_total' : section_total,
-                       'format' : format,
-                       'subtitle' : subtitle,
-                       'due' : s.get("due") or "",
-                       'graded' : graded,
-                       }
-                sections.append(section_score)
+                if settings.GENERATE_PROFILE_SCORES:
+                    if total > 1:
+                        correct = random.randrange(max(total - 2, 1), total + 1)
+                    else:
+                        correct = total
+
+                if not total > 0:
+                    #We simply cannot grade a problem that is 12/0, because we might need it as a percentage
+                    graded = False
+
+                scores.append(Score(correct, total, graded, module.metadata.get('display_name')))
+
+            section_total, graded_total = graders.aggregate_scores(scores, s.metadata.get('display_name'))
+            #Add the graded total to totaled_scores
+            format = s.metadata.get('format', "")
+            if format and graded_total.possible > 0:
+                format_scores = totaled_scores.get(format, [])
+                format_scores.append(graded_total)
+                totaled_scores[format] = format_scores
+
+            sections.append({
+                'section': s.metadata.get('display_name'),
+                'scores': scores,
+                'section_total': section_total,
+                'format': format,
+                'due': s.metadata.get("due", ""),
+                'graded': graded,
+            })
+
+        chapters.append({'course': course.metadata.get('display_name'),
+                         'chapter': c.metadata.get('display_name'),
+                         'sections': sections})
 
-        chapters.append({'course':course,
-                         'chapter' : c.get("name"),
-                         'sections' : sections,})
-    
-        
     grader = course_settings.GRADER
     grade_summary = grader.grade(totaled_scores)
-    
-    return {'courseware_summary' : chapters,
-            'grade_summary' : grade_summary}
 
-def get_score(user, problem, cache, course=None):
-    ## HACK: assumes max score is fixed per problem
-    id = problem.get('id')
+    return {'courseware_summary': chapters,
+            'grade_summary': grade_summary}
+
+
+def get_score(user, problem, cache):
+    """
+    Return the score for a user on a problem
+
+    user: a Student object
+    problem: an XModule
+    cache: A StudentModuleCache
+    """
     correct = 0.0
-    
-    # If the ID is not in the cache, add the item
-    if id not in cache:
-        module = StudentModule(module_type = 'problem',  # TODO: Move into StudentModule.__init__?
-                               module_id = id,
-                               student = user, 
-                               state = None, 
-                               grade = 0,
-                               max_grade = None,
-                               done = 'i')
-        cache[id] = module
 
-    # Grab the # correct from cache
-    if id in cache:
-        response = cache[id]
-        if response.grade!=None:
-            correct=float(response.grade)
-        
-    # Grab max grade from cache, or if it doesn't exist, compute and save to DB
-    if id in cache and response.max_grade is not None:
-        total = response.max_grade
-    else:
-        ## HACK 1: We shouldn't specifically reference capa_module
-        ## HACK 2: Backwards-compatibility: This should be written when a grade is saved, and removed from the system
-        # TODO: These are no longer correct params for I4xSystem -- figure out what this code
-        # does, clean it up.
-        from module_render import I4xSystem
-        system = I4xSystem(None, None, None, course=course)
-        total=float(xmodule.capa_module.Module(system, etree.tostring(problem), "id").max_score())
-        response.max_grade = total
-        response.save()
-        
-    #Now we re-weight the problem, if specified
-    weight = problem.get("weight", None)
-    if weight:
-        weight = float(weight) 
-        correct = correct * weight / total
-        total = weight        
+    # If the ID is not in the cache, add the item
+    instance_module = cache.lookup(problem.category, problem.id)
+    if instance_module is None:
+        instance_module = StudentModule(module_type=problem.category,
+                                        module_state_key=problem.id,
+                                        student=user,
+                                        state=None,
+                                        grade=0,
+                                        max_grade=problem.max_score(),
+                                        done='i')
+        cache.append(instance_module)
+        instance_module.save()
+
+    # If this problem is ungraded/ungradable, bail
+    if instance_module.max_grade is None:
+        return (None, None)
+
+    correct = instance_module.grade if instance_module.grade is not None else 0
+    total = instance_module.max_grade
+
+    if correct is not None and total is not None:
+        #Now we re-weight the problem, if specified
+        weight = getattr(problem, 'weight', 1)
+        if weight != 1:
+            correct = correct * weight / total
+            total = weight
 
     return (correct, total)

lms/djangoapps/courseware/management/commands/check_course.py

@@ -6,50 +6,93 @@ from django.core.management.base import BaseCommand
 from django.conf import settings
 from django.contrib.auth.models import User
 
-from courseware.content_parser import course_file
-import courseware.module_render
 import xmodule
 
+import mitxmako.middleware as middleware
+middleware.MakoMiddleware()
+from xmodule.modulestore.django import modulestore
+from courseware.models import StudentModuleCache
+from courseware.module_render import get_module
+
+
+def check_rendering(module):
+    '''Check that all modules render'''
+    all_ok = True
+    print "Confirming all modules render. Nothing should print during this step. "
+
+    def _check_module(module):
+        try:
+            module.get_html()
+        except Exception as ex:
+            print "==============> Error in ", module.id
+            print ""
+            print ex
+            all_ok = False
+        for child in module.get_children():
+            _check_module(child)
+    _check_module(module)
+    print "Module render check finished"
+    return all_ok
+
+
+def check_sections(course):
+    all_ok = True
+    sections_dir = settings.DATA_DIR + "/sections"
+    print "Checking that all sections exist and parse properly"
+    if os.path.exists(sections_dir):
+        print "Checking all section includes are valid XML"
+        for f in os.listdir(sections_dir):
+            sectionfile = sections_dir + '/' + f
+            #print sectionfile
+            # skip non-xml files:
+            if not sectionfile.endswith('xml'):
+                continue
+            try:
+                etree.parse(sectionfile)
+            except Exception as ex:
+                print "================> Error parsing ", sectionfile
+                print ex
+                all_ok = False
+        print "checked all sections"
+    else:
+        print "Skipping check of include files -- no section includes dir (" + sections_dir + ")"
+    return all_ok
+
+
 class Command(BaseCommand):
     help = "Does basic validity tests on course.xml."
+
     def handle(self, *args, **options):
-        check = True
+        all_ok = True
+
+        # TODO (vshnayder): create dummy user objects.  Anon, authenticated, staff.
+        # Check that everything works for each.
+        # The objects probably shouldn't be actual django users to avoid unneeded
+        # dependency on django.
+
+        # TODO: use args as list of files to check.  Fix loading to work for other files.
+
         sample_user = User.objects.all()[0]
+
         print "Attempting to load courseware"
-        course = course_file(sample_user)
-        print "Confirming all problems have alphanumeric names"
-        for problem in course.xpath('//problem'):
-            filename = problem.get('filename')
-            if not filename.isalnum():
-                print "==============> Invalid (non-alphanumeric) filename", filename
-                check = False
-        print "Confirming all modules render. Nothing should print during this step. "
-        for module in course.xpath('//problem|//html|//video|//vertical|//sequential|/tab'):
-            module_class = xmodule.modx_modules[module.tag]
-            # TODO: Abstract this out in render_module.py
-            try: 
-                module_class(etree.tostring(module), 
-                             module.get('id'), 
-                             ajax_url='',
-                             state=None, 
-                             track_function = lambda x,y,z:None, 
-                             render_function = lambda x: {'content':'','type':'video'})
-            except:
-                print "==============> Error in ", etree.tostring(module)
-                check = False
-        print "Module render check finished"
-        sections_dir = settings.DATA_DIR+"sections"
-        if os.path.exists(sections_dir):
-            print "Checking all section includes are valid XML"
-            for f in os.listdir(sections_dir):
-                print f
-                etree.parse(sections_dir+'/'+f)
-        else:
-            print "Skipping check of include files -- no section includes dir ("+sections_dir+")"
+
+        # TODO (cpennington): Get coursename in a legitimate way
+        course_location = 'i4x://edx/6002xs12/course/6.002_Spring_2012'
+        student_module_cache = StudentModuleCache(sample_user, modulestore().get_item(course_location))
+        (course, _, _, _) = get_module(sample_user, None, course_location, student_module_cache)
+
+        to_run = [
+            #TODO (vshnayder) : make check_rendering work (use module_render.py),
+            # turn it on
+            check_rendering,
+            check_sections,
+        ]
+        for check in to_run:
+            all_ok = check(course) and all_ok
+
         # TODO: print "Checking course properly annotated with preprocess.py"
-        
-            
-        if check:
+
+        if all_ok:
             print 'Courseware passes all checks!'
-        else: 
+        else:
             print "Courseware fails some checks"

lms/djangoapps/courseware/models.py

@@ -13,7 +13,6 @@ ASSUMPTIONS: modules have unique IDs, even across different module_types
 
 """
 from django.db import models
-from django.db.models.signals import post_save, post_delete
 #from django.core.cache import cache
 from django.contrib.auth.models import User
 
@@ -21,72 +20,106 @@ from django.contrib.auth.models import User
 
 #CACHE_TIMEOUT = 60 * 60 * 4 # Set the cache timeout to be four hours
 
+
 class StudentModule(models.Model):
     # For a homework problem, contains a JSON
     # object consisting of state
-    MODULE_TYPES = (('problem','problem'),
-                    ('video','video'),
-                    ('html','html'),
+    MODULE_TYPES = (('problem', 'problem'),
+                    ('video', 'video'),
+                    ('html', 'html'),
                     )
     ## These three are the key for the object
     module_type = models.CharField(max_length=32, choices=MODULE_TYPES, default='problem', db_index=True)
-    module_id = models.CharField(max_length=255, db_index=True) # Filename for homeworks, etc. 
+
+    # Key used to share state. By default, this is the module_id,
+    # but for abtests and the like, this can be set to a shared value
+    # for many instances of the module.
+    # Filename for homeworks, etc.
+    module_state_key = models.CharField(max_length=255, db_index=True, db_column='module_id')
     student = models.ForeignKey(User, db_index=True)
+
     class Meta:
-        unique_together = (('student', 'module_id'),)
+        unique_together = (('student', 'module_state_key'),)
 
     ## Internal state of the object
     state = models.TextField(null=True, blank=True)
 
-    ## Grade, and are we done? 
+    ## Grade, and are we done?
     grade = models.FloatField(null=True, blank=True, db_index=True)
     max_grade = models.FloatField(null=True, blank=True)
-    DONE_TYPES = (('na','NOT_APPLICABLE'),
-                    ('f','FINISHED'),
-                    ('i','INCOMPLETE'),
+    DONE_TYPES = (('na', 'NOT_APPLICABLE'),
+                    ('f', 'FINISHED'),
+                    ('i', 'INCOMPLETE'),
                     )
     done = models.CharField(max_length=8, choices=DONE_TYPES, default='na', db_index=True)
 
-    # DONE_TYPES = (('done','DONE'),          # Finished
-    #               ('incomplete','NOTDONE'), # Not finished
-    #               ('na','NA'))              # Not applicable (e.g. vertical)
-    # done = models.CharField(max_length=16, choices=DONE_TYPES)
-
     created = models.DateTimeField(auto_now_add=True, db_index=True)
     modified = models.DateTimeField(auto_now=True, db_index=True)
 
     def __unicode__(self):
-        return self.module_type+'/'+self.student.username+"/"+self.module_id+'/'+str(self.state)[:20]
-
-    # @classmethod
-    # def get_with_caching(cls, student, module_id):
-    #     k = cls.key_for(student, module_id)
-    #     student_module = cache.get(k)
-    #     if student_module is None:
-    #         student_module = StudentModule.objects.filter(student=student,
-    #                                                       module_id=module_id)[0]
-    #         # It's possible it really doesn't exist...
-    #         if student_module is not None:
-    #             cache.set(k, student_module, CACHE_TIMEOUT)
-
-    #     return student_module
-
-    @classmethod
-    def key_for(cls, student, module_id):
-        return "StudentModule-student_id:{0};module_id:{1}".format(student.id, module_id)
+        return '/'.join([self.module_type, self.student.username, self.module_state_key, str(self.state)[:20]])
 
 
-# def clear_cache_by_student_and_module_id(sender, instance, *args, **kwargs):
-#     k = sender.key_for(instance.student, instance.module_id)
-#     cache.delete(k)
-
-# def update_cache_by_student_and_module_id(sender, instance, *args, **kwargs):
-#     k = sender.key_for(instance.student, instance.module_id)
-#     cache.set(k, instance, CACHE_TIMEOUT)
+# TODO (cpennington): Remove these once the LMS switches to using XModuleDescriptors
 
 
-#post_save.connect(update_cache_by_student_and_module_id, sender=StudentModule, weak=False)
-#post_delete.connect(clear_cache_by_student_and_module_id, sender=StudentModule, weak=False)
 
-#cache_model(StudentModule)
+class StudentModuleCache(object):
+    """
+    A cache of StudentModules for a specific student
+    """
+    def __init__(self, user, descriptor, depth=None):
+        '''
+        Find any StudentModule objects that are needed by any child modules of the
+        supplied descriptor. Avoids making multiple queries to the database
+        '''
+        if user.is_authenticated():
+            module_ids = self._get_module_state_keys(descriptor, depth)
 
+            # This works around a limitation in sqlite3 on the number of parameters
+            # that can be put into a single query
+            self.cache = []
+            chunk_size = 500
+            for id_chunk in [module_ids[i:i+chunk_size] for i in xrange(0, len(module_ids), chunk_size)]:
+                self.cache.extend(StudentModule.objects.filter(
+                    student=user,
+                    module_state_key__in=id_chunk)
+                )
+
+        else:
+            self.cache = []
+
+    def _get_module_state_keys(self, descriptor, depth):
+        '''
+        Get a list of the state_keys needed for StudentModules
+        required for this chunk of module xml
+        '''
+        keys = [descriptor.location.url()]
+
+        shared_state_key = getattr(descriptor, 'shared_state_key', None)
+        if shared_state_key is not None:
+            keys.append(shared_state_key)
+
+        if depth is None or depth > 0:
+            new_depth = depth - 1 if depth is not None else depth
+
+            for child in descriptor.get_children():
+                keys.extend(self._get_module_state_keys(child, new_depth))
+
+        return keys
+
+    def lookup(self, module_type, module_state_key):
+        '''
+        Look for a student module with the given type and id in the cache.
+
+        cache -- list of student modules
+
+        returns first found object, or None
+        '''
+        for o in self.cache:
+            if o.module_type == module_type and o.module_state_key == module_state_key:
+                return o
+        return None
+
+    def append(self, student_module):
+        self.cache.append(student_module)

lms/djangoapps/courseware/module_render.py

@@ -2,30 +2,22 @@ import json
 import logging
 import os
 
-from lxml import etree
-
+from django.conf import settings
 from django.http import Http404
 from django.http import HttpResponse
-from django.shortcuts import redirect
+from lxml import etree
 
-from fs.osfs import OSFS
-
-from django.conf import settings
-from mitxmako.shortcuts import render_to_string, render_to_response
-
-from models import StudentModule
-from multicourse import multicourse_settings
-from util.views import accepts
-
-import courseware.content_parser as content_parser
-import xmodule
+from xmodule.modulestore.django import modulestore
+from mitxmako.shortcuts import render_to_string
+from models import StudentModule, StudentModuleCache
 
 log = logging.getLogger("mitx.courseware")
 
+
 class I4xSystem(object):
     '''
-    This is an abstraction such that x_modules can function independent 
-    of the courseware (e.g. import into other types of courseware, LMS, 
+    This is an abstraction such that x_modules can function independent
+    of the courseware (e.g. import into other types of courseware, LMS,
     or if we want to have a sandbox server for user-contributed content)
 
     I4xSystem objects are passed to x_modules to provide access to system
@@ -34,9 +26,9 @@ class I4xSystem(object):
     Note that these functions can be closures over e.g. a django request
     and user, or other environment-specific info.
     '''
-    def __init__(self, ajax_url, track_function, render_function,
-                 module_from_xml, render_template, request=None,
-                 filestore=None, course=None):
+    def __init__(self, ajax_url, track_function,
+                 get_module, render_template, user=None,
+                 filestore=None):
         '''
         Create a closure around the system environment.
 
@@ -45,40 +37,28 @@ class I4xSystem(object):
                          or otherwise tracking the event.
                          TODO: Not used, and has inconsistent args in different
                          files.  Update or remove.
-        module_from_xml - function that takes (module_xml) and returns a corresponding
+        get_module - function that takes (location) and returns a corresponding
                           module instance object.
-        render_function - function that takes (module_xml) and renders it,
-                          returning a dictionary with a context for rendering the
-                          module to html.  Dictionary will contain keys 'content'
-                          and 'type'.
         render_template - a function that takes (template_file, context), and returns
                           rendered html.
-        request - the request in progress
+        user - The user to base the seed off of for this request
         filestore - A filestore ojbect.  Defaults to an instance of OSFS based at
                     settings.DATA_DIR.
         '''
-        self.course = course
         self.ajax_url = ajax_url
         self.track_function = track_function
-        if not filestore: 
-            self.filestore = OSFS(settings.DATA_DIR)
-        else:
-            self.filestore = filestore
-            if settings.DEBUG:
-                log.info("[courseware.module_render.I4xSystem] filestore path = %s",
-                         filestore)
-        self.module_from_xml = module_from_xml
-        self.render_function = render_function
+        self.filestore = filestore
+        self.get_module = get_module
         self.render_template = render_template
         self.exception404 = Http404
         self.DEBUG = settings.DEBUG
-        self.id = request.user.id if request is not None else 0
+        self.seed = user.id if user is not None else 0
 
     def get(self, attr):
         '''	provide uniform access to attributes (like etree).'''
         return self.__dict__.get(attr)
-    
-    def set(self,attr,val):
+
+    def set(self, attr, val):
         '''provide uniform access to attributes (like etree)'''
         self.__dict__[attr] = val
 
@@ -88,21 +68,9 @@ class I4xSystem(object):
     def __str__(self):
         return str(self.__dict__)
 
-def smod_cache_lookup(cache, module_type, module_id):
-    '''
-    Look for a student module with the given type and id in the cache.
-
-    cache -- list of student modules
-
-    returns first found object, or None
-    '''
-    for o in cache: 
-        if o.module_type == module_type and o.module_id == module_id:
-            return o
-    return None
 
 def make_track_function(request):
-    ''' 
+    '''
     Make a tracking function that logs what happened.
     For use in I4xSystem.
     '''
@@ -112,8 +80,9 @@ def make_track_function(request):
         return track.views.server_track(request, event_type, event, page='x_module')
     return f
 
+
 def grade_histogram(module_id):
-    ''' Print out a histogram of grades on a given problem. 
+    ''' Print out a histogram of grades on a given problem.
         Part of staff member debug info.
     '''
     from django.db import connection
@@ -134,18 +103,81 @@ def grade_histogram(module_id):
     return grades
 
 
-def make_module_from_xml_fn(user, request, student_module_cache, course, position):
-    '''Create the make_from_xml() function'''
-    def module_from_xml(xml):
-        '''Modules need a way to convert xml to instance objects.
-        Pass the rest of the context through.'''
-        (instance, sm, module_type) = get_module(
-            user, request, xml, student_module_cache, course, position)
-        return instance
-    return module_from_xml
+def toc_for_course(user, request, course, active_chapter, active_section):
+    '''
+    Create a table of contents from the module store
+
+    Return format:
+    [ {'name': name, 'sections': SECTIONS, 'active': bool}, ... ]
+
+    where SECTIONS is a list
+    [ {'name': name, 'format': format, 'due': due, 'active' : bool}, ...]
+
+    active is set for the section and chapter corresponding to the passed
+    parameters.  Everything else comes from the xml, or defaults to "".
+
+    chapters with name 'hidden' are skipped.
+    '''
+
+    student_module_cache = StudentModuleCache(user, course, depth=2)
+    (course, _, _, _) = get_module(user, request, course.location, student_module_cache)
+
+    chapters = list()
+    for chapter in course.get_display_items():
+        sections = list()
+        for section in chapter.get_display_items():
+
+            active = (chapter.metadata.get('display_name') == active_chapter and
+                      section.metadata.get('display_name') == active_section)
+
+            sections.append({'name': section.metadata.get('display_name'),
+                             'format': section.metadata.get('format', ''),
+                             'due': section.metadata.get('due', ''),
+                             'active': active})
+
+        chapters.append({'name': chapter.metadata.get('display_name'),
+                         'sections': sections,
+                         'active': chapter.metadata.get('display_name') == active_chapter})
+    return chapters
 
 
-def get_module(user, request, module_xml, student_module_cache, course, position=None):
+def get_section(course, chapter, section):
+    """
+    Returns the xmodule descriptor for the name course > chapter > section,
+    or None if this doesn't specify a valid section
+
+    course: Course url
+    chapter: Chapter name
+    section: Section name
+    """
+    try:
+        course_module = modulestore().get_item(course)
+    except:
+        log.exception("Unable to load course_module")
+        return None
+
+    if course_module is None:
+        return
+
+    chapter_module = None
+    for _chapter in course_module.get_children():
+        if _chapter.metadata.get('display_name') == chapter:
+            chapter_module = _chapter
+            break
+
+    if chapter_module is None:
+        return
+
+    section_module = None
+    for _section in chapter_module.get_children():
+        if _section.metadata.get('display_name') == section:
+            section_module = _section
+            break
+
+    return section_module
+
+
+def get_module(user, request, location, student_module_cache, position=None):
     ''' Get an instance of the xmodule class corresponding to module_xml,
     setting the state based on an existing StudentModule, or creating one if none
     exists.
@@ -154,197 +186,125 @@ def get_module(user, request, module_xml, student_module_cache, course, position
       - user                  : current django User
       - request               : current django HTTPrequest
       - module_xml            : lxml etree of xml subtree for the requested module
-      - student_module_cache  : list of StudentModule objects, one of which may
-                                match this module type and id
-      - position   	          : extra information from URL for user-specified
+      - student_module_cache  : a StudentModuleCache
+      - position              : extra information from URL for user-specified
                                 position within module
 
     Returns:
-      - a tuple (xmodule instance, student module, module type).
+      - a tuple (xmodule instance, instance_module, shared_module, module type).
+        instance_module is a StudentModule specific to this module for this student
+        shared_module is a StudentModule specific to all modules with the same 'shared_state_key' attribute, or None if the module doesn't elect to share state
     '''
-    module_type = module_xml.tag
-    module_class = xmodule.get_module_class(module_type)
-    module_id = module_xml.get('id')
+    descriptor = modulestore().get_item(location)
 
-    # Grab xmodule state from StudentModule cache
-    smod = smod_cache_lookup(student_module_cache, module_type, module_id)
-    state = smod.state if smod else None
+    instance_module = student_module_cache.lookup(descriptor.category, descriptor.location.url())
+    shared_state_key = getattr(descriptor, 'shared_state_key', None)
+    if shared_state_key is not None:
+        shared_module = student_module_cache.lookup(descriptor.category, shared_state_key)
+    else:
+        shared_module = None
 
-    # get coursename if present in request
-    # coursename = multicourse_settings.get_coursename_from_request(request)
-
-    # if coursename and settings.ENABLE_MULTICOURSE:
-    #     # path to XML for the course
-    #     xp = multicourse_settings.get_course_xmlpath(coursename)
-    #     data_root = settings.DATA_DIR + xp
-    # else:
-
-    data_root = course.path
+    instance_state = instance_module.state if instance_module is not None else None
+    shared_state = shared_module.state if shared_module is not None else None
 
     # Setup system context for module instance
-    ajax_url = settings.MITX_ROOT_URL + '/modx/' + module_type + '/' + module_id + '/'
+    ajax_url = settings.MITX_ROOT_URL + '/modx/' + descriptor.location.url() + '/'
 
-    module_from_xml = make_module_from_xml_fn(
-        user, request, student_module_cache, course, position)
-    
-    system = I4xSystem(track_function = make_track_function(request), 
-                       render_function = lambda xml: render_x_module(
-                           user, request, xml, student_module_cache, course, position),
-                       render_template = render_to_string,
-                       ajax_url = ajax_url,
-                       request = request,
-                       filestore = OSFS(data_root),
-                       module_from_xml = module_from_xml,
+    def _get_module(location):
+        (module, _, _, _) = get_module(user, request, location, student_module_cache, position)
+        return module
+
+    system = I4xSystem(track_function=make_track_function(request),
+                       render_template=render_to_string,
+                       ajax_url=ajax_url,
+                       # TODO (cpennington): Figure out how to share info between systems
+                       filestore=descriptor.system.resources_fs,
+                       get_module=_get_module,
+                       user=user,
                        )
     # pass position specified in URL to module through I4xSystem
-    system.set('position', position) 
-    instance = module_class(system, 
-                            etree.tostring(module_xml), 
-                            module_id, 
-                            state=state)
+    system.set('position', position)
+
+    module = descriptor.xmodule_constructor(system)(instance_state, shared_state)
+
+    if settings.MITX_FEATURES.get('DISPLAY_HISTOGRAMS_TO_STAFF') and user.is_staff:
+        module = add_histogram(module)
 
     # If StudentModule for this instance wasn't already in the database,
     # and this isn't a guest user, create it.
-    if not smod and user.is_authenticated():
-        smod = StudentModule(student=user, module_type = module_type,
-                           module_id=module_id, state=instance.get_state())
-        smod.save()
-        # Add to cache. The caller and the system context have references
-        # to it, so the change persists past the return
-        student_module_cache.append(smod)
+    if user.is_authenticated():
+        if not instance_module:
+            instance_module = StudentModule(
+                student=user,
+                module_type=descriptor.category,
+                module_state_key=module.id,
+                state=module.get_instance_state(),
+                max_grade=module.max_score())
+            instance_module.save()
+            # Add to cache. The caller and the system context have references
+            # to it, so the change persists past the return
+            student_module_cache.append(instance_module)
+        if not shared_module and shared_state_key is not None:
+            shared_module = StudentModule(
+                student=user,
+                module_type=descriptor.category,
+                module_state_key=shared_state_key,
+                state=module.get_shared_state())
+            shared_module.save()
+            student_module_cache.append(shared_module)
 
-    return (instance, smod, module_type)
+    return (module, instance_module, shared_module, descriptor.category)
 
-def render_x_module(user, request, module_xml, student_module_cache, course, position=None):
-    ''' Generic module for extensions. This renders to HTML.
 
-    modules include sequential, vertical, problem, video, html
+def add_histogram(module):
+    original_get_html = module.get_html
 
-    Note that modules can recurse.  problems, video, html, can be inside sequential or vertical.
-
-    Arguments:
-
-      - user                  : current django User
-      - request               : current django HTTPrequest
-      - module_xml            : lxml etree of xml subtree for the current module
-      - student_module_cache : list of StudentModule objects, one of which may match this module type and id
-      - position   	      : extra information from URL for user-specified position within module
-
-    Returns:
-
-      - dict which is context for HTML rendering of the specified module.  Will have
-      key 'content', and will have 'type' key if passed a valid module.
-    '''
-    if module_xml is None :
-        return {"content": ""}
-
-    (instance, smod, module_type) = get_module(
-        user, request, module_xml, student_module_cache, course, position)
-
-    content = instance.get_html()
-
-    # special extra information about each problem, only for users who are staff 
-    if settings.MITX_FEATURES.get('DISPLAY_HISTOGRAMS_TO_STAFF') and user.is_staff:
-        module_id = module_xml.get('id')
+    def get_html():
+        module_id = module.id
         histogram = grade_histogram(module_id)
         render_histogram = len(histogram) > 0
-        staff_context = {'xml': etree.tostring(module_xml), 
-                         'module_id': module_id,
+
+        # Cast module.definition and module.metadata to dicts so that json can dump them
+        # even though they are lazily loaded
+        staff_context = {'definition': json.dumps(dict(module.definition), indent=4),
+                         'metadata': json.dumps(dict(module.metadata), indent=4),
+                         'element_id': module.location.html_id(),
                          'histogram': json.dumps(histogram),
-                         'render_histogram': render_histogram}
-        content += render_to_string("staff_problem_info.html", staff_context)
+                         'render_histogram': render_histogram,
+                         'module_content': original_get_html()}
+        return render_to_string("staff_problem_info.html", staff_context)
 
-    context = {'content': content, 'type': module_type}
-    return context
+    module.get_html = get_html
+    return module
 
-def modx_dispatch(request, module=None, dispatch=None, id=None):
+
+def modx_dispatch(request, dispatch=None, id=None):
     ''' Generic view for extensions. This is where AJAX calls go.
 
     Arguments:
 
       - request -- the django request.
-      - module -- the type of the module, as used in the course configuration xml.
-                  e.g. 'problem', 'video', etc
       - dispatch -- the command string to pass through to the module's handle_ajax call
            (e.g. 'problem_reset').  If this string contains '?', only pass
            through the part before the first '?'.
-      - id -- the module id.  Used to look up the student module.
-            e.g. filenamexformularesponse
+      - id -- the module id. Used to look up the XModule instance
     '''
     # ''' (fix emacs broken parsing)
-    if not request.user.is_authenticated():
-        return redirect('/')
-
-    # python concats adjacent strings
-    error_msg = ("We're sorry, this module is temporarily unavailable. "
-                 "Our staff is working to fix it as soon as possible")
-
-
-    # Grab the student information for the module from the database
-    s = StudentModule.objects.filter(student=request.user, 
-                                     module_id=id)
-
-    if s is None or len(s) == 0:
-        log.debug("Couldn't find module '%s' for user '%s' and id '%s'",
-                  module, request.user, id)
-        raise Http404
-    s = s[0]
-
-    oldgrade = s.grade
-    oldstate = s.state
 
     # If there are arguments, get rid of them
     dispatch, _, _ = dispatch.partition('?')
 
-    ajax_url = '{root}/modx/{module}/{id}'.format(root = settings.MITX_ROOT_URL,
-                                                  module=module, id=id)
-    coursename = multicourse_settings.get_coursename_from_request(request)
-    if coursename and settings.ENABLE_MULTICOURSE:
-        xp = multicourse_settings.get_course_xmlpath(coursename)
-        data_root = settings.DATA_DIR + xp
-    else:
-        data_root = settings.DATA_DIR
+    student_module_cache = StudentModuleCache(request.user, modulestore().get_item(id))
+    instance, instance_module, shared_module, module_type = get_module(request.user, request, id, student_module_cache)
 
-    # Grab the XML corresponding to the request from course.xml
-    try:
-        xml = content_parser.module_xml(request.user, module, 'id', id, coursename)
-    except:
-        log.exception(
-            "Unable to load module during ajax call. module=%s, dispatch=%s, id=%s",
-            module, dispatch, id)
-        if accepts(request, 'text/html'):
-            return render_to_response("module-error.html", {})
-        else:
-            response = HttpResponse(json.dumps({'success': error_msg}))
-        return response
+    if instance_module is None:
+        log.debug("Couldn't find module '%s' for user '%s'",
+                  id, request.user)
+        raise Http404
 
-    # TODO: This doesn't have a cache of child student modules.  Just
-    # passing the current one.  If ajax calls end up needing children,
-    # this won't work (but fixing it may cause performance issues...)
-    # Figure out :)
-    module_from_xml = make_module_from_xml_fn(
-        request.user, request, [s], None)
-
-    # Create the module
-    system = I4xSystem(track_function = make_track_function(request), 
-                       render_function = None,
-                       module_from_xml = module_from_xml,
-                       render_template = render_to_string,
-                       ajax_url = ajax_url,
-                       request = request,
-                       filestore = OSFS(data_root),
-                       )
-
-    try:
-        module_class = xmodule.get_module_class(module)
-        instance = module_class(system, xml, id, state=oldstate)
-    except:
-        log.exception("Unable to load module instance during ajax call")
-        if accepts(request, 'text/html'):
-            return render_to_response("module-error.html", {})
-        else:
-            response = HttpResponse(json.dumps({'success': error_msg}))
-        return response
+    oldgrade = instance_module.grade
+    old_instance_state = instance_module.state
+    old_shared_state = shared_module.state if shared_module is not None else None
 
     # Let the module handle the AJAX
     try:
@@ -354,10 +314,16 @@ def modx_dispatch(request, module=None, dispatch=None, id=None):
         raise
 
     # Save the state back to the database
-    s.state = instance.get_state()
-    if instance.get_score(): 
-        s.grade = instance.get_score()['score']
-    if s.grade != oldgrade or s.state != oldstate:
-        s.save()
+    instance_module.state = instance.get_instance_state()
+    if instance.get_score():
+        instance_module.grade = instance.get_score()['score']
+    if instance_module.grade != oldgrade or instance_module.state != old_instance_state:
+        instance_module.save()
+
+    if shared_module is not None:
+        shared_module.state = instance.get_shared_state()
+        if shared_module.state != old_shared_state:
+            shared_module.save()
+
     # Return whatever the module wanted to return to the client/caller
     return HttpResponse(ajax_return)

lms/djangoapps/courseware/views.py

@@ -1,8 +1,6 @@
 import logging
 import urllib
 
-from fs.osfs import OSFS
-
 from django.conf import settings
 from django.core.context_processors import csrf
 from django.contrib.auth.models import User
@@ -14,24 +12,42 @@ from mitxmako.shortcuts import render_to_response, render_to_string
 from django_future.csrf import ensure_csrf_cookie
 from django.views.decorators.cache import cache_control
 
-from lxml import etree
-
-from module_render import render_x_module, make_track_function, I4xSystem
-from models import StudentModule
+from module_render import toc_for_course, get_module, get_section
+from models import StudentModuleCache
 from student.models import UserProfile
 from multicourse import multicourse_settings
-import xmodule
+from xmodule.modulestore.django import modulestore
 
-import courseware.content_parser as content_parser
-
-import courseware.grades as grades
+from util.cache import cache
+from student.models import UserTestGroup
+from courseware import grades
 
 log = logging.getLogger("mitx.courseware")
 
-etree.set_default_parser(etree.XMLParser(dtd_validation=False, load_dtd=False,
-                                         remove_comments = True))
+template_imports = {'urllib': urllib}
+
+
+def user_groups(user):
+    if not user.is_authenticated():
+        return []
+
+    # TODO: Rewrite in Django
+    key = 'user_group_names_{user.id}'.format(user=user)
+    cache_expiration = 60 * 60  # one hour
+
+    # Kill caching on dev machines -- we switch groups a lot
+    group_names = cache.get(key)
+
+    if group_names is None:
+        group_names = [u.name for u in UserTestGroup.objects.filter(users=user)]
+        cache.set(key, group_names, cache_expiration)
+
+    return group_names
+
+
+def format_url_params(params):
+    return [urllib.quote(string.replace(' ', '_')) for string in params]
 
-template_imports={'urllib':urllib}
 
 @ensure_csrf_cookie
 def courses(request):
@@ -43,37 +59,47 @@ def courses(request):
 
 @cache_control(no_cache=True, no_store=True, must_revalidate=True)
 def gradebook(request):
-    if 'course_admin' not in content_parser.user_groups(request.user):
+    if 'course_admin' not in user_groups(request.user):
         raise Http404
 
     course = settings.COURSES_BY_ID[course_id]
 
     student_objects = User.objects.all()[:100]
-    student_info = [{'username': s.username,
-                     'id': s.id,
-                     'email': s.email,
-                     'grade_info': grades.grade_sheet(s, course),
-                     'realname': UserProfile.objects.get(user = s).name
-                     } for s in student_objects]
+    student_info = []
+
+    for student in student_objects:
+        # TODO (cpennington): do the right thing with courses
+        student_module_cache = StudentModuleCache(student, modulestore().get_item(course_location))
+        course, _, _, _ = get_module(request.user, request, course_location, student_module_cache)
+        student_info.append({
+            'username': student.username,
+            'id': student.id,
+            'email': student.email,
+            'grade_info': grades.grade_sheet(student, course, student_module_cache),
+            'realname': UserProfile.objects.get(user=student).name
+        })
 
     return render_to_response('gradebook.html', {'students': student_info, 'course': course})
 
+
 @login_required
 @cache_control(no_cache=True, no_store=True, must_revalidate=True)
-def profile(request, course_id=None, student_id=None):
+def profile(request, student_id=None):
     ''' User profile. Show username, location, etc, as well as grades .
         We need to allow the user to change some of these settings .'''
     course = settings.COURSES_BY_ID[course_id]
     if student_id is None:
         student = request.user
     else:
-        if 'course_admin' not in content_parser.user_groups(request.user):
+        if 'course_admin' not in user_groups(request.user):
             raise Http404
-        student = User.objects.get( id = int(student_id))
+        student = User.objects.get(id=int(student_id))
 
-    user_info = UserProfile.objects.get(user=student) # request.user.profile_cache #
+    user_info = UserProfile.objects.get(user=student)
 
-    # coursename = multicourse_settings.get_coursename_from_request(request)
+    # TODO (cpennington): do the right thing with courses
+    student_module_cache = StudentModuleCache(request.user, modulestore().get_item(course_location))
+    course, _, _, _ = get_module(request.user, request, course_location, student_module_cache)
 
     context = {'name': user_info.name,
                'username': student.username,
@@ -84,7 +110,7 @@ def profile(request, course_id=None, student_id=None):
                'format_url_params': content_parser.format_url_params,
                'csrf': csrf(request)['csrf_token']
                }
-    context.update(grades.grade_sheet(student, course))
+    context.update(grades.grade_sheet(student, course, student_module_cache))
 
     return render_to_response('profile.html', context)
 
@@ -96,91 +122,24 @@ def render_accordion(request, course, chapter, section):
         If chapter and section are '' or None, renders a default accordion.
 
         Returns (initialization_javascript, content)'''
-    # if not course:
-    #     course = "6.002 Spring 2012"
 
-    toc = content_parser.toc_from_xml(
-        content_parser.course_file(request.user, course), chapter, section)
+    # TODO (cpennington): do the right thing with courses
+    toc = toc_for_course(request.user, request, course_location, chapter, section)
 
     active_chapter = 1
     for i in range(len(toc)):
         if toc[i]['active']:
             active_chapter = i
 
-    log.info(course.title)
-    context=dict([('active_chapter', active_chapter),
-                  ('toc', toc),
-                  ('course_name', course.title),
-                  ('course_id', course.id),
-                  ('format_url_params', content_parser.format_url_params),
-                  ('csrf', csrf(request)['csrf_token'])] +
-                     template_imports.items())
+    context = dict([('active_chapter', active_chapter),
+                    ('toc', toc),
+                    ('course_name', course.title),
+                    ('course_id', course.id),
+                    ('format_url_params', format_url_params),
+                    ('csrf', csrf(request)['csrf_token'])] + template_imports.items())
     return render_to_string('accordion.html', context)
 
 
-@cache_control(no_cache=True, no_store=True, must_revalidate=True)
-def render_section(request, section):
-    ''' TODO: Consolidate with index
-    '''
-    user = request.user
-    if not settings.COURSEWARE_ENABLED:
-        return redirect('/')
-
-    coursename = multicourse_settings.get_coursename_from_request(request)
-
-    try:
-        dom = content_parser.section_file(user, section, coursename)
-    except:
-        log.exception("Unable to parse courseware xml")
-        return render_to_response('courseware-error.html', {})
-
-    context = {
-        'csrf': csrf(request)['csrf_token'],
-        'accordion': render_accordion(request, course, '', '')
-    }
-
-    module_ids = dom.xpath("//@id")
-
-    if user.is_authenticated():
-        student_module_cache = list(StudentModule.objects.filter(student=user,
-                                                                  module_id__in=module_ids))
-    else:
-        student_module_cache = []
-
-    try:
-        module = render_x_module(user, request, dom, student_module_cache)
-    except:
-        log.exception("Unable to load module")
-        context.update({
-            'init': '',
-            'content': render_to_string("module-error.html", {}),
-        })
-        return render_to_response('courseware.html', context)
-
-    context.update({
-        'init': module.get('init_js', ''),
-        'content': module['content'],
-    })
-
-    result = render_to_response('courseware.html', context)
-    return result
-
-def get_course(request, course):
-    ''' Figure out what the correct course is.
-
-    Needed to preserve backwards compatibility with non-multi-course version.
-    TODO: Can this go away once multicourse becomes standard?
-    '''
-
-    if course==None:
-        if not settings.ENABLE_MULTICOURSE:
-            course = "6.002 Spring 2012"
-        elif 'coursename' in request.session:
-            course = request.session['coursename']
-        else:
-            course = settings.COURSE_DEFAULT
-    return course
-
 def get_module_xml(user, course, chapter, section):
     ''' Look up the module xml for the given course/chapter/section path.
 
@@ -239,69 +198,9 @@ def index(request, course=None, chapter=None, section=None,
         '''
         return s.replace('_', ' ') if s is not None else None
 
-    def get_submodule_ids(module_xml):
-        '''
-        Get a list with ids of the modules within this module.
-        '''
-        return module_xml.xpath("//@id")
-
-    def preload_student_modules(module_xml):
-        '''
-        Find any StudentModule objects for this user that match
-        one of the given module_ids.  Used as a cache to avoid having
-        each rendered module hit the db separately.
-
-        Returns the list, or None on error.
-        '''
-        if request.user.is_authenticated():
-            module_ids = get_submodule_ids(module_xml)
-            return list(StudentModule.objects.filter(student=request.user,
-                                                     module_id__in=module_ids))
-        else:
-            return []
-
-    def get_module_context():
-        '''
-        Look up the module object and render it.  If all goes well, returns
-        {'init': module-init-js, 'content': module-rendered-content}
-
-        If there's an error, returns
-        {'content': module-error message}
-        '''
-        user = request.user
-
-        module_xml = get_module_xml(user, course, chapter, section)
-        if module_xml is None:
-            log.exception("couldn't get module_xml: course/chapter/section: '%s/%s/%s'",
-                          course, chapter, section)
-            return {'content' : render_to_string("module-error.html", {})}
-
-        student_module_cache = preload_student_modules(module_xml)
-
-        try:
-            module_context = render_x_module(user, request, module_xml,
-                                             student_module_cache, course, position)
-        except:
-            log.exception("Unable to load module")
-            return {'content' : render_to_string("module-error.html", {})}
-
-        return {'init': module_context.get('init_js', ''),
-                'content': module_context['content']}
-
     if not settings.COURSEWARE_ENABLED:
         return redirect('/')
 
-    # course = clean(get_course(request, course))
-    # if not multicourse_settings.is_valid_course(course):
-    #     return redirect('/')
-    try:
-        course = settings.COURSES_BY_ID[course_id]
-    except KeyError:
-        raise Http404("Course not found")
-
-    # keep track of current course being viewed in django's request.session
-    request.session['coursename'] = course.title
-
     chapter = clean(chapter)
     section = clean(section)
 
@@ -316,11 +215,16 @@ def index(request, course=None, chapter=None, section=None,
 
     look_for_module = chapter is not None and section is not None
     if look_for_module:
-        context.update(get_module_context())
+        # TODO (cpennington): Pass the right course in here
+        section = get_section(course_location, chapter, section)
+        student_module_cache = StudentModuleCache(request.user, section)
+        module, _, _, _ = get_module(request.user, request, section.location, student_module_cache)
+        context['content'] = module.get_html()
 
     result = render_to_response('courseware.html', context)
     return result
 
+
 def jump_to(request, probname=None):
     '''
     Jump to viewing a specific problem.  The problem is specified by a
@@ -343,7 +247,8 @@ def jump_to(request, probname=None):
 
     # look for problem of given name
     pxml = xml.xpath('//problem[@filename="%s"]' % probname)
-    if pxml: pxml = pxml[0]
+    if pxml:
+        pxml = pxml[0]
 
     # get the parent element
     parent = pxml.getparent()
@@ -352,7 +257,7 @@ def jump_to(request, probname=None):
     chapter = None
     section = None
     branch = parent
-    for k in range(4):	# max depth of recursion
+    for k in range(4):  # max depth of recursion
         if branch.tag == 'section':
             section = branch.get('name')
         if branch.tag == 'chapter':
@@ -361,19 +266,13 @@ def jump_to(request, probname=None):
 
     position = None
     if parent.tag == 'sequential':
-        position = parent.index(pxml) + 1	# position in sequence
+        position = parent.index(pxml) + 1  # position in sequence
 
     return index(request,
                  course=coursename, chapter=chapter,
                  section=section, position=position)
 
 
-@ensure_csrf_cookie
-def course_info(request):
-  csrf_token = csrf(request)['csrf_token']
-  # TODO: Couse should be a model
-  return render_to_response('portal/course_info.html', {'csrf': csrf_token })
-
 @ensure_csrf_cookie
 def course_info(request, course_id):
     # This is the advertising page for a student to look at the course before signing up

lms/djangoapps/multicourse/multicourse_settings.py

@@ -31,11 +31,13 @@ if hasattr(settings,'COURSE_SETTINGS'):    	# in the future, this could be repla
 elif hasattr(settings,'COURSE_NAME'):		# backward compatibility
     COURSE_SETTINGS = {settings.COURSE_NAME: {'number': settings.COURSE_NUMBER,
                                               'title':  settings.COURSE_TITLE,
+                                              'location': settings.COURSE_LOCATION,
                                               },
                        }
 else:						# default to 6.002_Spring_2012
     COURSE_SETTINGS = {'6.002_Spring_2012': {'number': '6.002x',
                                               'title':  'Circuits and Electronics',
+                                              'location': 'i4x://edx/6002xs12/course/6.002 Spring 2012',
                                               },
                        }
 
@@ -51,31 +53,47 @@ def get_coursename_from_request(request):
 
 def get_course_settings(coursename):
     if not coursename:
-        if hasattr(settings,'COURSE_DEFAULT'):
+        if hasattr(settings, 'COURSE_DEFAULT'):
             coursename = settings.COURSE_DEFAULT
         else:
             coursename = '6.002_Spring_2012'
-    if coursename in COURSE_SETTINGS: return COURSE_SETTINGS[coursename]
-    coursename = coursename.replace(' ','_')
-    if coursename in COURSE_SETTINGS: return COURSE_SETTINGS[coursename]
+    if coursename in COURSE_SETTINGS:
+        return COURSE_SETTINGS[coursename]
+    coursename = coursename.replace(' ', '_')
+    if coursename in COURSE_SETTINGS:
+        return COURSE_SETTINGS[coursename]
     return None
 
+
 def is_valid_course(coursename):
     return get_course_settings(coursename) != None
 
-def get_course_property(coursename,property):
+
+def get_course_property(coursename, property):
     cs = get_course_settings(coursename)
-    if not cs: return ''	# raise exception instead?
-    if property in cs: return cs[property]
-    return ''	# default
+
+    # raise exception instead?
+    if not cs:
+        return ''
+
+    if property in cs:
+        return cs[property]
+
+    # default
+    return ''
+
 
 def get_course_xmlpath(coursename):
-    return get_course_property(coursename,'xmlpath')
+    return get_course_property(coursename, 'xmlpath')
+
 
 def get_course_title(coursename):
-    return get_course_property(coursename,'title')
+    return get_course_property(coursename, 'title')
+
 
 def get_course_number(coursename):
-    return get_course_property(coursename,'number')
-    
+    return get_course_property(coursename, 'number')
 
+
+def get_course_location(coursename):
+    return get_course_property(coursename, 'location')

lms/envs/common.py

@@ -138,10 +138,25 @@ COURSE_DEFAULT = '6.002_Spring_2012'
 COURSE_SETTINGS =  {'6.002_Spring_2012': {'number' : '6.002x',
                                           'title'  :  'Circuits and Electronics',
                                           'xmlpath': '6002x/',
+                                          'location': 'i4x://edx/6002xs12/course/6.002_Spring_2012',
                                           }
                     }
 
 
+############################### XModule Store ##################################
+MODULESTORE = {
+    'default': {
+        'ENGINE': 'xmodule.modulestore.xml.XMLModuleStore',
+        'OPTIONS': {
+            'org': 'edx',
+            'course': '6002xs12',
+            'data_dir': DATA_DIR,
+            'default_class': 'xmodule.hidden_module.HiddenDescriptor',
+        }
+    }
+}
+
+
 ############################### DJANGO BUILT-INS ###############################
 # Change DEBUG/TEMPLATE_DEBUG in your environment settings files, not here
 DEBUG = False

lms/envs/dev.py

@@ -11,7 +11,7 @@ from .common import *
 from .logsettings import get_logger_config
 
 DEBUG = True
-TEMPLATE_DEBUG = True
+TEMPLATE_DEBUG = False
 
 LOGGING = get_logger_config(ENV_ROOT / "log", 
                             logging_env="dev",

lms/envs/logsettings.py

@@ -77,7 +77,7 @@ def get_logger_config(log_dir,
                 'level' : 'DEBUG',
                 'propagate' : False,
             },
-            'root' : {
+            '' : {
                 'handlers' : handlers,
                 'level' : 'DEBUG',
                 'propagate' : False

lms/lib/dogfood/views.py

@@ -174,7 +174,7 @@ def quickedit(request, id=None, qetemplate='quickedit.html',coursename=None):
         module = 'problem'
         xml = content_parser.module_xml(request.user, module, 'id', id, coursename)
     
-        ajax_url = settings.MITX_ROOT_URL + '/modx/'+module+'/'+id+'/'
+        ajax_url = settings.MITX_ROOT_URL + '/modx/'+id+'/'
     
         # Create the module (instance of capa_module.Module)
         system = I4xSystem(track_function = make_track_function(request), 
@@ -184,29 +184,29 @@ def quickedit(request, id=None, qetemplate='quickedit.html',coursename=None):
                            filestore = OSFS(settings.DATA_DIR + xp),
                            #role = 'staff' if request.user.is_staff else 'student',		# TODO: generalize this
                            )
-        instance=xmodule.get_module_class(module)(system, 
-                                                             xml, 
+        instance = xmodule.get_module_class(module)(system,
+                                                             xml,
                                                              id,
                                                              state=None)
         log.info('ajax_url = ' + instance.ajax_url)
 
         # create empty student state for this problem, if not previously existing
-        s = StudentModule.objects.filter(student=request.user, 
-                                         module_id=id)
+        s = StudentModule.objects.filter(student=request.user,
+                                         module_state_key=id)
         if len(s) == 0 or s is None:
-            smod=StudentModule(student=request.user, 
-                               module_type = 'problem',
-                               module_id=id, 
-                               state=instance.get_state())
+            smod = StudentModule(student=request.user,
+                                 module_type='problem',
+                                 module_state_key=id,
+                                 state=instance.get_instance_state())
             smod.save()
 
         lcp = instance.lcp
         pxml = lcp.tree
-        pxmls = etree.tostring(pxml,pretty_print=True)
+        pxmls = etree.tostring(pxml, pretty_print=True)
 
         return instance, pxmls
 
-    instance, pxmls = get_lcp(coursename,id)
+    instance, pxmls = get_lcp(coursename, id)
 
     # if there was a POST, then process it
     msg = ''
@@ -246,8 +246,6 @@ def quickedit(request, id=None, qetemplate='quickedit.html',coursename=None):
     # get the rendered problem HTML
     phtml = instance.get_html()
     # phtml = instance.get_problem_html()
-    # init_js = instance.get_init_js()
-    # destory_js = instance.get_destroy_js()
 
     context = {'id':id,
                'msg' : msg,

lms/static/coffee/spec/calculator_spec.coffee

@@ -12,8 +12,8 @@ describe 'Calculator', ->
 
     it 'bind the help button', ->
       # These events are bind by $.hover()
-      expect($('div.help-wrapper a')).toHandleWith 'mouseenter', @calculator.helpToggle
-      expect($('div.help-wrapper a')).toHandleWith 'mouseleave', @calculator.helpToggle
+      expect($('div.help-wrapper a')).toHandleWith 'mouseover', @calculator.helpToggle
+      expect($('div.help-wrapper a')).toHandleWith 'mouseout', @calculator.helpToggle
 
     it 'prevent default behavior on help button', ->
       $('div.help-wrapper a').click (e) ->

lms/static/coffee/spec/courseware_spec.coffee

@@ -34,7 +34,7 @@ describe 'Courseware', ->
         <div class="course-content">
           <div id="video_1" class="video" data-streams="1.0:abc1234"></div>
           <div id="video_2" class="video" data-streams="1.0:def5678"></div>
-          <div id="problem_3" class="problems-wrapper" data-url="/example/url/">
+          <div id="problem_3" class="problems-wrapper" data-problem-id="3" data-url="/example/url/">
             <div id="histogram_3" class="histogram" data-histogram="[[0, 1]]" style="height: 20px; display: block;">
           </div>
         </div>
@@ -46,7 +46,7 @@ describe 'Courseware', ->
       expect(window.Video).toHaveBeenCalledWith('2', '1.0:def5678')
 
     it 'detect the problem element and convert it', ->
-      expect(window.Problem).toHaveBeenCalledWith('3', '/example/url/')
+      expect(window.Problem).toHaveBeenCalledWith(3, 'problem_3', '/example/url/')
 
     it 'detect the histrogram element and convert it', ->
       expect(window.Histogram).toHaveBeenCalledWith('3', [[0, 1]])

lms/static/coffee/spec/helper.coffee

@@ -1,14 +1,14 @@
 jasmine.getFixtures().fixturesPath = "/_jasmine/fixtures/"
 
 jasmine.stubbedMetadata =
-  abc123:
-    id: 'abc123'
-    duration: 100
-  def456:
-    id: 'def456'
+  slowerSpeedYoutubeId:
+    id: 'slowerSpeedYoutubeId'
+    duration: 300
+  normalSpeedYoutubeId:
+    id: 'normalSpeedYoutubeId'
     duration: 200
   bogus:
-    duration: 300
+    duration: 100
 
 jasmine.stubbedCaption =
   start: [0, 10000, 20000, 30000]
@@ -20,10 +20,12 @@ jasmine.stubRequests = ->
       settings.success data: jasmine.stubbedMetadata[match[1]]
     else if match = settings.url.match /static\/subs\/(.+)\.srt\.sjson/
       settings.success jasmine.stubbedCaption
+    else if settings.url.match /modx\/.+\/problem_get$/
+      settings.success html: readFixtures('problem_content.html')
     else if settings.url == '/calculate' ||
-      settings.url == '/6002x/modx/sequence/1/goto_position' ||
+      settings.url.match(/modx\/.+\/goto_position$/) ||
       settings.url.match(/event$/) ||
-      settings.url.match(/6002x\/modx\/problem\/.+\/problem_(check|reset|show|save)$/)
+      settings.url.match(/modx\/.+\/problem_(check|reset|show|save)$/)
       # do nothing
     else
       throw "External request attempted for #{settings.url}, which is not defined."
@@ -47,10 +49,10 @@ jasmine.stubVideoPlayer = (context, enableParts, createPlayer=true) ->
   loadFixtures 'video.html'
   jasmine.stubRequests()
   YT.Player = undefined
-  context.video = new Video 'example', '.75:abc123,1.0:def456'
+  context.video = new Video 'example', '.75:slowerSpeedYoutubeId,1.0:normalSpeedYoutubeId'
   jasmine.stubYoutubePlayer()
   if createPlayer
-    return new VideoPlayer context.video
+    return new VideoPlayer(video: context.video)
 
 spyOn(window, 'onunload')
 

lms/static/coffee/spec/logger_spec.coffee

@@ -25,7 +25,7 @@ describe 'Logger', ->
 
     it 'send a request to log event', ->
       spyOn($, 'ajax')
-      $(window).trigger('onunload')
+      window.onunload()
       expect($.ajax).toHaveBeenCalledWith
         url: "#{Courseware.prefix}/event",
         data: