Merge remote-tracking branch 'origin/master' into bugfix/brian/openid_provider_post

common/djangoapps/cache_toolbox/core.py

@@ -12,7 +12,7 @@ from django.core.cache import cache
 from django.db import DEFAULT_DB_ALIAS
 
 from . import app_settings
-
+from xmodule.contentstore.content import StaticContent
 
 def get_instance(model, instance_or_pk, timeout=None, using=None):
     """
@@ -108,14 +108,11 @@ def instance_key(model, instance_or_pk):
         getattr(instance_or_pk, 'pk', instance_or_pk),
     )
 
-def content_key(filename):
-    return 'content:%s' % (filename)
-
 def set_cached_content(content):
-    cache.set(content_key(content.filename), content)
+    cache.set(str(content.location), content)
 
-def get_cached_content(filename):
-    return cache.get(content_key(filename))
+def get_cached_content(location):
+    return cache.get(str(location))
 
-def del_cached_content(filename):
-    cache.delete(content_key(filename))
+def del_cached_content(location):
+    cache.delete(str(location))

common/djangoapps/contentserver/middleware.py

@@ -12,14 +12,14 @@ from xmodule.exceptions import NotFoundError
 class StaticContentServer(object):
     def process_request(self, request):
         # look to see if the request is prefixed with 'c4x' tag
-        if request.path.startswith('/' + XASSET_LOCATION_TAG):
-
+        if request.path.startswith('/' + XASSET_LOCATION_TAG +'/'):
+            loc = StaticContent.get_location_from_path(request.path)
             # first look in our cache so we don't have to round-trip to the DB
-            content = get_cached_content(request.path)
+            content = get_cached_content(loc)
             if content is None:
                 # nope, not in cache, let's fetch from DB
                 try:
-                    content = contentstore().find(request.path)
+                    content = contentstore().find(loc)
                 except NotFoundError:
                     raise Http404
 

common/djangoapps/external_auth/views.py

@@ -215,6 +215,52 @@ def ssl_dn_extract_info(dn):
     else:
         return None
     return (user, email, fullname)
+    
+
+def ssl_get_cert_from_request(request):
+    """
+    Extract user information from certificate, if it exists, returning (user, email, fullname).
+    Else return None.
+    """
+    certkey = "SSL_CLIENT_S_DN"  # specify the request.META field to use
+
+    cert = request.META.get(certkey, '')
+    if not cert:
+        cert = request.META.get('HTTP_' + certkey, '')
+    if not cert:
+        try:
+            # try the direct apache2 SSL key
+            cert = request._req.subprocess_env.get(certkey, '')
+        except Exception:
+            return ''
+
+    return cert
+
+    (user, email, fullname) = ssl_dn_extract_info(cert)
+    return (user, email, fullname)
+
+
+def ssl_login_shortcut(fn):
+    """
+    Python function decorator for login procedures, to allow direct login
+    based on existing ExternalAuth record and MIT ssl certificate.
+    """
+    def wrapped(*args, **kwargs):
+        if not settings.MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES']:
+            return fn(*args, **kwargs)
+        request = args[0]
+        cert = ssl_get_cert_from_request(request)
+        if not cert:		# no certificate information - show normal login window
+            return fn(*args, **kwargs)
+
+        (user, email, fullname) = ssl_dn_extract_info(cert)
+        return external_login_or_signup(request,
+                                        external_id=email,
+                                        external_domain="ssl:MIT",
+                                        credentials=cert,
+                                        email=email,
+                                        fullname=fullname)
+    return wrapped
 
 
 @csrf_exempt
@@ -234,17 +280,7 @@ def ssl_login(request):
 
     Else continues on with student.views.index, and no authentication.
     """
-    certkey = "SSL_CLIENT_S_DN"  # specify the request.META field to use
-
-    cert = request.META.get(certkey, '')
-    if not cert:
-        cert = request.META.get('HTTP_' + certkey, '')
-    if not cert:
-        try:
-            # try the direct apache2 SSL key
-            cert = request._req.subprocess_env.get(certkey, '')
-        except Exception:
-            cert = None
+    cert = ssl_get_cert_from_request(request)
 
     if not cert:
         # no certificate information - go onward to main index

common/djangoapps/heartbeat/urls.py

@@ -0,0 +1,5 @@
+from django.conf.urls import *
+
+urlpatterns = patterns('',
+    url(r'^$', 'heartbeat.views.heartbeat', name='heartbeat'),
+)

common/djangoapps/heartbeat/views.py

@@ -0,0 +1,15 @@
+import json
+from datetime import datetime
+from django.http import HttpResponse
+from xmodule.modulestore.django import modulestore
+
+
+def heartbeat(request):
+    """
+    Simple view that a loadbalancer can check to verify that the app is up
+    """
+    output = {
+        'date': datetime.now().isoformat(),
+        'courses': [course.location.url() for course in modulestore().get_courses()],
+    }
+    return HttpResponse(json.dumps(output, indent=4))

common/djangoapps/static_replace.py

@@ -5,6 +5,10 @@ from staticfiles.storage import staticfiles_storage
 from staticfiles import finders
 from django.conf import settings
 
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.xml import XMLModuleStore
+from xmodule.contentstore.content import StaticContent
+
 log = logging.getLogger(__name__)
 
 def try_staticfiles_lookup(path):
@@ -22,7 +26,7 @@ def try_staticfiles_lookup(path):
     return url
 
 
-def replace(static_url, prefix=None):
+def replace(static_url, prefix=None, course_namespace=None):
     if prefix is None:
         prefix = ''
     else:
@@ -41,13 +45,24 @@ def replace(static_url, prefix=None):
         return static_url.group(0)
     else:
         # don't error if file can't be found
-        url = try_staticfiles_lookup(prefix + static_url.group('rest'))
-        return "".join([quote, url, quote])
+        # cdodge: to support the change over to Mongo backed content stores, lets
+        # use the utility functions in StaticContent.py
+        if static_url.group('prefix') == '/static/' and not isinstance(modulestore(), XMLModuleStore):
+            if course_namespace is None:
+                raise BaseException('You must pass in course_namespace when remapping static content urls with MongoDB stores')
+            url = StaticContent.convert_legacy_static_url(static_url.group('rest'), course_namespace)
+        else:
+            url = try_staticfiles_lookup(prefix + static_url.group('rest'))
+
+        new_link = "".join([quote, url, quote])
+        return new_link
 
 
-def replace_urls(text, staticfiles_prefix=None, replace_prefix='/static/'):
+
+def replace_urls(text, staticfiles_prefix=None, replace_prefix='/static/', course_namespace=None):
+
     def replace_url(static_url):
-        return replace(static_url, staticfiles_prefix)
+        return replace(static_url, staticfiles_prefix, course_namespace = course_namespace)
 
     return re.sub(r"""
         (?x)                 # flags=re.VERBOSE

common/djangoapps/student/models.py

@@ -53,6 +53,7 @@ from django.forms import ModelForm, forms
 
 import comment_client as cc
 
+
 log = logging.getLogger(__name__)
 
 

common/djangoapps/util/converters.py

@@ -0,0 +1,24 @@
+import time, datetime
+import re
+import calendar
+
+def time_to_date(time_obj):
+    """
+    Convert a time.time_struct to a true universal time (can pass to js Date constructor)
+    """
+    # TODO change to using the isoformat() function on datetime. js date can parse those
+    return calendar.timegm(time_obj) * 1000
+
+def jsdate_to_time(field):
+    """
+    Convert a universal time (iso format) or msec since epoch to a time obj
+    """
+    if field is None:
+        return field
+    elif isinstance(field, basestring):  # iso format but ignores time zone assuming it's Z
+        d=datetime.datetime(*map(int, re.split('[^\d]', field)[:6])) # stop after seconds. Debatable  
+        return d.utctimetuple()
+    elif isinstance(field, int) or isinstance(field, float):
+        return time.gmtime(field / 1000)
+    elif isinstance(field, time.struct_time):
+        return field

common/djangoapps/util/json_request.py

@@ -13,7 +13,7 @@ def expect_json(view_function):
     def expect_json_with_cloned_request(request, *args, **kwargs):
         # cdodge: fix postback errors in CMS. The POST 'content-type' header can include additional information
         # e.g. 'charset', so we can't do a direct string compare
-        if request.META['CONTENT_TYPE'].lower().startswith("application/json"):
+        if request.META.get('CONTENT_TYPE','').lower().startswith("application/json"):
             cloned_request = copy.copy(request)
             cloned_request.POST = cloned_request.POST.copy()
             cloned_request.POST.update(json.loads(request.body))

common/djangoapps/xmodule_modifiers.py

@@ -12,7 +12,7 @@ from xmodule.vertical_module import VerticalModule
 
 log = logging.getLogger("mitx.xmodule_modifiers")
 
-def wrap_xmodule(get_html, module, template):
+def wrap_xmodule(get_html, module, template, context=None):
     """
     Wraps the results of get_html in a standard <section> with identifying
     data so that the appropriate javascript module can be loaded onto it.
@@ -21,17 +21,23 @@ def wrap_xmodule(get_html, module, template):
     module: An XModule
     template: A template that takes the variables:
         content: the results of get_html,
+        display_name: the display name of the xmodule, if available (None otherwise)
         class_: the module class name
         module_name: the js_module_name of the module
     """
+    if context is None:
+        context = {}
 
     @wraps(get_html)
     def _get_html():
-        return render_to_string(template, {
+        context.update({
             'content': get_html(),
+            'display_name' : module.metadata.get('display_name') if module.metadata is not None else None,
             'class_': module.__class__.__name__,
             'module_name': module.js_module_name
         })
+
+        return render_to_string(template, context)
     return _get_html
 
 
@@ -46,7 +52,7 @@ def replace_course_urls(get_html, course_id):
         return replace_urls(get_html(), staticfiles_prefix='/courses/'+course_id, replace_prefix='/course/')
     return _get_html
 
-def replace_static_urls(get_html, prefix):
+def replace_static_urls(get_html, prefix, course_namespace=None):
     """
     Updates the supplied module with a new get_html function that wraps
     the old get_html function and substitutes urls of the form /static/...
@@ -55,7 +61,7 @@ def replace_static_urls(get_html, prefix):
 
     @wraps(get_html)
     def _get_html():
-        return replace_urls(get_html(), staticfiles_prefix=prefix)
+        return replace_urls(get_html(), staticfiles_prefix=prefix, course_namespace = course_namespace)
     return _get_html
 
 

common/lib/.gitignore

@@ -0,0 +1 @@
+*/jasmine_test_runner.html

common/lib/capa/capa/capa_problem.py

@@ -34,6 +34,8 @@ import chem
 import chem.chemcalc
 import chem.chemtools
 import chem.miller
+import verifiers
+import verifiers.draganddrop
 
 import calc
 from correctmap import CorrectMap
@@ -69,7 +71,8 @@ global_context = {'random': random,
                   'eia': eia,
                   'chemcalc': chem.chemcalc,
                   'chemtools': chem.chemtools,
-                  'miller': chem.miller}
+                  'miller': chem.miller,
+                  'draganddrop': verifiers.draganddrop}
 
 # These should be removed from HTML output, including all subelements
 html_problem_semantics = ["codeparam", "responseparam", "answer", "script", "hintgroup", "openendedparam","openendedrubric"]

common/lib/capa/capa/inputtypes.py

@@ -13,6 +13,9 @@ Module containing the problem elements which render into input objects
 - imageinput  (for clickable image)
 - optioninput (for option list)
 - filesubmission (upload a file)
+- crystallography
+- vsepr_input
+- drag_and_drop
 
 These are matched by *.html files templates/*.html which are mako templates with the
 actual html.
@@ -41,6 +44,7 @@ from lxml import etree
 import re
 import shlex  # for splitting quoted strings
 import sys
+import os
 
 from registry import TagRegistry
 
@@ -692,7 +696,7 @@ class VseprInput(InputTypeBase):
     @classmethod
     def get_attributes(cls):
         """
-        Note: height, width are required.
+        Note: height, width, molecules and geometries are required.
         """
         return [Attribute('height'),
                 Attribute('width'),
@@ -735,3 +739,93 @@ class ChemicalEquationInput(InputTypeBase):
 registry.register(ChemicalEquationInput)
 
 #-----------------------------------------------------------------------------
+
+class DragAndDropInput(InputTypeBase):
+    """
+    Input for drag and drop problems. Allows student to drag and drop images and
+    labels to base image.
+    """
+
+    template = 'drag_and_drop_input.html'
+    tags = ['drag_and_drop_input']
+
+    def setup(self):
+
+        def parse(tag, tag_type):
+            """Parses <tag ... /> xml element to dictionary. Stores
+                'draggable' and 'target' tags with attributes to dictionary and
+                returns last.
+
+                Args:
+                    tag: xml etree element <tag...> with attributes
+
+                    tag_type: 'draggable' or 'target'.
+
+                    If tag_type is 'draggable' : all attributes except id
+                    (name or label or icon or can_reuse) are optional
+
+                    If tag_type is 'target' all attributes (name, x, y, w, h)
+                    are required. (x, y) - coordinates of center of target,
+                    w, h - weight and height of target.
+
+                Returns:
+                    Dictionary of vaues of attributes:
+                    dict{'name': smth, 'label': smth, 'icon': smth,
+                    'can_reuse': smth}.
+            """
+            tag_attrs = dict()
+            tag_attrs['draggable'] = {'id': Attribute._sentinel,
+                                      'label': "", 'icon': "",
+                                      'can_reuse': ""}
+
+            tag_attrs['target'] = {'id': Attribute._sentinel,
+                                    'x': Attribute._sentinel,
+                                    'y': Attribute._sentinel,
+                                    'w': Attribute._sentinel,
+                                    'h': Attribute._sentinel}
+
+            dic = dict()
+
+            for attr_name in tag_attrs[tag_type].keys():
+                dic[attr_name] = Attribute(attr_name,
+                    default=tag_attrs[tag_type][attr_name]).parse_from_xml(tag)
+
+            if tag_type == 'draggable' and not self.no_labels:
+                dic['label'] = dic['label'] or dic['id']
+
+            return dic
+
+        # add labels to images?:
+        self.no_labels = Attribute('no_labels',
+                                        default="False").parse_from_xml(self.xml)
+
+        to_js = dict()
+
+        # image drag and drop onto
+        to_js['base_image'] = Attribute('img').parse_from_xml(self.xml)
+
+        # outline places on image where to drag adn drop
+        to_js['target_outline'] = Attribute('target_outline',
+                                        default="False").parse_from_xml(self.xml)
+        # one draggable per target?
+        to_js['one_per_target'] = Attribute('one_per_target',
+                                        default="True").parse_from_xml(self.xml)
+        # list of draggables
+        to_js['draggables'] = [parse(draggable, 'draggable') for draggable in
+                                                self.xml.iterchildren('draggable')]
+        # list of targets
+        to_js['targets'] = [parse(target, 'target') for target in
+                                                self.xml.iterchildren('target')]
+
+        # custom background color for labels:
+        label_bg_color = Attribute('label_bg_color',
+                                   default=None).parse_from_xml(self.xml)
+        if label_bg_color:
+            to_js['label_bg_color'] = label_bg_color
+
+        self.loaded_attributes['drag_and_drop_json'] = json.dumps(to_js)
+        self.to_render.add('drag_and_drop_json')
+
+registry.register(DragAndDropInput)
+
+#--------------------------------------------------------------------------------------------------------------------

common/lib/capa/capa/responsetypes.py

@@ -33,7 +33,7 @@ from correctmap import CorrectMap
 from datetime import datetime
 from util import *
 from lxml import etree
-from lxml.html.soupparser import fromstring as fromstring_bs	 # uses Beautiful Soup!!! FIXME?
+from lxml.html.soupparser import fromstring as fromstring_bs     # uses Beautiful Soup!!! FIXME?
 import xqueue_interface
 
 log = logging.getLogger('mitx.' + __name__)
@@ -101,7 +101,6 @@ class LoncapaResponse(object):
 
       - hint_tag             : xhtml tag identifying hint associated with this response inside
                                hintgroup
-
     """
     __metaclass__ = abc.ABCMeta  # abc = Abstract Base Class
 
@@ -185,6 +184,11 @@ class LoncapaResponse(object):
         '''
         # render ourself as a <span> + our content
         tree = etree.Element('span')
+
+        # problem author can make this span display:inline
+        if self.xml.get('inline',''):
+            tree.set('class','inline')
+            
         for item in self.xml:
             # call provided procedure to do the rendering
             item_xhtml = renderer(item)
@@ -869,7 +873,9 @@ def sympy_check2():
 
     response_tag = 'customresponse'
 
-    allowed_inputfields = ['textline', 'textbox', 'crystallography', 'chemicalequationinput', 'vsepr_input']
+    allowed_inputfields = ['textline', 'textbox', 'crystallography',
+                            'chemicalequationinput', 'vsepr_input',
+                            'drag_and_drop_input']
 
     def setup_response(self):
         xml = self.xml
@@ -1044,7 +1050,7 @@ def sympy_check2():
                                          pretty_print=True)
                     #msg = etree.tostring(fromstring_bs(msg),pretty_print=True)
                     msg = msg.replace('&#13;', '')
-                    #msg = re.sub('<html>(.*)</html>','\\1',msg,flags=re.M|re.DOTALL)	# python 2.7
+                    #msg = re.sub('<html>(.*)</html>','\\1',msg,flags=re.M|re.DOTALL)   # python 2.7
                     msg = re.sub('(?ms)<html>(.*)</html>', '\\1', msg)
 
                 messages[0] = msg
@@ -1141,7 +1147,13 @@ class CodeResponse(LoncapaResponse):
         xml = self.xml
         # TODO: XML can override external resource (grader/queue) URL
         self.url = xml.get('url', None)
-        self.queue_name = xml.get('queuename', self.system.xqueue['default_queuename'])
+
+        # We do not support xqueue within Studio.
+        if self.system.xqueue is not None:
+            default_queuename = self.system.xqueue['default_queuename']
+        else:
+            default_queuename = None
+        self.queue_name = xml.get('queuename', default_queuename)
 
         # VS[compat]:
         #   Check if XML uses the ExternalResponse format or the generic CodeResponse format
@@ -1230,6 +1242,13 @@ class CodeResponse(LoncapaResponse):
                 (err, self.answer_id, convert_files_to_filenames(student_answers)))
             raise Exception(err)
 
+        # We do not support xqueue within Studio.
+        if self.system.xqueue is None:
+            cmap = CorrectMap()
+            cmap.set(self.answer_id, queuestate=None,
+                msg='Error checking problem: no external queueing server is configured.')
+            return cmap
+
         # Prepare xqueue request
         #------------------------------------------------------------
 
@@ -1763,7 +1782,7 @@ class ImageResponse(LoncapaResponse):
     def get_score(self, student_answers):
         correct_map = CorrectMap()
         expectedset = self.get_answers()
-        for aid in self.answer_ids:	 # loop through IDs of <imageinput>
+        for aid in self.answer_ids:  # loop through IDs of <imageinput>
         #  fields in our stanza
             given = student_answers[aid]  # this should be a string of the form '[x,y]'
             correct_map.set(aid, 'incorrect')
@@ -1815,6 +1834,7 @@ class ImageResponse(LoncapaResponse):
         return (dict([(ie.get('id'), ie.get('rectangle')) for ie in self.ielements]),
                 dict([(ie.get('id'), ie.get('regions')) for ie in self.ielements]))
 #-----------------------------------------------------------------------------
+
 # TEMPORARY: List of all response subclasses
 # FIXME: To be replaced by auto-registration
 

common/lib/capa/capa/templates/drag_and_drop_input.html

@@ -0,0 +1,46 @@
+<section id="inputtype_${id}" class="capa_inputtype" >
+    <div class="drag_and_drop_problem_div" id="drag_and_drop_div_${id}"
+     data-plain-id="${id}">
+    </div>
+
+    <div class="drag_and_drop_problem_json" id="drag_and_drop_json_${id}"
+        style="display:none;">${drag_and_drop_json}</div>
+
+    <div class="script_placeholder" data-src="/static/js/capa/drag_and_drop.js"></div>
+
+    % if status == 'unsubmitted':
+        <div class="unanswered" id="status_${id}">
+    % elif status == 'correct':
+        <div class="correct" id="status_${id}">
+    % elif status == 'incorrect':
+        <div class="incorrect" id="status_${id}">
+    % elif status == 'incomplete':
+        <div class="incorrect" id="status_${id}">
+    % endif
+
+
+    <input type="text" name="input_${id}" id="input_${id}" value="${value|h}"
+    style="display:none;"/>
+
+    <p class="status">
+        % if status == 'unsubmitted':
+        unanswered
+        % elif status == 'correct':
+        correct
+        % elif status == 'incorrect':
+        incorrect
+        % elif status == 'incomplete':
+        incomplete
+        % endif
+    </p>
+
+    <p id="answer_${id}" class="answer"></p>
+
+    % if msg:
+        <span class="message">${msg|n}</span>
+    % endif
+
+    % if status in ['unsubmitted', 'correct', 'incorrect', 'incomplete']:
+        </div>
+    % endif
+</section>

common/lib/capa/capa/tests/test_inputtypes.py

@@ -9,13 +9,14 @@ TODO:
 - check rendering -- e.g. msg should appear in the rendered output.  If possible, test that
   templates are escaping things properly.
 
-  
+
 - test unicode in values, parameters, etc.
 - test various html escapes
 - test funny xml chars -- should never get xml parse error if things are escaped properly.
 
 """
 
+import json
 from lxml import etree
 import unittest
 import xml.sax.saxutils as saxutils
@@ -501,3 +502,70 @@ class ChemicalEquationTest(unittest.TestCase):
                     }
         self.assertEqual(context, expected)
 
+
+class DragAndDropTest(unittest.TestCase):
+    '''
+    Check that drag and drop inputs work
+    '''
+
+    def test_rendering(self):
+        path_to_images = '/static/images/'
+
+        xml_str = """
+        <drag_and_drop_input id="prob_1_2" img="{path}about_1.png" target_outline="false">
+            <draggable id="1" label="Label 1"/>
+            <draggable id="name_with_icon" label="cc" icon="{path}cc.jpg"/>
+            <draggable id="with_icon" label="arrow-left" icon="{path}arrow-left.png" />
+            <draggable id="5" label="Label2" />
+            <draggable id="2" label="Mute" icon="{path}mute.png" />
+            <draggable id="name_label_icon3" label="spinner" icon="{path}spinner.gif" />
+            <draggable id="name4" label="Star" icon="{path}volume.png" />
+            <draggable id="7" label="Label3" />
+
+            <target id="t1" x="210" y="90" w="90" h="90"/>
+            <target id="t2" x="370" y="160" w="90" h="90"/>
+
+        </drag_and_drop_input>
+        """.format(path=path_to_images)
+
+        element = etree.fromstring(xml_str)
+
+        value = 'abc'
+        state = {'value': value,
+                 'status': 'unsubmitted'}
+
+        user_input = {  # order matters, for string comparison
+                        "target_outline": "false",
+                        "base_image": "/static/images/about_1.png",
+                        "draggables": [
+{"can_reuse": "", "label": "Label 1", "id": "1", "icon": ""},
+{"can_reuse": "", "label": "cc", "id": "name_with_icon", "icon": "/static/images/cc.jpg", },
+{"can_reuse": "", "label": "arrow-left", "id": "with_icon", "icon": "/static/images/arrow-left.png", "can_reuse": ""},
+{"can_reuse": "", "label": "Label2", "id": "5", "icon": "", "can_reuse": ""},
+{"can_reuse": "", "label": "Mute", "id": "2", "icon": "/static/images/mute.png", "can_reuse": ""},
+{"can_reuse": "", "label": "spinner", "id": "name_label_icon3", "icon": "/static/images/spinner.gif", "can_reuse": ""},
+{"can_reuse": "", "label": "Star", "id": "name4", "icon": "/static/images/volume.png", "can_reuse": ""},
+{"can_reuse": "", "label": "Label3", "id": "7", "icon": "", "can_reuse": ""}],
+                        "one_per_target": "True",
+                        "targets": [
+                {"y": "90", "x": "210", "id": "t1", "w": "90", "h": "90"},
+                {"y": "160", "x": "370", "id": "t2", "w": "90", "h": "90"}
+                                    ]
+                    }
+
+        the_input = lookup_tag('drag_and_drop_input')(test_system, element, state)
+
+        context = the_input._get_render_context()
+        expected = {'id': 'prob_1_2',
+                    'value': value,
+                    'status': 'unsubmitted',
+                    'msg': '',
+                    'drag_and_drop_json': json.dumps(user_input)
+                    }
+
+        # as we are dumping 'draggables' dicts while dumping user_input, string
+        # comparison will fail, as order of keys is random.
+        self.assertEqual(json.loads(context['drag_and_drop_json']), user_input)
+        context.pop('drag_and_drop_json')
+        expected.pop('drag_and_drop_json')
+        self.assertEqual(context, expected)

common/lib/capa/capa/verifiers/draganddrop.py

@@ -0,0 +1,376 @@
+""" Grader of drag and drop input.
+
+Client side behavior: user can drag and drop images from list on base image.
+
+
+ Then json returned from client is:
+ {
+    "draggable": [
+        { "image1": "t1"  },
+        { "ant": "t2"  },
+        { "molecule": "t3"  },
+                ]
+}
+values are target names.
+
+or:
+ {
+    "draggable": [
+        { "image1": "[10, 20]"  },
+        { "ant": "[30, 40]"  },
+        { "molecule": "[100, 200]"  },
+                ]
+}
+values are (x,y) coordinates of centers of dragged images.
+"""
+
+import json
+
+
+class PositionsCompare(list):
+    """ Class for comparing positions.
+
+        Args:
+                list or string::
+                    "abc" - target
+                    [10, 20] - list of integers
+                    [[10,20], 200] list of list and integer
+
+    """
+    def __eq__(self, other):
+        """ Compares two arguments.
+
+        Default lists behavior is conversion of string "abc" to  list
+        ["a", "b", "c"]. We will use that.
+
+        If self or other is empty - returns False.
+
+        Args:
+                self, other: str, unicode, list, int, float
+
+        Returns: bool
+        """
+        # checks if self or other is not empty list (empty lists  = false)
+        if not self or not other:
+            return False
+
+        if (isinstance(self[0], (list, int, float)) and
+            isinstance(other[0], (list, int, float))):
+            return self.coordinate_positions_compare(other)
+
+        elif (isinstance(self[0], (unicode, str)) and
+              isinstance(other[0], (unicode, str))):
+            return ''.join(self) == ''.join(other)
+        else:  # improper argument types: no (float / int or lists of list
+            #and float / int pair) or two string / unicode lists pair
+            return False
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def coordinate_positions_compare(self, other, r=10):
+        """ Checks if self is equal to other inside radius of forgiveness
+            (default 10 px).
+
+            Args:
+                self, other: [x, y] or [[x, y], r], where r is radius of
+                             forgiveness;
+                             x, y, r: int
+
+            Returns: bool.
+        """
+        # get max radius of forgiveness
+        if isinstance(self[0], list):  # [(x, y), r] case
+            r = max(self[1], r)
+            x1, y1 = self[0]
+        else:
+            x1, y1 = self
+
+        if isinstance(other[0], list):  # [(x, y), r] case
+            r = max(other[1], r)
+            x2, y2 = other[0]
+        else:
+            x2, y2 = other
+
+        if (x2 - x1) ** 2 + (y2 - y1) ** 2 > r * r:
+            return False
+
+        return True
+
+
+class DragAndDrop(object):
+    """ Grader class for drag and drop inputtype.
+    """
+
+    def grade(self):
+        ''' Grader user answer.
+
+        Checks if every draggable isplaced on proper target or  on proper
+        coordinates within radius of forgiveness (default is 10).
+
+        Returns: bool.
+        '''
+        for draggable in self.excess_draggables:
+            if not self.excess_draggables[draggable]:
+                return False  # user answer has more draggables than correct answer
+
+        # Number of draggables in user_groups may be differ that in
+        # correct_groups, that is incorrect, except special case with 'number'
+        for groupname, draggable_ids in self.correct_groups.items():
+
+            # 'number' rule special case
+            # for reusable draggables we may get in self.user_groups
+            # {'1': [u'2', u'2', u'2'], '0': [u'1', u'1'], '2': [u'3']}
+            # if '+number' is in rule - do not remove duplicates and strip
+            # '+number' from rule
+            current_rule = self.correct_positions[groupname].keys()[0]
+            if 'number' in current_rule:
+                rule_values = self.correct_positions[groupname][current_rule]
+                # clean rule, do not do clean duplicate items
+                self.correct_positions[groupname].pop(current_rule, None)
+                parsed_rule = current_rule.replace('+', '').replace('number', '')
+                self.correct_positions[groupname][parsed_rule] = rule_values
+            else:  # remove dublicates
+                self.user_groups[groupname] = list(set(self.user_groups[groupname]))
+
+            if sorted(draggable_ids) != sorted(self.user_groups[groupname]):
+                return False
+
+        # Check that in every group, for rule of that group, user positions of
+        # every element are equal with correct positions
+        for groupname in self.correct_groups:
+            rules_executed = 0
+            for rule in ('exact', 'anyof', 'unordered_equal'):
+                # every group has only one rule
+                if self.correct_positions[groupname].get(rule, None):
+                    rules_executed += 1
+                    if not self.compare_positions(
+                            self.correct_positions[groupname][rule],
+                            self.user_positions[groupname]['user'], flag=rule):
+                        return False
+            if not rules_executed:  # no correct rules for current group
+            # probably xml content mistake - wrong rules names
+                return False
+
+        return True
+
+    def compare_positions(self, correct, user, flag):
+        """ Compares two lists of positions with flag rules. Order of
+        correct/user arguments is matter only in 'anyof' flag.
+
+        Rules description:
+
+            'exact' means 1-1 ordered relationship::
+
+                [el1, el2, el3] is 'exact' equal to [el5, el6, el7] when
+                el1 == el5, el2 == el6, el3 == el7.
+                Equality function is custom, see below.
+
+
+            'anyof' means subset relationship::
+
+                user = [el1, el2] is 'anyof' equal to correct = [el1, el2, el3]
+                when
+                        set(user) <= set(correct).
+
+                'anyof' is ordered relationship. It always checks if user
+                is subset of correct
+
+                Equality function is custom, see below.
+
+                Examples:
+
+                     - many draggables per position:
+                    user ['1','2','2','2'] is 'anyof' equal to ['1', '2', '3']
+
+                     - draggables can be placed in any order:
+                    user ['1','2','3','4'] is 'anyof' equal to ['4', '2', '1', 3']
+
+            'unordered_equal' is same as 'exact' but disregards on order
+
+        Equality functions:
+
+        Equality functon depends on type of element. They declared in
+        PositionsCompare class. For position like targets
+        ids ("t1", "t2", etc..) it is string equality function. For coordinate
+        positions ([1,2] or [[1,2], 15]) it is coordinate_positions_compare
+        function (see docstrings in PositionsCompare class)
+
+        Args:
+            correst, user: lists of positions
+
+        Returns: True if within rule lists are equal, otherwise False.
+        """
+        if flag == 'exact':
+            if len(correct) != len(user):
+                return False
+            for el1, el2 in zip(correct, user):
+                if PositionsCompare(el1) != PositionsCompare(el2):
+                    return False
+
+        if flag == 'anyof':
+            for u_el in user:
+                for c_el in correct:
+                    if PositionsCompare(u_el) == PositionsCompare(c_el):
+                        break
+                else:
+                    # General: the else is executed after the for,
+                    # only if the for terminates normally (not by a break)
+
+                    # In this case, 'for' is terminated normally if every element
+                    # from 'correct' list isn't equal to concrete element from
+                    # 'user' list. So as we found one element from 'user' list,
+                    # that not in 'correct' list - we return False
+                    return False
+
+        if flag == 'unordered_equal':
+            if len(correct) != len(user):
+                return False
+            temp = correct[:]
+            for u_el in user:
+                for c_el in temp:
+                    if PositionsCompare(u_el) == PositionsCompare(c_el):
+                        temp.remove(c_el)
+                        break
+                else:
+                    # same as upper -  if we found element from 'user' list,
+                    # that not in 'correct' list - we return False.
+                    return False
+
+        return True
+
+    def __init__(self, correct_answer, user_answer):
+        """ Populates DragAndDrop variables from user_answer and correct_answer.
+        If correct_answer is dict, converts it to list.
+        Correct answer in dict form is simpe structure for fast and simple
+        grading. Example of correct answer dict example::
+
+            correct_answer = {'name4': 't1',
+                            'name_with_icon': 't1',
+                            '5': 't2',
+                            '7':'t2'}
+
+            It is draggable_name: dragable_position mapping.
+
+            Advanced form converted from simple form uses 'exact' rule
+            for matching.
+
+        Correct answer in list form is designed for advanced cases::
+
+        correct_answers = [
+        {
+        'draggables': ['1', '2', '3', '4', '5', '6'],
+        'targets': [
+           's_left', 's_right', 's_sigma', 's_sigma_star', 'p_pi_1',  'p_pi_2'],
+        'rule': 'anyof'},
+        {
+        'draggables': ['7', '8', '9', '10'],
+        'targets': ['p_left_1', 'p_left_2', 'p_right_1', 'p_right_2'],
+        'rule': 'anyof'
+        }
+                        ]
+
+        Advanced answer in list form is list of dicts, and every dict must have
+        3 keys: 'draggables', 'targets' and 'rule'. 'Draggables' value is
+        list of draggables ids, 'targes' values are list of targets ids, 'rule'
+        value one of 'exact', 'anyof', 'unordered_equal', 'anyof+number',
+        'unordered_equal+number'
+
+        Advanced form uses "all dicts must match with their rule" logic.
+
+        Same draggable cannot appears more that in one dict.
+
+        Behavior is more widely explained in sphinx documentation.
+
+        Args:
+            user_answer: json
+            correct_answer: dict  or list
+        """
+
+        self.correct_groups = dict()  # correct groups from xml
+        self.correct_positions = dict()  # correct positions for comparing
+        self.user_groups = dict()  # will be populated from user answer
+        self.user_positions = dict()  # will be populated from user answer
+
+        # convert from dict answer format to list format
+        if isinstance(correct_answer, dict):
+            tmp = []
+            for key, value in correct_answer.items():
+                tmp_dict = {'draggables': [], 'targets': [], 'rule': 'exact'}
+                tmp_dict['draggables'].append(key)
+                tmp_dict['targets'].append(value)
+                tmp.append(tmp_dict)
+            correct_answer = tmp
+
+        user_answer = json.loads(user_answer)
+
+        # check if we have draggables that are not in correct answer:
+        self.excess_draggables = {}
+
+        # create identical data structures from user answer and correct answer
+        for i in xrange(0, len(correct_answer)):
+            groupname = str(i)
+            self.correct_groups[groupname] = correct_answer[i]['draggables']
+            self.correct_positions[groupname] = {correct_answer[i]['rule']:
+                                                correct_answer[i]['targets']}
+            self.user_groups[groupname] = []
+            self.user_positions[groupname] = {'user': []}
+            for draggable_dict in user_answer['draggables']:
+                # draggable_dict is 1-to-1 {draggable_name: position}
+                draggable_name = draggable_dict.keys()[0]
+                if draggable_name in self.correct_groups[groupname]:
+                    self.user_groups[groupname].append(draggable_name)
+                    self.user_positions[groupname]['user'].append(
+                                            draggable_dict[draggable_name])
+                    self.excess_draggables[draggable_name] = True
+                else:
+                    self.excess_draggables[draggable_name] = \
+                    self.excess_draggables.get(draggable_name, False)
+
+
+def grade(user_input, correct_answer):
+    """ Creates DragAndDrop instance from user_input and correct_answer and
+        calls DragAndDrop.grade for grading.
+
+        Supports two interfaces for correct_answer: dict and list.
+
+        Args:
+            user_input: json. Format::
+
+                { "draggables":
+                [{"1": [10, 10]}, {"name_with_icon": [20, 20]}]}'
+
+                or
+
+                {"draggables": [{"1": "t1"}, \
+                {"name_with_icon": "t2"}]}
+
+            correct_answer: dict or list.
+
+                Dict form::
+
+                        {'1': 't1',  'name_with_icon': 't2'}
+
+                        or
+
+                        {'1': '[10, 10]',  'name_with_icon': '[[10, 10], 20]'}
+
+                List form::
+
+                    correct_answer = [
+                    {
+                        'draggables':  ['l3_o', 'l10_o'],
+                        'targets':  ['t1_o', 't9_o'],
+                        'rule': 'anyof'
+                    },
+                    {
+                        'draggables': ['l1_c','l8_c'],
+                        'targets': ['t5_c','t6_c'],
+                        'rule': 'anyof'
+                    }
+                    ]
+
+        Returns: bool
+    """
+    return DragAndDrop(correct_answer=correct_answer,
+                       user_answer=user_input).grade()

common/lib/capa/capa/verifiers/tests_draganddrop.py

@@ -0,0 +1,603 @@
+import unittest
+
+import draganddrop
+from draganddrop import PositionsCompare
+
+
+class Test_PositionsCompare(unittest.TestCase):
+    """ describe"""
+
+    def test_nested_list_and_list1(self):
+        self.assertEqual(PositionsCompare([[1, 2], 40]), PositionsCompare([1, 3]))
+
+    def test_nested_list_and_list2(self):
+        self.assertNotEqual(PositionsCompare([1, 12]), PositionsCompare([1, 1]))
+
+    def test_list_and_list1(self):
+        self.assertNotEqual(PositionsCompare([[1, 2], 12]), PositionsCompare([1, 15]))
+
+    def test_list_and_list2(self):
+        self.assertEqual(PositionsCompare([1, 11]), PositionsCompare([1, 1]))
+
+    def test_numerical_list_and_string_list(self):
+        self.assertNotEqual(PositionsCompare([1, 2]), PositionsCompare(["1"]))
+
+    def test_string_and_string_list1(self):
+        self.assertEqual(PositionsCompare("1"), PositionsCompare(["1"]))
+
+    def test_string_and_string_list2(self):
+        self.assertEqual(PositionsCompare("abc"), PositionsCompare("abc"))
+
+    def test_string_and_string_list3(self):
+        self.assertNotEqual(PositionsCompare("abd"), PositionsCompare("abe"))
+
+    def test_float_and_string(self):
+        self.assertNotEqual(PositionsCompare([3.5, 5.7]), PositionsCompare(["1"]))
+
+    def test_floats_and_ints(self):
+        self.assertEqual(PositionsCompare([3.5, 4.5]), PositionsCompare([5, 7]))
+
+
+class Test_DragAndDrop_Grade(unittest.TestCase):
+
+    def test_targets_true(self):
+        user_input = '{"draggables": [{"1": "t1"}, \
+         {"name_with_icon": "t2"}]}'
+        correct_answer = {'1':              't1', 'name_with_icon': 't2'}
+        self.assertTrue(draganddrop.grade(user_input, correct_answer))
+
+    def test_targets_false(self):
+        user_input = '{"draggables": [{"1": "t1"}, \
+        {"name_with_icon": "t2"}]}'
+        correct_answer = {'1':              't3', 'name_with_icon': 't2'}
+        self.assertFalse(draganddrop.grade(user_input, correct_answer))
+
+    def test_multiple_images_per_target_true(self):
+        user_input = '{\
+        "draggables": [{"1": "t1"}, {"name_with_icon": "t2"}, \
+        {"2": "t1"}]}'
+        correct_answer = {'1':              't1', 'name_with_icon': 't2',
+        '2': 't1'}
+        self.assertTrue(draganddrop.grade(user_input, correct_answer))
+
+    def test_multiple_images_per_target_false(self):
+        user_input = '{\
+        "draggables": [{"1": "t1"}, {"name_with_icon": "t2"}, \
+        {"2": "t1"}]}'
+        correct_answer = {'1':              't2', 'name_with_icon': 't2',
+        '2': 't1'}
+        self.assertFalse(draganddrop.grade(user_input, correct_answer))
+
+    def test_targets_and_positions(self):
+        user_input = '{"draggables": [{"1": [10,10]}, \
+         {"name_with_icon": [[10,10],4]}]}'
+        correct_answer = {'1':  [10, 10], 'name_with_icon': [[10, 10], 4]}
+        self.assertTrue(draganddrop.grade(user_input, correct_answer))
+
+    def test_position_and_targets(self):
+        user_input = '{"draggables": [{"1": "t1"}, {"name_with_icon": "t2"}]}'
+        correct_answer = {'1':    't1', 'name_with_icon': 't2'}
+        self.assertTrue(draganddrop.grade(user_input, correct_answer))
+
+    def test_positions_exact(self):
+        user_input = '{"draggables": \
+        [{"1": [10, 10]}, {"name_with_icon": [20, 20]}]}'
+        correct_answer = {'1':   [10, 10], 'name_with_icon': [20, 20]}
+        self.assertTrue(draganddrop.grade(user_input, correct_answer))
+
+    def test_positions_false(self):
+        user_input = '{"draggables": \
+        [{"1": [10, 10]}, {"name_with_icon": [20, 20]}]}'
+        correct_answer = {'1':   [25, 25], 'name_with_icon': [20, 20]}
+        self.assertFalse(draganddrop.grade(user_input, correct_answer))
+
+    def test_positions_true_in_radius(self):
+        user_input = '{"draggables": \
+        [{"1": [10, 10]}, {"name_with_icon": [20, 20]}]}'
+        correct_answer = {'1':   [14, 14], 'name_with_icon': [20, 20]}
+        self.assertTrue(draganddrop.grade(user_input, correct_answer))
+
+    def test_positions_true_in_manual_radius(self):
+        user_input = '{"draggables": \
+        [{"1": [10, 10]}, {"name_with_icon": [20, 20]}]}'
+        correct_answer = {'1':   [[40, 10], 30], 'name_with_icon': [20, 20]}
+        self.assertTrue(draganddrop.grade(user_input, correct_answer))
+
+    def test_positions_false_in_manual_radius(self):
+        user_input = '{"draggables": \
+        [{"1": [10, 10]}, {"name_with_icon": [20, 20]}]}'
+        correct_answer = {'1': [[40, 10], 29], 'name_with_icon': [20, 20]}
+        self.assertFalse(draganddrop.grade(user_input, correct_answer))
+
+    def test_correct_answer_not_has_key_from_user_answer(self):
+        user_input = '{"draggables": [{"1": "t1"}, \
+        {"name_with_icon": "t2"}]}'
+        correct_answer = {'3':   't3', 'name_with_icon': 't2'}
+        self.assertFalse(draganddrop.grade(user_input, correct_answer))
+
+    def test_anywhere(self):
+        """Draggables can be places anywhere on base image.
+            Place grass in the middle of the image and ant in the
+            right upper corner."""
+        user_input = '{"draggables": \
+        [{"ant":[610.5,57.449951171875]},{"grass":[322.5,199.449951171875]}]}'
+
+        correct_answer = {'grass':     [[300, 200], 200], 'ant': [[500, 0], 200]}
+        self.assertTrue(draganddrop.grade(user_input, correct_answer))
+
+    def test_lcao_correct(self):
+        """Describe carbon molecule in LCAO-MO"""
+        user_input = '{"draggables":[{"1":"s_left"}, \
+        {"5":"s_right"},{"4":"s_sigma"},{"6":"s_sigma_star"},{"7":"p_left_1"}, \
+        {"8":"p_left_2"},{"10":"p_right_1"},{"9":"p_right_2"}, \
+        {"2":"p_pi_1"},{"3":"p_pi_2"},{"11":"s_sigma_name"}, \
+        {"13":"s_sigma_star_name"},{"15":"p_pi_name"},{"16":"p_pi_star_name"}, \
+        {"12":"p_sigma_name"},{"14":"p_sigma_star_name"}]}'
+
+        correct_answer = [{
+        'draggables': ['1', '2', '3', '4', '5', '6'],
+        'targets': [
+            's_left', 's_right', 's_sigma', 's_sigma_star', 'p_pi_1', 'p_pi_2'
+        ],
+            'rule': 'anyof'
+        }, {
+            'draggables': ['7', '8', '9', '10'],
+            'targets': ['p_left_1', 'p_left_2', 'p_right_1', 'p_right_2'],
+            'rule': 'anyof'
+        }, {
+            'draggables': ['11', '12'],
+            'targets': ['s_sigma_name', 'p_sigma_name'],
+            'rule': 'anyof'
+        }, {
+            'draggables': ['13', '14'],
+            'targets': ['s_sigma_star_name', 'p_sigma_star_name'],
+            'rule': 'anyof'
+        }, {
+            'draggables': ['15'],
+            'targets': ['p_pi_name'],
+            'rule': 'anyof'
+        }, {
+            'draggables': ['16'],
+            'targets': ['p_pi_star_name'],
+            'rule': 'anyof'
+        }]
+
+        self.assertTrue(draganddrop.grade(user_input, correct_answer))
+
+    def test_lcao_extra_element_incorrect(self):
+        """Describe carbon molecule in LCAO-MO"""
+        user_input = '{"draggables":[{"1":"s_left"}, \
+        {"5":"s_right"},{"4":"s_sigma"},{"6":"s_sigma_star"},{"7":"p_left_1"}, \
+        {"8":"p_left_2"},{"17":"p_left_3"},{"10":"p_right_1"},{"9":"p_right_2"}, \
+        {"2":"p_pi_1"},{"3":"p_pi_2"},{"11":"s_sigma_name"}, \
+        {"13":"s_sigma_star_name"},{"15":"p_pi_name"},{"16":"p_pi_star_name"}, \
+        {"12":"p_sigma_name"},{"14":"p_sigma_star_name"}]}'
+
+        correct_answer = [{
+        'draggables': ['1', '2', '3', '4', '5', '6'],
+        'targets': [
+            's_left', 's_right', 's_sigma', 's_sigma_star', 'p_pi_1', 'p_pi_2'
+            ],
+            'rule': 'anyof'
+        }, {
+            'draggables': ['7', '8', '9', '10'],
+            'targets': ['p_left_1', 'p_left_2', 'p_right_1', 'p_right_2'],
+            'rule': 'anyof'
+        }, {
+            'draggables': ['11', '12'],
+            'targets': ['s_sigma_name', 'p_sigma_name'],
+            'rule': 'anyof'
+        }, {
+            'draggables': ['13', '14'],
+            'targets': ['s_sigma_star_name', 'p_sigma_star_name'],
+            'rule': 'anyof'
+        }, {
+            'draggables': ['15'],
+            'targets': ['p_pi_name'],
+            'rule': 'anyof'
+        }, {
+            'draggables': ['16'],
+            'targets': ['p_pi_star_name'],
+            'rule': 'anyof'
+        }]
+
+        self.assertFalse(draganddrop.grade(user_input, correct_answer))
+
+    def test_reuse_draggable_no_mupliples(self):
+        """Test reusable draggables (no mupltiple draggables per target)"""
+        user_input = '{"draggables":[{"1":"target1"}, \
+        {"2":"target2"},{"1":"target3"},{"2":"target4"},{"2":"target5"}, \
+        {"3":"target6"}]}'
+        correct_answer = [
+        {
+            'draggables': ['1'],
+            'targets': ['target1',  'target3'],
+            'rule': 'anyof'
+        },
+        {
+            'draggables': ['2'],
+            'targets': ['target2', 'target4', 'target5'],
+            'rule': 'anyof'
+        },
+        {
+            'draggables': ['3'],
+            'targets': ['target6'],
+            'rule': 'anyof'
+        }]
+        self.assertTrue(draganddrop.grade(user_input, correct_answer))
+
+    def test_reuse_draggable_with_mupliples(self):
+        """Test reusable draggables with mupltiple draggables per target"""
+        user_input = '{"draggables":[{"1":"target1"}, \
+        {"2":"target2"},{"1":"target1"},{"2":"target4"},{"2":"target4"}, \
+        {"3":"target6"}]}'
+        correct_answer = [
+        {
+            'draggables': ['1'],
+            'targets': ['target1',  'target3'],
+            'rule': 'anyof'
+        },
+        {
+            'draggables': ['2'],
+            'targets': ['target2', 'target4'],
+            'rule': 'anyof'
+        },
+        {
+            'draggables': ['3'],
+            'targets': ['target6'],
+            'rule': 'anyof'
+        }]
+        self.assertTrue(draganddrop.grade(user_input, correct_answer))
+
+    def test_reuse_many_draggable_with_mupliples(self):
+        """Test reusable draggables with mupltiple draggables per target"""
+        user_input = '{"draggables":[{"1":"target1"}, \
+        {"2":"target2"},{"1":"target1"},{"2":"target4"},{"2":"target4"}, \
+        {"3":"target6"}, {"4": "target3"}, {"5": "target4"}, \
+        {"5": "target5"}, {"6": "target2"}]}'
+        correct_answer = [
+        {
+            'draggables': ['1', '4'],
+            'targets': ['target1',  'target3'],
+            'rule': 'anyof'
+        },
+        {
+            'draggables': ['2', '6'],
+            'targets': ['target2', 'target4'],
+            'rule': 'anyof'
+        },
+        {
+            'draggables': ['5'],
+            'targets': ['target4', 'target5'],
+            'rule': 'anyof'
+        },
+        {
+            'draggables': ['3'],
+            'targets': ['target6'],
+            'rule': 'anyof'
+        }]
+        self.assertTrue(draganddrop.grade(user_input, correct_answer))
+
+    def test_reuse_many_draggable_with_mupliples_wrong(self):
+        """Test reusable draggables with mupltiple draggables per target"""
+        user_input = '{"draggables":[{"1":"target1"}, \
+        {"2":"target2"},{"1":"target1"}, \
+        {"2":"target3"}, \
+        {"2":"target4"}, \
+        {"3":"target6"}, {"4": "target3"}, {"5": "target4"}, \
+        {"5": "target5"}, {"6": "target2"}]}'
+        correct_answer = [
+        {
+            'draggables': ['1', '4'],
+            'targets': ['target1',  'target3'],
+            'rule': 'anyof'
+        },
+        {
+            'draggables': ['2', '6'],
+            'targets': ['target2', 'target4'],
+            'rule': 'anyof'
+        },
+        {
+            'draggables': ['5'],
+            'targets': ['target4', 'target5'],
+            'rule': 'anyof'
+        },
+        {
+            'draggables': ['3'],
+            'targets': ['target6'],
+            'rule': 'anyof'
+        }]
+        self.assertFalse(draganddrop.grade(user_input, correct_answer))
+
+    def test_label_10_targets_with_a_b_c_false(self):
+        """Test reusable draggables (no mupltiple draggables per target)"""
+        user_input = '{"draggables":[{"a":"target1"}, \
+        {"b":"target2"},{"c":"target3"},{"a":"target4"},{"b":"target5"}, \
+        {"c":"target6"}, {"a":"target7"},{"b":"target8"},{"c":"target9"}, \
+         {"a":"target1"}]}'
+        correct_answer = [
+        {
+            'draggables': ['a'],
+            'targets': ['target1',  'target4', 'target7', 'target10'],
+            'rule': 'unordered_equal'
+        },
+        {
+            'draggables': ['b'],
+            'targets': ['target2', 'target5', 'target8'],
+            'rule': 'unordered_equal'
+        },
+        {
+            'draggables': ['c'],
+            'targets': ['target3', 'target6', 'target9'],
+            'rule': 'unordered_equal'
+        }]
+        self.assertFalse(draganddrop.grade(user_input, correct_answer))
+
+    def test_label_10_targets_with_a_b_c_(self):
+        """Test reusable draggables (no mupltiple draggables per target)"""
+        user_input = '{"draggables":[{"a":"target1"}, \
+        {"b":"target2"},{"c":"target3"},{"a":"target4"},{"b":"target5"}, \
+        {"c":"target6"}, {"a":"target7"},{"b":"target8"},{"c":"target9"}, \
+         {"a":"target10"}]}'
+        correct_answer = [
+        {
+            'draggables': ['a'],
+            'targets': ['target1',  'target4', 'target7', 'target10'],
+            'rule': 'unordered_equal'
+        },
+        {
+            'draggables': ['b'],
+            'targets': ['target2', 'target5', 'target8'],
+            'rule': 'unordered_equal'
+        },
+        {
+            'draggables': ['c'],
+            'targets': ['target3', 'target6', 'target9'],
+            'rule': 'unordered_equal'
+        }]
+        self.assertTrue(draganddrop.grade(user_input, correct_answer))
+
+    def test_label_10_targets_with_a_b_c_multiple(self):
+        """Test reusable draggables  (mupltiple draggables per target)"""
+        user_input = '{"draggables":[{"a":"target1"}, \
+        {"b":"target2"},{"c":"target3"},{"b":"target5"}, \
+        {"c":"target6"}, {"a":"target7"},{"b":"target8"},{"c":"target9"}, \
+         {"a":"target1"}]}'
+        correct_answer = [
+        {
+            'draggables': ['a', 'a', 'a'],
+            'targets': ['target1',  'target4', 'target7', 'target10'],
+            'rule': 'anyof+number'
+        },
+        {
+            'draggables': ['b', 'b', 'b'],
+            'targets': ['target2', 'target5', 'target8'],
+            'rule': 'anyof+number'
+        },
+        {
+            'draggables': ['c', 'c', 'c'],
+            'targets': ['target3', 'target6', 'target9'],
+            'rule': 'anyof+number'
+        }]
+        self.assertTrue(draganddrop.grade(user_input, correct_answer))
+
+    def test_label_10_targets_with_a_b_c_multiple_false(self):
+        """Test reusable draggables  (mupltiple draggables per target)"""
+        user_input = '{"draggables":[{"a":"target1"}, \
+        {"b":"target2"},{"c":"target3"},{"a":"target4"},{"b":"target5"}, \
+        {"c":"target6"}, {"a":"target7"},{"b":"target8"},{"c":"target9"}, \
+         {"a":"target1"}]}'
+        correct_answer = [
+        {
+            'draggables': ['a', 'a', 'a'],
+            'targets': ['target1',  'target4', 'target7', 'target10'],
+            'rule': 'anyof+number'
+        },
+        {
+            'draggables': ['b', 'b', 'b'],
+            'targets': ['target2', 'target5', 'target8'],
+            'rule': 'anyof+number'
+        },
+        {
+            'draggables': ['c', 'c', 'c'],
+            'targets': ['target3', 'target6', 'target9'],
+            'rule': 'anyof+number'
+        }]
+        self.assertFalse(draganddrop.grade(user_input, correct_answer))
+
+    def test_label_10_targets_with_a_b_c_reused(self):
+        """Test a b c in 10 labels reused"""
+        user_input = '{"draggables":[{"a":"target1"}, \
+        {"b":"target2"},{"c":"target3"},{"b":"target5"}, \
+        {"c":"target6"}, {"b":"target8"},{"c":"target9"}, \
+         {"a":"target10"}]}'
+        correct_answer = [
+        {
+            'draggables': ['a', 'a'],
+            'targets': ['target1',   'target10'],
+            'rule': 'unordered_equal+number'
+        },
+        {
+            'draggables': ['b', 'b', 'b'],
+            'targets': ['target2', 'target5', 'target8'],
+            'rule': 'unordered_equal+number'
+        },
+        {
+            'draggables': ['c', 'c', 'c'],
+            'targets': ['target3', 'target6', 'target9'],
+            'rule': 'unordered_equal+number'
+        }]
+        self.assertTrue(draganddrop.grade(user_input, correct_answer))
+
+    def test_label_10_targets_with_a_b_c_reused_false(self):
+        """Test a b c in 10 labels reused false"""
+        user_input = '{"draggables":[{"a":"target1"}, \
+        {"b":"target2"},{"c":"target3"},{"b":"target5"}, {"a":"target8"},\
+        {"c":"target6"}, {"b":"target8"},{"c":"target9"}, \
+         {"a":"target10"}]}'
+        correct_answer = [
+        {
+            'draggables': ['a', 'a'],
+            'targets': ['target1',   'target10'],
+            'rule': 'unordered_equal+number'
+        },
+        {
+            'draggables': ['b', 'b', 'b'],
+            'targets': ['target2', 'target5', 'target8'],
+            'rule': 'unordered_equal+number'
+        },
+        {
+            'draggables': ['c', 'c', 'c'],
+            'targets': ['target3', 'target6', 'target9'],
+            'rule': 'unordered_equal+number'
+        }]
+        self.assertFalse(draganddrop.grade(user_input, correct_answer))
+
+    def test_mixed_reuse_and_not_reuse(self):
+        """Test reusable draggables """
+        user_input = '{"draggables":[{"a":"target1"}, \
+        {"b":"target2"},{"c":"target3"}, {"a":"target4"},\
+         {"a":"target5"}]}'
+        correct_answer = [
+        {
+            'draggables': ['a', 'b'],
+            'targets': ['target1',   'target2', 'target4', 'target5'],
+            'rule': 'anyof'
+        },
+        {
+            'draggables': ['c'],
+            'targets': ['target3'],
+            'rule': 'exact'
+        }]
+        self.assertTrue(draganddrop.grade(user_input, correct_answer))
+
+    def test_mixed_reuse_and_not_reuse_number(self):
+        """Test reusable draggables with number """
+        user_input = '{"draggables":[{"a":"target1"}, \
+        {"b":"target2"},{"c":"target3"}, {"a":"target4"}]}'
+        correct_answer = [
+        {
+            'draggables': ['a', 'a', 'b'],
+            'targets': ['target1',   'target2', 'target4'],
+            'rule': 'anyof+number'
+        },
+        {
+            'draggables': ['c'],
+            'targets': ['target3'],
+            'rule': 'exact'
+        }]
+        self.assertTrue(draganddrop.grade(user_input, correct_answer))
+
+    def test_mixed_reuse_and_not_reuse_number_false(self):
+        """Test reusable draggables with numbers, but wrong"""
+        user_input = '{"draggables":[{"a":"target1"}, \
+        {"b":"target2"},{"c":"target3"}, {"a":"target4"}, {"a":"target10"}]}'
+        correct_answer = [
+        {
+            'draggables': ['a', 'a', 'b'],
+            'targets': ['target1',   'target2', 'target4', 'target10'],
+            'rule': 'anyof_number'
+        },
+        {
+            'draggables': ['c'],
+            'targets': ['target3'],
+            'rule': 'exact'
+        }]
+        self.assertFalse(draganddrop.grade(user_input, correct_answer))
+
+    def test_alternative_correct_answer(self):
+        user_input = '{"draggables":[{"name_with_icon":"t1"},\
+        {"name_with_icon":"t1"},{"name_with_icon":"t1"},{"name4":"t1"}, \
+        {"name4":"t1"}]}'
+        correct_answer = [
+        {'draggables': ['name4'], 'targets': ['t1', 't1'], 'rule': 'exact'},
+        {'draggables': ['name_with_icon'], 'targets': ['t1', 't1', 't1'],
+        'rule': 'exact'}
+        ]
+        self.assertTrue(draganddrop.grade(user_input, correct_answer))
+
+
+class Test_DragAndDrop_Populate(unittest.TestCase):
+
+    def test_1(self):
+        correct_answer = {'1': [[40, 10], 29], 'name_with_icon': [20, 20]}
+        user_input = '{"draggables": \
+        [{"1": [10, 10]}, {"name_with_icon": [20, 20]}]}'
+        dnd = draganddrop.DragAndDrop(correct_answer, user_input)
+
+        correct_groups = {'1': ['name_with_icon'], '0': ['1']}
+        correct_positions = {'1': {'exact': [[20, 20]]}, '0': {'exact': [[[40, 10], 29]]}}
+        user_groups = {'1': [u'name_with_icon'], '0': [u'1']}
+        user_positions = {'1': {'user': [[20, 20]]}, '0': {'user': [[10, 10]]}}
+
+        self.assertEqual(correct_groups, dnd.correct_groups)
+        self.assertEqual(correct_positions, dnd.correct_positions)
+        self.assertEqual(user_groups, dnd.user_groups)
+        self.assertEqual(user_positions, dnd.user_positions)
+
+
+class Test_DraAndDrop_Compare_Positions(unittest.TestCase):
+
+    def test_1(self):
+        dnd = draganddrop.DragAndDrop({'1': 't1'}, '{"draggables": [{"1": "t1"}]}')
+        self.assertTrue(dnd.compare_positions(correct=[[1, 1], [2, 3]],
+                                              user=[[2, 3], [1, 1]],
+                                              flag='anyof'))
+
+    def test_2a(self):
+        dnd = draganddrop.DragAndDrop({'1': 't1'}, '{"draggables": [{"1": "t1"}]}')
+        self.assertTrue(dnd.compare_positions(correct=[[1, 1], [2, 3]],
+                                               user=[[2, 3], [1, 1]],
+                                               flag='exact'))
+
+    def test_2b(self):
+        dnd = draganddrop.DragAndDrop({'1': 't1'}, '{"draggables": [{"1": "t1"}]}')
+        self.assertFalse(dnd.compare_positions(correct=[[1, 1], [2, 3]],
+                                               user=[[2, 13], [1, 1]],
+                                               flag='exact'))
+
+    def test_3(self):
+        dnd = draganddrop.DragAndDrop({'1': 't1'}, '{"draggables": [{"1": "t1"}]}')
+        self.assertFalse(dnd.compare_positions(correct=["a", "b"],
+                                               user=["a", "b", "c"],
+                                               flag='anyof'))
+
+    def test_4(self):
+        dnd = draganddrop.DragAndDrop({'1': 't1'}, '{"draggables": [{"1": "t1"}]}')
+        self.assertTrue(dnd.compare_positions(correct=["a", "b", "c"],
+                                              user=["a", "b"],
+                                              flag='anyof'))
+
+    def test_5(self):
+        dnd = draganddrop.DragAndDrop({'1': 't1'}, '{"draggables": [{"1": "t1"}]}')
+        self.assertFalse(dnd.compare_positions(correct=["a", "b", "c"],
+                                               user=["a", "c", "b"],
+                                               flag='exact'))
+
+    def test_6(self):
+        dnd = draganddrop.DragAndDrop({'1': 't1'}, '{"draggables": [{"1": "t1"}]}')
+        self.assertTrue(dnd.compare_positions(correct=["a", "b", "c"],
+                                               user=["a", "c", "b"],
+                                               flag='anyof'))
+
+    def test_7(self):
+        dnd = draganddrop.DragAndDrop({'1': 't1'}, '{"draggables": [{"1": "t1"}]}')
+        self.assertFalse(dnd.compare_positions(correct=["a", "b", "b"],
+                                               user=["a", "c", "b"],
+                                               flag='anyof'))
+
+
+def suite():
+
+    testcases = [Test_PositionsCompare,
+                   Test_DragAndDrop_Populate,
+                   Test_DragAndDrop_Grade,
+                   Test_DraAndDrop_Compare_Positions
+                 ]
+    suites = []
+    for testcase in testcases:
+        suites.append(unittest.TestLoader().loadTestsFromTestCase(testcase))
+    return unittest.TestSuite(suites)
+
+if __name__ == "__main__":
+    unittest.TextTestRunner(verbosity=2).run(suite())

common/lib/logsettings.py

@@ -0,0 +1,148 @@
+import os
+import platform
+import sys
+from logging.handlers import SysLogHandler
+
+LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
+
+def get_logger_config(log_dir,
+                      logging_env="no_env",
+                      tracking_filename="tracking.log",
+                      edx_filename="edx.log",
+                      dev_env=False,
+                      syslog_addr=None,
+                      debug=False,
+                      local_loglevel='INFO',
+                      console_loglevel=None):
+
+    """
+
+    Return the appropriate logging config dictionary. You should assign the
+    result of this to the LOGGING var in your settings. The reason it's done
+    this way instead of registering directly is because I didn't want to worry
+    about resetting the logging state if this is called multiple times when
+    settings are extended.
+
+    If dev_env is set to true logging will not be done via local rsyslogd,
+    instead, tracking and application logs will be dropped in log_dir.
+
+    "tracking_filename" and "edx_filename" are ignored unless dev_env
+    is set to true since otherwise logging is handled by rsyslogd.
+
+    """
+
+    # Revert to INFO if an invalid string is passed in
+    if local_loglevel not in LOG_LEVELS:
+        local_loglevel = 'INFO'
+
+    if console_loglevel is None or console_loglevel not in LOG_LEVELS:
+        console_loglevel = 'DEBUG' if debug else 'INFO'
+
+    hostname = platform.node().split(".")[0]
+    syslog_format = ("[%(name)s][env:{logging_env}] %(levelname)s "
+                     "[{hostname}  %(process)d] [%(filename)s:%(lineno)d] "
+                     "- %(message)s").format(
+                        logging_env=logging_env, hostname=hostname)
+
+    handlers = ['console', 'local'] if debug else ['console',
+                                'syslogger-remote', 'local']
+
+    logger_config = {
+        'version': 1,
+        'disable_existing_loggers': False,
+        'formatters': {
+            'standard': {
+                'format': '%(asctime)s %(levelname)s %(process)d '
+                          '[%(name)s] %(filename)s:%(lineno)d - %(message)s',
+            },
+            'syslog_format': {'format': syslog_format},
+            'raw': {'format': '%(message)s'},
+        },
+        'handlers': {
+            'console': {
+                'level': console_loglevel,
+                'class': 'logging.StreamHandler',
+                'formatter': 'standard',
+                'stream': sys.stdout,
+            },
+            'syslogger-remote': {
+                'level': 'INFO',
+                'class': 'logging.handlers.SysLogHandler',
+                'address': syslog_addr,
+                'formatter': 'syslog_format',
+            },
+            'newrelic': {
+                'level': 'ERROR',
+                'class': 'newrelic_logging.NewRelicHandler',
+                'formatter': 'raw',
+            }
+        },
+        'loggers': {
+            'django': {
+                'handlers': handlers,
+                'propagate': True,
+                'level': 'INFO'
+            },
+            'tracking': {
+                'handlers': ['tracking'],
+                'level': 'DEBUG',
+                'propagate': False,
+            },
+            '': {
+                'handlers': handlers,
+                'level': 'DEBUG',
+                'propagate': False
+            },
+            'mitx': {
+                'handlers': handlers,
+                'level': 'DEBUG',
+                'propagate': False
+            },
+            'keyedcache': {
+                'handlers': handlers,
+                'level': 'DEBUG',
+                'propagate': False
+            },
+        }
+    }
+
+    if dev_env:
+        tracking_file_loc = os.path.join(log_dir, tracking_filename)
+        edx_file_loc = os.path.join(log_dir, edx_filename)
+        logger_config['handlers'].update({
+            'local': {
+                'class': 'logging.handlers.RotatingFileHandler',
+                'level': local_loglevel,
+                'formatter': 'standard',
+                'filename': edx_file_loc,
+                'maxBytes': 1024 * 1024 * 2,
+                'backupCount': 5,
+            },
+            'tracking': {
+                'level': 'DEBUG',
+                'class': 'logging.handlers.RotatingFileHandler',
+                'filename': tracking_file_loc,
+                'formatter': 'raw',
+                'maxBytes': 1024 * 1024 * 2,
+                'backupCount': 5,
+            },
+        })
+    else:
+        logger_config['handlers'].update({
+            'local': {
+                'level': local_loglevel,
+                'class': 'logging.handlers.SysLogHandler',
+                'address': '/dev/log',
+                'formatter': 'syslog_format',
+                'facility': SysLogHandler.LOG_LOCAL0,
+            },
+            'tracking': {
+                'level': 'DEBUG',
+                'class': 'logging.handlers.SysLogHandler',
+                'address': '/dev/log',
+                'facility': SysLogHandler.LOG_LOCAL1,
+                'formatter': 'raw',
+            },
+        })
+
+    return logger_config

common/lib/rooted_paths.py

@@ -0,0 +1,18 @@
+import glob2
+
+
+def rooted_glob(root, glob):
+    """
+    Returns the results of running `glob` rooted in the directory `root`.
+    All returned paths are relative to `root`.
+
+    Uses glob2 globbing
+    """
+    return remove_root(root, glob2.glob('{root}/{glob}'.format(root=root, glob=glob)))
+
+
+def remove_root(root, paths):
+    """
+    Returns `paths` made relative to `root`
+    """
+    return [pth.replace(root + '/', '') for pth in paths]

common/lib/xmodule/jasmine_test_runner.html.erb

@@ -0,0 +1,48 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+  "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<head>
+  <title>Jasmine Test Runner</title>
+  <link rel="stylesheet" type="text/css" href="<%= phantom_jasmine_path %>/vendor/jasmine-1.2.0/jasmine.css">
+  <script type="text/javascript" src="<%= phantom_jasmine_path %>/vendor/jasmine-1.2.0/jasmine.js"></script>
+  <script type="text/javascript" src="<%= phantom_jasmine_path %>/vendor/jasmine-1.2.0/jasmine-html.js"></script>
+
+  <script type="text/javascript" src="<%= phantom_jasmine_path %>/lib/console-runner.js"></script>
+  <script type="text/javascript" src="<%= common_coffee_root %>/ajax_prefix.js"></script>
+  <script type="text/javascript" src="<%= common_coffee_root %>/logger.js"></script>
+  <script type="text/javascript" src="<%= common_js_root %>/vendor/jquery.min.js"></script>
+  <script type="text/javascript" src="<%= common_js_root %>/vendor/jasmine-jquery.js"></script>
+  <script type="text/javascript" src="<%= common_js_root %>/vendor/jquery.cookie.js"></script>
+  <script type="text/javascript" src="<%= common_js_root %>/vendor/CodeMirror/codemirror.js"></script>
+  <script type="text/javascript" src="<%= common_js_root %>/vendor/mathjax-MathJax-c9db6ac/MathJax.js"></script>
+  <script type="text/javascript" src="<%= common_js_root %>/vendor/tiny_mce/jquery.tinymce.js"></script>
+  <script type="text/javascript" src="<%= common_js_root %>/vendor/tiny_mce/tiny_mce.js"></script>
+
+  <script type="text/javascript">
+    AjaxPrefix.addAjaxPrefix(jQuery, function() {
+       return "";
+    });
+  </script>
+
+  <!-- SOURCE FILES -->
+  <% for src in js_source %>
+  <script type="text/javascript" src="<%= src %>"></script>
+  <% end %>
+
+  <!-- SPEC FILES -->
+  <% for src in js_specs %>
+  <script type="text/javascript" src="<%= src %>"></script>
+  <% end %>
+</head>
+<body>
+
+<script type="text/javascript">
+  var console_reporter = new jasmine.ConsoleReporter()
+  jasmine.getEnv().addReporter(new jasmine.TrivialReporter());
+  jasmine.getEnv().addReporter(console_reporter);
+  jasmine.getEnv().execute();
+</script>
+
+</body>
+</html>
+

common/lib/xmodule/setup.py

@@ -36,7 +36,11 @@ setup(
             "videodev = xmodule.backcompat_module:TranslateCustomTagDescriptor",
             "videosequence = xmodule.seq_module:SequenceDescriptor",
             "discussion = xmodule.discussion_module:DiscussionDescriptor",
-            "graphical_slider_tool = xmodule.gst_module:GraphicalSliderToolDescriptor",
+            "course_info = xmodule.html_module:CourseInfoDescriptor",
+            "static_tab = xmodule.html_module:StaticTabDescriptor",
+            "custom_tag_template = xmodule.raw_module:RawDescriptor",
+            "about = xmodule.html_module:AboutDescriptor",
+            "graphical_slider_tool = xmodule.gst_module:GraphicalSliderToolDescriptor"
         ]
     }
 )

common/lib/xmodule/xmodule/abtest_module.py

@@ -52,9 +52,10 @@ class ABTestModule(XModule):
     def get_shared_state(self):
         return json.dumps({'group': self.group})
     
-    def get_children_locations(self):
-        return self.definition['data']['group_content'][self.group]
-        
+    def get_child_descriptors(self):
+        active_locations = set(self.definition['data']['group_content'][self.group])
+        return [desc for desc in self.descriptor.get_children() if desc.location.url() in active_locations]
+
     def displayable_items(self):
         # Most modules return "self" as the displayable_item. We never display ourself
         # (which is why we don't implement get_html). We only display our children.
@@ -66,7 +67,7 @@ class ABTestModule(XModule):
 class ABTestDescriptor(RawDescriptor, XmlDescriptor):
     module_class = ABTestModule
 
-#    template_dir_name = "abtest"
+    template_dir_name = "abtest"
 
     def __init__(self, system, definition=None, **kwargs):
         """

common/lib/xmodule/xmodule/capa_module.py

@@ -10,7 +10,6 @@ import sys
 
 from datetime import timedelta
 from lxml import etree
-from lxml.html import rewrite_links
 from pkg_resources import resource_string
 
 from capa.capa_problem import LoncapaProblem
@@ -83,7 +82,8 @@ class CapaModule(XModule):
                      resource_string(__name__, 'js/src/javascript_loader.coffee'),
                     ],
           'js': [resource_string(__name__, 'js/src/capa/imageinput.js'),
-                 resource_string(__name__, 'js/src/capa/schematic.js')]}
+                 resource_string(__name__, 'js/src/capa/schematic.js')
+                 ]}
 
     js_module_name = "Problem"
     css = {'scss': [resource_string(__name__, 'css/capa/display.scss')]}
@@ -352,16 +352,8 @@ class CapaModule(XModule):
             html = '<div id="problem_{id}" class="problem" data-url="{ajax_url}">'.format(
                 id=self.location.html_id(), ajax_url=self.system.ajax_url) + html + "</div>"
 
-        # cdodge: OK, we have to do two rounds of url reference subsitutions
-        # one which uses the 'asset library' that is served by the contentstore and the
-        # more global /static/ filesystem based static content.
-        # NOTE: rewrite_content_links is defined in XModule
-        # This is a bit unfortunate and I'm sure we'll try to considate this into
-        # a one step process.
-        html = rewrite_links(html, self.rewrite_content_links)
-
         # now do the substitutions which are filesystem based, e.g. '/static/' prefixes
-        return self.system.replace_urls(html, self.metadata['data_dir'])
+        return self.system.replace_urls(html, self.metadata['data_dir'], course_namespace=self.location)
 
     def handle_ajax(self, dispatch, get):
         '''
@@ -466,7 +458,7 @@ class CapaModule(XModule):
         new_answers = dict()
         for answer_id in answers:
             try:
-                new_answer = {answer_id: self.system.replace_urls(answers[answer_id], self.metadata['data_dir'])}
+                new_answer = {answer_id: self.system.replace_urls(answers[answer_id], self.metadata['data_dir'], course_namespace=self.location)}
             except TypeError:
                 log.debug('Unable to perform URL substitution on answers[%s]: %s' % (answer_id, answers[answer_id]))
                 new_answer = {answer_id: answers[answer_id]}
@@ -665,11 +657,29 @@ class CapaDescriptor(RawDescriptor):
     stores_state = True
     has_score = True
     template_dir_name = 'problem'
+    mako_template = "widgets/problem-edit.html"
+    js = {'coffee': [resource_string(__name__, 'js/src/problem/edit.coffee')]}
+    js_module_name = "MarkdownEditingDescriptor"
+    css = {'scss': [resource_string(__name__, 'css/editor/edit.scss'), resource_string(__name__, 'css/problem/edit.scss')]}
 
     # Capa modules have some additional metadata:
     # TODO (vshnayder): do problems have any other metadata?  Do they
     # actually use type and points?
     metadata_attributes = RawDescriptor.metadata_attributes + ('type', 'points')
+    
+    def get_context(self):
+        _context = RawDescriptor.get_context(self)
+        _context.update({'markdown': self.metadata.get('markdown', '')})
+        return _context
+    
+    @property
+    def editable_metadata_fields(self):
+        """Remove metadata from the editable fields since it has its own editor"""
+        subset = super(CapaDescriptor,self).editable_metadata_fields
+        if 'markdown' in subset:
+            subset.remove('markdown') 
+        return subset
+
 
     # VS[compat]
     # TODO (cpennington): Delete this method once all fall 2012 course are being

common/lib/xmodule/xmodule/contentstore/content.py

@@ -1,26 +1,155 @@
 XASSET_LOCATION_TAG = 'c4x'
 XASSET_SRCREF_PREFIX = 'xasset:'
 
+XASSET_THUMBNAIL_TAIL_NAME = '.jpg'
+
+import os
+import logging
+import StringIO
+
+from xmodule.modulestore import Location
+from .django import contentstore
+from PIL import Image
+
 class StaticContent(object):
-    def __init__(self, filename, name, content_type, data, last_modified_at=None):
-        self.filename = filename
-        self.name = name
+    def __init__(self, loc, name, content_type, data, last_modified_at=None, thumbnail_location=None, import_path=None):
+        self.location = loc
+        self.name = name #a display string which can be edited, and thus not part of the location which needs to be fixed
         self.content_type = content_type
         self.data = data
         self.last_modified_at = last_modified_at
+        self.thumbnail_location = Location(thumbnail_location) if thumbnail_location is not None else None
+        # optional information about where this file was imported from. This is needed to support import/export 
+        # cycles
+        self.import_path = import_path
+
+    @property
+    def is_thumbnail(self):
+        return self.location.category == 'thumbnail'
 
     @staticmethod
-    def compute_location_filename(org, course, name):
-        return '/{0}/{1}/{2}/asset/{3}'.format(XASSET_LOCATION_TAG, org, course, name)
+    def generate_thumbnail_name(original_name):
+        return ('{0}'+XASSET_THUMBNAIL_TAIL_NAME).format(os.path.splitext(original_name)[0])
+
+    @staticmethod
+    def compute_location(org, course, name, revision=None, is_thumbnail=False):
+        name = name.replace('/', '_')
+        return Location([XASSET_LOCATION_TAG, org, course, 'asset' if not is_thumbnail else 'thumbnail', Location.clean(name), revision])
+
+    def get_id(self):
+        return StaticContent.get_id_from_location(self.location)
+
+    def get_url_path(self):
+        return StaticContent.get_url_path_from_location(self.location)
+    
+    @staticmethod
+    def get_url_path_from_location(location):
+        if location is not None:
+            return "/{tag}/{org}/{course}/{category}/{name}".format(**location.dict())
+        else:
+            return None
+
+    @staticmethod
+    def get_base_url_path_for_course_assets(loc):
+        if loc is not None:
+            return "/c4x/{org}/{course}/asset".format(**loc.dict())
+
+    @staticmethod
+    def get_id_from_location(location):
+        return { 'tag':location.tag, 'org' : location.org, 'course' : location.course,
+                   'category' : location.category, 'name' : location.name, 
+                   'revision' : location.revision}
+    @staticmethod
+    def get_location_from_path(path):
+        # remove leading / character if it is there one
+        if path.startswith('/'):
+            path = path[1:]
+        
+        return Location(path.split('/'))
+
+    @staticmethod
+    def get_id_from_path(path):
+        return get_id_from_location(get_location_from_path(path))
+
+    @staticmethod
+    def convert_legacy_static_url(path, course_namespace):
+        loc = StaticContent.compute_location(course_namespace.org, course_namespace.course, path)
+        return StaticContent.get_url_path_from_location(loc)
+
+
+    
 
-'''
-Abstraction for all ContentStore providers (e.g. MongoDB)
-'''
 class ContentStore(object):
+    '''
+    Abstraction for all ContentStore providers (e.g. MongoDB)
+    '''
     def save(self, content):
         raise NotImplementedError
 
     def find(self, filename):
         raise NotImplementedError
 
-            
+    def get_all_content_for_course(self, location):
+        '''
+        Returns a list of all static assets for a course. The return format is a list of dictionary elements. Example:
+
+            [
+
+            {u'displayname': u'profile.jpg', u'chunkSize': 262144, u'length': 85374, 
+            u'uploadDate': datetime.datetime(2012, 10, 3, 5, 41, 54, 183000), u'contentType': u'image/jpeg', 
+            u'_id': {u'category': u'asset', u'name': u'profile.jpg', u'course': u'6.002x', u'tag': u'c4x', 
+            u'org': u'MITx', u'revision': None}, u'md5': u'36dc53519d4b735eb6beba51cd686a0e'}, 
+
+            {u'displayname': u'profile.thumbnail.jpg', u'chunkSize': 262144, u'length': 4073, 
+            u'uploadDate': datetime.datetime(2012, 10, 3, 5, 41, 54, 196000), u'contentType': u'image/jpeg', 
+            u'_id': {u'category': u'asset', u'name': u'profile.thumbnail.jpg', u'course': u'6.002x', u'tag': u'c4x', 
+            u'org': u'MITx', u'revision': None}, u'md5': u'ff1532598830e3feac91c2449eaa60d6'},
+
+            ....
+
+            ]
+        '''
+        raise NotImplementedError
+
+    def generate_thumbnail(self, content):
+        thumbnail_content = None
+        # use a naming convention to associate originals with the thumbnail
+        thumbnail_name = StaticContent.generate_thumbnail_name(content.location.name)
+
+        thumbnail_file_location = StaticContent.compute_location(content.location.org, content.location.course,
+            thumbnail_name, is_thumbnail = True)
+
+        # if we're uploading an image, then let's generate a thumbnail so that we can
+        # serve it up when needed without having to rescale on the fly
+        if content.content_type is not None and content.content_type.split('/')[0] == 'image':
+            try:
+                # use PIL to do the thumbnail generation (http://www.pythonware.com/products/pil/)
+                # My understanding is that PIL will maintain aspect ratios while restricting
+                # the max-height/width to be whatever you pass in as 'size'
+                # @todo: move the thumbnail size to a configuration setting?!?
+                im = Image.open(StringIO.StringIO(content.data))
+
+                # I've seen some exceptions from the PIL library when trying to save palletted 
+                # PNG files to JPEG. Per the google-universe, they suggest converting to RGB first.
+                im = im.convert('RGB')
+                size = 128, 128
+                im.thumbnail(size, Image.ANTIALIAS)
+                thumbnail_file = StringIO.StringIO()
+                im.save(thumbnail_file, 'JPEG')
+                thumbnail_file.seek(0)
+
+                # store this thumbnail as any other piece of content
+                thumbnail_content = StaticContent(thumbnail_file_location, thumbnail_name, 
+                                                  'image/jpeg', thumbnail_file)
+
+                contentstore().save(thumbnail_content)
+
+            except Exception, e:
+                # log and continue as thumbnails are generally considered as optional
+                logging.exception("Failed to generate thumbnail for {0}. Exception: {1}".format(content.location, str(e)))
+
+        return thumbnail_content, thumbnail_file_location
+
+
+
+

common/lib/xmodule/xmodule/contentstore/mongo.py

@@ -1,32 +1,111 @@
+from bson.son import SON
 from pymongo import Connection
 import gridfs
 from gridfs.errors import NoFile
 
+from xmodule.modulestore.mongo import location_to_query, Location
+from xmodule.contentstore.content import XASSET_LOCATION_TAG
+
 import sys
 import logging
 
 from .content import StaticContent, ContentStore
 from xmodule.exceptions import NotFoundError
+from fs.osfs import OSFS
+import os
 
 
 class MongoContentStore(ContentStore):
-    def __init__(self, host, db, port=27017):
+    def __init__(self, host, db, port=27017, user=None, password=None, **kwargs):
         logging.debug( 'Using MongoDB for static content serving at host={0} db={1}'.format(host,db))
-        _db = Connection(host=host, port=port)[db]
+        _db = Connection(host=host, port=port, **kwargs)[db]
+
+        if user is not None and password is not None:
+            _db.authenticate(user, password)
+
         self.fs = gridfs.GridFS(_db)
+        self.fs_files = _db["fs.files"] # the underlying collection GridFS uses
+
 
     def save(self, content):
-        with self.fs.new_file(filename=content.filename, content_type=content.content_type, displayname=content.name) as fp:
+        id = content.get_id()
+
+        # Seems like with the GridFS we can't update existing ID's we have to do a delete/add pair
+        self.delete(id)
+
+        with self.fs.new_file(_id = id, filename=content.get_url_path(), content_type=content.content_type, 
+            displayname=content.name, thumbnail_location=content.thumbnail_location, import_path=content.import_path) as fp:
+
             fp.write(content.data)
-            return content
         
-    
-    def find(self, filename):
+        return content
+        
+    def delete(self, id):
+        if self.fs.exists({"_id" : id}):
+            self.fs.delete(id)
+
+    def find(self, location):
+        id = StaticContent.get_id_from_location(location)
         try:
-            with self.fs.get_last_version(filename) as fp:
-                return StaticContent(fp.filename, fp.displayname, fp.content_type, fp.read(), fp.uploadDate)
+            with self.fs.get(id) as fp:
+                return StaticContent(location, fp.displayname, fp.content_type, fp.read(), 
+                    fp.uploadDate, thumbnail_location = fp.thumbnail_location if hasattr(fp, 'thumbnail_location') else None,
+                    import_path = fp.import_path if hasattr(fp, 'import_path') else None)
         except NoFile:
             raise NotFoundError()
 
+    def export(self, location, output_directory):
+        content = self.find(location)
+
+        if content.import_path is not None:
+            output_directory = output_directory + '/' + os.path.dirname(content.import_path)
+
+        if not os.path.exists(output_directory):
+            os.makedirs(output_directory)
+
+        disk_fs = OSFS(output_directory)
+
+        with disk_fs.open(content.name, 'wb') as asset_file:
+            asset_file.write(content.data)
+
+    def export_all_for_course(self, course_location, output_directory):
+        assets = self.get_all_content_for_course(course_location)
+
+        for asset in assets:
+            asset_location = Location(asset['_id'])
+            self.export(asset_location, output_directory)
+
+    def get_all_content_thumbnails_for_course(self, location):
+        return self._get_all_content_for_course(location, get_thumbnails = True)
+
+    def get_all_content_for_course(self, location):
+        return self._get_all_content_for_course(location, get_thumbnails = False)
+
+    def _get_all_content_for_course(self, location, get_thumbnails = False):
+        '''
+        Returns a list of all static assets for a course. The return format is a list of dictionary elements. Example:
+
+            [
+
+            {u'displayname': u'profile.jpg', u'chunkSize': 262144, u'length': 85374, 
+            u'uploadDate': datetime.datetime(2012, 10, 3, 5, 41, 54, 183000), u'contentType': u'image/jpeg', 
+            u'_id': {u'category': u'asset', u'name': u'profile.jpg', u'course': u'6.002x', u'tag': u'c4x', 
+            u'org': u'MITx', u'revision': None}, u'md5': u'36dc53519d4b735eb6beba51cd686a0e'}, 
+
+            {u'displayname': u'profile.thumbnail.jpg', u'chunkSize': 262144, u'length': 4073, 
+            u'uploadDate': datetime.datetime(2012, 10, 3, 5, 41, 54, 196000), u'contentType': u'image/jpeg', 
+            u'_id': {u'category': u'asset', u'name': u'profile.thumbnail.jpg', u'course': u'6.002x', u'tag': u'c4x', 
+            u'org': u'MITx', u'revision': None}, u'md5': u'ff1532598830e3feac91c2449eaa60d6'},
+
+            ....
+
+            ]
+        '''
+        course_filter = Location(XASSET_LOCATION_TAG, category="asset" if not get_thumbnails else "thumbnail",
+            course=location.course,org=location.org)
+        # 'borrow' the function 'location_to_query' from the Mongo modulestore implementation
+        items = self.fs_files.find(location_to_query(course_filter))
+        return list(items)
+
 
         

common/lib/xmodule/xmodule/course_module.py

@@ -1,4 +1,5 @@
 import logging
+from cStringIO import StringIO
 from math import exp, erf
 from lxml import etree
 from path import path  # NOTE (THK): Only used for detecting presence of syllabus
@@ -6,18 +7,33 @@ import requests
 import time
 from datetime import datetime
 
-from xmodule.util.decorators import lazyproperty
-from xmodule.graders import load_grading_policy
 from xmodule.modulestore import Location
 from xmodule.seq_module import SequenceDescriptor, SequenceModule
 from xmodule.timeparse import parse_time, stringify_time
+from xmodule.util.decorators import lazyproperty
+from xmodule.graders import grader_from_conf
+from datetime import datetime
+import json
+import logging
+import requests
+import time
+import copy
+
 
 log = logging.getLogger(__name__)
 
 
+edx_xml_parser = etree.XMLParser(dtd_validation=False, load_dtd=False,
+                                 remove_comments=True, remove_blank_text=True)
+
+_cached_toc = {}
+
+
 class CourseDescriptor(SequenceDescriptor):
     module_class = SequenceModule
 
+    template_dir_name = 'course'
+
     class Textbook:
         def __init__(self, title, book_url):
             self.title = title
@@ -45,6 +61,24 @@ class CourseDescriptor(SequenceDescriptor):
             """
             toc_url = self.book_url + 'toc.xml'
 
+            # cdodge: I've added this caching of TOC because in Mongo-backed instances (but not Filesystem stores)
+            # course modules have a very short lifespan and are constantly being created and torn down.
+            # Since this module in the __init__() method does a synchronous call to AWS to get the TOC
+            # this is causing a big performance problem. So let's be a bit smarter about this and cache
+            # each fetch and store in-mem for 10 minutes.
+            # NOTE: I have to get this onto sandbox ASAP as we're having runtime failures. I'd like to swing back and
+            # rewrite to use the traditional Django in-memory cache.
+            try:
+                # see if we already fetched this
+                if toc_url in _cached_toc:
+                    (table_of_contents, timestamp) = _cached_toc[toc_url]
+                    age = datetime.now() - timestamp
+                    # expire every 10 minutes
+                    if age.seconds < 600:
+                        return table_of_contents
+            except Exception as err:
+                pass
+
             # Get the table of contents from S3
             log.info("Retrieving textbook table of contents from %s" % toc_url)
             try:
@@ -57,6 +91,7 @@ class CourseDescriptor(SequenceDescriptor):
             # TOC is XML. Parse it
             try:
                 table_of_contents = etree.fromstring(r.text)
+                _cached_toc[toc_url] = (table_of_contents, datetime.now())
             except Exception as err:
                 msg = 'Error %s: Unable to parse XML for textbook table of contents at %s' % (err, toc_url)
                 log.error(msg)
@@ -66,7 +101,6 @@ class CourseDescriptor(SequenceDescriptor):
 
     def __init__(self, system, definition=None, **kwargs):
         super(CourseDescriptor, self).__init__(system, definition, **kwargs)
-
         self.textbooks = []
         for title, book_url in self.definition['data']['textbooks']:
             try:
@@ -87,16 +121,13 @@ class CourseDescriptor(SequenceDescriptor):
             log.critical(msg)
             system.error_tracker(msg)
 
-        self.enrollment_start = self._try_parse_time("enrollment_start")
-        self.enrollment_end = self._try_parse_time("enrollment_end")
-        self.end = self._try_parse_time("end")
-
         # NOTE: relies on the modulestore to call set_grading_policy() right after
         # init.  (Modulestore is in charge of figuring out where to load the policy from)
 
         # NOTE (THK): This is a last-minute addition for Fall 2012 launch to dynamically
         #   disable the syllabus content for courses that do not provide a syllabus
         self.syllabus_present = self.system.resources_fs.exists(path('syllabus'))
+        self.set_grading_policy(self.definition['data'].get('grading_policy', None))
 
         self.test_center_exams = []
         test_center_info = self.metadata.get('testcenter_info')
@@ -112,18 +143,121 @@ class CourseDescriptor(SequenceDescriptor):
                     log.error(msg)
                     continue
 
+    def defaut_grading_policy(self):
+        """
+        Return a dict which is a copy of the default grading policy
+        """
+        default = {"GRADER" : [
+                {
+                    "type" : "Homework",
+                    "min_count" : 12,
+                    "drop_count" : 2,
+                    "short_label" : "HW",
+                    "weight" : 0.15
+                },
+                {
+                    "type" : "Lab",
+                    "min_count" : 12,
+                    "drop_count" : 2,
+                    "weight" : 0.15
+                },
+                {
+                    "type" : "Midterm Exam",
+                    "short_label" : "Midterm",
+                    "min_count" : 1,
+                    "drop_count" : 0,
+                    "weight" : 0.3
+                },
+                {
+                    "type" : "Final Exam",
+                    "short_label" : "Final",
+                    "min_count" : 1,
+                    "drop_count" : 0,
+                    "weight" : 0.4
+                }
+            ],
+            "GRADE_CUTOFFS" : {
+                "Pass" : 0.5
+            }}
+        return copy.deepcopy(default)
+
+    def set_grading_policy(self, course_policy):
+        """
+        The JSON object can have the keys GRADER and GRADE_CUTOFFS. If either is
+        missing, it reverts to the default.
+        """
+        if course_policy is None:
+            course_policy = {}
+
+        # Load the global settings as a dictionary
+        grading_policy = self.defaut_grading_policy()
+
+        # Override any global settings with the course settings
+        grading_policy.update(course_policy)
+
+        # Here is where we should parse any configurations, so that we can fail early
+        grading_policy['RAW_GRADER'] = grading_policy['GRADER']  # used for cms access
+        grading_policy['GRADER'] = grader_from_conf(grading_policy['GRADER'])
+        self._grading_policy = grading_policy
+
+
+
+    @classmethod
+    def read_grading_policy(cls, paths, system):
+        """Load a grading policy from the specified paths, in order, if it exists."""
+        # Default to a blank policy dict
+        policy_str = '{}'
+
+        for policy_path in paths:
+            if not system.resources_fs.exists(policy_path):
+                continue
+            log.debug("Loading grading policy from {0}".format(policy_path))
+            try:
+                with system.resources_fs.open(policy_path) as grading_policy_file:
+                    policy_str = grading_policy_file.read()
+                    # if we successfully read the file, stop looking at backups
+                    break
+            except (IOError):
+                msg = "Unable to load course settings file from '{0}'".format(policy_path)
+                log.warning(msg)
+
+        return policy_str
+
+    
+    @classmethod
+    def from_xml(cls, xml_data, system, org=None, course=None):
+        instance = super(CourseDescriptor, cls).from_xml(xml_data, system, org, course)
+
+        # bleh, have to parse the XML here to just pull out the url_name attribute
+        # I don't think it's stored anywhere in the instance.
+        course_file = StringIO(xml_data.encode('ascii','ignore'))
+        xml_obj = etree.parse(course_file,parser=edx_xml_parser).getroot()
+
+        policy_dir = None
+        url_name = xml_obj.get('url_name', xml_obj.get('slug'))
+        if url_name:
+            policy_dir = 'policies/' + url_name
+
+        # Try to load grading policy
+        paths = ['grading_policy.json']
+        if policy_dir:
+            paths = [policy_dir + '/grading_policy.json'] + paths
 
-    def set_grading_policy(self, policy_str):
-        """Parse the policy specified in policy_str, and save it"""
         try:
-            self._grading_policy = load_grading_policy(policy_str)
-        except Exception, err:
-            log.exception('Failed to load grading policy:')
-            self.system.error_tracker("Failed to load grading policy")
-            # Setting this to an empty dictionary will lead to errors when
-            # grading needs to happen, but should allow course staff to see
-            # the error log.
-            self._grading_policy = {}
+            policy = json.loads(cls.read_grading_policy(paths, system))
+        except ValueError:
+            system.error_tracker("Unable to decode grading policy as json")
+            policy = None
+        
+        # cdodge: import the grading policy information that is on disk and put into the
+        # descriptor 'definition' bucket as a dictionary so that it is persisted in the DB
+        instance.definition['data']['grading_policy'] = policy
+
+        # now set the current instance. set_grading_policy() will apply some inheritance rules
+        instance.set_grading_policy(policy)
+
+        return instance
+
 
     @classmethod
     def definition_from_xml(cls, xml_object, system):
@@ -159,13 +293,53 @@ class CourseDescriptor(SequenceDescriptor):
     def has_started(self):
         return time.gmtime() > self.start
 
+    @property
+    def end(self):
+        return self._try_parse_time("end")
+    @end.setter
+    def end(self, value):
+        if isinstance(value, time.struct_time):
+            self.metadata['end'] = stringify_time(value)
+    @property
+    def enrollment_start(self):
+        return self._try_parse_time("enrollment_start")
+        
+    @enrollment_start.setter
+    def enrollment_start(self, value):
+        if isinstance(value, time.struct_time):
+            self.metadata['enrollment_start'] = stringify_time(value)
+    @property
+    def enrollment_end(self):        
+        return self._try_parse_time("enrollment_end")
+        
+    @enrollment_end.setter
+    def enrollment_end(self, value):
+        if isinstance(value, time.struct_time):
+            self.metadata['enrollment_end'] = stringify_time(value)
+        
     @property
     def grader(self):
         return self._grading_policy['GRADER']
+    
+    @property
+    def raw_grader(self):
+        return self._grading_policy['RAW_GRADER']
+    
+    @raw_grader.setter
+    def raw_grader(self, value):
+        # NOTE WELL: this change will not update the processed graders. If we need that, this needs to call grader_from_conf
+        self._grading_policy['RAW_GRADER'] = value
+        self.definition['data'].setdefault('grading_policy',{})['GRADER'] = value
 
     @property
     def grade_cutoffs(self):
         return self._grading_policy['GRADE_CUTOFFS']
+    
+    @grade_cutoffs.setter
+    def grade_cutoffs(self, value):
+        self._grading_policy['GRADE_CUTOFFS'] = value
+        self.definition['data'].setdefault('grading_policy',{})['GRADE_CUTOFFS'] = value
+    
 
     @property
     def lowest_passing_grade(self):
@@ -178,6 +352,10 @@ class CourseDescriptor(SequenceDescriptor):
         """
         return self.metadata.get('tabs')
 
+    @tabs.setter
+    def tabs(self, value):
+        self.metadata['tabs'] = value
+
     @property
     def show_calculator(self):
         return self.metadata.get("show_calculator", None) == "Yes"
@@ -502,3 +680,4 @@ class CourseDescriptor(SequenceDescriptor):
     @property
     def org(self):
         return self.location.org
+

common/lib/xmodule/xmodule/css/codemirror/codemirror.scss

@@ -0,0 +1,5 @@
+.CodeMirror {
+  background: #fff;
+  font-size: 13px;
+  color: #3c3c3c;
+}

common/lib/xmodule/xmodule/css/editor/edit.scss

@@ -0,0 +1,63 @@
+// This is shared CSS between the xmodule problem editor and the xmodule HTML editor.
+.editor {
+  position: relative;
+
+  .row {
+    position: relative;
+  }
+
+  .editor-bar {
+    position: relative;
+    @include linear-gradient(top, #d4dee8, #c9d5e2);
+    padding: 5px;
+    border: 1px solid #3c3c3c;
+    border-radius: 3px 3px 0 0;
+    border-bottom-color: #a5aaaf;
+    @include clearfix;
+
+    a {
+      display: block;
+      float: left;
+      padding: 3px 10px 7px;
+      margin-left: 7px;
+      border-radius: 2px;
+
+      &:hover {
+        background: rgba(255, 255, 255, .5);
+      }
+    }
+  }
+
+  .editor-tabs {
+    position: absolute;
+    top: 10px;
+    right: 10px;
+
+    li {
+      float: left;
+      margin-right: 5px;
+
+      &:last-child {
+        margin-right: 0;
+      }
+    }
+
+    .tab {
+      display: block;
+      height: 24px;
+      padding: 7px 20px 3px;
+      border: 1px solid #a5aaaf;
+      border-radius: 3px 3px 0 0;
+      @include linear-gradient(top, rgba(0, 0, 0, 0) 87%, rgba(0, 0, 0, .06));
+      background-color: #e5ecf3;
+      font-size: 13px;
+      color: #3c3c3c;
+      box-shadow: 1px -1px 1px rgba(0, 0, 0, .05);
+
+      &.current {
+        background: #fff;
+        border-bottom-color: #fff;
+      }
+    }
+  }
+}

common/lib/xmodule/xmodule/css/html/edit.scss

@@ -0,0 +1,29 @@
+.html-editor {
+  @include clearfix();
+
+  .CodeMirror {
+    @include box-sizing(border-box);
+    position: absolute;
+    top: 46px;
+    width: 100%;
+    height: 379px;
+    border: 1px solid #3c3c3c;
+    border-top: 1px solid #8891a1;
+    background: #fff;
+    color: #3c3c3c;
+  }
+
+  .CodeMirror-scroll {
+    height: 100%;
+  }
+
+  .editor-tabs {
+    top: 11px !important;
+    right: 10px;
+    z-index: 99;
+  }
+
+  .is-inactive {
+    display: none;
+  }
+}

common/lib/xmodule/xmodule/css/problem/edit.scss

@@ -0,0 +1,143 @@
+.editor-bar {
+
+  .editor-tabs {
+
+    .advanced-toggle {
+      @include white-button;
+      height: auto;
+      margin-top: -1px;
+      padding: 3px 9px;
+      font-size: 12px;
+
+      &.current {
+        border: 1px solid $lightGrey !important;
+        border-radius: 3px !important;
+        background: $lightGrey !important;
+        color: $darkGrey !important;
+        pointer-events: none;
+        cursor: none;
+        
+        &:hover {
+          box-shadow: 0 0 0 0 !important;
+        }
+      }
+    }
+
+    .cheatsheet-toggle {
+      width: 21px;
+      height: 21px;
+      padding: 0;
+      margin: 0 5px 0 15px;
+      border-radius: 22px;
+      border: 1px solid #a5aaaf;
+      background: #e5ecf3;
+      font-size: 13px;
+      font-weight: 700;
+      color: #565d64;
+      text-align: center;
+    }
+  }
+}
+
+.simple-editor-cheatsheet {
+  position: absolute;
+  top: 0;
+  left: 100%;
+  width: 0;
+  border-radius: 0 3px 3px 0;
+  @include linear-gradient(left, rgba(0, 0, 0, .1), rgba(0, 0, 0, 0) 4px);
+  background-color: #fff;
+  overflow: hidden;
+  @include transition(width .3s);
+
+  &.shown {
+    width: 300px;
+    height: 100%;
+    overflow-y: scroll;
+  }
+
+  .cheatsheet-wrapper {
+    width: 240px;
+    padding: 20px 30px;
+  }
+
+  h6 {
+    margin-bottom: 7px;
+    font-size: 15px;
+    font-weight: 700;
+  }
+
+  .row {
+    @include clearfix;
+    padding-bottom: 5px !important;
+    margin-bottom: 10px !important;
+    border-bottom: 1px solid #ddd !important;
+
+    &:last-child {
+      border-bottom: none !important;
+      margin-bottom: 0 !important;
+    }
+  }
+
+  .col {
+    float: left;
+
+    &.sample {
+      width: 60px;
+      margin-right: 30px;
+    }
+  }
+
+  pre {
+    font-size: 12px;
+    line-height: 18px;
+  }
+
+  code {
+    padding: 0;
+    background: none;
+  }
+}
+
+.problem-editor-icon {
+  display: inline-block;
+  width: 26px;
+  height: 21px;
+  vertical-align: middle;
+  background: url(../img/problem-editor-icons.png) no-repeat;
+}
+
+.problem-editor-icon.header {
+  width: 18px;
+  background-position: -265px 0;
+}
+
+.problem-editor-icon.multiple-choice {
+  background-position: 0 0;
+}
+
+.problem-editor-icon.checks {
+  background-position: -56px 0;
+}
+
+.problem-editor-icon.string {
+  width: 28px;
+  background-position: -111px 0;
+}
+
+.problem-editor-icon.number {
+  width: 24px;
+  background-position: -168px 0;
+}
+
+.problem-editor-icon.dropdown {
+  width: 17px;
+  background-position: -220px 0;
+}
+
+.problem-editor-icon.explanation {
+  width: 17px;
+  background-position: -307px 0;
+}
+
+

common/lib/xmodule/xmodule/css/sequence/display.scss

@@ -356,7 +356,7 @@ nav.sequence-bottom {
   }
 }
 
-div.course-wrapper section.course-content ol.vert-mod > li ul.sequence-nav-buttons {
+.xmodule_VerticalModule ol.vert-mod > li ul.sequence-nav-buttons {
   list-style: none !important;
 }
 

common/lib/xmodule/xmodule/css/video/display.scss

@@ -1,3 +1,7 @@
+& {
+    margin-bottom: 30px;
+}
+
 div.video {
   @include clearfix();
   background: #f3f3f3;
@@ -28,7 +32,7 @@ div.video {
     }
 
     section.video-controls {
-      @extend .clearfix;
+      @include clearfix();
       background: #333;
       border: 1px solid #000;
       border-top: 0;
@@ -42,7 +46,7 @@ div.video {
       }
 
       div.slider {
-        @extend .clearfix;
+        @include clearfix();
         background: #c2c2c2;
         border: 1px solid #000;
         @include border-radius(0);

common/lib/xmodule/xmodule/editing_module.py

@@ -30,6 +30,8 @@ class XMLEditingDescriptor(EditingDescriptor):
     any validation of its definition
     """
 
+    css = {'scss': [resource_string(__name__, 'css/codemirror/codemirror.scss')]}
+
     js = {'coffee': [resource_string(__name__, 'js/src/raw/edit/xml.coffee')]}
     js_module_name = "XMLEditingDescriptor"
 
@@ -40,5 +42,7 @@ class JSONEditingDescriptor(EditingDescriptor):
     any validation of its definition
     """
 
+    css = {'scss': [resource_string(__name__, 'css/codemirror/codemirror.scss')]}
+
     js = {'coffee': [resource_string(__name__, 'js/src/raw/edit/json.coffee')]}
     js_module_name = "JSONEditingDescriptor"

common/lib/xmodule/xmodule/graders.py

@@ -1,6 +1,5 @@
 import abc
 import inspect
-import json
 import logging
 import random
 import sys
@@ -13,69 +12,6 @@ log = logging.getLogger("mitx.courseware")
 # Section either indicates the name of the problem or the name of the section
 Score = namedtuple("Score", "earned possible graded section")
 
-def load_grading_policy(course_policy_string):
-    """
-    This loads a grading policy from a string (usually read from a file),
-    which can be a JSON object or an empty string.
-
-    The JSON object can have the keys GRADER and GRADE_CUTOFFS. If either is
-    missing, it reverts to the default.
-    """
-
-    default_policy_string = """
-    {
-        "GRADER" : [
-            {
-                "type" : "Homework",
-                "min_count" : 12,
-                "drop_count" : 2,
-                "short_label" : "HW",
-                "weight" : 0.15
-            },
-            {
-                "type" : "Lab",
-                "min_count" : 12,
-                "drop_count" : 2,
-                "category" : "Labs",
-                "weight" : 0.15
-            },
-            {
-                "type" : "Midterm",
-                "name" : "Midterm Exam",
-                "short_label" : "Midterm",
-                "weight" : 0.3
-            },
-            {
-                "type" : "Final",
-                "name" : "Final Exam",
-                "short_label" : "Final",
-                "weight" : 0.4
-            }
-        ],
-        "GRADE_CUTOFFS" : {
-            "A" : 0.87,
-            "B" : 0.7,
-            "C" : 0.6
-        }
-    }
-    """
-
-    # Load the global settings as a dictionary
-    grading_policy = json.loads(default_policy_string)
-
-    # Load the course policies as a dictionary
-    course_policy = {}
-    if course_policy_string:
-        course_policy = json.loads(course_policy_string)
-
-    # Override any global settings with the course settings
-    grading_policy.update(course_policy)
-
-    # Here is where we should parse any configurations, so that we can fail early
-    grading_policy['GRADER'] = grader_from_conf(grading_policy['GRADER'])
-
-    return grading_policy
-
 
 def aggregate_scores(scores, section_name="summary"):
     """
@@ -129,17 +65,27 @@ def grader_from_conf(conf):
     for subgraderconf in conf:
         subgraderconf = subgraderconf.copy()
         weight = subgraderconf.pop("weight", 0)
+        # NOTE: 'name' used to exist in SingleSectionGrader. We are deprecating SingleSectionGrader
+        # and converting everything into an AssignmentFormatGrader by adding 'min_count' and
+        # 'drop_count'. AssignmentFormatGrader does not expect 'name', so if it appears
+        # in bad_args, go ahead remove it (this causes no errors). Eventually, SingleSectionGrader
+        # should be completely removed.
+        name = 'name'
         try:
             if 'min_count' in subgraderconf:
                 #This is an AssignmentFormatGrader
                 subgrader_class = AssignmentFormatGrader
-            elif 'name' in subgraderconf:
+            elif name in subgraderconf:
                 #This is an SingleSectionGrader
                 subgrader_class = SingleSectionGrader
             else:
                 raise ValueError("Configuration has no appropriate grader class.")
             
             bad_args = invalid_args(subgrader_class.__init__, subgraderconf)
+            # See note above concerning 'name'.
+            if bad_args.issuperset({name}):
+                bad_args = bad_args - {name}
+                del subgraderconf[name]
             if len(bad_args) > 0:
                 log.warning("Invalid arguments for a subgrader: %s", bad_args)
                 for key in bad_args:

common/lib/xmodule/xmodule/html_module.py

@@ -4,7 +4,6 @@ import logging
 import os
 import sys
 from lxml import etree
-from lxml.html import rewrite_links
 from path import path
 
 from pkg_resources import resource_string
@@ -28,8 +27,7 @@ class HtmlModule(XModule):
     js_module_name = "HTMLModule"
     
     def get_html(self):
-        # cdodge: perform link substitutions for any references to course static content (e.g. images)
-        return rewrite_links(self.html, self.rewrite_content_links)
+        return self.html
 
     def __init__(self, system, location, definition, descriptor,
                  instance_state=None, shared_state=None, **kwargs):
@@ -50,6 +48,7 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
 
     js = {'coffee': [resource_string(__name__, 'js/src/html/edit.coffee')]}
     js_module_name = "HTMLEditingDescriptor"
+    css = {'scss': [resource_string(__name__, 'css/editor/edit.scss'), resource_string(__name__, 'css/html/edit.scss')]}
 
     # VS[compat] TODO (cpennington): Delete this method once all fall 2012 course
     # are being edited in the cms
@@ -161,7 +160,7 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
         filepath = u'{category}/{pathname}.html'.format(category=self.category,
                                                     pathname=pathname)
 
-        resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
+        resource_fs.makedir(os.path.dirname(filepath), recursive=True, allow_recreate=True)
         with resource_fs.open(filepath, 'w') as file:
             file.write(self.definition['data'].encode('utf-8'))
 
@@ -171,3 +170,25 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
         elt = etree.Element('html')
         elt.set("filename", relname)
         return elt
+
+
+class AboutDescriptor(HtmlDescriptor):
+    """
+    These pieces of course content are treated as HtmlModules but we need to overload where the templates are located
+    in order to be able to create new ones
+    """
+    template_dir_name = "about"
+
+class StaticTabDescriptor(HtmlDescriptor):
+    """
+    These pieces of course content are treated as HtmlModules but we need to overload where the templates are located
+    in order to be able to create new ones
+    """
+    template_dir_name = "statictab"
+
+class CourseInfoDescriptor(HtmlDescriptor):
+    """
+    These pieces of course content are treated as HtmlModules but we need to overload where the templates are located
+    in order to be able to create new ones
+    """
+    template_dir_name = "courseinfo"

common/lib/xmodule/xmodule/js/fixtures/html-edit-formattingbug.html

@@ -0,0 +1,18 @@
+<section class="html-edit">
+    <textarea class="tiny-mce">dummy</textarea>
+<!--
+    The text passed in is the escaped version of
+    &lt;problem>
+    &lt;p>&lt;/p>
+    &lt;multiplechoiceresponse>
+    <pre>&lt;problem>
+        &lt;p>&lt;/p></pre>
+    <div><foo>bar</foo></div>
+-->
+    <textarea name="" class="edit-box">&amp;lt;problem&gt;
+&amp;lt;p&gt;&amp;lt;/p&gt;
+&amp;lt;multiplechoiceresponse&gt;
+&lt;pre&gt;&amp;lt;problem&gt;
+    &amp;lt;p&gt;&amp;lt;/p&gt;</pre>
+<div><foo>bar</foo></div></textarea>
+</section>

common/lib/xmodule/xmodule/js/fixtures/html-edit.html

@@ -0,0 +1,10 @@
+<section class="html-edit">
+    <ul class="editor-tabs">
+        <li><a href="#" class="visual-tab tab current" data-tab="visual">Visual</a></li>
+        <li><a href="#" class="html-tab tab" data-tab="advanced">HTML</a></li>
+    </ul>
+    <div class="row">
+        <textarea class="tiny-mce">dummy text</textarea>
+        <textarea name="" class="edit-box">Advanced Editor Text</textarea>
+    </div>
+</section>

common/lib/xmodule/xmodule/js/fixtures/problem-with-markdown.html

@@ -0,0 +1,6 @@
+<section class="problem-editor editor">
+    <div class="row">
+        <textarea class="markdown-box">markdown</textarea>
+        <textarea class="xml-box" rows="8" cols="40">xml</textarea>
+    </div>
+</section>

common/lib/xmodule/xmodule/js/fixtures/problem-without-markdown.html

@@ -0,0 +1,5 @@
+<section class="problem-editor editor">
+    <div class="row">
+        <textarea class="xml-box" rows="8" cols="40">xml only</textarea>
+    </div>
+</section>

common/lib/xmodule/xmodule/js/fixtures/problem.html

@@ -1 +1,7 @@
-<section id="problem_1" class="problems-wrapper" data-url="/problem/url/"></section>
+<section class='xmodule_display xmodule_CapaModule' data-type='Problem'>
+  <section id='problem_1' 
+           class='problems-wrapper' 
+           data-problem-id='i4x://edX/101/problem/Problem1' 
+           data-url='/problem/Problem1'>
+  </section>
+</section>

common/lib/xmodule/xmodule/js/spec/.gitignore

@@ -0,0 +1,2 @@
+*.js
+

common/lib/xmodule/xmodule/js/spec/capa/display_spec.coffee

@@ -8,25 +8,43 @@ describe 'Problem', ->
     MathJax.Hub.getAllJax.andReturn [@stubbedJax]
     window.update_schematics = ->
 
+    # Load this function from spec/helper.coffee
+    # Note that if your test fails with a message like:
+    # 'External request attempted for blah, which is not defined.'
+    # this msg is coming from the stubRequests function else clause.
+    jasmine.stubRequests()
+
+    # note that the fixturesPath is set in spec/helper.coffee
     loadFixtures 'problem.html'
+ 
     spyOn Logger, 'log'
     spyOn($.fn, 'load').andCallFake (url, callback) ->
       $(@).html readFixtures('problem_content.html')
       callback()
-    jasmine.stubRequests()
 
   describe 'constructor', ->
-    beforeEach ->
-      @problem = new Problem 1, "problem_1", "/problem/url/"
 
-    it 'set the element', ->
-      expect(@problem.el).toBe '#problem_1'
+    it 'set the element from html', ->
+      @problem999 = new Problem ("
+        <section class='xmodule_display xmodule_CapaModule' data-type='Problem'>
+          <section id='problem_999' 
+                   class='problems-wrapper' 
+                   data-problem-id='i4x://edX/999/problem/Quiz' 
+                   data-url='/problem/quiz/'>
+          </section>
+        </section>
+        ")      
+      expect(@problem999.element_id).toBe 'problem_999'
+
+    it 'set the element from loadFixtures', ->
+      @problem1 = new Problem($('.xmodule_display'))
+      expect(@problem1.element_id).toBe 'problem_1'
 
   describe 'bind', ->
     beforeEach ->
       spyOn window, 'update_schematics'
       MathJax.Hub.getAllJax.andReturn [@stubbedJax]
-      @problem = new Problem 1, "problem_1", "/problem/url/"
+      @problem = new Problem($('.xmodule_display'))
 
     it 'set mathjax typeset', ->
       expect(MathJax.Hub.Queue).toHaveBeenCalled()
@@ -38,7 +56,7 @@ describe 'Problem', ->
       expect($('section.action input:button')).toHandleWith 'click', @problem.refreshAnswers
 
     it 'bind the check button', ->
-      expect($('section.action input.check')).toHandleWith 'click', @problem.check
+      expect($('section.action input.check')).toHandleWith 'click', @problem.check_fd
 
     it 'bind the reset button', ->
       expect($('section.action input.reset')).toHandleWith 'click', @problem.reset
@@ -52,7 +70,8 @@ describe 'Problem', ->
     it 'bind the math input', ->
       expect($('input.math')).toHandleWith 'keyup', @problem.refreshMath
 
-    it 'replace math content on the page', ->
+    # TODO: figure out why failing
+    xit 'replace math content on the page', ->
       expect(MathJax.Hub.Queue.mostRecentCall.args).toEqual [
         ['Text', @stubbedJax, ''],
         [@problem.updateMathML, @stubbedJax, $('#input_example_1').get(0)]
@@ -60,7 +79,7 @@ describe 'Problem', ->
 
   describe 'render', ->
     beforeEach ->
-      @problem = new Problem 1, "problem_1", "/problem/url/"
+      @problem = new Problem($('.xmodule_display'))
       @bind = @problem.bind
       spyOn @problem, 'bind'
 
@@ -86,9 +105,13 @@ describe 'Problem', ->
       it 're-bind the content', ->
         expect(@problem.bind).toHaveBeenCalled()
 
+  describe 'check_fd', ->
+    xit 'should have specs written for this functionality', ->
+      expect(false)
+
   describe 'check', ->
     beforeEach ->
-      @problem = new Problem 1, "problem_1", "/problem/url/"
+      @problem = new Problem($('.xmodule_display'))
       @problem.answers = 'foo=1&bar=2'
 
     it 'log the problem_check event', ->
@@ -98,30 +121,35 @@ describe 'Problem', ->
     it 'submit the answer for check', ->
       spyOn $, 'postWithPrefix'
       @problem.check()
-      expect($.postWithPrefix).toHaveBeenCalledWith '/modx/1/problem_check', 'foo=1&bar=2', jasmine.any(Function)
+      expect($.postWithPrefix).toHaveBeenCalledWith '/problem/Problem1/problem_check',
+          'foo=1&bar=2', jasmine.any(Function)
 
     describe 'when the response is correct', ->
       it 'call render with returned content', ->
-        spyOn($, 'postWithPrefix').andCallFake (url, answers, callback) -> callback(success: 'correct', contents: 'Correct!')
+        spyOn($, 'postWithPrefix').andCallFake (url, answers, callback) -> 
+          callback(success: 'correct', contents: 'Correct!')
         @problem.check()
         expect(@problem.el.html()).toEqual 'Correct!'
 
     describe 'when the response is incorrect', ->
       it 'call render with returned content', ->
-        spyOn($, 'postWithPrefix').andCallFake (url, answers, callback) -> callback(success: 'incorrect', contents: 'Correct!')
+        spyOn($, 'postWithPrefix').andCallFake (url, answers, callback) -> 
+          callback(success: 'incorrect', contents: 'Incorrect!')
         @problem.check()
-        expect(@problem.el.html()).toEqual 'Correct!'
+        expect(@problem.el.html()).toEqual 'Incorrect!'
 
-    describe 'when the response is undetermined', ->
+    # TODO: figure out why failing
+    xdescribe 'when the response is undetermined', ->
       it 'alert the response', ->
         spyOn window, 'alert'
-        spyOn($, 'postWithPrefix').andCallFake (url, answers, callback) -> callback(success: 'Number Only!')
+        spyOn($, 'postWithPrefix').andCallFake (url, answers, callback) ->
+          callback(success: 'Number Only!')
         @problem.check()
         expect(window.alert).toHaveBeenCalledWith 'Number Only!'
 
   describe 'reset', ->
     beforeEach ->
-      @problem = new Problem 1, "problem_1", "/problem/url/"
+      @problem = new Problem($('.xmodule_display'))
 
     it 'log the problem_reset event', ->
       @problem.answers = 'foo=1&bar=2'
@@ -131,7 +159,8 @@ describe 'Problem', ->
     it 'POST to the problem reset page', ->
       spyOn $, 'postWithPrefix'
       @problem.reset()
-      expect($.postWithPrefix).toHaveBeenCalledWith '/modx/1/problem_reset', { id: 1 }, jasmine.any(Function)
+      expect($.postWithPrefix).toHaveBeenCalledWith '/problem/Problem1/problem_reset', 
+          { id: 'i4x://edX/101/problem/Problem1' }, jasmine.any(Function)
 
     it 'render the returned content', ->
       spyOn($, 'postWithPrefix').andCallFake (url, answers, callback) ->
@@ -141,7 +170,7 @@ describe 'Problem', ->
 
   describe 'show', ->
     beforeEach ->
-      @problem = new Problem 1, "problem_1", "/problem/url/"
+      @problem = new Problem($('.xmodule_display'))
       @problem.el.prepend '<div id="answer_1_1" /><div id="answer_1_2" />'
 
     describe 'when the answer has not yet shown', ->
@@ -150,12 +179,14 @@ describe 'Problem', ->
 
       it 'log the problem_show event', ->
         @problem.show()
-        expect(Logger.log).toHaveBeenCalledWith 'problem_show', problem: 1
+        expect(Logger.log).toHaveBeenCalledWith 'problem_show', 
+            problem: 'i4x://edX/101/problem/Problem1'
 
       it 'fetch the answers', ->
         spyOn $, 'postWithPrefix'
         @problem.show()
-        expect($.postWithPrefix).toHaveBeenCalledWith '/modx/1/problem_show', jasmine.any(Function)
+        expect($.postWithPrefix).toHaveBeenCalledWith '/problem/Problem1/problem_show',
+            jasmine.any(Function)
 
       it 'show the answers', ->
         spyOn($, 'postWithPrefix').andCallFake (url, callback) ->
@@ -220,7 +251,7 @@ describe 'Problem', ->
 
   describe 'save', ->
     beforeEach ->
-      @problem = new Problem 1, "problem_1", "/problem/url/"
+      @problem = new Problem($('.xmodule_display'))
       @problem.answers = 'foo=1&bar=2'
 
     it 'log the problem_save event', ->
@@ -230,9 +261,11 @@ describe 'Problem', ->
     it 'POST to save problem', ->
       spyOn $, 'postWithPrefix'
       @problem.save()
-      expect($.postWithPrefix).toHaveBeenCalledWith '/modx/1/problem_save', 'foo=1&bar=2', jasmine.any(Function)
+      expect($.postWithPrefix).toHaveBeenCalledWith '/problem/Problem1/problem_save', 
+          'foo=1&bar=2', jasmine.any(Function)
 
-    it 'alert to the user', ->
+    # TODO: figure out why failing
+    xit 'alert to the user', ->
       spyOn window, 'alert'
       spyOn($, 'postWithPrefix').andCallFake (url, answers, callback) -> callback(success: 'OK')
       @problem.save()
@@ -240,7 +273,7 @@ describe 'Problem', ->
 
   describe 'refreshMath', ->
     beforeEach ->
-      @problem = new Problem 1, "problem_1", "/problem/url/"
+      @problem = new Problem($('.xmodule_display'))
       $('#input_example_1').val 'E=mc^2'
       @problem.refreshMath target: $('#input_example_1').get(0)
 
@@ -250,7 +283,7 @@ describe 'Problem', ->
 
   describe 'updateMathML', ->
     beforeEach ->
-      @problem = new Problem 1, "problem_1", "/problem/url/"
+      @problem = new Problem($('.xmodule_display'))
       @stubbedJax.root.toMathML.andReturn '<MathML>'
 
     describe 'when there is no exception', ->
@@ -270,7 +303,7 @@ describe 'Problem', ->
 
   describe 'refreshAnswers', ->
     beforeEach ->
-      @problem = new Problem 1, "problem_1", "/problem/url/"
+      @problem = new Problem($('.xmodule_display'))
       @problem.el.html '''
         <textarea class="CodeMirror" />
         <input id="input_1_1" name="input_1_1" class="schematic" value="one" />
@@ -290,6 +323,10 @@ describe 'Problem', ->
       @problem.refreshAnswers()
       expect(@stubCodeMirror.save).toHaveBeenCalled()
 
-    it 'serialize all answers', ->
+    # TODO: figure out why failing
+    xit 'serialize all answers', ->
       @problem.refreshAnswers()
       expect(@problem.answers).toEqual "input_1_1=one&input_1_2=two"
+
+
+

common/lib/xmodule/xmodule/js/spec/helper.coffee

@@ -0,0 +1,76 @@
+# Stub Youtube API
+window.YT =
+  PlayerState:
+    UNSTARTED: -1
+    ENDED: 0
+    PLAYING: 1
+    PAUSED: 2
+    BUFFERING: 3
+    CUED: 5
+
+jasmine.getFixtures().fixturesPath = 'xmodule/js/fixtures'
+
+jasmine.stubbedMetadata =
+  slowerSpeedYoutubeId:
+    id: 'slowerSpeedYoutubeId'
+    duration: 300
+  normalSpeedYoutubeId:
+    id: 'normalSpeedYoutubeId'
+    duration: 200
+  bogus:
+    duration: 100
+
+jasmine.stubbedCaption =
+  start: [0, 10000, 20000, 30000]
+  text: ['Caption at 0', 'Caption at 10000', 'Caption at 20000', 'Caption at 30000']
+
+jasmine.stubRequests = ->
+  spyOn($, 'ajax').andCallFake (settings) ->
+    if match = settings.url.match /youtube\.com\/.+\/videos\/(.+)\?v=2&alt=jsonc/
+      settings.success data: jasmine.stubbedMetadata[match[1]]
+    else if match = settings.url.match /static\/subs\/(.+)\.srt\.sjson/
+      settings.success jasmine.stubbedCaption
+    else if settings.url.match /.+\/problem_get$/
+      settings.success html: readFixtures('problem_content.html')
+    else if settings.url == '/calculate' ||
+      settings.url.match(/.+\/goto_position$/) ||
+      settings.url.match(/event$/) ||
+      settings.url.match(/.+\/problem_(check|reset|show|save)$/)
+      # do nothing
+    else
+      throw "External request attempted for #{settings.url}, which is not defined."
+
+jasmine.stubYoutubePlayer = ->
+  YT.Player = -> jasmine.createSpyObj 'YT.Player', ['cueVideoById', 'getVideoEmbedCode',
+    'getCurrentTime', 'getPlayerState', 'getVolume', 'setVolume', 'loadVideoById',
+    'playVideo', 'pauseVideo', 'seekTo']
+
+jasmine.stubVideoPlayer = (context, enableParts, createPlayer=true) ->
+  enableParts = [enableParts] unless $.isArray(enableParts)
+
+  suite = context.suite
+  currentPartName = suite.description while suite = suite.parentSuite
+  enableParts.push currentPartName
+
+  for part in ['VideoCaption', 'VideoSpeedControl', 'VideoVolumeControl', 'VideoProgressSlider']
+    unless $.inArray(part, enableParts) >= 0
+      spyOn window, part
+
+  loadFixtures 'video.html'
+  jasmine.stubRequests()
+  YT.Player = undefined
+  context.video = new Video 'example', '.75:slowerSpeedYoutubeId,1.0:normalSpeedYoutubeId'
+  jasmine.stubYoutubePlayer()
+  if createPlayer
+    return new VideoPlayer(video: context.video)
+
+spyOn(window, 'onunload')
+
+# Stub jQuery.cookie
+$.cookie = jasmine.createSpy('jQuery.cookie').andReturn '1.0'
+
+# Stub jQuery.qtip
+$.fn.qtip = jasmine.createSpy 'jQuery.qtip'
+
+# Stub jQuery.scrollTo
+$.fn.scrollTo = jasmine.createSpy 'jQuery.scrollTo'

common/lib/xmodule/xmodule/js/spec/html/edit_spec.coffee

@@ -0,0 +1,90 @@
+describe 'HTMLEditingDescriptor', ->
+  describe 'Read data from server, create Editor, and get data back out', ->
+    it 'Does not munge &lt', ->
+#     This is a test for Lighthouse #22,
+#     "html names are automatically converted to the symbols they describe"
+#     A better test would be a Selenium test to avoid duplicating the
+#     mako template structure in html-edit-formattingbug.html.
+#     However, we currently have no working Selenium tests.
+      loadFixtures 'html-edit-formattingbug.html'
+      @descriptor = new HTMLEditingDescriptor($('.html-edit'))
+      visualEditorStub =
+        isDirty: () -> false
+      spyOn(@descriptor, 'getVisualEditor').andCallFake () ->
+        visualEditorStub
+      data = @descriptor.save().data
+      expect(data).toEqual("""<problem>
+                           <p></p>
+                           <multiplechoiceresponse>
+                           <pre>&lt;problem>
+                               &lt;p>&lt;/p></pre>
+                           <div><foo>bar</foo></div>""")
+  describe 'Saves HTML', ->
+    beforeEach ->
+      loadFixtures 'html-edit.html'
+      @descriptor = new HTMLEditingDescriptor($('.html-edit'))
+    it 'Returns data from Advanced Editor if Visual Editor is not dirty', ->
+      visualEditorStub =
+        isDirty: () -> false
+      spyOn(@descriptor, 'getVisualEditor').andCallFake () ->
+        visualEditorStub
+      expect(@descriptor.showingVisualEditor).toEqual(true)
+      data = @descriptor.save().data
+      expect(data).toEqual('Advanced Editor Text')
+    it 'Returns data from Advanced Editor if Visual Editor is not showing (even if Visual Editor is dirty)', ->
+      visualEditorStub =
+        isDirty: () -> true
+      spyOn(@descriptor, 'getVisualEditor').andCallFake () ->
+        visualEditorStub
+      @descriptor.showingVisualEditor = false
+      data = @descriptor.save().data
+      expect(data).toEqual('Advanced Editor Text')
+    it 'Returns data from Visual Editor if Visual Editor is dirty and showing', ->
+      visualEditorStub =
+        isDirty: () -> true
+        getContent: () -> 'from visual editor'
+      spyOn(@descriptor, 'getVisualEditor').andCallFake () ->
+        visualEditorStub
+      expect(@descriptor.showingVisualEditor).toEqual(true)
+      data = @descriptor.save().data
+      expect(data).toEqual('from visual editor')
+  describe 'Can switch to Advanced Editor', ->
+    beforeEach ->
+      loadFixtures 'html-edit.html'
+      @descriptor = new HTMLEditingDescriptor($('.html-edit'))
+    it 'Populates from Visual Editor if Advanced Visual is dirty', ->
+      expect(@descriptor.showingVisualEditor).toEqual(true)
+      visualEditorStub =
+        isDirty: () -> true
+        getContent: () -> 'from visual editor'
+      @descriptor.showAdvancedEditor(visualEditorStub)
+      expect(@descriptor.showingVisualEditor).toEqual(false)
+      expect(@descriptor.advanced_editor.getValue()).toEqual('from visual editor')
+    it 'Does not populate from Visual Editor if Visual Editor is not dirty', ->
+      expect(@descriptor.showingVisualEditor).toEqual(true)
+      visualEditorStub =
+        isDirty: () -> false
+        getContent: () -> 'from visual editor'
+      @descriptor.showAdvancedEditor(visualEditorStub)
+      expect(@descriptor.showingVisualEditor).toEqual(false)
+      expect(@descriptor.advanced_editor.getValue()).toEqual('Advanced Editor Text')
+  describe 'Can switch to Visual Editor', ->
+    it 'Always populates from the Advanced Editor', ->
+      loadFixtures 'html-edit.html'
+      @descriptor = new HTMLEditingDescriptor($('.html-edit'))
+      @descriptor.showingVisualEditor = false
+
+      visualEditorStub =
+        isNotDirty: false
+        content: 'not set'
+        startContent: 'not set',
+        focus: () -> true
+        isDirty: () -> not @isNotDirty
+        setContent: (x) -> @content = x
+        getContent: -> @content
+
+      @descriptor.showVisualEditor(visualEditorStub)
+      expect(@descriptor.showingVisualEditor).toEqual(true)
+      expect(visualEditorStub.isDirty()).toEqual(false)
+      expect(visualEditorStub.getContent()).toEqual('Advanced Editor Text')
+      expect(visualEditorStub.startContent).toEqual('Advanced Editor Text')

common/lib/xmodule/xmodule/js/spec/problem/edit_spec.coffee

@@ -0,0 +1,372 @@
+describe 'MarkdownEditingDescriptor', ->
+  describe 'save stores the correct data', ->
+    it 'saves markdown from markdown editor', ->
+      loadFixtures 'problem-with-markdown.html'
+      @descriptor = new MarkdownEditingDescriptor($('.problem-editor'))
+      saveResult = @descriptor.save()
+      expect(saveResult.metadata.markdown).toEqual('markdown')
+      expect(saveResult.data).toEqual('<problem>\n<p>markdown</p>\n</problem>')
+    it 'clears markdown when xml editor is selected', ->
+      loadFixtures 'problem-with-markdown.html'
+      @descriptor = new MarkdownEditingDescriptor($('.problem-editor'))
+      @descriptor.createXMLEditor('replace with markdown')
+      saveResult = @descriptor.save()
+      expect(saveResult.metadata.markdown).toEqual(null)
+      expect(saveResult.data).toEqual('replace with markdown')
+    it 'saves xml from the xml editor', ->
+      loadFixtures 'problem-without-markdown.html'
+      @descriptor = new MarkdownEditingDescriptor($('.problem-editor'))
+      saveResult = @descriptor.save()
+      expect(saveResult.metadata.markdown).toEqual(null)
+      expect(saveResult.data).toEqual('xml only')
+
+  describe 'insertMultipleChoice', ->
+    it 'inserts the template if selection is empty', ->
+      revisedSelection = MarkdownEditingDescriptor.insertMultipleChoice('')
+      expect(revisedSelection).toEqual(MarkdownEditingDescriptor.multipleChoiceTemplate)
+    it 'wraps existing text', ->
+      revisedSelection = MarkdownEditingDescriptor.insertMultipleChoice('foo\nbar')
+      expect(revisedSelection).toEqual('( ) foo\n( ) bar\n')
+    it 'recognizes x as a selection if there is non-whitespace after x', ->
+      revisedSelection = MarkdownEditingDescriptor.insertMultipleChoice('a\nx b\nc\nx \nd\n x e')
+      expect(revisedSelection).toEqual('( ) a\n(x) b\n( ) c\n( ) x \n( ) d\n(x) e\n')
+    it 'recognizes x as a selection if it is first non whitespace and has whitespace with other non-whitespace', ->
+      revisedSelection = MarkdownEditingDescriptor.insertMultipleChoice(' x correct\n x \nex post facto\nb x c\nx c\nxxp')
+      expect(revisedSelection).toEqual('(x) correct\n( )  x \n( ) ex post facto\n( ) b x c\n(x) c\n( ) xxp\n')
+    it 'removes multiple newlines but not last one', ->
+      revisedSelection = MarkdownEditingDescriptor.insertMultipleChoice('a\nx b\n\n\nc\n')
+      expect(revisedSelection).toEqual('( ) a\n(x) b\n( ) c\n')
+
+  describe 'insertCheckboxChoice', ->
+    # Note, shares code with insertMultipleChoice. Therefore only doing smoke test.
+    it 'inserts the template if selection is empty', ->
+      revisedSelection = MarkdownEditingDescriptor.insertCheckboxChoice('')
+      expect(revisedSelection).toEqual(MarkdownEditingDescriptor.checkboxChoiceTemplate)
+    it 'wraps existing text', ->
+      revisedSelection = MarkdownEditingDescriptor.insertCheckboxChoice('foo\nbar')
+      expect(revisedSelection).toEqual('[ ] foo\n[ ] bar\n')
+
+  describe 'insertStringInput', ->
+    it 'inserts the template if selection is empty', ->
+      revisedSelection = MarkdownEditingDescriptor.insertStringInput('')
+      expect(revisedSelection).toEqual(MarkdownEditingDescriptor.stringInputTemplate)
+    it 'wraps existing text', ->
+      revisedSelection = MarkdownEditingDescriptor.insertStringInput('my text')
+      expect(revisedSelection).toEqual('= my text')
+
+  describe 'insertNumberInput', ->
+    it 'inserts the template if selection is empty', ->
+      revisedSelection = MarkdownEditingDescriptor.insertNumberInput('')
+      expect(revisedSelection).toEqual(MarkdownEditingDescriptor.numberInputTemplate)
+    it 'wraps existing text', ->
+      revisedSelection = MarkdownEditingDescriptor.insertNumberInput('my text')
+      expect(revisedSelection).toEqual('= my text')
+
+  describe 'insertSelect', ->
+    it 'inserts the template if selection is empty', ->
+      revisedSelection = MarkdownEditingDescriptor.insertSelect('')
+      expect(revisedSelection).toEqual(MarkdownEditingDescriptor.selectTemplate)
+    it 'wraps existing text', ->
+      revisedSelection = MarkdownEditingDescriptor.insertSelect('my text')
+      expect(revisedSelection).toEqual('[[my text]]')
+
+  describe 'insertHeader', ->
+    it 'inserts the template if selection is empty', ->
+      revisedSelection = MarkdownEditingDescriptor.insertHeader('')
+      expect(revisedSelection).toEqual(MarkdownEditingDescriptor.headerTemplate)
+    it 'wraps existing text', ->
+      revisedSelection = MarkdownEditingDescriptor.insertHeader('my text')
+      expect(revisedSelection).toEqual('my text\n====\n')
+
+  describe 'insertExplanation', ->
+    it 'inserts the template if selection is empty', ->
+      revisedSelection = MarkdownEditingDescriptor.insertExplanation('')
+      expect(revisedSelection).toEqual(MarkdownEditingDescriptor.explanationTemplate)
+    it 'wraps existing text', ->
+      revisedSelection = MarkdownEditingDescriptor.insertExplanation('my text')
+      expect(revisedSelection).toEqual('[explanation]\nmy text\n[explanation]')
+
+  describe 'markdownToXml', ->
+    it 'converts raw text to paragraph', ->
+      data = MarkdownEditingDescriptor.markdownToXml('foo')
+      expect(data).toEqual('<problem>\n<p>foo</p>\n</problem>')
+    # test default templates
+    it 'converts numerical response to xml', ->
+      data = MarkdownEditingDescriptor.markdownToXml("""A numerical response problem accepts a line of text input from the student, and evaluates the input for correctness based on its numerical value.
+
+        The answer is correct if it is within a specified numerical tolerance of the expected answer.
+
+        Enter the numerical value of Pi:
+        = 3.14159 +- .02
+
+        Enter the approximate value of 502*9:
+        = 4518 +- 15%
+
+        Enter the number of fingers on a human hand:
+        = 5
+        
+        [Explanation]
+        Pi, or the the ratio between a circle's circumference to its diameter, is an irrational number known to extreme precision. It is value is approximately equal to 3.14.
+        
+        Although you can get an exact value by typing 502*9 into a calculator, the result will be close to 500*10, or 5,000. The grader accepts any response within 15% of the true value, 4518, so that you can use any estimation technique that you like.
+        
+        If you look at your hand, you can count that you have five fingers.
+        [Explanation]
+        """)
+      expect(data).toEqual("""<problem>
+        <p>A numerical response problem accepts a line of text input from the student, and evaluates the input for correctness based on its numerical value.</p>
+        
+        <p>The answer is correct if it is within a specified numerical tolerance of the expected answer.</p>
+        
+        <p>Enter the numerical value of Pi:</p>
+        <numericalresponse answer="3.14159 ">
+          <responseparam type="tolerance" default=".02" />
+          <textline />
+        </numericalresponse>
+        
+        <p>Enter the approximate value of 502*9:</p>
+        <numericalresponse answer="4518 ">
+          <responseparam type="tolerance" default="15%" />
+          <textline />
+        </numericalresponse>
+        
+        <p>Enter the number of fingers on a human hand:</p>
+        <numericalresponse answer="5">
+          <textline />
+        </numericalresponse>
+        
+        <solution>
+        <div class="detailed-solution">
+        <p>Explanation</p>
+        
+        <p>Pi, or the the ratio between a circle's circumference to its diameter, is an irrational number known to extreme precision. It is value is approximately equal to 3.14.</p>
+        
+        <p>Although you can get an exact value by typing 502*9 into a calculator, the result will be close to 500*10, or 5,000. The grader accepts any response within 15% of the true value, 4518, so that you can use any estimation technique that you like.</p>
+        
+        <p>If you look at your hand, you can count that you have five fingers.</p>
+
+        </div>
+        </solution>
+        </problem>""")
+    it 'converts multiple choice to xml', ->
+      data = MarkdownEditingDescriptor.markdownToXml("""A multiple choice problem presents radio buttons for student input. Students can only select a single option presented. Multiple Choice questions have been the subject of many areas of research due to the early invention and adoption of bubble sheets.
+        
+        One of the main elements that goes into a good multiple choice question is the existence of good distractors. That is, each of the alternate responses presented to the student should be the result of a plausible mistake that a student might make.
+        
+        What Apple device competed with the portable CD player?
+        ( ) The iPad
+        ( ) Napster
+        (x) The iPod
+        ( ) The vegetable peeler
+        ( ) Android
+        ( ) The Beatles
+        
+        [Explanation]
+        The release of the iPod allowed consumers to carry their entire music library with them in a format that did not rely on fragile and energy-intensive spinning disks.
+        [Explanation]
+        """)
+      expect(data).toEqual("""<problem>
+        <p>A multiple choice problem presents radio buttons for student input. Students can only select a single option presented. Multiple Choice questions have been the subject of many areas of research due to the early invention and adoption of bubble sheets.</p>
+        
+        <p>One of the main elements that goes into a good multiple choice question is the existence of good distractors. That is, each of the alternate responses presented to the student should be the result of a plausible mistake that a student might make.</p>
+        
+        <p>What Apple device competed with the portable CD player?</p>
+        <multiplechoiceresponse>
+          <choicegroup type="MultipleChoice">
+            <choice correct="false">The iPad</choice>
+            <choice correct="false">Napster</choice>
+            <choice correct="true">The iPod</choice>
+            <choice correct="false">The vegetable peeler</choice>
+            <choice correct="false">Android</choice>
+            <choice correct="false">The Beatles</choice>
+          </choicegroup>
+        </multiplechoiceresponse>
+        
+        <solution>
+        <div class="detailed-solution">
+        <p>Explanation</p>
+        
+        <p>The release of the iPod allowed consumers to carry their entire music library with them in a format that did not rely on fragile and energy-intensive spinning disks.</p>
+
+        </div>
+        </solution>
+        </problem>""")        
+    it 'converts OptionResponse to xml', ->
+      data = MarkdownEditingDescriptor.markdownToXml("""OptionResponse gives a limited set of options for students to respond with, and presents those options in a format that encourages them to search for a specific answer rather than being immediately presented with options from which to recognize the correct answer.
+        
+        The answer options and the identification of the correct answer is defined in the <b>optioninput</b> tag.
+        
+        Translation between Option Response and __________ is extremely straightforward:
+        [[(Multiple Choice), String Response, Numerical Response, External Response, Image Response]]
+        
+        [Explanation]
+        Multiple Choice also allows students to select from a variety of pre-written responses, although the format makes it easier for students to read very long response options. Optionresponse also differs slightly because students are more likely to think of an answer and then search for it rather than relying purely on recognition to answer the question.
+        [Explanation]
+        """)
+      expect(data).toEqual("""<problem>
+        <p>OptionResponse gives a limited set of options for students to respond with, and presents those options in a format that encourages them to search for a specific answer rather than being immediately presented with options from which to recognize the correct answer.</p>
+        
+        <p>The answer options and the identification of the correct answer is defined in the <b>optioninput</b> tag.</p>
+        
+        <p>Translation between Option Response and __________ is extremely straightforward:</p>
+        
+        <optionresponse>
+          <optioninput options="('Multiple Choice','String Response','Numerical Response','External Response','Image Response')" correct="Multiple Choice"></optioninput>
+        </optionresponse>
+        
+        <solution>
+        <div class="detailed-solution">
+        <p>Explanation</p>
+        
+        <p>Multiple Choice also allows students to select from a variety of pre-written responses, although the format makes it easier for students to read very long response options. Optionresponse also differs slightly because students are more likely to think of an answer and then search for it rather than relying purely on recognition to answer the question.</p>
+
+        </div>
+        </solution>
+        </problem>""")        
+    it 'converts StringResponse to xml', ->
+      data = MarkdownEditingDescriptor.markdownToXml("""A string response problem accepts a line of text input from the student, and evaluates the input for correctness based on an expected answer within each input box.
+        
+        The answer is correct if it matches every character of the expected answer. This can be a problem with international spelling, dates, or anything where the format of the answer is not clear.
+        
+        Which US state has Lansing as its capital?
+        = Michigan
+        
+        [Explanation]
+        Lansing is the capital of Michigan, although it is not Michgan's largest city, or even the seat of the county in which it resides.
+        [Explanation]
+        """)
+      expect(data).toEqual("""<problem>
+        <p>A string response problem accepts a line of text input from the student, and evaluates the input for correctness based on an expected answer within each input box.</p>
+        
+        <p>The answer is correct if it matches every character of the expected answer. This can be a problem with international spelling, dates, or anything where the format of the answer is not clear.</p>
+        
+        <p>Which US state has Lansing as its capital?</p>
+        <stringresponse answer="Michigan" type="ci">
+          <textline size="20"/>
+        </stringresponse>
+        
+        <solution>
+        <div class="detailed-solution">
+        <p>Explanation</p>
+        
+        <p>Lansing is the capital of Michigan, although it is not Michgan's largest city, or even the seat of the county in which it resides.</p>
+
+        </div>
+        </solution>
+        </problem>""")
+    # test oddities
+    it 'converts headers and oddities to xml', ->
+      data = MarkdownEditingDescriptor.markdownToXml("""Not a header
+        A header
+        ==============
+        
+        Multiple choice w/ parentheticals
+        ( ) option (with parens)
+        ( ) xd option (x)
+        ()) parentheses inside
+        () no space b4 close paren
+        
+        Choice checks
+        [ ] option1 [x]
+        [x] correct
+        [x] redundant
+        [(] distractor
+        [] no space
+        
+        Option with multiple correct ones
+        [[one option, (correct one), (should not be correct)]]
+        
+        Option with embedded parens
+        [[My (heart), another, (correct)]]
+        
+        What happens w/ empty correct options?
+        [[()]]
+
+        [Explanation]see[/expLanation]
+
+        [explanation]
+        orphaned start
+        
+        No p tags in the below
+        <script type='javascript'>
+           var two = 2;
+
+           console.log(two * 2);
+        </script>
+        
+        But in this there should be
+        <div>
+        Great ideas require offsetting.
+        
+        bad tests require drivel
+        </div>
+        """)
+      expect(data).toEqual("""<problem>
+        <p>Not a header</p>
+        <h1>A header</h1>
+        
+        <p>Multiple choice w/ parentheticals</p>
+        <multiplechoiceresponse>
+          <choicegroup type="MultipleChoice">
+            <choice correct="false">option (with parens)</choice>
+            <choice correct="false">xd option (x)</choice>
+            <choice correct="false">parentheses inside</choice>
+            <choice correct="false">no space b4 close paren</choice>
+          </choicegroup>
+        </multiplechoiceresponse>
+        
+        <p>Choice checks</p>
+        <choiceresponse>
+          <checkboxgroup direction="vertical">
+            <choice correct="false">option1 [x]</choice>
+            <choice correct="true">correct</choice>
+            <choice correct="true">redundant</choice>
+            <choice correct="false">distractor</choice>
+            <choice correct="false">no space</choice>
+          </checkboxgroup>
+        </choiceresponse>
+        
+        <p>Option with multiple correct ones</p>
+        
+        <optionresponse>
+          <optioninput options="('one option','correct one','should not be correct')" correct="correct one"></optioninput>
+        </optionresponse>
+        
+        <p>Option with embedded parens</p>
+        
+        <optionresponse>
+          <optioninput options="('My (heart)','another','correct')" correct="correct"></optioninput>
+        </optionresponse>
+        
+        <p>What happens w/ empty correct options?</p>
+        
+        <optionresponse>
+          <optioninput options="('')" correct=""></optioninput>
+        </optionresponse>
+        
+        <solution>
+        <div class="detailed-solution">
+        <p>Explanation</p>
+
+        <p>see</p>
+        </div>
+        </solution>
+
+        <p>[explanation]</p>
+        <p>orphaned start</p>
+
+        <p>No p tags in the below</p>
+        <script type='javascript'>
+           var two = 2;
+
+           console.log(two * 2);
+        </script>
+        
+        <p>But in this there should be</p>
+        <div>
+        <p>Great ideas require offsetting.</p>
+        
+        <p>bad tests require drivel</p>
+        </div>
+        </problem>""")
+    # failure tests

common/lib/xmodule/xmodule/js/spec/sequence/display_spec.coffee

@@ -1,4 +1,5 @@
-describe 'Sequence', ->
+# TODO: figure out why failing
+xdescribe 'Sequence', ->
   beforeEach ->
     # Stub MathJax
     window.MathJax = { Hub: { Queue: -> } }

common/lib/xmodule/xmodule/js/spec/video/display/video_caption_spec.coffee

@@ -1,4 +1,5 @@
-describe 'VideoCaption', ->
+# TODO: figure out why failing
+xdescribe 'VideoCaption', ->
   beforeEach ->
     jasmine.stubVideoPlayer @
     $('.subtitles').remove()

common/lib/xmodule/xmodule/js/spec/video/display/video_control_spec.coffee

@@ -1,4 +1,5 @@
-describe 'VideoControl', ->
+# TODO: figure out why failing
+xdescribe 'VideoControl', ->
   beforeEach ->
     jasmine.stubVideoPlayer @
     $('.video-controls').html ''

common/lib/xmodule/xmodule/js/spec/video/display/video_player_spec.coffee

@@ -1,4 +1,5 @@
-describe 'VideoPlayer', ->
+# TODO: figure out why failing
+xdescribe 'VideoPlayer', ->
   beforeEach ->
     jasmine.stubVideoPlayer @, [], false
 

common/lib/xmodule/xmodule/js/spec/video/display/video_progress_slider_spec.coffee

@@ -1,4 +1,5 @@
-describe 'VideoProgressSlider', ->
+# TODO: figure out why failing
+xdescribe 'VideoProgressSlider', ->
   beforeEach ->
     jasmine.stubVideoPlayer @
 

common/lib/xmodule/xmodule/js/spec/video/display/video_speed_control_spec.coffee

@@ -1,4 +1,5 @@
-describe 'VideoSpeedControl', ->
+# TODO: figure out why failing
+xdescribe 'VideoSpeedControl', ->
   beforeEach ->
     jasmine.stubVideoPlayer @
     $('.speeds').remove()

common/lib/xmodule/xmodule/js/spec/video/display/video_volume_control_spec.coffee

@@ -1,4 +1,5 @@
-describe 'VideoVolumeControl', ->
+# TODO: figure out why failing
+xdescribe 'VideoVolumeControl', ->
   beforeEach ->
     jasmine.stubVideoPlayer @
     $('.volume').remove()

common/lib/xmodule/xmodule/js/spec/video/display_spec.coffee

@@ -1,4 +1,5 @@
-describe 'Video', ->
+# TODO: figure out why failing
+xdescribe 'Video', ->
   beforeEach ->
     loadFixtures 'video.html'
     jasmine.stubRequests()

common/lib/xmodule/xmodule/js/src/.gitignore

@@ -0,0 +1,2 @@
+*.js
+

common/lib/xmodule/xmodule/js/src/discussion/display.coffee

@@ -1,4 +1,4 @@
-class @InlineDiscussion
+class @InlineDiscussion extends XModule.Descriptor
   constructor: (element) ->
     @el = $(element).find('.discussion-module')
     @view = new DiscussionModuleView(el: @el)

common/lib/xmodule/xmodule/js/src/html/display.coffee

@@ -4,6 +4,7 @@ class @HTMLModule
     @el = $(@element)
     JavascriptLoader.executeModuleScripts(@el)
     Collapsible.setCollapsibles(@el)
+    MathJax.Hub.Queue ["Typeset", MathJax.Hub, @el[0]]
 
   $: (selector) ->
     $(selector, @el)

common/lib/xmodule/xmodule/js/src/html/edit.coffee

@@ -1,7 +1,92 @@
 class @HTMLEditingDescriptor
-  constructor: (@element) ->
-    @edit_box = CodeMirror.fromTextArea($(".edit-box", @element)[0], {
+  @isInactiveClass : "is-inactive"
+
+  constructor: (element) ->
+    @element = element;
+
+    @advanced_editor = CodeMirror.fromTextArea($(".edit-box", @element)[0], {
       mode: "text/html"
+      lineNumbers: true
+      lineWrapping: true
     })
 
-  save: -> @edit_box.getValue()
+    $(@advanced_editor.getWrapperElement()).addClass(HTMLEditingDescriptor.isInactiveClass)
+
+#   This is a workaround for the fact that tinyMCE's baseURL property is not getting correctly set on AWS
+#   instances (like sandbox). It is not necessary to explicitly set baseURL when running locally.
+    tinyMCE.baseURL = '/static/js/vendor/tiny_mce'
+    @tiny_mce_textarea = $(".tiny-mce", @element).tinymce({
+      script_url : '/static/js/vendor/tiny_mce/tiny_mce.js',
+      theme : "advanced",
+      skin: 'studio',
+      schema: "html5",
+      # TODO: we should share this CSS with studio (and LMS)
+      content_css : "/static/css/tiny-mce.css",
+      # We may want to add "styleselect" when we collect all styles used throughout the LMS
+      theme_advanced_buttons1 : "formatselect,bold,italic,underline,bullist,numlist,outdent,indent,blockquote,link,unlink",
+      theme_advanced_toolbar_location : "top",
+      theme_advanced_toolbar_align : "left",
+      theme_advanced_statusbar_location : "none",
+      theme_advanced_resizing : true,
+      theme_advanced_blockformats : "p,code,h2,h3,blockquote",
+      width: '100%',
+      height: '400px',
+      # Cannot get access to tinyMCE Editor instance (for focusing) until after it is rendered.
+      # The tinyMCE callback passes in the editor as a paramter.
+      init_instance_callback: @focusVisualEditor
+    })
+
+    @showingVisualEditor = true
+    @element.on('click', '.editor-tabs .tab', @onSwitchEditor)
+
+  onSwitchEditor: (e)=>
+    e.preventDefault();
+
+    if not $(e.currentTarget).hasClass('current')
+      $('.editor-tabs .current').removeClass('current')
+      $(e.currentTarget).addClass('current')
+      $('table.mceToolbar').toggleClass(HTMLEditingDescriptor.isInactiveClass)
+      $(@advanced_editor.getWrapperElement()).toggleClass(HTMLEditingDescriptor.isInactiveClass)
+
+      visualEditor = @getVisualEditor()
+      if $(e.currentTarget).attr('data-tab') is 'visual'
+        @showVisualEditor(visualEditor)
+      else
+        @showAdvancedEditor(visualEditor)
+
+  # Show the Advanced (codemirror) Editor. Pulled out as a helper method for unit testing.
+  showAdvancedEditor: (visualEditor) ->
+    if visualEditor.isDirty()
+      @advanced_editor.setValue(visualEditor.getContent({no_events: 1}))
+      @advanced_editor.setCursor(0)
+    @advanced_editor.refresh()
+    @advanced_editor.focus()
+    @showingVisualEditor = false
+
+  # Show the Visual (tinyMCE) Editor. Pulled out as a helper method for unit testing.
+  showVisualEditor: (visualEditor) ->
+    visualEditor.setContent(@advanced_editor.getValue())
+    # In order for isDirty() to return true ONLY if edits have been made after setting the text,
+    # both the startContent must be sync'ed up and the dirty flag set to false.
+    visualEditor.startContent = visualEditor.getContent({format: "raw", no_events: 1});
+    visualEditor.isNotDirty = true
+    @focusVisualEditor(visualEditor)
+    @showingVisualEditor = true
+
+  focusVisualEditor: (visualEditor) ->
+    visualEditor.focus()
+
+  getVisualEditor: ->
+    ###
+    Returns the instance of TinyMCE.
+    This is different from the textarea that exists in the HTML template (@tiny_mce_textarea.
+    ###
+    return tinyMCE.get($('.tiny-mce', this.element).attr('id'))
+
+  save: ->
+    @element.off('click', '.editor-tabs .tab', @onSwitchEditor)
+    text = @advanced_editor.getValue()
+    visualEditor = @getVisualEditor()
+    if @showingVisualEditor and visualEditor.isDirty()
+      text = visualEditor.getContent({no_events: 1})
+    data: text

common/lib/xmodule/xmodule/js/src/problem/edit.coffee

@@ -0,0 +1,298 @@
+class @MarkdownEditingDescriptor extends XModule.Descriptor
+  # TODO really, these templates should come from or also feed the cheatsheet
+  @multipleChoiceTemplate : "( ) incorrect\n( ) incorrect\n(x) correct\n"
+  @checkboxChoiceTemplate: "[x] correct\n[ ] incorrect\n[x] correct\n"
+  @stringInputTemplate: "= answer\n"
+  @numberInputTemplate: "= answer +- x%\n"
+  @selectTemplate: "[[incorrect, (correct), incorrect]]\n"
+  @headerTemplate: "Header\n=====\n"
+  @explanationTemplate: "[explanation]\nShort explanation\n[explanation]\n"
+
+  constructor: (element) ->
+    @element = element
+
+    if $(".markdown-box", @element).length != 0
+      @markdown_editor = CodeMirror.fromTextArea($(".markdown-box", element)[0], {
+      lineWrapping: true
+      mode: null
+      })
+      @setCurrentEditor(@markdown_editor)
+      # Add listeners for toolbar buttons (only present for markdown editor)
+      @element.on('click', '.xml-tab', @onShowXMLButton)
+      @element.on('click', '.format-buttons a', @onToolbarButton)
+      @element.on('click', '.cheatsheet-toggle', @toggleCheatsheet)
+      # Hide the XML text area
+      $(@element.find('.xml-box')).hide()
+    else
+      @createXMLEditor()
+
+  ###
+  Creates the XML Editor and sets it as the current editor. If text is passed in,
+  it will replace the text present in the HTML template.
+
+  text: optional argument to override the text passed in via the HTML template
+  ###
+  createXMLEditor: (text) ->
+    @xml_editor = CodeMirror.fromTextArea($(".xml-box", @element)[0], {
+    mode: "xml"
+    lineNumbers: true
+    lineWrapping: true
+    })
+    if text
+      @xml_editor.setValue(text)
+    @setCurrentEditor(@xml_editor)
+
+  ###
+  User has clicked to show the XML editor. Before XML editor is swapped in,
+  the user will need to confirm the one-way conversion.
+  ###
+  onShowXMLButton: (e) =>
+    e.preventDefault();
+    if @confirmConversionToXml()
+      @createXMLEditor(MarkdownEditingDescriptor.markdownToXml(@markdown_editor.getValue()))
+      # Need to refresh to get line numbers to display properly (and put cursor position to 0)
+      @xml_editor.setCursor(0)
+      @xml_editor.refresh()
+      # Hide markdown-specific toolbar buttons
+      $(@element.find('.editor-bar')).hide()
+
+  ###
+  Have the user confirm the one-way conversion to XML.
+  Returns true if the user clicked OK, else false.
+  ###
+  confirmConversionToXml: ->
+    # TODO: use something besides a JavaScript confirm dialog?
+    return confirm("If you use the Advanced Editor, this problem will be converted to XML and you will not be able to return to the Simple Editor Interface.\n\nProceed to the Advanced Editor and convert this problem to XML?")
+
+  ###
+  Event listener for toolbar buttons (only possible when markdown editor is visible).
+  ###
+  onToolbarButton: (e) =>
+    e.preventDefault();
+    selection = @markdown_editor.getSelection()
+    revisedSelection = null
+    switch $(e.currentTarget).attr('class')
+      when "multiple-choice-button" then revisedSelection = MarkdownEditingDescriptor.insertMultipleChoice(selection)
+      when "string-button" then revisedSelection = MarkdownEditingDescriptor.insertStringInput(selection)
+      when "number-button" then revisedSelection = MarkdownEditingDescriptor.insertNumberInput(selection)
+      when "checks-button" then revisedSelection = MarkdownEditingDescriptor.insertCheckboxChoice(selection)
+      when "dropdown-button" then revisedSelection = MarkdownEditingDescriptor.insertSelect(selection)
+      when "header-button" then revisedSelection = MarkdownEditingDescriptor.insertHeader(selection)
+      when "explanation-button" then revisedSelection = MarkdownEditingDescriptor.insertExplanation(selection)
+      else # ignore click
+
+    if revisedSelection != null
+      @markdown_editor.replaceSelection(revisedSelection)
+      @markdown_editor.focus()
+
+  ###
+  Event listener for toggling cheatsheet (only possible when markdown editor is visible).
+  ###
+  toggleCheatsheet: (e) =>
+    e.preventDefault();
+    if !$(@markdown_editor.getWrapperElement()).find('.simple-editor-cheatsheet')[0]
+      @cheatsheet = $($('#simple-editor-cheatsheet').html())
+      $(@markdown_editor.getWrapperElement()).append(@cheatsheet)
+
+    setTimeout (=> @cheatsheet.toggleClass('shown')), 10
+
+  ###
+  Stores the current editor and hides the one that is not displayed.
+  ###
+  setCurrentEditor: (editor) ->
+    if @current_editor
+      $(@current_editor.getWrapperElement()).hide()
+    @current_editor = editor
+    $(@current_editor.getWrapperElement()).show()
+    $(@current_editor).focus();
+
+  ###
+  Called when save is called. Listeners are unregistered because editing the block again will
+  result in a new instance of the descriptor. Note that this is NOT the case for cancel--
+  when cancel is called the instance of the descriptor is reused if edit is selected again.
+  ###
+  save: ->
+    @element.off('click', '.xml-tab', @changeEditor)
+    @element.off('click', '.format-buttons a', @onToolbarButton)
+    @element.off('click', '.cheatsheet-toggle', @toggleCheatsheet)
+    if @current_editor == @markdown_editor
+        {
+            data: MarkdownEditingDescriptor.markdownToXml(@markdown_editor.getValue())
+            metadata:
+            	markdown: @markdown_editor.getValue()
+        }
+    else
+       {
+            data: @xml_editor.getValue()
+            metadata:
+               markdown: null
+       }
+
+  @insertMultipleChoice: (selectedText) ->
+    return MarkdownEditingDescriptor.insertGenericChoice(selectedText, '(', ')', MarkdownEditingDescriptor.multipleChoiceTemplate)
+
+  @insertCheckboxChoice: (selectedText) ->
+    return MarkdownEditingDescriptor.insertGenericChoice(selectedText, '[', ']', MarkdownEditingDescriptor.checkboxChoiceTemplate)
+
+  @insertGenericChoice: (selectedText, choiceStart, choiceEnd, template) ->
+    if selectedText.length > 0
+      # Replace adjacent newlines with a single newline, strip any trailing newline
+      cleanSelectedText = selectedText.replace(/\n+/g, '\n').replace(/\n$/,'')
+      lines =  cleanSelectedText.split('\n')
+      revisedLines = ''
+      for line in lines
+        revisedLines += choiceStart
+        # a stand alone x before other text implies that this option is "correct"
+        if /^\s*x\s+(\S)/i.test(line)
+          # Remove the x and any initial whitespace as long as there's more text on the line
+          line = line.replace(/^\s*x\s+(\S)/i, '$1')
+          revisedLines += 'x'
+        else
+          revisedLines += ' '
+        revisedLines += choiceEnd + ' ' + line + '\n'
+      return revisedLines
+    else
+      return template
+
+  @insertStringInput: (selectedText) ->
+    return MarkdownEditingDescriptor.insertGenericInput(selectedText, '= ', '', MarkdownEditingDescriptor.stringInputTemplate)
+
+  @insertNumberInput: (selectedText) ->
+    return MarkdownEditingDescriptor.insertGenericInput(selectedText, '= ', '', MarkdownEditingDescriptor.numberInputTemplate)
+
+  @insertSelect: (selectedText) ->
+    return MarkdownEditingDescriptor.insertGenericInput(selectedText, '[[', ']]', MarkdownEditingDescriptor.selectTemplate)
+
+  @insertHeader: (selectedText) ->
+    return MarkdownEditingDescriptor.insertGenericInput(selectedText, '', '\n====\n', MarkdownEditingDescriptor.headerTemplate)
+
+  @insertExplanation: (selectedText) ->
+    return MarkdownEditingDescriptor.insertGenericInput(selectedText, '[explanation]\n', '\n[explanation]', MarkdownEditingDescriptor.explanationTemplate)
+
+  @insertGenericInput: (selectedText, lineStart, lineEnd, template) ->
+    if selectedText.length > 0
+      # TODO: should this insert a newline afterwards?
+      return lineStart + selectedText + lineEnd
+    else
+      return template
+
+# We may wish to add insertHeader. Here is Tom's code.
+# function makeHeader() {
+#  var selection = simpleEditor.getSelection();
+#  var revisedSelection = selection + '\n';
+#  for(var i = 0; i < selection.length; i++) {
+#revisedSelection += '=';
+#  }
+#  simpleEditor.replaceSelection(revisedSelection);
+#}
+#
+  @markdownToXml: (markdown)->
+    toXml = `function(markdown) {
+      var xml = markdown;
+
+      // replace headers
+      xml = xml.replace(/(^.*?$)(?=\n\=\=+$)/gm, '<h1>$1</h1>');
+      xml = xml.replace(/\n^\=\=+$/gm, '');
+
+      // group multiple choice answers
+      xml = xml.replace(/(^\s*\(.?\).*?$\n*)+/gm, function(match, p) {
+        var groupString = '<multiplechoiceresponse>\n';
+        groupString += '  <choicegroup type="MultipleChoice">\n';
+        var options = match.split('\n');
+        for(var i = 0; i < options.length; i++) {
+          if(options[i].length > 0) {
+            var value = options[i].split(/^\s*\(.?\)\s*/)[1];
+            var correct = /^\s*\(x\)/i.test(options[i]);
+            groupString += '    <choice correct="' + correct + '">' + value + '</choice>\n';
+          }
+        }
+        groupString += '  </choicegroup>\n';
+        groupString += '</multiplechoiceresponse>\n\n';
+        return groupString;
+      });
+
+      // group check answers
+      xml = xml.replace(/(^\s*\[.?\].*?$\n*)+/gm, function(match, p) {
+        var groupString = '<choiceresponse>\n';
+        groupString += '  <checkboxgroup direction="vertical">\n';
+        var options = match.split('\n');
+        for(var i = 0; i < options.length; i++) {
+          if(options[i].length > 0) {
+            var value = options[i].split(/^\s*\[.?\]\s*/)[1];
+            var correct = /^\s*\[x\]/i.test(options[i]);
+            groupString += '    <choice correct="' + correct + '">' + value + '</choice>\n';
+          }
+        }
+        groupString += '  </checkboxgroup>\n';
+        groupString += '</choiceresponse>\n\n';
+        return groupString;
+      });
+
+      // replace string and numerical
+      xml = xml.replace(/^\=\s*(.*?$)/gm, function(match, p) {
+        var string;
+        var params = /(.*?)\+\-\s*(.*?$)/.exec(p);
+        if(parseFloat(p)) {
+          if(params) {
+            string = '<numericalresponse answer="' + params[1] + '">\n';
+            string += '  <responseparam type="tolerance" default="' + params[2] + '" />\n';
+          } else {
+            string = '<numericalresponse answer="' + p + '">\n';
+          }
+          string += '  <textline />\n';
+          string += '</numericalresponse>\n\n';
+        } else {
+          string = '<stringresponse answer="' + p + '" type="ci">\n  <textline size="20"/>\n</stringresponse>\n\n';
+        }
+        return string;
+      });
+
+      // replace selects
+      xml = xml.replace(/\[\[(.+?)\]\]/g, function(match, p) {
+        var selectString = '\n<optionresponse>\n';
+        selectString += '  <optioninput options="(';
+        var options = p.split(/\,\s*/g);
+        for(var i = 0; i < options.length; i++) {
+          selectString += "'" + options[i].replace(/(?:^|,)\s*\((.*?)\)\s*(?:$|,)/g, '$1') + "'" + (i < options.length -1 ? ',' : '');
+        }
+        selectString += ')" correct="';
+        var correct = /(?:^|,)\s*\((.*?)\)\s*(?:$|,)/g.exec(p);
+        if (correct) selectString += correct[1];
+        selectString += '"></optioninput>\n';
+        selectString += '</optionresponse>\n\n';
+        return selectString;
+      });
+      
+      // replace explanations
+      xml = xml.replace(/\[explanation\]\n?([^\]]*)\[\/?explanation\]/gmi, function(match, p1) {
+          var selectString = '<solution>\n<div class="detailed-solution">\nExplanation\n\n' + p1 + '\n</div>\n</solution>';
+          return selectString;
+      });
+
+      // split scripts and wrap paragraphs
+      var splits = xml.split(/(\<\/?script.*?\>)/g);
+      var scriptFlag = false;
+      for(var i = 0; i < splits.length; i++) {
+        if(/\<script/.test(splits[i])) {
+          scriptFlag = true;
+        }
+        if(!scriptFlag) {
+          splits[i] = splits[i].replace(/(^(?!\s*\<|$).*$)/gm, '<p>$1</p>');
+        }
+        if(/\<\/script/.test(splits[i])) {
+          scriptFlag = false;
+        }
+      }
+      xml = splits.join('');
+
+      // rid white space
+      xml = xml.replace(/\n\n\n/g, '\n');
+      
+      // surround w/ problem tag
+      xml = '<problem>\n' + xml + '\n</problem>';
+
+      return xml;
+    }
+    `
+    return toXml markdown
+

common/lib/xmodule/xmodule/js/src/raw/edit/json.coffee

@@ -1,7 +1,10 @@
-class @JSONEditingDescriptor
+class @JSONEditingDescriptor extends XModule.Descriptor
   constructor: (@element) ->
     @edit_box = CodeMirror.fromTextArea($(".edit-box", @element)[0], {
       mode: { name: "javascript", json: true }
+      lineNumbers: true
+      lineWrapping: true
     })
 
-  save: -> JSON.parse @edit_box.getValue()
+  save: ->
+    data: JSON.parse @edit_box.getValue()

common/lib/xmodule/xmodule/js/src/raw/edit/xml.coffee

@@ -1,7 +1,10 @@
-class @XMLEditingDescriptor
+class @XMLEditingDescriptor extends XModule.Descriptor
   constructor: (@element) ->
     @edit_box = CodeMirror.fromTextArea($(".edit-box", @element)[0], {
       mode: "xml"
+      lineNumbers: true
+      lineWrapping: true
     })
 
-  save: -> @edit_box.getValue()
+  save: ->
+    data: @edit_box.getValue()

common/lib/xmodule/xmodule/js/src/sequence/display.coffee

@@ -84,7 +84,7 @@ class @Sequence
 
       @mark_active new_position
       @$('#seq_content').html @contents.eq(new_position - 1).text()
-      XModule.loadModules('display', @$('#seq_content'))
+      XModule.loadModules(@$('#seq_content'))
 
       MathJax.Hub.Queue(["Typeset", MathJax.Hub, "seq_content"]) # NOTE: Actually redundant. Some other MathJax call also being performed
       window.update_schematics() # For embedded circuit simulator exercises in 6.002x

common/lib/xmodule/xmodule/js/src/sequence/display/jquery.sequence.js

@@ -1,6 +1,3 @@
-
-
-
 var SequenceNav = function($element) {
 	var _this = this;
 	var $element = $element;
@@ -44,7 +41,7 @@ var SequenceNav = function($element) {
 
 		var leftPercent = clamp(-left / padding, 0, 1);
 		$leftShadow.css('opacity', leftPercent);
-	
+
 		var rightPercent = clamp((maxScroll + left) / padding, 0, 1);
 		$rightShadow.css('opacity', rightPercent);
 	};
@@ -95,5 +92,5 @@ var SequenceNav = function($element) {
 	$(window).bind('resize', updateWidths);
 	setTimeout(function() {
 		checkPosition();
-	}, 200);	
-}
+	}, 200);
+};

common/lib/xmodule/xmodule/js/src/sequence/edit.coffee

@@ -0,0 +1,9 @@
+class @SequenceDescriptor extends XModule.Descriptor
+  constructor: (@element) ->
+    @$tabs = $(@element).find("#sequence-list")
+    @$tabs.sortable(
+      update: (event, ui) => @update()
+    )
+
+  save: ->
+    children: $('#sequence-list li a', @element).map((idx, el) -> $(el).data('id')).toArray()

common/lib/xmodule/xmodule/js/src/vertical/edit.coffee

@@ -0,0 +1,9 @@
+class @VerticalDescriptor extends XModule.Descriptor
+  constructor: (@element) ->
+    @$items = $(@element).find(".vert-mod")
+    @$items.sortable(
+      update: (event, ui) => @update()
+    )
+
+  save: ->
+    children: $('.vert-mod li', @element).map((idx, el) -> $(el).data('id')).toArray()

common/lib/xmodule/xmodule/js/src/video/display.coffee

@@ -5,6 +5,7 @@ class @Video
     @start = @el.data('start')
     @end = @el.data('end')
     @caption_data_dir = @el.data('caption-data-dir')
+    @caption_asset_path = @el.data('caption-asset-path')
     @show_captions = @el.data('show-captions') == "true"
     window.player = null
     @el = $("#video_#{@id}")
@@ -19,7 +20,7 @@ class @Video
       @embed()
     else
       window.onYouTubePlayerAPIReady = =>
-        $('.course-content .video').each ->
+        @el.each ->
           $(this).data('video').embed()
 
   youtubeId: (speed)->

common/lib/xmodule/xmodule/js/src/video/display/video_caption.coffee

@@ -10,7 +10,7 @@ class @VideoCaption extends Subview
       .bind('DOMMouseScroll', @onMovement)
 
   captionURL: ->
-    "/static/#{@captionDataDir}/subs/#{@youtubeId}.srt.sjson"
+    "#{@captionAssetPath}#{@youtubeId}.srt.sjson"
 
   render: ->
     # TODO: make it so you can have a video with no captions.

common/lib/xmodule/xmodule/js/src/video/display/video_player.coffee

@@ -31,7 +31,7 @@ class @VideoPlayer extends Subview
         el: @el
         youtubeId: @video.youtubeId('1.0')
         currentSpeed: @currentSpeed()
-        captionDataDir: @video.caption_data_dir
+        captionAssetPath: @video.caption_asset_path
     unless onTouchBasedDevice()
       @volumeControl = new VideoVolumeControl el: @$('.secondary-controls')
     @speedControl = new VideoSpeedControl el: @$('.secondary-controls'), speeds: @video.speeds, currentSpeed: @currentSpeed()

common/lib/xmodule/xmodule/js/src/xmodule.coffee

@@ -0,0 +1,76 @@
+@XModule =
+  ###
+  Load a single module (either an edit module or a display module)
+  from the supplied element, which should have a data-type attribute
+  specifying the class to load
+  ###
+  loadModule: (element) ->
+    moduleType = $(element).data('type')
+    if moduleType == 'None'
+      return
+
+    try
+      module = new window[moduleType](element)
+      if $(element).hasClass('xmodule_edit')
+        $(document).trigger('XModule.loaded.edit', [element, module])
+
+      if $(element).hasClass('xmodule_display')
+        $(document).trigger('XModule.loaded.display', [element, module])
+
+      return module
+
+    catch error
+      console.error "Unable to load #{moduleType}: #{error.message}" if console
+
+  ###
+  Load all modules on the page of the specified type.
+  If container is provided, only load modules inside that element
+  Type is one of 'display' or 'edit'
+  ###
+  loadModules: (container) ->
+    selector = ".xmodule_edit, .xmodule_display"
+
+    if container?
+      modules = $(container).find(selector)
+    else
+      modules = $(selector)
+
+    modules.each((idx, element) -> XModule.loadModule element)
+
+class @XModule.Descriptor
+
+  ###
+  Register a callback method to be called when the state of this
+  descriptor is updated. The callback will be passed the results
+  of calling the save method on this descriptor.
+  ###
+  onUpdate: (callback) ->
+    if ! @callbacks?
+      @callbacks = []
+
+    @callbacks.push(callback)
+
+  ###
+  Notify registered callbacks that the state of this descriptor has changed
+  ###
+  update: =>
+    data = @save()
+    callback(data) for callback in @callbacks
+
+  ###
+  Bind the module to an element. This may be called multiple times,
+  if the element content has changed and so the module needs to be rebound
+
+  @method: constructor
+  @param {html element} the .xmodule_edit section containing all of the descriptor content
+  ###
+  constructor: (@element) -> return
+
+  ###
+  Return the current state of the descriptor (to be written to the module store)
+
+  @method: save
+  @returns {object} An object containing children and data attributes (both optional).
+                    The contents of the attributes will be saved to the server
+  ###
+  save: -> return {}

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -153,7 +153,7 @@ class Location(_LocationBase):
             def check(val, regexp):
                 if val is not None and regexp.search(val) is not None:
                     log.debug('invalid characters val="%s", list_="%s"' % (val, list_))
-                    raise InvalidLocationError(location)
+                    raise InvalidLocationError("Invalid characters in '%s'." % (val))
 
             list_ = list(list_)
             for val in list_[:4] + [list_[5]]:
@@ -240,11 +240,15 @@ class ModuleStore(object):
     An abstract interface for a database backend that stores XModuleDescriptor
     instances
     """
+    def has_item(self, location):
+        """
+        Returns True if location exists in this ModuleStore.
+        """
+        raise NotImplementedError
+
     def get_item(self, location, depth=0):
         """
         Returns an XModuleDescriptor instance for the item at location.
-        If location.revision is None, returns the item with the most
-        recent revision
 
         If any segment of the location is None except revision, raises
             xmodule.modulestore.exceptions.InsufficientSpecificationError
@@ -261,7 +265,7 @@ class ModuleStore(object):
         """
         raise NotImplementedError
 
-    def get_instance(self, course_id, location):
+    def get_instance(self, course_id, location, depth=0):
         """
         Get an instance of this location, with policy for course_id applied.
         TODO (vshnayder): this may want to live outside the modulestore eventually
@@ -280,7 +284,7 @@ class ModuleStore(object):
         """
         raise NotImplementedError
 
-    def get_items(self, location, depth=0):
+    def get_items(self, location, course_id=None, depth=0):
         """
         Returns a list of XModuleDescriptor instances for the items
         that match location. Any element of location that is None is treated
@@ -332,11 +336,26 @@ class ModuleStore(object):
         """
         raise NotImplementedError
 
+    def delete_item(self, location):
+        """
+        Delete an item from this modulestore
+
+        location: Something that can be passed to Location
+        """
+        raise NotImplementedError
+
     def get_courses(self):
         '''
         Returns a list containing the top level XModuleDescriptors of the courses
         in this modulestore.
         '''
+        course_filter = Location("i4x", category="course")
+        return self.get_items(course_filter)
+
+    def get_course(self, course_id):
+        '''
+        Look for a specific course id.  Returns the course descriptor, or None if not found.
+        '''
         raise NotImplementedError
 
     def get_course(self, course_id):
@@ -370,6 +389,7 @@ class ModuleStore(object):
         return courses
 
 
+
 class ModuleStoreBase(ModuleStore):
     '''
     Implement interface functionality that can be shared.

common/lib/xmodule/xmodule/modulestore/draft.py

@@ -0,0 +1,196 @@
+from datetime import datetime
+
+from . import ModuleStoreBase, Location
+from .exceptions import ItemNotFoundError
+
+DRAFT = 'draft'
+
+
+def as_draft(location):
+    """
+    Returns the Location that is the draft for `location`
+    """
+    return Location(location)._replace(revision=DRAFT)
+
+
+def wrap_draft(item):
+    """
+    Sets `item.metadata['is_draft']` to `True` if the item is a
+    draft, and false otherwise. Sets the item's location to the
+    non-draft location in either case
+    """
+    item.metadata['is_draft'] = item.location.revision == DRAFT
+    item.location = item.location._replace(revision=None)
+    return item
+
+
+class DraftModuleStore(ModuleStoreBase):
+    """
+    This mixin modifies a modulestore to give it draft semantics.
+    That is, edits made to units are stored to locations that have the revision DRAFT,
+    and when reads are made, they first read with revision DRAFT, and then fall back
+    to the baseline revision only if DRAFT doesn't exist.
+
+    This module also includes functionality to promote DRAFT modules (and optionally
+    their children) to published modules.
+    """
+
+    def get_item(self, location, depth=0):
+        """
+        Returns an XModuleDescriptor instance for the item at location.
+        If location.revision is None, returns the item with the most
+        recent revision
+
+        If any segment of the location is None except revision, raises
+            xmodule.modulestore.exceptions.InsufficientSpecificationError
+
+        If no object is found at that location, raises
+            xmodule.modulestore.exceptions.ItemNotFoundError
+
+        location: Something that can be passed to Location
+
+        depth (int): An argument that some module stores may use to prefetch
+            descendents of the queried modules for more efficient results later
+            in the request. The depth is counted in the number of calls to
+            get_children() to cache. None indicates to cache all descendents
+        """
+
+        # cdodge: we're forcing depth=0 here as the Draft store is not handling caching well
+        try:
+            return wrap_draft(super(DraftModuleStore, self).get_item(as_draft(location), depth=0))
+        except ItemNotFoundError:
+            return wrap_draft(super(DraftModuleStore, self).get_item(location, depth=0))
+
+    def get_instance(self, course_id, location, depth=0):
+        """
+        Get an instance of this location, with policy for course_id applied.
+        TODO (vshnayder): this may want to live outside the modulestore eventually
+        """
+
+        # cdodge: we're forcing depth=0 here as the Draft store is not handling caching well 
+        try:
+            return wrap_draft(super(DraftModuleStore, self).get_instance(course_id, as_draft(location), depth=0))
+        except ItemNotFoundError:
+            return wrap_draft(super(DraftModuleStore, self).get_instance(course_id, location, depth=0))
+
+    def get_items(self, location, depth=0):
+        """
+        Returns a list of XModuleDescriptor instances for the items
+        that match location. Any element of location that is None is treated
+        as a wildcard that matches any value
+
+        location: Something that can be passed to Location
+
+        depth: An argument that some module stores may use to prefetch
+            descendents of the queried modules for more efficient results later
+            in the request. The depth is counted in the number of calls to
+            get_children() to cache. None indicates to cache all descendents
+        """
+        draft_loc = as_draft(location)
+
+        # cdodge: we're forcing depth=0 here as the Draft store is not handling caching well
+        draft_items = super(DraftModuleStore, self).get_items(draft_loc, depth=0)
+        items = super(DraftModuleStore, self).get_items(location, depth=0)
+
+        draft_locs_found = set(item.location._replace(revision=None) for item in draft_items)
+        non_draft_items = [
+            item
+            for item in items
+            if (item.location.revision != DRAFT
+                and item.location._replace(revision=None) not in draft_locs_found)
+        ]
+        return [wrap_draft(item) for item in draft_items + non_draft_items]
+
+    def clone_item(self, source, location):
+        """
+        Clone a new item that is a copy of the item at the location `source`
+        and writes it to `location`
+        """
+        return wrap_draft(super(DraftModuleStore, self).clone_item(source, as_draft(location)))
+
+    def update_item(self, location, data):
+        """
+        Set the data in the item specified by the location to
+        data
+
+        location: Something that can be passed to Location
+        data: A nested dictionary of problem data
+        """
+        draft_loc = as_draft(location)
+        draft_item = self.get_item(location)
+        if not draft_item.metadata['is_draft']:
+            self.clone_item(location, draft_loc)
+
+        return super(DraftModuleStore, self).update_item(draft_loc, data)
+
+    def update_children(self, location, children):
+        """
+        Set the children for the item specified by the location to
+        children
+
+        location: Something that can be passed to Location
+        children: A list of child item identifiers
+        """
+        draft_loc = as_draft(location)
+        draft_item = self.get_item(location)
+        if not draft_item.metadata['is_draft']:
+            self.clone_item(location, draft_loc)
+
+        return super(DraftModuleStore, self).update_children(draft_loc, children)
+
+    def update_metadata(self, location, metadata):
+        """
+        Set the metadata for the item specified by the location to
+        metadata
+
+        location: Something that can be passed to Location
+        metadata: A nested dictionary of module metadata
+        """
+        draft_loc = as_draft(location)
+        draft_item = self.get_item(location)
+
+        if not draft_item.metadata['is_draft']:
+            self.clone_item(location, draft_loc)
+
+        if 'is_draft' in metadata:
+            del metadata['is_draft']
+
+        return super(DraftModuleStore, self).update_metadata(draft_loc, metadata)
+
+    def delete_item(self, location):
+        """
+        Delete an item from this modulestore
+
+        location: Something that can be passed to Location
+        """
+        return super(DraftModuleStore, self).delete_item(as_draft(location))
+
+
+    def get_parent_locations(self, location, course_id):
+        '''Find all locations that are the parents of this location.  Needed
+        for path_to_location().
+
+        returns an iterable of things that can be passed to Location.
+        '''
+        return super(DraftModuleStore, self).get_parent_locations(location, course_id)
+
+    def publish(self, location, published_by_id):
+        """
+        Save a current draft to the underlying modulestore
+        """
+        draft = self.get_item(location)
+        metadata = {}
+        metadata.update(draft.metadata)
+        metadata['published_date'] = tuple(datetime.utcnow().timetuple())
+        metadata['published_by'] = published_by_id
+        super(DraftModuleStore, self).update_item(location, draft.definition.get('data', {}))
+        super(DraftModuleStore, self).update_children(location, draft.definition.get('children', []))
+        super(DraftModuleStore, self).update_metadata(location, metadata)
+        self.delete_item(location)
+
+    def unpublish(self, location):
+        """
+        Turn the published version into a draft, removing the published version
+        """
+        super(DraftModuleStore, self).clone_item(location, as_draft(location))
+        super(DraftModuleStore, self).delete_item(location)

common/lib/xmodule/xmodule/modulestore/mongo.py

@@ -1,5 +1,6 @@
 import pymongo
 import sys
+import logging
 
 from bson.son import SON
 from fs.osfs import OSFS
@@ -13,6 +14,7 @@ from xmodule.mako_module import MakoDescriptorSystem
 from xmodule.error_module import ErrorDescriptor
 
 from . import ModuleStoreBase, Location
+from .draft import DraftModuleStore
 from .exceptions import (ItemNotFoundError,
                          DuplicateItemError)
 
@@ -49,6 +51,9 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
         self.modulestore = modulestore
         self.module_data = module_data
         self.default_class = default_class
+        # cdodge: other Systems have a course_id attribute defined. To keep things consistent, let's
+        # define an attribute here as well, even though it's None
+        self.course_id = None
 
     def load_item(self, location):
         location = Location(location)
@@ -69,21 +74,34 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
                 )
 
 
-def location_to_query(location):
+def location_to_query(location, wildcard=True):
     """
     Takes a Location and returns a SON object that will query for that location.
     Fields in location that are None are ignored in the query
+
+    If `wildcard` is True, then a None in a location is treated as a wildcard
+    query. Otherwise, it is searched for literally
     """
-    query = SON()
-    # Location dict is ordered by specificity, and SON
-    # will preserve that order for queries
-    for key, val in Location(location).dict().iteritems():
-        if val is not None:
-            query['_id.{key}'.format(key=key)] = val
+    query = namedtuple_to_son(Location(location), prefix='_id.')
+
+    if wildcard:
+        for key, value in query.items():
+            if value is None:
+                del query[key]
 
     return query
 
 
+def namedtuple_to_son(namedtuple, prefix=''):
+    """
+    Converts a namedtuple into a SON object with the same key order
+    """
+    son = SON()
+    for idx, field_name in enumerate(namedtuple._fields):
+        son[prefix + field_name] = namedtuple[idx]
+    return son
+
+
 class MongoModuleStore(ModuleStoreBase):
     """
     A Mongodb backed ModuleStore
@@ -92,15 +110,21 @@ class MongoModuleStore(ModuleStoreBase):
     # TODO (cpennington): Enable non-filesystem filestores
     def __init__(self, host, db, collection, fs_root, render_template,
                  port=27017, default_class=None,
-                 error_tracker=null_error_tracker):
+                 error_tracker=null_error_tracker,
+                 user=None, password=None, **kwargs):
 
         ModuleStoreBase.__init__(self)
 
         self.collection = pymongo.connection.Connection(
             host=host,
-            port=port
+            port=port,
+            **kwargs
         )[db][collection]
 
+        if user is not None and password is not None:
+            self.collection.database.authenticate(user, password)
+
+
         # Force mongo to report errors, at the expense of performance
         self.collection.safe = True
 
@@ -134,6 +158,7 @@ class MongoModuleStore(ModuleStoreBase):
         If depth is None, will load all the children.
         This will make a number of queries that is linear in the depth.
         """
+
         data = {}
         to_process = list(items)
         while to_process and depth is None or depth >= 0:
@@ -147,8 +172,10 @@ class MongoModuleStore(ModuleStoreBase):
             # http://www.mongodb.org/display/DOCS/Advanced+Queries#AdvancedQueries-%24or
             # for or-query syntax
             if children:
-                to_process = list(self.collection.find(
-                    {'_id': {'$in': [Location(child).dict() for child in children]}}))
+                query = {
+                    '_id': {'$in': [namedtuple_to_son(Location(child)) for child in children]}
+                }
+                to_process = self.collection.find(query)
             else:
                 to_process = []
             # If depth is None, then we just recurse until we hit all the descendents
@@ -202,18 +229,27 @@ class MongoModuleStore(ModuleStoreBase):
         ItemNotFoundError.
         '''
         item = self.collection.find_one(
-            location_to_query(location),
+            location_to_query(location, wildcard=False),
             sort=[('revision', pymongo.ASCENDING)],
         )
         if item is None:
             raise ItemNotFoundError(location)
         return item
 
+    def has_item(self, location):
+        """
+        Returns True if location exists in this ModuleStore.
+        """
+        location = Location.ensure_fully_specified(location)
+        try:
+            self._find_one(location)
+            return True
+        except ItemNotFoundError:
+            return False
+
     def get_item(self, location, depth=0):
         """
         Returns an XModuleDescriptor instance for the item at location.
-        If location.revision is None, returns the item with the most
-        recent revision.
 
         If any segment of the location is None except revision, raises
             xmodule.modulestore.exceptions.InsufficientSpecificationError
@@ -231,14 +267,19 @@ class MongoModuleStore(ModuleStoreBase):
         item = self._find_one(location)
         return self._load_items([item], depth)[0]
 
-    def get_instance(self, course_id, location):
+    def get_instance(self, course_id, location, depth=0):
         """
         TODO (vshnayder): implement policy tracking in mongo.
         For now, just delegate to get_item and ignore policy.
-        """
-        return self.get_item(location)
 
-    def get_items(self, location, depth=0):
+        depth (int): An argument that some module stores may use to prefetch
+            descendents of the queried modules for more efficient results later
+            in the request. The depth is counted in the number of
+            calls to get_children() to cache. None indicates to cache all descendents.
+        """
+        return self.get_item(location, depth=depth)
+
+    def get_items(self, location, course_id=None, depth=0):
         items = self.collection.find(
             location_to_query(location),
             sort=[('revision', pymongo.ASCENDING)],
@@ -255,10 +296,49 @@ class MongoModuleStore(ModuleStoreBase):
             source_item = self.collection.find_one(location_to_query(source))
             source_item['_id'] = Location(location).dict()
             self.collection.insert(source_item)
-            return self._load_items([source_item])[0]
+            item = self._load_items([source_item])[0]
+
+            # VS[compat] cdodge: This is a hack because static_tabs also have references from the course module, so
+            # if we add one then we need to also add it to the policy information (i.e. metadata)
+            # we should remove this once we can break this reference from the course to static tabs
+            if location.category == 'static_tab':
+                course = self.get_course_for_item(item.location)
+                existing_tabs = course.tabs or []
+                existing_tabs.append({'type':'static_tab', 'name' : item.metadata.get('display_name'), 'url_slug' : item.location.name})
+                course.tabs = existing_tabs
+                self.update_metadata(course.location, course.metadata)
+
+            return item
         except pymongo.errors.DuplicateKeyError:
             raise DuplicateItemError(location)
 
+
+    def get_course_for_item(self, location):
+        '''
+        VS[compat]
+        cdodge: for a given Xmodule, return the course that it belongs to
+        NOTE: This makes a lot of assumptions about the format of the course location
+        Also we have to assert that this module maps to only one course item - it'll throw an
+        assert if not
+        This is only used to support static_tabs as we need to be course module aware
+        '''
+
+        # @hack! We need to find the course location however, we don't
+        # know the 'name' parameter in this context, so we have
+        # to assume there's only one item in this query even though we are not specifying a name
+        course_search_location = ['i4x', location.org, location.course, 'course', None]
+        courses = self.get_items(course_search_location)
+
+        # make sure we found exactly one match on this above course search
+        found_cnt = len(courses)
+        if found_cnt == 0:
+            raise BaseException('Could not find course at {0}'.format(course_search_location))
+
+        if found_cnt > 1:
+            raise BaseException('Found more than one course at {0}. There should only be one!!! Dump = {1}'.format(course_search_location, courses))
+
+        return courses[0]
+
     def _update_single_item(self, location, update):
         """
         Set update on the specified item, and raises ItemNotFoundError
@@ -306,23 +386,47 @@ class MongoModuleStore(ModuleStoreBase):
         location: Something that can be passed to Location
         metadata: A nested dictionary of module metadata
         """
+        # VS[compat] cdodge: This is a hack because static_tabs also have references from the course module, so
+        # if we add one then we need to also add it to the policy information (i.e. metadata)
+        # we should remove this once we can break this reference from the course to static tabs
+        loc = Location(location)
+        if loc.category == 'static_tab':
+            course = self.get_course_for_item(loc)
+            existing_tabs = course.tabs or []
+            for tab in existing_tabs:
+                if tab.get('url_slug') == loc.name:
+                    tab['name'] = metadata.get('display_name')
+                    break
+            course.tabs = existing_tabs
+            self.update_metadata(course.location, course.metadata)
 
         self._update_single_item(location, {'metadata': metadata})
 
+
+    def delete_item(self, location):
+        """
+        Delete an item from this modulestore
+
+        location: Something that can be passed to Location
+        """
+        # VS[compat] cdodge: This is a hack because static_tabs also have references from the course module, so
+        # if we add one then we need to also add it to the policy information (i.e. metadata)
+        # we should remove this once we can break this reference from the course to static tabs
+        if location.category == 'static_tab':
+            item = self.get_item(location)
+            course = self.get_course_for_item(item.location)
+            existing_tabs = course.tabs or []
+            course.tabs = [tab for tab in existing_tabs if tab.get('url_slug') != location.name]
+            self.update_metadata(course.location, course.metadata)
+
+        self.collection.remove({'_id': Location(location).dict()})
+
+
     def get_parent_locations(self, location, course_id):
         '''Find all locations that are the parents of this location in this 
         course.  Needed for path_to_location().
-
-        If there is no data at location in this modulestore, raise
-            ItemNotFoundError.
-
-        returns an iterable of things that can be passed to Location.  This may
-        be empty if there are no parents.
         '''
         location = Location.ensure_fully_specified(location)
-        # Check that it's actually in this modulestore.
-        self._find_one(location)
-        # now get the parents
         items = self.collection.find({'definition.children': location.url()},
                                     {'_id': True})
         return [i['_id'] for i in items]
@@ -333,3 +437,8 @@ class MongoModuleStore(ModuleStoreBase):
         are loaded on demand, rather than up front
         """
         return {}
+
+
+# DraftModuleStore is first, because it needs to intercept calls to MongoModuleStore
+class DraftMongoModuleStore(DraftModuleStore, MongoModuleStore):
+    pass

common/lib/xmodule/xmodule/modulestore/search.py

@@ -81,6 +81,9 @@ def path_to_location(modulestore, course_id, location):
         # If we're here, there is no path
         return None
 
+    if not modulestore.has_item(location):
+        raise ItemNotFoundError
+
     path = find_path_to_course()
     if path is None:
         raise NoPathToItem(location)

common/lib/xmodule/xmodule/modulestore/store_utilities.py

@@ -0,0 +1,130 @@
+import logging
+from xmodule.contentstore.content import StaticContent
+from xmodule.modulestore import Location
+from xmodule.modulestore.mongo import MongoModuleStore
+
+def clone_course(modulestore, contentstore, source_location, dest_location, delete_original=False):
+  # first check to see if the modulestore is Mongo backed
+  if not isinstance(modulestore, MongoModuleStore):
+    raise Exception("Expected a MongoModuleStore in the runtime. Aborting....")
+
+  # check to see if the dest_location exists as an empty course
+  # we need an empty course because the app layers manage the permissions and users
+  if not modulestore.has_item(dest_location):
+    raise Exception("An empty course at {0} must have already been created. Aborting...".format(dest_location))
+
+  # verify that the dest_location really is an empty course, which means only one 
+  dest_modules = modulestore.get_items([dest_location.tag, dest_location.org, dest_location.course, None, None, None])
+
+  if len(dest_modules) != 1:
+    raise Exception("Course at destination {0} is not an empty course. You can only clone into an empty course. Aborting...".format(dest_location))
+
+  # check to see if the source course is actually there
+  if not modulestore.has_item(source_location):
+    raise Exception("Cannot find a course at {0}. Aborting".format(source_location))
+
+  # Get all modules under this namespace which is (tag, org, course) tuple
+
+  modules = modulestore.get_items([source_location.tag, source_location.org, source_location.course, None, None, None])
+
+  for module in modules:
+    original_loc = Location(module.location)
+
+    if original_loc.category != 'course':
+      module.location = module.location._replace(tag = dest_location.tag, org = dest_location.org, 
+        course = dest_location.course)
+    else:
+      # on the course module we also have to update the module name
+      module.location = module.location._replace(tag = dest_location.tag, org = dest_location.org, 
+        course = dest_location.course, name=dest_location.name)
+
+    print "Cloning module {0} to {1}....".format(original_loc, module.location)
+
+    if 'data' in module.definition:
+      modulestore.update_item(module.location, module.definition['data'])
+
+    # repoint children
+    if 'children' in module.definition:
+      new_children = []
+      for child_loc_url in module.definition['children']:
+        child_loc = Location(child_loc_url)
+        child_loc = child_loc._replace(tag = dest_location.tag, org = dest_location.org, 
+        course = dest_location.course)
+        new_children = new_children + [child_loc.url()]
+
+      modulestore.update_children(module.location, new_children)
+
+    # save metadata
+    modulestore.update_metadata(module.location, module.metadata)
+
+  # now iterate through all of the assets and clone them
+  # first the thumbnails
+  thumbs = contentstore.get_all_content_thumbnails_for_course(source_location)
+  for thumb in thumbs:
+    thumb_loc = Location(thumb["_id"])
+    content = contentstore.find(thumb_loc)
+    content.location = content.location._replace(org = dest_location.org, 
+        course = dest_location.course)
+
+    print "Cloning thumbnail {0} to {1}".format(thumb_loc, content.location)
+
+    contentstore.save(content)
+
+  # now iterate through all of the assets, also updating the thumbnail pointer
+
+  assets = contentstore.get_all_content_for_course(source_location)
+  for asset in assets:
+    asset_loc = Location(asset["_id"])
+    content = contentstore.find(asset_loc)
+    content.location = content.location._replace(org = dest_location.org, 
+        course = dest_location.course)
+
+    # be sure to update the pointer to the thumbnail
+    if content.thumbnail_location is not None:
+      content.thumbnail_location = content.thumbnail_location._replace(org = dest_location.org, 
+        course = dest_location.course)
+
+    print "Cloning asset {0} to {1}".format(asset_loc, content.location)
+
+    contentstore.save(content)
+
+  return True
+
+def delete_course(modulestore, contentstore, source_location):
+    # first check to see if the modulestore is Mongo backed
+  if not isinstance(modulestore, MongoModuleStore):
+    raise Exception("Expected a MongoModuleStore in the runtime. Aborting....")
+
+  # check to see if the source course is actually there
+  if not modulestore.has_item(source_location):
+    raise Exception("Cannot find a course at {0}. Aborting".format(source_location))
+
+  # first delete all of the thumbnails
+  thumbs = contentstore.get_all_content_thumbnails_for_course(source_location)
+  for thumb in thumbs:
+    thumb_loc = Location(thumb["_id"])
+    id = StaticContent.get_id_from_location(thumb_loc)
+    print "Deleting {0}...".format(id)
+    contentstore.delete(id)
+
+  # then delete all of the assets
+  assets = contentstore.get_all_content_for_course(source_location)
+  for asset in assets:
+    asset_loc = Location(asset["_id"])
+    id = StaticContent.get_id_from_location(asset_loc)
+    print "Deleting {0}...".format(id)
+    contentstore.delete(id)
+
+  # then delete all course modules
+  modules = modulestore.get_items([source_location.tag, source_location.org, source_location.course, None, None, None])
+
+  for module in modules:
+    if module.category != 'course': # save deleting the course module for last
+      print "Deleting {0}...".format(module.location)
+      modulestore.delete_item(module.location)
+
+  # finally delete the top-level course module itself
+  print "Deleting {0}...".format(source_location)
+  modulestore.delete_item(source_location)
+
+  return True

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -4,6 +4,7 @@ import logging
 import os
 import re
 import sys
+import glob
 
 from collections import defaultdict
 from cStringIO import StringIO
@@ -12,11 +13,14 @@ from importlib import import_module
 from lxml import etree
 from path import path
 
+from xmodule.error_module import ErrorDescriptor
 from xmodule.errortracker import make_error_tracker, exc_info_to_str
 from xmodule.course_module import CourseDescriptor
 from xmodule.mako_module import MakoDescriptorSystem
 from xmodule.x_module import XModuleDescriptor, XMLParsingSystem
 
+from xmodule.html_module import HtmlDescriptor
+
 from . import ModuleStoreBase, Location
 from .exceptions import ItemNotFoundError
 
@@ -50,6 +54,8 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
         self.unnamed = defaultdict(int)     # category -> num of new url_names for that category
         self.used_names = defaultdict(set)  # category -> set of used url_names
         self.org, self.course, self.url_name = course_id.split('/')
+        # cdodge: adding the course_id as passed in for later reference rather than having to recomine the org/course/url_name
+        self.course_id = course_id
         self.load_error_modules = load_error_modules
 
         def process_xml(xml):
@@ -162,8 +168,6 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
                 # Didn't load properly.  Fall back on loading as an error
                 # descriptor.  This should never error due to formatting.
 
-                # Put import here to avoid circular import errors
-                from xmodule.error_module import ErrorDescriptor
 
                 msg = "Error loading from xml. " + str(err)[:200]
                 log.warning(msg)
@@ -302,11 +306,11 @@ class XMLModuleStore(ModuleStoreBase):
         try:
             course_descriptor = self.load_course(course_dir, errorlog.tracker)
         except Exception as e:
-            msg = "Failed to load course '{0}': {1}".format(course_dir, str(e))
+            msg = "ERROR: Failed to load course '{0}': {1}".format(course_dir, str(e))
             log.exception(msg)
             errorlog.tracker(msg)
 
-        if course_descriptor is not None:
+        if course_descriptor is not None and not isinstance(course_descriptor, ErrorDescriptor):
             self.courses[course_dir] = course_descriptor
             self._location_errors[course_descriptor.location] = errorlog
             self.parent_trackers[course_descriptor.id].make_known(course_descriptor.location)
@@ -333,37 +337,15 @@ class XMLModuleStore(ModuleStoreBase):
         if not os.path.exists(policy_path):
             return {}
         try:
-            log.debug("Loading policy from {0}".format(policy_path))
             with open(policy_path) as f:
                 return json.load(f)
         except (IOError, ValueError) as err:
-            msg = "Error loading course policy from {0}".format(policy_path)
+            msg = "ERROR: loading course policy from {0}".format(policy_path)
             tracker(msg)
             log.warning(msg + " " + str(err))
         return {}
 
 
-    def read_grading_policy(self, paths, tracker):
-        """Load a grading policy from the specified paths, in order, if it exists."""
-        # Default to a blank policy
-        policy_str = ""
-
-        for policy_path in paths:
-            if not os.path.exists(policy_path):
-                continue
-            log.debug("Loading grading policy from {0}".format(policy_path))
-            try:
-                with open(policy_path) as grading_policy_file:
-                    policy_str = grading_policy_file.read()
-                    # if we successfully read the file, stop looking at backups
-                    break
-            except (IOError):
-                msg = "Unable to load course settings file from '{0}'".format(policy_path)
-                tracker(msg)
-                log.warning(msg)
-
-        return policy_str
-
 
     def load_course(self, course_dir, tracker):
         """
@@ -409,6 +391,7 @@ class XMLModuleStore(ModuleStoreBase):
             if url_name:
                 policy_dir = self.data_dir / course_dir / 'policies' / url_name
                 policy_path = policy_dir / 'policy.json'
+
                 policy = self.load_policy(policy_path, tracker)
 
                 # VS[compat]: remove once courses use the policy dirs.
@@ -426,7 +409,6 @@ class XMLModuleStore(ModuleStoreBase):
                     raise ValueError("Can't load a course without a 'url_name' "
                                      "(or 'name') set.  Set url_name.")
 
-
             course_id = CourseDescriptor.make_id(org, course, url_name)
             system = ImportSystem(
                 self,
@@ -440,24 +422,65 @@ class XMLModuleStore(ModuleStoreBase):
 
             course_descriptor = system.process_xml(etree.tostring(course_data, encoding='unicode'))
 
+            # If we fail to load the course, then skip the rest of the loading steps
+            if isinstance(course_descriptor, ErrorDescriptor):
+                return course_descriptor
+
             # NOTE: The descriptors end up loading somewhat bottom up, which
             # breaks metadata inheritance via get_children().  Instead
             # (actually, in addition to, for now), we do a final inheritance pass
             # after we have the course descriptor.
             XModuleDescriptor.compute_inherited_metadata(course_descriptor)
 
-            # Try to load grading policy
-            paths = [self.data_dir / course_dir / 'grading_policy.json']
-            if policy_dir:
-                paths = [policy_dir / 'grading_policy.json'] + paths
+            # now import all pieces of course_info which is expected to be stored
+            # in <content_dir>/info or <content_dir>/info/<url_name>
+            self.load_extra_content(system, course_descriptor, 'course_info', self.data_dir / course_dir / 'info', course_dir, url_name)
 
-            policy_str = self.read_grading_policy(paths, tracker)
-            course_descriptor.set_grading_policy(policy_str)
+            # now import all static tabs which are expected to be stored in
+            # in <content_dir>/tabs or <content_dir>/tabs/<url_name>           
+            self.load_extra_content(system, course_descriptor, 'static_tab', self.data_dir / course_dir / 'tabs', course_dir, url_name)
+
+            self.load_extra_content(system, course_descriptor, 'custom_tag_template', self.data_dir / course_dir / 'custom_tags', course_dir, url_name)
+
+            self.load_extra_content(system, course_descriptor, 'about', self.data_dir / course_dir / 'about', course_dir, url_name)
 
             log.debug('========> Done with course import from {0}'.format(course_dir))
             return course_descriptor
 
 
+    def load_extra_content(self, system, course_descriptor, category, base_dir, course_dir, url_name):
+
+        self._load_extra_content(system, course_descriptor, category, base_dir, course_dir)
+
+         # then look in a override folder based on the course run
+        if os.path.isdir(base_dir / url_name):
+            self._load_extra_content(system, course_descriptor, category, base_dir / url_name, course_dir)       
+
+
+    def _load_extra_content(self, system, course_descriptor, category, path, course_dir):
+
+        for filepath in glob.glob(path/ '*'):
+            if not os.path.isdir(filepath):
+                with open(filepath) as f:
+                    try:
+                        html = f.read().decode('utf-8')
+                        # tabs are referenced in policy.json through a 'slug' which is just the filename without the .html suffix
+                        slug = os.path.splitext(os.path.basename(filepath))[0]
+                        loc = Location('i4x', course_descriptor.location.org, course_descriptor.location.course, category, slug)
+                        module = HtmlDescriptor(system, definition={'data' : html}, **{'location' : loc})
+                        # VS[compat]:
+                        # Hack because we need to pull in the 'display_name' for static tabs (because we need to edit them)
+                        # from the course policy
+                        if category == "static_tab":
+                            for tab in course_descriptor.tabs or []:
+                                if tab.get('url_slug') == slug:
+                                    module.metadata['display_name'] = tab['name']
+                        module.metadata['data_dir'] = course_dir
+                        self.modules[course_descriptor.id][module.location] = module
+                    except Exception, e:
+                        logging.exception("Failed to load {0}. Skipping... Exception: {1}".format(filepath, str(e)))
+                        system.error_tracker("ERROR: " + str(e))
+
     def get_instance(self, course_id, location, depth=0):
         """
         Returns an XModuleDescriptor instance for the item at
@@ -479,11 +502,16 @@ class XMLModuleStore(ModuleStoreBase):
         except KeyError:
             raise ItemNotFoundError(location)
 
+    def has_item(self, location):
+        """
+        Returns True if location exists in this ModuleStore.
+        """
+        location = Location(location)
+        return any(location in course_modules for course_modules in self.modules.values())
+
     def get_item(self, location, depth=0):
         """
         Returns an XModuleDescriptor instance for the item at location.
-        If location.revision is None, returns the most item with the most
-        recent revision
 
         If any segment of the location is None except revision, raises
             xmodule.modulestore.exceptions.InsufficientSpecificationError
@@ -496,6 +524,24 @@ class XMLModuleStore(ModuleStoreBase):
         raise NotImplementedError("XMLModuleStores can't guarantee that definitions"
                                   " are unique. Use get_instance.")
 
+    def get_items(self, location, course_id=None, depth=0):
+        items = []
+
+        def _add_get_items(self, location, modules):
+            for mod_loc, module in modules.iteritems():
+                # Locations match if each value in `location` is None or if the value from `location`
+                # matches the value from `mod_loc`
+                if all(goal is None or goal == value for goal, value in zip(location, mod_loc)):
+                    items.append(module)
+
+        if course_id is None:
+            for _, modules in self.modules.iteritems():
+                _add_get_items(self, location, modules)
+        else:
+            _add_get_items(self, location, self.modules[course_id])
+
+        return items
+
 
     def get_courses(self, depth=0):
         """
@@ -547,9 +593,6 @@ class XMLModuleStore(ModuleStoreBase):
         '''Find all locations that are the parents of this location in this 
         course.  Needed for path_to_location().
 
-        If there is no data at location in this modulestore, raise
-            ItemNotFoundError.
-
         returns an iterable of things that can be passed to Location.  This may
         be empty if there are no parents.
         '''

common/lib/xmodule/xmodule/modulestore/xml_exporter.py

@@ -0,0 +1,20 @@
+import logging
+from xmodule.modulestore import Location
+from xmodule.modulestore.django import modulestore
+from fs.osfs import OSFS
+
+def export_to_xml(modulestore, contentstore, course_location, root_dir, course_dir):
+
+  course = modulestore.get_item(course_location)
+
+  fs = OSFS(root_dir)
+  export_fs = fs.makeopendir(course_dir)
+
+  xml = course.export_to_xml(export_fs)
+  with export_fs.open('course.xml', 'w') as course_xml:
+      course_xml.write(xml)
+
+  # export the static assets
+  contentstore.export_all_for_course(course_location, root_dir + '/' + course_dir + '/static/')
+
+  

common/lib/xmodule/xmodule/modulestore/xml_importer.py

@@ -1,14 +1,100 @@
 import logging
+import os
+import mimetypes
+from lxml.html import rewrite_links as lxml_rewrite_links
+from path import path
 
 from .xml import XMLModuleStore
 from .exceptions import DuplicateItemError
+from xmodule.modulestore import Location
+from xmodule.contentstore.content import StaticContent, XASSET_SRCREF_PREFIX
 
 log = logging.getLogger(__name__)
 
+def import_static_content(modules, course_loc, course_data_path, static_content_store, target_location_namespace, 
+    subpath = 'static', verbose=False):
+    
+    remap_dict = {}
+
+    # now import all static assets
+    static_dir = course_data_path / subpath
+
+    for dirname, dirnames, filenames in os.walk(static_dir):
+        for filename in filenames:
+
+            try:
+                content_path = os.path.join(dirname, filename)
+                if verbose:
+                    log.debug('importing static content {0}...'.format(content_path))
+
+                fullname_with_subpath = content_path.replace(static_dir, '')  # strip away leading path from the name
+                if fullname_with_subpath.startswith('/'):
+                    fullname_with_subpath = fullname_with_subpath[1:]
+                content_loc = StaticContent.compute_location(target_location_namespace.org, target_location_namespace.course, fullname_with_subpath)
+                mime_type = mimetypes.guess_type(filename)[0]
+
+                with open(content_path, 'rb') as f:
+                    data = f.read()
+
+                content = StaticContent(content_loc, filename, mime_type, data, import_path = fullname_with_subpath)
+
+                # first let's save a thumbnail so we can get back a thumbnail location
+                (thumbnail_content, thumbnail_location) = static_content_store.generate_thumbnail(content)
+
+                if thumbnail_content is not None:
+                    content.thumbnail_location = thumbnail_location
+
+                #then commit the content
+                static_content_store.save(content)
+
+                #store the remapping information which will be needed to subsitute in the module data
+                remap_dict[fullname_with_subpath] = content_loc.name
+            except:
+                raise 
+
+    return remap_dict
+
+def verify_content_links(module, base_dir, static_content_store, link, remap_dict = None):
+    if link.startswith('/static/'):
+        # yes, then parse out the name
+        path = link[len('/static/'):]
+
+        static_pathname = base_dir / path
+
+        if os.path.exists(static_pathname):
+            try:
+                content_loc = StaticContent.compute_location(module.location.org, module.location.course, path)
+                filename = os.path.basename(path)
+                mime_type = mimetypes.guess_type(filename)[0]
+
+                with open(static_pathname, 'rb') as f:
+                    data = f.read()
+
+                content = StaticContent(content_loc, filename, mime_type, data, import_path = path) 
+
+                # first let's save a thumbnail so we can get back a thumbnail location
+                (thumbnail_content, thumbnail_location) = static_content_store.generate_thumbnail(content)
+
+                if thumbnail_content is not None:
+                    content.thumbnail_location = thumbnail_location
+
+                #then commit the content
+                static_content_store.save(content)   
+
+                new_link = StaticContent.get_url_path_from_location(content_loc)   
+
+                if remap_dict is not None:
+                    remap_dict[link] = new_link
+
+                return new_link                 
+            except Exception, e:
+                logging.exception('Skipping failed content load from {0}. Exception: {1}'.format(path, e))
+
+    return link
 
 def import_from_xml(store, data_dir, course_dirs=None, 
                     default_class='xmodule.raw_module.RawDescriptor',
-                    load_error_modules=True):
+                    load_error_modules=True, static_content_store=None, target_location_namespace=None, verbose=False):
     """
     Import the specified xml data_dir into the "store" modulestore,
     using org and course as the location org and course.
@@ -16,22 +102,245 @@ def import_from_xml(store, data_dir, course_dirs=None,
     course_dirs: If specified, the list of course_dirs to load. Otherwise, load
     all course dirs
 
+    target_location_namespace is the namespace [passed as Location] (i.e. {tag},{org},{course}) that all modules in the should be remapped to
+    after import off disk. We do this remapping as a post-processing step because there's logic in the importing which
+    expects a 'url_name' as an identifier to where things are on disk e.g. ../policies/<url_name>/policy.json as well as metadata keys in
+    the policy.json. so we need to keep the original url_name during import
+
     """
+    
     module_store = XMLModuleStore(
         data_dir,
         default_class=default_class,
         course_dirs=course_dirs,
-        load_error_modules=load_error_modules,
+        load_error_modules=load_error_modules
     )
+
+    # NOTE: the XmlModuleStore does not implement get_items() which would be a preferable means
+    # to enumerate the entire collection of course modules. It will be left as a TBD to implement that
+    # method on XmlModuleStore.
+    course_items = []
     for course_id in module_store.modules.keys():
+
+        course_data_path = None
+        course_location = None
+
+        if verbose:
+            log.debug("Scanning {0} for course module...".format(course_id))
+
+        # Quick scan to get course module as we need some info from there. Also we need to make sure that the
+        # course module is committed first into the store
+        for module in module_store.modules[course_id].itervalues():
+            if module.category == 'course':
+                course_data_path = path(data_dir) / module.metadata['data_dir']
+                course_location = module.location
+
+                module = remap_namespace(module, target_location_namespace)
+
+                # HACK: for now we don't support progress tabs. There's a special metadata configuration setting for this.
+                module.metadata['hide_progress_tab'] = True
+
+                # cdodge: more hacks (what else). Seems like we have a problem when importing a course (like 6.002) which 
+                # does not have any tabs defined in the policy file. The import goes fine and then displays fine in LMS, 
+                # but if someone tries to add a new tab in the CMS, then the LMS barfs because it expects that - 
+                # if there is *any* tabs - then there at least needs to be some predefined ones
+                if module.tabs is None or len(module.tabs) == 0:
+                    module.tabs = [{"type": "courseware"}, 
+                        {"type": "course_info", "name": "Course Info"}, 
+                        {"type": "discussion", "name": "Discussion"},
+                        {"type": "wiki", "name": "Wiki"}]  # note, add 'progress' when we can support it on Edge
+
+
+                store.update_item(module.location, module.definition['data'])
+                if 'children' in module.definition:
+                    store.update_children(module.location, module.definition['children'])
+                store.update_metadata(module.location, dict(module.own_metadata))
+
+                # a bit of a hack, but typically the "course image" which is shown on marketing pages is hard coded to /images/course_image.jpg
+                # so let's make sure we import in case there are no other references to it in the modules
+                verify_content_links(module, course_data_path, static_content_store, '/static/images/course_image.jpg')
+
+                course_items.append(module)
+
+                
+        # then import all the static content
+        if static_content_store is not None:
+            _namespace_rename = target_location_namespace if target_location_namespace is not None else course_location
+            
+            # first pass to find everything in /static/
+            import_static_content(module_store.modules[course_id], course_location, course_data_path, static_content_store, 
+                _namespace_rename, subpath='static', verbose=verbose)
+
+        # finally loop through all the modules
         for module in module_store.modules[course_id].itervalues():
 
+            if module.category == 'course':
+                # we've already saved the course module up at the top of the loop
+                # so just skip over it in the inner loop
+                continue
+
+            # remap module to the new namespace
+            if target_location_namespace is not None:
+                module = remap_namespace(module, target_location_namespace)
+
+            if verbose:
+                log.debug('importing module location {0}'.format(module.location))
+
             if 'data' in module.definition:
-                store.update_item(module.location, module.definition['data'])
+                module_data = module.definition['data']
+
+                # cdodge: now go through any link references to '/static/' and make sure we've imported
+                # it as a StaticContent asset
+                try:   
+                    remap_dict = {}
+
+                    # use the rewrite_links as a utility means to enumerate through all links
+                    # in the module data. We use that to load that reference into our asset store
+                    # IMPORTANT: There appears to be a bug in lxml.rewrite_link which makes us not be able to
+                    # do the rewrites natively in that code.
+                    # For example, what I'm seeing is <img src='foo.jpg' />   ->   <img src='bar.jpg'>  
+                    # Note the dropped element closing tag. This causes the LMS to fail when rendering modules - that's
+                    # no good, so we have to do this kludge
+                    if isinstance(module_data, str) or isinstance(module_data, unicode):   # some module 'data' fields are non strings which blows up the link traversal code
+                        lxml_rewrite_links(module_data, lambda link: verify_content_links(module, course_data_path, 
+                            static_content_store, link, remap_dict))                     
+
+                        for key in remap_dict.keys():
+                            module_data = module_data.replace(key, remap_dict[key])
+
+                except Exception, e:
+                    logging.exception("failed to rewrite links on {0}. Continuing...".format(module.location))
+
+                store.update_item(module.location, module_data)
+
             if 'children' in module.definition:
                 store.update_children(module.location, module.definition['children'])
+
             # NOTE: It's important to use own_metadata here to avoid writing
             # inherited metadata everywhere.
             store.update_metadata(module.location, dict(module.own_metadata))
 
-    return module_store
+    return module_store, course_items
+
+def remap_namespace(module, target_location_namespace):
+    if target_location_namespace is None:
+        return module
+        
+    # This looks a bit wonky as we need to also change the 'name' of the imported course to be what
+    # the caller passed in
+    if module.location.category != 'course':
+        module.location = module.location._replace(tag=target_location_namespace.tag, org=target_location_namespace.org, 
+            course=target_location_namespace.course)
+    else:
+        module.location = module.location._replace(tag=target_location_namespace.tag, org=target_location_namespace.org, 
+            course=target_location_namespace.course, name=target_location_namespace.name)
+
+    # then remap children pointers since they too will be re-namespaced
+    children_locs = module.definition.get('children')
+    if children_locs is not None:
+        new_locs = []
+        for child in children_locs:
+            child_loc = Location(child)
+            new_child_loc = child_loc._replace(tag=target_location_namespace.tag, org=target_location_namespace.org, 
+                course=target_location_namespace.course)
+
+            new_locs.append(new_child_loc.url())
+
+        module.definition['children'] = new_locs    
+
+    return module
+
+def validate_category_hierarcy(module_store, course_id, parent_category, expected_child_category):
+    err_cnt = 0
+
+    parents = []
+    # get all modules of parent_category
+    for module in module_store.modules[course_id].itervalues():
+        if module.location.category == parent_category:
+            parents.append(module)
+
+    for parent in parents:
+        for child_loc in [Location(child) for child in parent.definition.get('children', [])]:
+            if child_loc.category != expected_child_category:
+                err_cnt += 1
+                print 'ERROR: child {0} of parent {1} was expected to be category of {2} but was {3}'.format(
+                    child_loc, parent.location, expected_child_category, child_loc.category)
+
+    return err_cnt
+
+def validate_data_source_path_existence(path, is_err = True, extra_msg = None):
+    _cnt = 0
+    if not os.path.exists(path):
+        print ("{0}: Expected folder at {1}. {2}".format('ERROR' if is_err == True else 'WARNING', path, extra_msg if 
+            extra_msg is not None else ''))
+        _cnt = 1
+    return _cnt
+
+def validate_data_source_paths(data_dir, course_dir):
+    # check that there is a '/static/' directory
+    course_path = data_dir / course_dir
+    err_cnt = 0
+    warn_cnt = 0
+    err_cnt += validate_data_source_path_existence(course_path / 'static')
+    warn_cnt += validate_data_source_path_existence(course_path / 'static/subs', is_err = False, 
+        extra_msg = 'Video captions (if they are used) will not work unless they are static/subs.')
+    return err_cnt, warn_cnt
+
+
+def perform_xlint(data_dir, course_dirs, 
+                    default_class='xmodule.raw_module.RawDescriptor',
+                    load_error_modules=True):
+    err_cnt = 0
+    warn_cnt = 0
+
+    module_store = XMLModuleStore(
+        data_dir,
+        default_class=default_class,
+        course_dirs=course_dirs,
+        load_error_modules=load_error_modules
+    )
+
+    # check all data source path information
+    for course_dir in course_dirs:
+        _err_cnt, _warn_cnt = validate_data_source_paths(path(data_dir), course_dir)
+        err_cnt += _err_cnt
+        warn_cnt += _warn_cnt
+
+    # first count all errors and warnings as part of the XMLModuleStore import
+    for err_log in module_store._location_errors.itervalues():
+        for err_log_entry in err_log.errors:
+            msg = err_log_entry[0]
+            if msg.startswith('ERROR:'):
+                err_cnt+=1
+            else:
+                warn_cnt+=1
+
+    # then count outright all courses that failed to load at all
+    for err_log in module_store.errored_courses.itervalues():
+        for err_log_entry in err_log.errors:
+            msg = err_log_entry[0]
+            print msg
+            if msg.startswith('ERROR:'):
+                err_cnt+=1
+            else:
+                warn_cnt+=1
+
+    for course_id in module_store.modules.keys():
+        # constrain that courses only have 'chapter' children
+        err_cnt += validate_category_hierarcy(module_store, course_id, "course", "chapter")
+        # constrain that chapters only have 'sequentials'
+        err_cnt += validate_category_hierarcy(module_store, course_id, "chapter", "sequential")
+        # constrain that sequentials only have 'verticals'
+        err_cnt += validate_category_hierarcy(module_store, course_id, "sequential", "vertical")
+
+    print "\n\n------------------------------------------\nVALIDATION SUMMARY: {0} Errors   {1} Warnings\n".format(err_cnt, warn_cnt)
+
+    if err_cnt > 0:
+        print "This course is not suitable for importing. Please fix courseware according to specifications before importing."
+    elif warn_cnt > 0:
+        print "This course can be imported, but some errors may occur during the run of the course. It is recommend that you fix your courseware before importing"
+    else:
+        print "This course can be imported successfully."
+
+       
+

common/lib/xmodule/xmodule/self_assessment_module.py

@@ -283,6 +283,7 @@ class SelfAssessmentDescriptor(XmlDescriptor, EditingDescriptor):
 
     js = {'coffee': [resource_string(__name__, 'js/src/html/edit.coffee')]}
     js_module_name = "HTMLEditingDescriptor"
+    css = {'scss': [resource_string(__name__, 'css/editor/edit.scss'), resource_string(__name__, 'css/html/edit.scss')]}
 
     @classmethod
     def definition_from_xml(cls, xml_object, system):

common/lib/xmodule/xmodule/seq_module.py

@@ -86,6 +86,7 @@ class SequenceModule(XModule):
                 'progress_status': Progress.to_js_status_str(progress),
                 'progress_detail': Progress.to_js_detail_str(progress),
                 'type': child.get_icon_class(),
+                'id': child.id,
             }
             if childinfo['title']=='':
                 childinfo['title'] = child.metadata.get('display_name','')
@@ -117,7 +118,8 @@ class SequenceDescriptor(MakoModuleDescriptor, XmlDescriptor):
 
     stores_state = True # For remembering where in the sequence the student is
 
-    template_dir_name = 'sequence'
+    js = {'coffee': [resource_string(__name__, 'js/src/sequence/edit.coffee')]}
+    js_module_name = "SequenceDescriptor"
 
     @classmethod
     def definition_from_xml(cls, xml_object, system):
@@ -125,8 +127,10 @@ class SequenceDescriptor(MakoModuleDescriptor, XmlDescriptor):
         for child in xml_object:
             try:
                 children.append(system.process_xml(etree.tostring(child, encoding='unicode')).location.url())
-            except:
+            except Exception as e:
                 log.exception("Unable to load child when parsing Sequence. Continuing...")
+                if system.error_tracker is not None:
+                    system.error_tracker("ERROR: " + str(e))
                 continue
         return {'children': children}
 

common/lib/xmodule/xmodule/static_content.py

@@ -0,0 +1,107 @@
+"""
+This module has utility functions for gathering up the static content
+that is defined by XModules and XModuleDescriptors (javascript and css)
+"""
+
+import hashlib
+import os
+import errno
+from collections import defaultdict
+
+from .x_module import XModuleDescriptor
+
+
+def write_module_styles(output_root, extra_descriptors):
+    return _write_styles('.xmodule_display', output_root, _list_modules(extra_descriptors))
+
+
+def write_module_js(output_root, extra_descriptors):
+    return _write_js(output_root, _list_modules(extra_descriptors))
+
+
+def write_descriptor_styles(output_root, extra_descriptors):
+    return _write_styles('.xmodule_edit', output_root, _list_descriptors(extra_descriptors))
+
+
+def write_descriptor_js(output_root, extra_descriptors):
+    return _write_js(output_root, _list_descriptors(extra_descriptors))
+
+
+def _list_descriptors(extra_descriptors):
+    return [
+        desc for desc in [
+            desc for (_, desc) in XModuleDescriptor.load_classes()
+        ] + extra_descriptors
+    ]
+
+
+def _list_modules(extra_descriptors):
+    return [
+        desc.module_class
+        for desc
+        in _list_descriptors(extra_descriptors)
+    ]
+
+
+def _ensure_dir(dir_):
+    try:
+        os.makedirs(dir_)
+    except OSError as exc:
+        if exc.errno == errno.EEXIST:
+            pass
+        else:
+            raise
+
+
+def _write_styles(selector, output_root, classes):
+    _ensure_dir(output_root)
+
+    css_fragments = defaultdict(set)
+    for class_ in classes:
+        class_css = class_.get_css()
+        for filetype in ('sass', 'scss', 'css'):
+            for idx, fragment in enumerate(class_css.get(filetype, [])):
+                css_fragments[idx, filetype, fragment].add(class_.__name__)
+    css_imports = defaultdict(set)
+    for (idx, filetype, fragment), classes in sorted(css_fragments.items()):
+        fragment_name = "{idx}-{hash}.{type}".format(
+            idx=idx,
+            hash=hashlib.md5(fragment).hexdigest(),
+            type=filetype)
+        # Prepend _ so that sass just includes the files into a single file
+        with open(output_root / '_' + fragment_name, 'w') as css_file:
+            css_file.write(fragment)
+
+        for class_ in classes:
+            css_imports[class_].add(fragment_name)
+
+    with open(output_root / '_module-styles.scss', 'w') as module_styles:
+        for class_, fragment_names in css_imports.items():
+            imports = "\n".join('@import "{0}";'.format(name) for name in fragment_names)
+            module_styles.write("""{selector}.xmodule_{class_} {{ {imports} }}""".format(
+                class_=class_, imports=imports, selector=selector
+            ))
+
+
+def _write_js(output_root, classes):
+    _ensure_dir(output_root)
+
+    js_fragments = set()
+    for class_ in classes:
+        module_js = class_.get_javascript()
+        for filetype in ('coffee', 'js'):
+            for idx, fragment in enumerate(module_js.get(filetype, [])):
+                js_fragments.add((idx, filetype, fragment))
+
+    module_js = []
+    for idx, filetype, fragment in sorted(js_fragments):
+        path = output_root / "{idx}-{hash}.{type}".format(
+            idx=idx,
+            hash=hashlib.md5(fragment).hexdigest(),
+            type=filetype)
+        with open(path, 'w') as js_file:
+            js_file.write(fragment)
+
+        module_js.append(path)
+
+    return module_js

common/lib/xmodule/xmodule/template_module.py

@@ -2,7 +2,8 @@ from xmodule.x_module import XModule
 from xmodule.raw_module import RawDescriptor
 from lxml import etree
 from mako.template import Template
-
+from xmodule.modulestore.django import modulestore
+import logging
 
 class CustomTagModule(XModule):
     """
@@ -38,9 +39,9 @@ class CustomTagModule(XModule):
 class CustomTagDescriptor(RawDescriptor):
     """ Descriptor for custom tags.  Loads the template when created."""
     module_class = CustomTagModule
+    template_dir_name = 'customtag'
 
-    @staticmethod
-    def render_template(system, xml_data):
+    def render_template(self, system, xml_data):
         '''Render the template, given the definition xml_data'''
         xmltree = etree.fromstring(xml_data)
         if 'impl' in xmltree.attrib:
@@ -56,15 +57,23 @@ class CustomTagDescriptor(RawDescriptor):
                                 .format(location))
 
         params = dict(xmltree.items())
-        with system.resources_fs.open('custom_tags/{name}'
-                                   .format(name=template_name)) as template:
-            return Template(template.read().decode('utf-8')).render(**params)
+
+        # cdodge: look up the template as a module
+        template_loc = self.location._replace(category='custom_tag_template', name=template_name)
+
+        template_module = self.system.load_item(template_loc)
+        template_module_data = template_module.definition['data']
+        template = Template(template_module_data)
+        return template.render(**params)
 
 
     def __init__(self, system, definition, **kwargs):
         '''Render and save the template for this descriptor instance'''
         super(CustomTagDescriptor, self).__init__(system, definition, **kwargs)
-        self.rendered_html = self.render_template(system, definition['data'])
+
+    @property
+    def rendered_html(self):
+        return self.render_template(self.system, self.definition['data'])
 
     def export_to_file(self):
         """

common/lib/xmodule/xmodule/templates.py

@@ -65,8 +65,15 @@ def update_templates():
             template_location = Location('i4x', 'edx', 'templates', category, Location.clean_for_url_name(template.metadata['display_name']))
 
             try:
-                json_data = template._asdict()
+                json_data = {
+                    'definition': {
+                        'data': template.data,
+                        'children': template.children
+                    },
+                    'metadata': template.metadata
+                }
                 json_data['location'] = template_location.dict()
+
                 XModuleDescriptor.load_from_json(json_data, TemplateTestSystem())
             except:
                 log.warning('Unable to instantiate {cat} from template {template}, skipping'.format(
@@ -75,6 +82,6 @@ def update_templates():
                 ), exc_info=True)
                 continue
 
-            modulestore().update_item(template_location, template.data)
-            modulestore().update_children(template_location, template.children)
-            modulestore().update_metadata(template_location, template.metadata)
+            modulestore('direct').update_item(template_location, template.data)
+            modulestore('direct').update_children(template_location, template.children)
+            modulestore('direct').update_metadata(template_location, template.metadata)

common/lib/xmodule/xmodule/templates/about/empty.yaml

@@ -0,0 +1,5 @@
+---
+metadata:
+    display_name: Empty
+data: "<p>This is where you can add additional information about your course.</p>"
+children: []

common/lib/xmodule/xmodule/templates/course/empty.yaml

@@ -0,0 +1,6 @@
+---
+metadata:
+    display_name: Empty
+    start: 2020-10-10T10:00
+data: { 'textbooks' : [ ], 'wiki_slug' : null }
+children: []

common/lib/xmodule/xmodule/templates/courseinfo/empty.yaml

@@ -0,0 +1,5 @@
+---
+metadata:
+    display_name: Empty
+data: "<ol></ol>"
+children: []

common/lib/xmodule/xmodule/templates/discussion/default.yaml

@@ -1,6 +1,9 @@
 ---
 metadata:
     display_name: Discussion Tag
+    for: Topic-Level Student-Visible Label 
+    id: 6002x_group_discussion_by_this 
+    discussion_category: Week 1
 data: |
   <discussion for="Topic-Level Student-Visible Label" id="6002x_group_discussion_by_this" discussion_category="Week 1" />
 children: []

common/lib/xmodule/xmodule/templates/html/announcement.yaml

@@ -9,7 +9,7 @@ data: |
       <section class='update-description'>
       	       <section class='primary'>
       	       		<p> Words of encouragement! This is a short note that most students will read. </p>
-      	       		<p class='author'>&mdash; Anant  Agarwal (6.002x Principle Instructor)</p>
+      	       		<p class='author'>Anant  Agarwal (6.002x Principle Instructor)</p>
       	       </section>		  
       	       <p><h3>Primary versus Secondary Updates:</h3> Unfortunately, the internet throws a lot of text at students, and they 
       	       do not read everything that they are given. However, many students <em>do</em> read all that they are

common/lib/xmodule/xmodule/templates/html/empty.yaml

@@ -0,0 +1,7 @@
+---
+metadata:
+      display_name: Empty
+
+data: |
+
+children: []

common/lib/xmodule/xmodule/templates/html/latex_html.yaml

@@ -0,0 +1,23 @@
+---
+metadata:
+    display_name: E-text Written in LaTeX 
+    source_processor_url: https://qisx.mit.edu:5443/latex2edx
+    source_code: |
+        \subsection{Example of E-text in LaTeX}
+
+        It is very convenient to write complex equations in LaTeX.        
+
+        \begin{equation}
+                x = \frac{-b\pm\sqrt{b^2-4*a*c}}{2a}
+        \end{equation}
+
+        Seize the moment.
+
+data: |
+    <html>
+        <h2>Example: E-text page</h2>
+        <p>
+          It is very convenient to write complex equations in LaTeX.
+        </p>
+    </html>
+children: []

common/lib/xmodule/xmodule/templates/problem/circuitschematic.yaml

@@ -1,131 +1,62 @@
 ---
 metadata:
     display_name: Circuit Schematic
+    rerandomize: never
+    showanswer: always
 data: |
- <!-- Logic gate: cjt 2/13/12 -->
-    
-    <problem>
-      <startouttext />
-      Your goal for this lab is to design a circuit that implements a
-      <!-- \overline doesn't seem to render correctly -->
-      3-input logic gate that implements \(Z = \lnot{(C(A+B))}\) where the
-      \(\lnot\) symbol stands for logical negation. This function is
-      enumerated in the following truth table:
-      
-      <center><pre>
-      C B A | Z
-      =========
-      0 0 0 | 1
-      0 0 1 | 1
-      0 1 0 | 1
-      0 1 1 | 1
-      1 0 0 | 1
-      1 0 1 | 0
-      1 1 0 | 0
-      1 1 1 | 0
-      </pre></center>
-      
-      The schematic diagram below includes the resistive pullup for the logic
-      gate and some voltage sources that serve as the power supply and 
-      generators for the signals that will be the inputs to the gate.
-      The voltage sources generate the three input signals (A, B and C), timed so that all
-      possible combinations of the inputs will be generated over a \(4\mu s\)
-      interval.
-      
-      <br/><br/>Please add the appropriate pulldown network of mosfet
-      switches connected to node Z to implement the truth table above, with
-      \(R_{ON}\) of the mosfets chosen so that \(V_{ol}\) of the logic gate is less than
-      \(0.25V\) for any combination of inputs.  In the schematic tool, the mosfet
-      model has \(V_{th} = 0.5V\), so \(V_{ol} \lt 0.25V\)
-      will ensure that when the output of the logic gate is 0, if it is used
-      as the input to some other logic gate, the mosfet to which
-      it connects will be off.
-      
-      <br/><br/>On <A href="/book-shifted/305">page 305</A> of the text we
-      see from Equation (6) that \(R_{ON} = R_n \frac{L}{W}\).  In the
-      schematic tool, the mosfet model has \(R_n \approx 26.5k\Omega\) when
-      using a \(3V\) power supply.  To adjust \(R_{ON}\), double click the
-      mosfet and select an appropriate value for the W/L parameter.  For
-      example, setting a mosfet's W/L to 10 would result in \(R_{ON} =
-      2.65k\Omega\).
-      
-      <br/><br/>Note that the "Plot offset" property of the scope probes
-      on the A, B and C signals has been set so that the plots will not
-      overlap, making it easier to see what's happening.
-      
-      <br/><br/>Please do not change the voltages of the voltage sources or the
-      resistance of the pullup resistor.
-      
-      <endouttext />
-      
-      <schematicresponse>
-    <center>
-    <schematic height="500" width="600" parts="g,n,s" analyses="dc,tran"
-        submit_analyses="{&quot;tran&quot;:[[&quot;Z&quot;,0.0000004,0.0000009,0.0000014,0.0000019,0.0000024,0.0000029,0.0000034,0.000039]]}"
-        initial_value="[[&quot;w&quot;,[112,96,128,96]],[&quot;w&quot;,[256,96,240,96]],[&quot;w&quot;,[192,96,240,96]],[&quot;s&quot;,[240,96,0],{&quot;color&quot;:&quot;cyan&quot;,&quot;offset&quot;:&quot;&quot;,&quot;plot offset&quot;:&quot;0&quot;,&quot;_json_&quot;:3},[&quot;Z&quot;]],[&quot;w&quot;,[32,224,192,224]],[&quot;w&quot;,[96,48,192,48]],[&quot;L&quot;,[256,96,3],{&quot;label&quot;:&quot;Z&quot;,&quot;_json_&quot;:6},[&quot;Z&quot;]],[&quot;r&quot;,[192,48,0],{&quot;name&quot;:&quot;Rpullup&quot;,&quot;r&quot;:&quot;10K&quot;,&quot;_json_&quot;:7},[&quot;1&quot;,&quot;Z&quot;]],[&quot;w&quot;,[32,144,32,192]],[&quot;w&quot;,[32,224,32,192]],[&quot;w&quot;,[48,192,32,192]],[&quot;w&quot;,[32,96,32,144]],[&quot;w&quot;,[48,144,32,144]],[&quot;w&quot;,[32,48,32,96]],[&quot;w&quot;,[48,96,32,96]],[&quot;w&quot;,[32,48,48,48]],[&quot;g&quot;,[32,224,0],{&quot;_json_&quot;:16},[&quot;0&quot;]],[&quot;v&quot;,[96,192,1],{&quot;name&quot;:&quot;VC&quot;,&quot;value&quot;:&quot;square(3,0,250K)&quot;,&quot;_json_&quot;:17},[&quot;C&quot;,&quot;0&quot;]],[&quot;v&quot;,[96,144,1],{&quot;name&quot;:&quot;VB&quot;,&quot;value&quot;:&quot;square(3,0,500K)&quot;,&quot;_json_&quot;:18},[&quot;B&quot;,&quot;0&quot;]],[&quot;v&quot;,[96,96,1],{&quot;name&quot;:&quot;VA&quot;,&quot;value&quot;:&quot;square(3,0,1000K)&quot;,&quot;_json_&quot;:19},[&quot;A&quot;,&quot;0&quot;]],[&quot;v&quot;,[96,48,1],{&quot;name&quot;:&quot;Vpwr&quot;,&quot;value&quot;:&quot;dc(3)&quot;,&quot;_json_&quot;:20},[&quot;1&quot;,&quot;0&quot;]],[&quot;L&quot;,[96,96,2],{&quot;label&quot;:&quot;A&quot;,&quot;_json_&quot;:21},[&quot;A&quot;]],[&quot;w&quot;,[96,96,104,96]],[&quot;L&quot;,[96,144,2],{&quot;label&quot;:&quot;B&quot;,&quot;_json_&quot;:23},[&quot;B&quot;]],[&quot;w&quot;,[96,144,104,144]],[&quot;L&quot;,[96,192,2],{&quot;label&quot;:&quot;C&quot;,&quot;_json_&quot;:25},[&quot;C&quot;]],[&quot;w&quot;,[96,192,104,192]],[&quot;w&quot;,[192,96,192,112]],[&quot;s&quot;,[112,96,0],{&quot;color&quot;:&quot;red&quot;,&quot;offset&quot;:&quot;15&quot;,&quot;plot offset&quot;:&quot;0&quot;,&quot;_json_&quot;:28},[&quot;A&quot;]],[&quot;w&quot;,[104,96,112,96]],[&quot;s&quot;,[112,144,0],{&quot;color&quot;:&quot;green&quot;,&quot;offset&quot;:&quot;10&quot;,&quot;plot offset&quot;:&quot;0&quot;,&quot;_json_&quot;:30},[&quot;B&quot;]],[&quot;w&quot;,[104,144,112,144]],[&quot;w&quot;,[128,144,112,144]],[&quot;s&quot;,[112,192,0],{&quot;color&quot;:&quot;blue&quot;,&quot;offset&quot;:&quot;5&quot;,&quot;plot offset&quot;:&quot;0&quot;,&quot;_json_&quot;:33},[&quot;C&quot;]],[&quot;w&quot;,[104,192,112,192]],[&quot;w&quot;,[128,192,112,192]],[&quot;view&quot;,0,0,2,&quot;5&quot;,&quot;10&quot;,&quot;10MEG&quot;,null,&quot;100&quot;,&quot;4us&quot;]]"
-       />
-    </center>
- <answer type="loncapa/python">
-   # for a schematic response, submission[i] is the json representation
-   # of the diagram and analysis results for the i-th schematic tag
-   
-   def get_tran(json,signal):
-     for element in json:
-       if element[0] == 'transient':
-         return element[1].get(signal,[])
-     return []
-   
-   def get_value(at,output):
-     for (t,v) in output:
-       if at == t: return v
-     return None
-   
-   output = get_tran(submission[0],'Z')
-   okay = True
-   
-   # output should be 1, 1, 1, 1, 1, 0, 0, 0
-   if get_value(0.0000004,output) &lt; 2.7: okay = False;
-   if get_value(0.0000009,output) &lt; 2.7: okay = False;
-   if get_value(0.0000014,output) &lt; 2.7: okay = False;
-   if get_value(0.0000019,output) &lt; 2.7: okay = False;
-   if get_value(0.0000024,output) &lt; 2.7: okay = False;
-   if get_value(0.0000029,output) &gt; 0.25: okay = False;
-   if get_value(0.0000034,output) &gt; 0.25: okay = False;
-   if get_value(0.0000039,output) &gt; 0.25: okay = False;
-   
-   correct = ['correct' if okay else 'incorrect']
-   
- </answer>
-      </schematicresponse>
-      
-      <startouttext />
-      
-      When your circuit is ready for testing, run a \(4\mu s\) transient
-      simulation to verify correct functionality and appropriate \(V_{ol}\)
-      when the output of the gate is logic 0.  To submit, please click
-      CHECK.  The checker will be verifying the voltage of the
-      output node at several different times, so you'll earn a checkmark
-      only <i>after</i> you've performed the transient simulation so that
-      the checker will have a waveform to check!
-      
-      <br/><br/>Hint: you'll only need 3 mosfet switches to implement the gate.
-      
-      <br/><br/>When the gate is correctly implemented, the plot produced by the transient
-      analysis should like similar to the following figure.
-      
-      <center>
- <img src="/static/Lab3_1.png"/>
- <br/>Figure 1.  Example plot output
-      </center>
-      
-      <br/><br/>Food for thought: You'll notice there are little spikes,
-      sometimes called <i>glitches</i>, in the output waveform (see the
-      bottom cyan-colored waveform in Figure 1).  These only occur when the
-      A and B inputs are changing simultaneously and the C input is high.
-      Can you explain why?  Think about what is happening in the pulldown
-      circuitry at the time the glitches occur.
-      
-      <endouttext />
-      
-    </problem>
+  <problem >
+    Please make a voltage divider that splits the provided voltage evenly.
+
+  <schematicresponse>
+  <center>
+  <schematic height="500" width="600" parts="g,r" analyses="dc"
+  initial_value="[[&quot;v&quot;,[168,144,0],{&quot;value&quot;:&quot;dc(1)&quot;,&quot;_json_&quot;:0},[&quot;1&quot;,&quot;0&quot;]],[&quot;r&quot;,[296,120,0],{&quot;r&quot;:&quot;1&quot;,&quot;_json_&quot;:1},[&quot;1&quot;,&quot;output&quot;]],[&quot;L&quot;,[296,168,3],{&quot;label&quot;:&quot;output&quot;,&quot;_json_&quot;:2},[&quot;output&quot;]],[&quot;w&quot;,[296,216,168,216]],[&quot;w&quot;,[168,216,168,192]],[&quot;w&quot;,[168,144,168,120]],[&quot;w&quot;,[168,120,296,120]],[&quot;g&quot;,[168,216,0],{&quot;_json_&quot;:7},[&quot;0&quot;]],[&quot;view&quot;,-67.49999999999994,-78.49999999999994,1.6000000000000003,&quot;50&quot;,&quot;10&quot;,&quot;1G&quot;,null,&quot;100&quot;,&quot;1&quot;,&quot;1000&quot;]]"
+  />
+  </center>
+  <answer type="loncapa/python">
+  dc_value = "dc analysis not found"
+  for response in submission[0]:
+    if response[0] == 'dc':
+        for node in response[1:]:
+            dc_value = node['output']
+
+  if dc_value == .5:
+    correct = ['correct']
+  else:
+    correct = ['incorrect']
+  </answer>
+  </schematicresponse>
+  <schematicresponse>
+  <p>Make a high pass filter</p>
+  <center>
+  <schematic height="500" width="600" parts="g,r,s,c" analyses="ac"
+  submit_analyses="{&quot;ac&quot;:[[&quot;NodeA&quot;,1,9]]}"
+  initial_value="[[&quot;v&quot;,[160,152,0],{&quot;name&quot;:&quot;v1&quot;,&quot;value&quot;:&quot;sin(0,1,1,0,0)&quot;,&quot;_json_&quot;:0},[&quot;1&quot;,&quot;0&quot;]],[&quot;w&quot;,[160,200,240,200]],[&quot;g&quot;,[160,200,0],{&quot;_json_&quot;:2},[&quot;0&quot;]],[&quot;L&quot;,[240,152,3],{&quot;label&quot;:&quot;NodeA&quot;,&quot;_json_&quot;:3},[&quot;NodeA&quot;]],[&quot;s&quot;,[240,152,0],{&quot;color&quot;:&quot;cyan&quot;,&quot;offset&quot;:&quot;0&quot;,&quot;_json_&quot;:4},[&quot;NodeA&quot;]],[&quot;view&quot;,64.55878906250004,54.114697265625054,2.5000000000000004,&quot;50&quot;,&quot;10&quot;,&quot;1G&quot;,null,&quot;100&quot;,&quot;1&quot;,&quot;1000&quot;]]"/>
+  </center>
+  <answer type="loncapa/python">
+  ac_values = None
+  for response in submission[0]:
+    if response[0] == 'ac':
+        for node in response[1:]:
+            ac_values = node['NodeA']
+  print "the ac analysis value:", ac_values
+  if ac_values == None:
+    correct = ['incorrect']
+  elif ac_values[0][1] &lt; ac_values[1][1]:
+    correct = ['correct']
+  else:
+    correct = ['incorrect']
+  </answer>
+  </schematicresponse>
+
+      <solution>
+          <div class="detailed-solution">
+              <p>Explanation</p>
+              <p>A voltage divider that evenly divides the input voltage can be formed with two identically valued resistors, with the sampled voltage taken in between the two.</p>
+              <p><img src="/static/images/voltage_divider.png"/></p>
+              <p>A simple high-pass filter without any further constaints can be formed by simply putting a resister in series with a capacitor. The actual values of the components do not really matter in order to meet the constraints of the problem.</p>
+              <p><img src="/static/images/high_pass_filter.png"/></p>
+          </div>
+      </solution>
+  </problem>
 children: []

common/lib/xmodule/xmodule/templates/problem/customgrader.yaml

@@ -1,31 +1,49 @@
 ---
 metadata:
     display_name: Custom Grader
+    rerandomize: never
+    showanswer: always
 data: |
- <problem>
- <text>
-   <h2>Example: Custom Response Problem</h2>
-   
-   <p>
-     A custom response problem accepts one or more lines of text input from the
-     student, and evaluates the inputs for correctness based on evaluation using a 
-     python script embedded within the problem.
-   </p>
-   
-   <script type="loncapa/python">def test_add(expect,ans):
-        (a1,a2) = map(float,ans)
-        return (a1+a2)==10
-     
-   </script>
-   
-   <text>
-     Enter two integers which sum to 10: <br/>
-     <customresponse cfn="test_add">
-              <textline size="40" correct_answer="3"/><br/>
-              <textline size="40" correct_answer="7"/>
-     </customresponse>
-   </text>
-   
- </text>
- </problem>
+  <problem>
+  <p>
+  A custom response problem accepts one or more lines of text input from the
+  student, and evaluates the inputs for correctness based on evaluation using a 
+  python script embedded within the problem.
+  </p>
+
+  <script type="loncapa/python">
+
+  def test_add_to_ten(expect,ans):
+    a1=float(ans[0])
+    a2=float(ans[1])
+    return (a1+a2)==10
+
+  def test_add(expect,ans):
+    a1=float(ans[0])
+    a2=float(ans[1])
+    return (a1+a2)== float(expect)
+
+  </script>
+
+  <p>Enter two integers which sum to 10: </p>
+  <customresponse cfn="test_add_to_ten">
+          <textline size="40" correct_answer="3"/><br/>
+          <textline size="40" correct_answer="7"/>
+  </customresponse>
+
+  <p>Enter two integers which sum to 20: </p>
+  <customresponse cfn="test_add" expect="20">
+          <textline size="40" correct_answer="11"/><br/>
+          <textline size="40" correct_answer="9"/>
+  </customresponse>
+
+      <solution>
+          <div class="detailed-solution">
+              <p>Explanation</p>
+              <p>Any set of values on the line \(y = 10 - x\) and \(y = 20 - x\) satisfy these constraints.</p>
+              <img src="/static/images/simple_graph.png"/>
+          </div>
+      </solution>
+  </problem>
+
 children: []

common/lib/xmodule/xmodule/templates/problem/empty.yaml

@@ -0,0 +1,11 @@
+---
+metadata:
+    display_name: Empty
+    rerandomize: never
+    showanswer: always
+    markdown: ""
+data: |
+  <problem>
+  </problem>
+
+children: []

common/lib/xmodule/xmodule/templates/problem/forumularesponse.yaml

@@ -1,38 +1,47 @@
 ---
 metadata:
-    display_name: Formula Repsonse
+    display_name: Formula Response
+    rerandomize: never
+    showanswer: always
 data: |
- <problem>
- <text>
-   <h2>Example: Formula Response Problem</h2>
-   
-   <p>
-     A formula response problem accepts a line of text input from the
-     student, and evaluates the input for correctness based on numerical sampling of 
-     the symbolic formula which is given.
-   </p>
-   
-   <p>
-     The answer is correct if it is within a specified numerical tolerance
-     of the expected answer.
-   </p>
-   
-   <p>This kind of response checking can handle symbolic expressions, but places an extra burden
-     on the problem author to specify the allowed variables in the expression, and the
-     numerical ranges over which the variables must be sampled to test for correctness.</p>
-   
-   <script type="loncapa/python">I = "m*c^2"</script>
-   
-   <text>
-     <br/>
-     Give an equation for the relativistic energy of an object with mass m.  Explicitly indicate multiplication with a <tt>*</tt> symbol.<br/>
-   </text>
-   <formularesponse type="cs" samples="m,c@1,2:3,4#10" answer="$I">
-         <responseparam description="Numerical Tolerance" type="tolerance"
-      default="0.00001" name="tol" /> 
-         <br/><text>E =</text> <textline size="40" math="1" />    
-          </formularesponse>
-   
- </text>
- </problem>
+  <problem>
+  <p>
+  A formula response problem accepts a line of text representing a mathematical expression from the
+  student, and evaluates the input for equivalence to a mathematical expression provided by the 
+  grader. Correctness is based on numerical sampling of the symbolic expressions.
+  </p>
+  <p>
+  The answer is correct if both the student provided response and the grader's mathematical
+  expression are equivalent to specified numerical tolerance, over a specified range of values for each
+  variable.
+  </p>
+
+  <p>This kind of response checking can handle symbolic expressions, but places an extra burden
+  on the problem author to specify the allowed variables in the expression, and the
+  numerical ranges over which the variables must be sampled in order to test for correctness.</p>
+
+  <script type="loncapa/python">
+  VoVi = "(R_1*R_2)/R_3"
+  </script>
+  <p>Give an equation for the relativistic energy of an object with mass m.  Explicitly indicate multiplication with a <tt>*</tt> symbol.</p>
+  <formularesponse type="cs" samples="m,c@1,2:3,4#10" answer="m*c^2">
+      <responseparam type="tolerance" default="0.00001"/> 
+      <br/><text>E =</text> <textline size="40" math="1" />    
+  </formularesponse>
+
+  <p>The answer to this question is (R_1*R_2)/R_3. </p>
+
+  <formularesponse type="ci" samples="R_1,R_2,R_3@1,2,3:3,4,5#10" answer="$VoVi">
+      <responseparam type="tolerance" default="0.00001"/> 
+      <textline size="40" math="1" />    
+  </formularesponse>
+      <solution>
+          <div class="detailed-solution">
+              <p>Explanation</p>
+              <p>The mathematical summary of many of the theory of relativity developed by Einstein is that the amount of energy contained in a mass m is the mass time the speed of light squared.</p>
+              <p>As you can see with the formula entry, the answer is \(\frac{R_1*R_2}{R_3}\)</p>
+          </div>
+      </solution>
+  </problem>
+
 children: []

common/lib/xmodule/xmodule/templates/problem/imageresponse.yaml

@@ -1,26 +1,26 @@
 ---
 metadata:
     display_name: Image Response
+    rerandomize: never
+    showanswer: always
 data: |
- <problem>
-      <text>
- <h2>Example: Image Response Problem</h2>
- 
- <p>
-   When teaching conventionally, a common check for understanding is to ask the student to point 
-   at something which satisfies a set of contraints. This use case is captured in the imageresponse. 
-   An image response problem presents an image for the student.  Input is
-   given by the location of mouse clicks on the image. Correctness of input can only be evaluated based on expected dimensions of a rectangle.
-   
- </p>
- 
- <text>
-   Click on the cow in this image:
-   <imageresponse>
-     <imageinput src="/static/cow.png" width="715" height="511" rectangle="(404,150)-(715,480)" />
-   </imageresponse>
- </text>
-      </text>
- </problem>
+  <problem>
+  <p>
+  An image response problem presents an image for the student.  Input is
+  given by the location of mouse clicks on the image. Correctness of input can be evaluated based on expected dimensions of a rectangle.
+  </p>
+
+  <p>Which object in this image is required by the fire code?</p>
+      <imageresponse>
+        <imageinput src="/static/images/firecode.jpg" width="640" height="480" rectangle="(365,42)-(423,66)" />
+      </imageresponse>
+      <solution>
+          <div class="detailed-solution">
+              <p>Explanation</p>
+              <p>The fire code requires that all exits be clearly marked, so the red exit sign is the correct answer.</p>
+          </div>
+      </solution>
+  </problem>
+
     
 children: []

common/lib/xmodule/xmodule/templates/problem/latex_problem.yaml

@@ -0,0 +1,218 @@
+---
+metadata:
+    display_name: Problem Written in LaTeX 
+    source_processor_url: https://qisx.mit.edu:5443/latex2edx
+    source_code: |
+        % Nearly any kind of edX problem can be authored using Latex as
+        % the source language.  Write latex as usual, including equations.  The
+        % key new feature is the \edXabox{} macro, which specifies an "Answer
+        % Box" that queries students for a response, and specifies what the
+        % epxected (correct) answer is.
+        
+        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+        \subsection{Example "option" problem}
+        
+        Where is the earth?
+        
+        \edXabox{options='up','down' expect='down'}
+        
+        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+        \subsection{Example "symbolic" problem}
+        
+        What is Einstein's equation for the energy equivalent of a mass $m$?
+        
+        \edXabox{type='symbolic' size='90' expect='m*c^2' }
+        
+        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+        \subsection{Example "numerical" problem}
+        
+        Estimate the energy savings (in J/y) if all the people
+        ($3\times 10^8$) in the U.~S. switched from U.~S. code to low flow
+        shower heads.
+        
+        \edXinline{Energy saved = }\edXabox{expect="0.52" type="numerical" tolerance='0.02' inline='1' } %
+        \edXinline{~EJ/year}
+        
+        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+        \subsection{Example "multiple choice" problem}
+        
+        What color is a banana?
+        
+        \edXabox{ type="multichoice" expect="Yellow" options="Red","Green","Yellow","Blue" }
+        
+        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+        \subsection{Example "string response" problem}
+        
+        In what U.S. state is Detroit located?
+        
+        \edXabox{ type="string" expect="Michigan" options="ci" }
+        
+        An explanation of the answer can be provided by using the edXsolution
+        macro.  Click on "Show Answer" to see the solution.
+        
+        \begin{edXsolution}
+        Detroit is near Canada, but it is actually in the United States.
+        \end{edXsolution}
+                
+        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+        \subsection{Example "custom response" problem}
+        
+        This problem demonstrates the use of a custom python script used for
+        checking the answer.
+        
+        \begin{edXscript}
+        def sumtest(expect,ans):
+            (a1,a2) = map(float,eval(ans))
+            return (a1+a2)==10
+        \end{edXscript}
+        
+        Enter a python list of two numbers which sum to 10, eg [9,1]:
+        
+        \edXabox{expect="[1,9]" type="custom" cfn="sumtest"}
+        
+        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+        \subsection{Example image}
+        
+        Include image by using the edXxml macro:
+        
+        \edXxml{<img src="http://autoid.mit.edu/images/mit_dome.jpg"/>}
+        
+        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+        \subsection{Example show/hide explanation}
+        
+        Extra explanations can be tucked away behind a "showhide" toggle flag:
+        
+        \edXshowhide{sh1}{More explanation}{This is a hidden explanation.  It
+        can contain equations: $\alpha = \frac{2}{\sqrt{1+\gamma}}$ }
+        
+        This is some text after the showhide example.
+
+data: |
+    <?xml version="1.0"?>
+    <problem>
+      <text>
+        <p>
+          <h4>Example "option" problem</h4>
+        </p>
+        <p>
+    Where is the earth? </p>
+        <p>
+          <optionresponse>
+            <optioninput options="('up','down')" correct="down"/>
+          </optionresponse>
+        </p>
+        <p>
+          <h4>Example "symbolic" problem</h4>
+        </p>
+        <p>
+    What is Einstein's equation for the energy equivalent of a mass [mathjaxinline]m[/mathjaxinline]? </p>
+        <p>
+          <symbolicresponse expect="m*c^2">
+            <textline size="90" correct_answer="m*c^2" math="1"/>
+          </symbolicresponse>
+        </p>
+        <p>
+          <h4>Example "numerical" problem</h4>
+        </p>
+        <p>
+    Estimate the energy savings (in J/y) if all the people ([mathjaxinline]3\times 10^8[/mathjaxinline]) in the U.&#xA0;S. switched from U.&#xA0;S. code to low flow shower heads. </p>
+        <p>
+          <p style="display:inline">Energy saved = </p>
+          <numericalresponse inline="1" answer="0.52">
+            <textline inline="1">
+              <responseparam description="Numerical Tolerance" type="tolerance" default="0.02" name="tol"/>
+            </textline>
+          </numericalresponse>
+          <p style="display:inline">&#xA0;EJ/year</p>
+        </p>
+        <p>
+          <h4>Example "multiple choice" problem</h4>
+        </p>
+        <p>
+    What color is a banana? </p>
+        <p>
+          <choiceresponse>
+            <checkboxgroup>
+              <choice correct="false" name="1">
+                <text>Red</text>
+              </choice>
+              <choice correct="false" name="2">
+                <text>Green</text>
+              </choice>
+              <choice correct="true" name="3">
+                <text>Yellow</text>
+              </choice>
+              <choice correct="false" name="4">
+                <text>Blue</text>
+              </choice>
+            </checkboxgroup>
+          </choiceresponse>
+        </p>
+        <p>
+          <h4>Example "string response" problem</h4>
+        </p>
+        <p>
+    In what U.S. state is Detroit located? </p>
+        <p>
+          <stringresponse answer="Michigan">
+            <textline/>
+          </stringresponse>
+        </p>
+        <p>
+    An explanation of the answer can be provided by using the edXsolution macro: </p>
+        <p>
+          <solution>
+            <font color="blue">Answer: </font>
+            <font color="blue">Detroit is near Canada, but it is actually in the United States. </font>
+          </solution>
+        </p>
+        <p>
+          <h4>Example "custom response" problem</h4>
+        </p>
+        <p>
+    This problem demonstrates the use of a custom python script used for checking the answer. </p>
+        <script type="text/python" system_path="python_lib">
+    def sumtest(expect,ans):
+        (a1,a2) = map(float,eval(ans))
+        return (a1+a2)==10
+    </script>
+        <p>
+    Enter a python list of two numbers which sum to 10, eg [9,1]: </p>
+        <p>
+          <customresponse cfn="sumtest" expect="[1,9]">
+            <textline correct_answer="[1,9]"/>
+          </customresponse>
+        </p>
+        <p>
+          <h4>Example image</h4>
+        </p>
+        <p>
+    Include image by using the edXxml macro: </p>
+        <p>
+          <img src="http://autoid.mit.edu/images/mit_dome.jpg"/>
+        </p>
+        <p>
+          <h4>Example show/hide explanation</h4>
+        </p>
+        <p>
+    Extra explanations can be tucked away behind a "showhide" toggle flag: </p>
+        <p>
+          <table class="wikitable collapsible collapsed">
+            <tbody>
+              <tr>
+                <th> More explanation [<a href="javascript:$('#sh1').toggle()" id="sh1l">show</a>]</th>
+              </tr>
+              <tr id="sh1" style="display:none">
+                <td>
+                  <p>
+    This is a hidden explanation. It can contain equations: [mathjaxinline]\alpha = \frac{2}{\sqrt {1+\gamma }}[/mathjaxinline]  </p>
+                  <p>
+    This is some text after the showhide example. </p>
+                </td>
+              </tr>
+            </tbody>
+          </table>
+        </p>
+      </text>
+    </problem>
+children: []

common/lib/xmodule/xmodule/templates/problem/multiline.yaml

@@ -1,7 +0,0 @@
----
-metadata:
-    display_name: Multiline XML
-data: |
-    <problem>
-    </problem>
-children: []

common/lib/xmodule/xmodule/templates/problem/multiplechoice.yaml

@@ -1,27 +1,63 @@
 ---
 metadata:
     display_name: Multiple Choice
+    rerandomize: never
+    showanswer: always
+    markdown: 
+       "A multiple choice problem presents radio buttons for student input. Students can only select a single 
+       option presented. Multiple Choice questions have been the subject of many areas of research due to the early 
+       invention and adoption of bubble sheets.
+       
+       
+       One of the main elements that goes into a good multiple choice question is the existence of good distractors. 
+       That is, each of the alternate responses presented to the student should be the result of a plausible mistake 
+       that a student might make.
+        
+    
+       What Apple device competed with the portable CD player?
+       
+            ( ) The iPad
+            
+            ( ) Napster
+            
+            (x) The iPod
+            
+            ( ) The vegetable peeler
+            
+            ( ) Android
+            
+            ( ) The Beatles
+            
+            
+       [explanation]
+       The release of the iPod allowed consumers to carry their entire music library with them in a 
+       format that did not rely on fragile and energy-intensive spinning disks.
+       [explanation]
+       "
 data: |
- <problem>
-      <text>
- <h2>Example: Multiple Choice Response Problem</h2>
- 
- <p>
-   A multiple choice response problem presents radio buttons for student 
-   input.  <!-->One or more of the choice may be correct.-->  Correctness of
-   input is evaluated based on expected answers specified within each
-   "choice" stanza.
- </p>
- 
- <p>Select the correct choice.  Grass is:</p>
- <multiplechoiceresponse direction="vertical" randomize="yes">
-   <choicegroup type="MultipleChoice">
-            <choice location="random" correct="false" name="red">Red</choice>
-            <choice location="random" correct="true" name="green">Green</choice>
-            <choice location="random" correct="false" name="yellow">Yellow</choice>
-            <choice location="bottom" correct="false" name="blue">Blue</choice>
-   </choicegroup>
- </multiplechoiceresponse>
-      </text>
- </problem>
+   <problem>
+   <p>
+   A multiple choice problem presents radio buttons for student
+   input. Students can only select a single option presented. Multiple Choice questions have been the subject of many areas of research due to the early invention and adoption of bubble sheets.</p>
+   <p> One of the main elements that goes into a good multiple choice question is the existence of good distractors. That is, each of the alternate responses presented to the student should be the result of a plausible mistake that a student might make. 
+   </p>
+
+   <p>What Apple device competed with the portable CD player?</p>
+       <multiplechoiceresponse>
+        <choicegroup type="MultipleChoice">
+           <choice correct="false" name="ipad">The iPad</choice>
+           <choice correct="false" name="beatles">Napster</choice>
+           <choice correct="true" name="ipod">The iPod</choice>
+           <choice correct="false" name="peeler">The vegetable peeler</choice>
+           <choice correct="false" name="android">Android</choice>
+           <choice correct="false" name="beatles">The Beatles</choice>
+        </choicegroup>
+       </multiplechoiceresponse>
+       <solution>
+           <div class="detailed-solution">
+               <p>Explanation</p>
+               <p>The release of the iPod allowed consumers to carry their entire music library with them in a format that did not rely on fragile and energy-intensive spinning disks. </p>
+           </div>
+       </solution>
+   </problem>
 children: []