Merge branch 'master' into ihoover/feature_flag_auto_auth

cms/djangoapps/contentstore/features/checklists.feature

@@ -10,6 +10,7 @@ Feature: Course checklists
     Then I can check and uncheck tasks in a checklist
     And They are correctly selected after I reload the page
 
+  @skip
   Scenario: A task can link to a location within Studio
     Given I have opened Checklists
     When I select a link to the course outline

cms/djangoapps/contentstore/features/common.py

@@ -209,7 +209,8 @@ def i_created_a_video_component(step):
     world.create_component_instance(
         step, '.large-video-icon',
         'video',
-        '.xmodule_VideoModule'
+        '.xmodule_VideoModule',
+        has_multiple_templates=False
     )
 
 

cms/djangoapps/contentstore/features/component_settings_editor_helpers.py

@@ -7,10 +7,16 @@ from terrain.steps import reload_the_page
 
 
 @world.absorb
-def create_component_instance(step, component_button_css, category, expected_css, boilerplate=None):
-    click_new_component_button(step, component_button_css)
-    click_component_from_menu(category, boilerplate, expected_css)
+def create_component_instance(step, component_button_css, category,
+                              expected_css, boilerplate=None,
+                              has_multiple_templates=True):
 
+    click_new_component_button(step, component_button_css)
+
+    if has_multiple_templates:
+        click_component_from_menu(category, boilerplate, expected_css)
+
+    assert_equal(1, len(world.css_find(expected_css)))
 
 @world.absorb
 def click_new_component_button(step, component_button_css):
@@ -34,7 +40,6 @@ def click_component_from_menu(category, boilerplate, expected_css):
     elements = world.css_find(elem_css)
     assert_equal(len(elements), 1)
     world.css_click(elem_css)
-    assert_equal(1, len(world.css_find(expected_css)))
 
 
 @world.absorb

cms/djangoapps/contentstore/features/discussion-editor.py

@@ -9,7 +9,8 @@ def i_created_discussion_tag(step):
     world.create_component_instance(
         step, '.large-discussion-icon',
         'discussion',
-        '.xmodule_DiscussionModule'
+        '.xmodule_DiscussionModule',
+        has_multiple_templates=False
     )
 
 

cms/djangoapps/contentstore/features/problem-editor.py

@@ -170,7 +170,8 @@ def edit_latex_source(step):
 @step('my change to the High Level Source is persisted')
 def high_level_source_persisted(step):
     def verify_text(driver):
-        return world.css_text('.problem') == 'hi'
+        css_sel = '.problem div>span'
+        return world.css_text(css_sel) == 'hi'
 
     world.wait_for(verify_text)
 

cms/djangoapps/contentstore/tests/test_crud.py

@@ -0,0 +1,186 @@
+'''
+Created on May 7, 2013
+
+@author: dmitchell
+'''
+import unittest
+from xmodule import templates
+from xmodule.modulestore.tests import persistent_factories
+from xmodule.course_module import CourseDescriptor
+from xmodule.modulestore.django import modulestore
+from xmodule.seq_module import SequenceDescriptor
+from xmodule.x_module import XModuleDescriptor
+from xmodule.capa_module import CapaDescriptor
+from xmodule.modulestore.locator import CourseLocator, BlockUsageLocator
+from xmodule.modulestore.exceptions import ItemNotFoundError
+from xmodule.html_module import HtmlDescriptor
+
+
+class TemplateTests(unittest.TestCase):
+    """
+    Test finding and using the templates (boilerplates) for xblocks.
+    """
+
+    def test_get_templates(self):
+        found = templates.all_templates()
+        self.assertIsNotNone(found.get('course'))
+        self.assertIsNotNone(found.get('about'))
+        self.assertIsNotNone(found.get('html'))
+        self.assertIsNotNone(found.get('problem'))
+        self.assertEqual(len(found.get('course')), 0)
+        self.assertEqual(len(found.get('about')), 1)
+        self.assertGreaterEqual(len(found.get('html')), 2)
+        self.assertGreaterEqual(len(found.get('problem')), 10)
+        dropdown = None
+        for template in found['problem']:
+            self.assertIn('metadata', template)
+            self.assertIn('display_name', template['metadata'])
+            if template['metadata']['display_name'] == 'Dropdown':
+                dropdown = template
+                break
+        self.assertIsNotNone(dropdown)
+        self.assertIn('markdown', dropdown['metadata'])
+        self.assertIn('data', dropdown)
+        self.assertRegexpMatches(dropdown['metadata']['markdown'], r'^Dropdown.*')
+        self.assertRegexpMatches(dropdown['data'], r'<problem>\s*<p>Dropdown.*')
+
+    def test_get_some_templates(self):
+        self.assertEqual(len(SequenceDescriptor.templates()), 0)
+        self.assertGreater(len(HtmlDescriptor.templates()), 0)
+        self.assertIsNone(SequenceDescriptor.get_template('doesntexist.yaml'))
+        self.assertIsNone(HtmlDescriptor.get_template('doesntexist.yaml'))
+        self.assertIsNotNone(HtmlDescriptor.get_template('announcement.yaml'))
+
+    def test_factories(self):
+        test_course = persistent_factories.PersistentCourseFactory.create(org='testx', prettyid='tempcourse',
+            display_name='fun test course', user_id='testbot')
+        self.assertIsInstance(test_course, CourseDescriptor)
+        self.assertEqual(test_course.display_name, 'fun test course')
+        index_info = modulestore('split').get_course_index_info(test_course.location)
+        self.assertEqual(index_info['org'], 'testx')
+        self.assertEqual(index_info['prettyid'], 'tempcourse')
+
+        test_chapter = persistent_factories.ItemFactory.create(display_name='chapter 1',
+            parent_location=test_course.location)
+        self.assertIsInstance(test_chapter, SequenceDescriptor)
+        # refetch parent which should now point to child
+        test_course = modulestore('split').get_course(test_chapter.location)
+        self.assertIn(test_chapter.location.usage_id, test_course.children)
+
+    def test_temporary_xblocks(self):
+        """
+        Test using load_from_json to create non persisted xblocks
+        """
+        test_course = persistent_factories.PersistentCourseFactory.create(org='testx', prettyid='tempcourse',
+            display_name='fun test course', user_id='testbot')
+
+        test_chapter = XModuleDescriptor.load_from_json({'category': 'chapter',
+            'metadata': {'display_name': 'chapter n'}},
+            test_course.system, parent_xblock=test_course)
+        self.assertIsInstance(test_chapter, SequenceDescriptor)
+        self.assertEqual(test_chapter.display_name, 'chapter n')
+        self.assertIn(test_chapter, test_course.get_children())
+
+        # test w/ a definition (e.g., a problem)
+        test_def_content = '<problem>boo</problem>'
+        test_problem = XModuleDescriptor.load_from_json({'category': 'problem',
+            'definition': {'data': test_def_content}},
+            test_course.system, parent_xblock=test_chapter)
+        self.assertIsInstance(test_problem, CapaDescriptor)
+        self.assertEqual(test_problem.data, test_def_content)
+        self.assertIn(test_problem, test_chapter.get_children())
+        test_problem.display_name = 'test problem'
+        self.assertEqual(test_problem.display_name, 'test problem')
+
+    def test_persist_dag(self):
+        """
+        try saving temporary xblocks
+        """
+        test_course = persistent_factories.PersistentCourseFactory.create(org='testx', prettyid='tempcourse',
+            display_name='fun test course', user_id='testbot')
+        test_chapter = XModuleDescriptor.load_from_json({'category': 'chapter',
+            'metadata': {'display_name': 'chapter n'}},
+            test_course.system, parent_xblock=test_course)
+        test_def_content = '<problem>boo</problem>'
+        test_problem = XModuleDescriptor.load_from_json({'category': 'problem',
+            'definition': {'data': test_def_content}},
+            test_course.system, parent_xblock=test_chapter)
+        # better to pass in persisted parent over the subdag so
+        # subdag gets the parent pointer (otherwise 2 ops, persist dag, update parent children,
+        # persist parent
+        persisted_course = modulestore('split').persist_xblock_dag(test_course, 'testbot')
+        self.assertEqual(len(persisted_course.children), 1)
+        persisted_chapter = persisted_course.get_children()[0]
+        self.assertEqual(persisted_chapter.category, 'chapter')
+        self.assertEqual(persisted_chapter.display_name, 'chapter n')
+        self.assertEqual(len(persisted_chapter.children), 1)
+        persisted_problem = persisted_chapter.get_children()[0]
+        self.assertEqual(persisted_problem.category, 'problem')
+        self.assertEqual(persisted_problem.data, test_def_content)
+
+    def test_delete_course(self):
+        test_course = persistent_factories.PersistentCourseFactory.create(
+            org='testx',
+            prettyid='edu.harvard.history.doomed',
+            display_name='doomed test course',
+            user_id='testbot')
+        persistent_factories.ItemFactory.create(display_name='chapter 1',
+            parent_location=test_course.location)
+
+        id_locator = CourseLocator(course_id=test_course.location.course_id, revision='draft')
+        guid_locator = CourseLocator(version_guid=test_course.location.version_guid)
+        # verify it can be retireved by id
+        self.assertIsInstance(modulestore('split').get_course(id_locator), CourseDescriptor)
+        # and by guid
+        self.assertIsInstance(modulestore('split').get_course(guid_locator), CourseDescriptor)
+        modulestore('split').delete_course(id_locator.course_id)
+        # test can no longer retrieve by id
+        self.assertRaises(ItemNotFoundError, modulestore('split').get_course, id_locator)
+        # but can by guid
+        self.assertIsInstance(modulestore('split').get_course(guid_locator), CourseDescriptor)
+
+    def test_block_generations(self):
+        """
+        Test get_block_generations
+        """
+        test_course = persistent_factories.PersistentCourseFactory.create(
+            org='testx',
+            prettyid='edu.harvard.history.hist101',
+            display_name='history test course',
+            user_id='testbot')
+        chapter = persistent_factories.ItemFactory.create(display_name='chapter 1',
+            parent_location=test_course.location, user_id='testbot')
+        sub = persistent_factories.ItemFactory.create(display_name='subsection 1',
+            parent_location=chapter.location, user_id='testbot', category='vertical')
+        first_problem = persistent_factories.ItemFactory.create(display_name='problem 1',
+            parent_location=sub.location, user_id='testbot', category='problem', data="<problem></problem>")
+        first_problem.max_attempts = 3
+        updated_problem = modulestore('split').update_item(first_problem, 'testbot')
+        updated_loc = modulestore('split').delete_item(updated_problem.location, 'testbot')
+
+        second_problem = persistent_factories.ItemFactory.create(display_name='problem 2',
+            parent_location=BlockUsageLocator(updated_loc, usage_id=sub.location.usage_id),
+            user_id='testbot', category='problem', data="<problem></problem>")
+
+        # course root only updated 2x
+        version_history = modulestore('split').get_block_generations(test_course.location)
+        self.assertEqual(version_history.locator.version_guid, test_course.location.version_guid)
+        self.assertEqual(len(version_history.children), 1)
+        self.assertEqual(version_history.children[0].children, [])
+        self.assertEqual(version_history.children[0].locator.version_guid, chapter.location.version_guid)
+
+        # sub changed on add, add problem, delete problem, add problem in strict linear seq
+        version_history = modulestore('split').get_block_generations(sub.location)
+        self.assertEqual(len(version_history.children), 1)
+        self.assertEqual(len(version_history.children[0].children), 1)
+        self.assertEqual(len(version_history.children[0].children[0].children), 1)
+        self.assertEqual(len(version_history.children[0].children[0].children[0].children), 0)
+
+        # first and second problem may show as same usage_id; so, need to ensure their histories are right
+        version_history = modulestore('split').get_block_generations(updated_problem.location)
+        self.assertEqual(version_history.locator.version_guid, first_problem.location.version_guid)
+        self.assertEqual(len(version_history.children), 1)  # updated max_attempts
+        self.assertEqual(len(version_history.children[0].children), 0)
+
+        version_history = modulestore('split').get_block_generations(second_problem.location)
+        self.assertNotEqual(version_history.locator.version_guid, first_problem.location.version_guid)

cms/envs/dev.py

@@ -33,6 +33,10 @@ MODULESTORE = {
     'direct': {
         'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
         'OPTIONS': modulestore_options
+    },
+    'split': {
+        'ENGINE': 'xmodule.modulestore.split_mongo.SplitMongoModuleStore',
+        'OPTIONS': modulestore_options
     }
 }
 

cms/envs/test.py

@@ -63,6 +63,10 @@ MODULESTORE = {
     'draft': {
         'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
         'OPTIONS': MODULESTORE_OPTIONS
+    },
+    'split': {
+        'ENGINE': 'xmodule.modulestore.split_mongo.SplitMongoModuleStore',
+        'OPTIONS': MODULESTORE_OPTIONS
     }
 }
 

common/lib/xmodule/xmodule/capa_module.py

@@ -309,7 +309,13 @@ class CapaModule(CapaFields, XModule):
         d = self.get_score()
         score = d['score']
         total = d['total']
+
         if total > 0:
+            if self.weight is not None:
+                # scale score and total by weight/total:
+                score = score * self.weight / total
+                total = self.weight
+
             try:
                 return Progress(score, total)
             except (TypeError, ValueError):
@@ -321,11 +327,13 @@ class CapaModule(CapaFields, XModule):
         """
         Return some html with data about the module
         """
+        progress = self.get_progress()
         return self.system.render_template('problem_ajax.html', {
             'element_id': self.location.html_id(),
             'id': self.id,
             'ajax_url': self.system.ajax_url,
-            'progress': Progress.to_js_status_str(self.get_progress())
+            'progress_status': Progress.to_js_status_str(progress),
+            'progress_detail': Progress.to_js_detail_str(progress),
         })
 
     def check_button_name(self):
@@ -485,8 +493,7 @@ class CapaModule(CapaFields, XModule):
         """
         Return html for the problem.
 
-        Adds check, reset, save buttons as necessary based on the problem config
-        and state.
+        Adds check, reset, save buttons as necessary based on the problem config and state.
         """
 
         try:
@@ -516,13 +523,12 @@ class CapaModule(CapaFields, XModule):
                    'reset_button': self.should_show_reset_button(),
                    'save_button': self.should_show_save_button(),
                    'answer_available': self.answer_available(),
-                   'ajax_url': self.system.ajax_url,
                    'attempts_used': self.attempts,
                    'attempts_allowed': self.max_attempts,
-                   'progress': self.get_progress(),
                    }
 
         html = self.system.render_template('problem.html', context)
+
         if encapsulate:
             html = u'<div id="problem_{id}" class="problem" data-url="{ajax_url}">'.format(
                 id=self.location.html_id(), ajax_url=self.system.ajax_url
@@ -584,6 +590,7 @@ class CapaModule(CapaFields, XModule):
         result.update({
             'progress_changed': after != before,
             'progress_status': Progress.to_js_status_str(after),
+            'progress_detail': Progress.to_js_detail_str(after),
         })
 
         return json.dumps(result, cls=ComplexEncoder)
@@ -614,6 +621,7 @@ class CapaModule(CapaFields, XModule):
         Problem can be completely wrong.
         Pressing RESET button makes this function to return False.
         """
+        # used by conditional module
         return self.lcp.done
 
     def is_attempted(self):
@@ -757,6 +765,7 @@ class CapaModule(CapaFields, XModule):
         """
         return {'html': self.get_problem_html(encapsulate=False)}
 
+
     @staticmethod
     def make_dict_of_responses(data):
         """

common/lib/xmodule/xmodule/course_module.py

@@ -15,6 +15,7 @@ import json
 
 from xblock.core import Scope, List, String, Dict, Boolean
 from .fields import Date
+from xmodule.modulestore.locator import CourseLocator
 from django.utils.timezone import UTC
 from xmodule.util import date_utils
 
@@ -372,7 +373,10 @@ class CourseDescriptor(CourseFields, SequenceDescriptor):
         super(CourseDescriptor, self).__init__(*args, **kwargs)
 
         if self.wiki_slug is None:
-            self.wiki_slug = self.location.course
+            if isinstance(self.location, Location):
+                self.wiki_slug = self.location.course
+            elif isinstance(self.location, CourseLocator):
+                self.wiki_slug = self.location.course_id or self.display_name
 
         msg = None
 

common/lib/xmodule/xmodule/css/capa/display.scss

@@ -3,6 +3,7 @@ h2 {
   margin-bottom: 15px;
 
   &.problem-header {
+    display: inline-block;
     section.staff {
       margin-top: 30px;
       font-size: 80%;
@@ -28,6 +29,13 @@ iframe[seamless]{
   color: darken($error-red, 11%);
 }
 
+section.problem-progress {
+  display: inline-block;
+  color: #999;
+  font-size: em(16);
+  font-weight: 100;
+  padding-left: 5px;
+}
 
 section.problem {
   @media print {

common/lib/xmodule/xmodule/error_module.py

@@ -79,8 +79,10 @@ class ErrorDescriptor(ErrorFields, JSONEditingDescriptor):
     @classmethod
     def _construct(cls, system, contents, error_msg, location):
 
-        if location.name is None:
-            location = location._replace(
+        if isinstance(location, dict) and 'course' in location:
+            location = Location(location)
+        if isinstance(location, Location) and location.name is None:
+            location = location.replace(
                 category='error',
                 # Pick a unique url_name -- the sha1 hash of the contents.
                 # NOTE: We could try to pull out the url_name of the errored descriptor,
@@ -94,7 +96,7 @@ class ErrorDescriptor(ErrorFields, JSONEditingDescriptor):
         model_data = {
             'error_msg': str(error_msg),
             'contents': contents,
-            'display_name': 'Error: ' + location.name,
+            'display_name': 'Error: ' + location.url(),
             'location': location,
             'category': 'error'
         }

common/lib/xmodule/xmodule/js/fixtures/problem.html

@@ -1,6 +1,6 @@
 <section class='xmodule_display xmodule_CapaModule' data-type='Problem'>
   <section id='problem_1' 
-           class='problems-wrapper' 
+           class='problems-wrapper'
            data-problem-id='i4x://edX/101/problem/Problem1' 
            data-url='/problem/Problem1'>
   </section>

common/lib/xmodule/xmodule/js/fixtures/problem_content.html

@@ -1,5 +1,8 @@
 <h2 class="problem-header">Problem Header</h2>
 
+<section class='problem-progress'>
+</section>
+
 <section class="problem">
   <p>Problem Content</p>
 

common/lib/xmodule/xmodule/js/spec/capa/display_spec.coffee

@@ -77,6 +77,25 @@ describe 'Problem', ->
         [@problem.updateMathML, @stubbedJax, $('#input_example_1').get(0)]
       ]
 
+  describe 'renderProgressState', ->
+    beforeEach ->
+      @problem = new Problem($('.xmodule_display'))
+      #@renderProgressState = @problem.renderProgressState
+
+    describe 'with a status of "none"', ->
+      it 'reports the number of points possible', ->
+        @problem.el.data('progress_status', 'none')
+        @problem.el.data('progress_detail', '0/1')
+        @problem.renderProgressState()
+        expect(@problem.$('.problem-progress').html()).toEqual "(1 point possible)"
+
+    describe 'with any other valid status', ->
+      it 'reports the current score', ->
+        @problem.el.data('progress_status', 'foo')
+        @problem.el.data('progress_detail', '1/1')
+        @problem.renderProgressState()
+        expect(@problem.$('.problem-progress').html()).toEqual "(1/1 points)"
+
   describe 'render', ->
     beforeEach ->
       @problem = new Problem($('.xmodule_display'))

common/lib/xmodule/xmodule/js/src/capa/display.coffee

@@ -35,15 +35,34 @@ class @Problem
       @$('input.math').each (index, element) =>
         MathJax.Hub.Queue [@refreshMath, null, element]
 
+  renderProgressState: =>
+    detail = @el.data('progress_detail')
+    status = @el.data('progress_status')
+    # i18n
+    progress = "(#{detail} points)"
+    if status == 'none' and detail? and detail.indexOf('/') > 0
+        a = detail.split('/')
+        possible = parseInt(a[1])
+        if possible == 1
+            # i18n
+            progress = "(#{possible} point possible)"
+        else
+            # i18n
+            progress = "(#{possible} points possible)"
+    @$('.problem-progress').html(progress)
+
   updateProgress: (response) =>
     if response.progress_changed
-        @el.attr progress: response.progress_status
+        @el.data('progress_status', response.progress_status)
+        @el.data('progress_detail', response.progress_detail)
         @el.trigger('progressChanged')
+    @renderProgressState()
 
   forceUpdate: (response) =>
-    @el.attr progress: response.progress_status
+    @el.data('progress_status', response.progress_status)
+    @el.data('progress_detail', response.progress_detail)
     @el.trigger('progressChanged')
-
+    @renderProgressState()
 
   queueing: =>
     @queued_items = @$(".xqueue")
@@ -113,7 +132,7 @@ class @Problem
           @setupInputTypes()
           @bind()
           @queueing()
-
+          @forceUpdate response
 
   # TODO add hooks for problem types here by inspecting response.html and doing
   # stuff if a div w a class is found

common/lib/xmodule/xmodule/js/src/sequence/display.coffee

@@ -45,7 +45,7 @@ class @Sequence
     new_progress = "NA"
     _this = this
     $('.problems-wrapper').each (index) ->
-      progress = $(this).attr 'progress'
+      progress = $(this).data 'progress_status'
       new_progress = _this.mergeProgress progress, new_progress
 
     @progressTable[@position] = new_progress

common/lib/xmodule/xmodule/modulestore/exceptions.py

@@ -7,10 +7,18 @@ class ItemNotFoundError(Exception):
     pass
 
 
+class ItemWriteConflictError(Exception):
+    pass
+
+
 class InsufficientSpecificationError(Exception):
     pass
 
 
+class OverSpecificationError(Exception):
+    pass
+
+
 class InvalidLocationError(Exception):
     pass
 
@@ -21,3 +29,13 @@ class NoPathToItem(Exception):
 
 class DuplicateItemError(Exception):
     pass
+
+
+class VersionConflictError(Exception):
+    """
+    The caller asked for either draft or published head and gave a version which conflicted with it.
+    """
+    def __init__(self, requestedLocation, currentHead):
+        super(VersionConflictError, self).__init__()
+        self.requestedLocation = requestedLocation
+        self.currentHead = currentHead

common/lib/xmodule/xmodule/modulestore/inheritance.py

@@ -50,6 +50,8 @@ def inherit_metadata(descriptor, model_data):
 
 
 def own_metadata(module):
+    # IN SPLIT MONGO this is just ['metadata'] as it keeps ['_inherited_metadata'] separate!
+    # FIXME move into kvs? will that work for xml mongo?
     """
     Return a dictionary that contains only non-inherited field keys,
     mapped to their values

common/lib/xmodule/xmodule/modulestore/locator.py

@@ -0,0 +1,465 @@
+"""
+Created on Mar 13, 2013
+
+@author: dmitchell
+"""
+from __future__ import absolute_import
+import logging
+import inspect
+from abc import ABCMeta, abstractmethod
+from urllib import quote
+
+from bson.objectid import ObjectId
+from bson.errors import InvalidId
+
+from xmodule.modulestore.exceptions import InsufficientSpecificationError, OverSpecificationError
+
+from .parsers import parse_url, parse_course_id, parse_block_ref
+
+log = logging.getLogger(__name__)
+
+
+class Locator(object):
+    """
+    A locator is like a URL, it refers to a course resource.
+
+    Locator is an abstract base class: do not instantiate
+    """
+
+    __metaclass__ = ABCMeta
+
+    @abstractmethod
+    def url(self):
+        """
+        Return a string containing the URL for this location. Raises
+        InsufficientSpecificationError if the instance doesn't have a
+        complete enough specification to generate a url
+        """
+        raise InsufficientSpecificationError()
+
+    def quoted_url(self):
+        return quote(self.url(), '@;#')
+
+    def __eq__(self, other):
+        return self.__dict__ == other.__dict__
+
+    def __repr__(self):
+        '''
+        repr(self) returns something like this: CourseLocator("edu.mit.eecs.6002x")
+        '''
+        classname = self.__class__.__name__
+        if classname.find('.') != -1:
+            classname = classname.split['.'][-1]
+        return '%s("%s")' % (classname, unicode(self))
+
+    def __str__(self):
+        '''
+        str(self) returns something like this: "edu.mit.eecs.6002x"
+        '''
+        return unicode(self).encode('utf8')
+
+    def __unicode__(self):
+        '''
+        unicode(self) returns something like this: "edu.mit.eecs.6002x"
+        '''
+        return self.url()
+
+    @abstractmethod
+    def version(self):
+        """
+        Returns the ObjectId referencing this specific location.
+        Raises InsufficientSpecificationError if the instance
+        doesn't have a complete enough specification.
+        """
+        raise InsufficientSpecificationError()
+
+    def set_property(self, property_name, new):
+        """
+        Initialize property to new value.
+        If property has already been initialized to a different value, raise an exception.
+        """
+        current = getattr(self, property_name)
+        if current and current != new:
+            raise OverSpecificationError('%s cannot be both %s and %s' %
+                                         (property_name, current, new))
+        setattr(self, property_name, new)
+
+
+class CourseLocator(Locator):
+    """
+    Examples of valid CourseLocator specifications:
+     CourseLocator(version_guid=ObjectId('519665f6223ebd6980884f2b'))
+     CourseLocator(course_id='edu.mit.eecs.6002x')
+     CourseLocator(course_id='edu.mit.eecs.6002x;published')
+     CourseLocator(course_id='edu.mit.eecs.6002x', revision='published')
+     CourseLocator(url='edx://@519665f6223ebd6980884f2b')
+     CourseLocator(url='edx://edu.mit.eecs.6002x')
+     CourseLocator(url='edx://edu.mit.eecs.6002x;published')
+
+    Should have at lease a specific course_id (id for the course as if it were a project w/
+    versions) with optional 'revision' (must be 'draft', 'published', or None),
+    or version_guid (which points to a specific version). Can contain both in which case
+    the persistence layer may raise exceptions if the given version != the current such version
+    of the course.
+    """
+
+    # Default values
+    version_guid = None
+    course_id = None
+    revision = None
+
+    def __unicode__(self):
+        """
+        Return a string representing this location.
+        """
+        if self.course_id:
+            result = self.course_id
+            if self.revision:
+                result += ';' + self.revision
+            return result
+        elif self.version_guid:
+            return '@' + str(self.version_guid)
+        else:
+            # raise InsufficientSpecificationError("missing course_id or version_guid")
+            return '<InsufficientSpecificationError: missing course_id or version_guid>'
+
+    def url(self):
+        """
+        Return a string containing the URL for this location.
+        """
+        return 'edx://' + unicode(self)
+
+    # -- unused args which are used via inspect
+    # pylint: disable= W0613
+    def validate_args(self, url, version_guid, course_id, revision):
+        """
+        Validate provided arguments.
+        """
+        need_oneof = set(('url', 'version_guid', 'course_id'))
+        args, _, _, values = inspect.getargvalues(inspect.currentframe())
+        provided_args = [a for a in args if a != 'self' and values[a] is not None]
+        if len(need_oneof.intersection(provided_args)) == 0:
+            raise InsufficientSpecificationError("Must provide one of these args: %s " %
+                                                 list(need_oneof))
+
+    def is_fully_specified(self):
+        """
+        Returns True if either version_guid is specified, or course_id+revision
+        are specified.
+        This should always return True, since this should be validated in the constructor.
+        """
+        return self.version_guid is not None \
+            or (self.course_id is not None and self.revision is not None)
+
+    def set_course_id(self, new):
+        """
+        Initialize course_id to new value.
+        If course_id has already been initialized to a different value, raise an exception.
+        """
+        self.set_property('course_id', new)
+
+    def set_revision(self, new):
+        """
+        Initialize revision to new value.
+        If revision has already been initialized to a different value, raise an exception.
+        """
+        self.set_property('revision', new)
+
+    def set_version_guid(self, new):
+        """
+        Initialize version_guid to new value.
+        If version_guid has already been initialized to a different value, raise an exception.
+        """
+        self.set_property('version_guid', new)
+
+    def as_course_locator(self):
+        """
+        Returns a copy of itself (downcasting) as a CourseLocator.
+        The copy has the same CourseLocator fields as the original.
+        The copy does not include subclass information, such as
+        a usage_id (a property of BlockUsageLocator).
+        """
+        return CourseLocator(course_id=self.course_id,
+                             version_guid=self.version_guid,
+                             revision=self.revision)
+
+    def __init__(self, url=None, version_guid=None, course_id=None, revision=None):
+        """
+        Construct a CourseLocator
+        Caller may provide url (but no other parameters).
+        Caller may provide version_guid (but no other parameters).
+        Caller may provide course_id (optionally provide revision).
+
+        Resulting CourseLocator will have either a version_guid property
+        or a course_id (with optional revision) property, or both.
+
+        version_guid must be an instance of bson.objectid.ObjectId or None
+        url, course_id, and revision must be strings or None
+
+        """
+        self.validate_args(url, version_guid, course_id, revision)
+        if url:
+            self.init_from_url(url)
+        if version_guid:
+            self.init_from_version_guid(version_guid)
+        if course_id or revision:
+            self.init_from_course_id(course_id, revision)
+        assert self.version_guid or self.course_id, \
+            "Either version_guid or course_id should be set."
+
+    @classmethod
+    def as_object_id(cls, value):
+        """
+        Attempts to cast value as a bson.objectid.ObjectId.
+        If cast fails, raises ValueError
+        """
+        if isinstance(value, ObjectId):
+            return value
+        try:
+            return ObjectId(value)
+        except InvalidId:
+            raise ValueError('"%s" is not a valid version_guid' % value)
+
+    def init_from_url(self, url):
+        """
+        url must be a string beginning with 'edx://' and containing
+        either a valid version_guid or course_id (with optional revision)
+        If a block ('#HW3') is present, it is ignored.
+        """
+        if isinstance(url, Locator):
+            url = url.url()
+        assert isinstance(url, basestring), \
+            '%s is not an instance of basestring' % url
+        parse = parse_url(url)
+        assert parse, 'Could not parse "%s" as a url' % url
+        if 'version_guid' in parse:
+            new_guid = parse['version_guid']
+            self.set_version_guid(self.as_object_id(new_guid))
+        else:
+            self.set_course_id(parse['id'])
+            self.set_revision(parse['revision'])
+
+    def init_from_version_guid(self, version_guid):
+        """
+        version_guid must be an instance of bson.objectid.ObjectId,
+        or able to be cast as one.
+        If it's a string, attempt to cast it as an ObjectId first.
+        """
+        version_guid = self.as_object_id(version_guid)
+
+        assert isinstance(version_guid, ObjectId), \
+            '%s is not an instance of ObjectId' % version_guid
+        self.set_version_guid(version_guid)
+
+    def init_from_course_id(self, course_id, explicit_revision=None):
+        """
+        Course_id is a string like 'edu.mit.eecs.6002x' or 'edu.mit.eecs.6002x;published'.
+
+        Revision (optional) is a string like 'published'.
+        It may be provided explicitly (explicit_revision) or embedded into course_id.
+        If revision is part of course_id ("...;published"), parse it out separately.
+        If revision is provided both ways, that's ok as long as they are the same value.
+
+        If a block ('#HW3') is a part of course_id, it is ignored.
+
+        """
+
+        if course_id:
+            if isinstance(course_id, CourseLocator):
+                course_id = course_id.course_id
+                assert course_id, "%s does not have a valid course_id"
+
+            parse = parse_course_id(course_id)
+            assert parse, 'Could not parse "%s" as a course_id' % course_id
+            self.set_course_id(parse['id'])
+            rev = parse['revision']
+            if rev:
+                self.set_revision(rev)
+        if explicit_revision:
+            self.set_revision(explicit_revision)
+
+    def version(self):
+        """
+        Returns the ObjectId referencing this specific location.
+        """
+        return self.version_guid
+
+    def html_id(self):
+        """
+        Generate a discussion group id based on course
+
+        To make compatible with old Location object functionality. I don't believe this behavior fits at this
+        place, but I have no way to override. If this is really needed, it should probably use the pretty_id to seed
+        the name although that's mutable. We should also clearly define the purpose and restrictions of this
+        (e.g., I'm assuming periods are fine).
+        """
+        return self.course_id
+
+
+class BlockUsageLocator(CourseLocator):
+    """
+    Encodes a location.
+
+    Locations address modules (aka blocks) which are definitions situated in a
+    course instance. Thus, a Location must identify the course and the occurrence of
+    the defined element in the course. Courses can be a version of an offering, the
+    current draft head, or the current production version.
+
+    Locators can contain both a version and a course_id w/ revision. The split mongo functions
+    may raise errors if these conflict w/ the current db state (i.e., the course's revision !=
+    the version_guid)
+
+    Locations can express as urls as well as dictionaries. They consist of
+        course_identifier: course_guid | version_guid
+        block : guid
+        revision : 'draft' | 'published' (optional)
+    """
+
+    # Default value
+    usage_id = None
+
+    def __init__(self, url=None, version_guid=None, course_id=None,
+                 revision=None, usage_id=None):
+        """
+        Construct a BlockUsageLocator
+        Caller may provide url, version_guid, or course_id, and optionally provide revision.
+
+        The usage_id may be specified, either explictly or as part of
+        the url or course_id. If omitted, the locator is created but it
+        has not yet been initialized.
+
+        Resulting BlockUsageLocator will have a usage_id property.
+        It will have either a version_guid property or a course_id (with optional revision) property, or both.
+
+        version_guid must be an instance of bson.objectid.ObjectId or None
+        url, course_id, revision, and usage_id must be strings or None
+
+        """
+        self.validate_args(url, version_guid, course_id, revision)
+        if url:
+            self.init_block_ref_from_url(url)
+        if course_id:
+            self.init_block_ref_from_course_id(course_id)
+        if usage_id:
+            self.init_block_ref(usage_id)
+        CourseLocator.__init__(self,
+                               url=url,
+                               version_guid=version_guid,
+                               course_id=course_id,
+                               revision=revision)
+
+    def is_initialized(self):
+        """
+        Returns True if usage_id has been initialized, else returns False
+        """
+        return self.usage_id is not None
+
+    def version_agnostic(self):
+        """
+        Returns a copy of itself.
+        If both version_guid and course_id are known, use a blank course_id in the copy.
+
+        We don't care if the locator's version is not the current head; so, avoid version conflict
+        by reducing info.
+
+        :param block_locator:
+        """
+        if self.course_id and self.version_guid:
+            return BlockUsageLocator(version_guid=self.version_guid,
+                                     revision=self.revision,
+                                     usage_id=self.usage_id)
+        else:
+            return BlockUsageLocator(course_id=self.course_id,
+                                     revision=self.revision,
+                                     usage_id=self.usage_id)
+
+    def set_usage_id(self, new):
+        """
+        Initialize usage_id to new value.
+        If usage_id has already been initialized to a different value, raise an exception.
+        """
+        self.set_property('usage_id', new)
+
+    def init_block_ref(self, block_ref):
+        parse = parse_block_ref(block_ref)
+        assert parse, 'Could not parse "%s" as a block_ref' % block_ref
+        self.set_usage_id(parse['block'])
+
+    def init_block_ref_from_url(self, url):
+        if isinstance(url, Locator):
+            url = url.url()
+        parse = parse_url(url)
+        assert parse, 'Could not parse "%s" as a url' % url
+        block = parse.get('block', None)
+        if block:
+            self.set_usage_id(block)
+
+    def init_block_ref_from_course_id(self, course_id):
+        if isinstance(course_id, CourseLocator):
+            course_id = course_id.course_id
+            assert course_id, "%s does not have a valid course_id"
+        parse = parse_course_id(course_id)
+        assert parse, 'Could not parse "%s" as a course_id' % course_id
+        block = parse.get('block', None)
+        if block:
+            self.set_usage_id(block)
+
+    def __unicode__(self):
+        """
+        Return a string representing this location.
+        """
+        rep = CourseLocator.__unicode__(self)
+        if self.usage_id is None:
+            # usage_id has not been initialized
+            return rep + '#NONE'
+        else:
+            return rep + '#' + self.usage_id
+
+
+class DescriptionLocator(Locator):
+    """
+    Container for how to locate a description
+    """
+
+    def __init__(self, definition_id):
+        self.definition_id = definition_id
+
+    def __unicode__(self):
+        '''
+        Return a string representing this location.
+        unicode(self) returns something like this: "@519665f6223ebd6980884f2b"
+        '''
+        return '@' + str(self.definition_guid)
+
+    def url(self):
+        """
+        Return a string containing the URL for this location.
+        url(self) returns something like this: 'edx://@519665f6223ebd6980884f2b'
+        """
+        return 'edx://' + unicode(self)
+
+    def version(self):
+        """
+        Returns the ObjectId referencing this specific location.
+        """
+        return self.definition_guid
+
+
+class VersionTree(object):
+    """
+    Holds trees of Locators to represent version histories.
+    """
+    def __init__(self, locator, tree_dict=None):
+        """
+        :param locator: must be version specific (Course has version_guid or definition had id)
+        """
+        assert isinstance(locator, Locator) and not inspect.isabstract(locator), \
+            "locator must be a concrete subclass of Locator"
+        assert locator.version(), \
+            "locator must be version specific (Course has version_guid or definition had id)"
+        self.locator = locator
+        if tree_dict is None:
+            self.children = []
+        else:
+            self.children = [VersionTree(child, tree_dict)
+                             for child in tree_dict.get(locator.version(), [])]

common/lib/xmodule/xmodule/modulestore/mongo/base.py

@@ -105,15 +105,6 @@ class MongoKeyValueStore(KeyValueStore):
         else:
             raise InvalidScopeError(key.scope)
 
-    def set_many(self, update_dict):
-        """set_many method. Implementations should accept an `update_dict` of
-        key-value pairs, and set all the `keys` to the given `value`s."""
-        # `set` simply updates an in-memory db, rather than calling down to a real db,
-        # as mongo bulk save is handled elsewhere. A future improvement would be to pull
-        # the mongo-specific bulk save logic into this method.
-        for key, value in update_dict.iteritems():
-            self.set(key, value)
-
     def delete(self, key):
         if key.scope == Scope.children:
             self._children = []

common/lib/xmodule/xmodule/modulestore/parsers.py

@@ -0,0 +1,115 @@
+import re
+
+URL_RE = re.compile(r'^edx://(.+)$', re.IGNORECASE)
+
+
+def parse_url(string):
+    """
+    A url must begin with 'edx://' (case-insensitive match),
+    followed by either a version_guid or a course_id.
+
+    Examples:
+        'edx://@0123FFFF'
+        'edx://edu.mit.eecs.6002x'
+        'edx://edu.mit.eecs.6002x;published'
+        'edx://edu.mit.eecs.6002x;published#HW3'
+
+    This returns None if string cannot be parsed.
+
+    If it can be parsed as a version_guid, returns a dict
+    with key 'version_guid' and the value,
+
+    If it can be parsed as a course_id, returns a dict
+    with keys 'id' and 'revision' (value of 'revision' may be None),
+
+    """
+    match = URL_RE.match(string)
+    if not match:
+        return None
+    path = match.group(1)
+    if path[0] == '@':
+        return parse_guid(path[1:])
+    return parse_course_id(path)
+
+
+BLOCK_RE = re.compile(r'^\w+$', re.IGNORECASE)
+
+
+def parse_block_ref(string):
+    r"""
+    A block_ref is a string of word_chars.
+
+    <word_chars> matches one or more Unicode word characters; this includes most
+    characters that can be part of a word in any language, as well as numbers
+    and the underscore. (see definition of \w in python regular expressions,
+    at http://docs.python.org/dev/library/re.html)
+
+    If string is a block_ref, returns a dict with key 'block_ref' and the value,
+    otherwise returns None.
+    """
+    if len(string) > 0 and BLOCK_RE.match(string):
+        return {'block': string}
+    return None
+
+
+GUID_RE = re.compile(r'^(?P<version_guid>[A-F0-9]+)(#(?P<block>\w+))?$', re.IGNORECASE)
+
+
+def parse_guid(string):
+    """
+    A version_guid is a string of hex digits (0-F).
+
+    If string is a version_guid, returns a dict with key 'version_guid' and the value,
+    otherwise returns None.
+    """
+    m = GUID_RE.match(string)
+    if m is not None:
+        return m.groupdict()
+    else:
+        return None
+
+
+COURSE_ID_RE = re.compile(r'^(?P<id>(\w+)(\.\w+\w*)*)(;(?P<revision>\w+))?(#(?P<block>\w+))?$', re.IGNORECASE)
+
+
+def parse_course_id(string):
+    r"""
+
+    A course_id has a main id component.
+    There may also be an optional revision (;published or ;draft).
+    There may also be an optional block (#HW3 or #Quiz2).
+
+    Examples of valid course_ids:
+
+      'edu.mit.eecs.6002x'
+      'edu.mit.eecs.6002x;published'
+      'edu.mit.eecs.6002x#HW3'
+      'edu.mit.eecs.6002x;published#HW3'
+
+
+    Syntax:
+
+      course_id = main_id [; revision] [# block]
+
+      main_id = name [. name]*
+
+      revision = name
+
+      block = name
+
+      name = <word_chars>
+
+    <word_chars> matches one or more Unicode word characters; this includes most
+    characters that can be part of a word in any language, as well as numbers
+    and the underscore. (see definition of \w in python regular expressions,
+    at http://docs.python.org/dev/library/re.html)
+
+    If string is a course_id, returns a dict with keys 'id', 'revision', and 'block'.
+    Revision is optional: if missing returned_dict['revision'] is None.
+    Block is optional: if missing returned_dict['block'] is None.
+    Else returns None.
+    """
+    match = COURSE_ID_RE.match(string)
+    if not match:
+        return None
+    return match.groupdict()

common/lib/xmodule/xmodule/modulestore/split_mongo/__init__.py

@@ -0,0 +1 @@
+from split import SplitMongoModuleStore

common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py

@@ -0,0 +1,119 @@
+import sys
+import logging
+from xmodule.mako_module import MakoDescriptorSystem
+from xmodule.x_module import XModuleDescriptor
+from xmodule.modulestore.locator import BlockUsageLocator
+from xmodule.error_module import ErrorDescriptor
+from xmodule.errortracker import exc_info_to_str
+from xblock.runtime import DbModel
+from ..exceptions import ItemNotFoundError
+from .split_mongo_kvs import SplitMongoKVS, SplitMongoKVSid
+
+log = logging.getLogger(__name__)
+
+# TODO should this be here or w/ x_module or ???
+class CachingDescriptorSystem(MakoDescriptorSystem):
+    """
+    A system that has a cache of a course version's json that it will use to load modules
+    from, with a backup of calling to the underlying modulestore for more data.
+
+    Computes the metadata inheritance upon creation.
+    """
+    def __init__(self, modulestore, course_entry, module_data, lazy,
+        default_class, error_tracker, render_template):
+        """
+        Computes the metadata inheritance and sets up the cache.
+
+        modulestore: the module store that can be used to retrieve additional
+        modules
+
+        module_data: a dict mapping Location -> json that was cached from the
+            underlying modulestore
+
+        default_class: The default_class to use when loading an
+            XModuleDescriptor from the module_data
+
+        resources_fs: a filesystem, as per MakoDescriptorSystem
+
+        error_tracker: a function that logs errors for later display to users
+
+        render_template: a function for rendering templates, as per
+            MakoDescriptorSystem
+        """
+        # TODO find all references to resources_fs and make handle None
+        super(CachingDescriptorSystem, self).__init__(
+                self._load_item, None, error_tracker, render_template)
+        self.modulestore = modulestore
+        self.course_entry = course_entry
+        self.lazy = lazy
+        self.module_data = module_data
+        self.default_class = default_class
+        # TODO see if self.course_id is needed: is already in course_entry but could be > 1 value
+        # Compute inheritance
+        modulestore.inherit_metadata(course_entry.get('blocks', {}),
+            course_entry.get('blocks', {})
+            .get(course_entry.get('root')))
+
+    def _load_item(self, usage_id, course_entry_override=None):
+        # TODO ensure all callers of system.load_item pass just the id
+        json_data = self.module_data.get(usage_id)
+        if json_data is None:
+            # deeper than initial descendant fetch or doesn't exist
+            self.modulestore.cache_items(self, [usage_id], lazy=self.lazy)
+            json_data = self.module_data.get(usage_id)
+            if json_data is None:
+                raise ItemNotFoundError
+
+        class_ = XModuleDescriptor.load_class(
+            json_data.get('category'),
+            self.default_class
+        )
+        return self.xblock_from_json(class_, usage_id, json_data, course_entry_override)
+
+    def xblock_from_json(self, class_, usage_id, json_data, course_entry_override=None):
+        if course_entry_override is None:
+            course_entry_override = self.course_entry
+        # most likely a lazy loader but not the id directly
+        definition = json_data.get('definition', {})
+        metadata = json_data.get('metadata', {})
+
+        block_locator = BlockUsageLocator(
+            version_guid=course_entry_override['_id'],
+            usage_id=usage_id,
+            course_id=course_entry_override.get('course_id'),
+            revision=course_entry_override.get('revision')
+        )
+
+        kvs = SplitMongoKVS(
+            definition,
+            json_data.get('children', []),
+            metadata,
+            json_data.get('_inherited_metadata'),
+            block_locator,
+            json_data.get('category'))
+        model_data = DbModel(kvs, class_, None,
+            SplitMongoKVSid(
+                # DbModel req's that these support .url()
+                block_locator,
+                self.modulestore.definition_locator(definition)))
+
+        try:
+            module = class_(self, model_data)
+        except Exception:
+            log.warning("Failed to load descriptor", exc_info=True)
+            if usage_id is None:
+                usage_id = "MISSING"
+            return ErrorDescriptor.from_json(
+                json_data,
+                self,
+                BlockUsageLocator(version_guid=course_entry_override['_id'],
+                    usage_id=usage_id),
+                error_msg=exc_info_to_str(sys.exc_info())
+            )
+
+        module.edited_by = json_data.get('edited_by')
+        module.edited_on = json_data.get('edited_on')
+        module.previous_version = json_data.get('previous_version')
+        module.update_version = json_data.get('update_version')
+        module.definition_locator = self.modulestore.definition_locator(definition)
+        return module

common/lib/xmodule/xmodule/modulestore/split_mongo/definition_lazy_loader.py

@@ -0,0 +1,26 @@
+from xmodule.modulestore.locator import DescriptionLocator
+
+
+class DefinitionLazyLoader(object):
+    """
+    A placeholder to put into an xblock in place of its definition which
+    when accessed knows how to get its content. Only useful if the containing
+    object doesn't force access during init but waits until client wants the
+    definition. Only works if the modulestore is a split mongo store.
+    """
+    def __init__(self, modulestore, definition_id):
+        """
+        Simple placeholder for yet-to-be-fetched data
+        :param modulestore: the pymongo db connection with the definitions
+        :param definition_locator: the id of the record in the above to fetch
+        """
+        self.modulestore = modulestore
+        self.definition_locator = DescriptionLocator(definition_id)
+
+    def fetch(self):
+        """
+        Fetch the definition. Note, the caller should replace this lazy
+        loader pointer with the result so as not to fetch more than once
+        """
+        return self.modulestore.definitions.find_one(
+            {'_id': self.definition_locator.definition_id})

common/lib/xmodule/xmodule/modulestore/split_mongo/split.py

@@ -0,0 +1,1240 @@
+import threading
+import datetime
+import logging
+import pymongo
+import re
+from importlib import import_module
+from path import path
+
+from xmodule.errortracker import null_error_tracker
+from xmodule.x_module import XModuleDescriptor
+from xmodule.modulestore.locator import BlockUsageLocator, DescriptionLocator, CourseLocator, VersionTree
+from xmodule.modulestore.exceptions import InsufficientSpecificationError, VersionConflictError
+from xmodule.modulestore import inheritance
+
+from .. import ModuleStoreBase
+from ..exceptions import ItemNotFoundError
+from .definition_lazy_loader import DefinitionLazyLoader
+from .caching_descriptor_system import CachingDescriptorSystem
+
+log = logging.getLogger(__name__)
+#==============================================================================
+# Documentation is at
+# https://edx-wiki.atlassian.net/wiki/display/ENG/Mongostore+Data+Structure
+#
+# Known issue:
+#    Inheritance for cached kvs doesn't work on edits. Use case.
+#     1) attribute foo is inheritable
+#     2) g.children = [p], p.children = [a]
+#     3) g.foo = 1 on load
+#     4) if g.foo > 0, if p.foo > 0, if a.foo > 0 all eval True
+#     5) p.foo = -1
+#     6) g.foo > 0, p.foo <= 0 all eval True BUT
+#     7) BUG: a.foo > 0 still evals True but should be False
+#     8) reread and everything works right
+#     9) p.del(foo), p.foo > 0 is True! works
+#    10) BUG: a.foo < 0!
+#   Local fix wont' permanently work b/c xblock may cache a.foo...
+#
+#==============================================================================
+
+
+class SplitMongoModuleStore(ModuleStoreBase):
+    """
+    A Mongodb backed ModuleStore supporting versions, inheritance,
+    and sharing.
+    """
+    def __init__(self, host, db, collection, fs_root, render_template,
+                 port=27017, default_class=None,
+                 error_tracker=null_error_tracker,
+                 user=None, password=None,
+                 **kwargs):
+
+        ModuleStoreBase.__init__(self)
+
+        self.db = pymongo.database.Database(pymongo.MongoClient(
+            host=host,
+            port=port,
+            tz_aware=True,
+            **kwargs
+        ), db)
+
+        # TODO add caching of structures to thread_cache to prevent repeated fetches (but not index b/c
+        # it changes w/o having a change in id)
+        self.course_index = self.db[collection + '.active_versions']
+        self.structures = self.db[collection + '.structures']
+        self.definitions = self.db[collection + '.definitions']
+
+        # ??? Code review question: those familiar w/ python threading. Should I instead
+        # use django cache? How should I expire entries?
+        # _add_cache could use a lru mechanism to control the cache size?
+        self.thread_cache = threading.local()
+
+        if user is not None and password is not None:
+            self.db.authenticate(user, password)
+
+        # every app has write access to the db (v having a flag to indicate r/o v write)
+        # Force mongo to report errors, at the expense of performance
+        # pymongo docs suck but explanation:
+        # http://api.mongodb.org/java/2.10.1/com/mongodb/WriteConcern.html
+        self.course_index.write_concern = {'w': 1}
+        self.structures.write_concern = {'w': 1}
+        self.definitions.write_concern = {'w': 1}
+
+        if default_class is not None:
+            module_path, _, class_name = default_class.rpartition('.')
+            class_ = getattr(import_module(module_path), class_name)
+            self.default_class = class_
+        else:
+            self.default_class = None
+        self.fs_root = path(fs_root)
+        self.error_tracker = error_tracker
+        self.render_template = render_template
+
+    def cache_items(self, system, base_usage_ids, depth=0, lazy=True):
+        '''
+        Handles caching of items once inheritance and any other one time
+        per course per fetch operations are done.
+        :param system: a CachingDescriptorSystem
+        :param base_usage_ids: list of usage_ids to fetch
+        :param depth: how deep below these to prefetch
+        :param lazy: whether to fetch definitions or use placeholders
+        '''
+        new_module_data = {}
+        for usage_id in base_usage_ids:
+            new_module_data = self.descendants(system.course_entry['blocks'],
+                                               usage_id,
+                                               depth,
+                                               new_module_data)
+
+        # remove any which were already in module_data (not sure if there's a better way)
+        for newkey in new_module_data.iterkeys():
+            if newkey in system.module_data:
+                del new_module_data[newkey]
+
+        if lazy:
+            for block in new_module_data.itervalues():
+                block['definition'] = DefinitionLazyLoader(self,
+                                                           block['definition'])
+        else:
+            # Load all descendants by id
+            descendent_definitions = self.definitions.find({
+                '_id': {'$in': [block['definition']
+                                for block in new_module_data.itervalues()]}})
+            # turn into a map
+            definitions = {definition['_id']: definition
+                           for definition in descendent_definitions}
+
+            for block in new_module_data.itervalues():
+                if block['definition'] in definitions:
+                    block['definition'] = definitions[block['definition']]
+
+        system.module_data.update(new_module_data)
+        return system.module_data
+
+    def _load_items(self, course_entry, usage_ids, depth=0, lazy=True):
+        '''
+        Load & cache the given blocks from the course. Prefetch down to the
+        given depth. Load the definitions into each block if lazy is False;
+        otherwise, use the lazy definition placeholder.
+        '''
+        system = self._get_cache(course_entry['_id'])
+        if system is None:
+            system = CachingDescriptorSystem(
+                self,
+                course_entry,
+                {},
+                lazy,
+                self.default_class,
+                self.error_tracker,
+                self.render_template
+            )
+            self._add_cache(course_entry['_id'], system)
+            self.cache_items(system, usage_ids, depth, lazy)
+        return [system.load_item(usage_id, course_entry) for usage_id in usage_ids]
+
+    def _get_cache(self, course_version_guid):
+        """
+        Find the descriptor cache for this course if it exists
+        :param course_version_guid:
+        """
+        if not hasattr(self.thread_cache, 'course_cache'):
+            self.thread_cache.course_cache = {}
+        system = self.thread_cache.course_cache
+        return system.get(course_version_guid)
+
+    def _add_cache(self, course_version_guid, system):
+        """
+        Save this cache for subsequent access
+        :param course_version_guid:
+        :param system:
+        """
+        if not hasattr(self.thread_cache, 'course_cache'):
+            self.thread_cache.course_cache = {}
+        self.thread_cache.course_cache[course_version_guid] = system
+        return system
+
+    def _clear_cache(self):
+        """
+        Should only be used by testing or something which implements transactional boundary semantics
+        """
+        self.thread_cache.course_cache = {}
+
+    def _lookup_course(self, course_locator):
+        '''
+        Decode the locator into the right series of db access. Does not
+        return the CourseDescriptor! It returns the actual db json from
+        structures.
+
+        Semantics: if course_id and revision given, then it will get that revision. If
+        also give a version_guid, it will see if the current head of that revision == that guid. If not
+        it raises VersionConflictError (the version now differs from what it was when you got your
+        reference)
+
+        :param course_locator: any subclass of CourseLocator
+        '''
+        # NOTE: if and when this uses cache, the update if changed logic will break if the cache
+        # holds the same objects as the descriptors!
+        if not course_locator.is_fully_specified():
+            raise InsufficientSpecificationError('Not fully specified: %s' % course_locator)
+
+        if course_locator.course_id is not None and course_locator.revision is not None:
+            # use the course_id
+            index = self.course_index.find_one({'_id': course_locator.course_id})
+            if index is None:
+                raise ItemNotFoundError(course_locator)
+            if course_locator.revision not in index['versions']:
+                raise ItemNotFoundError(course_locator)
+            version_guid = index['versions'][course_locator.revision]
+            if course_locator.version_guid is not None and version_guid != course_locator.version_guid:
+                # This may be a bit too touchy but it's hard to infer intent
+                raise VersionConflictError(course_locator, CourseLocator(course_locator, version_guid=version_guid))
+        else:
+            # TODO should this raise an exception if revision was provided?
+            version_guid = course_locator.version_guid
+
+        # cast string to ObjectId if necessary
+        version_guid = course_locator.as_object_id(version_guid)
+        entry = self.structures.find_one({'_id': version_guid})
+
+        # b/c more than one course can use same structure, the 'course_id' is not intrinsic to structure
+        # and the one assoc'd w/ it by another fetch may not be the one relevant to this fetch; so,
+        # fake it by explicitly setting it in the in memory structure.
+
+        if course_locator.course_id:
+            entry['course_id'] = course_locator.course_id
+            entry['revision'] = course_locator.revision
+        return entry
+
+    def get_courses(self, revision, qualifiers=None):
+        '''
+        Returns a list of course descriptors matching any given qualifiers.
+
+        qualifiers should be a dict of keywords matching the db fields or any
+        legal query for mongo to use against the active_versions collection.
+
+        Note, this is to find the current head of the named revision type
+        (e.g., 'draft'). To get specific versions via guid use get_course.
+        '''
+        if qualifiers is None:
+            qualifiers = {}
+        qualifiers.update({"versions.{}".format(revision): {"$exists": True}})
+        matching = self.course_index.find(qualifiers)
+
+        # collect ids and then query for those
+        version_guids = []
+        id_version_map = {}
+        for course_entry in matching:
+            version_guid = course_entry['versions'][revision]
+            version_guids.append(version_guid)
+            id_version_map[version_guid] = course_entry['_id']
+
+        course_entries = self.structures.find({'_id': {'$in': version_guids}})
+
+        # get the block for the course element (s/b the root)
+        result = []
+        for entry in course_entries:
+            # structures are course agnostic but the caller wants to know course, so add it in here
+            entry['course_id'] = id_version_map[entry['_id']]
+            root = entry['root']
+            result.extend(self._load_items(entry, [root], 0, lazy=True))
+        return result
+
+    def get_course(self, course_locator):
+        '''
+        Gets the course descriptor for the course identified by the locator
+        which may or may not be a blockLocator.
+
+        raises InsufficientSpecificationError
+        '''
+        course_entry = self._lookup_course(course_locator)
+        root = course_entry['root']
+        result = self._load_items(course_entry, [root], 0, lazy=True)
+        return result[0]
+
+    def get_course_for_item(self, location):
+        '''
+        Provided for backward compatibility. Is equivalent to calling get_course
+        :param location:
+        '''
+        return self.get_course(location)
+
+    def has_item(self, block_location):
+        """
+        Returns True if location exists in its course. Returns false if
+        the course or the block w/in the course do not exist for the given version.
+        raises InsufficientSpecificationError if the locator does not id a block
+        """
+        if block_location.usage_id is None:
+            raise InsufficientSpecificationError(block_location)
+        try:
+            course_structure = self._lookup_course(block_location)
+        except ItemNotFoundError:
+            # this error only occurs if the course does not exist
+            return False
+
+        return course_structure['blocks'].get(block_location.usage_id) is not None
+
+    def get_item(self, location, depth=0):
+        """
+        depth (int): An argument that some module stores may use to prefetch
+            descendants of the queried modules for more efficient results later
+            in the request. The depth is counted in the number of
+            calls to get_children() to cache. None indicates to cache all
+            descendants.
+        raises InsufficientSpecificationError or ItemNotFoundError
+        """
+        assert isinstance(location, BlockUsageLocator)
+        if not location.is_initialized():
+            raise InsufficientSpecificationError("Not yet initialized: %s" % location)
+        course = self._lookup_course(location)
+        items = self._load_items(course, [location.usage_id], depth, lazy=True)
+        if len(items) == 0:
+            raise ItemNotFoundError(location)
+        return items[0]
+
+    # TODO refactor this and get_courses to use a constructed query
+    def get_items(self, locator, qualifiers):
+        '''
+        Get all of the modules in the given course matching the qualifiers. The
+        qualifiers should only be fields in the structures collection (sorry).
+        There will be a separate search method for searching through
+        definitions.
+
+        Common qualifiers are category, definition (provide definition id),
+        metadata: {display_name ..}, children (return
+        block if its children includes the one given value). If you want
+        substring matching use {$regex: /acme.*corp/i} type syntax.
+
+        Although these
+        look like mongo queries, it is all done in memory; so, you cannot
+        try arbitrary queries.
+
+        :param locator: CourseLocator or BlockUsageLocator restricting search scope
+        :param qualifiers: a dict restricting which elements should match
+        '''
+        # TODO extend to only search a subdag of the course?
+        course = self._lookup_course(locator)
+        items = []
+        for usage_id, value in course['blocks'].iteritems():
+            if self._block_matches(value, qualifiers):
+                items.append(usage_id)
+
+        if len(items) > 0:
+            return self._load_items(course, items, 0, lazy=True)
+        else:
+            return []
+
+    # What's the use case for usage_id being separate?
+    def get_parent_locations(self, locator, usage_id=None):
+        '''
+        Return the locations (Locators w/ usage_ids) for the parents of this location in this
+        course. Could use get_items(location, {'children': usage_id}) but this is slightly faster.
+        NOTE: does not actually ensure usage_id exists
+        If usage_id is None, then the locator must specify the usage_id
+        '''
+        if usage_id is None:
+            usage_id = locator.usage_id
+        course = self._lookup_course(locator)
+        items = []
+        for parent_id, value in course['blocks'].iteritems():
+            for child_id in value['children']:
+                if usage_id == child_id:
+                    locator = locator.as_course_locator()
+                    items.append(BlockUsageLocator(url=locator, usage_id=parent_id))
+        return items
+
+    def get_course_index_info(self, course_locator):
+        """
+        The index records the initial creation of the indexed course and tracks the current version
+        heads. This function is primarily for test verification but may serve some
+        more general purpose.
+        :param course_locator: must have a course_id set
+        :return {'org': , 'prettyid': ,
+            versions: {'draft': the head draft version id,
+                'published': the head published version id if any,
+            },
+            'edited_by': who created the course originally (named edited for consistency),
+            'edited_on': when the course was originally created
+        }
+        """
+        if course_locator.course_id is None:
+            return None
+        index = self.course_index.find_one({'_id': course_locator.course_id})
+        return index
+
+    # TODO figure out a way to make this info accessible from the course descriptor
+    def get_course_history_info(self, course_locator):
+        """
+        Because xblocks doesn't give a means to separate the course structure's meta information from
+        the course xblock's, this method will get that info for the structure as a whole.
+        :param course_locator:
+        :return {'original_version': the version guid of the original version of this course,
+            'previous_version': the version guid of the previous version,
+            'edited_by': who made the last change,
+            'edited_on': when the change was made
+        }
+        """
+        course = self._lookup_course(course_locator)
+        return {'original_version': course['original_version'],
+            'previous_version': course['previous_version'],
+            'edited_by': course['edited_by'],
+            'edited_on': course['edited_on']
+        }
+
+    def get_definition_history_info(self, definition_locator):
+        """
+        Because xblocks doesn't give a means to separate the definition's meta information from
+        the usage xblock's, this method will get that info for the definition
+        :return {'original_version': the version guid of the original version of this course,
+            'previous_version': the version guid of the previous version,
+            'edited_by': who made the last change,
+            'edited_on': when the change was made
+        }
+        """
+        definition = self.definitions.find_one({'_id': definition_locator.definition_id})
+        if definition is None:
+            return None
+        return {'original_version': definition['original_version'],
+            'previous_version': definition['previous_version'],
+            'edited_by': definition['edited_by'],
+            'edited_on': definition['edited_on']
+        }
+
+    def get_course_successors(self, course_locator, version_history_depth=1):
+        '''
+        Find the version_history_depth next versions of this course. Return as a VersionTree
+        Mostly makes sense when course_locator uses a version_guid, but because it finds all relevant
+        next versions, these do include those created for other courses.
+        :param course_locator:
+        '''
+        if version_history_depth < 1:
+            return None
+        if course_locator.version_guid is None:
+            course = self._lookup_course(course_locator)
+            version_guid = course.version_guid
+        else:
+            version_guid = course_locator.version_guid
+
+        # TODO if depth is significant, it may make sense to get all that have the same original_version
+        # and reconstruct the subtree from version_guid
+        next_entries = self.structures.find({'previous_version' : version_guid})
+        # must only scan cursor's once
+        next_versions = [struct for struct in next_entries]
+        result = {version_guid: [CourseLocator(version_guid=struct['_id']) for struct in next_versions]}
+        depth = 1
+        while depth < version_history_depth and len(next_versions) > 0:
+            depth += 1
+            next_entries = self.structures.find({'previous_version':
+                {'$in': [struct['_id'] for struct in next_versions]}})
+            next_versions = [struct for struct in next_entries]
+            for course_structure in next_versions:
+                result.setdefault(course_structure['previous_version'], []).append(
+                    CourseLocator(version_guid=struct['_id']))
+        return VersionTree(CourseLocator(course_locator, version_guid=version_guid), result)
+
+
+    def get_block_generations(self, block_locator):
+        '''
+        Find the history of this block. Return as a VersionTree of each place the block changed (except
+        deletion).
+
+        The block's history tracks its explicit changes; so, changes in descendants won't be reflected
+        as new iterations.
+        '''
+        block_locator = block_locator.version_agnostic()
+        course_struct = self._lookup_course(block_locator)
+        usage_id = block_locator.usage_id
+        update_version_field = 'blocks.{}.update_version'.format(usage_id)
+        all_versions_with_block = self.structures.find({'original_version': course_struct['original_version'],
+            update_version_field: {'$exists': True}})
+        # find (all) root versions and build map previous: [successors]
+        possible_roots = []
+        result = {}
+        for version in all_versions_with_block:
+            if version['_id'] == version['blocks'][usage_id]['update_version']:
+                if version['blocks'][usage_id].get('previous_version') is None:
+                    possible_roots.append(version['blocks'][usage_id]['update_version'])
+                else:
+                    result.setdefault(version['blocks'][usage_id]['previous_version'], set()).add(
+                        version['blocks'][usage_id]['update_version'])
+        # more than one possible_root means usage was added and deleted > 1x.
+        if len(possible_roots) > 1:
+            # find the history segment including block_locator's version
+            element_to_find = course_struct['blocks'][usage_id]['update_version']
+            if element_to_find in possible_roots:
+                possible_roots = [element_to_find]
+            for possibility in possible_roots:
+                if self._find_local_root(element_to_find, possibility, result):
+                    possible_roots = [possibility]
+                    break
+        elif len(possible_roots) == 0:
+            return None
+        # convert the results value sets to locators
+        for k, versions in result.iteritems():
+            result[k] = [BlockUsageLocator(version_guid=version, usage_id=usage_id)
+                for version in versions]
+        return VersionTree(BlockUsageLocator(version_guid=possible_roots[0], usage_id=usage_id), result)
+
+    def get_definition_successors(self, definition_locator, version_history_depth=1):
+        '''
+        Find the version_history_depth next versions of this definition. Return as a VersionTree
+        '''
+        # TODO implement
+        pass
+
+    def create_definition_from_data(self, new_def_data, category, user_id):
+        """
+        Pull the definition fields out of descriptor and save to the db as a new definition
+        w/o a predecessor and return the new id.
+
+        :param user_id: request.user object
+        """
+        document = {"category" : category,
+            "data": new_def_data,
+            "edited_by": user_id,
+            "edited_on": datetime.datetime.utcnow(),
+            "previous_version": None,
+            "original_version": None}
+        new_id = self.definitions.insert(document)
+        definition_locator = DescriptionLocator(new_id)
+        document['original_version'] = new_id
+        self.definitions.update({'_id': new_id}, {'$set': {"original_version": new_id}})
+        return definition_locator
+
+    def update_definition_from_data(self, definition_locator, new_def_data, user_id):
+        """
+        See if new_def_data differs from the persisted version. If so, update
+        the persisted version and return the new id.
+
+        :param user_id: request.user
+        """
+        def needs_saved():
+            if isinstance(new_def_data, dict):
+                for key, value in new_def_data.iteritems():
+                    if key not in old_definition['data'] or value != old_definition['data'][key]:
+                        return True
+                for key, value in old_definition['data'].iteritems():
+                    if key not in new_def_data:
+                        return True
+            else:
+                return new_def_data != old_definition['data']
+
+        # if this looks in cache rather than fresh fetches, then it will probably not detect
+        # actual change b/c the descriptor and cache probably point to the same objects
+        old_definition = self.definitions.find_one({'_id': definition_locator.definition_id})
+        if old_definition is None:
+            raise ItemNotFoundError(definition_locator.url())
+        del old_definition['_id']
+
+        if needs_saved():
+            old_definition['data'] = new_def_data
+            old_definition['edited_by'] = user_id
+            old_definition['edited_on'] = datetime.datetime.utcnow()
+            old_definition['previous_version'] = definition_locator.definition_id
+            new_id = self.definitions.insert(old_definition)
+            return DescriptionLocator(new_id), True
+        else:
+            return definition_locator, False
+
+    def _generate_usage_id(self, course_blocks, category):
+        """
+        Generate a somewhat readable block id unique w/in this course using the category
+        :param course_blocks: the current list of blocks.
+        :param category:
+        """
+        # NOTE: a potential bug is that a block is deleted and another created which gets the old
+        # block's id. a possible fix is to cache the last serial in a dict in the structure
+        # {category: last_serial...}
+        # A potential confusion is if the name incorporates the parent's name, then if the child
+        # moves, its id won't change and will be confusing
+        serial = 1
+        while category + str(serial) in course_blocks:
+            serial += 1
+        return category + str(serial)
+
+    def _generate_course_id(self, id_root):
+        """
+        Generate a somewhat readable course id unique w/in this db using the id_root
+        :param course_blocks: the current list of blocks.
+        :param category:
+        """
+        existing_uses = self.course_index.find({"_id": {"$regex": id_root}})
+        if existing_uses.count() > 0:
+            max_found = 0
+            matcher = re.compile(id_root + r'(\d+)')
+            for entry in existing_uses:
+                serial = re.search(matcher, entry['_id'])
+                if serial is not None and serial.groups > 0:
+                    value = int(serial.group(1))
+                    if value > max_found:
+                        max_found = value
+            return id_root + str(max_found + 1)
+        else:
+            return id_root
+
+    # TODO I would love to write this to take a real descriptor and persist it BUT descriptors, kvs, and dbmodel
+    #      all assume locators are set and unique! Having this take the model contents piecemeal breaks the separation
+    #      of model from persistence layer
+    def create_item(self, course_or_parent_locator, category, user_id, definition_locator=None, new_def_data=None,
+        metadata=None, force=False):
+        """
+        Add a descriptor to persistence as the last child of the optional parent_location or just as an element
+        of the course (if no parent provided). Return the resulting post saved version with populated locators.
+
+        If the locator is a BlockUsageLocator, then it's assumed to be the parent. If it's a CourseLocator, then it's
+        merely the containing course.
+
+        raises InsufficientSpecificationError if there is no course locator.
+        raises VersionConflictError if course_id and version_guid given and the current version head != version_guid
+            and force is not True.
+        force: fork the structure and don't update the course draftVersion if the above
+
+        The incoming definition_locator should either be None to indicate this is a brand new definition or
+        a pointer to the existing definition to which this block should point or from which this was derived.
+        If new_def_data is None, then definition_locator must have a value meaning that this block points
+        to the existing definition. If new_def_data is not None and definition_location is not None, then
+        new_def_data is assumed to be a new payload for definition_location.
+
+        Creates a new version of the course structure, creates and inserts the new block, makes the block point
+        to the definition which may be new or a new version of an existing or an existing.
+        Rules for course locator:
+
+        * If the course locator specifies a course_id and either it doesn't
+          specify version_guid or the one it specifies == the current draft, it progresses the course to point
+          to the new draft and sets the active version to point to the new draft
+        * If the locator has a course_id but its version_guid != current draft, it raises VersionConflictError.
+
+        NOTE: using a version_guid will end up creating a new version of the course. Your new item won't be in
+        the course id'd by version_guid but instead in one w/ a new version_guid. Ensure in this case that you get
+        the new version_guid from the locator in the returned object!
+        """
+        # find course_index entry if applicable and structures entry
+        index_entry = self._get_index_if_valid(course_or_parent_locator, force)
+        structure = self._lookup_course(course_or_parent_locator)
+
+        # persist the definition if persisted != passed
+        if (definition_locator is None or definition_locator.definition_id is None):
+            definition_locator = self.create_definition_from_data(new_def_data, category, user_id)
+        elif new_def_data is not None:
+            definition_locator, _ = self.update_definition_from_data(definition_locator, new_def_data, user_id)
+
+        # copy the structure and modify the new one
+        new_structure = self._version_structure(structure, user_id)
+        # generate an id
+        new_usage_id = self._generate_usage_id(new_structure['blocks'], category)
+        update_version_keys = ['blocks.{}.update_version'.format(new_usage_id)]
+        if isinstance(course_or_parent_locator, BlockUsageLocator) and course_or_parent_locator.usage_id is not None:
+            parent = new_structure['blocks'][course_or_parent_locator.usage_id]
+            parent['children'].append(new_usage_id)
+            parent['edited_on'] = datetime.datetime.utcnow()
+            parent['edited_by'] = user_id
+            parent['previous_version'] = parent['update_version']
+            update_version_keys.append('blocks.{}.update_version'.format(course_or_parent_locator.usage_id))
+        new_structure['blocks'][new_usage_id] = {
+            "children": [],
+            "category": category,
+            "definition": definition_locator.definition_id,
+            "metadata": metadata if metadata else {},
+            'edited_on': datetime.datetime.utcnow(),
+            'edited_by': user_id,
+            'previous_version': None
+            }
+        new_id = self.structures.insert(new_structure)
+        update_version_payload = {key: new_id for key in update_version_keys}
+        self.structures.update({'_id': new_id},
+            {'$set': update_version_payload})
+
+        # update the index entry if appropriate
+        if index_entry is not None:
+            self._update_head(index_entry, course_or_parent_locator.revision, new_id)
+            course_parent = course_or_parent_locator.as_course_locator()
+        else:
+            course_parent = None
+
+        # fetch and return the new item--fetching is unnecessary but a good qc step
+        return self.get_item(BlockUsageLocator(course_id=course_parent,
+                                               usage_id=new_usage_id,
+                                               version_guid=new_id))
+
+    def create_course(self, org, prettyid, user_id, id_root=None, metadata=None, course_data=None,
+        master_version='draft', versions_dict=None, root_category='course'):
+        """
+        Create a new entry in the active courses index which points to an existing or new structure. Returns
+        the course root of the resulting entry (the location has the course id)
+
+        id_root: allows the caller to specify the course_id. It's a root in that, if it's already taken,
+        this method will append things to the root to make it unique. (defaults to org)
+
+        metadata: if provided, will set the metadata of the root course object in the new draft course. If both
+        metadata and a starting version are provided, it will generate a successor version to the given version,
+        and update the metadata with any provided values (via update not setting).
+
+        course_data: if provided, will update the data of the new course xblock definition to this. Like metadata,
+        if provided, this will cause a new version of any given version as well as a new version of the
+        definition (which will point to the existing one if given a version). If not provided and given
+        a draft_version, it will reuse the same definition as the draft course (obvious since it's reusing the draft
+        course). If not provided and no draft is given, it will be empty and get the field defaults (hopefully) when
+        loaded.
+
+        master_version: the tag (key) for the version name in the dict which is the 'draft' version. Not the actual
+        version guid, but what to call it.
+
+        versions_dict: the starting version ids where the keys are the tags such as 'draft' and 'published'
+        and the values are structure guids. If provided, the new course will reuse this version (unless you also
+        provide any overrides such as metadata, see above). if not provided, will create a mostly empty course
+        structure with just a category course root xblock.
+        """
+        if metadata is None:
+            metadata = {}
+        # build from inside out: definition, structure, index entry
+        # if building a wholly new structure
+        if versions_dict is None or master_version not in versions_dict:
+            # create new definition and structure
+            if course_data is None:
+                course_data = {}
+            definition_entry = {
+                'category': root_category,
+                'data': course_data,
+                'edited_by': user_id,
+                'edited_on': datetime.datetime.utcnow(),
+                'previous_version': None,
+                }
+            definition_id = self.definitions.insert(definition_entry)
+            definition_entry['original_version'] = definition_id
+            self.definitions.update({'_id': definition_id}, {'$set': {"original_version": definition_id}})
+
+            draft_structure = {
+                'root': 'course',
+                'previous_version': None,
+                'edited_by': user_id,
+                'edited_on': datetime.datetime.utcnow(),
+                'blocks': {
+                    'course': {
+                        'children':[],
+                        'category': 'course',
+                        'definition': definition_id,
+                        'metadata': metadata,
+                        'edited_on': datetime.datetime.utcnow(),
+                        'edited_by': user_id,
+                        'previous_version': None}}}
+            new_id = self.structures.insert(draft_structure)
+            draft_structure['original_version'] = new_id
+            self.structures.update({'_id': new_id},
+                {'$set': {"original_version": new_id,
+                    'blocks.course.update_version': new_id}})
+            if versions_dict is None:
+                versions_dict = {master_version: new_id}
+            else:
+                versions_dict[master_version] = new_id
+
+        else:
+            # just get the draft_version structure
+            draft_version = CourseLocator(version_guid=versions_dict[master_version])
+            draft_structure = self._lookup_course(draft_version)
+            if course_data is not None or metadata:
+                draft_structure = self._version_structure(draft_structure, user_id)
+                root_block = draft_structure['blocks'][draft_structure['root']]
+                if metadata is not None:
+                    root_block['metadata'].update(metadata)
+                if course_data is not None:
+                    definition = self.definitions.find_one({'_id': root_block['definition']})
+                    definition['data'].update(course_data)
+                    definition['previous_version'] = definition['_id']
+                    definition['edited_by'] = user_id
+                    definition['edited_on'] = datetime.datetime.utcnow()
+                    del definition['_id']
+                    root_block['definition'] = self.definitions.insert(definition)
+                    root_block['edited_on'] = datetime.datetime.utcnow()
+                    root_block['edited_by'] = user_id
+                    root_block['previous_version'] = root_block.get('update_version')
+                # insert updates the '_id' in draft_structure
+                new_id = self.structures.insert(draft_structure)
+                versions_dict[master_version] = new_id
+                self.structures.update({'_id': new_id},
+                    {'$set': {'blocks.{}.update_version'.format(draft_structure['root']): new_id}})
+        # create the index entry
+        if id_root is None:
+            id_root = org
+        new_id = self._generate_course_id(id_root)
+
+        index_entry = {
+            '_id': new_id,
+            'org': org,
+            'prettyid': prettyid,
+            'edited_by': user_id,
+            'edited_on': datetime.datetime.utcnow(),
+            'versions': versions_dict}
+        new_id = self.course_index.insert(index_entry)
+        return self.get_course(CourseLocator(course_id=new_id, revision=master_version))
+
+    def update_item(self, descriptor, user_id, force=False):
+        """
+        Save the descriptor's definition, metadata, & children references (i.e., it doesn't descend the tree).
+        Return the new descriptor (updated location).
+
+        raises ItemNotFoundError if the location does not exist.
+
+        Creates a new course version. If the descriptor's location has a course_id, it moves the course head
+        pointer. If the version_guid of the descriptor points to a non-head version and there's been an intervening
+        change to this item, it raises a VersionConflictError unless force is True. In the force case, it forks
+        the course but leaves the head pointer where it is (this change will not be in the course head).
+
+        The implementation tries to detect which, if any changes, actually need to be saved and thus won't version
+        the definition, structure, nor course if they didn't change.
+        """
+        original_structure = self._lookup_course(descriptor.location)
+        index_entry = self._get_index_if_valid(descriptor.location, force)
+
+        descriptor.definition_locator, is_updated = self.update_definition_from_data(
+            descriptor.definition_locator, descriptor.xblock_kvs.get_data(), user_id)
+        # check children
+        original_entry = original_structure['blocks'][descriptor.location.usage_id]
+        if (not is_updated and descriptor.has_children
+            and not self._xblock_lists_equal(original_entry['children'], descriptor.children)):
+            is_updated = True
+        # check metadata
+        if not is_updated:
+            is_updated = self._compare_metadata(descriptor.xblock_kvs.get_own_metadata(), original_entry['metadata'])
+
+        # if updated, rev the structure
+        if is_updated:
+            new_structure = self._version_structure(original_structure, user_id)
+            block_data = new_structure['blocks'][descriptor.location.usage_id]
+            if descriptor.has_children:
+                block_data["children"] = [self._usage_id(child) for child in descriptor.children]
+
+            block_data["definition"] = descriptor.definition_locator.definition_id
+            block_data["metadata"] = descriptor.xblock_kvs.get_own_metadata()
+            block_data['edited_on'] = datetime.datetime.utcnow()
+            block_data['edited_by'] = user_id
+            block_data['previous_version'] = block_data['update_version']
+            new_id = self.structures.insert(new_structure)
+            self.structures.update({'_id': new_id},
+                {'$set': {'blocks.{}.update_version'.format(descriptor.location.usage_id): new_id}})
+
+            # update the index entry if appropriate
+            if index_entry is not None:
+                self._update_head(index_entry, descriptor.location.revision, new_id)
+
+            # fetch and return the new item--fetching is unnecessary but a good qc step
+            return self.get_item(BlockUsageLocator(descriptor.location, version_guid=new_id))
+        else:
+            # nothing changed, just return the one sent in
+            return descriptor
+
+    def persist_xblock_dag(self, xblock, user_id, force=False):
+        """
+        create or update the xblock and all of its children. The xblock's location must specify a course.
+        If it doesn't specify a usage_id, then it's presumed to be new and need creation. This function
+        descends the children performing the same operation for any that are xblocks. Any children which
+        are usage_ids just update the children pointer.
+
+        All updates go into the same course version (bulk updater).
+
+        Updates the objects which came in w/ updated location and definition_location info.
+
+        returns the post-persisted version of the incoming xblock. Note that its children will be ids not
+        objects.
+
+        :param xblock:
+        :param user_id:
+        """
+        # find course_index entry if applicable and structures entry
+        index_entry = self._get_index_if_valid(xblock.location, force)
+        structure = self._lookup_course(xblock.location)
+        new_structure = self._version_structure(structure, user_id)
+
+        changed_blocks = self._persist_subdag(xblock, user_id, new_structure['blocks'])
+
+        if changed_blocks:
+            new_id = self.structures.insert(new_structure)
+            update_command = {}
+            for usage_id in changed_blocks:
+                update_command['blocks.{}.update_version'.format(usage_id)] = new_id
+            self.structures.update({'_id': new_id}, {'$set': update_command})
+
+            # update the index entry if appropriate
+            if index_entry is not None:
+                self._update_head(index_entry, xblock.location.revision, new_id)
+
+            # fetch and return the new item--fetching is unnecessary but a good qc step
+            return self.get_item(BlockUsageLocator(xblock.location, version_guid=new_id))
+        else:
+            return xblock
+
+    def _persist_subdag(self, xblock, user_id, structure_blocks):
+        # persist the definition if persisted != passed
+        new_def_data = xblock.xblock_kvs.get_data()
+        if (xblock.definition_locator is None or xblock.definition_locator.definition_id is None):
+            xblock.definition_locator = self.create_definition_from_data(new_def_data,
+                xblock.category, user_id)
+            is_updated = True
+        elif new_def_data is not None:
+            xblock.definition_locator, is_updated = self.update_definition_from_data(xblock.definition_locator,
+                new_def_data, user_id)
+
+        if xblock.location.usage_id is None:
+            # generate an id
+            is_new = True
+            is_updated = True
+            usage_id = self._generate_usage_id(structure_blocks, xblock.category)
+            xblock.location.usage_id = usage_id
+        else:
+            is_new = False
+            usage_id = xblock.location.usage_id
+            if (not is_updated and xblock.has_children
+                and not self._xblock_lists_equal(structure_blocks[usage_id]['children'], xblock.children)):
+                is_updated = True
+
+        children = []
+        updated_blocks = []
+        if xblock.has_children:
+            for child in xblock.children:
+                if isinstance(child, XModuleDescriptor):
+                    updated_blocks += self._persist_subdag(child, user_id, structure_blocks)
+                    children.append(child.location.usage_id)
+                else:
+                    children.append(child)
+
+        is_updated = is_updated or updated_blocks
+        metadata = xblock.xblock_kvs.get_own_metadata()
+        if not is_new and not is_updated:
+            is_updated = self._compare_metadata(metadata, structure_blocks[usage_id]['metadata'])
+
+        if is_updated:
+            structure_blocks[usage_id] = {
+                "children": children,
+                "category": xblock.category,
+                "definition": xblock.definition_locator.definition_id,
+                "metadata": metadata if metadata else {},
+                'previous_version': structure_blocks.get(usage_id, {}).get('update_version'),
+                'edited_by': user_id,
+                'edited_on': datetime.datetime.utcnow()
+            }
+            updated_blocks.append(usage_id)
+
+        return updated_blocks
+
+    def _compare_metadata(self, metadata, original_metadata):
+        original_keys = original_metadata.keys()
+        if len(metadata) != len(original_keys):
+            return True
+        else:
+            new_keys = metadata.keys()
+            for key in original_keys:
+                if key not in new_keys or original_metadata[key] != metadata[key]:
+                    return True
+
+    # TODO change all callers to update_item
+    def update_children(self, course_id, location, children):
+        raise NotImplementedError()
+
+    # TODO change all callers to update_item
+    def update_metadata(self, course_id, location, metadata):
+        raise NotImplementedError()
+
+    def update_course_index(self, course_locator, new_values_dict, update_versions=False):
+        """
+        Change the given course's index entry for the given fields. new_values_dict
+        should be a subset of the dict returned by get_course_index_info.
+        It cannot include '_id' (will raise IllegalArgument).
+        Provide update_versions=True if you intend this to replace the versions hash.
+        Note, this operation can be dangerous and break running courses.
+
+        If the dict includes versions and not update_versions, it will raise an exception.
+
+        If the dict includes edited_on or edited_by, it will raise an exception
+
+        Does not return anything useful.
+        """
+        # TODO how should this log the change? edited_on and edited_by for this entry
+        # has the semantic of who created the course and when; so, changing those will lose
+        # that information.
+        if '_id' in new_values_dict:
+            raise ValueError("Cannot override _id")
+        if 'edited_on' in new_values_dict or 'edited_by' in new_values_dict:
+            raise ValueError("Cannot set edited_on or edited_by")
+        if not update_versions and 'versions' in new_values_dict:
+            raise ValueError("Cannot override versions without setting update_versions")
+        self.course_index.update({'_id': course_locator.course_id},
+            {'$set': new_values_dict})
+
+    def delete_item(self, usage_locator, user_id, force=False):
+        """
+        Delete the tree rooted at block and any references w/in the course to the block
+        from a new version of the course structure.
+
+        returns CourseLocator for new version
+
+        raises ItemNotFoundError if the location does not exist.
+        raises ValueError if usage_locator points to the structure root
+
+        Creates a new course version. If the descriptor's location has a course_id, it moves the course head
+        pointer. If the version_guid of the descriptor points to a non-head version and there's been an intervening
+        change to this item, it raises a VersionConflictError unless force is True. In the force case, it forks
+        the course but leaves the head pointer where it is (this change will not be in the course head).
+        """
+        assert isinstance(usage_locator, BlockUsageLocator) and usage_locator.is_initialized()
+        original_structure = self._lookup_course(usage_locator)
+        if original_structure['root'] == usage_locator.usage_id:
+            raise ValueError("Cannot delete the root of a course")
+        index_entry = self._get_index_if_valid(usage_locator, force)
+        new_structure = self._version_structure(original_structure, user_id)
+        new_blocks = new_structure['blocks']
+        parents = self.get_parent_locations(usage_locator)
+        update_version_keys = []
+        for parent in parents:
+            parent_block = new_blocks[parent.usage_id]
+            parent_block['children'].remove(usage_locator.usage_id)
+            parent_block['edited_on'] = datetime.datetime.utcnow()
+            parent_block['edited_by'] = user_id
+            parent_block['previous_version'] = parent_block['update_version']
+            update_version_keys.append('blocks.{}.update_version'.format(parent.usage_id))
+        # remove subtree
+        def remove_subtree(usage_id):
+            for child in new_blocks[usage_id]['children']:
+                remove_subtree(child)
+            del new_blocks[usage_id]
+        remove_subtree(usage_locator.usage_id)
+
+        # update index if appropriate and structures
+        new_id = self.structures.insert(new_structure)
+        if update_version_keys:
+            update_version_payload = {key: new_id for key in update_version_keys}
+            self.structures.update({'_id': new_id}, {'$set': update_version_payload})
+
+        result = CourseLocator(version_guid=new_id)
+
+        # update the index entry if appropriate
+        if index_entry is not None:
+            self._update_head(index_entry, usage_locator.revision, new_id)
+            result.course_id = usage_locator.course_id
+            result.revision = usage_locator.revision
+
+        return result
+
+    def delete_course(self, course_id):
+        """
+        Remove the given course from the course index.
+
+        Only removes the course from the index. The data remains. You can use create_course
+        with a versions hash to restore the course; however, the edited_on and
+        edited_by won't reflect the originals, of course.
+
+        :param course_id: uses course_id rather than locator to emphasize its global effect
+        """
+        index = self.course_index.find_one({'_id': course_id})
+        if index is None:
+            raise ItemNotFoundError(course_id)
+        # this is the only real delete in the system. should it do something else?
+        self.course_index.remove(index['_id'])
+
+    # TODO remove all callers and then this
+    def get_errored_courses(self):
+        """
+        This function doesn't make sense for the mongo modulestore, as structures
+        are loaded on demand, rather than up front
+        """
+        return {}
+
+    def inherit_metadata(self, block_map, block, inheriting_metadata=None):
+        """
+        Updates block with any value
+        that exist in inheriting_metadata and don't appear in block['metadata'],
+        and then inherits block['metadata'] to all of the children in
+        block['children']. Filters by inheritance.INHERITABLE_METADATA
+        """
+        if block is None:
+            return
+
+        if inheriting_metadata is None:
+            inheriting_metadata = {}
+
+        # the currently passed down values take precedence over any previously cached ones
+        # NOTE: this should show the values which all fields would have if inherited: i.e.,
+        # not set to the locally defined value but to value set by nearest ancestor who sets it
+        block.setdefault('_inherited_metadata', {}).update(inheriting_metadata)
+
+        # update the inheriting w/ what should pass to children
+        inheriting_metadata = block['_inherited_metadata'].copy()
+        for field in inheritance.INHERITABLE_METADATA:
+            if field in block['metadata']:
+                inheriting_metadata[field] = block['metadata'][field]
+
+        for child in block.get('children', []):
+            self.inherit_metadata(block_map, block_map[child], inheriting_metadata)
+
+    def descendants(self, block_map, usage_id, depth, descendent_map):
+        """
+        adds block and its descendants out to depth to descendent_map
+        Depth specifies the number of levels of descendants to return
+        (0 => this usage only, 1 => this usage and its children, etc...)
+        A depth of None returns all descendants
+        """
+        if usage_id not in block_map:
+            return descendent_map
+
+        if usage_id not in descendent_map:
+            descendent_map[usage_id] = block_map[usage_id]
+
+        if depth is None or depth > 0:
+            depth = depth - 1 if depth is not None else None
+            for child in block_map[usage_id].get('children', []):
+                descendent_map = self.descendants(block_map, child, depth,
+                    descendent_map)
+
+        return descendent_map
+
+    def definition_locator(self, definition):
+        '''
+        Pull the id out of the definition w/ correct semantics for its
+        representation
+        '''
+        if isinstance(definition, DefinitionLazyLoader):
+            return definition.definition_locator
+        elif '_id' not in definition:
+            return None
+        else:
+            return DescriptionLocator(definition['_id'])
+
+    def _block_matches(self, value, qualifiers):
+        '''
+        Return True or False depending on whether the value (block contents)
+        matches the qualifiers as per get_items
+        :param value:
+        :param qualifiers:
+        '''
+        for key, criteria in qualifiers.iteritems():
+            if key in value:
+                target = value[key]
+                if not self._value_matches(target, criteria):
+                    return False
+            elif criteria is not None:
+                return False
+        return True
+
+    def _value_matches(self, target, criteria):
+        ''' helper for _block_matches '''
+        if isinstance(target, list):
+            return any(self._value_matches(ele, criteria)
+                for ele in target)
+        elif isinstance(criteria, dict):
+            if '$regex' in criteria:
+                return re.search(criteria['$regex'], target) is not None
+            elif not isinstance(target, dict):
+                return False
+            else:
+                return (isinstance(target, dict) and
+                        self._block_matches(target, criteria))
+        else:
+            return criteria == target
+
+    def _xblock_lists_equal(self, lista, listb):
+        """
+        Do the 2 lists refer to the same xblocks in the same order (presumes they're from the
+        same course)
+
+        :param lista:
+        :param listb:
+        """
+        if len(lista) != len(listb):
+            return False
+        for idx in enumerate(lista):
+            if lista[idx] != listb[idx]:
+                itema = self._usage_id(lista[idx])
+                if itema != self._usage_id(listb[idx]):
+                    return False
+        return True
+
+    def _usage_id(self, xblock_or_id):
+        """
+        arg is either an xblock or an id. If an xblock, get the usage_id from its location. Otherwise, return itself.
+        :param xblock_or_id:
+        """
+        if isinstance(xblock_or_id, XModuleDescriptor):
+            return xblock_or_id.location.usage_id
+        else:
+            return xblock_or_id
+
+    def _get_index_if_valid(self, locator, force=False):
+        """
+        If the locator identifies a course and points to its draft (or plausibly its draft),
+        then return the index entry.
+
+        raises VersionConflictError if not the right version
+
+        :param locator:
+        """
+        if locator.course_id is None or locator.revision is None:
+            return None
+        else:
+            index_entry = self.course_index.find_one({'_id': locator.course_id})
+            if (locator.version_guid is not None
+                and index_entry['versions'][locator.revision] != locator.version_guid
+                and not force):
+                raise VersionConflictError(
+                    locator,
+                    CourseLocator(
+                        course_id=index_entry['_id'],
+                        version_guid=index_entry['versions'][locator.revision],
+                        revision=locator.revision))
+            else:
+                return index_entry
+
+    def _version_structure(self, structure, user_id):
+        """
+        Copy the structure and update the history info (edited_by, edited_on, previous_version)
+        :param structure:
+        :param user_id:
+        """
+        new_structure = structure.copy()
+        new_structure['blocks'] = new_structure['blocks'].copy()
+        del new_structure['_id']
+        new_structure['previous_version'] = structure['_id']
+        new_structure['edited_by'] = user_id
+        new_structure['edited_on'] = datetime.datetime.utcnow()
+        return new_structure
+
+    def _find_local_root(self, element_to_find, possibility, tree):
+        if possibility not in tree:
+            return False
+        if element_to_find in tree[possibility]:
+            return True
+        for subtree in tree[possibility]:
+            if self._find_local_root(element_to_find, subtree, tree):
+                return True
+        return False
+
+
+    def _update_head(self, index_entry, revision, new_id):
+        """
+        Update the active index for the given course's revision to point to new_id
+
+        :param index_entry:
+        :param course_locator:
+        :param new_id:
+        """
+        self.course_index.update(
+            {"_id": index_entry["_id"]},
+            {"$set": {"versions.{}".format(revision): new_id}})

common/lib/xmodule/xmodule/modulestore/split_mongo/split_mongo_kvs.py

@@ -0,0 +1,163 @@
+import copy
+from xblock.core import Scope
+from collections import namedtuple
+from xblock.runtime import KeyValueStore, InvalidScopeError
+from .definition_lazy_loader import DefinitionLazyLoader
+
+# id is a BlockUsageLocator, def_id is the definition's guid
+SplitMongoKVSid = namedtuple('SplitMongoKVSid', 'id, def_id')
+
+
+# TODO should this be here or w/ x_module or ???
+class SplitMongoKVS(KeyValueStore):
+    """
+    A KeyValueStore that maps keyed data access to one of the 3 data areas
+    known to the MongoModuleStore (data, children, and metadata)
+    """
+    def __init__(self, definition, children, metadata, _inherited_metadata, location, category):
+        """
+
+        :param definition:
+        :param children:
+        :param metadata: the locally defined value for each metadata field
+        :param _inherited_metadata: the value of each inheritable field from above this.
+            Note, metadata may override and disagree w/ this b/c this says what the value
+            should be if metadata is undefined for this field.
+        """
+        # ensure kvs's don't share objects w/ others so that changes can't appear in separate ones
+        # the particular use case was that changes to kvs's were polluting caches. My thinking was
+        # that kvs's should be independent thus responsible for the isolation.
+        if isinstance(definition, DefinitionLazyLoader):
+            self._definition = definition
+        else:
+            self._definition = copy.copy(definition)
+        self._children = copy.copy(children)
+        self._metadata = copy.copy(metadata)
+        self._inherited_metadata = _inherited_metadata
+        self._location = location
+        self._category = category
+
+    def get(self, key):
+        if key.scope == Scope.children:
+            return self._children
+        elif key.scope == Scope.parent:
+            return None
+        elif key.scope == Scope.settings:
+            if key.field_name in self._metadata:
+                return self._metadata[key.field_name]
+            elif key.field_name in self._inherited_metadata:
+                return self._inherited_metadata[key.field_name]
+            else:
+                raise KeyError()
+        elif key.scope == Scope.content:
+            if key.field_name == 'location':
+                return self._location
+            elif key.field_name == 'category':
+                return self._category
+            else:
+                if isinstance(self._definition, DefinitionLazyLoader):
+                    self._definition = self._definition.fetch()
+                if (key.field_name == 'data' and
+                        not isinstance(self._definition.get('data'), dict)):
+                    return self._definition.get('data')
+                elif 'data' not in self._definition or key.field_name not in self._definition['data']:
+                    raise KeyError()
+                else:
+                    return self._definition['data'][key.field_name]
+        else:
+            raise InvalidScopeError(key.scope)
+
+    def set(self, key, value):
+        # TODO cache db update implications & add method to invoke
+        if key.scope == Scope.children:
+            self._children = value
+            # TODO remove inheritance from any orphaned exchildren
+            # TODO add inheritance to any new children
+        elif key.scope == Scope.settings:
+            # TODO if inheritable, push down to children who don't override
+            self._metadata[key.field_name] = value
+        elif key.scope == Scope.content:
+            if key.field_name == 'location':
+                self._location = value
+            elif key.field_name == 'category':
+                self._category = value
+            else:
+                if isinstance(self._definition, DefinitionLazyLoader):
+                    self._definition = self._definition.fetch()
+                if (key.field_name == 'data' and
+                        not isinstance(self._definition.get('data'), dict)):
+                    self._definition.get('data')
+                else:
+                    self._definition.setdefault('data', {})[key.field_name] = value
+        else:
+            raise InvalidScopeError(key.scope)
+
+    def delete(self, key):
+        # TODO cache db update implications & add method to invoke
+        if key.scope == Scope.children:
+            self._children = []
+        elif key.scope == Scope.settings:
+            # TODO if inheritable, ensure _inherited_metadata has value from above and
+            # revert children to that value
+            if key.field_name in self._metadata:
+                del self._metadata[key.field_name]
+        elif key.scope == Scope.content:
+            # don't allow deletion of location nor category
+            if key.field_name == 'location':
+                pass
+            elif key.field_name == 'category':
+                pass
+            else:
+                if isinstance(self._definition, DefinitionLazyLoader):
+                    self._definition = self._definition.fetch()
+                if (key.field_name == 'data' and
+                        not isinstance(self._definition.get('data'), dict)):
+                    self._definition.setdefault('data', None)
+                else:
+                    try:
+                        del self._definition['data'][key.field_name]
+                    except KeyError:
+                        pass
+        else:
+            raise InvalidScopeError(key.scope)
+
+    def has(self, key):
+        if key.scope in (Scope.children, Scope.parent):
+            return True
+        elif key.scope == Scope.settings:
+            return key.field_name in self._metadata or key.field_name in self._inherited_metadata
+        elif key.scope == Scope.content:
+            if key.field_name == 'location':
+                return True
+            elif key.field_name == 'category':
+                return self._category is not None
+            else:
+                if isinstance(self._definition, DefinitionLazyLoader):
+                    self._definition = self._definition.fetch()
+                if (key.field_name == 'data' and
+                        not isinstance(self._definition.get('data'), dict)):
+                    return self._definition.get('data') is not None
+                else:
+                    return key.field_name in self._definition.get('data', {})
+        else:
+            return False
+
+    def get_data(self):
+        """
+        Intended only for use by persistence layer to get the native definition['data'] rep
+        """
+        if isinstance(self._definition, DefinitionLazyLoader):
+            self._definition = self._definition.fetch()
+        return self._definition.get('data')
+
+    def get_own_metadata(self):
+        """
+        Get the metadata explicitly set on this element.
+        """
+        return self._metadata
+
+    def get_inherited_metadata(self):
+        """
+        Get the metadata set by the ancestors (which own metadata may override or not)
+        """
+        return self._inherited_metadata

common/lib/xmodule/xmodule/modulestore/tests/persistent_factories.py

@@ -0,0 +1,96 @@
+from xmodule.modulestore.django import modulestore
+from xmodule.course_module import CourseDescriptor
+from xmodule.x_module import XModuleDescriptor
+import factory
+
+
+# [dhm] I'm not sure why we're using factory_boy if we're not following its pattern. If anyone
+# assumes they can call build, it will completely fail, for example.
+# pylint: disable=W0232
+class PersistentCourseFactory(factory.Factory):
+    """
+    Create a new course (not a new version of a course, but a whole new index entry).
+
+    keywords:
+    * org: defaults to textX
+    * prettyid: defaults to 999
+    * display_name
+    * user_id
+    * data (optional) the data payload to save in the course item
+    * metadata (optional) the metadata payload. If display_name is in the metadata, that takes
+    precedence over any display_name provided directly.
+    """
+    FACTORY_FOR = CourseDescriptor
+
+    org = 'testX'
+    prettyid = '999'
+    display_name = 'Robot Super Course'
+    user_id = "test_user"
+    data = None
+    metadata = None
+    master_version = 'draft'
+
+    # pylint: disable=W0613
+    @classmethod
+    def _create(cls, target_class, *args, **kwargs):
+
+        org = kwargs.get('org')
+        prettyid = kwargs.get('prettyid')
+        display_name = kwargs.get('display_name')
+        user_id = kwargs.get('user_id')
+        data = kwargs.get('data')
+        metadata = kwargs.get('metadata', {})
+        if metadata is None:
+            metadata = {}
+        if 'display_name' not in metadata:
+            metadata['display_name'] = display_name
+
+        # Write the data to the mongo datastore
+        new_course = modulestore('split').create_course(
+            org, prettyid, user_id, metadata=metadata, course_data=data, id_root=prettyid,
+            master_version=kwargs.get('master_version'))
+
+        return new_course
+
+    @classmethod
+    def _build(cls, target_class, *args, **kwargs):
+        raise NotImplementedError()
+
+
+class ItemFactory(factory.Factory):
+    FACTORY_FOR = XModuleDescriptor
+
+    category = 'chapter'
+    user_id = 'test_user'
+    display_name = factory.LazyAttributeSequence(lambda o, n: "{} {}".format(o.category, n))
+
+    # pylint: disable=W0613
+    @classmethod
+    def _create(cls, target_class, *args, **kwargs):
+        """
+        Uses *kwargs*:
+
+        *parent_location* (required): the location of the course & possibly parent
+
+        *category* (defaults to 'chapter')
+
+        *data* (optional): the data for the item
+
+        definition_locator (optional): the DescriptorLocator for the definition this uses or branches
+
+        *display_name* (optional): the display name of the item
+
+        *metadata* (optional): dictionary of metadata attributes (display_name here takes
+        precedence over the above attr)
+        """
+        metadata = kwargs.get('metadata', {})
+        if 'display_name' not in metadata and 'display_name' in kwargs:
+            metadata['display_name'] = kwargs['display_name']
+
+        return modulestore('split').create_item(kwargs['parent_location'], kwargs['category'],
+            kwargs['user_id'], definition_locator=kwargs.get('definition_locator'),
+            new_def_data=kwargs.get('data'), metadata=metadata)
+
+    @classmethod
+    def _build(cls, target_class, *args, **kwargs):
+        raise NotImplementedError()

common/lib/xmodule/xmodule/modulestore/tests/test_locators.py

@@ -0,0 +1,539 @@
+'''
+Created on Mar 14, 2013
+
+@author: dmitchell
+'''
+from unittest import TestCase
+from nose.plugins.skip import SkipTest
+
+from bson.objectid import ObjectId
+from xmodule.modulestore.locator import Locator, CourseLocator, BlockUsageLocator
+from xmodule.modulestore.exceptions import InvalidLocationError, \
+    InsufficientSpecificationError, OverSpecificationError
+
+
+class LocatorTest(TestCase):
+
+    def test_cant_instantiate_abstract_class(self):
+        self.assertRaises(TypeError, Locator)
+
+    def test_course_constructor_overspecified(self):
+        self.assertRaises(
+            OverSpecificationError,
+            CourseLocator,
+            url='edx://edu.mit.eecs.6002x',
+            course_id='edu.harvard.history',
+            revision='published',
+            version_guid=ObjectId())
+        self.assertRaises(
+            OverSpecificationError,
+            CourseLocator,
+            url='edx://edu.mit.eecs.6002x',
+            course_id='edu.harvard.history',
+            version_guid=ObjectId())
+        self.assertRaises(
+            OverSpecificationError,
+            CourseLocator,
+            url='edx://edu.mit.eecs.6002x;published',
+            revision='draft')
+        self.assertRaises(
+            OverSpecificationError,
+            CourseLocator,
+            course_id='edu.mit.eecs.6002x;published',
+            revision='draft')
+
+    def test_course_constructor_underspecified(self):
+        self.assertRaises(InsufficientSpecificationError, CourseLocator)
+        self.assertRaises(InsufficientSpecificationError, CourseLocator, revision='published')
+
+    def test_course_constructor_bad_version_guid(self):
+        self.assertRaises(ValueError, CourseLocator, version_guid="012345")
+        self.assertRaises(InsufficientSpecificationError, CourseLocator, version_guid=None)
+
+    def test_course_constructor_version_guid(self):
+        # generate a random location
+        test_id_1 = ObjectId()
+        test_id_1_loc = str(test_id_1)
+        testobj_1 = CourseLocator(version_guid=test_id_1)
+        self.check_course_locn_fields(testobj_1, 'version_guid', version_guid=test_id_1)
+        self.assertEqual(str(testobj_1.version_guid), test_id_1_loc)
+        self.assertEqual(str(testobj_1), '@' + test_id_1_loc)
+        self.assertEqual(testobj_1.url(), 'edx://@' + test_id_1_loc)
+
+        # Test using a given string
+        test_id_2_loc = '519665f6223ebd6980884f2b'
+        test_id_2 = ObjectId(test_id_2_loc)
+        testobj_2 = CourseLocator(version_guid=test_id_2)
+        self.check_course_locn_fields(testobj_2, 'version_guid', version_guid=test_id_2)
+        self.assertEqual(str(testobj_2.version_guid), test_id_2_loc)
+        self.assertEqual(str(testobj_2), '@' + test_id_2_loc)
+        self.assertEqual(testobj_2.url(), 'edx://@' + test_id_2_loc)
+
+    def test_course_constructor_bad_course_id(self):
+        """
+        Test all sorts of badly-formed course_ids (and urls with those course_ids)
+        """
+        for bad_id in ('edu.mit.',
+                       ' edu.mit.eecs',
+                       'edu.mit.eecs ',
+                       '@edu.mit.eecs',
+                       '#edu.mit.eecs',
+                       'edu.mit.ee cs',
+                       'edu.mit.ee,cs',
+                       'edu.mit.ee/cs',
+                       'edu.mit.ee$cs',
+                       'edu.mit.ee&cs',
+                       'edu.mit.ee()cs',
+                       ';this',
+                       'edu.mit.eecs;',
+                       'edu.mit.eecs;this;that',
+                       'edu.mit.eecs;this;',
+                       'edu.mit.eecs;this ',
+                       'edu.mit.eecs;th%is ',
+                       ):
+            self.assertRaises(AssertionError, CourseLocator, course_id=bad_id)
+            self.assertRaises(AssertionError, CourseLocator, url='edx://' + bad_id)
+
+    def test_course_constructor_bad_url(self):
+        for bad_url in ('edx://',
+                        'edx:/edu.mit.eecs',
+                        'http://edu.mit.eecs',
+                        'edu.mit.eecs',
+                        'edx//edu.mit.eecs'):
+            self.assertRaises(AssertionError, CourseLocator, url=bad_url)
+
+    def test_course_constructor_redundant_001(self):
+        testurn = 'edu.mit.eecs.6002x'
+        testobj = CourseLocator(course_id=testurn, url='edx://' + testurn)
+        self.check_course_locn_fields(testobj, 'course_id', course_id=testurn)
+
+    def test_course_constructor_redundant_002(self):
+        testurn = 'edu.mit.eecs.6002x;published'
+        expected_urn = 'edu.mit.eecs.6002x'
+        expected_rev = 'published'
+        testobj = CourseLocator(course_id=testurn, url='edx://' + testurn)
+        self.check_course_locn_fields(testobj, 'course_id',
+                                      course_id=expected_urn,
+                                      revision=expected_rev)
+
+    def test_course_constructor_course_id_no_revision(self):
+        testurn = 'edu.mit.eecs.6002x'
+        testobj = CourseLocator(course_id=testurn)
+        self.check_course_locn_fields(testobj, 'course_id', course_id=testurn)
+        self.assertEqual(testobj.course_id, testurn)
+        self.assertEqual(str(testobj), testurn)
+        self.assertEqual(testobj.url(), 'edx://' + testurn)
+
+    def test_course_constructor_course_id_with_revision(self):
+        testurn = 'edu.mit.eecs.6002x;published'
+        expected_id = 'edu.mit.eecs.6002x'
+        expected_revision = 'published'
+        testobj = CourseLocator(course_id=testurn)
+        self.check_course_locn_fields(testobj, 'course_id with revision',
+                                      course_id=expected_id,
+                                      revision=expected_revision,
+                                      )
+        self.assertEqual(testobj.course_id, expected_id)
+        self.assertEqual(testobj.revision, expected_revision)
+        self.assertEqual(str(testobj), testurn)
+        self.assertEqual(testobj.url(), 'edx://' + testurn)
+
+    def test_course_constructor_course_id_separate_revision(self):
+        test_id = 'edu.mit.eecs.6002x'
+        test_revision = 'published'
+        expected_urn = 'edu.mit.eecs.6002x;published'
+        testobj = CourseLocator(course_id=test_id, revision=test_revision)
+        self.check_course_locn_fields(testobj, 'course_id with separate revision',
+                                      course_id=test_id,
+                                      revision=test_revision,
+                                      )
+        self.assertEqual(testobj.course_id, test_id)
+        self.assertEqual(testobj.revision, test_revision)
+        self.assertEqual(str(testobj), expected_urn)
+        self.assertEqual(testobj.url(), 'edx://' + expected_urn)
+
+    def test_course_constructor_course_id_repeated_revision(self):
+        """
+        The same revision appears in the course_id and the revision field.
+        """
+        test_id = 'edu.mit.eecs.6002x;published'
+        test_revision = 'published'
+        expected_id = 'edu.mit.eecs.6002x'
+        expected_urn = 'edu.mit.eecs.6002x;published'
+        testobj = CourseLocator(course_id=test_id, revision=test_revision)
+        self.check_course_locn_fields(testobj, 'course_id with repeated revision',
+                                      course_id=expected_id,
+                                      revision=test_revision,
+                                      )
+        self.assertEqual(testobj.course_id, expected_id)
+        self.assertEqual(testobj.revision, test_revision)
+        self.assertEqual(str(testobj), expected_urn)
+        self.assertEqual(testobj.url(), 'edx://' + expected_urn)
+
+    def test_block_constructor(self):
+        testurn = 'edu.mit.eecs.6002x;published#HW3'
+        expected_id = 'edu.mit.eecs.6002x'
+        expected_revision = 'published'
+        expected_block_ref = 'HW3'
+        testobj = BlockUsageLocator(course_id=testurn)
+        self.check_block_locn_fields(testobj, 'test_block constructor',
+                                     course_id=expected_id,
+                                     revision=expected_revision,
+                                     block=expected_block_ref)
+        self.assertEqual(str(testobj), testurn)
+        self.assertEqual(testobj.url(), 'edx://' + testurn)
+
+    # ------------------------------------------------------------
+    # Disabled tests
+
+    def test_course_urls(self):
+        '''
+        Test constructor and property accessors.
+        '''
+        raise SkipTest()
+        self.assertRaises(TypeError, CourseLocator, 'empty constructor')
+
+        # url inits
+        testurn = 'edx://org/course/category/name'
+        self.assertRaises(InvalidLocationError, CourseLocator, url=testurn)
+        testurn = 'unknown/versionid/blockid'
+        self.assertRaises(InvalidLocationError, CourseLocator, url=testurn)
+
+        testurn = 'cvx/versionid'
+        testobj = CourseLocator(testurn)
+        self.check_course_locn_fields(testobj, testurn, 'versionid')
+        self.assertEqual(testobj, CourseLocator(testobj),
+                         'initialization from another instance')
+
+        testurn = 'cvx/versionid/'
+        testobj = CourseLocator(testurn)
+        self.check_course_locn_fields(testobj, testurn, 'versionid')
+
+        testurn = 'cvx/versionid/blockid'
+        testobj = CourseLocator(testurn)
+        self.check_course_locn_fields(testobj, testurn, 'versionid')
+
+        testurn = 'cvx/versionid/blockid/extraneousstuff?including=args'
+        testobj = CourseLocator(testurn)
+        self.check_course_locn_fields(testobj, testurn, 'versionid')
+
+        testurn = 'cvx://versionid/blockid'
+        testobj = CourseLocator(testurn)
+        self.check_course_locn_fields(testobj, testurn, 'versionid')
+
+        testurn = 'crx/courseid/blockid'
+        testobj = CourseLocator(testurn)
+        self.check_course_locn_fields(testobj, testurn, course_id='courseid')
+
+        testurn = 'crx/courseid@revision/blockid'
+        testobj = CourseLocator(testurn)
+        self.check_course_locn_fields(testobj, testurn, course_id='courseid',
+                                      revision='revision')
+        self.assertEqual(testobj, CourseLocator(testobj),
+                         'run initialization from another instance')
+
+    def test_course_keyword_setters(self):
+        raise SkipTest()
+        # arg list inits
+        testobj = CourseLocator(version_guid='versionid')
+        self.check_course_locn_fields(testobj, 'versionid arg', 'versionid')
+
+        testobj = CourseLocator(course_id='courseid')
+        self.check_course_locn_fields(testobj, 'courseid arg',
+                                      course_id='courseid')
+
+        testobj = CourseLocator(course_id='courseid', revision='rev')
+        self.check_course_locn_fields(testobj, 'rev arg',
+                                      course_id='courseid',
+                                      revision='rev')
+        # ignores garbage
+        testobj = CourseLocator(course_id='courseid', revision='rev',
+                                potato='spud')
+        self.check_course_locn_fields(testobj, 'extra keyword arg',
+                                      course_id='courseid',
+                                      revision='rev')
+
+        # url w/ keyword override
+        testurn = 'crx/courseid@revision/blockid'
+        testobj = CourseLocator(testurn, revision='rev')
+        self.check_course_locn_fields(testobj, 'rev override',
+                                      course_id='courseid',
+                                      revision='rev')
+
+    def test_course_dict(self):
+        raise SkipTest()
+        # dict init w/ keyword overwrites
+        testobj = CourseLocator({"version_guid": 'versionid'})
+        self.check_course_locn_fields(testobj, 'versionid dict', 'versionid')
+
+        testobj = CourseLocator({"course_id": 'courseid'})
+        self.check_course_locn_fields(testobj, 'courseid dict',
+                                      course_id='courseid')
+
+        testobj = CourseLocator({"course_id": 'courseid', "revision": 'rev'})
+        self.check_course_locn_fields(testobj, 'rev dict',
+                                      course_id='courseid',
+                                      revision='rev')
+        # ignores garbage
+        testobj = CourseLocator({"course_id": 'courseid', "revision": 'rev',
+                                 "potato": 'spud'})
+        self.check_course_locn_fields(testobj, 'extra keyword dict',
+                                      course_id='courseid',
+                                      revision='rev')
+        testobj = CourseLocator({"course_id": 'courseid', "revision": 'rev'},
+                                revision='alt')
+        self.check_course_locn_fields(testobj, 'rev dict',
+                                      course_id='courseid',
+                                      revision='alt')
+
+        # urn init w/ dict & keyword overwrites
+        testobj = CourseLocator('crx/notcourse@notthis',
+                                {"course_id": 'courseid'},
+                                revision='alt')
+        self.check_course_locn_fields(testobj, 'rev dict',
+                                      course_id='courseid',
+                                      revision='alt')
+
+    def test_url(self):
+        '''
+        Ensure CourseLocator generates expected urls.
+        '''
+        raise SkipTest()
+
+        testobj = CourseLocator(version_guid='versionid')
+        self.assertEqual(testobj.url(), 'cvx/versionid', 'versionid')
+        self.assertEqual(testobj, CourseLocator(testobj.url()),
+                         'versionid conversion through url')
+
+        testobj = CourseLocator(course_id='courseid')
+        self.assertEqual(testobj.url(), 'crx/courseid', 'courseid')
+        self.assertEqual(testobj, CourseLocator(testobj.url()),
+                         'courseid conversion through url')
+
+        testobj = CourseLocator(course_id='courseid', revision='rev')
+        self.assertEqual(testobj.url(), 'crx/courseid@rev', 'rev')
+        self.assertEqual(testobj, CourseLocator(testobj.url()),
+                         'rev conversion through url')
+
+    def test_html(self):
+        '''
+        Ensure CourseLocator generates expected urls.
+        '''
+        raise SkipTest()
+        testobj = CourseLocator(version_guid='versionid')
+        self.assertEqual(testobj.html_id(), 'cvx/versionid', 'versionid')
+        self.assertEqual(testobj, CourseLocator(testobj.html_id()),
+                         'versionid conversion through html_id')
+
+        testobj = CourseLocator(course_id='courseid')
+        self.assertEqual(testobj.html_id(), 'crx/courseid', 'courseid')
+        self.assertEqual(testobj, CourseLocator(testobj.html_id()),
+                         'courseid conversion through html_id')
+
+        testobj = CourseLocator(course_id='courseid', revision='rev')
+        self.assertEqual(testobj.html_id(), 'crx/courseid%40rev', 'rev')
+        self.assertEqual(testobj, CourseLocator(testobj.html_id()),
+                         'rev conversion through html_id')
+
+    def test_block_locator(self):
+        '''
+        Test constructor and property accessors.
+        '''
+        raise SkipTest()
+        self.assertIsInstance(BlockUsageLocator(), BlockUsageLocator,
+                              'empty constructor')
+
+        # url inits
+        testurn = 'edx://org/course/category/name'
+        self.assertRaises(InvalidLocationError, BlockUsageLocator, testurn)
+        testurn = 'unknown/versionid/blockid'
+        self.assertRaises(InvalidLocationError, BlockUsageLocator, testurn)
+
+        testurn = 'cvx/versionid'
+        testobj = BlockUsageLocator(testurn)
+        self.check_block_locn_fields(testobj, testurn, 'versionid')
+        self.assertEqual(testobj, BlockUsageLocator(testobj),
+                         'initialization from another instance')
+
+        testurn = 'cvx/versionid/'
+        testobj = BlockUsageLocator(testurn)
+        self.check_block_locn_fields(testobj, testurn, 'versionid')
+
+        testurn = 'cvx/versionid/blockid'
+        testobj = BlockUsageLocator(testurn)
+        self.check_block_locn_fields(testobj, testurn, 'versionid',
+                                     block='blockid')
+
+        testurn = 'cvx/versionid/blockid/extraneousstuff?including=args'
+        testobj = BlockUsageLocator(testurn)
+        self.check_block_locn_fields(testobj, testurn, 'versionid',
+                                     block='blockid')
+
+        testurn = 'cvx://versionid/blockid'
+        testobj = BlockUsageLocator(testurn)
+        self.check_block_locn_fields(testobj, testurn, 'versionid',
+                                     block='blockid')
+
+        testurn = 'crx/courseid/blockid'
+        testobj = BlockUsageLocator(testurn)
+        self.check_block_locn_fields(testobj, testurn, course_id='courseid',
+                                     block='blockid')
+
+        testurn = 'crx/courseid@revision/blockid'
+        testobj = BlockUsageLocator(testurn)
+        self.check_block_locn_fields(testobj, testurn, course_id='courseid',
+                                     revision='revision', block='blockid')
+        self.assertEqual(testobj, BlockUsageLocator(testobj),
+                         'run initialization from another instance')
+
+    def test_block_keyword_init(self):
+        # arg list inits
+        raise SkipTest()
+        testobj = BlockUsageLocator(version_guid='versionid')
+        self.check_block_locn_fields(testobj, 'versionid arg', 'versionid')
+
+        testobj = BlockUsageLocator(version_guid='versionid', usage_id='myblock')
+        self.check_block_locn_fields(testobj, 'versionid arg', 'versionid',
+                                     block='myblock')
+
+        testobj = BlockUsageLocator(course_id='courseid')
+        self.check_block_locn_fields(testobj, 'courseid arg',
+                                     course_id='courseid')
+
+        testobj = BlockUsageLocator(course_id='courseid', revision='rev')
+        self.check_block_locn_fields(testobj, 'rev arg',
+                                     course_id='courseid',
+                                     revision='rev')
+        # ignores garbage
+        testobj = BlockUsageLocator(course_id='courseid', revision='rev',
+                                    usage_id='this_block', potato='spud')
+        self.check_block_locn_fields(testobj, 'extra keyword arg',
+                                     course_id='courseid', block='this_block', revision='rev')
+
+        # url w/ keyword override
+        testurn = 'crx/courseid@revision/blockid'
+        testobj = BlockUsageLocator(testurn, revision='rev')
+        self.check_block_locn_fields(testobj, 'rev override',
+                                     course_id='courseid', block='blockid',
+                                     revision='rev')
+
+    def test_block_keywords(self):
+        # dict init w/ keyword overwrites
+        raise SkipTest()
+        testobj = BlockUsageLocator({"version_guid": 'versionid',
+                                     'usage_id': 'dictblock'})
+        self.check_block_locn_fields(testobj, 'versionid dict', 'versionid',
+                                     block='dictblock')
+
+        testobj = BlockUsageLocator({"course_id": 'courseid',
+                                     'usage_id': 'dictblock'})
+        self.check_block_locn_fields(testobj, 'courseid dict',
+                                     block='dictblock', course_id='courseid')
+
+        testobj = BlockUsageLocator({"course_id": 'courseid', "revision": 'rev',
+                                     'usage_id': 'dictblock'})
+        self.check_block_locn_fields(testobj, 'rev dict',
+                                     course_id='courseid', block='dictblock',
+                                     revision='rev')
+        # ignores garbage
+        testobj = BlockUsageLocator({"course_id": 'courseid', "revision": 'rev',
+                                     'usage_id': 'dictblock', "potato": 'spud'})
+        self.check_block_locn_fields(testobj, 'extra keyword dict',
+                                     course_id='courseid', block='dictblock',
+                                     revision='rev')
+        testobj = BlockUsageLocator({"course_id": 'courseid', "revision": 'rev',
+                                     'usage_id': 'dictblock'}, revision='alt', usage_id='anotherblock')
+        self.check_block_locn_fields(testobj, 'rev dict',
+                                     course_id='courseid', block='anotherblock',
+                                     revision='alt')
+
+        # urn init w/ dict & keyword overwrites
+        testobj = BlockUsageLocator('crx/notcourse@notthis/northis',
+                                    {"course_id": 'courseid'}, revision='alt', usage_id='anotherblock')
+        self.check_block_locn_fields(testobj, 'rev dict',
+                                     course_id='courseid', block='anotherblock',
+                                     revision='alt')
+
+    def test_ensure_fully_specd(self):
+        '''
+        Test constructor and property accessors.
+        '''
+        raise SkipTest()
+        self.assertRaises(InsufficientSpecificationError,
+                          BlockUsageLocator.ensure_fully_specified, BlockUsageLocator())
+
+        # url inits
+        testurn = 'edx://org/course/category/name'
+        self.assertRaises(InvalidLocationError,
+                          BlockUsageLocator.ensure_fully_specified, testurn)
+        testurn = 'unknown/versionid/blockid'
+        self.assertRaises(InvalidLocationError,
+                          BlockUsageLocator.ensure_fully_specified, testurn)
+
+        testurn = 'cvx/versionid'
+        self.assertRaises(InsufficientSpecificationError,
+                          BlockUsageLocator.ensure_fully_specified, testurn)
+
+        testurn = 'cvx/versionid/'
+        self.assertRaises(InsufficientSpecificationError,
+                          BlockUsageLocator.ensure_fully_specified, testurn)
+
+        testurn = 'cvx/versionid/blockid'
+        self.assertIsInstance(BlockUsageLocator.ensure_fully_specified(testurn),
+                              BlockUsageLocator, testurn)
+
+        testurn = 'cvx/versionid/blockid/extraneousstuff?including=args'
+        self.assertIsInstance(BlockUsageLocator.ensure_fully_specified(testurn),
+                              BlockUsageLocator, testurn)
+
+        testurn = 'cvx://versionid/blockid'
+        self.assertIsInstance(BlockUsageLocator.ensure_fully_specified(testurn),
+                              BlockUsageLocator, testurn)
+
+        testurn = 'crx/courseid/blockid'
+        self.assertIsInstance(BlockUsageLocator.ensure_fully_specified(testurn),
+                              BlockUsageLocator, testurn)
+
+        testurn = 'crx/courseid@revision/blockid'
+        self.assertIsInstance(BlockUsageLocator.ensure_fully_specified(testurn),
+                              BlockUsageLocator, testurn)
+
+    def test_ensure_fully_via_keyword(self):
+        # arg list inits
+        raise SkipTest()
+        testobj = BlockUsageLocator(version_guid='versionid')
+        self.assertRaises(InsufficientSpecificationError,
+                          BlockUsageLocator.ensure_fully_specified, testobj)
+
+        testurn = 'crx/courseid@revision/blockid'
+        testobj = BlockUsageLocator(version_guid='versionid', usage_id='myblock')
+        self.assertIsInstance(BlockUsageLocator.ensure_fully_specified(testurn),
+                              BlockUsageLocator, testurn)
+
+        testobj = BlockUsageLocator(course_id='courseid')
+        self.assertRaises(InsufficientSpecificationError,
+                          BlockUsageLocator.ensure_fully_specified, testobj)
+
+        testobj = BlockUsageLocator(course_id='courseid', revision='rev')
+        self.assertRaises(InsufficientSpecificationError,
+                          BlockUsageLocator.ensure_fully_specified, testobj)
+
+        testobj = BlockUsageLocator(course_id='courseid', revision='rev',
+                                    usage_id='this_block')
+        self.assertIsInstance(BlockUsageLocator.ensure_fully_specified(testurn),
+                              BlockUsageLocator, testurn)
+
+    # ------------------------------------------------------------------
+    # Utilities
+
+    def check_course_locn_fields(self, testobj, msg, version_guid=None,
+                                 course_id=None, revision=None):
+        self.assertEqual(testobj.version_guid, version_guid, msg)
+        self.assertEqual(testobj.course_id, course_id, msg)
+        self.assertEqual(testobj.revision, revision, msg)
+
+    def check_block_locn_fields(self, testobj, msg, version_guid=None,
+                                course_id=None, revision=None, block=None):
+        self.check_course_locn_fields(testobj, msg, version_guid, course_id,
+                                      revision)
+        self.assertEqual(testobj.usage_id, block)

common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py

@@ -0,0 +1,992 @@
+'''
+Created on Mar 25, 2013
+
+@author: dmitchell
+'''
+import datetime
+import subprocess
+import unittest
+import uuid
+from importlib import import_module
+
+from xblock.core import Scope
+from xmodule.course_module import CourseDescriptor
+from xmodule.modulestore.exceptions import InsufficientSpecificationError, ItemNotFoundError, VersionConflictError
+from xmodule.modulestore.locator import CourseLocator, BlockUsageLocator, VersionTree, DescriptionLocator
+from pytz import UTC
+from path import path
+import re
+
+
+class SplitModuleTest(unittest.TestCase):
+    '''
+    The base set of tests manually populates a db w/ courses which have
+    versions. It creates unique collection names and removes them after all
+    tests finish.
+    '''
+    # Snippet of what would be in the django settings envs file
+    modulestore_options = {
+        'default_class': 'xmodule.raw_module.RawDescriptor',
+        'host': 'localhost',
+        'db': 'test_xmodule',
+        'collection': 'modulestore{0}'.format(uuid.uuid4().hex),
+        'fs_root': '',
+    }
+
+    MODULESTORE = {
+        'ENGINE': 'xmodule.modulestore.split_mongo.SplitMongoModuleStore',
+        'OPTIONS': modulestore_options
+    }
+
+    # don't create django dependency; so, duplicates common.py in envs
+    match = re.search(r'(.*?/common)(?:$|/)', path(__file__))
+    COMMON_ROOT = match.group(1)
+
+    modulestore = None
+
+    # These version_guids correspond to values hard-coded in fixture files
+    # used for these tests. The files live in mitx/fixtures/splitmongo_json/*
+
+    GUID_D0 = "1d00000000000000dddd0000"  # v12345d
+    GUID_D1 = "1d00000000000000dddd1111"  # v12345d1
+    GUID_D2 = "1d00000000000000dddd2222"  # v23456d
+    GUID_D3 = "1d00000000000000dddd3333"  # v12345d0
+    GUID_D4 = "1d00000000000000dddd4444"  # v23456d0
+    GUID_D5 = "1d00000000000000dddd5555"  # v345679d
+    GUID_P = "1d00000000000000eeee0000"  # v23456p
+
+    @staticmethod
+    def bootstrapDB():
+        '''
+        Loads the initial data into the db ensuring the collection name is
+        unique.
+        '''
+        collection_prefix = SplitModuleTest.MODULESTORE['OPTIONS']['collection'] + '.'
+        dbname = SplitModuleTest.MODULESTORE['OPTIONS']['db']
+        processes = [
+            subprocess.Popen([
+                'mongoimport', '-d', dbname, '-c',
+                collection_prefix + collection, '--jsonArray',
+                '--file',
+                SplitModuleTest.COMMON_ROOT + '/test/data/splitmongo_json/' + collection + '.json'
+            ])
+            for collection in ('active_versions', 'structures', 'definitions')]
+        for p in processes:
+            if p.wait() != 0:
+                raise Exception("DB did not init correctly")
+
+    @classmethod
+    def tearDownClass(cls):
+        collection_prefix = SplitModuleTest.MODULESTORE['OPTIONS']['collection'] + '.'
+        if SplitModuleTest.modulestore:
+            for collection in ('active_versions', 'structures', 'definitions'):
+                modulestore().db.drop_collection(collection_prefix + collection)
+            # drop the modulestore to force re init
+            SplitModuleTest.modulestore = None
+
+    def findByIdInResult(self, collection, _id):
+        """
+        Result is a collection of descriptors. Find the one whose block id
+        matches the _id.
+        """
+        for element in collection:
+            if element.location.usage_id == _id:
+                return element
+
+
+class SplitModuleCourseTests(SplitModuleTest):
+    '''
+    Course CRUD operation tests
+    '''
+
+    def test_get_courses(self):
+        courses = modulestore().get_courses('draft')
+        # should have gotten 3 draft courses
+        self.assertEqual(len(courses), 3, "Wrong number of courses")
+        # check metadata -- NOTE no promised order
+        course = self.findByIdInResult(courses, "head12345")
+        self.assertEqual(course.location.course_id, "GreekHero")
+        self.assertEqual(
+            str(course.location.version_guid), self.GUID_D0,
+            "course version mismatch"
+        )
+        self.assertEqual(course.category, 'course', 'wrong category')
+        self.assertEqual(len(course.tabs), 6, "wrong number of tabs")
+        self.assertEqual(
+            course.display_name, "The Ancient Greek Hero",
+            "wrong display name"
+        )
+        self.assertEqual(
+            course.advertised_start, "Fall 2013",
+            "advertised_start"
+        )
+        self.assertEqual(
+            len(course.children), 3,
+            "children")
+        self.assertEqual(course.definition_locator.definition_id, "head12345_12")
+        # check dates and graders--forces loading of descriptor
+        self.assertEqual(course.edited_by, "testassist@edx.org")
+        self.assertEqual(str(course.previous_version), self.GUID_D1)
+        self.assertDictEqual(course.grade_cutoffs, {"Pass": 0.45})
+
+    def test_revision_requests(self):
+        # query w/ revision qualifier (both draft and published)
+        courses_published = modulestore().get_courses('published')
+        self.assertEqual(len(courses_published), 1, len(courses_published))
+        course = self.findByIdInResult(courses_published, "head23456")
+        self.assertIsNotNone(course, "published courses")
+        self.assertEqual(course.location.course_id, "wonderful")
+        self.assertEqual(str(course.location.version_guid), self.GUID_P,
+                         course.location.version_guid)
+        self.assertEqual(course.category, 'course', 'wrong category')
+        self.assertEqual(len(course.tabs), 4, "wrong number of tabs")
+        self.assertEqual(course.display_name, "The most wonderful course",
+                         course.display_name)
+        self.assertIsNone(course.advertised_start)
+        self.assertEqual(len(course.children), 0,
+                         "children")
+
+    def test_search_qualifiers(self):
+        # query w/ search criteria
+        courses = modulestore().get_courses('draft', qualifiers={'org': 'testx'})
+        self.assertEqual(len(courses), 2)
+        self.assertIsNotNone(self.findByIdInResult(courses, "head12345"))
+        self.assertIsNotNone(self.findByIdInResult(courses, "head23456"))
+
+        courses = modulestore().get_courses(
+            'draft',
+            qualifiers={'edited_on': {"$lt": datetime.datetime(2013, 3, 28, 15)}})
+        self.assertEqual(len(courses), 2)
+
+        courses = modulestore().get_courses(
+            'draft',
+            qualifiers={'org': 'testx', "prettyid": "test_course"})
+        self.assertEqual(len(courses), 1)
+        self.assertIsNotNone(self.findByIdInResult(courses, "head12345"))
+
+    def test_get_course(self):
+        '''
+        Test the various calling forms for get_course
+        '''
+        locator = CourseLocator(version_guid=self.GUID_D1)
+        course = modulestore().get_course(locator)
+        self.assertIsNone(course.location.course_id)
+        self.assertEqual(str(course.location.version_guid), self.GUID_D1)
+        self.assertEqual(course.category, 'course')
+        self.assertEqual(len(course.tabs), 6)
+        self.assertEqual(course.display_name, "The Ancient Greek Hero")
+        self.assertIsNone(course.advertised_start)
+        self.assertEqual(len(course.children), 0)
+        self.assertEqual(course.definition_locator.definition_id, "head12345_11")
+        # check dates and graders--forces loading of descriptor
+        self.assertEqual(course.edited_by, "testassist@edx.org")
+        self.assertDictEqual(course.grade_cutoffs, {"Pass": 0.55})
+
+        locator = CourseLocator(course_id='GreekHero', revision='draft')
+        course = modulestore().get_course(locator)
+        self.assertEqual(course.location.course_id, "GreekHero")
+        self.assertEqual(str(course.location.version_guid), self.GUID_D0)
+        self.assertEqual(course.category, 'course')
+        self.assertEqual(len(course.tabs), 6)
+        self.assertEqual(course.display_name, "The Ancient Greek Hero")
+        self.assertEqual(course.advertised_start, "Fall 2013")
+        self.assertEqual(len(course.children), 3)
+        # check dates and graders--forces loading of descriptor
+        self.assertEqual(course.edited_by, "testassist@edx.org")
+        self.assertDictEqual(course.grade_cutoffs, {"Pass": 0.45})
+
+        locator = CourseLocator(course_id='wonderful', revision='published')
+        course = modulestore().get_course(locator)
+        self.assertEqual(course.location.course_id, "wonderful")
+        self.assertEqual(str(course.location.version_guid), self.GUID_P)
+
+        locator = CourseLocator(course_id='wonderful', revision='draft')
+        course = modulestore().get_course(locator)
+        self.assertEqual(str(course.location.version_guid), self.GUID_D2)
+
+    def test_get_course_negative(self):
+        # Now negative testing
+        self.assertRaises(InsufficientSpecificationError,
+                          modulestore().get_course, CourseLocator(course_id='edu.meh.blah'))
+        self.assertRaises(ItemNotFoundError,
+                          modulestore().get_course, CourseLocator(course_id='nosuchthing', revision='draft'))
+        self.assertRaises(ItemNotFoundError,
+                          modulestore().get_course,
+                          CourseLocator(course_id='GreekHero', revision='published'))
+
+    def test_course_successors(self):
+        """
+        get_course_successors(course_locator, version_history_depth=1)
+        """
+        locator = CourseLocator(version_guid=self.GUID_D3)
+        result = modulestore().get_course_successors(locator)
+        self.assertIsInstance(result, VersionTree)
+        self.assertIsNone(result.locator.course_id)
+        self.assertEqual(str(result.locator.version_guid), self.GUID_D3)
+        self.assertEqual(len(result.children), 1)
+        self.assertEqual(str(result.children[0].locator.version_guid), self.GUID_D1)
+        self.assertEqual(len(result.children[0].children), 0, "descended more than one level")
+        result = modulestore().get_course_successors(locator, version_history_depth=2)
+        self.assertEqual(len(result.children), 1)
+        self.assertEqual(str(result.children[0].locator.version_guid), self.GUID_D1)
+        self.assertEqual(len(result.children[0].children), 1)
+        result = modulestore().get_course_successors(locator, version_history_depth=99)
+        self.assertEqual(len(result.children), 1)
+        self.assertEqual(str(result.children[0].locator.version_guid), self.GUID_D1)
+        self.assertEqual(len(result.children[0].children), 1)
+
+
+class SplitModuleItemTests(SplitModuleTest):
+    '''
+    Item read tests including inheritance
+    '''
+
+    def test_has_item(self):
+        '''
+        has_item(BlockUsageLocator)
+        '''
+        # positive tests of various forms
+        locator = BlockUsageLocator(version_guid=self.GUID_D1, usage_id='head12345')
+        self.assertTrue(modulestore().has_item(locator),
+                        "couldn't find in %s" % self.GUID_D1)
+
+        locator = BlockUsageLocator(course_id='GreekHero', usage_id='head12345', revision='draft')
+        self.assertTrue(
+            modulestore().has_item(locator),
+            "couldn't find in 12345"
+        )
+        self.assertTrue(
+            modulestore().has_item(BlockUsageLocator(
+                course_id=locator.course_id,
+                revision='draft',
+                usage_id=locator.usage_id
+            )),
+            "couldn't find in draft 12345"
+        )
+        self.assertFalse(
+            modulestore().has_item(BlockUsageLocator(
+                course_id=locator.course_id,
+                revision='published',
+                usage_id=locator.usage_id)),
+            "found in published 12345"
+        )
+        locator.revision = 'draft'
+        self.assertTrue(
+            modulestore().has_item(locator),
+            "not found in draft 12345"
+        )
+
+        # not a course obj
+        locator = BlockUsageLocator(course_id='GreekHero', usage_id='chapter1', revision='draft')
+        self.assertTrue(
+            modulestore().has_item(locator),
+            "couldn't find chapter1"
+        )
+
+        # in published course
+        locator = BlockUsageLocator(course_id="wonderful", usage_id="head23456", revision='draft')
+        self.assertTrue(modulestore().has_item(BlockUsageLocator(course_id=locator.course_id,
+                                                                 usage_id=locator.usage_id,
+                                                                 revision='published')),
+                        "couldn't find in 23456")
+        locator.revision = 'published'
+        self.assertTrue(modulestore().has_item(locator), "couldn't find in 23456")
+
+    def test_negative_has_item(self):
+        # negative tests--not found
+        # no such course or block
+        locator = BlockUsageLocator(course_id="doesnotexist", usage_id="head23456", revision='draft')
+        self.assertFalse(modulestore().has_item(locator))
+        locator = BlockUsageLocator(course_id="wonderful", usage_id="doesnotexist", revision='draft')
+        self.assertFalse(modulestore().has_item(locator))
+
+        # negative tests--insufficient specification
+        self.assertRaises(InsufficientSpecificationError, BlockUsageLocator)
+        self.assertRaises(InsufficientSpecificationError,
+                          modulestore().has_item, BlockUsageLocator(version_guid=self.GUID_D1))
+        self.assertRaises(InsufficientSpecificationError,
+                          modulestore().has_item, BlockUsageLocator(course_id='GreekHero'))
+
+    def test_get_item(self):
+        '''
+        get_item(blocklocator)
+        '''
+        # positive tests of various forms
+        locator = BlockUsageLocator(version_guid=self.GUID_D1, usage_id='head12345')
+        block = modulestore().get_item(locator)
+        self.assertIsInstance(block, CourseDescriptor)
+
+        locator = BlockUsageLocator(course_id='GreekHero', usage_id='head12345', revision='draft')
+        block = modulestore().get_item(locator)
+        self.assertEqual(block.location.course_id, "GreekHero")
+        # look at this one in detail
+        self.assertEqual(len(block.tabs), 6, "wrong number of tabs")
+        self.assertEqual(block.display_name, "The Ancient Greek Hero")
+        self.assertEqual(block.advertised_start, "Fall 2013")
+        self.assertEqual(len(block.children), 3)
+        self.assertEqual(block.definition_locator.definition_id, "head12345_12")
+        # check dates and graders--forces loading of descriptor
+        self.assertEqual(block.edited_by, "testassist@edx.org")
+        self.assertDictEqual(
+            block.grade_cutoffs, {"Pass": 0.45},
+        )
+
+        # try to look up other revisions
+        self.assertRaises(ItemNotFoundError,
+                          modulestore().get_item,
+                          BlockUsageLocator(course_id=locator.as_course_locator(),
+                                            usage_id=locator.usage_id,
+                                            revision='published'))
+        locator.revision = 'draft'
+        self.assertIsInstance(
+            modulestore().get_item(locator),
+            CourseDescriptor
+        )
+
+    def test_get_non_root(self):
+        # not a course obj
+        locator = BlockUsageLocator(course_id='GreekHero', usage_id='chapter1', revision='draft')
+        block = modulestore().get_item(locator)
+        self.assertEqual(block.location.course_id, "GreekHero")
+        self.assertEqual(block.category, 'chapter')
+        self.assertEqual(block.definition_locator.definition_id, "chapter12345_1")
+        self.assertEqual(block.display_name, "Hercules")
+        self.assertEqual(block.edited_by, "testassist@edx.org")
+
+        # in published course
+        locator = BlockUsageLocator(course_id="wonderful", usage_id="head23456", revision='published')
+        self.assertIsInstance(
+            modulestore().get_item(locator),
+            CourseDescriptor
+        )
+
+        # negative tests--not found
+        # no such course or block
+        locator = BlockUsageLocator(course_id="doesnotexist", usage_id="head23456", revision='draft')
+        with self.assertRaises(ItemNotFoundError):
+            modulestore().get_item(locator)
+        locator = BlockUsageLocator(course_id="wonderful", usage_id="doesnotexist", revision='draft')
+        with self.assertRaises(ItemNotFoundError):
+            modulestore().get_item(locator)
+
+        # negative tests--insufficient specification
+        with self.assertRaises(InsufficientSpecificationError):
+            modulestore().get_item(BlockUsageLocator(version_guid=self.GUID_D1))
+        with self.assertRaises(InsufficientSpecificationError):
+            modulestore().get_item(BlockUsageLocator(course_id='GreekHero', revision='draft'))
+
+    # pylint: disable=W0212
+    def test_matching(self):
+        '''
+        test the block and value matches help functions
+        '''
+        self.assertTrue(modulestore()._value_matches('help', 'help'))
+        self.assertFalse(modulestore()._value_matches('help', 'Help'))
+        self.assertTrue(modulestore()._value_matches(['distract', 'help', 'notme'], 'help'))
+        self.assertFalse(modulestore()._value_matches(['distract', 'Help', 'notme'], 'help'))
+        self.assertFalse(modulestore()._value_matches({'field': ['distract', 'Help', 'notme']}, {'field': 'help'}))
+        self.assertFalse(modulestore()._value_matches(['distract', 'Help', 'notme'], {'field': 'help'}))
+        self.assertTrue(modulestore()._value_matches(
+            {'field': ['distract', 'help', 'notme'],
+                'irrelevant': 2},
+            {'field': 'help'}))
+        self.assertTrue(modulestore()._value_matches('I need some help', {'$regex': 'help'}))
+        self.assertTrue(modulestore()._value_matches(['I need some help', 'today'], {'$regex': 'help'}))
+        self.assertFalse(modulestore()._value_matches('I need some help', {'$regex': 'Help'}))
+        self.assertFalse(modulestore()._value_matches(['I need some help', 'today'], {'$regex': 'Help'}))
+
+        self.assertTrue(modulestore()._block_matches({'a': 1, 'b': 2}, {'a': 1}))
+        self.assertTrue(modulestore()._block_matches({'a': 1, 'b': 2}, {'c': None}))
+        self.assertTrue(modulestore()._block_matches({'a': 1, 'b': 2}, {'a': 1, 'c': None}))
+        self.assertFalse(modulestore()._block_matches({'a': 1, 'b': 2}, {'a': 2}))
+        self.assertFalse(modulestore()._block_matches({'a': 1, 'b': 2}, {'c': 1}))
+        self.assertFalse(modulestore()._block_matches({'a': 1, 'b': 2}, {'a': 1, 'c': 1}))
+
+    def test_get_items(self):
+        '''
+        get_items(locator, qualifiers, [revision])
+        '''
+        locator = CourseLocator(version_guid=self.GUID_D0)
+        # get all modules
+        matches = modulestore().get_items(locator, {})
+        self.assertEqual(len(matches), 6)
+        matches = modulestore().get_items(locator, {'category': 'chapter'})
+        self.assertEqual(len(matches), 3)
+        matches = modulestore().get_items(locator, {'category': 'garbage'})
+        self.assertEqual(len(matches), 0)
+        matches = modulestore().get_items(
+            locator,
+            {
+                'category': 'chapter',
+                'metadata': {'display_name': {'$regex': 'Hera'}}
+            }
+        )
+        self.assertEqual(len(matches), 2)
+
+        matches = modulestore().get_items(locator, {'children': 'chapter2'})
+        self.assertEqual(len(matches), 1)
+        self.assertEqual(matches[0].location.usage_id, 'head12345')
+
+    def test_get_parents(self):
+        '''
+        get_parent_locations(locator, [usage_id], [revision]): [BlockUsageLocator]
+        '''
+        locator = CourseLocator(course_id="GreekHero", revision='draft')
+        parents = modulestore().get_parent_locations(locator, usage_id='chapter1')
+        self.assertEqual(len(parents), 1)
+        self.assertEqual(parents[0].usage_id, 'head12345')
+        self.assertEqual(parents[0].course_id, "GreekHero")
+        locator.usage_id = 'chapter2'
+        parents = modulestore().get_parent_locations(locator)
+        self.assertEqual(len(parents), 1)
+        self.assertEqual(parents[0].usage_id, 'head12345')
+        parents = modulestore().get_parent_locations(locator, usage_id='nosuchblock')
+        self.assertEqual(len(parents), 0)
+
+    def test_get_children(self):
+        """
+        Test the existing get_children method on xdescriptors
+        """
+        locator = BlockUsageLocator(course_id="GreekHero", usage_id="head12345", revision='draft')
+        block = modulestore().get_item(locator)
+        children = block.get_children()
+        expected_ids = [
+            "chapter1", "chapter2", "chapter3"
+        ]
+        for child in children:
+            self.assertEqual(child.category, "chapter")
+            self.assertIn(child.location.usage_id, expected_ids)
+            expected_ids.remove(child.location.usage_id)
+        self.assertEqual(len(expected_ids), 0)
+
+
+class TestItemCrud(SplitModuleTest):
+    """
+    Test create update and delete of items
+    """
+    # TODO do I need to test this case which I believe won't work:
+    #  1) fetch a course and some of its blocks
+    #  2) do a series of CRUD operations on those previously fetched elements
+    # The problem here will be that the version_guid of the items will be the version at time of fetch.
+    # Each separate save will change the head version; so, the 2nd piecemeal change will flag the version
+    # conflict. That is, if versions are v0..vn and start as v0 in initial fetch, the first CRUD op will
+    # say it's changing an object from v0, splitMongo will process it and make the current head v1, the next
+    # crud op will pass in its v0 element and splitMongo will flag the version conflict.
+    # What I don't know is how realistic this test is and whether to wrap the modulestore with a higher level
+    # transactional operation which manages the version change or make the threading cache reason out whether or
+    # not the changes are independent and additive and thus non-conflicting.
+    # A use case I expect is
+    # (client) change this metadata
+    # (server) done, here's the new info which, btw, updates the course version to v1
+    # (client) add these children to this other node (which says it came from v0 or
+    #          will the client have refreshed the version before doing the op?)
+    # In this case, having a server side transactional model won't help b/c the bug is a long-transaction on the
+    # on the client where it would be a mistake for the server to assume anything about client consistency. The best
+    # the server could do would be to see if the parent's children changed at all since v0.
+
+    def test_create_minimal_item(self):
+        """
+        create_item(course_or_parent_locator, category, user, definition_locator=None, new_def_data=None,
+        metadata=None): new_desciptor
+        """
+        # grab link to course to ensure new versioning works
+        locator = CourseLocator(course_id="GreekHero", revision='draft')
+        premod_course = modulestore().get_course(locator)
+        premod_time = datetime.datetime.now(UTC) - datetime.timedelta(seconds=1)
+        # add minimal one w/o a parent
+        category = 'sequential'
+        new_module = modulestore().create_item(
+            locator, category, 'user123',
+            metadata={'display_name': 'new sequential'}
+        )
+        # check that course version changed and course's previous is the other one
+        self.assertEqual(new_module.location.course_id, "GreekHero")
+        self.assertNotEqual(new_module.location.version_guid, premod_course.location.version_guid)
+        self.assertIsNone(locator.version_guid, "Version inadvertently filled in")
+        current_course = modulestore().get_course(locator)
+        self.assertEqual(new_module.location.version_guid, current_course.location.version_guid)
+
+        history_info = modulestore().get_course_history_info(current_course.location)
+        self.assertEqual(history_info['previous_version'], premod_course.location.version_guid)
+        self.assertEqual(str(history_info['original_version']), self.GUID_D3)
+        self.assertEqual(history_info['edited_by'], "user123")
+        self.assertGreaterEqual(history_info['edited_on'], premod_time)
+        self.assertLessEqual(history_info['edited_on'], datetime.datetime.now(UTC))
+        # check block's info: category, definition_locator, and display_name
+        self.assertEqual(new_module.category, 'sequential')
+        self.assertIsNotNone(new_module.definition_locator)
+        self.assertEqual(new_module.display_name, 'new sequential')
+        # check that block does not exist in previous version
+        locator = BlockUsageLocator(
+            version_guid=premod_course.location.version_guid,
+            usage_id=new_module.location.usage_id
+        )
+        self.assertRaises(ItemNotFoundError, modulestore().get_item, locator)
+
+    def test_create_parented_item(self):
+        """
+        Test create_item w/ specifying the parent of the new item
+        """
+        locator = BlockUsageLocator(course_id="wonderful", usage_id="head23456", revision='draft')
+        premod_course = modulestore().get_course(locator)
+        category = 'chapter'
+        new_module = modulestore().create_item(
+            locator, category, 'user123',
+            metadata={'display_name': 'new chapter'},
+            definition_locator=DescriptionLocator("chapter12345_2")
+        )
+        # check that course version changed and course's previous is the other one
+        self.assertNotEqual(new_module.location.version_guid, premod_course.location.version_guid)
+        parent = modulestore().get_item(locator)
+        self.assertIn(new_module.location.usage_id, parent.children)
+        self.assertEqual(new_module.definition_locator.definition_id, "chapter12345_2")
+
+    def test_unique_naming(self):
+        """
+        Check that 2 modules of same type get unique usage_ids. Also check that if creation provides
+        a definition id and new def data that it branches the definition in the db.
+        Actually, this tries to test all create_item features not tested above.
+        """
+        locator = BlockUsageLocator(course_id="contender", usage_id="head345679", revision='draft')
+        category = 'problem'
+        premod_time = datetime.datetime.now(UTC) - datetime.timedelta(seconds=1)
+        new_payload = "<problem>empty</problem>"
+        new_module = modulestore().create_item(
+            locator, category, 'anotheruser',
+            metadata={'display_name': 'problem 1'},
+            new_def_data=new_payload
+        )
+        another_payload = "<problem>not empty</problem>"
+        another_module = modulestore().create_item(
+            locator, category, 'anotheruser',
+            metadata={'display_name': 'problem 2'},
+            definition_locator=DescriptionLocator("problem12345_3_1"),
+            new_def_data=another_payload
+        )
+        # check that course version changed and course's previous is the other one
+        parent = modulestore().get_item(locator)
+        self.assertNotEqual(new_module.location.usage_id, another_module.location.usage_id)
+        self.assertIn(new_module.location.usage_id, parent.children)
+        self.assertIn(another_module.location.usage_id, parent.children)
+        self.assertEqual(new_module.data, new_payload)
+        self.assertEqual(another_module.data, another_payload)
+        # check definition histories
+        new_history = modulestore().get_definition_history_info(new_module.definition_locator)
+        self.assertIsNone(new_history['previous_version'])
+        self.assertEqual(new_history['original_version'], new_module.definition_locator.definition_id)
+        self.assertEqual(new_history['edited_by'], "anotheruser")
+        self.assertLessEqual(new_history['edited_on'], datetime.datetime.now(UTC))
+        self.assertGreaterEqual(new_history['edited_on'], premod_time)
+        another_history = modulestore().get_definition_history_info(another_module.definition_locator)
+        self.assertEqual(another_history['previous_version'], 'problem12345_3_1')
+    # TODO check that default fields are set
+
+    def test_update_metadata(self):
+        """
+        test updating an items metadata ensuring the definition doesn't version but the course does if it should
+        """
+        locator = BlockUsageLocator(course_id="GreekHero", usage_id="problem3_2", revision='draft')
+        problem = modulestore().get_item(locator)
+        pre_def_id = problem.definition_locator.definition_id
+        pre_version_guid = problem.location.version_guid
+        self.assertIsNotNone(pre_def_id)
+        self.assertIsNotNone(pre_version_guid)
+        premod_time = datetime.datetime.now(UTC) - datetime.timedelta(seconds=1)
+        self.assertNotEqual(problem.max_attempts, 4, "Invalidates rest of test")
+
+        problem.max_attempts = 4
+        updated_problem = modulestore().update_item(problem, 'changeMaven')
+        # check that course version changed and course's previous is the other one
+        self.assertEqual(updated_problem.definition_locator.definition_id, pre_def_id)
+        self.assertNotEqual(updated_problem.location.version_guid, pre_version_guid)
+        self.assertEqual(updated_problem.max_attempts, 4)
+        # refetch to ensure original didn't change
+        original_location = BlockUsageLocator(
+            version_guid=pre_version_guid,
+            usage_id=problem.location.usage_id
+        )
+        problem = modulestore().get_item(original_location)
+        self.assertNotEqual(problem.max_attempts, 4, "original changed")
+
+        current_course = modulestore().get_course(locator)
+        self.assertEqual(updated_problem.location.version_guid, current_course.location.version_guid)
+
+        history_info = modulestore().get_course_history_info(current_course.location)
+        self.assertEqual(history_info['previous_version'], pre_version_guid)
+        self.assertEqual(str(history_info['original_version']), self.GUID_D3)
+        self.assertEqual(history_info['edited_by'], "changeMaven")
+        self.assertGreaterEqual(history_info['edited_on'], premod_time)
+        self.assertLessEqual(history_info['edited_on'], datetime.datetime.now(UTC))
+
+    def test_update_children(self):
+        """
+        test updating an item's children ensuring the definition doesn't version but the course does if it should
+        """
+        locator = BlockUsageLocator(course_id="GreekHero", usage_id="chapter3", revision='draft')
+        block = modulestore().get_item(locator)
+        pre_def_id = block.definition_locator.definition_id
+        pre_version_guid = block.location.version_guid
+
+        # reorder children
+        self.assertGreater(len(block.children), 0, "meaningless test")
+        moved_child = block.children.pop()
+        updated_problem = modulestore().update_item(block, 'childchanger')
+        # check that course version changed and course's previous is the other one
+        self.assertEqual(updated_problem.definition_locator.definition_id, pre_def_id)
+        self.assertNotEqual(updated_problem.location.version_guid, pre_version_guid)
+        self.assertEqual(updated_problem.children, block.children)
+        self.assertNotIn(moved_child, updated_problem.children)
+        locator.usage_id = "chapter1"
+        other_block = modulestore().get_item(locator)
+        other_block.children.append(moved_child)
+        other_updated = modulestore().update_item(other_block, 'childchanger')
+        self.assertIn(moved_child, other_updated.children)
+
+    def test_update_definition(self):
+        """
+        test updating an item's definition: ensure it gets versioned as well as the course getting versioned
+        """
+        locator = BlockUsageLocator(course_id="GreekHero", usage_id="head12345", revision='draft')
+        block = modulestore().get_item(locator)
+        pre_def_id = block.definition_locator.definition_id
+        pre_version_guid = block.location.version_guid
+
+        block.grading_policy['GRADER'][0]['min_count'] = 13
+        updated_block = modulestore().update_item(block, 'definition_changer')
+
+        self.assertNotEqual(updated_block.definition_locator.definition_id, pre_def_id)
+        self.assertNotEqual(updated_block.location.version_guid, pre_version_guid)
+        self.assertEqual(updated_block.grading_policy['GRADER'][0]['min_count'], 13)
+
+    def test_update_manifold(self):
+        """
+        Test updating metadata, children, and definition in a single call ensuring all the versioning occurs
+        """
+        # first add 2 children to the course for the update to manipulate
+        locator = BlockUsageLocator(course_id="contender", usage_id="head345679", revision='draft')
+        category = 'problem'
+        new_payload = "<problem>empty</problem>"
+        modulestore().create_item(
+            locator, category, 'test_update_manifold',
+            metadata={'display_name': 'problem 1'},
+            new_def_data=new_payload
+        )
+        another_payload = "<problem>not empty</problem>"
+        modulestore().create_item(
+            locator, category, 'test_update_manifold',
+            metadata={'display_name': 'problem 2'},
+            definition_locator=DescriptionLocator("problem12345_3_1"),
+            new_def_data=another_payload
+        )
+        # pylint: disable=W0212
+        modulestore()._clear_cache()
+
+        # now begin the test
+        block = modulestore().get_item(locator)
+        pre_def_id = block.definition_locator.definition_id
+        pre_version_guid = block.location.version_guid
+
+        self.assertNotEqual(block.grading_policy['GRADER'][0]['min_count'], 13)
+        block.grading_policy['GRADER'][0]['min_count'] = 13
+        block.children = block.children[1:] + [block.children[0]]
+        block.advertised_start = "Soon"
+
+        updated_block = modulestore().update_item(block, "test_update_manifold")
+        self.assertNotEqual(updated_block.definition_locator.definition_id, pre_def_id)
+        self.assertNotEqual(updated_block.location.version_guid, pre_version_guid)
+        self.assertEqual(updated_block.grading_policy['GRADER'][0]['min_count'], 13)
+        self.assertEqual(updated_block.children[0], block.children[0])
+        self.assertEqual(updated_block.advertised_start, "Soon")
+
+    def test_delete_item(self):
+        course = self.create_course_for_deletion()
+        self.assertRaises(ValueError,
+                          modulestore().delete_item,
+                          course.location,
+                          'deleting_user')
+        reusable_location = BlockUsageLocator(
+            course_id=course.location.course_id,
+            usage_id=course.location.usage_id,
+            revision='draft')
+
+        # delete a leaf
+        problems = modulestore().get_items(reusable_location, {'category': 'problem'})
+        locn_to_del = problems[0].location
+        new_course_loc = modulestore().delete_item(locn_to_del, 'deleting_user')
+        deleted = BlockUsageLocator(course_id=reusable_location.course_id,
+                                    revision=reusable_location.revision,
+                                    usage_id=locn_to_del.usage_id)
+        self.assertFalse(modulestore().has_item(deleted))
+        self.assertRaises(VersionConflictError, modulestore().has_item, locn_to_del)
+        locator = BlockUsageLocator(
+            version_guid=locn_to_del.version_guid,
+            usage_id=locn_to_del.usage_id
+        )
+        self.assertTrue(modulestore().has_item(locator))
+        self.assertNotEqual(new_course_loc.version_guid, course.location.version_guid)
+
+        # delete a subtree
+        nodes = modulestore().get_items(reusable_location, {'category': 'chapter'})
+        new_course_loc = modulestore().delete_item(nodes[0].location, 'deleting_user')
+        # check subtree
+
+        def check_subtree(node):
+            if node:
+                node_loc = node.location
+                self.assertFalse(modulestore().has_item(
+                    BlockUsageLocator(
+                        course_id=node_loc.course_id,
+                        revision=node_loc.revision,
+                        usage_id=node.location.usage_id)))
+                locator = BlockUsageLocator(
+                    version_guid=node.location.version_guid,
+                    usage_id=node.location.usage_id)
+                self.assertTrue(modulestore().has_item(locator))
+                if node.has_children:
+                    for sub in node.get_children():
+                        check_subtree(sub)
+        check_subtree(nodes[0])
+
+    def create_course_for_deletion(self):
+        course = modulestore().create_course('nihilx', 'deletion', 'deleting_user')
+        root = BlockUsageLocator(
+            course_id=course.location.course_id,
+            usage_id=course.location.usage_id,
+            revision='draft')
+        for _ in range(4):
+            self.create_subtree_for_deletion(root, ['chapter', 'vertical', 'problem'])
+        return modulestore().get_item(root)
+
+    def create_subtree_for_deletion(self, parent, category_queue):
+        if not category_queue:
+            return
+        node = modulestore().create_item(parent, category_queue[0], 'deleting_user')
+        node_loc = BlockUsageLocator(parent.as_course_locator(), usage_id=node.location.usage_id)
+        for _ in range(4):
+            self.create_subtree_for_deletion(node_loc, category_queue[1:])
+
+
+class TestCourseCreation(SplitModuleTest):
+    """
+    Test create_course, duh :-)
+    """
+    def test_simple_creation(self):
+        """
+        The simplest case but probing all expected results from it.
+        """
+        # Oddly getting differences of 200nsec
+        pre_time = datetime.datetime.now(UTC) - datetime.timedelta(milliseconds=1)
+        new_course = modulestore().create_course('test_org', 'test_course', 'create_user')
+        new_locator = new_course.location
+        # check index entry
+        index_info = modulestore().get_course_index_info(new_locator)
+        self.assertEqual(index_info['org'], 'test_org')
+        self.assertEqual(index_info['prettyid'], 'test_course')
+        self.assertGreaterEqual(index_info["edited_on"], pre_time)
+        self.assertLessEqual(index_info["edited_on"], datetime.datetime.now(UTC))
+        self.assertEqual(index_info['edited_by'], 'create_user')
+        # check structure info
+        structure_info = modulestore().get_course_history_info(new_locator)
+        self.assertEqual(structure_info['original_version'], index_info['versions']['draft'])
+        self.assertIsNone(structure_info['previous_version'])
+        self.assertGreaterEqual(structure_info["edited_on"], pre_time)
+        self.assertLessEqual(structure_info["edited_on"], datetime.datetime.now(UTC))
+        self.assertEqual(structure_info['edited_by'], 'create_user')
+        # check the returned course object
+        self.assertIsInstance(new_course, CourseDescriptor)
+        self.assertEqual(new_course.category, 'course')
+        self.assertFalse(new_course.show_calculator)
+        self.assertTrue(new_course.allow_anonymous)
+        self.assertEqual(len(new_course.children), 0)
+        self.assertEqual(new_course.edited_by, "create_user")
+        self.assertEqual(len(new_course.grading_policy['GRADER']), 4)
+        self.assertDictEqual(new_course.grade_cutoffs, {"Pass": 0.5})
+
+    def test_cloned_course(self):
+        """
+        Test making a course which points to an existing draft and published but not making any changes to either.
+        """
+        pre_time = datetime.datetime.now(UTC)
+        original_locator = CourseLocator(course_id="wonderful", revision='draft')
+        original_index = modulestore().get_course_index_info(original_locator)
+        new_draft = modulestore().create_course(
+            'leech', 'best_course', 'leech_master', id_root='best',
+            versions_dict=original_index['versions'])
+        new_draft_locator = new_draft.location
+        self.assertRegexpMatches(new_draft_locator.course_id, r'best.*')
+        # the edited_by and other meta fields on the new course will be the original author not this one
+        self.assertEqual(new_draft.edited_by, 'test@edx.org')
+        self.assertLess(new_draft.edited_on, pre_time)
+        self.assertEqual(new_draft.location.version_guid, original_index['versions']['draft'])
+        # however the edited_by and other meta fields on course_index will be this one
+        new_index = modulestore().get_course_index_info(new_draft_locator)
+        self.assertGreaterEqual(new_index["edited_on"], pre_time)
+        self.assertLessEqual(new_index["edited_on"], datetime.datetime.now(UTC))
+        self.assertEqual(new_index['edited_by'], 'leech_master')
+
+        new_published_locator = CourseLocator(course_id=new_draft_locator.course_id, revision='published')
+        new_published = modulestore().get_course(new_published_locator)
+        self.assertEqual(new_published.edited_by, 'test@edx.org')
+        self.assertLess(new_published.edited_on, pre_time)
+        self.assertEqual(new_published.location.version_guid, original_index['versions']['published'])
+
+        # changing this course will not change the original course
+        # using new_draft.location will insert the chapter under the course root
+        new_item = modulestore().create_item(
+            new_draft.location, 'chapter', 'leech_master',
+            metadata={'display_name': 'new chapter'}
+        )
+        new_draft_locator.version_guid = None
+        new_index = modulestore().get_course_index_info(new_draft_locator)
+        self.assertNotEqual(new_index['versions']['draft'], original_index['versions']['draft'])
+        new_draft = modulestore().get_course(new_draft_locator)
+        self.assertEqual(new_item.edited_by, 'leech_master')
+        self.assertGreaterEqual(new_item.edited_on, pre_time)
+        self.assertNotEqual(new_item.location.version_guid, original_index['versions']['draft'])
+        self.assertNotEqual(new_draft.location.version_guid, original_index['versions']['draft'])
+        structure_info = modulestore().get_course_history_info(new_draft_locator)
+        self.assertGreaterEqual(structure_info["edited_on"], pre_time)
+        self.assertLessEqual(structure_info["edited_on"], datetime.datetime.now(UTC))
+        self.assertEqual(structure_info['edited_by'], 'leech_master')
+
+        original_course = modulestore().get_course(original_locator)
+        self.assertEqual(original_course.location.version_guid, original_index['versions']['draft'])
+        self.assertFalse(
+            modulestore().has_item(BlockUsageLocator(
+                original_locator,
+                usage_id=new_item.location.usage_id
+            ))
+        )
+
+    def test_derived_course(self):
+        """
+        Create a new course which overrides metadata and course_data
+        """
+        pre_time = datetime.datetime.now(UTC)
+        original_locator = CourseLocator(course_id="contender", revision='draft')
+        original = modulestore().get_course(original_locator)
+        original_index = modulestore().get_course_index_info(original_locator)
+        data_payload = {}
+        metadata_payload = {}
+        for field in original.fields:
+            if field.scope == Scope.content and field.name != 'location':
+                data_payload[field.name] = getattr(original, field.name)
+            elif field.scope == Scope.settings:
+                metadata_payload[field.name] = getattr(original, field.name)
+        data_payload['grading_policy']['GRADE_CUTOFFS'] = {'A': .9, 'B': .8, 'C': .65}
+        metadata_payload['display_name'] = 'Derivative'
+        new_draft = modulestore().create_course(
+            'leech', 'derivative', 'leech_master', id_root='counter',
+            versions_dict={'draft': original_index['versions']['draft']},
+            course_data=data_payload,
+            metadata=metadata_payload
+        )
+        new_draft_locator = new_draft.location
+        self.assertRegexpMatches(new_draft_locator.course_id, r'counter.*')
+        # the edited_by and other meta fields on the new course will be the original author not this one
+        self.assertEqual(new_draft.edited_by, 'leech_master')
+        self.assertGreaterEqual(new_draft.edited_on, pre_time)
+        self.assertNotEqual(new_draft.location.version_guid, original_index['versions']['draft'])
+        # however the edited_by and other meta fields on course_index will be this one
+        new_index = modulestore().get_course_index_info(new_draft_locator)
+        self.assertGreaterEqual(new_index["edited_on"], pre_time)
+        self.assertLessEqual(new_index["edited_on"], datetime.datetime.now(UTC))
+        self.assertEqual(new_index['edited_by'], 'leech_master')
+        self.assertEqual(new_draft.display_name, metadata_payload['display_name'])
+        self.assertDictEqual(
+            new_draft.grading_policy['GRADE_CUTOFFS'],
+            data_payload['grading_policy']['GRADE_CUTOFFS']
+        )
+
+    def test_update_course_index(self):
+        """
+        Test changing the org, pretty id, etc of a course. Test that it doesn't allow changing the id, etc.
+        """
+        locator = CourseLocator(course_id="GreekHero", revision='draft')
+        modulestore().update_course_index(locator, {'org': 'funkyU'})
+        course_info = modulestore().get_course_index_info(locator)
+        self.assertEqual(course_info['org'], 'funkyU')
+
+        modulestore().update_course_index(locator, {'org': 'moreFunky', 'prettyid': 'Ancient Greek Demagods'})
+        course_info = modulestore().get_course_index_info(locator)
+        self.assertEqual(course_info['org'], 'moreFunky')
+        self.assertEqual(course_info['prettyid'], 'Ancient Greek Demagods')
+
+        self.assertRaises(ValueError, modulestore().update_course_index, locator, {'_id': 'funkygreeks'})
+
+        with self.assertRaises(ValueError):
+            modulestore().update_course_index(
+                locator,
+                {'edited_on': datetime.datetime.now(UTC)}
+            )
+        with self.assertRaises(ValueError):
+            modulestore().update_course_index(
+                locator,
+                {'edited_by': 'sneak'}
+            )
+
+        self.assertRaises(ValueError, modulestore().update_course_index, locator,
+                          {'versions': {'draft': self.GUID_D1}})
+
+        # an allowed but not necessarily recommended way to revert the draft version
+        versions = course_info['versions']
+        versions['draft'] = self.GUID_D1
+        modulestore().update_course_index(locator, {'versions': versions}, update_versions=True)
+        course = modulestore().get_course(locator)
+        self.assertEqual(str(course.location.version_guid), self.GUID_D1)
+
+        # an allowed but not recommended way to publish a course
+        versions['published'] = self.GUID_D1
+        modulestore().update_course_index(locator, {'versions': versions}, update_versions=True)
+        course = modulestore().get_course(CourseLocator(course_id=locator.course_id, revision="published"))
+        self.assertEqual(str(course.location.version_guid), self.GUID_D1)
+
+
+class TestInheritance(SplitModuleTest):
+    """
+    Test the metadata inheritance mechanism.
+    """
+    def test_inheritance(self):
+        """
+        The actual test
+        """
+        # Note, not testing value where defined (course) b/c there's no
+        # defined accessor for it on CourseDescriptor.
+        locator = BlockUsageLocator(course_id="GreekHero", usage_id="problem3_2", revision='draft')
+        node = modulestore().get_item(locator)
+        # inherited
+        self.assertEqual(node.graceperiod, datetime.timedelta(hours=2))
+        locator = BlockUsageLocator(course_id="GreekHero", usage_id="problem1", revision='draft')
+        node = modulestore().get_item(locator)
+        # overridden
+        self.assertEqual(node.graceperiod, datetime.timedelta(hours=4))
+
+    # TODO test inheritance after set and delete of attrs
+
+
+#===========================================
+# This mocks the django.modulestore() function and is intended purely to disentangle
+# the tests from django
+def modulestore():
+    def load_function(path):
+        module_path, _, name = path.rpartition('.')
+        return getattr(import_module(module_path), name)
+
+    if SplitModuleTest.modulestore is None:
+        SplitModuleTest.bootstrapDB()
+        class_ = load_function(SplitModuleTest.MODULESTORE['ENGINE'])
+
+        options = {}
+
+        options.update(SplitModuleTest.MODULESTORE['OPTIONS'])
+        options['render_template'] = render_to_template_mock
+
+        # pylint: disable=W0142
+        SplitModuleTest.modulestore = class_(**options)
+
+    return SplitModuleTest.modulestore
+
+
+# pylint: disable=W0613
+def render_to_template_mock(*args):
+    pass

common/lib/xmodule/xmodule/tests/test_capa_module.py

@@ -1233,6 +1233,37 @@ class CapaModuleTest(unittest.TestCase):
             mock_log.exception.assert_called_once_with('Got bad progress')
             mock_log.reset_mock()
 
+    @patch('xmodule.capa_module.Progress')
+    def test_get_progress_calculate_progress_fraction(self, mock_progress):
+        """
+        Check that score and total are calculated correctly for the progress fraction.
+        """
+        module = CapaFactory.create()
+        module.weight = 1
+        module.get_progress()
+        mock_progress.assert_called_with(0, 1)
+
+        other_module = CapaFactory.create(correct=True)
+        other_module.weight = 1
+        other_module.get_progress()
+        mock_progress.assert_called_with(1, 1)
+
+    def test_get_html(self):
+        """
+        Check that get_html() calls get_progress() with no arguments.
+        """
+        module = CapaFactory.create()
+        module.get_progress = Mock(wraps=module.get_progress)
+        module.get_html()
+        module.get_progress.assert_called_once_with()
+
+    def test_get_problem(self):
+        """
+        Check that get_problem() returns the expected dictionary.
+        """
+        module = CapaFactory.create()
+        self.assertEquals(module.get_problem("data"), {'html': module.get_problem_html(encapsulate=False)})
+
 
 class ComplexEncoderTest(unittest.TestCase):
     def test_default(self):

common/lib/xmodule/xmodule/x_module.py

@@ -8,9 +8,10 @@ from collections import namedtuple
 from pkg_resources import resource_listdir, resource_string, resource_isdir
 
 from xmodule.modulestore import inheritance, Location
-from xmodule.modulestore.exceptions import ItemNotFoundError, InsufficientSpecificationError
+from xmodule.modulestore.exceptions import ItemNotFoundError, InsufficientSpecificationError, InvalidLocationError
 
 from xblock.core import XBlock, Scope, String, Integer, Float, ModelType
+from xmodule.modulestore.locator import BlockUsageLocator
 
 log = logging.getLogger(__name__)
 
@@ -27,7 +28,13 @@ class LocationField(ModelType):
         """
         Parse the json value as a Location
         """
-        return Location(value)
+        try:
+            return Location(value)
+        except InvalidLocationError:
+            if isinstance(value, BlockUsageLocator):
+                return value
+            else:
+                return BlockUsageLocator(value)
 
     def to_json(self, value):
         """
@@ -166,6 +173,10 @@ class XModule(XModuleFields, HTMLSnippet, XBlock):
             self.url_name = self.location.name
             if not hasattr(self, 'category'):
                 self.category = self.location.category
+        elif isinstance(self.location, BlockUsageLocator):
+            self.url_name = self.location.usage_id
+            if not hasattr(self, 'category'):
+                raise InsufficientSpecificationError()
         else:
             raise InsufficientSpecificationError()
         self._loaded_children = None
@@ -436,8 +447,17 @@ class XModuleDescriptor(XModuleFields, HTMLSnippet, ResourceTemplates, XBlock):
             self.url_name = self.location.name
             if not hasattr(self, 'category'):
                 self.category = self.location.category
+        elif isinstance(self.location, BlockUsageLocator):
+            self.url_name = self.location.usage_id
+            if not hasattr(self, 'category'):
+                raise InsufficientSpecificationError()
         else:
             raise InsufficientSpecificationError()
+        # update_version is the version which last updated this xblock v prev being the penultimate updater
+        # leaving off original_version since it complicates creation w/o any obv value yet and is computable
+        # by following previous until None
+        # definition_locator is only used by mongostores which separate definitions from blocks
+        self.edited_by = self.edited_on = self.previous_version = self.update_version = self.definition_locator = None
         self._child_instances = None
 
     @property
@@ -514,22 +534,30 @@ class XModuleDescriptor(XModuleFields, HTMLSnippet, ResourceTemplates, XBlock):
 
     # ================================= JSON PARSING ===========================
     @staticmethod
-    def load_from_json(json_data, system, default_class=None):
+    def load_from_json(json_data, system, default_class=None, parent_xblock=None):
         """
         This method instantiates the correct subclass of XModuleDescriptor based
-        on the contents of json_data.
+        on the contents of json_data. It does not persist it and can create one which
+        has no usage id.
 
-        json_data must contain a 'location' element, and must be suitable to be
-        passed into the subclasses `from_json` method as model_data
+        parent_xblock is used to compute inherited metadata as well as to append the new xblock.
+
+        json_data:
+        - 'location' : must have this field
+        - 'category': the xmodule category (required or location must be a Location)
+        - 'metadata': a dict of locally set metadata (not inherited)
+        - 'children': a list of children's usage_ids w/in this course
+        - 'definition':
+        - '_id' (optional): the usage_id of this. Will generate one if not given one.
         """
         class_ = XModuleDescriptor.load_class(
-            json_data['location']['category'],
+            json_data.get('category', json_data.get('location', {}).get('category')),
             default_class
         )
-        return class_.from_json(json_data, system)
+        return class_.from_json(json_data, system, parent_xblock)
 
     @classmethod
-    def from_json(cls, json_data, system):
+    def from_json(cls, json_data, system, parent_xblock=None):
         """
         Creates an instance of this descriptor from the supplied json_data.
         This may be overridden by subclasses
@@ -547,28 +575,25 @@ class XModuleDescriptor(XModuleFields, HTMLSnippet, ResourceTemplates, XBlock):
                Otherwise, it contains the single field 'data'
             4) Any value later in this list overrides a value earlier in this list
 
-        system: A DescriptorSystem for interacting with external resources
+        json_data:
+        - 'category': the xmodule category (required)
+        - 'metadata': a dict of locally set metadata (not inherited)
+        - 'children': a list of children's usage_ids w/in this course
+        - 'definition':
+        - '_id' (optional): the usage_id of this. Will generate one if not given one.
         """
-        model_data = {}
+        usage_id = json_data.get('_id', None)
+        if not '_inherited_metadata' in json_data and parent_xblock is not None:
+            json_data['_inherited_metadata'] = parent_xblock.xblock_kvs.get_inherited_metadata().copy()
+            json_metadata = json_data.get('metadata', {})
+            for field in inheritance.INHERITABLE_METADATA:
+                if field in json_metadata:
+                    json_data['_inherited_metadata'][field] = json_metadata[field]
 
-        for key, value in json_data.get('metadata', {}).items():
-            model_data[cls._translate(key)] = value
-
-        model_data.update(json_data.get('metadata', {}))
-
-        definition = json_data.get('definition', {})
-        if 'children' in definition:
-            model_data['children'] = definition['children']
-
-        if 'data' in definition:
-            if isinstance(definition['data'], dict):
-                model_data.update(definition['data'])
-            else:
-                model_data['data'] = definition['data']
-
-        model_data['location'] = json_data['location']
-
-        return cls(system, model_data)
+        new_block = system.xblock_from_json(cls, usage_id, json_data)
+        if parent_xblock is not None:
+            parent_xblock.children.append(new_block)
+        return new_block
 
     @classmethod
     def _translate(cls, key):
@@ -649,6 +674,8 @@ class XModuleDescriptor(XModuleFields, HTMLSnippet, ResourceTemplates, XBlock):
         """
         Use w/ caution. Really intended for use by the persistence layer.
         """
+        # if caller wants kvs, caller's assuming it's up to date; so, decache it
+        self.save()
         return self._model_data._kvs
 
     # =============================== BUILTIN METHODS ==========================

common/test/data/splitmongo_json/active_versions.json

@@ -0,0 +1,27 @@
+[{"_id" : "GreekHero", 
+  "org" : "testx", 
+  "prettyid" : "test_course", 
+  "versions" : {
+    "draft" : { "$oid" : "1d00000000000000dddd0000" }
+   },
+  "edited_on" : {"$date" : 1364481713238},
+  "edited_by" : "test@edx.org"},
+
+ {"_id" : "wonderful", 
+  "org" : "testx", 
+  "prettyid" : "another_course", 
+  "versions" : {
+    "draft" : { "$oid" : "1d00000000000000dddd2222" },
+    "published" : { "$oid" : "1d00000000000000eeee0000" }
+    },
+  "edited_on" : {"$date" : 1364481313238},
+  "edited_by" : "test@edx.org"},
+  
+ {"_id" : "contender", 
+  "org" : "guestx", 
+  "prettyid" : "test_course", 
+  "versions" : {
+        "draft" : { "$oid" : "1d00000000000000dddd5555" }},
+  "edited_on" : {"$date" : 1364491313238},
+  "edited_by" : "test@guestx.edu"}
+]

common/test/data/splitmongo_json/definitions.json

@@ -0,0 +1,334 @@
+[
+    {
+        "_id":"head12345_12",
+        "category":"course",
+        "data":{
+            "textbooks":[
+                
+            ],
+            "grading_policy":{
+                "GRADER":[
+                    {
+                        "min_count":4,
+                        "weight":0.15,
+                        "type":"Homework",
+                        "drop_count":2,
+                        "short_label":"HWa"
+                    },
+                    {
+                        "short_label":"",
+                        "min_count":12,
+                        "type":"Lab",
+                        "drop_count":2,
+                        "weight":0.15
+                    },
+                    {
+                        "short_label":"Midterm",
+                        "min_count":1,
+                        "type":"Midterm Exam",
+                        "drop_count":0,
+                        "weight":0.3
+                    },
+                    {
+                        "short_label":"Final",
+                        "min_count":1,
+                        "type":"Final Exam",
+                        "drop_count":0,
+                        "weight":0.4
+                    }
+                ],
+                "GRADE_CUTOFFS":{
+                    "Pass":0.45
+                }
+            },
+            "wiki_slug":null
+        },
+        "edited_by":"testassist@edx.org",
+        "edited_on":{"$date" : 1364481713238},
+        "previous_version":"head12345_11",
+        "original_version":"head12345_10"
+    },
+    {
+        "_id":"head12345_11",
+        "category":"course",
+        "data":{
+            "textbooks":[
+                
+            ],
+            "grading_policy":{
+                "GRADER":[
+                    {
+                        "min_count":5,
+                        "weight":0.15,
+                        "type":"Homework",
+                        "drop_count":1,
+                        "short_label":"HWa"
+                    },
+                    {
+                        "short_label":"",
+                        "min_count":12,
+                        "type":"Lab",
+                        "drop_count":2,
+                        "weight":0.15
+                    },
+                    {
+                        "short_label":"Midterm",
+                        "min_count":1,
+                        "type":"Midterm Exam",
+                        "drop_count":0,
+                        "weight":0.3
+                    },
+                    {
+                        "short_label":"Final",
+                        "min_count":1,
+                        "type":"Final Exam",
+                        "drop_count":0,
+                        "weight":0.4
+                    }
+                ],
+                "GRADE_CUTOFFS":{
+                    "Pass":0.55
+                }
+            },
+            "wiki_slug":null
+        },
+        "edited_by":"testassist@edx.org",
+        "edited_on":{"$date" : 1364481713238},
+        "previous_version":"head12345_10",
+        "original_version":"head12345_10"
+    },
+    {
+        "_id":"head12345_10",
+        "category":"course",
+        "data":{
+            "textbooks":[
+                
+            ],
+            "grading_policy":{
+                "GRADER":[
+                    {
+                        "min_count":5,
+                        "weight":0.15,
+                        "type":"Homework",
+                        "drop_count":1,
+                        "short_label":"HWa"
+                    },
+                    {
+                        "short_label":"",
+                        "min_count":2,
+                        "type":"Lab",
+                        "drop_count":0,
+                        "weight":0.15
+                    },
+                    {
+                        "short_label":"Midterm",
+                        "min_count":1,
+                        "type":"Midterm Exam",
+                        "drop_count":0,
+                        "weight":0.3
+                    },
+                    {
+                        "short_label":"Final",
+                        "min_count":1,
+                        "type":"Final Exam",
+                        "drop_count":0,
+                        "weight":0.4
+                    }
+                ],
+                "GRADE_CUTOFFS":{
+                    "Pass":0.75
+                }
+            },
+            "wiki_slug":null
+        },
+        "edited_by":"test@edx.org",
+        "edited_on":{"$date": 1364473713238},
+        "previous_version":null,
+        "original_version":"head12345_10"
+    },
+    {
+        "_id":"head23456_1",
+        "category":"course",
+        "data":{
+            "textbooks":[
+                
+            ],
+            "grading_policy":{
+                "GRADER":[
+                    {
+                        "min_count":14,
+                        "weight":0.25,
+                        "type":"Homework",
+                        "drop_count":1,
+                        "short_label":"HWa"
+                    },
+                    {
+                        "short_label":"",
+                        "min_count":12,
+                        "type":"Lab",
+                        "drop_count":2,
+                        "weight":0.25
+                    },
+                    {
+                        "short_label":"Midterm",
+                        "min_count":1,
+                        "type":"Midterm Exam",
+                        "drop_count":0,
+                        "weight":0.2
+                    },
+                    {
+                        "short_label":"Final",
+                        "min_count":1,
+                        "type":"Final Exam",
+                        "drop_count":0,
+                        "weight":0.3
+                    }
+                ],
+                "GRADE_CUTOFFS":{
+                    "Pass":0.45
+                }
+            },
+            "wiki_slug":null
+        },
+        "edited_by":"test@edx.org",
+        "edited_on":{"$date": 1364481313238},
+        "previous_version":"head23456_0",
+        "original_version":"head23456_0"
+    },
+    {
+        "_id":"head23456_0",
+        "category":"course",
+        "data":{
+            "textbooks":[
+                
+            ],
+            "grading_policy":{
+                "GRADER":[
+                    {
+                        "min_count":14,
+                        "weight":0.25,
+                        "type":"Homework",
+                        "drop_count":1,
+                        "short_label":"HWa"
+                    },
+                    {
+                        "short_label":"",
+                        "min_count":12,
+                        "type":"Lab",
+                        "drop_count":2,
+                        "weight":0.25
+                    },
+                    {
+                        "short_label":"Midterm",
+                        "min_count":1,
+                        "type":"Midterm Exam",
+                        "drop_count":0,
+                        "weight":0.2
+                    },
+                    {
+                        "short_label":"Final",
+                        "min_count":1,
+                        "type":"Final Exam",
+                        "drop_count":0,
+                        "weight":0.3
+                    }
+                ],
+                "GRADE_CUTOFFS":{
+                    "Pass":0.95
+                }
+            },
+            "wiki_slug":null
+        },
+        "edited_by":"test@edx.org",
+        "edited_on":{"$date" : 1364481313238},
+        "previous_version":null,
+        "original_version":"head23456_0"
+    },
+    {
+        "_id":"head345679_1",
+        "category":"course",
+        "data":{
+            "textbooks":[
+                
+            ],
+            "grading_policy":{
+                "GRADER":[
+                    {
+                        "min_count":4,
+                        "weight":0.25,
+                        "type":"Homework",
+                        "drop_count":0,
+                        "short_label":"HW"
+                    },
+                    {
+                        "short_label":"Midterm",
+                        "min_count":1,
+                        "type":"Midterm Exam",
+                        "drop_count":0,
+                        "weight":0.4
+                    },
+                    {
+                        "short_label":"Final",
+                        "min_count":1,
+                        "type":"Final Exam",
+                        "drop_count":0,
+                        "weight":0.35
+                    }
+                ],
+                "GRADE_CUTOFFS":{
+                    "Pass":0.25
+                }
+            },
+            "wiki_slug":null
+        },
+        "edited_by":"test@edx.org",
+        "edited_on":{"$date" : 1364481313238},
+        "previous_version":null,
+        "original_version":"head23456_0"
+    },
+    {
+        "_id":"chapter12345_1",
+        "category":"chapter",
+        "data":null,
+        "edited_by":"testassist@edx.org",
+        "edited_on":{"$date" : 1364483713238},
+        "previous_version":null,
+        "original_version":"chapter12345_1"
+    },
+    {
+        "_id":"chapter12345_2",
+        "category":"chapter",
+        "data":null,
+        "edited_by":"testassist@edx.org",
+        "edited_on":{"$date" : 1364483713238},
+        "previous_version":null,
+        "original_version":"chapter12345_2"
+    },
+    {
+        "_id":"chapter12345_3",
+        "category":"chapter",
+        "data":null,
+        "edited_by":"testassist@edx.org",
+        "edited_on":{"$date" : 1364483713238},
+        "previous_version":null,
+        "original_version":"chapter12345_3"
+    },
+    {
+        "_id":"problem12345_3_1",
+        "category":"problem",
+        "data":"",
+        "edited_by":"testassist@edx.org",
+        "edited_on":{"$date" : 1364483713238},
+        "previous_version":null,
+        "original_version":"problem12345_3_1"
+    },
+    {
+        "_id":"problem12345_3_2",
+        "category":"problem",
+        "data":"",
+        "edited_by":"testassist@edx.org",
+        "edited_on":{"$date" : 1364483713238},
+        "previous_version":null,
+        "original_version":"problem12345_3_2"
+    }
+]

common/test/data/splitmongo_json/structures.json

@@ -0,0 +1,471 @@
+[
+    {
+        "_id": { "$oid" : "1d00000000000000dddd0000"},
+        "root":"head12345",
+        "original_version":{ "$oid" : "1d00000000000000dddd3333" },
+        "previous_version":{ "$oid" : "1d00000000000000dddd1111" },
+        "edited_by":"testassist@edx.org",
+        "edited_on":{
+            "$date":1364483713238
+        },
+        "blocks":{
+            "head12345":{
+                "children":[
+                    "chapter1",
+                    "chapter2",
+                    "chapter3"
+                ],
+                "category":"course",
+                "definition":"head12345_12",
+                "metadata":{
+                    "end":"2013-06-13T04:30",
+                    "tabs":[
+                        {
+                            "type":"courseware"
+                        },
+                        {
+                            "type":"course_info",
+                            "name":"Course Info"
+                        },
+                        {
+                            "type":"discussion",
+                            "name":"Discussion"
+                        },
+                        {
+                            "type":"wiki",
+                            "name":"Wiki"
+                        },
+                        {
+                            "type":"static_tab",
+                            "name":"Syllabus",
+                            "url_slug":"01356a17b5924b17a04b7fc2426a3798"
+                        },
+                        {
+                            "type":"static_tab",
+                            "name":"Advice for Students",
+                            "url_slug":"57e9991c0d794ff58f7defae3e042e39"
+                        }
+                    ],
+                    "enrollment_start":"2013-01-01T05:00",
+                    "graceperiod":"2 hours 0 minutes 0 seconds",
+                    "start":"2013-02-14T05:00",
+                    "enrollment_end":"2013-03-02T05:00",
+                    "data_dir":"MITx-2-Base",
+                    "advertised_start":"Fall 2013",
+                    "display_name":"The Ancient Greek Hero"
+                },
+                "update_version":{ "$oid" : "1d00000000000000dddd0000" },
+                "previous_version":{ "$oid" : "1d00000000000000dddd1111" },
+                "edited_by":"testassist@edx.org",
+                "edited_on":{
+                    "$date":1364483713238
+                }
+            },
+            "chapter1":{
+                "children":[
+                    
+                ],
+                "category":"chapter",
+                "definition":"chapter12345_1",
+                "metadata":{
+                    "display_name":"Hercules"
+                },
+                "update_version":{ "$oid" : "1d00000000000000dddd0000" },
+                "previous_version":null,
+                "edited_by":"testassist@edx.org",
+                "edited_on":{
+                    "$date":1364483713238
+                }
+            },
+            "chapter2":{
+                "children":[
+                    
+                ],
+                "category":"chapter",
+                "definition":"chapter12345_2",
+                "metadata":{
+                    "display_name":"Hera heckles Hercules"
+                },
+                "update_version":{ "$oid" : "1d00000000000000dddd0000" },
+                "previous_version":null,
+                "edited_by":"testassist@edx.org",
+                "edited_on":{
+                    "$date":1364483713238
+                }
+            },
+            "chapter3":{
+                "children":[
+                    "problem1",
+                    "problem3_2"
+                ],
+                "category":"chapter",
+                "definition":"chapter12345_3",
+                "metadata":{
+                    "display_name":"Hera cuckolds Zeus"
+                },
+                "update_version":{ "$oid" : "1d00000000000000dddd0000" },
+                "previous_version":null,
+                "edited_by":"testassist@edx.org",
+                "edited_on":{
+                    "$date":1364483713238
+                }
+            },
+            "problem1":{
+                "children":[
+                    
+                ],
+                "category":"problem",
+                "definition":"problem12345_3_1",
+                "metadata":{
+                    "display_name":"Problem 3.1",
+                    "graceperiod":"4 hours 0 minutes 0 seconds"
+                },
+                "update_version":{ "$oid" : "1d00000000000000dddd0000" },
+                "previous_version":null,
+                "edited_by":"testassist@edx.org",
+                "edited_on":{
+                    "$date":1364483713238
+                }
+            },
+            "problem3_2":{
+                "children":[
+                    
+                ],
+                "category":"problem",
+                "definition":"problem12345_3_2",
+                "metadata":{
+                    "display_name":"Problem 3.2"
+                },
+                "update_version":{ "$oid" : "1d00000000000000dddd0000" },
+                "previous_version":null,
+                "edited_by":"testassist@edx.org",
+                "edited_on":{
+                    "$date":1364483713238
+                }
+            }
+        }
+    },
+    {
+        "_id": { "$oid" : "1d00000000000000dddd1111"},
+        "root":"head12345",
+        "original_version":{ "$oid" : "1d00000000000000dddd3333" },
+        "previous_version":{ "$oid" : "1d00000000000000dddd3333" },
+        "edited_by":"testassist@edx.org",
+        "edited_on":{
+            "$date":1364481713238
+        },
+        "blocks":{
+            "head12345":{
+                "children":[
+                    
+                ],
+                "category":"course",
+                "definition":"head12345_11",
+                "metadata":{
+                    "end":"2013-04-13T04:30",
+                    "tabs":[
+                        {
+                            "type":"courseware"
+                        },
+                        {
+                            "type":"course_info",
+                            "name":"Course Info"
+                        },
+                        {
+                            "type":"discussion",
+                            "name":"Discussion"
+                        },
+                        {
+                            "type":"wiki",
+                            "name":"Wiki"
+                        },
+                        {
+                            "type":"static_tab",
+                            "name":"Syllabus",
+                            "url_slug":"01356a17b5924b17a04b7fc2426a3798"
+                        },
+                        {
+                            "type":"static_tab",
+                            "name":"Advice for Students",
+                            "url_slug":"57e9991c0d794ff58f7defae3e042e39"
+                        }
+                    ],
+                    "enrollment_start":null,
+                    "graceperiod":"2 hours 0 minutes 0 seconds",
+                    "start":"2013-02-14T05:00",
+                    "enrollment_end":null,
+                    "data_dir":"MITx-2-Base",
+                    "advertised_start":null,
+                    "display_name":"The Ancient Greek Hero"
+                },
+                "update_version":{ "$oid" : "1d00000000000000dddd1111" },
+                "previous_version":{ "$oid" : "1d00000000000000dddd3333" },
+                "edited_by":"testassist@edx.org",
+                "edited_on":{
+                    "$date":1364481713238
+                }
+            }
+        }
+    },
+    {
+        "_id": { "$oid" : "1d00000000000000dddd3333"},
+        "root":"head12345",
+        "original_version":{ "$oid" : "1d00000000000000dddd3333" },
+        "previous_version":null,
+        "edited_by":"test@edx.org",
+        "edited_on":{
+            "$date":1364473713238
+        },
+        "blocks":{
+            "head12345":{
+                "children":[
+                    
+                ],
+                "category":"course",
+                "definition":"head12345_10",
+                "metadata":{
+                    "end":null,
+                    "tabs":[
+                        {
+                            "type":"courseware"
+                        },
+                        {
+                            "type":"course_info",
+                            "name":"Course Info"
+                        },
+                        {
+                            "type":"discussion",
+                            "name":"Discussion"
+                        },
+                        {
+                            "type":"wiki",
+                            "name":"Wiki"
+                        }
+                    ],
+                    "enrollment_start":null,
+                    "graceperiod":null,
+                    "start":"2013-02-14T05:00",
+                    "enrollment_end":null,
+                    "data_dir":"MITx-2-Base",
+                    "advertised_start":null,
+                    "display_name":"The Ancient Greek Hero"
+                },
+                "update_version":{ "$oid" : "1d00000000000000dddd3333" },
+                "previous_version":null,
+                "edited_by":"test@edx.org",
+                "edited_on":{
+                    "$date":1364473713238
+                }
+            }
+        }
+    },
+    {
+        "_id": { "$oid" : "1d00000000000000dddd2222"},
+        "root":"head23456",
+        "original_version":{ "$oid" : "1d00000000000000dddd4444" },
+        "previous_version":{ "$oid" : "1d00000000000000dddd4444" },
+        "edited_by":"test@edx.org",
+        "edited_on":{
+            "$date":1364481313238
+        },
+        "blocks":{
+            "head23456":{
+                "children":[
+                    
+                ],
+                "category":"course",
+                "definition":"head23456_1",
+                "metadata":{
+                    "end":null,
+                    "tabs":[
+                        {
+                            "type":"courseware"
+                        },
+                        {
+                            "type":"course_info",
+                            "name":"Course Info"
+                        },
+                        {
+                            "type":"discussion",
+                            "name":"Discussion"
+                        },
+                        {
+                            "type":"wiki",
+                            "name":"Wiki"
+                        }
+                    ],
+                    "enrollment_start":null,
+                    "graceperiod":null,
+                    "start":"2013-02-14T05:00",
+                    "enrollment_end":null,
+                    "data_dir":"MITx-2-Base",
+                    "advertised_start":null,
+                    "display_name":"The most wonderful course"
+                },
+                "update_version":{ "$oid" : "1d00000000000000dddd2222" },
+                "previous_version":{ "$oid" : "1d00000000000000dddd4444" },
+                "edited_by":"test@edx.org",
+                "edited_on":{
+                    "$date":1364481313238
+                }
+                
+            }
+        }
+    },
+    {
+        "_id": { "$oid" : "1d00000000000000dddd4444"},
+        "root":"head23456",
+        "original_version":{ "$oid" : "1d00000000000000dddd4444" },
+        "previous_version":null,
+        "edited_by":"test@edx.org",
+        "edited_on":{
+            "$date":1364480313238
+        },
+        "blocks":{
+            "head23456":{
+                "children":[
+                    
+                ],
+                "category":"course",
+                "definition":"head23456_0",
+                "metadata":{
+                    "end":null,
+                    "tabs":[
+                        {
+                            "type":"courseware"
+                        },
+                        {
+                            "type":"course_info",
+                            "name":"Course Info"
+                        },
+                        {
+                            "type":"discussion",
+                            "name":"Discussion"
+                        },
+                        {
+                            "type":"wiki",
+                            "name":"Wiki"
+                        }
+                    ],
+                    "enrollment_start":null,
+                    "graceperiod":null,
+                    "start":"2013-02-14T05:00",
+                    "enrollment_end":null,
+                    "data_dir":"MITx-2-Base",
+                    "advertised_start":null,
+                    "display_name":"A wonderful course"
+                },
+                "update_version":{ "$oid" : "1d00000000000000dddd4444" },
+                "previous_version":null,
+                "edited_by":"test@edx.org",
+                "edited_on":{
+                    "$date":1364480313238
+                }
+            }
+        }
+    },
+    {
+        "_id": { "$oid" : "1d00000000000000eeee0000"},
+        "root":"head23456",
+        "original_version":{ "$oid" : "1d00000000000000eeee0000" },
+        "previous_version":null,
+        "edited_by":"test@edx.org",
+        "edited_on":{
+            "$date":1364481333238
+        },
+        "blocks":{
+            "head23456":{
+                "children":[
+                    
+                ],
+                "category":"course",
+                "definition":"head23456_1",
+                "metadata":{
+                    "end":null,
+                    "tabs":[
+                        {
+                            "type":"courseware"
+                        },
+                        {
+                            "type":"course_info",
+                            "name":"Course Info"
+                        },
+                        {
+                            "type":"discussion",
+                            "name":"Discussion"
+                        },
+                        {
+                            "type":"wiki",
+                            "name":"Wiki"
+                        }
+                    ],
+                    "enrollment_start":null,
+                    "graceperiod":null,
+                    "start":"2013-02-14T05:00",
+                    "enrollment_end":null,
+                    "data_dir":"MITx-2-Base",
+                    "advertised_start":null,
+                    "display_name":"The most wonderful course"
+                },
+                "update_version":{ "$oid" : "1d00000000000000eeee0000" },
+                "previous_version":null,
+                "edited_by":"test@edx.org",
+                "edited_on":{
+                    "$date":1364481333238
+                }
+            }
+        }
+    },
+    {
+        "_id": { "$oid" : "1d00000000000000dddd5555"},
+        "root":"head345679",
+        "original_version":{ "$oid" : "1d00000000000000dddd5555" },
+        "previous_version":null,
+        "edited_by":"test@guestx.edu",
+        "edited_on":{
+            "$date":1364491313238
+        },
+        "blocks":{
+            "head345679":{
+                "children":[
+                    
+                ],
+                "category":"course",
+                "definition":"head345679_1",
+                "metadata":{
+                    "end":null,
+                    "tabs":[
+                        {
+                            "type":"courseware"
+                        },
+                        {
+                            "type":"course_info",
+                            "name":"Course Info"
+                        },
+                        {
+                            "type":"discussion",
+                            "name":"Discussion"
+                        },
+                        {
+                            "type":"wiki",
+                            "name":"Wiki"
+                        }
+                    ],
+                    "enrollment_start":null,
+                    "graceperiod":null,
+                    "start":"2013-03-14T05:00",
+                    "enrollment_end":null,
+                    "data_dir":"MITx-3-Base",
+                    "advertised_start":null,
+                    "display_name":"Yet another contender"
+                },
+                "update_version":{ "$oid" : "1d00000000000000dddd5555" },
+                "previous_version":null,
+                "edited_by":"test@guestx.edu",
+                "edited_on":{
+                    "$date":1364491313238
+                }
+            }
+        }
+    }
+]

docs/source/persistence.rst

@@ -0,0 +1,658 @@
+
+
+
+This document describes the split mongostore representation which
+separates course structure from content where each course run can have
+its own structure. It does not describe the original mongostore
+representation which combined structure and content and used the key
+to distinguish draft from published elements.
+
+This document does not describe mongo nor its operations. See
+`http://www.mongodb.org/`_ for information on Mongo.
+
+
+
+Product Goals and Discussion
+----------------------------
+
+(Mark Chang)
+
+This work was instigated by the studio team's need to correctly do
+metadata inheritance. As we moved from an on-startup load of the
+courseware, the system was able to inflate and perform an inheritance
+calculation step such that the intended properties of children could
+be set through inheritance. While not strictly a requirement from the
+studio authoring approach, where inheritance really rears its head is
+on import of existing courseware that was designed assuming
+inheritance.
+
+A short term patch was applied that allowed inheritance to act
+correctly, but it was felt that it was insufficient and this would be
+an opportunity to make a more clean datastore representation. After
+much difficulty with how draft objects would work, Calen Pennington
+worked through a split data store model ala FAT filesystem (Mark's
+metaphor, not Cale's) to split the structure from the content. The
+goal would be a sea of content documents that would not know about the
+structure they were utilized within. Cale began the work and handed it
+off to Don Mitchell.
+
+In the interim, great discussion was had at the Architect's Council
+that firmed up the design and strategy for implementation, adding
+great richness and completeness to the new data structure.
+
+The immediate
+needs are two, and only two.
+
+
+#. functioning metadata inheritance
+#. good groundwork for versioning
+
+
+While the discussions of the atomic unit of courseware available for
+sharing, how these are shared, and how they refer back to the parent
+definition are all valuable, they will not be built in the near term. I
+understand and expect there to be many refactorings, improvements, and
+migrations in the future. 
+
+I fully anticipate much more detail to be uncovered even in this first
+thin implementation. When that happens, we will need as much advice
+from those watching this page to make sure we move in the right
+direction. We also must have the right design artifacts to document
+where we stand relative to the overall design that has loftier goals.
+
+
+Representation
+--------------
+
+The xmodule collections:
+
+
++ `modulestore.active_versions`: this collection maps the org, course,
+  and run to the current draft and published versions of the course.
++ `modulestore.structures`: this collection has one entry per course
+  run and one for the template.
++ `modulestore.definitions`: this collection has one entry per
+  "module" or "block" version.
+
+modulestore.active_versions: 2 simple maps for dereferencing the
+correct course from the structures collection. Every course run will
+have a draft version. Not every course run will have a published
+version. No course run will have more than one of each of these.
+
+::
+
+    { '_id' : uniqueid,
+      'versions' : { <versionName> : versionGuid, ..}
+      'creator' : user_id,
+      'created' : date (native mongo rep)
+    }
+
+::
+
+
+
++ `id` is a unique id for finding this course run. It's a 
+  location-reference string, like 'edu.mit.eng.eecs.6002x.industry.spring2013'.
++ `versions`: These are references to `modulestore.structures`. A
+  location-reference like
+  `edu.mit.eng.eecs.6002x.industry.spring2013;draft` refers to the value
+  associated with `draft` for this document.
+
+    + `versionName` is `draft`, `published`, or another user-defined
+      string.
+    + `versionGuid` is a system generated globally unique id (hash). It
+      points to the entry in `modulestore.structures` ` `
+
+
+
+`draftVersion`: the design will try to generate a new draft version
+for each change to the course object: that is, for each move,
+deletion, node creation, or metadata change. Cloning a course
+(creating a new run of a course or such) will create a new entry in
+this table with just a `draftVersion` and will cause a copy of the
+corresponding entry in `modulestore.structures`. The entry in
+`structures` will point to its version parent in the source course.
+
+
+
+
+modulestore.structures : the entries in this collection follow this
+definition:
+
+::
+
+    { '_id' : course_guid,
+      'blocks' : 
+        { block_guid :  // the guid is an arbitrary id to represent this node in the course tree
+            { 'children' : [ block_guid* ],
+              'metadata' : { property map },
+              'definition' : definition_guid,
+              'category' : 'section' | 'sequence' | ... } 
+
+
+::
+
+          ...// more guids
+
+
+::
+
+        },
+      'root' : block_guid,
+      'original' : course_guid, // the first version of this course from which all others were derived
+      'previous' : course_guid | null, // the previous revision of this course (null if this is the original)
+      'version_entry' : uniqueid, // from the active_versions collection
+      'creator' : user_id 
+    }
+
+
+
++ `blocks`: each block is a node in the course such as the course, a
+  section, a subsection, a unit, or a component. The block ids remain
+  the same over edits (they're not versioned).
++ `root`: the true top of the course. Not all nodes without parents
+  are truly roots. Some are orphans.
++ `course_guid, block_guid, definition_guid` are not those specific
+  strings but instead some system generated globally unique id.
+
+    + The one which gets passed around and pointed to by urls is the
+      `block_guid`; so, it will be the one the system ensures is readable.
+      Unlike the other guids, this one stays the same over revisions and can
+      even be the same between course runs (although the course run
+      contextualizes it to distinguish its instantiated version).
+
++ `definition` points to the specific revision of the given element in
+  `modulestore.definitions` which this version of the course includes.
++ `children` lists the block_guids which are the children of this node
+  in the course tree. It's an error if the guid in the `children` list
+  does not occur in the `blocks` dictionary.
++ `metadata` is the node's explicitly defined metadata some of which
+  may be inherited by its children
+
+
+For debugging purposes, there may be value in adding a courseId field
+(org, course, run) for use via db browsers.
+
+modulestore.definitions : the data associated with each version of
+each node in the structures. Many courses may point to the same
+definition or may point to different versions derived from the same
+original definition.
+
+::
+
+    { '_id' : guid,
+      'data' : ..,
+      'default_settings' : {'display_name':..,..}, // a starting point for new uses of this definition
+      'category' : xblocktype, // the xmodule/xblock type such as course, problem, html, video, about
+      'original' : guid, // the first kept version of this definition from which all others were derived
+      'previous' : guid | null, // the previous revision of this definition (null if this is the original)
+      'creator' : user_id  // the id of whomever pressed the draft or publish button
+    }
+
+
+
++ `_id`: a guid to uniquely identify the definition.
++ `data` is the payload used by the xmodule and following the
+  xmodule's data representation.
++ `category` is the xmodule type and used to figure out which xmodule
+  to instantiate.
+
+
+There may be some debugging value to adding a courseId field, but it
+may also be misleading if the element is used in more than one course.
+
+
+Templates
+~~~~~~~~~
+
+(I'm refactoring templates quite a bit from their representation prior
+to this design)
+
+All field defaults will be defined through the xblock field.default
+mechanism. Templates, otoh, are for representing optional boilerplate
+usually for examples such as a multiple-choice problem or a video
+component with the fields all filled in. Templates are stored in yaml
+files which provide a template name, sorting and filtering information
+(e.g., requires advanced editor v allows simple editor), and then
+field: value pairs for setting xblocks' fields upon template
+selection.
+
+Most of the pre-existing templates including all of the 'empty' ones
+will go away. The ones which will stay are the ones truly just giving
+examples or starting points for variants. This change will require
+that the template choice code provide a default 'blank' choice to the
+user which just instantiates the model w/ its defaults versus a choice
+of the boilerplates. The client can therefore populate its own model
+of the xblock and then send a create-item request to the server when
+the user says he/she's ready to save it.
+
+
+Import/export
+~~~~~~~~~~~~~
+
+Export should allow the user to select the version of the course to
+export which can be any of the draft or published versions. At a
+minimum, the user should choose between draft or published.
+
+Import should import the course as a draft course regardless of
+whether it was exported as a published or draft one, I believe. If
+there's already a draft for the same course, in the best of all
+worlds, it would have the guid to see if the guid exists in the
+structures collection, and, if so, just make that the current
+draftVersion (don't do any actual data changes). If there's no guid or
+the guid doesn't exist in the structures collection, then we'll need
+to work out the logic for how to decide what definitions to create v
+update v point to.
+
+
+Course ID
+~~~~~~~~~
+
+Currently, we use a triple to identify a run of a course. The triple
+is organization, course name, and run identity (e.g., 2013Q1). The
+system does not care what the id consists of only that it uniquely
+identify an edition of the course. The system uses this id to organize
+the course composition and find the course elements. It distinguishes
+between a current being-edited version (aka, draft) and publicly
+viewable version (published). Not every course has a published
+version, but every course will have a draft version. The application
+specifies whether it wants the draft or published version. This system
+allows the application to easily switch between the 2; however, it
+will have a configuration in which it's impossible to access the draft
+so that we can add access optimizations and extraction filtering later
+if needed.
+
+
+Location
+~~~~~~~~
+
+The purpose of `Location` is to identify content. That is, to be able
+to locate content by providing sufficient addressing. The `Location`
+object is ubiquitous throughout the current code and thus will be
+difficult to adapt and make more flexible. Right now, it's a very
+simple `namedtuple` and a lot of code presumes this. This refactoring
+generalizes and subclasses it to handle various addressing schemes and
+remove direct manipulations.
+
+Our code needs to locate several types of things and should probably
+use several different types of locators for these. These are the types
+of things we need to address. Some of these can be the same as others,
+but I wanted to lay them out fairly fine grained here before proposing
+my distinctions:
+
+
+#. Courses: an object representing a course as an offering but not any
+   of its content. Used for dashboards and other such navigators. These
+   may specify a version or merely reference the idea of the course's
+   existence.
+#. Course structures: the names (and other metadata), `Locations`, and
+   children pointers but not definitions for all the blocks in a course
+   or a subtree of a course. Our applications often display contextual,
+   outline, or other such structural information which do not need to
+   include definitions but need to show display names, graded as, and
+   other status info. This document's design makes fetching these a
+   single document fetch; however, if it has to fetch the full course, it
+   will require far more work (getting all definitions too) than the apps
+   need.
+#. Blocks (uses of definitions within a version of a course including
+   metadata, pointers to children, and type specific content)
+#. Definitions: use independent definitions of content without
+   metadata (and currently w/o pointers to children).
+#. Version trees Fetching the time history portrayal of a definition,
+   course, or block including branching.
+#. Collections of courses, definitions, or blocks matching some
+   partial descriptors (e.g., all courses for org x, all definitions of
+   type foo, all blocks in course y of type x, all currently accessible
+   courses (published with startdate < today and enddate > today)).
+#. Fetching of courses, blocks, or definitions via "human readable"
+   urls. 
+#. (partial descriptors) may suffice for this as human readable
+   does not guarantee uniqueness.
+
+
+Some of these differ not so much in how to address them but in what
+should be returned. The content should be up to the functions not the
+addressing scheme. So, I think the addressable things are:
+
+
+#. Course as in #1 above: usually a specific offering of a course.
+   Often used as a context for the other queries.
+#. Blocks (aka usages) as in #3 above: a specific block contextualized
+   in a course
+#. Definitions (#4): a specific definition
+#. Collections of courses, blocks within a specific course, or
+   definitions matching a partial descriptor
+
+
+
+Course locator (course_loc)
+```````````````````````````
+
+There are 3 ways to locate a course:
+
+
+#. By its unique id in the `active_versions` collection with an
+   implied or specified selection of draft or published version.
+#. By its unique id in the `structures` collection.
+
+
+
+Block locator (block_loc)
+`````````````````````````
+
+A block locator finds a specific node in a specific version of a
+course. Thus, it needs a course locator plus a `usage_id`.
+
+
+Definition locator (definition_loc)
+```````````````````````````````````
+
+Just a `guid`.
+
+
+Partial descriptor collections locators (partial)
+`````````````````````````````````````````````````
+
+In the most general case, and to simplify implementation, these can be
+any payload passable to mongo for doing the lookup. The specification
+of which collection to look into can be implied by which lookup
+function your code calls (get_courses, get_blocks, get_definitions) or
+we could add it as another property. For now, I will leave this as
+merely a search string. Thus, to find all courses for org = mitx,
+`{"org": "mitx"}`. To find all blocks in a course whose display name
+contains "circuit example", call `get_blocks` with the course locator
+plus `{"metadata.display_name" : /circuit example/i}` (the i makes it
+case insensitive and is just an example). To find if a definition is
+used in a course, call get_blocks with the course locator plus
+`{definition : definition_guid}`. Note, this looks for a specific
+version of the definition. If you wanted to see if it used any of a
+set of versions, use `{definition : {"$in" : [definition_guid*]}}`
+
+
+i4x locator
+```````````
+
+To support existing xml based courses and any urls, we need to
+support i4x locators. These are tuples of `(org course category id
+['draft'])`. The trouble with these is that they don't uniquely
+identify a course run from which to dereference the element. There's
+also no requirement that `id` have any uniqueness outside the scope of
+the other elements. There's some debate as to whether these address
+blocks or definitions. To mean, they seem to address blocks; however,
+in the current system there is no distinction between blocks and
+definitions; so, either could be argued.
+
+This version will define an `i4x_location` class for representing
+these and using them for xml based courses if necessary.
+
+Current code munges strings to make them 'acceptable' by replacing
+'illegal' chars with underscores. I'd like to suggest leaving strings
+as is and using url escaping to make acceptable urls. As to making
+human readable names from display strings, that should be the
+responsibility of the naming module not the Location representation,
+imo.
+
+
+Use cases (expository)
+~~~~~~~~~~~~~~~~~~~~~~
+
+There's a section below walking through a specific use case. This one
+just tries to review potential functionality.
+
+
+Inheritance
+```````````
+
+Our system has the notion of policies which should control the
+behavior of whole courses or subtrees within courses. Such policies
+include graceperiods, discussion forum controls, dates, whether to
+show answers, how to randomize, etc. It's important that the course
+authors' intent propagates to all relevant course sections. The
+desired behavior is that (some? all?) metadata attributes on modules
+flow down to all children unless overridden.
+
+This design addresses inheritance by making course structure and
+metadata separate from content thus enabling a single or small number
+of db queries to get these and then compute the inheritance.
+
+
+Separating editing from live production
+```````````````````````````````````````
+
+Course authors should be able to make changes in isolation from
+production and then push out consistent chunks of changes for all
+students to see as atomic and consistent. The current system allows
+authors to change text and content without affecting production but
+not metadata nor course structure. This design separates all changes
+from production until pushed.
+
+
+Sharing of content, part 1
+``````````````````````````
+
+Authors want to share content between course runs and even between
+different courses. The current system requires copying all such
+content and losing the providence information which could be used to
+take advantage of other peoples' changes. This design allows multiple
+courses and multiple places within a course to point to the same
+definitions and thus potentially, at some day, see other changes to
+the content.
+
+
+Sharing of content, part 2: course structure
+````````````````````````````````````````````
+
+Because courses structures are separate from their identities, courses
+can share structure and track changes in the same way as definitions.
+That is, a new course run can point to an existing course instance
+with its version history and then branch it from there.
+
+
+Sharing of content, part 3: modules
+```````````````````````````````````
+
+Suppose a course includes a soldering tutorial (or a required lab
+safety lesson). Other courses want to use the same tutorial and
+possibly allow the student to skip it if the student succeeded at it
+in another course. As the tutorial updates, other courses may want to
+track the updates or choose to move to the updates without having to
+copy the modules from the module's authoritative parent course.
+
+This design enables sharing of composed modules but it does not track
+the revisions of those modules separately from their courses. It does
+not adequately address this but may be extendible enough to do so.
+That is, we could represent these shared units as separate "courses"
+and allow ids in block.children[] to point to courses as well as other
+blocks in the same course.
+
+We should decide on the behaviors we want. Such as, some times the
+student has to repeat the content or the student never has to repeat
+it or? progress should be tracked by the owning course or as a stand
+alone minicourse type element? Because it's a safety lesson, all
+courses should track the current published head and not have their own
+heads or they should choose when to promote the head?
+
+Are these shared elements rare and large grained enough to make the
+indirection not expensive or will it result in devolving to the
+current one entry per module design for deducing course structure?
+
+
+Functional differences from existing modulestore:
+-------------------------------------------------
+
+
++ Courses and definitions support trees of versions knowing from where
+  they were derived. For now, I will not implement the server functions
+  for retrieving and manipulating these version trees and will leave
+  those for a future effort. I will only implement functions which
+  extend the trees.
++ Changes to course structure don't immediately affect production:
+  note, we need to figure out the granularity of the user's publish
+  behavior for pushing out these actions. That is, do they publish a
+  whole subtree which may include new children in order to make these
+  effective, do they publish all structural (deletion, move) changes
+  under a subtree but not insertions as an action, do they publish each
+  action individually, or what? How do they know that any of these are
+  not yet published? Do we have phantom placeholders for deleted nodes
+  w/ "publish deletion" buttons?
+
+    + Element deletion
+    + Element move
+    + metadata changes
+
++ No location objects used as ids! This implementation will use guids
+  instead. There's a reasonable objection to guids as being too ugly,
+  long, and indecipherable. I will check mongy, pymongo, and python guid
+  generation mechanisms to find out if there's a way to make ones which
+  include a prepended string (such as course and run or an explicitly
+  stated prepend string) and minimize guid length (e.g., by using
+  sequential serial # from a global or local pool).
+
+
+
+Use case walkthrough:
+---------------------
+
+Simple course creation with no precursor course: Note, this shows that
+publishing creates subsets and side copies not in line versions of
+nodes.
+user db create course for org, course id, run id
+active_versions.draftVersion: add entry
+definitions: add entry C w/ category = 'course', no data
+structures: add entry w/ 1 child C, original = self, no previous,
+author = user
+add section S copy structures entry, new one points to old as original
+and previous
+active_versions.draftVersion points to new
+definitions: add entry S w/ category = 'section'
+structures entry:
+
++ add S to children of the course block,
+
+
+
++ add S to blocks w/ no children
+
+add subsection T copy structures entry, new one points to old as
+original and previous
+active_versions.draftVersion points to new
+definitions: add entry T w/ category = 'sequential'
+structures entry:
+
++ add T to children of the S block entry,
+
+
+
++ add T to blocks w/ no children
+
+add unit U copy structures entry, new one points to old as original
+and previous
+active_versions.draftVersion points to new
+definitions: add entry U w/ category = 'vertical'
+structures entry:
+
++ add U to children of the T block entry,
+
+
+
++ add U to blocks w/ no children
+
+publish U
+create structures entry, new one points to self as original (no
+pointer to draft course b/c it's not really a clone)
+active_versions.publishedVersion points to new
+block: add U, T, S, C pointers with each as respective child
+(regardless of other children they may have in draft), and their
+metadata
+add units V, W, X under T copy structures entry of the draftVersion,
+new one points to old as original and previous
+active_versions.draftVersion points to new
+definitions: add entries V, W, X w/ category = 'vertical'
+structures entry:
+
++ add V, W, X to children of the T block entry,
+
+
+
++ add V, W, X to blocks w/ no children
+
+edit U copy structures entry, new one points to old as original and
+previous
+active_versions.draftVersion points to new
+definitions: copy entry U to U_2 w/ updates, U_2 points to U as
+original and previous
+structures entry:
+
++ replace U w/ U_2 in children of the T block entry,
+
+
+
++ copy entry U in blocks to entry U_2 and remove U
+
+add subsection Z under S copy structures entry, new one points to old
+as original and previous
+active_versions.draftVersion points to new
+definitions: add entry Z w/ category = 'sequential'
+structures entry:
+
++ add Z to children of the S block entry,
+
+
+
++ add Z to blocks w/ no children
+
+edit S's name (metadata) copy structures entry, new one points to old
+as original and previous
+active_versions.draftVersion points to new
+structures entry: update S's metadata w/ new name publish U, V copy
+publishedCourse structures entry, new one points to old published as
+original and previous
+active_versions.publishedVersion points to new
+block: update T to point to new U & V and not old U
+Note: does not update S's name publish C copy publishedCourse
+structures entry, new one points to old published as original and
+previous
+active_versions.publishedVersion points to new
+blocks: note that C child S == published(S) but metadata !=, update
+metadata
+note that S has unpublished children: publish them (recurse on this)
+note that Z is unpublished: add pointer to blocks and children of S
+note that W, X unpublished: add to blocks, add to children of T edit C
+metadata (e.g., graceperiod) copy draft structures entry, new one
+points to old as original and previous
+active_versions.draftVersion points to new
+structures entry: update C's metadata add Y under Z ... publish C's
+metadata change copy publishedCourse structures entry, new one points
+to old published as original and previous
+active_versions.publishedVersion points to new
+blocks: update C's metadata
+Note: no copying of Y or any other changes to published move X under Z
+copy draft structures entry, new one points to old as original and
+previous
+active_versions.draftVersion points to new
+structures entry: remove X from T's children and add to Z's
+Note: making it persistently clear to the user that X still exists
+under T in the published version will be crucial delete W copy draft
+structures entry, new one points to old as original and previous
+active_versions.draftVersion points to new
+structures entry: remove W from T's children and remove W from blocks
+Note: no actual deletion of W, just no longer reachable w/in the draft
+course, but still in published; so, need to keep user aware of that.
+publish Z Note: the interesting thing here is that X cannot occur
+under both Z and T, but the user's not publishing T, here's where
+having a consistent definition of original may help. If the original
+of a new element == original of an existing, then it's an update?
+copy publishedCourse entry...
+definitions: add Y, copy/update Z, X if either have any data changes
+(they don't)
+blocks: remove X from T's children and add to Z's, add Y to Z, add Y
+publish deletion of W copy publishedCourse entry...
+structures entry: remove W from T's children and remove W from blocks
+Conflict detection:
+
+Need a scenario where 2 authors make edits to different parts of
+course, to parts while parents being moved, while parents being
+deleted, to same parts, ...
+
+.. _http://www.mongodb.org/: http://www.mongodb.org/
+ 

lms/djangoapps/courseware/features/problems.feature

@@ -129,3 +129,45 @@ Feature: Answer problems
         When I press the button with the label "Hide Answer(s)"
         Then the button with the label "Show Answer(s)" does appear
         And I should not see "4.14159" anywhere on the page
+    
+    Scenario: I can see my score on a problem when I answer it and after I reset it
+        Given I am viewing a "<ProblemType>" problem
+        When I answer a "<ProblemType>" problem "<Correctness>ly"
+        Then I should see a score of "<Score>"
+        When I reset the problem
+        Then I should see a score of "<Points Possible>"
+
+        Examples:
+        | ProblemType       | Correctness   | Score               | Points Possible    |
+        | drop down         | correct       | 1/1 points          | 1 point possible   |
+        | drop down         | incorrect     | 1 point possible    | 1 point possible   |
+        | multiple choice   | correct       | 1/1 points          | 1 point possible   |
+        | multiple choice   | incorrect     | 1 point possible    | 1 point possible   |
+        | checkbox          | correct       | 1/1 points          | 1 point possible   |
+        | checkbox          | incorrect     | 1 point possible    | 1 point possible   |
+        | radio             | correct       | 1/1 points          | 1 point possible   |
+        | radio             | incorrect     | 1 point possible    | 1 point possible   |
+        | string            | correct       | 1/1 points          | 1 point possible   |
+        | string            | incorrect     | 1 point possible    | 1 point possible   |
+        | numerical         | correct       | 1/1 points          | 1 point possible   |
+        | numerical         | incorrect     | 1 point possible    | 1 point possible   |
+        | formula           | correct       | 1/1 points          | 1 point possible   |
+        | formula           | incorrect     | 1 point possible    | 1 point possible   |
+        | script            | correct       | 2/2 points          | 2 points possible  |
+        | script            | incorrect     | 2 points possible   | 2 points possible  |
+
+    Scenario: I can see my score on a problem to which I submit a blank answer
+        Given I am viewing a "<ProblemType>" problem
+        When I check a problem
+        Then I should see a score of "<Points Possible>"
+
+        Examples:
+        | ProblemType       | Points Possible    |
+        | drop down         | 1 point possible   |
+        | multiple choice   | 1 point possible   |
+        | checkbox          | 1 point possible   |
+        | radio             | 1 point possible   |
+        | string            | 1 point possible   |
+        | numerical         | 1 point possible   |
+        | formula           | 1 point possible   |
+        | script            | 2 points possible  |

lms/djangoapps/courseware/features/problems.py

@@ -142,6 +142,11 @@ def button_with_label_present(_step, buttonname, doesnt_appear):
         assert world.browser.is_text_present(buttonname, wait_time=5)
 
 
+@step(u'I should see a score of "([^"]*)"$')
+def see_score(_step, score):
+    assert world.browser.is_text_present(score)
+
+
 @step(u'My "([^"]*)" answer is marked "([^"]*)"')
 def assert_answer_mark(step, problem_type, correctness):
     """

lms/templates/problem.html

@@ -1,11 +1,11 @@
 <%namespace name='static' file='static_content.html'/>
 <h2 class="problem-header">
   ${ problem['name'] }
-  % if problem['weight'] != 1 and problem['weight'] is not None:
-    : ${ problem['weight'] } points
-  % endif
 </h2>
 
+<section class="problem-progress">
+</section>
+
 <section class="problem">
   ${ problem['html'] }
 

lms/templates/problem_ajax.html

@@ -1 +1 @@
-<section id="problem_${element_id}" class="problems-wrapper" data-problem-id="${id}" data-url="${ajax_url}" progress="${progress}"></section>
+<section id="problem_${element_id}" class="problems-wrapper" data-problem-id="${id}" data-url="${ajax_url}" data-progress_status="${progress_status}" data-progress_detail="${progress_detail}"></section>

requirements/edx/github.txt

@@ -8,6 +8,6 @@
 -e git://github.com/eventbrite/zendesk.git@d53fe0e81b623f084e91776bcf6369f8b7b63879#egg=zendesk
 
 # Our libraries:
--e git+https://github.com/edx/XBlock.git@3974e999fe853a37dfa6fadf0611289434349409#egg=XBlock
+-e git+https://github.com/edx/XBlock.git@b697bebd45deebd0f868613fab6722a0460ca0c1#egg=XBlock
 -e git+https://github.com/edx/codejail.git@c08967fb44d1bcdb259d3ec58812e3ac592539c2#egg=codejail
 -e git+https://github.com/edx/diff-cover.git@v0.1.3#egg=diff_cover