Merge branch 'master' of github.com:edx/edx-platform into feature/ichuang/import-with-no-static

Conflicts:
	common/djangoapps/static_replace/__init__.py
	common/djangoapps/xmodule_modifiers.py
	lms/djangoapps/courseware/courses.py
	lms/djangoapps/courseware/module_render.py

CHANGELOG.rst

@@ -5,12 +5,31 @@ These are notable changes in edx-platform.  This is a rolling list of changes,
 in roughly chronological order, most recent first.  Add your entries at or near
 the top.  Include a label indicating the component affected.
 
+Blades: Took videoalpha out of alpha, replacing the old video player
+
+LMS: Enable beta instructor dashboard. The beta dashboard is a rearchitecture
+of the existing instructor dashboard and is available by clicking a link at
+the top right of the existing dashboard.
+
+Common: CourseEnrollment has new fields `is_active` and `mode`. The mode will be
+used to differentiate different kinds of enrollments (currently, all enrollments
+are honor certificate enrollments). The `is_active` flag will be used to
+deactivate enrollments without deleting them, so that we know what course you
+*were* enrolled in. Because of the latter change, enrollment and unenrollment
+logic has been consolidated into the model -- you should use new class methods
+to `enroll()`, `unenroll()`, and to check `is_enrolled()`, instead of creating
+CourseEnrollment objects or querying them directly.
+
 Studio: Email will be sent to admin address when a user requests course creator
 privileges for Studio (edge only).
 
 Studio: Studio course authors (both instructors and staff) will be auto-enrolled
 for their courses so that "View Live" works.
 
+Common: Add a new input type ``<formulaequationinput />`` for Formula/Numerical
+Responses. It periodically makes AJAX calls to preview and validate the
+student's input.
+
 Common: Added ratelimiting to our authentication backend.
 
 Common: Add additional logging to cover login attempts and logouts.
@@ -77,6 +96,8 @@ LMS: Removed press releases
 
 Common: Updated Sass and Bourbon libraries, added Neat library
 
+LMS: Add a MixedModuleStore to aggregate the XMLModuleStore and MongoMonduleStore
+
 LMS: Users are no longer auto-activated if they click "reset password"
 This is now done when they click on the link in the reset password
 email they receive (along with usual path through activation email).
@@ -214,6 +235,12 @@ LMS: Fixed failing numeric response (decimal but no trailing digits).
 
 LMS: XML Error module no longer shows students a stack trace.
 
+Studio: Add feedback to end user if there is a problem exporting a course
+
+Studio: Improve link re-writing on imports into a different course-id
+
+Studio: Allow for intracourse linking in Capa Problems
+
 Blades: Videoalpha.
 
 XModules: Added partial credit for foldit module.
@@ -222,6 +249,10 @@ XModules: Added "randomize" XModule to list of XModule types.
 
 XModules: Show errors with full descriptors.
 
+Studio: Add feedback to end user if there is a problem exporting a course
+
+Studio: Improve link re-writing on imports into a different course-id
+
 XQueue: Fixed (hopefully) worker crash when the connection to RabbitMQ is
 dropped suddenly.
 

cms/djangoapps/auth/tests/test_authz.py

@@ -181,7 +181,7 @@ class CourseGroupTest(TestCase):
         create_all_course_groups(self.creator, self.location)
         add_user_to_course_group(self.creator, self.staff, self.location, STAFF_ROLE_NAME)
 
-        location2 = 'i4x', 'mitX', '103', 'course2', 'test2'
+        location2 = 'i4x', 'mitX', '103', 'course', 'test2'
         staff2 = User.objects.create_user('teststaff2', 'teststaff2+courses@edx.org', 'foo')
         create_all_course_groups(self.creator, location2)
         add_user_to_course_group(self.creator, staff2, location2, STAFF_ROLE_NAME)
@@ -193,7 +193,7 @@ class CourseGroupTest(TestCase):
         create_all_course_groups(self.creator, self.location)
         add_user_to_course_group(self.creator, self.staff, self.location, STAFF_ROLE_NAME)
 
-        location2 = 'i4x', 'mitX', '103', 'course2', 'test2'
+        location2 = 'i4x', 'mitX', '103', 'course', 'test2'
         creator2 = User.objects.create_user('testcreator2', 'testcreator2+courses@edx.org', 'foo')
         staff2 = User.objects.create_user('teststaff2', 'teststaff2+courses@edx.org', 'foo')
         create_all_course_groups(creator2, location2)

cms/djangoapps/contentstore/features/courses.feature

@@ -8,6 +8,6 @@ Feature: Create Course
     And I am logged into Studio
     When I click the New Course button
     And I fill in the new course information
-    And I press the "Save" button
+    And I press the "Create" button
     Then the Courseware page has loaded in Studio
     And I see a link for adding a new section

cms/djangoapps/contentstore/features/grading.feature

@@ -84,3 +84,12 @@ Feature: Course Grading
         And I am viewing the grading settings
         When I change assignment type "Homework" to ""
         Then the save button is disabled
+
+    Scenario: User can edit grading range names
+        Given I have opened a new course in Studio
+        And I have populated the course
+        And I am viewing the grading settings
+        When I change the highest grade range to "Good"
+        And I press the "Save" notification button
+        And I reload the page
+        Then I see the highest grade range is "Good"

cms/djangoapps/contentstore/features/grading.py

@@ -117,6 +117,19 @@ def i_see_the_assignment_type(_step, name):
       assert name in types
 
 
+@step(u'I change the highest grade range to "(.*)"$')
+def change_grade_range(_step, range_name):
+    range_css = 'span.letter-grade'
+    grade = world.css_find(range_css).first
+    grade.value = range_name
+
+
+@step(u'I see the highest grade range is "(.*)"$')
+def i_see_highest_grade_range(_step, range_name):
+    range_css = 'span.letter-grade'
+    grade = world.css_find(range_css).first
+    assert grade.value == range_name
+
 def get_type_index(name):
     name_id = '#course-grading-assignment-name'
     all_types = world.css_find(name_id)

cms/djangoapps/contentstore/management/commands/check_course.py

@@ -5,6 +5,9 @@ from xmodule.course_module import CourseDescriptor
 
 from request_cache.middleware import RequestCache
 
+from django.core.cache import get_cache
+
+CACHE = get_cache('mongo_metadata_inheritance')
 
 class Command(BaseCommand):
     help = '''Enumerates through the course and find common errors'''
@@ -19,7 +22,10 @@ class Command(BaseCommand):
         store = modulestore()
 
         # setup a request cache so we don't throttle the DB with all the metadata inheritance requests
-        store.request_cache = RequestCache.get_request_cache()
+        store.set_modulestore_configuration({
+            'metadata_inheritance_cache_subsystem': CACHE,
+            'request_cache': RequestCache.get_request_cache()
+        })
 
         course = store.get_item(loc, depth=3)
 

cms/djangoapps/contentstore/management/commands/clone_course.py

@@ -15,10 +15,6 @@ from auth.authz import _copy_course_group
 from request_cache.middleware import RequestCache
 from django.core.cache import get_cache
 
-#
-# To run from command line: rake cms:delete_course LOC=MITx/111/Foo1
-#
-
 CACHE = get_cache('mongo_metadata_inheritance')
 
 class Command(BaseCommand):
@@ -36,8 +32,11 @@ class Command(BaseCommand):
         mstore = modulestore('direct')
         cstore = contentstore()
 
-        mstore.metadata_inheritance_cache_subsystem = CACHE
-        mstore.request_cache = RequestCache.get_request_cache()
+        mstore.set_modulestore_configuration({
+            'metadata_inheritance_cache_subsystem': CACHE,
+            'request_cache': RequestCache.get_request_cache()
+        })
+
         org, course_num, run = dest_course_id.split("/")
         mstore.ignore_write_events_on_courses.append('{0}/{1}'.format(org, course_num))
 

cms/djangoapps/contentstore/management/commands/delete_course.py

@@ -36,8 +36,11 @@ class Command(BaseCommand):
         ms = modulestore('direct')
         cs = contentstore()
 
-        ms.metadata_inheritance_cache_subsystem = CACHE
-        ms.request_cache = RequestCache.get_request_cache()
+        ms.set_modulestore_configuration({
+            'metadata_inheritance_cache_subsystem': CACHE,
+            'request_cache': RequestCache.get_request_cache()
+        })
+
         org, course_num, run = course_id.split("/")
         ms.ignore_write_events_on_courses.append('{0}/{1}'.format(org, course_num))
 
@@ -48,4 +51,7 @@ class Command(BaseCommand):
                     print 'removing User permissions from course....'
                     # in the django layer, we need to remove all the user permissions groups associated with this course
                     if commit:
-                        _delete_course_group(loc)
+                        try:
+                            _delete_course_group(loc)
+                        except Exception as err:
+                            print("Error in deleting course groups for {0}: {1}".format(loc, err))

cms/djangoapps/contentstore/module_info_model.py

@@ -1,6 +1,5 @@
 from static_replace import replace_static_urls
 from xmodule.modulestore.exceptions import ItemNotFoundError
-from xmodule.modulestore import Location
 
 
 def get_module_info(store, location, rewrite_static_links=False):
@@ -13,16 +12,12 @@ def get_module_info(store, location, rewrite_static_links=False):
 
     data = module.data
     if rewrite_static_links:
+        # we pass a partially bogus course_id as we don't have the RUN information passed yet
+        # through the CMS. Also the contentstore is also not RUN-aware at this point in time.
         data = replace_static_urls(
             module.data,
             None,
-            course_namespace=Location([
-                module.location.tag,
-                module.location.org,
-                module.location.course,
-                None,
-                None
-            ])
+            course_id=module.location.org + '/' + module.location.course + '/BOGUS_RUN_REPLACE_WHEN_AVAILABLE'
         )
 
     return {

cms/djangoapps/contentstore/tests/test_contentstore.py

@@ -49,7 +49,7 @@ import datetime
 from pytz import UTC
 from uuid import uuid4
 from pymongo import MongoClient
-from student.views import is_enrolled_in_course
+from student.models import CourseEnrollment
 
 TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
 TEST_DATA_CONTENTSTORE['OPTIONS']['db'] = 'test_xcontent_%s' % uuid4().hex
@@ -400,6 +400,20 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
         resp = self.client.get(url)
         self.assertEqual(resp.status_code, 200)
 
+    def test_video_module_caption_asset_path(self):
+        '''
+        This verifies that a video caption url is as we expect it to be
+        '''
+        direct_store = modulestore('direct')
+        import_from_xml(direct_store, 'common/test/data/', ['toy'])
+
+        # also try a custom response which will trigger the 'is this course in whitelist' logic
+        video_module_location = Location(['i4x', 'edX', 'toy', 'video', 'sample_video', None])
+        url = reverse('preview_component', kwargs={'location': video_module_location.url()})
+        resp = self.client.get(url)
+        self.assertEqual(resp.status_code, 200)
+        self.assertContains(resp, 'data-caption-asset-path="/c4x/edX/toy/asset/subs_"')
+
     def test_delete(self):
         direct_store = modulestore('direct')
         CourseFactory.create(org='edX', course='999', display_name='Robot Super Course')
@@ -1168,7 +1182,7 @@ class ContentStoreTest(ModuleStoreTestCase):
         self.assertNotIn('ErrMsg', data)
         self.assertEqual(data['id'], 'i4x://MITx/{0}/course/2013_Spring'.format(test_course_data['number']))
         # Verify that the creator is now registered in the course.
-        self.assertTrue(is_enrolled_in_course(self.user, self._get_course_id(test_course_data)))
+        self.assertTrue(CourseEnrollment.is_enrolled(self.user, self._get_course_id(test_course_data)))
         return test_course_data
 
     def test_create_course_check_forum_seeding(self):
@@ -1190,14 +1204,14 @@ class ContentStoreTest(ModuleStoreTestCase):
         Checks that the course did not get created
         """
         course_id = self._get_course_id(self.course_data)
-        initially_enrolled = is_enrolled_in_course(self.user, course_id)
+        initially_enrolled = CourseEnrollment.is_enrolled(self.user, course_id)
         resp = self.client.post(reverse('create_new_course'), self.course_data)
         self.assertEqual(resp.status_code, 200)
         data = parse_json(resp)
         self.assertEqual(data['ErrMsg'], error_message)
         # One test case involves trying to create the same course twice. Hence for that course,
         # the user will be enrolled. In the other cases, initially_enrolled will be False.
-        self.assertEqual(initially_enrolled, is_enrolled_in_course(self.user, course_id))
+        self.assertEqual(initially_enrolled, CourseEnrollment.is_enrolled(self.user, course_id))
 
     def test_create_course_duplicate_number(self):
         """Test new course creation - error path"""

cms/djangoapps/contentstore/tests/test_crud.py

@@ -1,19 +1,15 @@
-'''
-Created on May 7, 2013
-
-@author: dmitchell
-'''
 import unittest
 from xmodule import templates
 from xmodule.modulestore.tests import persistent_factories
 from xmodule.course_module import CourseDescriptor
 from xmodule.modulestore.django import modulestore
 from xmodule.seq_module import SequenceDescriptor
-from xmodule.x_module import XModuleDescriptor
 from xmodule.capa_module import CapaDescriptor
 from xmodule.modulestore.locator import CourseLocator, BlockUsageLocator
 from xmodule.modulestore.exceptions import ItemNotFoundError
 from xmodule.html_module import HtmlDescriptor
+from xmodule.modulestore import inheritance
+from xmodule.x_module import XModuleDescriptor
 
 
 class TemplateTests(unittest.TestCase):
@@ -74,8 +70,8 @@ class TemplateTests(unittest.TestCase):
         test_course = persistent_factories.PersistentCourseFactory.create(org='testx', prettyid='tempcourse',
             display_name='fun test course', user_id='testbot')
 
-        test_chapter = XModuleDescriptor.load_from_json({'category': 'chapter',
-            'metadata': {'display_name': 'chapter n'}},
+        test_chapter = self.load_from_json({'category': 'chapter',
+            'fields': {'display_name': 'chapter n'}},
             test_course.system, parent_xblock=test_course)
         self.assertIsInstance(test_chapter, SequenceDescriptor)
         self.assertEqual(test_chapter.display_name, 'chapter n')
@@ -83,8 +79,8 @@ class TemplateTests(unittest.TestCase):
 
         # test w/ a definition (e.g., a problem)
         test_def_content = '<problem>boo</problem>'
-        test_problem = XModuleDescriptor.load_from_json({'category': 'problem',
-            'definition': {'data': test_def_content}},
+        test_problem = self.load_from_json({'category': 'problem',
+            'fields': {'data': test_def_content}},
             test_course.system, parent_xblock=test_chapter)
         self.assertIsInstance(test_problem, CapaDescriptor)
         self.assertEqual(test_problem.data, test_def_content)
@@ -98,12 +94,13 @@ class TemplateTests(unittest.TestCase):
         """
         test_course = persistent_factories.PersistentCourseFactory.create(org='testx', prettyid='tempcourse',
             display_name='fun test course', user_id='testbot')
-        test_chapter = XModuleDescriptor.load_from_json({'category': 'chapter',
-            'metadata': {'display_name': 'chapter n'}},
+        test_chapter = self.load_from_json({'category': 'chapter',
+            'fields': {'display_name': 'chapter n'}},
             test_course.system, parent_xblock=test_course)
         test_def_content = '<problem>boo</problem>'
-        test_problem = XModuleDescriptor.load_from_json({'category': 'problem',
-            'definition': {'data': test_def_content}},
+        # create child
+        _ = self.load_from_json({'category': 'problem',
+            'fields': {'data': test_def_content}},
             test_course.system, parent_xblock=test_chapter)
         # better to pass in persisted parent over the subdag so
         # subdag gets the parent pointer (otherwise 2 ops, persist dag, update parent children,
@@ -152,15 +149,24 @@ class TemplateTests(unittest.TestCase):
             parent_location=test_course.location, user_id='testbot')
         sub = persistent_factories.ItemFactory.create(display_name='subsection 1',
             parent_location=chapter.location, user_id='testbot', category='vertical')
-        first_problem = persistent_factories.ItemFactory.create(display_name='problem 1',
-            parent_location=sub.location, user_id='testbot', category='problem', data="<problem></problem>")
+        first_problem = persistent_factories.ItemFactory.create(
+            display_name='problem 1', parent_location=sub.location, user_id='testbot', category='problem',
+            data="<problem></problem>"
+        )
         first_problem.max_attempts = 3
+        first_problem.save()  # decache the above into the kvs
         updated_problem = modulestore('split').update_item(first_problem, 'testbot')
-        updated_loc = modulestore('split').delete_item(updated_problem.location, 'testbot')
+        self.assertIsNotNone(updated_problem.previous_version)
+        self.assertEqual(updated_problem.previous_version, first_problem.update_version)
+        self.assertNotEqual(updated_problem.update_version, first_problem.update_version)
+        updated_loc = modulestore('split').delete_item(updated_problem.location, 'testbot', delete_children=True)
 
-        second_problem = persistent_factories.ItemFactory.create(display_name='problem 2',
+        second_problem = persistent_factories.ItemFactory.create(
+            display_name='problem 2',
             parent_location=BlockUsageLocator(updated_loc, usage_id=sub.location.usage_id),
-            user_id='testbot', category='problem', data="<problem></problem>")
+            user_id='testbot', category='problem',
+            data="<problem></problem>"
+        )
 
         # course root only updated 2x
         version_history = modulestore('split').get_block_generations(test_course.location)
@@ -184,3 +190,48 @@ class TemplateTests(unittest.TestCase):
 
         version_history = modulestore('split').get_block_generations(second_problem.location)
         self.assertNotEqual(version_history.locator.version_guid, first_problem.location.version_guid)
+
+    # ================================= JSON PARSING ===========================
+    # These are example methods for creating xmodules in memory w/o persisting them.
+    # They were in x_module but since xblock is not planning to support them but will
+    # allow apps to use this type of thing, I put it here.
+    @staticmethod
+    def load_from_json(json_data, system, default_class=None, parent_xblock=None):
+        """
+        This method instantiates the correct subclass of XModuleDescriptor based
+        on the contents of json_data. It does not persist it and can create one which
+        has no usage id.
+
+        parent_xblock is used to compute inherited metadata as well as to append the new xblock.
+
+        json_data:
+        - 'location' : must have this field
+        - 'category': the xmodule category (required or location must be a Location)
+        - 'metadata': a dict of locally set metadata (not inherited)
+        - 'children': a list of children's usage_ids w/in this course
+        - 'definition':
+        - '_id' (optional): the usage_id of this. Will generate one if not given one.
+        """
+        class_ = XModuleDescriptor.load_class(
+            json_data.get('category', json_data.get('location', {}).get('category')),
+            default_class
+        )
+        usage_id = json_data.get('_id', None)
+        if not '_inherited_settings' in json_data and parent_xblock is not None:
+            json_data['_inherited_settings'] = parent_xblock.xblock_kvs.get_inherited_settings().copy()
+            json_fields = json_data.get('fields', {})
+            for field in inheritance.INHERITABLE_METADATA:
+                if field in json_fields:
+                    json_data['_inherited_settings'][field] = json_fields[field]
+
+        new_block = system.xblock_from_json(class_, usage_id, json_data)
+        if parent_xblock is not None:
+            children = parent_xblock.children
+            children.append(new_block)
+            # trigger setter method by using top level field access
+            parent_xblock.children = children
+            # decache pending children field settings (Note, truly persisting at this point would break b/c
+            # persistence assumes children is a list of ids not actual xblocks)
+            parent_xblock.save()
+        return new_block
+

cms/djangoapps/contentstore/tests/test_users.py

@@ -6,7 +6,7 @@ from .utils import CourseTestCase
 from django.contrib.auth.models import User, Group
 from django.core.urlresolvers import reverse
 from auth.authz import get_course_groupname_for_role
-from student.views import is_enrolled_in_course
+from student.models import CourseEnrollment
 
 
 class UsersTestCase(CourseTestCase):
@@ -372,13 +372,13 @@ class UsersTestCase(CourseTestCase):
     def assert_not_enrolled(self):
         """ Asserts that self.ext_user is not enrolled in self.course. """
         self.assertFalse(
-            is_enrolled_in_course(self.ext_user, self.course.location.course_id),
+            CourseEnrollment.is_enrolled(self.ext_user, self.course.location.course_id),
             'Did not expect ext_user to be enrolled in course'
         )
 
     def assert_enrolled(self):
         """ Asserts that self.ext_user is enrolled in self.course. """
         self.assertTrue(
-            is_enrolled_in_course(self.ext_user, self.course.location.course_id),
+            CourseEnrollment.is_enrolled(self.ext_user, self.course.location.course_id),
             'User ext_user should have been enrolled in the course'
         )

cms/djangoapps/contentstore/tests/test_utils.py

@@ -5,8 +5,6 @@ import collections
 import copy
 from django.test import TestCase
 from django.test.utils import override_settings
-from xmodule.modulestore.tests.factories import CourseFactory
-from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 
 
 class LMSLinksTestCase(TestCase):
@@ -56,21 +54,28 @@ class LMSLinksTestCase(TestCase):
     def get_about_page_link(self):
         """ create mock course and return the about page link """
         location = 'i4x', 'mitX', '101', 'course', 'test'
-        utils.get_course_id = mock.Mock(return_value="mitX/101/test")
         return utils.get_lms_link_for_about_page(location)
 
     def lms_link_test(self):
         """ Tests get_lms_link_for_item. """
         location = 'i4x', 'mitX', '101', 'vertical', 'contacting_us'
-        utils.get_course_id = mock.Mock(return_value="mitX/101/test")
-        link = utils.get_lms_link_for_item(location, False)
+        link = utils.get_lms_link_for_item(location, False, "mitX/101/test")
         self.assertEquals(link, "//localhost:8000/courses/mitX/101/test/jump_to/i4x://mitX/101/vertical/contacting_us")
-        link = utils.get_lms_link_for_item(location, True)
+        link = utils.get_lms_link_for_item(location, True, "mitX/101/test")
         self.assertEquals(
             link,
             "//preview/courses/mitX/101/test/jump_to/i4x://mitX/101/vertical/contacting_us"
         )
 
+        # If no course_id is passed in, it is obtained from the location. This is the case for
+        # Studio dashboard.
+        location = 'i4x', 'mitX', '101', 'course', 'test'
+        link = utils.get_lms_link_for_item(location)
+        self.assertEquals(
+            link,
+            "//localhost:8000/courses/mitX/101/test/jump_to/i4x://mitX/101/course/test"
+        )
+
 
 class ExtraPanelTabTestCase(TestCase):
     """ Tests adding and removing extra course tabs. """
@@ -145,4 +150,3 @@ class ExtraPanelTabTestCase(TestCase):
                 changed, actual_tabs = utils.remove_extra_panel_tab(tab_type, course)
                 self.assertFalse(changed)
                 self.assertEqual(actual_tabs, expected_tabs)
-

cms/djangoapps/contentstore/utils.py

@@ -89,8 +89,17 @@ def get_course_for_item(location):
 
 
 def get_lms_link_for_item(location, preview=False, course_id=None):
+    """
+    Returns an LMS link to the course with a jump_to to the provided location.
+
+    :param location: the location to jump to
+    :param preview: True if the preview version of LMS should be returned. Default value is false.
+    :param course_id: the course_id within which the location lives. If not specified, the course_id is obtained
+           by calling Location(location).course_id; note that this only works for locations representing courses
+           instead of elements within courses.
+    """
     if course_id is None:
-        course_id = get_course_id(location)
+        course_id = Location(location).course_id
 
     if settings.LMS_BASE is not None:
         if preview:
@@ -136,7 +145,7 @@ def get_lms_link_for_about_page(location):
     if about_base is not None:
         lms_link = "//{about_base_url}/courses/{course_id}/about".format(
             about_base_url=about_base,
-            course_id=get_course_id(location)
+            course_id=Location(location).course_id
         )
     else:
         lms_link = None
@@ -144,14 +153,6 @@ def get_lms_link_for_about_page(location):
     return lms_link
 
 
-def get_course_id(location):
-    """
-    Returns the course_id from a given the location tuple.
-    """
-    # TODO: These will need to be changed to point to the particular instance of this problem in the particular course
-    return modulestore().get_containing_courses(Location(location))[0].id
-
-
 class UnitState(object):
     draft = 'draft'
     private = 'private'

cms/djangoapps/contentstore/views/access.py

@@ -26,12 +26,16 @@ def has_access(user, location, role=STAFF_ROLE_NAME):
     There is a super-admin permissions if user.is_staff is set
     Also, since we're unifying the user database between LMS and CAS,
     I'm presuming that the course instructor (formally known as admin)
-    will not be in both INSTRUCTOR and STAFF groups, so we have to cascade our queries here as INSTRUCTOR
-    has all the rights that STAFF do
+    will not be in both INSTRUCTOR and STAFF groups, so we have to cascade our
+    queries here as INSTRUCTOR has all the rights that STAFF do
     '''
     course_location = get_course_location_for_item(location)
     _has_access = is_user_in_course_group_role(user, course_location, role)
     # if we're not in STAFF, perhaps we're in INSTRUCTOR groups
     if not _has_access and role == STAFF_ROLE_NAME:
-        _has_access = is_user_in_course_group_role(user, course_location, INSTRUCTOR_ROLE_NAME)
+        _has_access = is_user_in_course_group_role(
+                user,
+                course_location,
+                INSTRUCTOR_ROLE_NAME
+        )
     return _has_access

cms/djangoapps/contentstore/views/assets.py

@@ -4,6 +4,7 @@ import os
 import tarfile
 import shutil
 import cgi
+from functools import partial
 from tempfile import mkdtemp
 from path import path
 
@@ -34,7 +35,8 @@ from .access import get_location_and_verify_access
 from util.json_request import JsonResponse
 
 
-__all__ = ['asset_index', 'upload_asset', 'import_course', 'generate_export_course', 'export_course']
+__all__ = ['asset_index', 'upload_asset', 'import_course',
+        'generate_export_course', 'export_course']
 
 
 def assets_to_json_dict(assets):
@@ -58,13 +60,14 @@ def assets_to_json_dict(assets):
             obj["thumbnail"] = thumbnail
         id_info = asset.get("_id")
         if id_info:
-            obj["id"] = "/{tag}/{org}/{course}/{revision}/{category}/{name}".format(
-                org=id_info.get("org", ""),
-                course=id_info.get("course", ""),
-                revision=id_info.get("revision", ""),
-                tag=id_info.get("tag", ""),
-                category=id_info.get("category", ""),
-                name=id_info.get("name", ""),
+            obj["id"] = "/{tag}/{org}/{course}/{revision}/{category}/{name}" \
+                .format(
+                    org=id_info.get("org", ""),
+                    course=id_info.get("course", ""),
+                    revision=id_info.get("revision", ""),
+                    tag=id_info.get("tag", ""),
+                    category=id_info.get("category", ""),
+                    name=id_info.get("name", ""),
             )
         ret.append(obj)
     return ret
@@ -132,14 +135,14 @@ def asset_index(request, org, course, name):
 @login_required
 def upload_asset(request, org, course, coursename):
     '''
-    This method allows for POST uploading of files into the course asset library, which will
-    be supported by GridFS in MongoDB.
+    This method allows for POST uploading of files into the course asset
+    library, which will be supported by GridFS in MongoDB.
     '''
     # construct a location from the passed in path
     location = get_location_and_verify_access(request, org, course, coursename)
 
-    # Does the course actually exist?!? Get anything from it to prove its existance
-
+    # Does the course actually exist?!? Get anything from it to prove its
+    # existence
     try:
         modulestore().get_item(location)
     except:
@@ -150,9 +153,10 @@ def upload_asset(request, org, course, coursename):
     if 'file' not in request.FILES:
         return HttpResponseBadRequest()
 
-    # compute a 'filename' which is similar to the location formatting, we're using the 'filename'
-    # nomenclature since we're using a FileSystem paradigm here. We're just imposing
-    # the Location string formatting expectations to keep things a bit more consistent
+    # compute a 'filename' which is similar to the location formatting, we're
+    # using the 'filename' nomenclature since we're using a FileSystem paradigm
+    # here. We're just imposing the Location string formatting expectations to
+    # keep things a bit more consistent
     upload_file = request.FILES['file']
     filename = upload_file.name
     mime_type = upload_file.content_type
@@ -160,20 +164,25 @@ def upload_asset(request, org, course, coursename):
     content_loc = StaticContent.compute_location(org, course, filename)
 
     chunked = upload_file.multiple_chunks()
+    sc_partial = partial(StaticContent, content_loc, filename, mime_type)
     if chunked:
-        content = StaticContent(content_loc, filename, mime_type, upload_file.chunks())
+        content = sc_partial(upload_file.chunks())
+        temp_filepath = upload_file.temporary_file_path()
     else:
-        content = StaticContent(content_loc, filename, mime_type, upload_file.read())
+        content = sc_partial(upload_file.read())
+        tempfile_path = None
 
     thumbnail_content = None
     thumbnail_location = None
 
     # first let's see if a thumbnail can be created
-    (thumbnail_content, thumbnail_location) = contentstore().generate_thumbnail(content,
-                                                                                tempfile_path=None if not chunked else
-                                                                                upload_file.temporary_file_path())
+    (thumbnail_content, thumbnail_location) = contentstore().generate_thumbnail(
+            content,
+            tempfile_path=tempfile_path
+    )
 
-    # delete cached thumbnail even if one couldn't be created this time (else the old thumbnail will continue to show)
+    # delete cached thumbnail even if one couldn't be created this time (else
+    # the old thumbnail will continue to show)
     del_cached_content(thumbnail_location)
     # now store thumbnail location only if we could create it
     if thumbnail_content is not None:
@@ -186,13 +195,15 @@ def upload_asset(request, org, course, coursename):
     # readback the saved content - we need the database timestamp
     readback = contentstore().find(content.location)
 
-    response_payload = {'displayname': content.name,
-                        'uploadDate': get_default_time_display(readback.last_modified_at),
-                        'url': StaticContent.get_url_path_from_location(content.location),
-                        'portable_url': StaticContent.get_static_path_from_location(content.location),
-                        'thumb_url': StaticContent.get_url_path_from_location(thumbnail_location) if thumbnail_content is not None else None,
-                        'msg': 'Upload completed'
-                        }
+    response_payload = {
+            'displayname': content.name,
+            'uploadDate': get_default_time_display(readback.last_modified_at),
+            'url': StaticContent.get_url_path_from_location(content.location),
+            'portable_url': StaticContent.get_static_path_from_location(content.location),
+            'thumb_url': StaticContent.get_url_path_from_location(thumbnail_location)
+                if thumbnail_content is not None else None,
+            'msg': 'Upload completed'
+    }
 
     response = JsonResponse(response_payload)
     return response
@@ -202,8 +213,8 @@ def upload_asset(request, org, course, coursename):
 @login_required
 def remove_asset(request, org, course, name):
     '''
-    This method will perform a 'soft-delete' of an asset, which is basically to copy the asset from
-    the main GridFS collection and into a Trashcan
+    This method will perform a 'soft-delete' of an asset, which is basically to
+    copy the asset from the main GridFS collection and into a Trashcan
     '''
     get_location_and_verify_access(request, org, course, name)
 
@@ -348,6 +359,8 @@ def generate_export_course(request, org, course, name):
     try:
         export_to_xml(modulestore('direct'), contentstore(), loc, root_dir, name, modulestore())
     except SerializationError, e:
+        logging.exception('There was an error exporting course {0}. {1}'.format(course_module.location, unicode(e)))
+
         unit = None
         failed_item = None
         parent = None
@@ -380,6 +393,7 @@ def generate_export_course(request, org, course, name):
             })
         })
     except Exception, e:
+        logging.exception('There was an error exporting course {0}. {1}'.format(course_module.location, unicode(e)))
         return render_to_response('export.html', {
             'context_course': course_module,
             'successful_import_redirect_url': '',

cms/djangoapps/contentstore/views/checklist.py

@@ -30,7 +30,8 @@ def get_checklists(request, org, course, name):
     modulestore = get_modulestore(location)
     course_module = modulestore.get_item(location)
 
-    # If course was created before checklists were introduced, copy them over from the template.
+    # If course was created before checklists were introduced, copy them over
+    # from the template.
     copied = False
     if not course_module.checklists:
         course_module.checklists = CourseDescriptor.checklists.default
@@ -68,7 +69,8 @@ def update_checklist(request, org, course, name, checklist_index=None):
         if checklist_index is not None and 0 <= int(checklist_index) < len(course_module.checklists):
             index = int(checklist_index)
             course_module.checklists[index] = json.loads(request.body)
-            # seeming noop which triggers kvs to record that the metadata is not default
+            # seeming noop which triggers kvs to record that the metadata is
+            # not default
             course_module.checklists = course_module.checklists
             checklists, _ = expand_checklist_action_urls(course_module)
             course_module.save()
@@ -76,10 +78,13 @@ def update_checklist(request, org, course, name, checklist_index=None):
             return JsonResponse(checklists[index])
         else:
             return HttpResponseBadRequest(
-                "Could not save checklist state because the checklist index was out of range or unspecified.",
-                content_type="text/plain")
+                ( "Could not save checklist state because the checklist index "
+                "was out of range or unspecified."),
+                content_type="text/plain"
+            )
     elif request.method == 'GET':
-        # In the JavaScript view initialize method, we do a fetch to get all the checklists.
+        # In the JavaScript view initialize method, we do a fetch to get all
+        # the checklists.
         checklists, modified = expand_checklist_action_urls(course_module)
         if modified:
             course_module.save()

cms/djangoapps/contentstore/views/component.py

@@ -2,13 +2,15 @@ import json
 import logging
 from collections import defaultdict
 
-from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseForbidden
+from django.http import ( HttpResponse, HttpResponseBadRequest, 
+        HttpResponseForbidden )
 from django.contrib.auth.decorators import login_required
 from django.views.decorators.http import require_http_methods
 from django.core.exceptions import PermissionDenied
 from django_future.csrf import ensure_csrf_cookie
 from django.conf import settings
-from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError
+from xmodule.modulestore.exceptions import ( ItemNotFoundError, 
+        InvalidLocationError )
 from mitxmako.shortcuts import render_to_response
 
 from xmodule.modulestore import Location
@@ -19,8 +21,8 @@ from xblock.core import Scope
 from util.json_request import expect_json, JsonResponse
 
 from contentstore.module_info_model import get_module_info, set_module_info
-from contentstore.utils import get_modulestore, get_lms_link_for_item, \
-    compute_unit_state, UnitState, get_course_for_item
+from contentstore.utils import ( get_modulestore, get_lms_link_for_item, 
+    compute_unit_state, UnitState, get_course_for_item )
 
 from models.settings.course_grading import CourseGradingModel
 
@@ -72,10 +74,15 @@ def edit_subsection(request, location):
     except ItemNotFoundError:
         return HttpResponseBadRequest()
 
-    lms_link = get_lms_link_for_item(location, course_id=course.location.course_id)
-    preview_link = get_lms_link_for_item(location, course_id=course.location.course_id, preview=True)
+    lms_link = get_lms_link_for_item(
+            location, course_id=course.location.course_id
+    )
+    preview_link = get_lms_link_for_item(
+            location, course_id=course.location.course_id, preview=True
+    )
 
-    # make sure that location references a 'sequential', otherwise return BadRequest
+    # make sure that location references a 'sequential', otherwise return
+    # BadRequest
     if item.location.category != 'sequential':
         return HttpResponseBadRequest()
 
@@ -83,18 +90,23 @@ def edit_subsection(request, location):
 
     # we're for now assuming a single parent
     if len(parent_locs) != 1:
-        logging.error('Multiple (or none) parents have been found for {0}'.format(location))
+        logging.error(
+                'Multiple (or none) parents have been found for %', 
+                location
+        )
 
     # this should blow up if we don't find any parents, which would be erroneous
     parent = modulestore().get_item(parent_locs[0])
 
-    # remove all metadata from the generic dictionary that is presented in a more normalized UI
+    # remove all metadata from the generic dictionary that is presented in a
+    # more normalized UI
 
     policy_metadata = dict(
         (field.name, field.read_from(item))
         for field
         in item.fields
-        if field.name not in ['display_name', 'start', 'due', 'format'] and field.scope == Scope.settings
+        if field.name not in ['display_name', 'start', 'due', 'format']
+            and field.scope == Scope.settings
     )
 
     can_view_live = False
@@ -105,19 +117,22 @@ def edit_subsection(request, location):
             can_view_live = True
             break
 
-    return render_to_response('edit_subsection.html',
-                              {'subsection': item,
-                               'context_course': course,
-                               'new_unit_category': 'vertical',
-                               'lms_link': lms_link,
-                               'preview_link': preview_link,
-                               'course_graders': json.dumps(CourseGradingModel.fetch(course.location).graders),
-                               'parent_location': course.location,
-                               'parent_item': parent,
-                               'policy_metadata': policy_metadata,
-                               'subsection_units': subsection_units,
-                               'can_view_live': can_view_live
-                               })
+    return render_to_response(
+        'edit_subsection.html',
+        {
+           'subsection': item,
+           'context_course': course,
+           'new_unit_category': 'vertical',
+           'lms_link': lms_link,
+           'preview_link': preview_link,
+           'course_graders': json.dumps(CourseGradingModel.fetch(course.location).graders),
+           'parent_location': course.location,
+           'parent_item': parent,
+           'policy_metadata': policy_metadata,
+           'subsection_units': subsection_units,
+           'can_view_live': can_view_live
+        }
+    )
 
 
 @login_required
@@ -125,7 +140,7 @@ def edit_unit(request, location):
     """
     Display an editing page for the specified module.
 
-    Expects a GET request with the parameter 'id'.
+    Expects a GET request with the parameter `id`.
 
     id: A Location URL
     """
@@ -141,7 +156,10 @@ def edit_unit(request, location):
         item = modulestore().get_item(location, depth=1)
     except ItemNotFoundError:
         return HttpResponseBadRequest()
-    lms_link = get_lms_link_for_item(item.location, course_id=course.location.course_id)
+    lms_link = get_lms_link_for_item(
+            item.location,
+            course_id=course.location.course_id
+    )
 
     component_templates = defaultdict(list)
     for category in COMPONENT_TYPES:
@@ -162,17 +180,19 @@ def edit_unit(request, location):
                 template.get('template_id')
             ))
 
-    # Check if there are any advanced modules specified in the course policy. These modules
-    # should be specified as a list of strings, where the strings are the names of the modules
-    # in ADVANCED_COMPONENT_TYPES that should be enabled for the course.
+    # Check if there are any advanced modules specified in the course policy.
+    # These modules should be specified as a list of strings, where the strings
+    # are the names of the modules in ADVANCED_COMPONENT_TYPES that should be
+    # enabled for the course.
     course_advanced_keys = course.advanced_modules
 
     # Set component types according to course policy file
     if isinstance(course_advanced_keys, list):
         for category in course_advanced_keys:
             if category in ADVANCED_COMPONENT_TYPES:
-                # Do I need to allow for boilerplates or just defaults on the class? i.e., can an advanced
-                # have more than one entry in the menu? one for default and others for prefilled boilerplates?
+                # Do I need to allow for boilerplates or just defaults on the
+                # class? i.e., can an advanced have more than one entry in the
+                # menu? one for default and others for prefilled boilerplates?
                 try:
                     component_class = XModuleDescriptor.load_class(category)
 
@@ -183,13 +203,17 @@ def edit_unit(request, location):
                         None  # don't override default data
                         ))
                 except PluginMissingError:
-                    # dhm: I got this once but it can happen any time the course author configures
-                    # an advanced component which does not exist on the server. This code here merely
-                    # prevents any authors from trying to instantiate the non-existent component type
-                    # by not showing it in the menu
+                    # dhm: I got this once but it can happen any time the
+                    # course author configures an advanced component which does
+                    # not exist on the server. This code here merely
+                    # prevents any authors from trying to instantiate the
+                    # non-existent component type by not showing it in the menu
                     pass
     else:
-        log.error("Improper format for course advanced keys! {0}".format(course_advanced_keys))
+        log.error(
+            "Improper format for course advanced keys! %",
+            course_advanced_keys
+        )
 
     components = [
         component.location.url()
@@ -201,16 +225,20 @@ def edit_unit(request, location):
     # this will need to change to check permissions correctly so as
     # to pick the correct parent subsection
 
-    containing_subsection_locs = modulestore().get_parent_locations(location, None)
+    containing_subsection_locs = modulestore().get_parent_locations(
+            location, None
+    )
     containing_subsection = modulestore().get_item(containing_subsection_locs[0])
-
-    containing_section_locs = modulestore().get_parent_locations(containing_subsection.location, None)
+    containing_section_locs = modulestore().get_parent_locations(
+            containing_subsection.location, None
+    )
     containing_section = modulestore().get_item(containing_section_locs[0])
 
     # cdodge hack. We're having trouble previewing drafts via jump_to redirect
     # so let's generate the link url here
 
-    # need to figure out where this item is in the list of children as the preview will need this
+    # need to figure out where this item is in the list of children as the
+    # preview will need this
     index = 1
     for child in containing_subsection.get_children():
         if child.location == item.location:
@@ -219,15 +247,19 @@ def edit_unit(request, location):
 
     preview_lms_base = settings.MITX_FEATURES.get('PREVIEW_LMS_BASE')
 
-    preview_lms_link = '//{preview_lms_base}/courses/{org}/{course}/{course_name}/courseware/{section}/{subsection}/{index}'.format(
-        preview_lms_base=preview_lms_base,
-        lms_base=settings.LMS_BASE,
-        org=course.location.org,
-        course=course.location.course,
-        course_name=course.location.name,
-        section=containing_section.location.name,
-        subsection=containing_subsection.location.name,
-        index=index)
+    preview_lms_link = (
+            '//{preview_lms_base}/courses/{org}/{course}/'
+            '{course_name}/courseware/{section}/{subsection}/{index}'
+        ).format(
+            preview_lms_base=preview_lms_base,
+            lms_base=settings.LMS_BASE,
+            org=course.location.org,
+            course=course.location.course,
+            course_name=course.location.name,
+            section=containing_section.location.name,
+            subsection=containing_subsection.location.name,
+            index=index
+        )
 
     unit_state = compute_unit_state(item)
 
@@ -240,11 +272,13 @@ def edit_unit(request, location):
         'draft_preview_link': preview_lms_link,
         'published_preview_link': lms_link,
         'subsection': containing_subsection,
-        'release_date': get_default_time_display(containing_subsection.lms.start) if containing_subsection.lms.start is not None else None,
+        'release_date': get_default_time_display(containing_subsection.lms.start)
+            if containing_subsection.lms.start is not None else None,
         'section': containing_section,
         'new_unit_category': 'vertical',
         'unit_state': unit_state,
-        'published_date': get_default_time_display(item.cms.published_date) if item.cms.published_date is not None else None
+        'published_date': get_default_time_display(item.cms.published_date)
+            if item.cms.published_date is not None else None
     })
 
 
@@ -253,17 +287,21 @@ def edit_unit(request, location):
 @require_http_methods(("GET", "POST", "PUT"))
 @ensure_csrf_cookie
 def assignment_type_update(request, org, course, category, name):
-    '''
-    CRUD operations on assignment types for sections and subsections and anything else gradable.
-    '''
+    """
+    CRUD operations on assignment types for sections and subsections and
+    anything else gradable.
+    """
     location = Location(['i4x', org, course, category, name])
     if not has_access(request.user, location):
         return HttpResponseForbidden()
 
     if request.method == 'GET':
-        return JsonResponse(CourseGradingModel.get_section_grader_type(location))
+        rsp = CourseGradingModel.get_section_grader_type(location)
     elif request.method in ('POST', 'PUT'):  # post or put, doesn't matter.
-        return JsonResponse(CourseGradingModel.update_section_grader_type(location, request.POST))
+        rsp = CourseGradingModel.update_section_grader_type(
+                    location, request.POST
+        )
+    return JsonResponse(rsp)
 
 
 @login_required
@@ -276,8 +314,8 @@ def create_draft(request):
     if not has_access(request.user, location):
         raise PermissionDenied()
 
-    # This clones the existing item location to a draft location (the draft is implicit,
-    # because modulestore is a Draft modulestore)
+    # This clones the existing item location to a draft location (the draft is
+    # implicit, because modulestore is a Draft modulestore)
     modulestore().convert_to_draft(location)
 
     return HttpResponse()
@@ -286,7 +324,9 @@ def create_draft(request):
 @login_required
 @expect_json
 def publish_draft(request):
-    "Publish a draft"
+    """
+    Publish a draft
+    """
     location = request.POST['id']
 
     # check permissions for this user within this course
@@ -294,7 +334,10 @@ def publish_draft(request):
         raise PermissionDenied()
 
     item = modulestore().get_item(location)
-    _xmodule_recurse(item, lambda i: modulestore().publish(i.location, request.user.id))
+    _xmodule_recurse(
+            item,
+            lambda i: modulestore().publish(i.location, request.user.id)
+    )
 
     return HttpResponse()
 
@@ -328,13 +371,24 @@ def module_info(request, module_location):
         raise PermissionDenied()
 
     rewrite_static_links = request.GET.get('rewrite_url_links', 'True') in ['True', 'true']
-    logging.debug('rewrite_static_links = {0} {1}'.format(request.GET.get('rewrite_url_links', 'False'), rewrite_static_links))
+    logging.debug('rewrite_static_links = {0} {1}'.format(
+        request.GET.get('rewrite_url_links', False),
+        rewrite_static_links)
+    )
 
     # check that logged in user has permissions to this item
     if not has_access(request.user, location):
         raise PermissionDenied()
 
     if request.method == 'GET':
-        return JsonResponse(get_module_info(get_modulestore(location), location, rewrite_static_links=rewrite_static_links))
+        rsp = get_module_info(
+            get_modulestore(location),
+            location,
+            rewrite_static_links=rewrite_static_links
+        )
     elif request.method in ("POST", "PUT"):
-        return JsonResponse(set_module_info(get_modulestore(location), location, request.POST))
+        rsp = set_module_info(
+            get_modulestore(location),
+            location, request.POST
+        )
+    return JsonResponse(rsp)

cms/djangoapps/contentstore/views/course.py

@@ -44,7 +44,7 @@ from .component import (
 
 from django_comment_common.utils import seed_permissions_roles
 
-from student.views import enroll_in_course
+from student.models import CourseEnrollment
 
 from xmodule.html_module import AboutDescriptor
 __all__ = ['course_index', 'create_new_course', 'course_info',
@@ -82,7 +82,9 @@ def course_index(request, org, course, name):
         'context_course': course,
         'lms_link': lms_link,
         'sections': sections,
-        'course_graders': json.dumps(CourseGradingModel.fetch(course.location).graders),
+        'course_graders': json.dumps(
+            CourseGradingModel.fetch(course.location).graders
+        ),
         'parent_location': course.location,
         'new_section_category': 'chapter',
         'new_subsection_category': 'sequential',
@@ -120,24 +122,31 @@ def create_new_course(request):
     except ItemNotFoundError:
         pass
     if existing_course is not None:
-        return JsonResponse(
-            {
-                'ErrMsg': _('There is already a course defined with the same organization, course number, and course run. Please change either organization or course number to be unique.'),
-                'OrgErrMsg': _('Please change either the organization or course number so that it is unique.'),
-                'CourseErrMsg': _('Please change either the organization or course number so that it is unique.'),
-            }
-        )
+        return JsonResponse({
+                'ErrMsg': _('There is already a course defined with the same '
+                    'organization, course number, and course run. Please '
+                    'change either organization or course number to be '
+                    'unique.'),
+                'OrgErrMsg': _('Please change either the organization or '
+                    'course number so that it is unique.'),
+                'CourseErrMsg': _('Please change either the organization or '
+                    'course number so that it is unique.'),
+        })
 
-    course_search_location = ['i4x', dest_location.org, dest_location.course, 'course', None]
+    course_search_location = ['i4x', dest_location.org, dest_location.course,
+            'course', None
+    ]
     courses = modulestore().get_items(course_search_location)
     if len(courses) > 0:
-        return JsonResponse(
-            {
-                'ErrMsg': _('There is already a course defined with the same organization and course number. Please change at least one field to be unique.'),
-                'OrgErrMsg': _('Please change either the organization or course number so that it is unique.'),
-                'CourseErrMsg': _('Please change either the organization or course number so that it is unique.'),
-            }
-        )
+        return JsonResponse({
+                'ErrMsg': _('There is already a course defined with the same '
+                    'organization and course number. Please '
+                    'change at least one field to be unique.'),
+                'OrgErrMsg': _('Please change either the organization or '
+                    'course number so that it is unique.'),
+                'CourseErrMsg': _('Please change either the organization or '
+                    'course number so that it is unique.'),
+        })
 
     # instantiate the CourseDescriptor and then persist it
     # note: no system to pass
@@ -145,11 +154,17 @@ def create_new_course(request):
         metadata = {}
     else:
         metadata = {'display_name': display_name}
-    modulestore('direct').create_and_save_xmodule(dest_location, metadata=metadata)
+    modulestore('direct').create_and_save_xmodule(
+            dest_location,
+            metadata=metadata
+    )
     new_course = modulestore('direct').get_item(dest_location)
 
     # clone a default 'about' overview module as well
-    dest_about_location = dest_location.replace(category='about', name='overview')
+    dest_about_location = dest_location.replace(
+            category='about',
+            name='overview'
+    )
     overview_template = AboutDescriptor.get_template('overview.yaml')
     modulestore('direct').create_and_save_xmodule(
         dest_about_location,
@@ -164,8 +179,9 @@ def create_new_course(request):
     # seed the forums
     seed_permissions_roles(new_course.location.course_id)
 
-    # auto-enroll the course creator in the course so that "View Live" will work.
-    enroll_in_course(request.user, new_course.location.course_id)
+    # auto-enroll the course creator in the course so that "View Live" will
+    # work.
+    CourseEnrollment.enroll(request.user, new_course.location.course_id)
 
     return JsonResponse({'id': new_course.location.url()})
 
@@ -174,7 +190,8 @@ def create_new_course(request):
 @ensure_csrf_cookie
 def course_info(request, org, course, name, provided_id=None):
     """
-    Send models and views as well as html for editing the course info to the client.
+    Send models and views as well as html for editing the course info to the
+    client.
 
     org, course, name: Attributes of the Location for the item to edit
     """
@@ -189,8 +206,7 @@ def course_info(request, org, course, name, provided_id=None):
         'context_course': course_module,
         'url_base': "/" + org + "/" + course + "/",
         'course_updates': json.dumps(get_course_updates(location)),
-        'handouts_location': Location(['i4x', org, course, 'course_info', 'handouts']).url()
-    })
+        'handouts_location': Location(['i4x', org, course, 'course_info', 'handouts']).url() })
 
 
 @expect_json
@@ -202,14 +218,13 @@ def course_info_updates(request, org, course, provided_id=None):
     restful CRUD operations on course_info updates.
 
     org, course: Attributes of the Location for the item to edit
-    provided_id should be none if it's new (create) and a composite of the update db id + index otherwise.
+    provided_id should be none if it's new (create) and a composite of the
+    update db id + index otherwise.
     """
     # ??? No way to check for access permission afaik
     # get current updates
     location = ['i4x', org, course, 'course_info', "updates"]
 
-    # Hmmm, provided_id is coming as empty string on create whereas I believe it used to be None :-(
-    # Possibly due to my removing the seemingly redundant pattern in urls.py
     if provided_id == '':
         provided_id = None
 
@@ -223,21 +238,27 @@ def course_info_updates(request, org, course, provided_id=None):
         try:
             return JsonResponse(delete_course_update(location, request.POST, provided_id))
         except:
-            return HttpResponseBadRequest("Failed to delete",
-                                          content_type="text/plain")
-    elif request.method in ('POST', 'PUT'):  # can be either and sometimes django is rewriting one to the other
+            return HttpResponseBadRequest(
+                "Failed to delete",
+                content_type="text/plain"
+            )
+    # can be either and sometimes django is rewriting one to the other:
+    elif request.method in ('POST', 'PUT'): 
         try:
             return JsonResponse(update_course_updates(location, request.POST, provided_id))
         except:
-            return HttpResponseBadRequest("Failed to save",
-                                          content_type="text/plain")
+            return HttpResponseBadRequest(
+                "Failed to save",
+                content_type="text/plain"
+            )
 
 
 @login_required
 @ensure_csrf_cookie
 def get_course_settings(request, org, course, name):
     """
-    Send models and views as well as html for editing the course settings to the client.
+    Send models and views as well as html for editing the course settings to
+    the client.
 
     org, course, name: Attributes of the Location for the item to edit
     """
@@ -253,7 +274,9 @@ def get_course_settings(request, org, course, name):
                                        "course": course,
                                        "name": name,
                                        "section": "details"}),
-        'about_page_editable': not settings.MITX_FEATURES.get('ENABLE_MKTG_SITE', False)
+        'about_page_editable': not settings.MITX_FEATURES.get(
+            'ENABLE_MKTG_SITE', False
+        )
     })
 
 
@@ -261,7 +284,8 @@ def get_course_settings(request, org, course, name):
 @ensure_csrf_cookie
 def course_config_graders_page(request, org, course, name):
     """
-    Send models and views as well as html for editing the course settings to the client.
+    Send models and views as well as html for editing the course settings to
+    the client.
 
     org, course, name: Attributes of the Location for the item to edit
     """
@@ -281,7 +305,8 @@ def course_config_graders_page(request, org, course, name):
 @ensure_csrf_cookie
 def course_config_advanced_page(request, org, course, name):
     """
-    Send models and views as well as html for editing the advanced course settings to the client.
+    Send models and views as well as html for editing the advanced course
+    settings to the client.
 
     org, course, name: Attributes of the Location for the item to edit
     """
@@ -301,8 +326,9 @@ def course_config_advanced_page(request, org, course, name):
 @ensure_csrf_cookie
 def course_settings_updates(request, org, course, name, section):
     """
-    restful CRUD operations on course settings. This differs from get_course_settings by communicating purely
-    through json (not rendering any html) and handles section level operations rather than whole page.
+    Restful CRUD operations on course settings. This differs from
+    get_course_settings by communicating purely through json (not rendering any
+    html) and handles section level operations rather than whole page.
 
     org, course: Attributes of the Location for the item to edit
     section: one of details, faculty, grading, problems, discussions
@@ -318,9 +344,15 @@ def course_settings_updates(request, org, course, name, section):
 
     if request.method == 'GET':
         # Cannot just do a get w/o knowing the course name :-(
-        return JsonResponse(manager.fetch(Location(['i4x', org, course, 'course', name])), encoder=CourseSettingsEncoder)
+        return JsonResponse(
+            manager.fetch(Location(['i4x', org, course, 'course', name])),
+            encoder=CourseSettingsEncoder
+        )
     elif request.method in ('POST', 'PUT'):  # post or put, doesn't matter.
-        return JsonResponse(manager.update_from_json(request.POST), encoder=CourseSettingsEncoder)
+        return JsonResponse(
+            manager.update_from_json(request.POST),
+            encoder=CourseSettingsEncoder
+        )
 
 
 @expect_json
@@ -329,8 +361,9 @@ def course_settings_updates(request, org, course, name, section):
 @ensure_csrf_cookie
 def course_grader_updates(request, org, course, name, grader_index=None):
     """
-    restful CRUD operations on course_info updates. This differs from get_course_settings by communicating purely
-    through json (not rendering any html) and handles section level operations rather than whole page.
+    Restful CRUD operations on course_info updates. This differs from
+    get_course_settings by communicating purely through json (not rendering any
+    html) and handles section level operations rather than whole page.
 
     org, course: Attributes of the Location for the item to edit
     """
@@ -339,13 +372,18 @@ def course_grader_updates(request, org, course, name, grader_index=None):
 
     if request.method == 'GET':
         # Cannot just do a get w/o knowing the course name :-(
-        return JsonResponse(CourseGradingModel.fetch_grader(Location(location), grader_index))
+        return JsonResponse(CourseGradingModel.fetch_grader(
+                Location(location), grader_index
+        ))
     elif request.method == "DELETE":
         # ??? Should this return anything? Perhaps success fail?
         CourseGradingModel.delete_grader(Location(location), grader_index)
         return JsonResponse()
     else:  # post or put, doesn't matter.
-        return JsonResponse(CourseGradingModel.update_grader_from_json(Location(location), request.POST))
+        return JsonResponse(CourseGradingModel.update_grader_from_json(
+                Location(location),
+                request.POST
+        ))
 
 
 # # NB: expect_json failed on ["key", "key2"] and json payload
@@ -354,8 +392,9 @@ def course_grader_updates(request, org, course, name, grader_index=None):
 @ensure_csrf_cookie
 def course_advanced_updates(request, org, course, name):
     """
-    restful CRUD operations on metadata. The payload is a json rep of the metadata dicts. For delete, otoh,
-    the payload is either a key or a list of keys to delete.
+    Restful CRUD operations on metadata. The payload is a json rep of the
+    metadata dicts. For delete, otoh, the payload is either a key or a list of
+    keys to delete.
 
     org, course: Attributes of the Location for the item to edit
     """
@@ -364,20 +403,26 @@ def course_advanced_updates(request, org, course, name):
     if request.method == 'GET':
         return JsonResponse(CourseMetadata.fetch(location))
     elif request.method == 'DELETE':
-        return JsonResponse(CourseMetadata.delete_key(location, json.loads(request.body)))
+        return JsonResponse(CourseMetadata.delete_key(
+                location,
+                json.loads(request.body)
+        ))
     else:
         # NOTE: request.POST is messed up because expect_json
-        # cloned_request.POST.copy() is creating a defective entry w/ the whole payload as the key
+        # cloned_request.POST.copy() is creating a defective entry w/ the whole
+        # payload as the key
         request_body = json.loads(request.body)
         # Whether or not to filter the tabs key out of the settings metadata
         filter_tabs = True
 
-        # Check to see if the user instantiated any advanced components. This is a hack
-        # that does the following :
-        #   1) adds/removes the open ended panel tab to a course automatically if the user
-        #   has indicated that they want to edit the combinedopendended or peergrading module
-        #   2) adds/removes the notes panel tab to a course automatically if the user has
-        #   indicated that they want the notes module enabled in their course
+        # Check to see if the user instantiated any advanced components. This
+        # is a hack that does the following :
+        #   1) adds/removes the open ended panel tab to a course automatically
+        #   if the user has indicated that they want to edit the
+        #   combinedopendended or peergrading module
+        #   2) adds/removes the notes panel tab to a course automatically if
+        #   the user has indicated that they want the notes module enabled in
+        #   their course
         # TODO refactor the above into distinct advanced policy settings
         if ADVANCED_COMPONENT_POLICY_KEY in request_body:
             # Get the course so that we can scrape current tabs
@@ -389,19 +434,25 @@ def course_advanced_updates(request, org, course, name):
                 'notes': NOTE_COMPONENT_TYPES,
             }
 
-            # Check to see if the user instantiated any notes or open ended components
+            # Check to see if the user instantiated any notes or open ended
+            # components
             for tab_type in tab_component_map.keys():
                 component_types = tab_component_map.get(tab_type)
                 found_ac_type = False
                 for ac_type in component_types:
                     if ac_type in request_body[ADVANCED_COMPONENT_POLICY_KEY]:
                         # Add tab to the course if needed
-                        changed, new_tabs = add_extra_panel_tab(tab_type, course_module)
-                        # If a tab has been added to the course, then send the metadata along to CourseMetadata.update_from_json
+                        changed, new_tabs = add_extra_panel_tab(
+                            tab_type,
+                            course_module
+                        )
+                        # If a tab has been added to the course, then send the
+                        # metadata along to CourseMetadata.update_from_json
                         if changed:
                             course_module.tabs = new_tabs
                             request_body.update({'tabs': new_tabs})
-                            # Indicate that tabs should not be filtered out of the metadata
+                            # Indicate that tabs should not be filtered out of
+                            # the metadata
                             filter_tabs = False
                         # Set this flag to avoid the tab removal code below.
                         found_ac_type = True
@@ -410,18 +461,26 @@ def course_advanced_updates(request, org, course, name):
                 # we may need to remove the tab from the course.
                 if not found_ac_type:
                     # Remove tab from the course if needed
-                    changed, new_tabs = remove_extra_panel_tab(tab_type, course_module)
+                    changed, new_tabs = remove_extra_panel_tab(
+                        tab_type, course_module
+                    )
                     if changed:
                         course_module.tabs = new_tabs
                         request_body.update({'tabs': new_tabs})
-                        # Indicate that tabs should *not* be filtered out of the metadata
+                        # Indicate that tabs should *not* be filtered out of
+                        # the metadata
                         filter_tabs = False
         try:
-            return JsonResponse(CourseMetadata.update_from_json(location,
-                                                                request_body,
-                                                                filter_tabs=filter_tabs))
+            return JsonResponse(CourseMetadata.update_from_json(
+                    location,
+                    request_body,
+                    filter_tabs=filter_tabs
+            ))
         except (TypeError, ValueError) as err:
-            return HttpResponseBadRequest("Incorrect setting format. " + str(err), content_type="text/plain")
+            return HttpResponseBadRequest(
+                "Incorrect setting format. " + str(err),
+                content_type="text/plain"
+            )
 
 
 class TextbookValidationError(Exception):
@@ -498,7 +557,8 @@ def textbook_index(request, org, course, name):
     if request.is_ajax():
         if request.method == 'GET':
             return JsonResponse(course_module.pdf_textbooks)
-        elif request.method in ('POST', 'PUT'):  # can be either and sometimes django is rewriting one to the other
+        # can be either and sometimes django is rewriting one to the other:
+        elif request.method in ('POST', 'PUT'):
             try:
                 textbooks = validate_textbooks_json(request.body)
             except TextbookValidationError as err:
@@ -517,7 +577,10 @@ def textbook_index(request, org, course, name):
             # Save the data that we've just changed to the underlying
             # MongoKeyValueStore before we update the mongo datastore.
             course_module.save()
-            store.update_metadata(course_module.location, own_metadata(course_module))
+            store.update_metadata(
+                    course_module.location,
+                    own_metadata(course_module)
+            )
             return JsonResponse(course_module.pdf_textbooks)
     else:
         upload_asset_url = reverse('upload_asset', kwargs={
@@ -599,7 +662,8 @@ def textbook_by_id(request, org, course, name, tid):
         if not textbook:
             return JsonResponse(status=404)
         return JsonResponse(textbook)
-    elif request.method in ('POST', 'PUT'):  # can be either and sometimes django is rewriting one to the other
+    elif request.method in ('POST', 'PUT'):  # can be either and sometimes
+                                        # django is rewriting one to the other
         try:
             new_textbook = validate_textbook_json(request.body)
         except TextbookValidationError as err:
@@ -616,7 +680,10 @@ def textbook_by_id(request, org, course, name, tid):
         # Save the data that we've just changed to the underlying
         # MongoKeyValueStore before we update the mongo datastore.
         course_module.save()
-        store.update_metadata(course_module.location, own_metadata(course_module))
+        store.update_metadata(
+            course_module.location,
+            own_metadata(course_module)
+        )
         return JsonResponse(new_textbook, status=201)
     elif request.method == 'DELETE':
         if not textbook:
@@ -626,5 +693,8 @@ def textbook_by_id(request, org, course, name, tid):
         new_textbooks.extend(course_module.pdf_textbooks[i + 1:])
         course_module.pdf_textbooks = new_textbooks
         course_module.save()
-        store.update_metadata(course_module.location, own_metadata(course_module))
+        store.update_metadata(
+            course_module.location,
+            own_metadata(course_module)
+        )
         return JsonResponse()

cms/djangoapps/contentstore/views/item.py

@@ -58,14 +58,12 @@ def save_item(request):
         # 'apply' the submitted metadata, so we don't end up deleting system metadata
         existing_item = modulestore().get_item(item_location)
         for metadata_key in request.POST.get('nullout', []):
-            # [dhm] see comment on _get_xblock_field
             _get_xblock_field(existing_item, metadata_key).write_to(existing_item, None)
 
         # update existing metadata with submitted metadata (which can be partial)
         # IMPORTANT NOTE: if the client passed 'null' (None) for a piece of metadata that means 'remove it'. If
         # the intent is to make it None, use the nullout field
         for metadata_key, value in request.POST.get('metadata', {}).items():
-            # [dhm] see comment on _get_xblock_field
             field = _get_xblock_field(existing_item, metadata_key)
 
             if value is None:
@@ -82,32 +80,15 @@ def save_item(request):
     return JsonResponse()
 
 
-# [DHM] A hack until we implement a permanent soln. Proposed perm solution is to make namespace fields also top level
-# fields in xblocks rather than requiring dereference through namespace but we'll need to consider whether there are
-# plausible use cases for distinct fields w/ same name in different namespaces on the same blocks.
-# The idea is that consumers of the xblock, and particularly the web client, shouldn't know about our internal
-# representation (namespaces as means of decorating all modules).
-# Given top-level access, the calls can simply be setattr(existing_item, field, value) ...
-# Really, this method should be elsewhere (e.g., xblock). We also need methods for has_value (v is_default)...
 def _get_xblock_field(xblock, field_name):
     """
     A temporary function to get the xblock field either from the xblock or one of its namespaces by name.
     :param xblock:
     :param field_name:
     """
-    def find_field(fields):
-        for field in fields:
-            if field.name == field_name:
-                return field
-
-    found = find_field(xblock.fields)
-    if found:
-        return found
-    for namespace in xblock.namespaces:
-        found = find_field(getattr(xblock, namespace).fields)
-        if found:
-            return found
-
+    for field in xblock.iterfields():
+        if field.name == field_name:
+            return field
 
 @login_required
 @expect_json

cms/djangoapps/contentstore/views/preview.py

@@ -116,7 +116,7 @@ def preview_module_system(request, preview_id, descriptor):
         get_module=partial(load_preview_module, request, preview_id),
         render_template=render_from_lms,
         debug=True,
-        replace_urls=partial(static_replace.replace_static_urls, data_directory=None, course_namespace=descriptor.location),
+        replace_urls=partial(static_replace.replace_static_urls, data_directory=None, course_id=course_id),
         user=request.user,
         xblock_model_data=preview_model_data,
         can_execute_unsafe_code=(lambda: can_execute_unsafe_code(course_id)),
@@ -155,10 +155,12 @@ def load_preview_module(request, preview_id, descriptor):
             "xmodule_display.html",
         )
 
+    # we pass a partially bogus course_id as we don't have the RUN information passed yet
+    # through the CMS. Also the contentstore is also not RUN-aware at this point in time.
     module.get_html = replace_static_urls(
         module.get_html,
         getattr(module, 'data_dir', module.location.course),
-        course_namespace=Location([module.location.tag, module.location.org, module.location.course, None, None])
+        course_id=module.location.org + '/' + module.location.course + '/BOGUS_RUN_REPLACE_WHEN_AVAILABLE'
     )
 
     module.get_html = save_module(

cms/djangoapps/contentstore/views/user.py

@@ -24,7 +24,7 @@ from course_creators.views import (
 
 from .access import has_access
 
-from student.views import enroll_in_course
+from student.models import CourseEnrollment
 
 
 @login_required
@@ -54,8 +54,7 @@ def index(request):
                 'name': course.location.name,
             }),
             get_lms_link_for_item(
-                course.location,
-                course_id=course.location.course_id,
+                course.location
             ),
             course.display_org_with_default,
             course.display_number_with_default,
@@ -208,7 +207,7 @@ def course_team_user(request, org, course, name, email):
         user.groups.add(groups["instructor"])
         user.save()
         # auto-enroll the course creator in the course so that "View Live" will work.
-        enroll_in_course(user, location.course_id)
+        CourseEnrollment.enroll(user, location.course_id)
     elif role == "staff":
         # if we're trying to downgrade a user from "instructor" to "staff",
         # make sure we have at least one other instructor in the course team.
@@ -223,7 +222,7 @@ def course_team_user(request, org, course, name, email):
         user.groups.add(groups["staff"])
         user.save()
         # auto-enroll the course creator in the course so that "View Live" will work.
-        enroll_in_course(user, location.course_id)
+        CourseEnrollment.enroll(user, location.course_id)
 
     return JsonResponse()
 

cms/envs/acceptance.py

@@ -75,6 +75,12 @@ DATABASES = {
 # Use the auto_auth workflow for creating users and logging them in
 MITX_FEATURES['AUTOMATIC_AUTH_FOR_TESTING'] = True
 
+
+# HACK
+# Setting this flag to false causes imports to not load correctly in the lettuce python files
+# We do not yet understand why this occurs. Setting this to true is a stopgap measure
+USE_I18N = True
+
 # Include the lettuce app for acceptance testing, including the 'harvest' django-admin command
 INSTALLED_APPS += ('lettuce.django',)
 LETTUCE_APPS = ('contentstore',)

cms/envs/common.py

@@ -25,7 +25,7 @@ Longer TODO:
 
 import sys
 import lms.envs.common
-from lms.envs.common import USE_TZ, TECH_SUPPORT_EMAIL
+from lms.envs.common import USE_TZ, TECH_SUPPORT_EMAIL, PLATFORM_NAME, BUGS_EMAIL
 from path import path
 
 ############################ FEATURE CONFIGURATION #############################
@@ -201,7 +201,7 @@ STATICFILES_DIRS = [
 TIME_ZONE = 'America/New_York'  # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
 LANGUAGE_CODE = 'en'  # http://www.i18nguy.com/unicode/language-identifiers.html
 
-USE_I18N = True
+USE_I18N = False
 USE_L10N = True
 
 # Localization strings (e.g. django.po) are under this directory
@@ -332,6 +332,9 @@ INSTALLED_APPS = (
     # Monitor the status of services
     'service_status',
 
+    # Testing
+    'django_nose',
+
     # For CMS
     'contentstore',
     'auth',
@@ -339,7 +342,7 @@ INSTALLED_APPS = (
     'student',  # misleading name due to sharing with lms
     'course_groups',  # not used in cms (yet), but tests run
 
-    # tracking
+    # Tracking
     'track',
 
     # For asset pipelining

cms/envs/test.py

@@ -18,7 +18,6 @@ from path import path
 from warnings import filterwarnings
 
 # Nose Test Runner
-INSTALLED_APPS += ('django_nose',)
 TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
 
 TEST_ROOT = path('test_root')

cms/one_time_startup.py

@@ -9,8 +9,11 @@ from django.core.cache import get_cache
 CACHE = get_cache('mongo_metadata_inheritance')
 for store_name in settings.MODULESTORE:
     store = modulestore(store_name)
-    store.metadata_inheritance_cache_subsystem = CACHE
-    store.request_cache = RequestCache.get_request_cache()
+
+    store.set_modulestore_configuration({
+        'metadata_inheritance_cache_subsystem': CACHE,
+        'request_cache': RequestCache.get_request_cache()
+    })
 
     modulestore_update_signal = Signal(providing_args=['modulestore', 'course_id', 'location'])
     store.modulestore_update_signal = modulestore_update_signal

cms/static/js/base.js

@@ -605,81 +605,118 @@ function cancelNewSection(e) {
 function addNewCourse(e) {
     e.preventDefault();
     $('.new-course-button').addClass('is-disabled');
+    $('.new-course-save').addClass('is-disabled');
     var $newCourse = $('.wrapper-create-course').addClass('is-shown');
     var $cancelButton = $newCourse.find('.new-course-cancel');
-    $newCourse.find('.new-course-name').focus().select();
-    $newCourse.find('form').bind('submit', saveNewCourse);
+    var $courseName = $('.new-course-name');
+    $courseName.focus().select();
+    $('.new-course-save').on('click', saveNewCourse);
     $cancelButton.bind('click', cancelNewCourse);
     $body.bind('keyup', {
         $cancelButton: $cancelButton
     }, checkForCancel);
+
+    // Check that a course (org, number, run) doesn't use any special characters
+    var validateCourseItemEncoding = function(item) {
+        var required = validateRequiredField(item);
+        if(required) {
+            return required;
+        }
+        if(item !== encodeURIComponent(item)) {
+            return gettext('Please do not use any spaces or special characters in this field.');
+        }
+        return '';
+    }
+
+    // Ensure that all items are less than 80 characters.
+    var validateTotalCourseItemsLength = function() {
+        var totalLength = _.reduce(
+            ['.new-course-name', '.new-course-org', '.new-course-number', '.new-course-run'],
+            function(sum, ele) {
+                return sum + $(ele).val().length;
+        }, 0
+        );
+        if(totalLength > 80) {
+            $('.wrap-error').addClass('is-shown');
+            $('#course_creation_error').html('<p>' + gettext('Course fields must have a combined length of no more than 80 characters.') + '</p>');
+            $('.new-course-save').addClass('is-disabled');
+        }
+        else {
+            $('.wrap-error').removeClass('is-shown');
+        }
+    }
+
+    // Handle validation asynchronously
+    _.each(
+        ['.new-course-org', '.new-course-number', '.new-course-run'],
+        function(ele) {
+            var $ele = $(ele);
+            $ele.on('keyup', function(event) {
+                // Don't bother showing "required field" error when
+                // the user tabs into a new field; this is distracting
+                // and unnecessary
+                if(event.keyCode === 9) {
+                    return;
+                }
+                var error = validateCourseItemEncoding($ele.val());
+                setNewCourseFieldInErr($ele.parent('li'), error);
+                validateTotalCourseItemsLength();
+            });
+        }
+    );
+    var $name = $('.new-course-name');
+    $name.on('keyup', function() {
+        var error = validateRequiredField($name.val());
+        setNewCourseFieldInErr($name.parent('li'), error);
+        validateTotalCourseItemsLength();
+    });
 }
 
+function validateRequiredField(msg) {
+    return msg.length === 0 ? gettext('Required field.') : '';
+}
+
+function setNewCourseFieldInErr(el, msg) {
+    if(msg) {
+        el.addClass('error');
+        el.children('span.tip-error').addClass('is-showing').removeClass('is-hiding').text(msg);
+        $('.new-course-save').addClass('is-disabled');
+    }
+    else {
+        el.removeClass('error');
+        el.children('span.tip-error').addClass('is-hiding').removeClass('is-showing');
+        // One "error" div is always present, but hidden or shown
+        if($('.error').length === 1) {
+            $('.new-course-save').removeClass('is-disabled');
+        }
+    }
+};
+
 function saveNewCourse(e) {
     e.preventDefault();
 
+    // One final check for empty values
+    var errors = _.reduce(
+        ['.new-course-name', '.new-course-org', '.new-course-number', '.new-course-run'],
+        function(acc, ele) {
+            var $ele = $(ele);
+            var error = validateRequiredField($ele.val());
+            setNewCourseFieldInErr($ele.parent('li'), error);
+            return error ? true : acc;
+        },
+        false
+    );
+
+    if(errors) {
+        return;
+    }
+
     var $newCourseForm = $(this).closest('#create-course-form');
     var display_name = $newCourseForm.find('.new-course-name').val();
     var org = $newCourseForm.find('.new-course-org').val();
     var number = $newCourseForm.find('.new-course-number').val();
     var run = $newCourseForm.find('.new-course-run').val();
 
-    var required_field_text = gettext('Required field');
-
-    var display_name_errMsg = (display_name === '') ? required_field_text : null;
-    var org_errMsg = (org === '') ? required_field_text : null;
-    var number_errMsg = (number === '') ? required_field_text : null;
-    var run_errMsg = (run === '') ? required_field_text : null;
-
-    var bInErr = (display_name_errMsg || org_errMsg || number_errMsg || run_errMsg);
-
-    // check for suitable encoding
-    if (!bInErr) {
-        var encoding_errMsg = gettext('Please do not use any spaces or special characters in this field.');
-
-        if (encodeURIComponent(org) != org)
-            org_errMsg = encoding_errMsg;
-        if (encodeURIComponent(number) != number)
-            number_errMsg = encoding_errMsg;
-        if (encodeURIComponent(run) != run)
-            run_errMsg = encoding_errMsg;
-
-        bInErr = (org_errMsg || number_errMsg || run_errMsg);
-    }
-
-    var header_err_msg = (bInErr) ? gettext('Please correct the fields below.') : null;
-
-    var setNewCourseErrMsgs = function(header_err_msg, display_name_errMsg, org_errMsg, number_errMsg, run_errMsg) {
-        if (header_err_msg) {
-            $('.wrapper-create-course').addClass('has-errors');
-            $('.wrap-error').addClass('is-shown');
-            $('#course_creation_error').html('<p>' + header_err_msg + '</p>');
-        } else {
-            $('.wrap-error').removeClass('is-shown');
-            $('#course_creation_error').html('');
-        }
-
-        var setNewCourseFieldInErr = function(el, msg) {
-            el.children('.tip-error').remove();
-            if (msg !== null && msg !== '') {
-                el.addClass('error');
-                el.append('<span class="tip tip-error">' + msg + '</span>');
-            } else {
-                el.removeClass('error');
-            }
-        };
-
-        setNewCourseFieldInErr($('#field-course-name'), display_name_errMsg);
-        setNewCourseFieldInErr($('#field-organization'), org_errMsg);
-        setNewCourseFieldInErr($('#field-course-number'), number_errMsg);
-        setNewCourseFieldInErr($('#field-course-run'), run_errMsg);
-    };
-
-    setNewCourseErrMsgs(header_err_msg, display_name_errMsg, org_errMsg, number_errMsg, run_errMsg);
-
-    if (bInErr)
-        return;
-
     analytics.track('Created a Course', {
         'org': org,
         'number': number,
@@ -697,9 +734,9 @@ function saveNewCourse(e) {
             if (data.id !== undefined) {
                 window.location = '/' + data.id.replace(/.*:\/\//, '');
             } else if (data.ErrMsg !== undefined) {
-                var orgErrMsg = (data.OrgErrMsg !== undefined) ? data.OrgErrMsg : null;
-                var courseErrMsg = (data.CourseErrMsg !== undefined) ? data.CourseErrMsg : null;
-                setNewCourseErrMsgs(data.ErrMsg, null, orgErrMsg, courseErrMsg, null);
+                $('.wrap-error').addClass('is-shown');
+                $('#course_creation_error').html('<p>' + data.ErrMsg + '</p>');
+                $('.new-course-save').addClass('is-disabled');
             }
         }
     );
@@ -709,6 +746,16 @@ function cancelNewCourse(e) {
     e.preventDefault();
     $('.new-course-button').removeClass('is-disabled');
     $('.wrapper-create-course').removeClass('is-shown');
+    // Clear out existing fields and errors
+    _.each(
+        ['.new-course-name', '.new-course-org', '.new-course-number', '.new-course-run'],
+        function(field) {
+            $(field).val('');
+        }
+    );
+    $('#course_creation_error').html('');
+    $('.wrap-error').removeClass('is-shown');
+    $('.new-course-save').off('click');
 }
 
 function addNewSubsection(e) {

cms/static/js/views/settings/settings_grading_view.js

@@ -8,7 +8,7 @@ CMS.Views.Settings.Grading = CMS.Views.ValidatingView.extend({
         // Leaving change in as fallback for older browsers
         "change input" : "updateModel",
         "change textarea" : "updateModel",
-        "change span[contenteditable=true]" : "updateDesignation",
+        "input span[contenteditable]" : "updateDesignation",
         "click .settings-extra header" : "showSettingsExtras",
         "click .new-grade-button" : "addNewGrade",
         "click .remove-button" : "removeGrade",

cms/static/sass/elements/_forms.scss

@@ -225,8 +225,15 @@ form[class^="create-"] {
           color: $red;
         }
 
+        .is-showing {
+            @extend .anim-fadeIn;
+        }
+
+        .is-hiding {
+            @extend .anim-fadeOut;
+        }
+
         .tip-error {
-          @extend .anim-fadeIn;
           display: block;
           color: $red;
         }

cms/templates/index.html

@@ -99,23 +99,27 @@
                   <label for="new-course-name">${_("Course Name")}</label>
                   <input class="new-course-name" id="new-course-name" type="text" name="new-course-name" aria-required="true" placeholder="${_('e.g. Introduction to Computer Science')}" />
                   <span class="tip tip-stacked">${_("The public display name for your course.")}</span>
+                  <span class="tip tip-error is-hiding"></span>
                 </li>
                 <li class="field field-inline text required" id="field-organization">
                   <label for="new-course-org">${_("Organization")}</label>
                   <input class="new-course-org" id="new-course-org" type="text" name="new-course-org" aria-required="true" placeholder="${_('e.g. MITX or IMF')}" />
                   <span class="tip tip-stacked">${_("The name of the organization sponsoring the course")}  - <strong>${_("Note: No spaces or special characters are allowed. This cannot be changed.")}</strong></span>
+                  <span class="tip tip-error is-hiding"></span>
                 </li>
 
                 <li class="field field-inline text required" id="field-course-number">
                   <label for="new-course-number">${_("Course Number")}</label>
                   <input class="new-course-number" id="new-course-number" type="text" name="new-course-number" aria-required="true" placeholder="${_('e.g. CS101')}" />
                   <span class="tip tip-stacked">${_("The unique number that identifies your course within your organization")} - <strong>${_("Note: No spaces or special characters are allowed. This cannot be changed.")}</strong></span>
+                  <span class="tip tip-error is-hiding"></span>
                 </li>
 
                 <li class="field field-inline text required" id="field-course-run">
                   <label for="new-course-run">${_("Course Run")}</label>
                   <input class="new-course-run" id="new-course-run" type="text" name="new-course-run" aria-required="true"placeholder="${_('e.g. 2013_Spring')}" />
                   <span class="tip tip-stacked">${_("The term in which your course will run")} - <strong>${_("Note: No spaces or special characters are allowed. This cannot be changed.")}</strong></span>
+                  <span class="tip tip-error is-hiding"></span>
                 </li>
               </ol>
 
@@ -123,7 +127,7 @@
           </div>
 
           <div class="actions">
-            <input type="submit" value="${_('Save')}" class="action action-primary new-course-save" />
+            <input type="submit" value="${_('Create')}" class="action action-primary new-course-save" />
             <input type="button" value="${_('Cancel')}" class="action action-secondary action-cancel new-course-cancel" />
           </div>
         </form>

cms/templates/settings_discussions_faculty.html

@@ -37,7 +37,7 @@ from contentstore import utils
             <section class="settings-faculty-members">
               <header>
                 <h3>${_("Faculty Members")}</h3>
-                <span class="detail">${_("Individuals instructing and help with this course")}</span>
+                <span class="detail">${_("Individuals instructing and helping with this course")}</span>
               </header>
 
               <div class="row">

common/djangoapps/django_comment_common/models.py

@@ -19,6 +19,20 @@ FORUM_ROLE_STUDENT = 'Student'
 
 @receiver(post_save, sender=CourseEnrollment)
 def assign_default_role(sender, instance, **kwargs):
+    # The code below would remove all forum Roles from a user when they unenroll
+    # from a course. Concerns were raised that it should apply only to students,
+    # or that even the history of student roles is important for research
+    # purposes. Since this was new functionality being added in this release,
+    # I'm just going to comment it out for now and let the forums team deal with
+    # implementing the right behavior.
+    #
+    # # We've unenrolled the student, so remove all roles for this course
+    # if not instance.is_active:
+    #     course_roles = list(Role.objects.filter(course_id=instance.course_id))
+    #     instance.user.roles.remove(*course_roles)
+    #     return
+
+    # We've enrolled the student, so make sure they have a default role
     if instance.user.is_staff:
         role = Role.objects.get_or_create(course_id=instance.course_id, name="Moderator")[0]
     else:

common/djangoapps/django_comment_common/tests.py

@@ -0,0 +1,58 @@
+from django.test import TestCase
+
+from django_comment_common.models import Role
+from student.models import CourseEnrollment, User
+
+class RoleAssignmentTest(TestCase):
+    """
+    Basic checks to make sure our Roles get assigned and unassigned as students
+    are enrolled and unenrolled from a course.
+    """
+
+    def setUp(self):
+        self.staff_user = User.objects.create_user(
+            "patty",
+            "patty@fake.edx.org",
+        )
+        self.staff_user.is_staff = True
+
+        self.student_user = User.objects.create_user(
+            "hacky",
+            "hacky@fake.edx.org"
+        )
+        self.course_id = "edX/Fake101/2012"
+        CourseEnrollment.enroll(self.staff_user, self.course_id)
+        CourseEnrollment.enroll(self.student_user, self.course_id)
+
+    def test_enrollment_auto_role_creation(self):
+        moderator_role = Role.objects.get(
+            course_id=self.course_id,
+            name="Moderator"
+        )
+        student_role = Role.objects.get(
+            course_id=self.course_id,
+            name="Student"
+        )
+        self.assertIn(moderator_role, self.staff_user.roles.all())
+
+        self.assertIn(student_role, self.student_user.roles.all())
+        self.assertNotIn(moderator_role, self.student_user.roles.all())
+
+    # The following was written on the assumption that unenrolling from a course
+    # should remove all forum Roles for that student for that course. This is
+    # not necessarily the case -- please see comments at the top of 
+    # django_comment_client.models.assign_default_role(). Leaving it for the
+    # forums team to sort out.
+    #
+    # def test_unenrollment_auto_role_removal(self):
+    #     another_student = User.objects.create_user("sol", "sol@fake.edx.org")
+    #     CourseEnrollment.enroll(another_student, self.course_id)
+    #
+    #     CourseEnrollment.unenroll(self.student_user, self.course_id)
+    #     # Make sure we didn't delete the actual Role
+    #     student_role = Role.objects.get(
+    #         course_id=self.course_id,
+    #         name="Student"
+    #     )
+    #     self.assertNotIn(student_role, self.student_user.roles.all())
+    #     self.assertIn(student_role, another_student.roles.all())

common/djangoapps/external_auth/tests/test_shib.py

@@ -431,12 +431,12 @@ class ShibSPTest(ModuleStoreTestCase):
                 # If course is not limited or student has correct shib extauth then enrollment should be allowed
                 if course is open_enroll_course or student is shib_student:
                     self.assertEqual(response.status_code, 200)
-                    self.assertEqual(CourseEnrollment.objects.filter(user=student, course_id=course.id).count(), 1)
+                    self.assertTrue(CourseEnrollment.is_enrolled(student, course.id))
                     # Clean up
-                    CourseEnrollment.objects.filter(user=student, course_id=course.id).delete()
+                    CourseEnrollment.unenroll(student, course.id)
                 else:
                     self.assertEqual(response.status_code, 400)
-                    self.assertEqual(CourseEnrollment.objects.filter(user=student, course_id=course.id).count(), 0)
+                    self.assertFalse(CourseEnrollment.is_enrolled(student, course.id))
 
     @unittest.skipUnless(settings.MITX_FEATURES.get('AUTH_USE_SHIB'), True)
     def test_shib_login_enrollment(self):
@@ -462,7 +462,7 @@ class ShibSPTest(ModuleStoreTestCase):
 
         # use django test client for sessions and url processing
         # no enrollment before trying
-        self.assertEqual(CourseEnrollment.objects.filter(user=student, course_id=course.id).count(), 0)
+        self.assertFalse(CourseEnrollment.is_enrolled(student, course.id))
         self.client.logout()
         request_kwargs = {'path': '/shib-login/',
                           'data': {'enrollment_action': 'enroll', 'course_id': course.id},
@@ -474,4 +474,4 @@ class ShibSPTest(ModuleStoreTestCase):
         self.assertEqual(response.status_code, 302)
         self.assertEqual(response['location'], 'http://testserver/')
         # now there is enrollment
-        self.assertEqual(CourseEnrollment.objects.filter(user=student, course_id=course.id).count(), 1)
+        self.assertTrue(CourseEnrollment.is_enrolled(student, course.id))

common/djangoapps/static_replace/__init__.py

@@ -6,7 +6,7 @@ from staticfiles import finders
 from django.conf import settings
 
 from xmodule.modulestore.django import modulestore
-from xmodule.modulestore.xml import XMLModuleStore
+from xmodule.modulestore import XML_MODULESTORE_TYPE
 from xmodule.contentstore.content import StaticContent
 
 log = logging.getLogger(__name__)
@@ -90,7 +90,7 @@ def replace_course_urls(text, course_id):
     return re.sub(_url_replace_regex('/course/'), replace_course_url, text)
 
 
-def replace_static_urls(text, data_directory, course_namespace=None, static_asset_path=''):
+def replace_static_urls(text, data_directory, course_id=None, static_asset_path=''):
     """
     Replace /static/$stuff urls either with their correct url as generated by collectstatic,
     (/static/$md5_hashed_stuff) or by the course-specific content static url
@@ -99,7 +99,7 @@ def replace_static_urls(text, data_directory, course_namespace=None, static_asse
 
     text: The source text to do the substitution in
     data_directory: The directory in which course data is stored
-    course_namespace: The course identifier used to distinguish static content for this course in studio
+    course_id: The course identifier used to distinguish static content for this course in studio
     static_asset_path: Path for static assets, which overrides data_directory and course_namespace, if nonempty
     """
 
@@ -117,7 +117,7 @@ def replace_static_urls(text, data_directory, course_namespace=None, static_asse
         if settings.DEBUG and finders.find(rest, True):
             return original
         # if we're running with a MongoBacked store course_namespace is not None, then use studio style urls
-        elif (not static_asset_path) and course_namespace is not None and not isinstance(modulestore(), XMLModuleStore):
+        elif course_id and modulestore().get_modulestore_type(course_id) != XML_MODULESTORE_TYPE:
             # first look in the static file pipeline and see if we are trying to reference
             # a piece of static content which is in the mitx repo (e.g. JS associated with an xmodule)
             if staticfiles_storage.exists(rest):
@@ -125,7 +125,7 @@ def replace_static_urls(text, data_directory, course_namespace=None, static_asse
             else:
                 # if not, then assume it's courseware specific content and then look in the
                 # Mongo-backed database
-                url = StaticContent.convert_legacy_static_url(rest, course_namespace)
+                url = StaticContent.convert_legacy_static_url_with_course_id(rest, course_id)
         # Otherwise, look the file up in staticfiles_storage, and append the data directory if needed
         else:
             course_path = "/".join((static_asset_path or data_directory, rest))

common/djangoapps/static_replace/test/test_static_replace.py

@@ -10,7 +10,6 @@ from xmodule.modulestore.xml import XMLModuleStore
 
 DATA_DIRECTORY = 'data_dir'
 COURSE_ID = 'org/course/run'
-NAMESPACE = Location('org', 'course', 'run', None, None)
 STATIC_SOURCE = '"/static/file.png"'
 
 
@@ -52,18 +51,18 @@ def test_storage_url_not_exists(mock_storage):
 def test_mongo_filestore(mock_modulestore, mock_static_content):
 
     mock_modulestore.return_value = Mock(MongoModuleStore)
-    mock_static_content.convert_legacy_static_url.return_value = "c4x://mock_url"
+    mock_static_content.convert_legacy_static_url_with_course_id.return_value = "c4x://mock_url"
 
     # No namespace => no change to path
     assert_equals('"/static/data_dir/file.png"', replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY))
 
     # Namespace => content url
     assert_equals(
-        '"' + mock_static_content.convert_legacy_static_url.return_value + '"',
-        replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY, NAMESPACE)
+        '"' + mock_static_content.convert_legacy_static_url_with_course_id.return_value + '"',
+        replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY, course_id=COURSE_ID)
     )
 
-    mock_static_content.convert_legacy_static_url.assert_called_once_with('file.png', NAMESPACE)
+    mock_static_content.convert_legacy_static_url_with_course_id.assert_called_once_with('file.png', COURSE_ID)
 
 
 @patch('static_replace.settings')

common/djangoapps/student/management/commands/anonymized_id_mapping.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+"""Dump username,unique_id_for_user pairs as CSV.
+
+Give instructors easy access to the mapping from anonymized IDs to user IDs
+with a simple Django management command to generate a CSV mapping. To run, use
+the following:
+
+rake django-admin[anonymized_id_mapping,x,y,z]
+
+[Naturally, substitute the appropriate values for x, y, and z. (I.e.,
+ lms, dev, and MITx/6.002x/Circuits)]"""
+
+import csv
+
+from django.contrib.auth.models import User
+from django.core.management.base import BaseCommand, CommandError
+
+from student.models import unique_id_for_user
+
+
+class Command(BaseCommand):
+    """Add our handler to the space where django-admin looks up commands."""
+
+    # It appears that with the way Rake invokes these commands, we can't
+    # have more than one arg passed through...annoying.
+    args = ("course_id", )
+
+    help = """Export a CSV mapping usernames to anonymized ids
+
+    Exports a CSV document mapping each username in the specified course to
+    the anonymized, unique user ID.
+    """
+
+    def handle(self, *args, **options):
+        if len(args) != 1:
+            raise CommandError("Usage: unique_id_mapping %s" %
+                               " ".join(("<%s>" % arg for arg in Command.args)))
+
+        course_id = args[0]
+
+        # Generate the output filename from the course ID.
+        # Change slashes to dashes first, and then append .csv extension.
+        output_filename = course_id.replace('/', '-') + ".csv"
+
+        # Figure out which students are enrolled in the course
+        students = User.objects.filter(courseenrollment__course_id=course_id)
+        if len(students) == 0:
+            self.stdout.write("No students enrolled in %s" % course_id)
+            return
+
+        # Write mapping to output file in CSV format with a simple header
+        try:
+            with open(output_filename, 'wb') as output_file:
+                csv_writer = csv.writer(output_file)
+                csv_writer.writerow(("User ID", "Anonymized user ID"))
+                for student in students:
+                    csv_writer.writerow((student.id, unique_id_for_user(student)))
+        except IOError:
+            raise CommandError("Error writing to file: %s" % output_filename)
+

common/djangoapps/student/management/commands/create_random_users.py

@@ -12,7 +12,7 @@ def create(n, course_id):
     for i in range(n):
         (user, user_profile, _) = _do_create_account(get_random_post_override())
         if course_id is not None:
-            CourseEnrollment.objects.create(user=user, course_id=course_id)
+            CourseEnrollment.enroll(user, course_id)
 
 
 class Command(BaseCommand):

common/djangoapps/student/management/commands/get_grades.py

@@ -0,0 +1,95 @@
+from courseware import grades, courses
+from django.test.client import RequestFactory
+from django.core.management.base import BaseCommand, CommandError
+import os
+from django.contrib.auth.models import User
+from optparse import make_option
+import datetime
+from django.core.handlers.base import BaseHandler
+import csv
+
+
+class RequestMock(RequestFactory):
+    def request(self, **request):
+        "Construct a generic request object."
+        request = RequestFactory.request(self, **request)
+        handler = BaseHandler()
+        handler.load_middleware()
+        for middleware_method in handler._request_middleware:
+            if middleware_method(request):
+                raise Exception("Couldn't create request mock object - "
+                                "request middleware returned a response")
+        return request
+
+
+class Command(BaseCommand):
+
+    help = """
+    Generate a list of grades for all students
+    that are enrolled in a course.
+
+    Outputs grades to a csv file.
+
+    Example:
+      sudo -u www-data SERVICE_VARIANT=lms /opt/edx/bin/django-admin.py get_grades \
+        -c MITx/Chi6.00intro/A_Taste_of_Python_Programming -o /tmp/20130813-6.00x.csv \
+        --settings=lms.envs.aws --pythonpath=/opt/wwc/edx-platform
+    """
+
+    option_list = BaseCommand.option_list + (
+        make_option('-c', '--course',
+                    metavar='COURSE_ID',
+                    dest='course',
+                    default=False,
+                    help='Course ID for grade distribution'),
+        make_option('-o', '--output',
+                    metavar='FILE',
+                    dest='output',
+                    default=False,
+                    help='Filename for grade output'))
+
+    def handle(self, *args, **options):
+        if os.path.exists(options['output']):
+            raise CommandError("File {0} already exists".format(
+                options['output']))
+
+        STATUS_INTERVAL = 100
+        course_id = options['course']
+        print "Fetching enrolled students for {0}".format(course_id)
+        enrolled_students = User.objects.filter(
+            courseenrollment__course_id=course_id).prefetch_related(
+                "groups").order_by('username')
+        factory = RequestMock()
+        request = factory.get('/')
+
+        total = enrolled_students.count()
+        print "Total enrolled: {0}".format(total)
+        course = courses.get_course_by_id(course_id)
+        total = enrolled_students.count()
+        start = datetime.datetime.now()
+        rows = []
+        header = None
+        for count, student in enumerate(enrolled_students):
+            count += 1
+            if count % STATUS_INTERVAL == 0:
+                # Print a status update with an approximation of
+                # how much time is left based on how long the last
+                # interval took
+                diff = datetime.datetime.now() - start
+                timeleft = diff * (total - count) / STATUS_INTERVAL
+                hours, remainder = divmod(timeleft.seconds, 3600)
+                minutes, seconds = divmod(remainder, 60)
+                print "{0}/{1} completed ~{2:02}:{3:02}m remaining".format(
+                    count, total, hours, minutes)
+                start = datetime.datetime.now()
+            request.user = student
+            grade = grades.grade(student, request, course)
+            if not header:
+                header = [section['label'] for section in grade[u'section_breakdown']]
+                rows.append(["email", "username"] + header)
+            percents = {section['label']: section['percent'] for section in grade[u'section_breakdown']}
+            row_percents = [percents[label] for label in header]
+            rows.append([student.email, student.username] + row_percents)
+        with open(options['output'], 'wb') as f:
+            writer = csv.writer(f)
+            writer.writerows(rows)

common/djangoapps/student/migrations/0027_add_active_flag_and_mode_to_courseware_enrollment.py

@@ -0,0 +1,183 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'CourseEnrollment.is_active'
+        db.add_column('student_courseenrollment', 'is_active',
+                      self.gf('django.db.models.fields.BooleanField')(default=True),
+                      keep_default=False)
+
+        # Adding field 'CourseEnrollment.mode'
+        db.add_column('student_courseenrollment', 'mode',
+                      self.gf('django.db.models.fields.CharField')(default='honor', max_length=100),
+                      keep_default=False)
+
+
+    def backwards(self, orm):
+        # Deleting field 'CourseEnrollment.is_active'
+        db.delete_column('student_courseenrollment', 'is_active')
+
+        # Deleting field 'CourseEnrollment.mode'
+        db.delete_column('student_courseenrollment', 'mode')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'student.courseenrollment': {
+            'Meta': {'ordering': "('user', 'course_id')", 'unique_together': "(('user', 'course_id'),)", 'object_name': 'CourseEnrollment'},
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'null': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'mode': ('django.db.models.fields.CharField', [], {'default': "'honor'", 'max_length': '100'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'student.courseenrollmentallowed': {
+            'Meta': {'unique_together': "(('email', 'course_id'),)", 'object_name': 'CourseEnrollmentAllowed'},
+            'auto_enroll': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'null': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'email': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'student.pendingemailchange': {
+            'Meta': {'object_name': 'PendingEmailChange'},
+            'activation_key': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '32', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'new_email': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True'})
+        },
+        'student.pendingnamechange': {
+            'Meta': {'object_name': 'PendingNameChange'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'new_name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'rationale': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True'})
+        },
+        'student.registration': {
+            'Meta': {'object_name': 'Registration', 'db_table': "'auth_registration'"},
+            'activation_key': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '32', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'unique': 'True'})
+        },
+        'student.testcenterregistration': {
+            'Meta': {'object_name': 'TestCenterRegistration'},
+            'accommodation_code': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}),
+            'accommodation_request': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}),
+            'authorization_id': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'db_index': 'True'}),
+            'client_authorization_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '20', 'db_index': 'True'}),
+            'confirmed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'eligibility_appointment_date_first': ('django.db.models.fields.DateField', [], {'db_index': 'True'}),
+            'eligibility_appointment_date_last': ('django.db.models.fields.DateField', [], {'db_index': 'True'}),
+            'exam_series_code': ('django.db.models.fields.CharField', [], {'max_length': '15', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'processed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
+            'testcenter_user': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'to': "orm['student.TestCenterUser']"}),
+            'updated_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'upload_error_message': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
+            'upload_status': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'blank': 'True'}),
+            'uploaded_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
+            'user_updated_at': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'})
+        },
+        'student.testcenteruser': {
+            'Meta': {'object_name': 'TestCenterUser'},
+            'address_1': ('django.db.models.fields.CharField', [], {'max_length': '40'}),
+            'address_2': ('django.db.models.fields.CharField', [], {'max_length': '40', 'blank': 'True'}),
+            'address_3': ('django.db.models.fields.CharField', [], {'max_length': '40', 'blank': 'True'}),
+            'candidate_id': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'db_index': 'True'}),
+            'city': ('django.db.models.fields.CharField', [], {'max_length': '32', 'db_index': 'True'}),
+            'client_candidate_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '50', 'db_index': 'True'}),
+            'company_name': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'blank': 'True'}),
+            'confirmed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
+            'country': ('django.db.models.fields.CharField', [], {'max_length': '3', 'db_index': 'True'}),
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'extension': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '8', 'blank': 'True'}),
+            'fax': ('django.db.models.fields.CharField', [], {'max_length': '35', 'blank': 'True'}),
+            'fax_country_code': ('django.db.models.fields.CharField', [], {'max_length': '3', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'middle_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'phone': ('django.db.models.fields.CharField', [], {'max_length': '35'}),
+            'phone_country_code': ('django.db.models.fields.CharField', [], {'max_length': '3', 'db_index': 'True'}),
+            'postal_code': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '16', 'blank': 'True'}),
+            'processed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
+            'salutation': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
+            'state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'blank': 'True'}),
+            'suffix': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'updated_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'upload_error_message': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
+            'upload_status': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'blank': 'True'}),
+            'uploaded_at': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'to': "orm['auth.User']", 'unique': 'True'}),
+            'user_updated_at': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'})
+        },
+        'student.userprofile': {
+            'Meta': {'object_name': 'UserProfile', 'db_table': "'auth_userprofile'"},
+            'allow_certificate': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'courseware': ('django.db.models.fields.CharField', [], {'default': "'course.xml'", 'max_length': '255', 'blank': 'True'}),
+            'gender': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '6', 'null': 'True', 'blank': 'True'}),
+            'goals': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'language': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'blank': 'True'}),
+            'level_of_education': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '6', 'null': 'True', 'blank': 'True'}),
+            'location': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'blank': 'True'}),
+            'mailing_address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'meta': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'profile'", 'unique': 'True', 'to': "orm['auth.User']"}),
+            'year_of_birth': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'})
+        },
+        'student.usertestgroup': {
+            'Meta': {'object_name': 'UserTestGroup'},
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '32', 'db_index': 'True'}),
+            'users': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.User']", 'db_index': 'True', 'symmetrical': 'False'})
+        }
+    }
+
+    complete_apps = ['student']

common/djangoapps/student/models.py

@@ -11,11 +11,11 @@ file and check it in at the same time as your model changes. To do that,
 3. Add the migration file created in edx-platform/common/djangoapps/student/migrations/
 """
 from datetime import datetime
+from random import randint
 import hashlib
 import json
 import logging
 import uuid
-from random import randint
 
 
 from django.conf import settings
@@ -645,16 +645,223 @@ class PendingEmailChange(models.Model):
 
 
 class CourseEnrollment(models.Model):
+    """
+    Represents a Student's Enrollment record for a single Course. You should
+    generally not manipulate CourseEnrollment objects directly, but use the
+    classmethods provided to enroll, unenroll, or check on the enrollment status
+    of a given student.
+
+    We're starting to consolidate course enrollment logic in this class, but
+    more should be brought in (such as checking against CourseEnrollmentAllowed,
+    checking course dates, user permissions, etc.) This logic is currently
+    scattered across our views.
+    """
     user = models.ForeignKey(User)
     course_id = models.CharField(max_length=255, db_index=True)
-
     created = models.DateTimeField(auto_now_add=True, null=True, db_index=True)
 
+    # If is_active is False, then the student is not considered to be enrolled
+    # in the course (is_enrolled() will return False)
+    is_active = models.BooleanField(default=True)
+
+    # Represents the modes that are possible. We'll update this later with a
+    # list of possible values.
+    mode = models.CharField(default="honor", max_length=100)
+
+
     class Meta:
         unique_together = (('user', 'course_id'),)
+        ordering = ('user', 'course_id')
 
     def __unicode__(self):
-        return "[CourseEnrollment] %s: %s (%s)" % (self.user, self.course_id, self.created)
+        return (
+            "[CourseEnrollment] {}: {} ({}); active: ({})"
+        ).format(self.user, self.course_id, self.created, self.is_active)
+
+    @classmethod
+    def create_enrollment(cls, user, course_id, mode="honor", is_active=False):
+        """
+        Create an enrollment for a user in a class. By default *this enrollment
+        is not active*. This is useful for when an enrollment needs to go
+        through some sort of approval process before being activated. If you
+        don't need this functionality, just call `enroll()` instead.
+
+        Returns a CoursewareEnrollment object.
+
+        `user` is a Django User object. If it hasn't been saved yet (no `.id`
+               attribute), this method will automatically save it before
+               adding an enrollment for it.
+
+        `course_id` is our usual course_id string (e.g. "edX/Test101/2013_Fall)
+
+        `mode` is a string specifying what kind of enrollment this is. The
+               default is "honor", meaning honor certificate. Future options
+               may include "audit", "verified_id", etc. Please don't use it
+               until we have these mapped out.
+
+        `is_active` is a boolean. If the CourseEnrollment object has
+                    `is_active=False`, then calling
+                    `CourseEnrollment.is_enrolled()` for that user/course_id
+                    will return False.
+
+        It is expected that this method is called from a method which has already
+        verified the user authentication and access.
+        """
+        # If we're passing in a newly constructed (i.e. not yet persisted) User,
+        # save it to the database so that it can have an ID that we can throw
+        # into our CourseEnrollment object. Otherwise, we'll get an
+        # IntegrityError for having a null user_id.
+        if user.id is None:
+            user.save()
+
+        enrollment, _ = CourseEnrollment.objects.get_or_create(
+            user=user,
+            course_id=course_id,
+        )
+        # In case we're reactivating a deactivated enrollment, or changing the
+        # enrollment mode.
+        if enrollment.mode != mode or enrollment.is_active != is_active:
+            enrollment.mode = mode
+            enrollment.is_active = is_active
+            enrollment.save()
+
+        return enrollment
+
+    @classmethod
+    def enroll(cls, user, course_id, mode="honor"):
+        """
+        Enroll a user in a course. This saves immediately.
+
+        Returns a CoursewareEnrollment object.
+
+        `user` is a Django User object. If it hasn't been saved yet (no `.id`
+               attribute), this method will automatically save it before
+               adding an enrollment for it.
+
+        `course_id` is our usual course_id string (e.g. "edX/Test101/2013_Fall)
+
+        `mode` is a string specifying what kind of enrollment this is. The
+               default is "honor", meaning honor certificate. Future options
+               may include "audit", "verified_id", etc. Please don't use it
+               until we have these mapped out.
+
+        It is expected that this method is called from a method which has already
+        verified the user authentication and access.
+        """
+        return cls.create_enrollment(user, course_id, mode, is_active=True)
+
+    @classmethod
+    def enroll_by_email(cls, email, course_id, mode="honor", ignore_errors=True):
+        """
+        Enroll a user in a course given their email. This saves immediately.
+
+        Note that  enrolling by email is generally done in big batches and the
+        error rate is high. For that reason, we supress User lookup errors by
+        default.
+
+        Returns a CoursewareEnrollment object. If the User does not exist and
+        `ignore_errors` is set to `True`, it will return None.
+
+        `email` Email address of the User to add to enroll in the course.
+
+        `course_id` is our usual course_id string (e.g. "edX/Test101/2013_Fall)
+
+        `mode` is a string specifying what kind of enrollment this is. The
+               default is "honor", meaning honor certificate. Future options
+               may include "audit", "verified_id", etc. Please don't use it
+               until we have these mapped out.
+
+        `ignore_errors` is a boolean indicating whether we should suppress
+                        `User.DoesNotExist` errors (returning None) or let it
+                        bubble up.
+
+        It is expected that this method is called from a method which has already
+        verified the user authentication and access.
+        """
+        try:
+            user = User.objects.get(email=email)
+            return cls.enroll(user, course_id, mode)
+        except User.DoesNotExist:
+            err_msg = u"Tried to enroll email {} into course {}, but user not found"
+            log.error(err_msg.format(email, course_id))
+            if ignore_errors:
+                return None
+            raise
+
+    @classmethod
+    def unenroll(cls, user, course_id):
+        """
+        Remove the user from a given course. If the relevant `CourseEnrollment`
+        object doesn't exist, we log an error but don't throw an exception.
+
+        `user` is a Django User object. If it hasn't been saved yet (no `.id`
+               attribute), this method will automatically save it before
+               adding an enrollment for it.
+
+        `course_id` is our usual course_id string (e.g. "edX/Test101/2013_Fall)
+        """
+        try:
+            record = CourseEnrollment.objects.get(user=user, course_id=course_id)
+            record.is_active = False
+            record.save()
+        except cls.DoesNotExist:
+            log.error("Tried to unenroll student {} from {} but they were not enrolled")
+
+    @classmethod
+    def unenroll_by_email(cls, email, course_id):
+        """
+        Unenroll a user from a course given their email. This saves immediately.
+        User lookup errors are logged but will not throw an exception.
+
+        `email` Email address of the User to unenroll from the course.
+
+        `course_id` is our usual course_id string (e.g. "edX/Test101/2013_Fall)
+        """
+        try:
+            user = User.objects.get(email=email)
+            return cls.unenroll(user, course_id)
+        except User.DoesNotExist:
+            err_msg = u"Tried to unenroll email {} from course {}, but user not found"
+            log.error(err_msg.format(email, course_id))
+
+    @classmethod
+    def is_enrolled(cls, user, course_id):
+        """
+        Remove the user from a given course. If the relevant `CourseEnrollment`
+        object doesn't exist, we log an error but don't throw an exception.
+
+        Returns True if the user is enrolled in the course (the entry must exist
+        and it must have `is_active=True`). Otherwise, returns False.
+
+        `user` is a Django User object. If it hasn't been saved yet (no `.id`
+               attribute), this method will automatically save it before
+               adding an enrollment for it.
+
+        `course_id` is our usual course_id string (e.g. "edX/Test101/2013_Fall)
+        """
+        try:
+            record = CourseEnrollment.objects.get(user=user, course_id=course_id)
+            return record.is_active
+        except cls.DoesNotExist:
+            return False
+
+    @classmethod
+    def enrollments_for_user(cls, user):
+        return CourseEnrollment.objects.filter(user=user, is_active=1)
+
+    def activate(self):
+        """Makes this `CourseEnrollment` record active. Saves immediately."""
+        if not self.is_active:
+            self.is_active = True
+            self.save()
+
+    def deactivate(self):
+        """Makes this `CourseEnrollment` record inactive. Saves immediately. An
+        inactive record means that the student is not enrolled in this course.
+        """
+        if self.is_active:
+            self.is_active = False
+            self.save()
 
 
 class CourseEnrollmentAllowed(models.Model):

common/djangoapps/student/tests/tests.py

@@ -21,9 +21,8 @@ from django.utils.http import int_to_base36
 from mock import Mock, patch
 from textwrap import dedent
 
-from student.models import unique_id_for_user
+from student.models import unique_id_for_user, CourseEnrollment
 from student.views import process_survey_link, _cert_info, password_reset, password_reset_confirm_wrapper
-from student.views import enroll_in_course, is_enrolled_in_course
 from student.tests.factories import UserFactory
 from student.tests.test_email import mock_render_to_string
 COURSE_1 = 'edX/toy/2012_Fall'
@@ -209,12 +208,127 @@ class CourseEndingTest(TestCase):
 
 
 class EnrollInCourseTest(TestCase):
-    """ Tests the helper method for enrolling a user in a class """
+    """Tests enrolling and unenrolling in courses."""
 
-    def test_enroll_in_course(self):
+    def test_enrollment(self):
         user = User.objects.create_user("joe", "joe@joe.com", "password")
-        user.save()
-        course_id = "course_id"
-        self.assertFalse(is_enrolled_in_course(user, course_id))
-        enroll_in_course(user, course_id)
-        self.assertTrue(is_enrolled_in_course(user, course_id))
+        course_id = "edX/Test101/2013"
+
+        # Test basic enrollment
+        self.assertFalse(CourseEnrollment.is_enrolled(user, course_id))
+        CourseEnrollment.enroll(user, course_id)
+        self.assertTrue(CourseEnrollment.is_enrolled(user, course_id))
+
+        # Enrolling them again should be harmless
+        CourseEnrollment.enroll(user, course_id)
+        self.assertTrue(CourseEnrollment.is_enrolled(user, course_id))
+
+        # Now unenroll the user
+        CourseEnrollment.unenroll(user, course_id)
+        self.assertFalse(CourseEnrollment.is_enrolled(user, course_id))
+
+        # Unenrolling them again should also be harmless
+        CourseEnrollment.unenroll(user, course_id)
+        self.assertFalse(CourseEnrollment.is_enrolled(user, course_id))
+
+        # The enrollment record should still exist, just be inactive
+        enrollment_record = CourseEnrollment.objects.get(
+            user=user,
+            course_id=course_id
+        )
+        self.assertFalse(enrollment_record.is_active)
+
+    def test_enrollment_non_existent_user(self):
+        # Testing enrollment of newly unsaved user (i.e. no database entry)
+        user = User(username="rusty", email="rusty@fake.edx.org")
+        course_id = "edX/Test101/2013"
+
+        self.assertFalse(CourseEnrollment.is_enrolled(user, course_id))
+
+        # Unenroll does nothing
+        CourseEnrollment.unenroll(user, course_id)
+
+        # Implicit save() happens on new User object when enrolling, so this
+        # should still work
+        CourseEnrollment.enroll(user, course_id)
+        self.assertTrue(CourseEnrollment.is_enrolled(user, course_id))
+
+    def test_enrollment_by_email(self):
+        user = User.objects.create(username="jack", email="jack@fake.edx.org")
+        course_id = "edX/Test101/2013"
+
+        CourseEnrollment.enroll_by_email("jack@fake.edx.org", course_id)
+        self.assertTrue(CourseEnrollment.is_enrolled(user, course_id))
+
+        # This won't throw an exception, even though the user is not found
+        self.assertIsNone(
+            CourseEnrollment.enroll_by_email("not_jack@fake.edx.org", course_id)
+        )
+
+        self.assertRaises(
+            User.DoesNotExist,
+            CourseEnrollment.enroll_by_email,
+            "not_jack@fake.edx.org",
+            course_id,
+            ignore_errors=False
+        )
+
+        # Now unenroll them by email
+        CourseEnrollment.unenroll_by_email("jack@fake.edx.org", course_id)
+        self.assertFalse(CourseEnrollment.is_enrolled(user, course_id))
+
+        # Harmless second unenroll
+        CourseEnrollment.unenroll_by_email("jack@fake.edx.org", course_id)
+        self.assertFalse(CourseEnrollment.is_enrolled(user, course_id))
+
+        # Unenroll on non-existent user shouldn't throw an error
+        CourseEnrollment.unenroll_by_email("not_jack@fake.edx.org", course_id)
+
+    def test_enrollment_multiple_classes(self):
+        user = User(username="rusty", email="rusty@fake.edx.org")
+        course_id1 = "edX/Test101/2013"
+        course_id2 = "MITx/6.003z/2012"
+
+        CourseEnrollment.enroll(user, course_id1)
+        CourseEnrollment.enroll(user, course_id2)
+        self.assertTrue(CourseEnrollment.is_enrolled(user, course_id1))
+        self.assertTrue(CourseEnrollment.is_enrolled(user, course_id2))
+
+        CourseEnrollment.unenroll(user, course_id1)
+        self.assertFalse(CourseEnrollment.is_enrolled(user, course_id1))
+        self.assertTrue(CourseEnrollment.is_enrolled(user, course_id2))
+
+        CourseEnrollment.unenroll(user, course_id2)
+        self.assertFalse(CourseEnrollment.is_enrolled(user, course_id1))
+        self.assertFalse(CourseEnrollment.is_enrolled(user, course_id2))
+
+    def test_activation(self):
+        user = User.objects.create(username="jack", email="jack@fake.edx.org")
+        course_id = "edX/Test101/2013"
+        self.assertFalse(CourseEnrollment.is_enrolled(user, course_id))
+
+        # Creating an enrollment doesn't actually enroll a student
+        # (calling CourseEnrollment.enroll() would have)
+        enrollment = CourseEnrollment.create_enrollment(user, course_id)
+        self.assertFalse(CourseEnrollment.is_enrolled(user, course_id))
+
+        # Until you explicitly activate it
+        enrollment.activate()
+        self.assertTrue(CourseEnrollment.is_enrolled(user, course_id))
+
+        # Activating something that's already active does nothing
+        enrollment.activate()
+        self.assertTrue(CourseEnrollment.is_enrolled(user, course_id))
+
+        # Now deactive
+        enrollment.deactivate()
+        self.assertFalse(CourseEnrollment.is_enrolled(user, course_id))
+
+        # Deactivating something that's already inactive does nothing
+        enrollment.deactivate()
+        self.assertFalse(CourseEnrollment.is_enrolled(user, course_id))
+
+        # A deactivated enrollment should be activated if enroll() is called
+        # for that user/course_id combination
+        CourseEnrollment.enroll(user, course_id)
+        self.assertTrue(CourseEnrollment.is_enrolled(user, course_id))

common/djangoapps/student/views.py

@@ -95,6 +95,7 @@ def index(request, extra_context={}, user=None):
     courses = sort_by_announcement(courses)
 
     context = {'courses': courses}
+
     context.update(extra_context)
     return render_to_response('index.html', context)
 
@@ -254,13 +255,12 @@ def register_user(request, extra_context=None):
 @ensure_csrf_cookie
 def dashboard(request):
     user = request.user
-    enrollments = CourseEnrollment.objects.filter(user=user)
 
     # Build our courses list for the user, but ignore any courses that no longer
     # exist (because the course IDs have changed). Still, we don't delete those
     # enrollments, because it could have been a data push snafu.
     courses = []
-    for enrollment in enrollments:
+    for enrollment in CourseEnrollment.enrollments_for_user(user):
         try:
             courses.append(course_from_id(enrollment.course_id))
         except ItemNotFoundError:
@@ -377,18 +377,13 @@ def change_enrollment(request):
                                "course:{0}".format(course_num),
                                "run:{0}".format(run)])
 
-        try:
-            enroll_in_course(user, course.id)
-        except IntegrityError:
-            # If we've already created this enrollment in a separate transaction,
-            # then just continue
-            pass
+        CourseEnrollment.enroll(user, course.id)
+
         return HttpResponse()
 
     elif action == "unenroll":
         try:
-            enrollment = CourseEnrollment.objects.get(user=user, course_id=course_id)
-            enrollment.delete()
+            CourseEnrollment.unenroll(user, course_id)
 
             org, course_num, run = course_id.split("/")
             statsd.increment("common.student.unenrollment",
@@ -402,30 +397,10 @@ def change_enrollment(request):
     else:
         return HttpResponseBadRequest(_("Enrollment action is invalid"))
 
-
-def enroll_in_course(user, course_id):
-    """
-    Helper method to enroll a user in a particular class.
-
-    It is expected that this method is called from a method which has already
-    verified the user authentication and access.
-    """
-    CourseEnrollment.objects.get_or_create(user=user, course_id=course_id)
-
-
-def is_enrolled_in_course(user, course_id):
-    """
-    Helper method that returns whether or not the user is enrolled in a particular course.
-    """
-    return CourseEnrollment.objects.filter(user=user, course_id=course_id).count() > 0
-
-
 @ensure_csrf_cookie
 def accounts_login(request, error=""):
-
     return render_to_response('login.html', {'error': error})
 
-
 # Need different levels of logging
 @ensure_csrf_cookie
 def login_user(request, error=""):
@@ -1008,13 +983,21 @@ def activate_account(request, key):
             ceas = CourseEnrollmentAllowed.objects.filter(email=student[0].email)
             for cea in ceas:
                 if cea.auto_enroll:
-                    course_id = cea.course_id
-                    _enrollment, _created = CourseEnrollment.objects.get_or_create(user_id=student[0].id, course_id=course_id)
+                    CourseEnrollment.enroll(student[0], cea.course_id)
 
-        resp = render_to_response("registration/activation_complete.html", {'user_logged_in': user_logged_in, 'already_active': already_active})
+        resp = render_to_response(
+            "registration/activation_complete.html",
+            {
+                'user_logged_in': user_logged_in,
+                'already_active': already_active
+            }
+        )
         return resp
     if len(r) == 0:
-        return render_to_response("registration/activation_invalid.html", {'csrf': csrf(request)['csrf_token']})
+        return render_to_response(
+            "registration/activation_invalid.html",
+            {'csrf': csrf(request)['csrf_token']}
+        )
     return HttpResponse(_("Unknown error. Please e-mail us to let us know how it happened."))
 
 
@@ -1037,7 +1020,11 @@ def password_reset(request):
                                         'error': _('Invalid e-mail or user')}))
 
 
-def password_reset_confirm_wrapper(request, uidb36=None, token=None):
+def password_reset_confirm_wrapper(
+    request,
+    uidb36=None,
+    token=None,
+):
     ''' A wrapper around django.contrib.auth.views.password_reset_confirm.
         Needed because we want to set the user as active at this step.
     '''
@@ -1049,7 +1036,12 @@ def password_reset_confirm_wrapper(request, uidb36=None, token=None):
         user.save()
     except (ValueError, User.DoesNotExist):
         pass
-    return password_reset_confirm(request, uidb36=uidb36, token=token)
+    # we also want to pass settings.PLATFORM_NAME in as extra_context
+
+    extra_context = {"platform_name": settings.PLATFORM_NAME}
+    return password_reset_confirm(
+        request, uidb36=uidb36, token=token, extra_context=extra_context
+    )
 
 
 def reactivation_email_for_user(user):

common/djangoapps/terrain/course_helpers.py

@@ -54,7 +54,7 @@ def register_by_course_id(course_id, is_staff=False):
     if is_staff:
         u.is_staff = True
         u.save()
-    CourseEnrollment.objects.get_or_create(user=u, course_id=course_id)
+    CourseEnrollment.enroll(u, course_id)
 
 
 @world.absorb

common/djangoapps/xmodule_modifiers.py

@@ -76,7 +76,7 @@ def replace_course_urls(get_html, course_id):
     return _get_html
 
 
-def replace_static_urls(get_html, data_dir, course_namespace=None, static_asset_path=''):
+def replace_static_urls(get_html, data_dir, course_id=None, static_asset_path=''):
     """
     Updates the supplied module with a new get_html function that wraps
     the old get_html function and substitutes urls of the form /static/...
@@ -85,7 +85,7 @@ def replace_static_urls(get_html, data_dir, course_namespace=None, static_asset_
 
     @wraps(get_html)
     def _get_html():
-        return static_replace.replace_static_urls(get_html(), data_dir, course_namespace, static_asset_path=static_asset_path)
+        return static_replace.replace_static_urls(get_html(), data_dir, course_id, static_asset_path=static_asset_path)
     return _get_html
 
 

common/lib/calc/calc.py

@@ -11,11 +11,6 @@ import numpy
 import scipy.constants
 import calcfunctions
 
-# Have numpy ignore errors on functions outside its domain.
-# See http://docs.scipy.org/doc/numpy/reference/generated/numpy.seterr.html
-# TODO worry about thread safety/changing a global setting
-numpy.seterr(all='ignore')  # Also: 'ignore', 'warn' (default), 'raise'
-
 from pyparsing import (
     Word, Literal, CaselessLiteral, ZeroOrMore, MatchFirst, Optional, Forward,
     Group, ParseResults, stringEnd, Suppress, Combine, alphas, nums, alphanums

common/lib/calc/tests/test_calc.py

@@ -7,6 +7,12 @@ import numpy
 import calc
 from pyparsing import ParseException
 
+# numpy's default behavior when it evaluates a function outside its domain
+# is to raise a warning (not an exception) which is then printed to STDOUT.
+# To prevent this from polluting the output of the tests, configure numpy to
+# ignore it instead.
+# See http://docs.scipy.org/doc/numpy/reference/generated/numpy.seterr.html
+numpy.seterr(all='ignore')  # Also: 'ignore', 'warn' (default), 'raise'
 
 class EvaluatorTest(unittest.TestCase):
     """
@@ -186,17 +192,16 @@ class EvaluatorTest(unittest.TestCase):
         arcsin_inputs = ['-0.707', '0', '0.5', '0.588', '1.298 + 0.635*j']
         arcsin_angles = [-0.785, 0, 0.524, 0.629, 1 + 1j]
         self.assert_function_values('arcsin', arcsin_inputs, arcsin_angles)
-        # Rather than throwing an exception, numpy.arcsin gives nan
-        # self.assertTrue(numpy.isnan(calc.evaluator({}, {}, 'arcsin(-1.1)')))
-        # self.assertTrue(numpy.isnan(calc.evaluator({}, {}, 'arcsin(1.1)')))
-        # Disabled for now because they are giving a runtime warning... :-/
+        # Rather than a complex number, numpy.arcsin gives nan
+        self.assertTrue(numpy.isnan(calc.evaluator({}, {}, 'arcsin(-1.1)')))
+        self.assertTrue(numpy.isnan(calc.evaluator({}, {}, 'arcsin(1.1)')))
 
         # Include those where the real part is between 0 and pi
         arccos_inputs = ['1', '0.866', '0.809', '0.834-0.989*j']
         arccos_angles = [0, 0.524, 0.628, 1 + 1j]
         self.assert_function_values('arccos', arccos_inputs, arccos_angles)
-        # self.assertTrue(numpy.isnan(calc.evaluator({}, {}, 'arccos(-1.1)')))
-        # self.assertTrue(numpy.isnan(calc.evaluator({}, {}, 'arccos(1.1)')))
+        self.assertTrue(numpy.isnan(calc.evaluator({}, {}, 'arccos(-1.1)')))
+        self.assertTrue(numpy.isnan(calc.evaluator({}, {}, 'arccos(1.1)')))
 
         # Has the same range as arcsin
         arctan_inputs = ['-1', '0', '0.577', '0.727', '0.272 + 1.084*j']
@@ -535,10 +540,10 @@ class EvaluatorTest(unittest.TestCase):
         # With case sensitive turned on, it should pick the right function
         functions = {'f': lambda x: x, 'F': lambda x: x + 1}
         self.assertEqual(
-            calc.evaluator({}, functions, 'f(6)', case_sensitive=True), 6
+            6, calc.evaluator({}, functions, 'f(6)', case_sensitive=True)
         )
         self.assertEqual(
-            calc.evaluator({}, functions, 'F(6)', case_sensitive=True), 7
+            7, calc.evaluator({}, functions, 'F(6)', case_sensitive=True)
         )
 
     def test_undefined_vars(self):

common/lib/capa/capa/templates/optioninput.html

@@ -29,4 +29,9 @@
     <span class="sr">Status: incomplete</span>
   </span>
   % endif
+
+  % if msg:
+      <span class="message">${msg|n}</span>
+  % endif
+
 </form>

common/lib/xmodule/xmodule/combined_open_ended_module.py

@@ -8,13 +8,16 @@ from .x_module import XModule
 from xblock.core import Integer, Scope, String, List, Float, Boolean
 from xmodule.open_ended_grading_classes.combined_open_ended_modulev1 import CombinedOpenEndedV1Module, CombinedOpenEndedV1Descriptor
 from collections import namedtuple
-from .fields import Date
+from .fields import Date, Timedelta
 import textwrap
 
 log = logging.getLogger("mitx.courseware")
 
-V1_SETTINGS_ATTRIBUTES = ["display_name", "max_attempts", "graded", "accept_file_upload",
-                          "skip_spelling_checks", "due", "graceperiod", "weight"]
+V1_SETTINGS_ATTRIBUTES = [
+    "display_name", "max_attempts", "graded", "accept_file_upload",
+    "skip_spelling_checks", "due", "graceperiod", "weight", "min_to_calibrate",
+    "max_to_calibrate", "peer_grader_count", "required_peer_grading",
+]
 
 V1_STUDENT_ATTRIBUTES = ["current_task_number", "task_states", "state",
                          "student_attempts", "ready_to_reset"]
@@ -37,7 +40,7 @@ DEFAULT_DATA = textwrap.dedent("""\
         </p>
 
         <p>
-        Write a persuasive essay to a newspaper reflecting your vies on censorship in libraries. Do you believe that certain materials, such as books, music, movies, magazines, etc., should be removed from the shelves if they are found offensive? Support your position with convincing arguments from your own experience, observations, and/or reading.
+        Write a persuasive essay to a newspaper reflecting your views on censorship in libraries. Do you believe that certain materials, such as books, music, movies, magazines, etc., should be removed from the shelves if they are found offensive? Support your position with convincing arguments from your own experience, observations, and/or reading.
         </p>
 
     </prompt>
@@ -229,7 +232,7 @@ class CombinedOpenEndedFields(object):
         default=None,
         scope=Scope.settings
     )
-    graceperiod = String(
+    graceperiod = Timedelta(
         help="Amount of time after the due date that submissions will be accepted",
         default=None,
         scope=Scope.settings
@@ -244,6 +247,34 @@ class CombinedOpenEndedFields(object):
         values={"min" : 0 , "step": ".1"},
         default=1
     )
+    min_to_calibrate = Integer(
+        display_name="Minimum Peer Grading Calibrations",
+        help="The minimum number of calibration essays each student will need to complete for peer grading.",
+        default=3,
+        scope=Scope.settings,
+        values={"min" : 1, "max" : 20, "step" : "1"}
+    )
+    max_to_calibrate = Integer(
+        display_name="Maximum Peer Grading Calibrations",
+        help="The maximum number of calibration essays each student will need to complete for peer grading.",
+        default=6,
+        scope=Scope.settings,
+        values={"min" : 1, "max" : 20, "step" : "1"}
+    )
+    peer_grader_count = Integer(
+        display_name="Peer Graders per Response",
+        help="The number of peers who will grade each submission.",
+        default=3,
+        scope=Scope.settings,
+        values={"min" : 1, "step" : "1", "max" : 5}
+    )
+    required_peer_grading = Integer(
+        display_name="Required Peer Grading",
+        help="The number of other students each student making a submission will have to grade.",
+        default=3,
+        scope=Scope.settings,
+        values={"min" : 1, "step" : "1", "max" : 5}
+    )
     markdown = String(
         help="Markdown source of this module",
         default=textwrap.dedent("""\

common/lib/xmodule/xmodule/contentstore/content.py

@@ -100,6 +100,16 @@ class StaticContent(object):
         loc = StaticContent.compute_location(course_namespace.org, course_namespace.course, path)
         return StaticContent.get_url_path_from_location(loc)
 
+    @staticmethod
+    def convert_legacy_static_url_with_course_id(path, course_id):
+        """
+        Returns a path to a piece of static content when we are provided with a filepath and
+        a course_id
+        """
+        org, course_num, __ = course_id.split("/")
+        loc = StaticContent.compute_location(org, course_num, path)
+        return StaticContent.get_url_path_from_location(loc)
+
     def stream_data(self):
         yield self._data
 

common/lib/xmodule/xmodule/fields.py

@@ -82,6 +82,9 @@ TIMEDELTA_REGEX = re.compile(r'^((?P<days>\d+?) day(?:s?))?(\s)?((?P<hours>\d+?)
 
 
 class Timedelta(ModelType):
+    # Timedeltas are immutable, see http://docs.python.org/2/library/datetime.html#available-types
+    MUTABLE = False
+
     def from_json(self, time_str):
         """
         time_str: A string with the following components:

common/lib/xmodule/xmodule/js/spec/video/video_caption_spec.js

@@ -443,7 +443,7 @@
         });
 
         it('trigger seek event with the correct time', function() {
-          expect(videoPlayer.currentTime).toEqual(15);
+          expect(videoPlayer.currentTime).toEqual(14.91);
         });
       });
 
@@ -455,7 +455,20 @@
         });
 
         it('trigger seek event with the correct time', function() {
-          expect(videoPlayer.currentTime).toEqual(15);
+          expect(videoPlayer.currentTime).toEqual(14.91);
+        });
+      });
+
+      describe('when the player type is Flash at speed 0.75x', function () {
+        beforeEach(function () {
+            initialize();
+            videoSpeedControl.currentSpeed = '0.75';
+            state.currentPlayerMode = 'flash';
+            $('.subtitles li[data-start="14910"]').trigger('click');
+        });
+
+        it('trigger seek event with the correct time', function () {
+            expect(videoPlayer.currentTime).toEqual(15);
         });
       });
     });

common/lib/xmodule/xmodule/js/src/video/03_video_player.js

@@ -315,7 +315,21 @@ function (HTML5Video) {
 
         this.videoPlayer.log('load_video');
 
-        availablePlaybackRates = this.videoPlayer.player.getAvailablePlaybackRates();
+        availablePlaybackRates = this.videoPlayer.player
+                                    .getAvailablePlaybackRates();
+
+        // Because of problems with muting sound outside of range 0.25 and
+        // 5.0, we should filter our available playback rates.
+        // Issues:
+        //   https://code.google.com/p/chromium/issues/detail?id=264341
+        //   https://bugzilla.mozilla.org/show_bug.cgi?id=840745
+        //   https://developer.mozilla.org/en-US/docs/DOM/HTMLMediaElement
+
+        availablePlaybackRates = _.filter(availablePlaybackRates, function(item){
+            var speed = Number(item);
+            return  speed > 0.25 && speed <= 5;
+        });
+
         if ((this.currentPlayerMode === 'html5') && (this.videoType === 'youtube')) {
             if (availablePlaybackRates.length === 1) {
                 // This condition is needed in cases when Firefox version is less than 20. In those versions

common/lib/xmodule/xmodule/js/src/video/07_video_volume_control.js

@@ -61,37 +61,71 @@ function () {
             slide: state.videoVolumeControl.onChange
         });
 
-        // Make sure that we can focus the actual volume slider while Tabing.
-        state.videoVolumeControl.volumeSliderEl.find('a').attr('tabindex', '0');
-
         state.videoVolumeControl.el.toggleClass('muted', state.videoVolumeControl.currentVolume === 0);
     }
 
-    // function _bindHandlers(state)
-    //
-    //     Bind any necessary function callbacks to DOM events (click, mousemove, etc.).
+    /**
+     * @desc Bind any necessary function callbacks to DOM events (click,
+     *     mousemove, etc.).
+     *
+     * @type {function}
+     * @access private
+     *
+     * @param {object} state The object containg the state of the video player.
+     *     All other modules, their parameters, public variables, etc. are
+     *     available via this object.
+     *
+     * @this {object} The global window object.
+     *
+     * @returns {undefined}
+     */
     function _bindHandlers(state) {
-        state.videoVolumeControl.buttonEl.on('click', state.videoVolumeControl.toggleMute);
+        state.videoVolumeControl.buttonEl
+            .on('click', state.videoVolumeControl.toggleMute);
 
         state.videoVolumeControl.el.on('mouseenter', function() {
-            $(this).addClass('open');
-        });
-
-        state.videoVolumeControl.buttonEl.on('focus', function() {
-            $(this).parent().addClass('open');
+            state.videoVolumeControl.el.addClass('open');
         });
 
         state.videoVolumeControl.el.on('mouseleave', function() {
-            $(this).removeClass('open');
-        });
-
-        state.videoVolumeControl.buttonEl.on('blur', function() {
-            state.videoVolumeControl.volumeSliderEl.find('a').focus();
-        });
-
-        state.videoVolumeControl.volumeSliderEl.find('a').on('blur', function () {
             state.videoVolumeControl.el.removeClass('open');
         });
+
+        // Attach a focus event to the volume button.
+        state.videoVolumeControl.buttonEl.on('blur', function() {
+            // If the focus is being trasnfered from the volume slider, then we
+            // don't do anything except for unsetting the special flag.
+            if (state.volumeBlur === true) {
+                state.volumeBlur = false;
+            }
+
+            //If the focus is comming from elsewhere, then we must show the
+            // volume slider and set focus to it.
+            else {
+                state.videoVolumeControl.el.addClass('open');
+                state.videoVolumeControl.volumeSliderEl.find('a').focus();
+            }
+        });
+
+        // Attach a blur event handler (loss of focus) to the volume slider
+        // element. More specifically, we are attaching to the handle on
+        // the slider with which you can change the volume.
+        state.videoVolumeControl.volumeSliderEl.find('a')
+            .on('blur', function () {
+                // Hide the volume slider. This is done so that we can
+                // continue to the next (or previous) element by tabbing.
+                // Otherwise, after next tab we would come back to the volume
+                // slider because it is the next element visible element that
+                // we can tab to after the volume button.
+                state.videoVolumeControl.el.removeClass('open');
+
+                // Set focus to the volume button.
+                state.videoVolumeControl.buttonEl.focus();
+
+                // We store the fact that previous element that lost focus was
+                // the volume clontrol.
+                state.volumeBlur = true;
+            });
     }
 
     // ***************************************************************

common/lib/xmodule/xmodule/js/src/video/08_video_speed_control.js

@@ -10,6 +10,12 @@ function () {
     return function (state) {
         state.videoSpeedControl = {};
 
+        if (state.videoType === 'html5' && !(_checkPlaybackRates())) {
+            _hideSpeedControl(state);
+
+            return;
+        }
+
         _makeFunctionsPublic(state);
         _renderElements(state);
         _bindHandlers(state);
@@ -61,46 +67,145 @@ function () {
         state.videoSpeedControl.setSpeed(state.speed);
     }
 
-    // function _bindHandlers(state)
-    //
-    //     Bind any necessary function callbacks to DOM events (click,
-    //     mousemove, etc.).
+    /**
+    * @desc Check if playbackRate supports by browser.
+    *
+    * @type {function}
+    * @access private
+    *
+    * @param {object} state The object containg the state of the video player.
+    *     All other modules, their parameters, public variables, etc. are
+    *     available via this object.
+    *
+    * @this {object} The global window object.
+    *
+    * @returns {Boolean}
+    *       true: Browser support playbackRate functionality.
+    *       false: Browser doesn't support playbackRate functionality.
+    */
+    function _checkPlaybackRates() {
+        var video = document.createElement('video');
+
+        // If browser supports, 1.0 should be returned by playbackRate property.
+        // In this case, function return True. Otherwise, False will be returned.
+        return Boolean(video.playbackRate);
+    }
+
+    // Hide speed control.
+    function _hideSpeedControl(state) {
+        state.el.find('div.speeds').hide();
+    }
+
+    /**
+     * @desc Bind any necessary function callbacks to DOM events (click,
+     *     mousemove, etc.).
+     *
+     * @type {function}
+     * @access private
+     *
+     * @param {object} state The object containg the state of the video player.
+     *     All other modules, their parameters, public variables, etc. are
+     *     available via this object.
+     *
+     * @this {object} The global window object.
+     *
+     * @returns {undefined}
+     */
     function _bindHandlers(state) {
+        var speedLinks;
+
         state.videoSpeedControl.videoSpeedsEl.find('a')
             .on('click', state.videoSpeedControl.changeVideoSpeed);
 
         if (onTouchBasedDevice()) {
             state.videoSpeedControl.el.on('click', function(event) {
+                // So that you can't highlight this control via a drag
+                // operation, we disable the default browser actions on a
+                // click event.
                 event.preventDefault();
-                $(this).toggleClass('open');
+
+                state.videoSpeedControl.el.toggleClass('open');
             });
         } else {
             state.videoSpeedControl.el
                 .on('mouseenter', function () {
-                    $(this).addClass('open');
+                    state.videoSpeedControl.el.addClass('open');
                 })
                 .on('mouseleave', function () {
-                    $(this).removeClass('open');
+                    state.videoSpeedControl.el.removeClass('open');
                 })
                 .on('click', function (event) {
+                    // So that you can't highlight this control via a drag
+                    // operation, we disable the default browser actions on a
+                    // click event.
                     event.preventDefault();
-                    $(this).removeClass('open');
+
+                    state.videoSpeedControl.el.removeClass('open');
                 });
 
+            // ******************************
+            // Attach 'focus', and 'blur' events to the speed button which
+            // either brings up the speed dialog with individual speed entries,
+            // or closes it.
             state.videoSpeedControl.el.children('a')
                 .on('focus', function () {
-                    $(this).parent().addClass('open');
+                    // If the focus is comming from the first speed entry, this
+                    // means we are tabbing backwards. In this case we have to
+                    // hide the speed entries which will allow us to change the
+                    // focus further backwards.
+                    if (state.firstSpeedBlur === true) {
+                        state.videoSpeedControl.el.removeClass('open');
+
+                        state.firstSpeedBlur = false;
+                    }
+
+                    // If the focus is comming from some other element, show
+                    // the drop down with the speed entries.
+                    else {
+                        state.videoSpeedControl.el.addClass('open');
+                    }
                 })
                 .on('blur', function () {
+                    // When the focus leaves this element, if the speed entries
+                    // dialog is shown (tabbing forwards), then we will set
+                    // focus to the first speed entry.
+                    //
+                    // If the selector does not select anything, then this
+                    // means that the speed entries dialog is closed, and we
+                    // are tabbing backwads. The browser will select the
+                    // previous element to tab to by itself.
                     state.videoSpeedControl.videoSpeedsEl
                         .find('a.speed_link:first')
                         .focus();
                 });
 
-            state.videoSpeedControl.videoSpeedsEl.find('a.speed_link:last')
-                .on('blur', function () {
-                    state.videoSpeedControl.el.removeClass('open');
-                });
+
+            // ******************************
+            // Attach 'focus', and 'blur' events to elements which represent
+            // individual speed entries.
+            speedLinks = state.videoSpeedControl.videoSpeedsEl
+                .find('a.speed_link');
+
+            speedLinks.last().on('blur', function () {
+                // If we have reached the last speed entry, and the focus
+                // changes to the next element, we need to hide the speeds
+                // control drop-down.
+                state.videoSpeedControl.el.removeClass('open');
+            });
+            speedLinks.first().on('blur', function () {
+                // This flag will indicate that the focus to the next
+                // element that will receive it is comming from the first
+                // speed entry.
+                //
+                // This flag will be used to correctly handle scenario of
+                // tabbing backwards.
+                state.firstSpeedBlur = true;
+            });
+            speedLinks.on('focus', function () {
+                // Clear the flag which is only set when we are un-focusing
+                // (the blur event) from the first speed entry.
+                state.firstSpeedBlur = false;
+            });
         }
     }
 
@@ -151,7 +256,7 @@ function () {
         $.each(this.videoSpeedControl.speeds, function(index, speed) {
             var link, listItem;
 
-            link = '<a href="#">' + speed + 'x</a>';
+            link = '<a class="speed_link" href="#">' + speed + 'x</a>';
 
             listItem = $('<li data-speed="' + speed + '">' + link + '</li>');
 
@@ -162,11 +267,9 @@ function () {
             _this.videoSpeedControl.videoSpeedsEl.prepend(listItem);
         });
 
-        this.videoSpeedControl.videoSpeedsEl.find('a')
-            .on('click', this.videoSpeedControl.changeVideoSpeed);
-
-        // TODO: After the control was re-rendered, we should attach 'focus'
-        // and 'blur' events once more.
+        // Re-attach all events with their appropriate callbacks to the
+        // newly generated elements.
+        _bindHandlers(this);
     }
 
 });

common/lib/xmodule/xmodule/js/src/video/09_video_caption.js

@@ -312,15 +312,34 @@ function () {
         var newIndex;
 
         if (this.videoCaption.loaded) {
-            time = Math.round(Time.convert(time, this.speed, '1.0') * 1000 + 250);
+            // Current mode === 'flash' can only be for YouTube videos. So, we
+            // don't have to also check for videoType === 'youtube'.
+            if (this.currentPlayerMode === 'flash') {
+                // Total play time changes with speed change. Also there is
+                // a 250 ms delay we have to take into account.
+                time = Math.round(
+                    Time.convert(time, this.speed, '1.0') * 1000 + 250
+                );
+            } else {
+                // Total play time remains constant when speed changes.
+                time = Math.round(parseInt(time, 10) * 1000);
+            }
+
             newIndex = this.videoCaption.search(time);
 
-            if (newIndex !== void 0 && this.videoCaption.currentIndex !== newIndex) {
+            if (
+                newIndex !== void 0 &&
+                this.videoCaption.currentIndex !== newIndex
+            ) {
                 if (this.videoCaption.currentIndex) {
-                    this.videoCaption.subtitlesEl.find('li.current').removeClass('current');
+                    this.videoCaption.subtitlesEl
+                        .find('li.current')
+                        .removeClass('current');
                 }
 
-                this.videoCaption.subtitlesEl.find("li[data-index='" + newIndex + "']").addClass('current');
+                this.videoCaption.subtitlesEl
+                    .find("li[data-index='" + newIndex + "']")
+                    .addClass('current');
 
                 this.videoCaption.currentIndex = newIndex;
 
@@ -333,9 +352,29 @@ function () {
         var time;
 
         event.preventDefault();
-        time = Math.round(Time.convert($(event.target).data('start'), '1.0', this.speed) / 1000);
 
-        this.trigger('videoPlayer.onCaptionSeek', {'type': 'onCaptionSeek', 'time': time});
+        // Current mode === 'flash' can only be for YouTube videos. So, we
+        // don't have to also check for videoType === 'youtube'.
+        if (this.currentPlayerMode === 'flash') {
+            // Total play time changes with speed change. Also there is
+            // a 250 ms delay we have to take into account.
+            time = Math.round(
+                Time.convert(
+                    $(event.target).data('start'), '1.0', this.speed
+                ) / 1000
+            );
+        } else {
+            // Total play time remains constant when speed changes.
+            time = parseInt($(event.target).data('start'), 10)/1000;
+        }
+
+        this.trigger(
+            'videoPlayer.onCaptionSeek',
+            {
+                'type': 'onCaptionSeek',
+                'time': time
+            }
+        );
     }
 
     function calculateOffset(element) {

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -14,6 +14,8 @@ from bson.son import SON
 
 log = logging.getLogger('mitx.' + 'modulestore')
 
+MONGO_MODULESTORE_TYPE = 'mongo'
+XML_MODULESTORE_TYPE = 'xml'
 
 URL_RE = re.compile("""
     (?P<tag>[^:]+)://?
@@ -235,8 +237,15 @@ class Location(_LocationBase):
 
     @property
     def course_id(self):
-        """Return the ID of the Course that this item belongs to by looking
-        at the location URL hierachy"""
+        """
+        Return the ID of the Course that this item belongs to by looking
+        at the location URL hierachy.
+
+        Throws an InvalidLocationError is this location does not represent a course.
+        """
+        if self.category != 'course':
+            raise InvalidLocationError('Cannot call course_id for {0} because it is not of category course'.format(self))
+
         return "/".join([self.org, self.course, self.name])
 
     def replace(self, **kwargs):
@@ -251,7 +260,7 @@ class ModuleStore(object):
     An abstract interface for a database backend that stores XModuleDescriptor
     instances
     """
-    def has_item(self, location):
+    def has_item(self, course_id, location):
         """
         Returns True if location exists in this ModuleStore.
         """
@@ -370,20 +379,26 @@ class ModuleStore(object):
         '''
         raise NotImplementedError
 
-    def get_containing_courses(self, location):
-        '''
-        Returns the list of courses that contains the specified location
+    def get_errored_courses(self):
+        """
+        Return a dictionary of course_dir -> [(msg, exception_str)], for each
+        course_dir where course loading failed.
+        """
+        raise NotImplementedError
 
-        TODO (cpennington): This should really take a module instance id,
-        rather than a location
+    def set_modulestore_configuration(self, config_dict):
         '''
-        courses = [
-            course
-            for course in self.get_courses()
-            if course.location.org == location.org and course.location.course == location.course
-        ]
+        Allows for runtime configuration of the modulestore. In particular this is how the
+        application (LMS/CMS) can pass down Django related configuration information, e.g. caches, etc.
+        '''
+        raise NotImplementedError
 
-        return courses
+    def get_modulestore_type(self, course_id):
+        """
+        Returns a type which identifies which modulestore is servicing the given
+        course_id. The return can be either "xml" (for XML based courses) or "mongo" for MongoDB backed courses
+        """
+        raise NotImplementedError
 
 
 class ModuleStoreBase(ModuleStore):
@@ -395,7 +410,7 @@ class ModuleStoreBase(ModuleStore):
         Set up the error-tracking logic.
         '''
         self._location_errors = {}  # location -> ErrorLog
-        self.metadata_inheritance_cache = None
+        self.modulestore_configuration = {}
         self.modulestore_update_signal = None  # can be set by runtime to route notifications of datastore changes
 
     def _get_errorlog(self, location):
@@ -424,6 +439,15 @@ class ModuleStoreBase(ModuleStore):
         errorlog = self._get_errorlog(location)
         return errorlog.errors
 
+    def get_errored_courses(self):
+        """
+        Returns an empty dict.
+
+        It is up to subclasses to extend this method if the concept
+        of errored courses makes sense for their implementation.
+        """
+        return {}
+
     def get_course(self, course_id):
         """Default impl--linear search through course list"""
         for c in self.get_courses():
@@ -431,6 +455,27 @@ class ModuleStoreBase(ModuleStore):
                 return c
         return None
 
+    @property
+    def metadata_inheritance_cache_subsystem(self):
+        """
+        Exposes an accessor to the runtime configuration for the metadata inheritance cache
+        """
+        return self.modulestore_configuration.get('metadata_inheritance_cache_subsystem', None)
+
+    @property
+    def request_cache(self):
+        """
+        Exposes an accessor to the runtime configuration for the request cache
+        """
+        return self.modulestore_configuration.get('request_cache', None)
+
+    def set_modulestore_configuration(self, config_dict):
+        """
+        This is the base implementation of the interface, all we need to do is store
+        two possible configurations as attributes on the class
+        """
+        self.modulestore_configuration = config_dict
+
 
 def namedtuple_to_son(namedtuple, prefix=''):
     """

common/lib/xmodule/xmodule/modulestore/django.py

@@ -25,24 +25,31 @@ def load_function(path):
     return getattr(import_module(module_path), name)
 
 
+def create_modulestore_instance(engine, options):
+    """
+    This will return a new instance of a modulestore given an engine and options
+    """
+    class_ = load_function(engine)
+
+    _options = {}
+    _options.update(options)
+
+    for key in FUNCTION_KEYS:
+        if key in _options and isinstance(_options[key], basestring):
+            _options[key] = load_function(_options[key])
+
+    return class_(
+        **_options
+    )
+
+
 def modulestore(name='default'):
+    """
+    This returns an instance of a modulestore of given name. This will wither return an existing
+    modulestore or create a new one
+    """
     if name not in _MODULESTORES:
-        class_ = load_function(settings.MODULESTORE[name]['ENGINE'])
-
-        options = {}
-
-        options.update(settings.MODULESTORE[name]['OPTIONS'])
-        for key in FUNCTION_KEYS:
-            if key in options:
-                options[key] = load_function(options[key])
-
-        _MODULESTORES[name] = class_(
-            **options
-        )
+        _MODULESTORES[name] = create_modulestore_instance(settings.MODULESTORE[name]['ENGINE'],
+                                                          settings.MODULESTORE[name]['OPTIONS'])
 
     return _MODULESTORES[name]
-
-# if 'DJANGO_SETTINGS_MODULE' in environ:
-#     # Initialize the modulestores immediately
-#     for store_name in settings.MODULESTORE:
-#         modulestore(store_name)

common/lib/xmodule/xmodule/modulestore/locator.py

@@ -1,8 +1,7 @@
 """
-Created on Mar 13, 2013
+Identifier for course resources.
+"""
 
-@author: dmitchell
-"""
 from __future__ import absolute_import
 import logging
 import inspect
@@ -15,6 +14,7 @@ from bson.errors import InvalidId
 from xmodule.modulestore.exceptions import InsufficientSpecificationError, OverSpecificationError
 
 from .parsers import parse_url, parse_course_id, parse_block_ref
+from .parsers import BRANCH_PREFIX, BLOCK_PREFIX, URL_VERSION_PREFIX
 
 log = logging.getLogger(__name__)
 
@@ -37,9 +37,6 @@ class Locator(object):
         """
         raise InsufficientSpecificationError()
 
-    def quoted_url(self):
-        return quote(self.url(), '@;#')
-
     def __eq__(self, other):
         return self.__dict__ == other.__dict__
 
@@ -90,11 +87,12 @@ class CourseLocator(Locator):
     Examples of valid CourseLocator specifications:
      CourseLocator(version_guid=ObjectId('519665f6223ebd6980884f2b'))
      CourseLocator(course_id='mit.eecs.6002x')
-     CourseLocator(course_id='mit.eecs.6002x;published')
+     CourseLocator(course_id='mit.eecs.6002x/branch/published')
      CourseLocator(course_id='mit.eecs.6002x', branch='published')
-     CourseLocator(url='edx://@519665f6223ebd6980884f2b')
+     CourseLocator(url='edx://version/519665f6223ebd6980884f2b')
      CourseLocator(url='edx://mit.eecs.6002x')
-     CourseLocator(url='edx://mit.eecs.6002x;published')
+     CourseLocator(url='edx://mit.eecs.6002x/branch/published')
+     CourseLocator(url='edx://mit.eecs.6002x/branch/published/version/519665f6223ebd6980884f2b')
 
     Should have at lease a specific course_id (id for the course as if it were a project w/
     versions) with optional 'branch',
@@ -115,10 +113,10 @@ class CourseLocator(Locator):
         if self.course_id:
             result = self.course_id
             if self.branch:
-                result += ';' + self.branch
+                result += BRANCH_PREFIX + self.branch
             return result
         elif self.version_guid:
-            return '@' + str(self.version_guid)
+            return URL_VERSION_PREFIX + str(self.version_guid)
         else:
             # raise InsufficientSpecificationError("missing course_id or version_guid")
             return '<InsufficientSpecificationError: missing course_id or version_guid>'
@@ -223,21 +221,18 @@ class CourseLocator(Locator):
     def init_from_url(self, url):
         """
         url must be a string beginning with 'edx://' and containing
-        either a valid version_guid or course_id (with optional branch)
-        If a block ('#HW3') is present, it is ignored.
+        either a valid version_guid or course_id (with optional branch), or both.
         """
         if isinstance(url, Locator):
             url = url.url()
-        assert isinstance(url, basestring), \
-            '%s is not an instance of basestring' % url
+        assert isinstance(url, basestring), '%s is not an instance of basestring' % url
         parse = parse_url(url)
         assert parse, 'Could not parse "%s" as a url' % url
-        if 'version_guid' in parse:
-            new_guid = parse['version_guid']
-            self.set_version_guid(self.as_object_id(new_guid))
-        else:
-            self.set_course_id(parse['id'])
-            self.set_branch(parse['branch'])
+        self._set_value(
+            parse, 'version_guid', lambda (new_guid): self.set_version_guid(self.as_object_id(new_guid))
+        )
+        self._set_value(parse, 'id', lambda (new_id): self.set_course_id(new_id))
+        self._set_value(parse, 'branch', lambda (new_branch): self.set_branch(new_branch))
 
     def init_from_version_guid(self, version_guid):
         """
@@ -253,14 +248,14 @@ class CourseLocator(Locator):
 
     def init_from_course_id(self, course_id, explicit_branch=None):
         """
-        Course_id is a string like 'mit.eecs.6002x' or 'mit.eecs.6002x;published'.
+        Course_id is a string like 'mit.eecs.6002x' or 'mit.eecs.6002x/branch/published'.
 
         Revision (optional) is a string like 'published'.
         It may be provided explicitly (explicit_branch) or embedded into course_id.
-        If branch is part of course_id ("...;published"), parse it out separately.
+        If branch is part of course_id (".../branch/published"), parse it out separately.
         If branch is provided both ways, that's ok as long as they are the same value.
 
-        If a block ('#HW3') is a part of course_id, it is ignored.
+        If a block ('/block/HW3') is a part of course_id, it is ignored.
 
         """
 
@@ -295,6 +290,16 @@ class CourseLocator(Locator):
         """
         return self.course_id
 
+    def _set_value(self, parse, key, setter):
+        """
+        Helper method that gets a value out of the dict returned by parse,
+        and then sets the corresponding bit of information in this locator
+        (via the supplied lambda 'setter'), unless the value is None.
+        """
+        value = parse.get(key, None)
+        if value:
+            setter(value)
+
 
 class BlockUsageLocator(CourseLocator):
     """
@@ -390,9 +395,7 @@ class BlockUsageLocator(CourseLocator):
             url = url.url()
         parse = parse_url(url)
         assert parse, 'Could not parse "%s" as a url' % url
-        block = parse.get('block', None)
-        if block:
-            self.set_usage_id(block)
+        self._set_value(parse, 'block', lambda(new_block): self.set_usage_id(new_block))
 
     def init_block_ref_from_course_id(self, course_id):
         if isinstance(course_id, CourseLocator):
@@ -400,9 +403,7 @@ class BlockUsageLocator(CourseLocator):
             assert course_id, "%s does not have a valid course_id"
         parse = parse_course_id(course_id)
         assert parse, 'Could not parse "%s" as a course_id' % course_id
-        block = parse.get('block', None)
-        if block:
-            self.set_usage_id(block)
+        self._set_value(parse, 'block', lambda(new_block): self.set_usage_id(new_block))
 
     def __unicode__(self):
         """
@@ -411,14 +412,14 @@ class BlockUsageLocator(CourseLocator):
         rep = CourseLocator.__unicode__(self)
         if self.usage_id is None:
             # usage_id has not been initialized
-            return rep + '#NONE'
+            return rep + BLOCK_PREFIX + 'NONE'
         else:
-            return rep + '#' + self.usage_id
+            return rep + BLOCK_PREFIX + self.usage_id
 
 
 class DescriptionLocator(Locator):
     """
-    Container for how to locate a description
+    Container for how to locate a description (the course-independent content).
     """
 
     def __init__(self, definition_id):
@@ -427,14 +428,14 @@ class DescriptionLocator(Locator):
     def __unicode__(self):
         '''
         Return a string representing this location.
-        unicode(self) returns something like this: "@519665f6223ebd6980884f2b"
+        unicode(self) returns something like this: "version/519665f6223ebd6980884f2b"
         '''
-        return '@' + str(self.definition_guid)
+        return URL_VERSION_PREFIX + str(self.definition_id)
 
     def url(self):
         """
         Return a string containing the URL for this location.
-        url(self) returns something like this: 'edx://@519665f6223ebd6980884f2b'
+        url(self) returns something like this: 'edx://version/519665f6223ebd6980884f2b'
         """
         return 'edx://' + unicode(self)
 
@@ -442,7 +443,7 @@ class DescriptionLocator(Locator):
         """
         Returns the ObjectId referencing this specific location.
         """
-        return self.definition_guid
+        return self.definition_id
 
 
 class VersionTree(object):

common/lib/xmodule/xmodule/modulestore/mixed.py

@@ -0,0 +1,158 @@
+"""
+MixedModuleStore allows for aggregation between multiple modulestores.
+
+In this way, courses can be served up both - say - XMLModuleStore or MongoModuleStore
+
+IMPORTANT: This modulestore only supports READONLY applications, e.g. LMS
+"""
+
+from . import ModuleStoreBase
+from django import create_modulestore_instance
+import logging
+
+log = logging.getLogger(__name__)
+
+
+class MixedModuleStore(ModuleStoreBase):
+    """
+    ModuleStore that can be backed by either XML or Mongo
+    """
+    def __init__(self, mappings, stores):
+        """
+        Initialize a MixedModuleStore. Here we look into our passed in kwargs which should be a
+        collection of other modulestore configuration informations
+        """
+        super(MixedModuleStore, self).__init__()
+
+        self.modulestores = {}
+        self.mappings = mappings
+        if 'default' not in stores:
+            raise Exception('Missing a default modulestore in the MixedModuleStore __init__ method.')
+
+        for key in stores:
+            self.modulestores[key] = create_modulestore_instance(stores[key]['ENGINE'],
+                                                                 stores[key]['OPTIONS'])
+
+    def _get_modulestore_for_courseid(self, course_id):
+        """
+        For a given course_id, look in the mapping table and see if it has been pinned
+        to a particular modulestore
+        """
+        mapping = self.mappings.get(course_id, 'default')
+        return self.modulestores[mapping]
+
+    def has_item(self, course_id, location):
+        return self._get_modulestore_for_courseid(course_id).has_item(course_id, location)
+
+    def get_item(self, location, depth=0):
+        """
+        This method is explicitly not implemented as we need a course_id to disambiguate
+        We should be able to fix this when the data-model rearchitecting is done
+        """
+        raise NotImplementedError
+
+    def get_instance(self, course_id, location, depth=0):
+        return self._get_modulestore_for_courseid(course_id).get_instance(course_id, location, depth)
+
+    def get_items(self, location, course_id=None, depth=0):
+        """
+        Returns a list of XModuleDescriptor instances for the items
+        that match location. Any element of location that is None is treated
+        as a wildcard that matches any value
+
+        location: Something that can be passed to Location
+
+        depth: An argument that some module stores may use to prefetch
+            descendents of the queried modules for more efficient results later
+            in the request. The depth is counted in the number of calls to
+            get_children() to cache. None indicates to cache all descendents
+        """
+        if not course_id:
+            raise Exception("Must pass in a course_id when calling get_items() with MixedModuleStore")
+
+        return self._get_modulestore_for_courseid(course_id).get_items(location, course_id, depth)
+
+    def update_item(self, location, data, allow_not_found=False):
+        """
+        MixedModuleStore is for read-only (aka LMS)
+        """
+        raise NotImplementedError
+
+    def update_children(self, location, children):
+        """
+        MixedModuleStore is for read-only (aka LMS)
+        """
+        raise NotImplementedError
+
+    def update_metadata(self, location, metadata):
+        """
+        MixedModuleStore is for read-only (aka LMS)
+        """
+        raise NotImplementedError
+
+    def delete_item(self, location):
+        """
+        MixedModuleStore is for read-only (aka LMS)
+        """
+        raise NotImplementedError
+
+    def get_courses(self):
+        '''
+        Returns a list containing the top level XModuleDescriptors of the courses
+        in this modulestore.
+        '''
+        courses = []
+        for key in self.modulestores:
+            store_courses = self.modulestores[key].get_courses()
+            # If the store has not been labeled as 'default' then we should
+            # only surface courses that have a mapping entry, for example the XMLModuleStore will
+            # slurp up anything that is on disk, however, we don't want to surface those to
+            # consumers *unless* there is an explicit mapping in the configuration
+            if key != 'default':
+                for course in store_courses:
+                    # make sure that the courseId is mapped to the store in question
+                    if key == self.mappings.get(course.location.course_id, 'default'):
+                        courses = courses + ([course])
+            else:
+                # if we're the 'default' store provider, then we surface all courses hosted in
+                # that store provider
+                courses = courses + (store_courses)
+
+        return courses
+
+    def get_course(self, course_id):
+        """
+        returns the course module associated with the course_id
+        """
+        return self._get_modulestore_for_courseid(course_id).get_course(course_id)
+
+    def get_parent_locations(self, location, course_id):
+        """
+        returns the parent locations for a given lcoation and course_id
+        """
+        return self._get_modulestore_for_courseid(course_id).get_parent_locations(location, course_id)
+
+    def set_modulestore_configuration(self, config_dict):
+        """
+        This implementation of the interface method will pass along the configuration to all ModuleStore
+        instances
+        """
+        for store in self.modulestores.values():
+            store.set_modulestore_configuration(config_dict)
+
+    def get_modulestore_type(self, course_id):
+        """
+        Returns a type which identifies which modulestore is servicing the given
+        course_id. The return can be either "xml" (for XML based courses) or "mongo" for MongoDB backed courses
+        """
+        return self._get_modulestore_for_courseid(course_id).get_modulestore_type(course_id)
+
+    def get_errored_courses(self):
+        """
+        Return a dictionary of course_dir -> [(msg, exception_str)], for each
+        course_dir where course loading failed.
+        """
+        errs = {}
+        for store in self.modulestores.values():
+            errs.update(store.get_errored_courses())
+        return errs

common/lib/xmodule/xmodule/modulestore/mongo/base.py

@@ -32,7 +32,7 @@ from xmodule.error_module import ErrorDescriptor
 from xblock.runtime import DbModel, KeyValueStore, InvalidScopeError
 from xblock.core import Scope
 
-from xmodule.modulestore import ModuleStoreBase, Location, namedtuple_to_son
+from xmodule.modulestore import ModuleStoreBase, Location, namedtuple_to_son, MONGO_MODULESTORE_TYPE
 from xmodule.modulestore.exceptions import ItemNotFoundError
 from xmodule.modulestore.inheritance import own_metadata, INHERITABLE_METADATA, inherit_metadata
 
@@ -270,8 +270,7 @@ class MongoModuleStore(ModuleStoreBase):
     def __init__(self, host, db, collection, fs_root, render_template,
                  port=27017, default_class=None,
                  error_tracker=null_error_tracker,
-                 user=None, password=None, request_cache=None,
-                 metadata_inheritance_cache_subsystem=None, **kwargs):
+                 user=None, password=None, **kwargs):
 
         super(MongoModuleStore, self).__init__()
 
@@ -303,8 +302,6 @@ class MongoModuleStore(ModuleStoreBase):
         self.error_tracker = error_tracker
         self.render_template = render_template
         self.ignore_write_events_on_courses = []
-        self.request_cache = request_cache
-        self.metadata_inheritance_cache_subsystem = metadata_inheritance_cache_subsystem
 
     def compute_metadata_inheritance_tree(self, location):
         '''
@@ -547,7 +544,7 @@ class MongoModuleStore(ModuleStoreBase):
             raise ItemNotFoundError(location)
         return item
 
-    def has_item(self, location):
+    def has_item(self, course_id, location):
         """
         Returns True if location exists in this ModuleStore.
         """
@@ -681,7 +678,7 @@ class MongoModuleStore(ModuleStoreBase):
         # we should remove this once we can break this reference from the course to static tabs
         # TODO move this special casing to app tier (similar to attaching new element to parent)
         if location.category == 'static_tab':
-            course = self.get_course_for_item(location)
+            course = self._get_course_for_item(location)
             existing_tabs = course.tabs or []
             existing_tabs.append({
                 'type': 'static_tab',
@@ -701,7 +698,7 @@ class MongoModuleStore(ModuleStoreBase):
             self.modulestore_update_signal.send(self, modulestore=self, course_id=course_id,
                                                 location=location)
 
-    def get_course_for_item(self, location, depth=0):
+    def _get_course_for_item(self, location, depth=0):
         '''
         VS[compat]
         cdodge: for a given Xmodule, return the course that it belongs to
@@ -790,7 +787,7 @@ class MongoModuleStore(ModuleStoreBase):
         # we should remove this once we can break this reference from the course to static tabs
         loc = Location(location)
         if loc.category == 'static_tab':
-            course = self.get_course_for_item(loc)
+            course = self._get_course_for_item(loc)
             existing_tabs = course.tabs or []
             for tab in existing_tabs:
                 if tab.get('url_slug') == loc.name:
@@ -818,7 +815,7 @@ class MongoModuleStore(ModuleStoreBase):
         # we should remove this once we can break this reference from the course to static tabs
         if location.category == 'static_tab':
             item = self.get_item(location)
-            course = self.get_course_for_item(item.location)
+            course = self._get_course_for_item(item.location)
             existing_tabs = course.tabs or []
             course.tabs = [tab for tab in existing_tabs if tab.get('url_slug') != location.name]
             # Save the updates to the course to the MongoKeyValueStore
@@ -841,12 +838,12 @@ class MongoModuleStore(ModuleStoreBase):
                                      {'_id': True})
         return [i['_id'] for i in items]
 
-    def get_errored_courses(self):
+    def get_modulestore_type(self, course_id):
         """
-        This function doesn't make sense for the mongo modulestore, as courses
-        are loaded on demand, rather than up front
+        Returns a type which identifies which modulestore is servicing the given
+        course_id. The return can be either "xml" (for XML based courses) or "mongo" for MongoDB backed courses
         """
-        return {}
+        return MONGO_MODULESTORE_TYPE
 
     def _create_new_model_data(self, category, location, definition_data, metadata):
         """
@@ -861,9 +858,9 @@ class MongoModuleStore(ModuleStoreBase):
         )
 
         class_ = XModuleDescriptor.load_class(
-                    category,
-                    self.default_class
-                )
+            category,
+            self.default_class
+        )
         model_data = DbModel(kvs, class_, None, MongoUsage(None, location))
         model_data['category'] = category
         model_data['location'] = location

common/lib/xmodule/xmodule/modulestore/parsers.py

@@ -1,5 +1,14 @@
 import re
 
+# Prefix for the branch portion of a locator URL
+BRANCH_PREFIX = "/branch/"
+# Prefix for the block portion of a locator URL
+BLOCK_PREFIX = "/block/"
+# Prefix for the version portion of a locator URL, when it is preceded by a course ID
+VERSION_PREFIX = "/version/"
+# Prefix for version when it begins the URL (no course ID).
+URL_VERSION_PREFIX = 'version/'
+
 URL_RE = re.compile(r'^edx://(.+)$', re.IGNORECASE)
 
 
@@ -9,26 +18,27 @@ def parse_url(string):
     followed by either a version_guid or a course_id.
 
     Examples:
-        'edx://@0123FFFF'
+        'edx://version/0123FFFF'
         'edx://edu.mit.eecs.6002x'
-        'edx://edu.mit.eecs.6002x;published'
-        'edx://edu.mit.eecs.6002x;published#HW3'
+        'edx://edu.mit.eecs.6002x/branch/published'
+        'edx://edu.mit.eecs.6002x/branch/published/version/519665f6223ebd6980884f2b/block/HW3'
+        'edx://edu.mit.eecs.6002x/branch/published/block/HW3'
 
     This returns None if string cannot be parsed.
 
-    If it can be parsed as a version_guid, returns a dict
+    If it can be parsed as a version_guid with no preceding course_id, returns a dict
     with key 'version_guid' and the value,
 
     If it can be parsed as a course_id, returns a dict
-    with keys 'id' and 'branch' (value of 'branch' may be None),
+    with key 'id' and optional keys 'branch' and 'version_guid'.
 
     """
     match = URL_RE.match(string)
     if not match:
         return None
     path = match.group(1)
-    if path[0] == '@':
-        return parse_guid(path[1:])
+    if path.startswith(URL_VERSION_PREFIX):
+        return parse_guid(path[len(URL_VERSION_PREFIX):])
     return parse_course_id(path)
 
 
@@ -52,7 +62,7 @@ def parse_block_ref(string):
     return None
 
 
-GUID_RE = re.compile(r'^(?P<version_guid>[A-F0-9]+)(#(?P<block>\w+))?$', re.IGNORECASE)
+GUID_RE = re.compile(r'^(?P<version_guid>[A-F0-9]+)(' + BLOCK_PREFIX + '(?P<block>\w+))?$', re.IGNORECASE)
 
 
 def parse_guid(string):
@@ -69,27 +79,34 @@ def parse_guid(string):
         return None
 
 
-COURSE_ID_RE = re.compile(r'^(?P<id>(\w+)(\.\w+\w*)*)(;(?P<branch>\w+))?(#(?P<block>\w+))?$', re.IGNORECASE)
+COURSE_ID_RE = re.compile(
+    r'^(?P<id>(\w+)(\.\w+\w*)*)(' +
+    BRANCH_PREFIX + '(?P<branch>\w+))?(' +
+    VERSION_PREFIX + '(?P<version_guid>[A-F0-9]+))?(' +
+    BLOCK_PREFIX + '(?P<block>\w+))?$', re.IGNORECASE
+)
 
 
 def parse_course_id(string):
     r"""
 
     A course_id has a main id component.
-    There may also be an optional branch (;published or ;draft).
-    There may also be an optional block (#HW3 or #Quiz2).
+    There may also be an optional branch (/branch/published or /branch/draft).
+    There may also be an optional version (/version/519665f6223ebd6980884f2b).
+    There may also be an optional block (/block/HW3 or /block/Quiz2).
 
     Examples of valid course_ids:
 
       'edu.mit.eecs.6002x'
-      'edu.mit.eecs.6002x;published'
-      'edu.mit.eecs.6002x#HW3'
-      'edu.mit.eecs.6002x;published#HW3'
+      'edu.mit.eecs.6002x/branch/published'
+      'edu.mit.eecs.6002x/block/HW3'
+      'edu.mit.eecs.6002x/branch/published/block/HW3'
+      'edu.mit.eecs.6002x/branch/published/version/519665f6223ebd6980884f2b/block/HW3'
 
 
     Syntax:
 
-      course_id = main_id [; branch] [# block]
+      course_id = main_id [/branch/ branch] [/version/ version ] [/block/ block]
 
       main_id = name [. name]*
 

common/lib/xmodule/xmodule/modulestore/search.py

@@ -81,7 +81,7 @@ def path_to_location(modulestore, course_id, location):
         # If we're here, there is no path
         return None
 
-    if not modulestore.has_item(location):
+    if not modulestore.has_item(course_id, location):
         raise ItemNotFoundError
 
     path = find_path_to_course()

common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py

@@ -11,18 +11,17 @@ from .split_mongo_kvs import SplitMongoKVS, SplitMongoKVSid
 
 log = logging.getLogger(__name__)
 
-# TODO should this be here or w/ x_module or ???
 class CachingDescriptorSystem(MakoDescriptorSystem):
     """
     A system that has a cache of a course version's json that it will use to load modules
     from, with a backup of calling to the underlying modulestore for more data.
 
-    Computes the metadata inheritance upon creation.
+    Computes the settings (nee 'metadata') inheritance upon creation.
     """
     def __init__(self, modulestore, course_entry, module_data, lazy,
         default_class, error_tracker, render_template):
         """
-        Computes the metadata inheritance and sets up the cache.
+        Computes the settings inheritance and sets up the cache.
 
         modulestore: the module store that can be used to retrieve additional
         modules
@@ -50,9 +49,10 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
         self.default_class = default_class
         # TODO see if self.course_id is needed: is already in course_entry but could be > 1 value
         # Compute inheritance
-        modulestore.inherit_metadata(course_entry.get('blocks', {}),
-            course_entry.get('blocks', {})
-            .get(course_entry.get('root')))
+        modulestore.inherit_settings(
+            course_entry.get('blocks', {}),
+            course_entry.get('blocks', {}).get(course_entry.get('root'))
+        )
 
     def _load_item(self, usage_id, course_entry_override=None):
         # TODO ensure all callers of system.load_item pass just the id
@@ -73,9 +73,8 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
     def xblock_from_json(self, class_, usage_id, json_data, course_entry_override=None):
         if course_entry_override is None:
             course_entry_override = self.course_entry
-        # most likely a lazy loader but not the id directly
+        # most likely a lazy loader or the id directly
         definition = json_data.get('definition', {})
-        metadata = json_data.get('metadata', {})
 
         block_locator = BlockUsageLocator(
             version_guid=course_entry_override['_id'],
@@ -86,9 +85,8 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
 
         kvs = SplitMongoKVS(
             definition,
-            json_data.get('children', []),
-            metadata,
-            json_data.get('_inherited_metadata'),
+            json_data.get('fields', {}),
+            json_data.get('_inherited_settings'),
             block_locator,
             json_data.get('category'))
         model_data = DbModel(kvs, class_, None,
@@ -111,10 +109,11 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
                 error_msg=exc_info_to_str(sys.exc_info())
             )
 
-        module.edited_by = json_data.get('edited_by')
-        module.edited_on = json_data.get('edited_on')
-        module.previous_version = json_data.get('previous_version')
-        module.update_version = json_data.get('update_version')
+        edit_info = json_data.get('edit_info', {})
+        module.edited_by = edit_info.get('edited_by')
+        module.edited_on = edit_info.get('edited_on')
+        module.previous_version = edit_info.get('previous_version')
+        module.update_version = edit_info.get('update_version')
         module.definition_locator = self.modulestore.definition_locator(definition)
         # decache any pending field settings
         module.save()

common/lib/xmodule/xmodule/modulestore/split_mongo/split.py

@@ -16,6 +16,9 @@ from .. import ModuleStoreBase
 from ..exceptions import ItemNotFoundError
 from .definition_lazy_loader import DefinitionLazyLoader
 from .caching_descriptor_system import CachingDescriptorSystem
+from xblock.core import Scope
+from pytz import UTC
+import collections
 
 log = logging.getLogger(__name__)
 #==============================================================================
@@ -102,10 +105,12 @@ class SplitMongoModuleStore(ModuleStoreBase):
         '''
         new_module_data = {}
         for usage_id in base_usage_ids:
-            new_module_data = self.descendants(system.course_entry['blocks'],
-                                               usage_id,
-                                               depth,
-                                               new_module_data)
+            new_module_data = self.descendants(
+                system.course_entry['blocks'],
+                usage_id,
+                depth,
+                new_module_data
+            )
 
         # remove any which were already in module_data (not sure if there's a better way)
         for newkey in new_module_data.iterkeys():
@@ -114,8 +119,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
 
         if lazy:
             for block in new_module_data.itervalues():
-                block['definition'] = DefinitionLazyLoader(self,
-                                                           block['definition'])
+                block['definition'] = DefinitionLazyLoader(self, block['definition'])
         else:
             # Load all descendants by id
             descendent_definitions = self.definitions.find({
@@ -127,7 +131,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
 
             for block in new_module_data.itervalues():
                 if block['definition'] in definitions:
-                    block['definition'] = definitions[block['definition']]
+                    block['fields'].update(definitions[block['definition']].get('fields'))
 
         system.module_data.update(new_module_data)
         return system.module_data
@@ -226,7 +230,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
             entry['branch'] = course_locator.branch
         return entry
 
-    def get_courses(self, branch, qualifiers=None):
+    def get_courses(self, branch='published', qualifiers=None):
         '''
         Returns a list of course descriptors matching any given qualifiers.
 
@@ -235,6 +239,9 @@ class SplitMongoModuleStore(ModuleStoreBase):
 
         Note, this is to find the current head of the named branch type
         (e.g., 'draft'). To get specific versions via guid use get_course.
+
+        :param branch: the branch for which to return courses. Default value is 'published'.
+        :param qualifiers: a optional dict restricting which elements should match
         '''
         if qualifiers is None:
             qualifiers = {}
@@ -279,7 +286,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
         '''
         return self.get_course(location)
 
-    def has_item(self, block_location):
+    def has_item(self, course_id, block_location):
         """
         Returns True if location exists in its course. Returns false if
         the course or the block w/in the course do not exist for the given version.
@@ -313,16 +320,15 @@ class SplitMongoModuleStore(ModuleStoreBase):
             raise ItemNotFoundError(location)
         return items[0]
 
-    # TODO refactor this and get_courses to use a constructed query
-    def get_items(self, locator, qualifiers):
-        '''
+    def get_items(self, locator, course_id=None, depth=0, qualifiers=None):
+        """
         Get all of the modules in the given course matching the qualifiers. The
         qualifiers should only be fields in the structures collection (sorry).
         There will be a separate search method for searching through
         definitions.
 
         Common qualifiers are category, definition (provide definition id),
-        metadata: {display_name ..}, children (return
+        display_name, anyfieldname, children (return
         block if its children includes the one given value). If you want
         substring matching use {$regex: /acme.*corp/i} type syntax.
 
@@ -331,9 +337,14 @@ class SplitMongoModuleStore(ModuleStoreBase):
         try arbitrary queries.
 
         :param locator: CourseLocator or BlockUsageLocator restricting search scope
+        :param course_id: ignored. Only included for API compatibility.
+        :param depth: ignored. Only included for API compatibility.
         :param qualifiers: a dict restricting which elements should match
-        '''
+
+        """
         # TODO extend to only search a subdag of the course?
+        if qualifiers is None:
+            qualifiers = {}
         course = self._lookup_course(locator)
         items = []
         for usage_id, value in course['blocks'].iteritems():
@@ -345,23 +356,35 @@ class SplitMongoModuleStore(ModuleStoreBase):
         else:
             return []
 
-    # What's the use case for usage_id being separate?
+    def get_instance(self, course_id, location, depth=0):
+        """
+        Get an instance of this location.
+
+        For now, just delegate to get_item and ignore course policy.
+
+        depth (int): An argument that some module stores may use to prefetch
+            descendants of the queried modules for more efficient results later
+            in the request. The depth is counted in the number of
+            calls to get_children() to cache. None indicates to cache all descendants.
+        """
+        return self.get_item(location, depth=depth)
+
     def get_parent_locations(self, locator, usage_id=None):
         '''
         Return the locations (Locators w/ usage_ids) for the parents of this location in this
         course. Could use get_items(location, {'children': usage_id}) but this is slightly faster.
-        NOTE: does not actually ensure usage_id exists
-        If usage_id is None, then the locator must specify the usage_id
+        NOTE: the locator must contain the usage_id, and this code does not actually ensure usage_id exists
+
+        :param locator: BlockUsageLocator restricting search scope
+        :param usage_id: ignored. Only included for API compatibility. Specify the usage_id within the locator.
         '''
-        if usage_id is None:
-            usage_id = locator.usage_id
+
         course = self._lookup_course(locator)
         items = []
         for parent_id, value in course['blocks'].iteritems():
-            for child_id in value['children']:
-                if usage_id == child_id:
-                    locator = locator.as_course_locator()
-                    items.append(BlockUsageLocator(url=locator, usage_id=parent_id))
+            for child_id in value['fields'].get('children', []):
+                if locator.usage_id == child_id:
+                    items.append(BlockUsageLocator(url=locator.as_course_locator(), usage_id=parent_id))
         return items
 
     def get_course_index_info(self, course_locator):
@@ -415,11 +438,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
         definition = self.definitions.find_one({'_id': definition_locator.definition_id})
         if definition is None:
             return None
-        return {'original_version': definition['original_version'],
-            'previous_version': definition['previous_version'],
-            'edited_by': definition['edited_by'],
-            'edited_on': definition['edited_on']
-        }
+        return definition['edit_info']
 
     def get_course_successors(self, course_locator, version_history_depth=1):
         '''
@@ -459,29 +478,29 @@ class SplitMongoModuleStore(ModuleStoreBase):
         Find the history of this block. Return as a VersionTree of each place the block changed (except
         deletion).
 
-        The block's history tracks its explicit changes; so, changes in descendants won't be reflected
-        as new iterations.
+        The block's history tracks its explicit changes but not the changes in its children.
+
         '''
         block_locator = block_locator.version_agnostic()
         course_struct = self._lookup_course(block_locator)
         usage_id = block_locator.usage_id
-        update_version_field = 'blocks.{}.update_version'.format(usage_id)
+        update_version_field = 'blocks.{}.edit_info.update_version'.format(usage_id)
         all_versions_with_block = self.structures.find({'original_version': course_struct['original_version'],
             update_version_field: {'$exists': True}})
         # find (all) root versions and build map previous: [successors]
         possible_roots = []
         result = {}
         for version in all_versions_with_block:
-            if version['_id'] == version['blocks'][usage_id]['update_version']:
-                if version['blocks'][usage_id].get('previous_version') is None:
-                    possible_roots.append(version['blocks'][usage_id]['update_version'])
+            if version['_id'] == version['blocks'][usage_id]['edit_info']['update_version']:
+                if version['blocks'][usage_id]['edit_info'].get('previous_version') is None:
+                    possible_roots.append(version['blocks'][usage_id]['edit_info']['update_version'])
                 else:
-                    result.setdefault(version['blocks'][usage_id]['previous_version'], set()).add(
-                        version['blocks'][usage_id]['update_version'])
+                    result.setdefault(version['blocks'][usage_id]['edit_info']['previous_version'], set()).add(
+                        version['blocks'][usage_id]['edit_info']['update_version'])
         # more than one possible_root means usage was added and deleted > 1x.
         if len(possible_roots) > 1:
             # find the history segment including block_locator's version
-            element_to_find = course_struct['blocks'][usage_id]['update_version']
+            element_to_find = course_struct['blocks'][usage_id]['edit_info']['update_version']
             if element_to_find in possible_roots:
                 possible_roots = [element_to_find]
             for possibility in possible_roots:
@@ -501,7 +520,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
         Find the version_history_depth next versions of this definition. Return as a VersionTree
         '''
         # TODO implement
-        pass
+        raise NotImplementedError()
 
     def create_definition_from_data(self, new_def_data, category, user_id):
         """
@@ -510,16 +529,21 @@ class SplitMongoModuleStore(ModuleStoreBase):
 
         :param user_id: request.user object
         """
-        document = {"category" : category,
-            "data": new_def_data,
-            "edited_by": user_id,
-            "edited_on": datetime.datetime.utcnow(),
-            "previous_version": None,
-            "original_version": None}
+        new_def_data = self._filter_special_fields(new_def_data)
+        document = {
+            "category" : category,
+            "fields": new_def_data,
+            "edit_info": {
+                "edited_by": user_id,
+                "edited_on": datetime.datetime.now(UTC),
+                "previous_version": None,
+                "original_version": None
+                }
+            }
         new_id = self.definitions.insert(document)
         definition_locator = DescriptionLocator(new_id)
-        document['original_version'] = new_id
-        self.definitions.update({'_id': new_id}, {'$set': {"original_version": new_id}})
+        document['edit_info']['original_version'] = new_id
+        self.definitions.update({'_id': new_id}, {'$set': {"edit_info.original_version": new_id}})
         return definition_locator
 
     def update_definition_from_data(self, definition_locator, new_def_data, user_id):
@@ -529,16 +553,14 @@ class SplitMongoModuleStore(ModuleStoreBase):
 
         :param user_id: request.user
         """
+        new_def_data = self._filter_special_fields(new_def_data)
         def needs_saved():
-            if isinstance(new_def_data, dict):
-                for key, value in new_def_data.iteritems():
-                    if key not in old_definition['data'] or value != old_definition['data'][key]:
-                        return True
-                for key, value in old_definition['data'].iteritems():
-                    if key not in new_def_data:
-                        return True
-            else:
-                return new_def_data != old_definition['data']
+            for key, value in new_def_data.iteritems():
+                if key not in old_definition['fields'] or value != old_definition['fields'][key]:
+                    return True
+            for key, value in old_definition.get('fields', {}).iteritems():
+                if key not in new_def_data:
+                    return True
 
         # if this looks in cache rather than fresh fetches, then it will probably not detect
         # actual change b/c the descriptor and cache probably point to the same objects
@@ -548,10 +570,10 @@ class SplitMongoModuleStore(ModuleStoreBase):
         del old_definition['_id']
 
         if needs_saved():
-            old_definition['data'] = new_def_data
-            old_definition['edited_by'] = user_id
-            old_definition['edited_on'] = datetime.datetime.utcnow()
-            old_definition['previous_version'] = definition_locator.definition_id
+            old_definition['fields'] = new_def_data
+            old_definition['edit_info']['edited_by'] = user_id
+            old_definition['edit_info']['edited_on'] = datetime.datetime.now(UTC)
+            old_definition['edit_info']['previous_version'] = definition_locator.definition_id
             new_id = self.definitions.insert(old_definition)
             return DescriptionLocator(new_id), True
         else:
@@ -593,11 +615,11 @@ class SplitMongoModuleStore(ModuleStoreBase):
         else:
             return id_root
 
-    # TODO I would love to write this to take a real descriptor and persist it BUT descriptors, kvs, and dbmodel
-    #      all assume locators are set and unique! Having this take the model contents piecemeal breaks the separation
-    #      of model from persistence layer
-    def create_item(self, course_or_parent_locator, category, user_id, definition_locator=None, new_def_data=None,
-        metadata=None, force=False):
+    # TODO Should I rewrite this to take a new xblock instance rather than to construct it? That is, require the
+    # caller to use XModuleDescriptor.load_from_json thus reducing similar code and making the object creation and
+    # validation behavior a responsibility of the model layer rather than the persistence layer.
+    def create_item(self, course_or_parent_locator, category, user_id, definition_locator=None, fields=None,
+        force=False):
         """
         Add a descriptor to persistence as the last child of the optional parent_location or just as an element
         of the course (if no parent provided). Return the resulting post saved version with populated locators.
@@ -612,9 +634,10 @@ class SplitMongoModuleStore(ModuleStoreBase):
 
         The incoming definition_locator should either be None to indicate this is a brand new definition or
         a pointer to the existing definition to which this block should point or from which this was derived.
-        If new_def_data is None, then definition_locator must have a value meaning that this block points
-        to the existing definition. If new_def_data is not None and definition_location is not None, then
-        new_def_data is assumed to be a new payload for definition_location.
+        If fields does not contain any Scope.content, then definition_locator must have a value meaning that this
+        block points
+        to the existing definition. If fields contains Scope.content and definition_locator is not None, then
+        the Scope.content fields are assumed to be a new payload for definition_locator.
 
         Creates a new version of the course structure, creates and inserts the new block, makes the block point
         to the definition which may be new or a new version of an existing or an existing.
@@ -633,6 +656,8 @@ class SplitMongoModuleStore(ModuleStoreBase):
         index_entry = self._get_index_if_valid(course_or_parent_locator, force)
         structure = self._lookup_course(course_or_parent_locator)
 
+        partitioned_fields = self._partition_fields_by_scope(category, fields)
+        new_def_data = partitioned_fields.get(Scope.content, {})
         # persist the definition if persisted != passed
         if (definition_locator is None or definition_locator.definition_id is None):
             definition_locator = self.create_definition_from_data(new_def_data, category, user_id)
@@ -643,23 +668,27 @@ class SplitMongoModuleStore(ModuleStoreBase):
         new_structure = self._version_structure(structure, user_id)
         # generate an id
         new_usage_id = self._generate_usage_id(new_structure['blocks'], category)
-        update_version_keys = ['blocks.{}.update_version'.format(new_usage_id)]
+        update_version_keys = ['blocks.{}.edit_info.update_version'.format(new_usage_id)]
         if isinstance(course_or_parent_locator, BlockUsageLocator) and course_or_parent_locator.usage_id is not None:
             parent = new_structure['blocks'][course_or_parent_locator.usage_id]
-            parent['children'].append(new_usage_id)
-            parent['edited_on'] = datetime.datetime.utcnow()
-            parent['edited_by'] = user_id
-            parent['previous_version'] = parent['update_version']
-            update_version_keys.append('blocks.{}.update_version'.format(course_or_parent_locator.usage_id))
+            parent['fields'].setdefault('children', []).append(new_usage_id)
+            parent['edit_info']['edited_on'] = datetime.datetime.now(UTC)
+            parent['edit_info']['edited_by'] = user_id
+            parent['edit_info']['previous_version'] = parent['edit_info']['update_version']
+            update_version_keys.append('blocks.{}.edit_info.update_version'.format(course_or_parent_locator.usage_id))
+        block_fields = partitioned_fields.get(Scope.settings, {})
+        if Scope.children in partitioned_fields:
+            block_fields.update(partitioned_fields[Scope.children])
         new_structure['blocks'][new_usage_id] = {
-            "children": [],
             "category": category,
             "definition": definition_locator.definition_id,
-            "metadata": metadata if metadata else {},
-            'edited_on': datetime.datetime.utcnow(),
-            'edited_by': user_id,
-            'previous_version': None
+            "fields": block_fields,
+            'edit_info': {
+                'edited_on': datetime.datetime.now(UTC),
+                'edited_by': user_id,
+                'previous_version': None
             }
+        }
         new_id = self.structures.insert(new_structure)
         update_version_payload = {key: new_id for key in update_version_keys}
         self.structures.update({'_id': new_id},
@@ -677,8 +706,9 @@ class SplitMongoModuleStore(ModuleStoreBase):
                                                usage_id=new_usage_id,
                                                version_guid=new_id))
 
-    def create_course(self, org, prettyid, user_id, id_root=None, metadata=None, course_data=None,
-        master_version='draft', versions_dict=None, root_category='course'):
+    def create_course(
+        self, org, prettyid, user_id, id_root=None, fields=None,
+        master_branch='draft', versions_dict=None, root_category='course'):
         """
         Create a new entry in the active courses index which points to an existing or new structure. Returns
         the course root of the resulting entry (the location has the course id)
@@ -686,93 +716,106 @@ class SplitMongoModuleStore(ModuleStoreBase):
         id_root: allows the caller to specify the course_id. It's a root in that, if it's already taken,
         this method will append things to the root to make it unique. (defaults to org)
 
-        metadata: if provided, will set the metadata of the root course object in the new draft course. If both
-        metadata and a starting version are provided, it will generate a successor version to the given version,
-        and update the metadata with any provided values (via update not setting).
+        fields: if scope.settings fields provided, will set the fields of the root course object in the
+        new course. If both
+        settings fields and a starting version are provided (via versions_dict), it will generate a successor version
+        to the given version,
+        and update the settings fields with any provided values (via update not setting).
 
-        course_data: if provided, will update the data of the new course xblock definition to this. Like metadata,
+        fields (content): if scope.content fields provided, will update the fields of the new course
+        xblock definition to this. Like settings fields,
         if provided, this will cause a new version of any given version as well as a new version of the
         definition (which will point to the existing one if given a version). If not provided and given
-        a draft_version, it will reuse the same definition as the draft course (obvious since it's reusing the draft
-        course). If not provided and no draft is given, it will be empty and get the field defaults (hopefully) when
+        a version_dict, it will reuse the same definition as that version's course
+        (obvious since it's reusing the
+        course). If not provided and no version_dict is given, it will be empty and get the field defaults
+        when
         loaded.
 
-        master_version: the tag (key) for the version name in the dict which is the 'draft' version. Not the actual
+        master_branch: the tag (key) for the version name in the dict which is the 'draft' version. Not the actual
         version guid, but what to call it.
 
         versions_dict: the starting version ids where the keys are the tags such as 'draft' and 'published'
         and the values are structure guids. If provided, the new course will reuse this version (unless you also
-        provide any overrides such as metadata, see above). if not provided, will create a mostly empty course
+        provide any fields overrides, see above). if not provided, will create a mostly empty course
         structure with just a category course root xblock.
         """
-        if metadata is None:
-            metadata = {}
+        partitioned_fields = self._partition_fields_by_scope('course', fields)
+        block_fields = partitioned_fields.setdefault(Scope.settings, {})
+        if Scope.children in partitioned_fields:
+            block_fields.update(partitioned_fields[Scope.children])
+        definition_fields = self._filter_special_fields(partitioned_fields.get(Scope.content, {}))
+
         # build from inside out: definition, structure, index entry
         # if building a wholly new structure
-        if versions_dict is None or master_version not in versions_dict:
+        if versions_dict is None or master_branch not in versions_dict:
             # create new definition and structure
-            if course_data is None:
-                course_data = {}
             definition_entry = {
                 'category': root_category,
-                'data': course_data,
-                'edited_by': user_id,
-                'edited_on': datetime.datetime.utcnow(),
-                'previous_version': None,
+                'fields': definition_fields,
+                'edit_info': {
+                    'edited_by': user_id,
+                    'edited_on': datetime.datetime.now(UTC),
+                    'previous_version': None,
                 }
+            }
             definition_id = self.definitions.insert(definition_entry)
-            definition_entry['original_version'] = definition_id
-            self.definitions.update({'_id': definition_id}, {'$set': {"original_version": definition_id}})
+            definition_entry['edit_info']['original_version'] = definition_id
+            self.definitions.update({'_id': definition_id}, {'$set': {"edit_info.original_version": definition_id}})
 
             draft_structure = {
                 'root': 'course',
                 'previous_version': None,
                 'edited_by': user_id,
-                'edited_on': datetime.datetime.utcnow(),
+                'edited_on': datetime.datetime.now(UTC),
                 'blocks': {
                     'course': {
-                        'children':[],
                         'category': 'course',
                         'definition': definition_id,
-                        'metadata': metadata,
-                        'edited_on': datetime.datetime.utcnow(),
-                        'edited_by': user_id,
-                        'previous_version': None}}}
+                        'fields': block_fields,
+                        'edit_info': {
+                            'edited_on': datetime.datetime.now(UTC),
+                            'edited_by': user_id,
+                            'previous_version': None
+                        }
+                    }
+                }
+            }
             new_id = self.structures.insert(draft_structure)
             draft_structure['original_version'] = new_id
             self.structures.update({'_id': new_id},
                 {'$set': {"original_version": new_id,
-                    'blocks.course.update_version': new_id}})
+                    'blocks.course.edit_info.update_version': new_id}})
             if versions_dict is None:
-                versions_dict = {master_version: new_id}
+                versions_dict = {master_branch: new_id}
             else:
-                versions_dict[master_version] = new_id
+                versions_dict[master_branch] = new_id
 
         else:
             # just get the draft_version structure
-            draft_version = CourseLocator(version_guid=versions_dict[master_version])
+            draft_version = CourseLocator(version_guid=versions_dict[master_branch])
             draft_structure = self._lookup_course(draft_version)
-            if course_data is not None or metadata:
+            if definition_fields or block_fields:
                 draft_structure = self._version_structure(draft_structure, user_id)
                 root_block = draft_structure['blocks'][draft_structure['root']]
-                if metadata is not None:
-                    root_block['metadata'].update(metadata)
-                if course_data is not None:
+                if block_fields is not None:
+                    root_block['fields'].update(block_fields)
+                if definition_fields is not None:
                     definition = self.definitions.find_one({'_id': root_block['definition']})
-                    definition['data'].update(course_data)
-                    definition['previous_version'] = definition['_id']
-                    definition['edited_by'] = user_id
-                    definition['edited_on'] = datetime.datetime.utcnow()
+                    definition['fields'].update(definition_fields)
+                    definition['edit_info']['previous_version'] = definition['_id']
+                    definition['edit_info']['edited_by'] = user_id
+                    definition['edit_info']['edited_on'] = datetime.datetime.now(UTC)
                     del definition['_id']
                     root_block['definition'] = self.definitions.insert(definition)
-                    root_block['edited_on'] = datetime.datetime.utcnow()
-                    root_block['edited_by'] = user_id
-                    root_block['previous_version'] = root_block.get('update_version')
+                    root_block['edit_info']['edited_on'] = datetime.datetime.now(UTC)
+                    root_block['edit_info']['edited_by'] = user_id
+                    root_block['edit_info']['previous_version'] = root_block['edit_info'].get('update_version')
                 # insert updates the '_id' in draft_structure
                 new_id = self.structures.insert(draft_structure)
-                versions_dict[master_version] = new_id
+                versions_dict[master_branch] = new_id
                 self.structures.update({'_id': new_id},
-                    {'$set': {'blocks.{}.update_version'.format(draft_structure['root']): new_id}})
+                    {'$set': {'blocks.{}.edit_info.update_version'.format(draft_structure['root']): new_id}})
         # create the index entry
         if id_root is None:
             id_root = org
@@ -783,14 +826,14 @@ class SplitMongoModuleStore(ModuleStoreBase):
             'org': org,
             'prettyid': prettyid,
             'edited_by': user_id,
-            'edited_on': datetime.datetime.utcnow(),
+            'edited_on': datetime.datetime.now(UTC),
             'versions': versions_dict}
         new_id = self.course_index.insert(index_entry)
-        return self.get_course(CourseLocator(course_id=new_id, branch=master_version))
+        return self.get_course(CourseLocator(course_id=new_id, branch=master_branch))
 
     def update_item(self, descriptor, user_id, force=False):
         """
-        Save the descriptor's definition, metadata, & children references (i.e., it doesn't descend the tree).
+        Save the descriptor's fields. it doesn't descend the course dag to save the children.
         Return the new descriptor (updated location).
 
         raises ItemNotFoundError if the location does not exist.
@@ -807,31 +850,38 @@ class SplitMongoModuleStore(ModuleStoreBase):
         index_entry = self._get_index_if_valid(descriptor.location, force)
 
         descriptor.definition_locator, is_updated = self.update_definition_from_data(
-            descriptor.definition_locator, descriptor.xblock_kvs.get_data(), user_id)
+            descriptor.definition_locator, descriptor.get_explicitly_set_fields_by_scope(Scope.content), user_id)
         # check children
         original_entry = original_structure['blocks'][descriptor.location.usage_id]
         if (not is_updated and descriptor.has_children
-            and not self._xblock_lists_equal(original_entry['children'], descriptor.children)):
+            and not self._xblock_lists_equal(original_entry['fields']['children'], descriptor.children)):
             is_updated = True
         # check metadata
         if not is_updated:
-            is_updated = self._compare_metadata(descriptor.xblock_kvs.get_own_metadata(), original_entry['metadata'])
+            is_updated = self._compare_settings(
+                descriptor.get_explicitly_set_fields_by_scope(Scope.settings),
+                original_entry['fields']
+            )
 
         # if updated, rev the structure
         if is_updated:
             new_structure = self._version_structure(original_structure, user_id)
             block_data = new_structure['blocks'][descriptor.location.usage_id]
-            if descriptor.has_children:
-                block_data["children"] = [self._usage_id(child) for child in descriptor.children]
 
             block_data["definition"] = descriptor.definition_locator.definition_id
-            block_data["metadata"] = descriptor.xblock_kvs.get_own_metadata()
-            block_data['edited_on'] = datetime.datetime.utcnow()
-            block_data['edited_by'] = user_id
-            block_data['previous_version'] = block_data['update_version']
+            block_data["fields"] = descriptor.get_explicitly_set_fields_by_scope(Scope.settings)
+            if descriptor.has_children:
+                block_data['fields']["children"] = [self._usage_id(child) for child in descriptor.children]
+
+            block_data['edit_info'] = {
+                'edited_on': datetime.datetime.now(UTC),
+                'edited_by': user_id,
+                'previous_version': block_data['edit_info']['update_version'],
+            }
             new_id = self.structures.insert(new_structure)
-            self.structures.update({'_id': new_id},
-                {'$set': {'blocks.{}.update_version'.format(descriptor.location.usage_id): new_id}})
+            self.structures.update(
+                {'_id': new_id},
+                {'$set': {'blocks.{}.edit_info.update_version'.format(descriptor.location.usage_id): new_id}})
 
             # update the index entry if appropriate
             if index_entry is not None:
@@ -857,8 +907,8 @@ class SplitMongoModuleStore(ModuleStoreBase):
         returns the post-persisted version of the incoming xblock. Note that its children will be ids not
         objects.
 
-        :param xblock:
-        :param user_id:
+        :param xblock: the head of the dag
+        :param user_id: who's doing the change
         """
         # find course_index entry if applicable and structures entry
         index_entry = self._get_index_if_valid(xblock.location, force)
@@ -871,7 +921,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
             new_id = self.structures.insert(new_structure)
             update_command = {}
             for usage_id in changed_blocks:
-                update_command['blocks.{}.update_version'.format(usage_id)] = new_id
+                update_command['blocks.{}.edit_info.update_version'.format(usage_id)] = new_id
             self.structures.update({'_id': new_id}, {'$set': update_command})
 
             # update the index entry if appropriate
@@ -885,14 +935,14 @@ class SplitMongoModuleStore(ModuleStoreBase):
 
     def _persist_subdag(self, xblock, user_id, structure_blocks):
         # persist the definition if persisted != passed
-        new_def_data = xblock.xblock_kvs.get_data()
+        new_def_data = self._filter_special_fields(xblock.get_explicitly_set_fields_by_scope(Scope.content))
         if (xblock.definition_locator is None or xblock.definition_locator.definition_id is None):
-            xblock.definition_locator = self.create_definition_from_data(new_def_data,
-                xblock.category, user_id)
+            xblock.definition_locator = self.create_definition_from_data(
+                new_def_data, xblock.category, user_id)
             is_updated = True
-        elif new_def_data is not None:
-            xblock.definition_locator, is_updated = self.update_definition_from_data(xblock.definition_locator,
-                new_def_data, user_id)
+        elif new_def_data:
+            xblock.definition_locator, is_updated = self.update_definition_from_data(
+                xblock.definition_locator, new_def_data, user_id)
 
         if xblock.location.usage_id is None:
             # generate an id
@@ -904,7 +954,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
             is_new = False
             usage_id = xblock.location.usage_id
             if (not is_updated and xblock.has_children
-                and not self._xblock_lists_equal(structure_blocks[usage_id]['children'], xblock.children)):
+                and not self._xblock_lists_equal(structure_blocks[usage_id]['fields']['children'], xblock.children)):
                 is_updated = True
 
         children = []
@@ -918,41 +968,52 @@ class SplitMongoModuleStore(ModuleStoreBase):
                     children.append(child)
 
         is_updated = is_updated or updated_blocks
-        metadata = xblock.xblock_kvs.get_own_metadata()
+        block_fields = xblock.get_explicitly_set_fields_by_scope(Scope.settings)
         if not is_new and not is_updated:
-            is_updated = self._compare_metadata(metadata, structure_blocks[usage_id]['metadata'])
+            is_updated = self._compare_settings(block_fields, structure_blocks[usage_id]['fields'])
+        if children:
+            block_fields['children'] = children
 
         if is_updated:
+            previous_version = None if is_new else structure_blocks[usage_id]['edit_info'].get('update_version')
             structure_blocks[usage_id] = {
-                "children": children,
                 "category": xblock.category,
                 "definition": xblock.definition_locator.definition_id,
-                "metadata": metadata if metadata else {},
-                'previous_version': structure_blocks.get(usage_id, {}).get('update_version'),
-                'edited_by': user_id,
-                'edited_on': datetime.datetime.utcnow()
+                "fields": block_fields,
+                'edit_info': {
+                    'previous_version': previous_version,
+                    'edited_by': user_id,
+                    'edited_on': datetime.datetime.now(UTC)
+                }
             }
             updated_blocks.append(usage_id)
 
         return updated_blocks
 
-    def _compare_metadata(self, metadata, original_metadata):
-        original_keys = original_metadata.keys()
-        if len(metadata) != len(original_keys):
+    def _compare_settings(self, settings, original_fields):
+        """
+        Return True if the settings are not == to the original fields
+        :param settings:
+        :param original_fields:
+        """
+        original_keys = original_fields.keys()
+        if 'children' in original_keys:
+            original_keys.remove('children')
+        if len(settings) != len(original_keys):
             return True
         else:
-            new_keys = metadata.keys()
+            new_keys = settings.keys()
             for key in original_keys:
-                if key not in new_keys or original_metadata[key] != metadata[key]:
+                if key not in new_keys or original_fields[key] != settings[key]:
                     return True
 
-    # TODO change all callers to update_item
-    def update_children(self, course_id, location, children):
-        raise NotImplementedError()
+    def update_children(self, location, children):
+        '''Deprecated, use update_item.'''
+        raise NotImplementedError('use update_item')
 
-    # TODO change all callers to update_item
-    def update_metadata(self, course_id, location, metadata):
-        raise NotImplementedError()
+    def update_metadata(self, location, metadata):
+        '''Deprecated, use update_item.'''
+        raise NotImplementedError('use update_item')
 
     def update_course_index(self, course_locator, new_values_dict, update_versions=False):
         """
@@ -980,9 +1041,9 @@ class SplitMongoModuleStore(ModuleStoreBase):
         self.course_index.update({'_id': course_locator.course_id},
             {'$set': new_values_dict})
 
-    def delete_item(self, usage_locator, user_id, force=False):
+    def delete_item(self, usage_locator, user_id, delete_children=False, force=False):
         """
-        Delete the tree rooted at block and any references w/in the course to the block
+        Delete the block or tree rooted at block (if delete_children) and any references w/in the course to the block
         from a new version of the course structure.
 
         returns CourseLocator for new version
@@ -1006,17 +1067,18 @@ class SplitMongoModuleStore(ModuleStoreBase):
         update_version_keys = []
         for parent in parents:
             parent_block = new_blocks[parent.usage_id]
-            parent_block['children'].remove(usage_locator.usage_id)
-            parent_block['edited_on'] = datetime.datetime.utcnow()
-            parent_block['edited_by'] = user_id
-            parent_block['previous_version'] = parent_block['update_version']
-            update_version_keys.append('blocks.{}.update_version'.format(parent.usage_id))
+            parent_block['fields']['children'].remove(usage_locator.usage_id)
+            parent_block['edit_info']['edited_on'] = datetime.datetime.now(UTC)
+            parent_block['edit_info']['edited_by'] = user_id
+            parent_block['edit_info']['previous_version'] = parent_block['edit_info']['update_version']
+            update_version_keys.append('blocks.{}.edit_info.update_version'.format(parent.usage_id))
         # remove subtree
         def remove_subtree(usage_id):
-            for child in new_blocks[usage_id]['children']:
+            for child in new_blocks[usage_id]['fields'].get('children', []):
                 remove_subtree(child)
             del new_blocks[usage_id]
-        remove_subtree(usage_locator.usage_id)
+        if delete_children:
+            remove_subtree(usage_locator.usage_id)
 
         # update index if appropriate and structures
         new_id = self.structures.insert(new_structure)
@@ -1050,7 +1112,6 @@ class SplitMongoModuleStore(ModuleStoreBase):
         # this is the only real delete in the system. should it do something else?
         self.course_index.remove(index['_id'])
 
-    # TODO remove all callers and then this
     def get_errored_courses(self):
         """
         This function doesn't make sense for the mongo modulestore, as structures
@@ -1058,32 +1119,31 @@ class SplitMongoModuleStore(ModuleStoreBase):
         """
         return {}
 
-    def inherit_metadata(self, block_map, block, inheriting_metadata=None):
+    def inherit_settings(self, block_map, block, inheriting_settings=None):
         """
-        Updates block with any value
-        that exist in inheriting_metadata and don't appear in block['metadata'],
-        and then inherits block['metadata'] to all of the children in
-        block['children']. Filters by inheritance.INHERITABLE_METADATA
+        Updates block with any inheritable setting set by an ancestor and recurses to children.
         """
         if block is None:
             return
 
-        if inheriting_metadata is None:
-            inheriting_metadata = {}
+        if inheriting_settings is None:
+            inheriting_settings = {}
 
         # the currently passed down values take precedence over any previously cached ones
         # NOTE: this should show the values which all fields would have if inherited: i.e.,
         # not set to the locally defined value but to value set by nearest ancestor who sets it
-        block.setdefault('_inherited_metadata', {}).update(inheriting_metadata)
+        # ALSO NOTE: no xblock should ever define a _inherited_settings field as it will collide w/ this logic.
+        block.setdefault('_inherited_settings', {}).update(inheriting_settings)
 
         # update the inheriting w/ what should pass to children
-        inheriting_metadata = block['_inherited_metadata'].copy()
+        inheriting_settings = block['_inherited_settings'].copy()
+        block_fields = block['fields']
         for field in inheritance.INHERITABLE_METADATA:
-            if field in block['metadata']:
-                inheriting_metadata[field] = block['metadata'][field]
+            if field in block_fields:
+                inheriting_settings[field] = block_fields[field]
 
-        for child in block.get('children', []):
-            self.inherit_metadata(block_map, block_map[child], inheriting_metadata)
+        for child in block_fields.get('children', []):
+            self.inherit_settings(block_map, block_map[child], inheriting_settings)
 
     def descendants(self, block_map, usage_id, depth, descendent_map):
         """
@@ -1100,7 +1160,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
 
         if depth is None or depth > 0:
             depth = depth - 1 if depth is not None else None
-            for child in block_map[usage_id].get('children', []):
+            for child in block_map[usage_id]['fields'].get('children', []):
                 descendent_map = self.descendants(block_map, child, depth,
                     descendent_map)
 
@@ -1213,7 +1273,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
         del new_structure['_id']
         new_structure['previous_version'] = structure['_id']
         new_structure['edited_by'] = user_id
-        new_structure['edited_on'] = datetime.datetime.utcnow()
+        new_structure['edited_on'] = datetime.datetime.now(UTC)
         return new_structure
 
     def _find_local_root(self, element_to_find, possibility, tree):
@@ -1238,3 +1298,31 @@ class SplitMongoModuleStore(ModuleStoreBase):
         self.course_index.update(
             {"_id": index_entry["_id"]},
             {"$set": {"versions.{}".format(branch): new_id}})
+
+    def _partition_fields_by_scope(self, category, fields):
+        """
+        Return dictionary of {scope: {field1: val, ..}..} for the fields of this potential xblock
+
+        :param category: the xblock category
+        :param fields: the dictionary of {fieldname: value}
+        """
+        if fields is None:
+            return {}
+        cls = XModuleDescriptor.load_class(category)
+        result = collections.defaultdict(dict)
+        for field_name, value in fields.iteritems():
+            field = getattr(cls, field_name)
+            result[field.scope][field_name] = value
+        return result
+
+    def _filter_special_fields(self, fields):
+        """
+        Remove any fields which split or its kvs computes or adds but does not want persisted.
+
+        :param fields: a dict of fields
+        """
+        if 'location' in fields:
+            del fields['location']
+        if 'category' in fields:
+            del fields['category']
+        return fields

common/lib/xmodule/xmodule/modulestore/split_mongo/split_mongo_kvs.py

@@ -8,156 +8,175 @@ from .definition_lazy_loader import DefinitionLazyLoader
 SplitMongoKVSid = namedtuple('SplitMongoKVSid', 'id, def_id')
 
 
-# TODO should this be here or w/ x_module or ???
+PROVENANCE_LOCAL = 'local'
+PROVENANCE_DEFAULT = 'default'
+PROVENANCE_INHERITED = 'inherited'
+
 class SplitMongoKVS(KeyValueStore):
     """
     A KeyValueStore that maps keyed data access to one of the 3 data areas
     known to the MongoModuleStore (data, children, and metadata)
     """
-    def __init__(self, definition, children, metadata, _inherited_metadata, location, category):
+
+    def __init__(self, definition, fields, _inherited_settings, location, category):
         """
 
-        :param definition:
-        :param children:
-        :param metadata: the locally defined value for each metadata field
-        :param _inherited_metadata: the value of each inheritable field from above this.
-            Note, metadata may override and disagree w/ this b/c this says what the value
-            should be if metadata is undefined for this field.
+        :param definition: either a lazyloader or definition id for the definition
+        :param fields: a dictionary of the locally set fields
+        :param _inherited_settings: the value of each inheritable field from above this.
+            Note, local fields may override and disagree w/ this b/c this says what the value
+            should be if the field is undefined.
         """
         # ensure kvs's don't share objects w/ others so that changes can't appear in separate ones
         # the particular use case was that changes to kvs's were polluting caches. My thinking was
         # that kvs's should be independent thus responsible for the isolation.
-        if isinstance(definition, DefinitionLazyLoader):
-            self._definition = definition
-        else:
-            self._definition = copy.copy(definition)
-        self._children = copy.copy(children)
-        self._metadata = copy.copy(metadata)
-        self._inherited_metadata = _inherited_metadata
+        self._definition = definition  # either a DefinitionLazyLoader or the db id of the definition.
+        # if the db id, then the definition is presumed to be loaded into _fields
+        self._fields = copy.copy(fields)
+        self._inherited_settings = _inherited_settings
         self._location = location
         self._category = category
 
     def get(self, key):
-        if key.scope == Scope.children:
-            return self._children
-        elif key.scope == Scope.parent:
+        # simplest case, field is directly set
+        if key.field_name in self._fields:
+            return self._fields[key.field_name]
+
+        # parent undefined in editing runtime (I think)
+        if key.scope == Scope.parent:
+            # see STUD-624. Right now copies MongoKeyValueStore.get's behavior of returning None
             return None
+        if key.scope == Scope.children:
+            # didn't find children in _fields; so, see if there's a default
+            raise KeyError()
         elif key.scope == Scope.settings:
-            if key.field_name in self._metadata:
-                return self._metadata[key.field_name]
-            elif key.field_name in self._inherited_metadata:
-                return self._inherited_metadata[key.field_name]
+            # didn't find in _fields; so, get from inheritance since not locally set
+            if key.field_name in self._inherited_settings:
+                return self._inherited_settings[key.field_name]
             else:
+                # or get default
                 raise KeyError()
         elif key.scope == Scope.content:
             if key.field_name == 'location':
                 return self._location
             elif key.field_name == 'category':
                 return self._category
-            else:
-                if isinstance(self._definition, DefinitionLazyLoader):
-                    self._definition = self._definition.fetch()
-                if (key.field_name == 'data' and
-                        not isinstance(self._definition.get('data'), dict)):
-                    return self._definition.get('data')
-                elif 'data' not in self._definition or key.field_name not in self._definition['data']:
-                    raise KeyError()
-                else:
-                    return self._definition['data'][key.field_name]
+            elif isinstance(self._definition, DefinitionLazyLoader):
+                self._load_definition()
+                if key.field_name in self._fields:
+                    return self._fields[key.field_name]
+
+            raise KeyError()
         else:
             raise InvalidScopeError(key.scope)
 
     def set(self, key, value):
-        # TODO cache db update implications & add method to invoke
-        if key.scope == Scope.children:
-            self._children = value
-            # TODO remove inheritance from any orphaned exchildren
-            # TODO add inheritance to any new children
-        elif key.scope == Scope.settings:
-            # TODO if inheritable, push down to children who don't override
-            self._metadata[key.field_name] = value
-        elif key.scope == Scope.content:
-            if key.field_name == 'location':
-                self._location = value
-            elif key.field_name == 'category':
-                self._category = value
-            else:
-                if isinstance(self._definition, DefinitionLazyLoader):
-                    self._definition = self._definition.fetch()
-                if (key.field_name == 'data' and
-                        not isinstance(self._definition.get('data'), dict)):
-                    self._definition.get('data')
-                else:
-                    self._definition.setdefault('data', {})[key.field_name] = value
-        else:
+        # handle any special cases
+        if key.scope not in [Scope.children, Scope.settings, Scope.content]:
             raise InvalidScopeError(key.scope)
+        if key.scope == Scope.content:
+            if key.field_name == 'location':
+                self._location = value  # is changing this legal?
+                return
+            elif key.field_name == 'category':
+                # TODO should this raise an exception? category is not changeable.
+                return
+            else:
+                self._load_definition()
+
+        # set the field
+        self._fields[key.field_name] = value
+
+        # handle any side effects -- story STUD-624
+        # if key.scope == Scope.children:
+            # STUD-624 remove inheritance from any exchildren
+            # STUD-624 add inheritance to any new children
+        # if key.scope == Scope.settings:
+            # STUD-624 if inheritable, push down to children
 
     def delete(self, key):
-        # TODO cache db update implications & add method to invoke
-        if key.scope == Scope.children:
-            self._children = []
-        elif key.scope == Scope.settings:
-            # TODO if inheritable, ensure _inherited_metadata has value from above and
-            # revert children to that value
-            if key.field_name in self._metadata:
-                del self._metadata[key.field_name]
-        elif key.scope == Scope.content:
-            # don't allow deletion of location nor category
-            if key.field_name == 'location':
-                pass
-            elif key.field_name == 'category':
-                pass
-            else:
-                if isinstance(self._definition, DefinitionLazyLoader):
-                    self._definition = self._definition.fetch()
-                if (key.field_name == 'data' and
-                        not isinstance(self._definition.get('data'), dict)):
-                    self._definition.setdefault('data', None)
-                else:
-                    try:
-                        del self._definition['data'][key.field_name]
-                    except KeyError:
-                        pass
-        else:
+        # handle any special cases
+        if key.scope not in [Scope.children, Scope.settings, Scope.content]:
             raise InvalidScopeError(key.scope)
+        if key.scope == Scope.content:
+            if key.field_name == 'location':
+                return  # noop
+            elif key.field_name == 'category':
+                # TODO should this raise an exception? category is not deleteable.
+                return  # noop
+            else:
+                self._load_definition()
+
+        # delete the field value
+        if key.field_name in self._fields:
+            del self._fields[key.field_name]
+
+        # handle any side effects
+        # if key.scope == Scope.children:
+            # STUD-624 remove inheritance from any exchildren
+        # if key.scope == Scope.settings:
+            # STUD-624 if inheritable, push down _inherited_settings value to children
 
     def has(self, key):
-        if key.scope in (Scope.children, Scope.parent):
-            return True
-        elif key.scope == Scope.settings:
-            return key.field_name in self._metadata or key.field_name in self._inherited_metadata
-        elif key.scope == Scope.content:
+        """
+        Is the given field explicitly set in this kvs (not inherited nor default)
+        """
+        # handle any special cases
+        if key.scope == Scope.content:
             if key.field_name == 'location':
                 return True
             elif key.field_name == 'category':
                 return self._category is not None
             else:
-                if isinstance(self._definition, DefinitionLazyLoader):
-                    self._definition = self._definition.fetch()
-                if (key.field_name == 'data' and
-                        not isinstance(self._definition.get('data'), dict)):
-                    return self._definition.get('data') is not None
-                else:
-                    return key.field_name in self._definition.get('data', {})
-        else:
-            return False
+                self._load_definition()
+        elif key.scope == Scope.parent:
+            return True
 
-    def get_data(self):
+        # it's not clear whether inherited values should return True. Right now they don't
+        # if someone changes it so that they do, then change any tests of field.name in xx._model_data
+        return key.field_name in self._fields
+
+    # would like to just take a key, but there's a bunch of magic in DbModel for constructing the key via
+    # a private method
+    def field_value_provenance(self, key_scope, key_name):
         """
-        Intended only for use by persistence layer to get the native definition['data'] rep
+        Where the field value comes from: one of [PROVENANCE_LOCAL, PROVENANCE_DEFAULT, PROVENANCE_INHERITED].
+        """
+        # handle any special cases
+        if key_scope == Scope.content:
+            if key_name == 'location':
+                return PROVENANCE_LOCAL
+            elif key_name == 'category':
+                return PROVENANCE_LOCAL
+            else:
+                self._load_definition()
+                if key_name in self._fields:
+                    return PROVENANCE_LOCAL
+                else:
+                    return PROVENANCE_DEFAULT
+        elif key_scope == Scope.parent:
+            return PROVENANCE_DEFAULT
+        # catch the locally set state
+        elif key_name in self._fields:
+            return PROVENANCE_LOCAL
+        elif key_scope == Scope.settings and key_name in self._inherited_settings:
+            return PROVENANCE_INHERITED
+        else:
+            return PROVENANCE_DEFAULT
+
+    def get_inherited_settings(self):
+        """
+        Get the settings set by the ancestors (which locally set fields may override or not)
+        """
+        return self._inherited_settings
+
+    def _load_definition(self):
+        """
+        Update fields w/ the lazily loaded definitions
         """
         if isinstance(self._definition, DefinitionLazyLoader):
-            self._definition = self._definition.fetch()
-        return self._definition.get('data')
-
-    def get_own_metadata(self):
-        """
-        Get the metadata explicitly set on this element.
-        """
-        return self._metadata
-
-    def get_inherited_metadata(self):
-        """
-        Get the metadata set by the ancestors (which own metadata may override or not)
-        """
-        return self._inherited_metadata
+            persisted_definition = self._definition.fetch()
+            if persisted_definition is not None:
+                self._fields.update(persisted_definition.get('fields'))
+                # do we want to cache any of the edit_info?
+            self._definition = None  # already loaded

common/lib/xmodule/xmodule/modulestore/store_utilities.py

@@ -146,13 +146,9 @@ def _clone_modules(modulestore, modules, source_location, dest_location):
 
 
 def clone_course(modulestore, contentstore, source_location, dest_location, delete_original=False):
-    # first check to see if the modulestore is Mongo backed
-    if not isinstance(modulestore, MongoModuleStore):
-        raise Exception("Expected a MongoModuleStore in the runtime. Aborting....")
-
     # check to see if the dest_location exists as an empty course
     # we need an empty course because the app layers manage the permissions and users
-    if not modulestore.has_item(dest_location):
+    if not modulestore.has_item(dest_location.course_id, dest_location):
         raise Exception("An empty course at {0} must have already been created. Aborting...".format(dest_location))
 
     # verify that the dest_location really is an empty course, which means only one with an optional 'overview'
@@ -171,7 +167,7 @@ def clone_course(modulestore, contentstore, source_location, dest_location, dele
         raise Exception("Course at destination {0} is not an empty course. You can only clone into an empty course. Aborting...".format(dest_location))
 
     # check to see if the source course is actually there
-    if not modulestore.has_item(source_location):
+    if not modulestore.has_item(source_location.course_id, source_location):
         raise Exception("Cannot find a course at {0}. Aborting".format(source_location))
 
     # Get all modules under this namespace which is (tag, org, course) tuple
@@ -250,7 +246,7 @@ def delete_course(modulestore, contentstore, source_location, commit=False):
     """
 
     # check to see if the source course is actually there
-    if not modulestore.has_item(source_location):
+    if not modulestore.has_item(source_location.course_id, source_location):
         raise Exception("Cannot find a course at {0}. Aborting".format(source_location))
 
     # first delete all of the thumbnails

common/lib/xmodule/xmodule/modulestore/tests/__init__.py

@@ -1,10 +0,0 @@
-from path import path
-
-# from ~/mitx_all/mitx/common/lib/xmodule/xmodule/modulestore/tests/
-# to   ~/mitx_all/mitx/common/test
-TEST_DIR = path(__file__).abspath().dirname()
-for i in range(5):
-    TEST_DIR = TEST_DIR.dirname()
-TEST_DIR = TEST_DIR / 'test'
-
-DATA_DIR = TEST_DIR / 'data'

common/lib/xmodule/xmodule/modulestore/tests/persistent_factories.py

@@ -11,44 +11,24 @@ class PersistentCourseFactory(factory.Factory):
     """
     Create a new course (not a new version of a course, but a whole new index entry).
 
-    keywords:
+    keywords: any xblock field plus (note, the below are filtered out; so, if they
+    become legitimate xblock fields, they won't be settable via this factory)
     * org: defaults to textX
     * prettyid: defaults to 999
-    * display_name
-    * user_id
-    * data (optional) the data payload to save in the course item
-    * metadata (optional) the metadata payload. If display_name is in the metadata, that takes
-    precedence over any display_name provided directly.
+    * master_branch: (optional) defaults to 'draft'
+    * user_id: (optional) defaults to 'test_user'
+    * display_name (xblock field): will default to 'Robot Super Course' unless provided
     """
     FACTORY_FOR = CourseDescriptor
 
-    org = 'testX'
-    prettyid = '999'
-    display_name = 'Robot Super Course'
-    user_id = "test_user"
-    data = None
-    metadata = None
-    master_version = 'draft'
-
     # pylint: disable=W0613
     @classmethod
-    def _create(cls, target_class, *args, **kwargs):
-
-        org = kwargs.get('org')
-        prettyid = kwargs.get('prettyid')
-        display_name = kwargs.get('display_name')
-        user_id = kwargs.get('user_id')
-        data = kwargs.get('data')
-        metadata = kwargs.get('metadata', {})
-        if metadata is None:
-            metadata = {}
-        if 'display_name' not in metadata:
-            metadata['display_name'] = display_name
+    def _create(cls, target_class, org='testX', prettyid='999', user_id='test_user', master_branch='draft', **kwargs):
 
         # Write the data to the mongo datastore
         new_course = modulestore('split').create_course(
-            org, prettyid, user_id, metadata=metadata, course_data=data, id_root=prettyid,
-            master_version=kwargs.get('master_version'))
+            org, prettyid, user_id, fields=kwargs, id_root=prettyid,
+            master_branch=master_branch)
 
         return new_course
 
@@ -60,36 +40,24 @@ class PersistentCourseFactory(factory.Factory):
 class ItemFactory(factory.Factory):
     FACTORY_FOR = XModuleDescriptor
 
-    category = 'chapter'
-    user_id = 'test_user'
     display_name = factory.LazyAttributeSequence(lambda o, n: "{} {}".format(o.category, n))
 
     # pylint: disable=W0613
     @classmethod
-    def _create(cls, target_class, *args, **kwargs):
+    def _create(cls, target_class, parent_location, category='chapter',
+        user_id='test_user', definition_locator=None, **kwargs):
         """
-        Uses *kwargs*:
+        passes *kwargs* as the new item's field values:
 
-        *parent_location* (required): the location of the course & possibly parent
+        :param parent_location: (required) the location of the course & possibly parent
 
-        *category* (defaults to 'chapter')
+        :param category: (defaults to 'chapter')
 
-        *data* (optional): the data for the item
-
-        definition_locator (optional): the DescriptorLocator for the definition this uses or branches
-
-        *display_name* (optional): the display name of the item
-
-        *metadata* (optional): dictionary of metadata attributes (display_name here takes
-        precedence over the above attr)
+        :param definition_locator (optional): the DescriptorLocator for the definition this uses or branches
         """
-        metadata = kwargs.get('metadata', {})
-        if 'display_name' not in metadata and 'display_name' in kwargs:
-            metadata['display_name'] = kwargs['display_name']
-
-        return modulestore('split').create_item(kwargs['parent_location'], kwargs['category'],
-            kwargs['user_id'], definition_locator=kwargs.get('definition_locator'),
-            new_def_data=kwargs.get('data'), metadata=metadata)
+        return modulestore('split').create_item(
+            parent_location, category, user_id, definition_locator, fields=kwargs
+        )
 
     @classmethod
     def _build(cls, target_class, *args, **kwargs):

common/lib/xmodule/xmodule/modulestore/tests/test_location.py

@@ -159,3 +159,12 @@ def test_clean_for_html():
 def test_html_id():
     loc = Location("tag://org/course/cat/name:more_name@rev")
     assert_equals(loc.html_id(), "tag-org-course-cat-name_more_name-rev")
+
+
+def test_course_id():
+    loc = Location('i4x', 'mitX', '103', 'course', 'test2')
+    assert_equals('mitX/103/test2', loc.course_id)
+
+    loc = Location('i4x', 'mitX', '103', '_not_a_course', 'test2')
+    with assert_raises(InvalidLocationError):
+        loc.course_id

common/lib/xmodule/xmodule/modulestore/tests/test_locators.py

@@ -1,16 +1,18 @@
-'''
-Created on Mar 14, 2013
-
-@author: dmitchell
-'''
+"""
+Tests for xmodule.modulestore.locator.
+"""
 from unittest import TestCase
 
 from bson.objectid import ObjectId
-from xmodule.modulestore.locator import Locator, CourseLocator, BlockUsageLocator
+from xmodule.modulestore.locator import Locator, CourseLocator, BlockUsageLocator, DescriptionLocator
+from xmodule.modulestore.parsers import BRANCH_PREFIX, BLOCK_PREFIX, VERSION_PREFIX, URL_VERSION_PREFIX
 from xmodule.modulestore.exceptions import InsufficientSpecificationError, OverSpecificationError
 
 
 class LocatorTest(TestCase):
+    """
+    Tests for subclasses of Locator.
+    """
 
     def test_cant_instantiate_abstract_class(self):
         self.assertRaises(TypeError, Locator)
@@ -32,12 +34,12 @@ class LocatorTest(TestCase):
         self.assertRaises(
             OverSpecificationError,
             CourseLocator,
-            url='edx://mit.eecs.6002x;published',
+            url='edx://mit.eecs.6002x' + BRANCH_PREFIX + 'published',
             branch='draft')
         self.assertRaises(
             OverSpecificationError,
             CourseLocator,
-            course_id='mit.eecs.6002x;published',
+            course_id='mit.eecs.6002x' + BRANCH_PREFIX + 'published',
             branch='draft')
 
     def test_course_constructor_underspecified(self):
@@ -55,8 +57,8 @@ class LocatorTest(TestCase):
         testobj_1 = CourseLocator(version_guid=test_id_1)
         self.check_course_locn_fields(testobj_1, 'version_guid', version_guid=test_id_1)
         self.assertEqual(str(testobj_1.version_guid), test_id_1_loc)
-        self.assertEqual(str(testobj_1), '@' + test_id_1_loc)
-        self.assertEqual(testobj_1.url(), 'edx://@' + test_id_1_loc)
+        self.assertEqual(str(testobj_1), URL_VERSION_PREFIX + test_id_1_loc)
+        self.assertEqual(testobj_1.url(), 'edx://' + URL_VERSION_PREFIX + test_id_1_loc)
 
         # Test using a given string
         test_id_2_loc = '519665f6223ebd6980884f2b'
@@ -64,8 +66,8 @@ class LocatorTest(TestCase):
         testobj_2 = CourseLocator(version_guid=test_id_2)
         self.check_course_locn_fields(testobj_2, 'version_guid', version_guid=test_id_2)
         self.assertEqual(str(testobj_2.version_guid), test_id_2_loc)
-        self.assertEqual(str(testobj_2), '@' + test_id_2_loc)
-        self.assertEqual(testobj_2.url(), 'edx://@' + test_id_2_loc)
+        self.assertEqual(str(testobj_2), URL_VERSION_PREFIX + test_id_2_loc)
+        self.assertEqual(testobj_2.url(), 'edx://' + URL_VERSION_PREFIX + test_id_2_loc)
 
     def test_course_constructor_bad_course_id(self):
         """
@@ -74,20 +76,20 @@ class LocatorTest(TestCase):
         for bad_id in ('mit.',
                        ' mit.eecs',
                        'mit.eecs ',
-                       '@mit.eecs',
-                       '#mit.eecs',
+                       URL_VERSION_PREFIX + 'mit.eecs',
+                       BLOCK_PREFIX + 'block/mit.eecs',
                        'mit.ee cs',
                        'mit.ee,cs',
                        'mit.ee/cs',
                        'mit.ee$cs',
                        'mit.ee&cs',
                        'mit.ee()cs',
-                       ';this',
-                       'mit.eecs;',
-                       'mit.eecs;this;that',
-                       'mit.eecs;this;',
-                       'mit.eecs;this ',
-                       'mit.eecs;th%is ',
+                       BRANCH_PREFIX + 'this',
+                       'mit.eecs' + BRANCH_PREFIX,
+                       'mit.eecs' + BRANCH_PREFIX + 'this' + BRANCH_PREFIX + 'that',
+                       'mit.eecs' + BRANCH_PREFIX + 'this' + BRANCH_PREFIX,
+                       'mit.eecs' + BRANCH_PREFIX + 'this ',
+                       'mit.eecs' + BRANCH_PREFIX + 'th%is ',
                        ):
             self.assertRaises(AssertionError, CourseLocator, course_id=bad_id)
             self.assertRaises(AssertionError, CourseLocator, url='edx://' + bad_id)
@@ -106,7 +108,7 @@ class LocatorTest(TestCase):
         self.check_course_locn_fields(testobj, 'course_id', course_id=testurn)
 
     def test_course_constructor_redundant_002(self):
-        testurn = 'mit.eecs.6002x;published'
+        testurn = 'mit.eecs.6002x' + BRANCH_PREFIX + 'published'
         expected_urn = 'mit.eecs.6002x'
         expected_rev = 'published'
         testobj = CourseLocator(course_id=testurn, url='edx://' + testurn)
@@ -114,6 +116,32 @@ class LocatorTest(TestCase):
                                       course_id=expected_urn,
                                       branch=expected_rev)
 
+    def test_course_constructor_url(self):
+        # Test parsing a url when it starts with a version ID and there is also a block ID.
+        # This hits the parsers parse_guid method.
+        test_id_loc = '519665f6223ebd6980884f2b'
+        testobj = CourseLocator(url="edx://" + URL_VERSION_PREFIX + test_id_loc + BLOCK_PREFIX + "hw3")
+        self.check_course_locn_fields(
+            testobj,
+            'test_block constructor',
+            version_guid=ObjectId(test_id_loc)
+        )
+
+    def test_course_constructor_url_course_id_and_version_guid(self):
+        test_id_loc = '519665f6223ebd6980884f2b'
+        testobj = CourseLocator(url='edx://mit.eecs.6002x' + VERSION_PREFIX + test_id_loc)
+        self.check_course_locn_fields(testobj, 'error parsing url with both course ID and version GUID',
+                                      course_id='mit.eecs.6002x',
+                                      version_guid=ObjectId(test_id_loc))
+
+    def test_course_constructor_url_course_id_branch_and_version_guid(self):
+        test_id_loc = '519665f6223ebd6980884f2b'
+        testobj = CourseLocator(url='edx://mit.eecs.6002x' + BRANCH_PREFIX + 'draft' + VERSION_PREFIX + test_id_loc)
+        self.check_course_locn_fields(testobj, 'error parsing url with both course ID branch, and version GUID',
+                                      course_id='mit.eecs.6002x',
+                                      branch='draft',
+                                      version_guid=ObjectId(test_id_loc))
+
     def test_course_constructor_course_id_no_branch(self):
         testurn = 'mit.eecs.6002x'
         testobj = CourseLocator(course_id=testurn)
@@ -123,7 +151,7 @@ class LocatorTest(TestCase):
         self.assertEqual(testobj.url(), 'edx://' + testurn)
 
     def test_course_constructor_course_id_with_branch(self):
-        testurn = 'mit.eecs.6002x;published'
+        testurn = 'mit.eecs.6002x' + BRANCH_PREFIX + 'published'
         expected_id = 'mit.eecs.6002x'
         expected_branch = 'published'
         testobj = CourseLocator(course_id=testurn)
@@ -139,7 +167,7 @@ class LocatorTest(TestCase):
     def test_course_constructor_course_id_separate_branch(self):
         test_id = 'mit.eecs.6002x'
         test_branch = 'published'
-        expected_urn = 'mit.eecs.6002x;published'
+        expected_urn = 'mit.eecs.6002x' + BRANCH_PREFIX + 'published'
         testobj = CourseLocator(course_id=test_id, branch=test_branch)
         self.check_course_locn_fields(testobj, 'course_id with separate branch',
                                       course_id=test_id,
@@ -154,10 +182,10 @@ class LocatorTest(TestCase):
         """
         The same branch appears in the course_id and the branch field.
         """
-        test_id = 'mit.eecs.6002x;published'
+        test_id = 'mit.eecs.6002x' + BRANCH_PREFIX + 'published'
         test_branch = 'published'
         expected_id = 'mit.eecs.6002x'
-        expected_urn = 'mit.eecs.6002x;published'
+        expected_urn = test_id
         testobj = CourseLocator(course_id=test_id, branch=test_branch)
         self.check_course_locn_fields(testobj, 'course_id with repeated branch',
                                       course_id=expected_id,
@@ -169,7 +197,7 @@ class LocatorTest(TestCase):
         self.assertEqual(testobj.url(), 'edx://' + expected_urn)
 
     def test_block_constructor(self):
-        testurn = 'mit.eecs.6002x;published#HW3'
+        testurn = 'mit.eecs.6002x' + BRANCH_PREFIX + 'published' + BLOCK_PREFIX + 'HW3'
         expected_id = 'mit.eecs.6002x'
         expected_branch = 'published'
         expected_block_ref = 'HW3'
@@ -181,6 +209,43 @@ class LocatorTest(TestCase):
         self.assertEqual(str(testobj), testurn)
         self.assertEqual(testobj.url(), 'edx://' + testurn)
 
+    def test_block_constructor_url_version_prefix(self):
+        test_id_loc = '519665f6223ebd6980884f2b'
+        testobj = BlockUsageLocator(
+            url='edx://mit.eecs.6002x' + VERSION_PREFIX + test_id_loc + BLOCK_PREFIX + 'lab2'
+        )
+        self.check_block_locn_fields(
+            testobj, 'error parsing URL with version and block',
+            course_id='mit.eecs.6002x',
+            block='lab2',
+            version_guid=ObjectId(test_id_loc)
+        )
+
+    def test_block_constructor_url_kitchen_sink(self):
+        test_id_loc = '519665f6223ebd6980884f2b'
+        testobj = BlockUsageLocator(
+            url='edx://mit.eecs.6002x' + BRANCH_PREFIX + 'draft' + VERSION_PREFIX + test_id_loc + BLOCK_PREFIX + 'lab2'
+        )
+        self.check_block_locn_fields(
+            testobj, 'error parsing URL with branch, version, and block',
+            course_id='mit.eecs.6002x',
+            branch='draft',
+            block='lab2',
+            version_guid=ObjectId(test_id_loc)
+        )
+
+    def test_repr(self):
+        testurn = 'mit.eecs.6002x' + BRANCH_PREFIX + 'published' + BLOCK_PREFIX + 'HW3'
+        testobj = BlockUsageLocator(course_id=testurn)
+        self.assertEqual('BlockUsageLocator("mit.eecs.6002x/branch/published/block/HW3")', repr(testobj))
+
+    def test_description_locator_url(self):
+        definition_locator = DescriptionLocator("chapter12345_2")
+        self.assertEqual('edx://' + URL_VERSION_PREFIX + 'chapter12345_2', definition_locator.url())
+
+    def test_description_locator_version(self):
+        definition_locator = DescriptionLocator("chapter12345_2")
+        self.assertEqual("chapter12345_2", definition_locator.version())
 
     # ------------------------------------------------------------------
     # Utilities

common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py

@@ -0,0 +1,242 @@
+from nose.tools import assert_equals, assert_raises, assert_false, assert_true, assert_not_equals
+import pymongo
+from uuid import uuid4
+
+from xmodule.tests import DATA_DIR
+from xmodule.modulestore import Location, MONGO_MODULESTORE_TYPE, XML_MODULESTORE_TYPE
+from xmodule.modulestore.exceptions import ItemNotFoundError
+from xmodule.modulestore.mixed import MixedModuleStore
+from xmodule.modulestore.xml_importer import import_from_xml
+
+
+HOST = 'localhost'
+PORT = 27017
+DB = 'test_mongo_%s' % uuid4().hex
+COLLECTION = 'modulestore'
+FS_ROOT = DATA_DIR  # TODO (vshnayder): will need a real fs_root for testing load_item
+DEFAULT_CLASS = 'xmodule.raw_module.RawDescriptor'
+RENDER_TEMPLATE = lambda t_n, d, ctx = None, nsp = 'main': ''
+
+IMPORT_COURSEID = 'MITx/999/2013_Spring'
+XML_COURSEID1 = 'edX/toy/2012_Fall'
+XML_COURSEID2 = 'edX/simple/2012_Fall'
+
+OPTIONS = {
+    'mappings': {
+        XML_COURSEID1: 'xml',
+        XML_COURSEID2: 'xml',
+        IMPORT_COURSEID: 'default'
+    },
+    'stores': {
+        'xml': {
+            'ENGINE': 'xmodule.modulestore.xml.XMLModuleStore',
+            'OPTIONS': {
+                'data_dir': DATA_DIR,
+                'default_class': 'xmodule.hidden_module.HiddenDescriptor',
+            }
+        },
+        'default': {
+            'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
+            'OPTIONS': {
+                'default_class': DEFAULT_CLASS,
+                'host': HOST,
+                'db': DB,
+                'collection': COLLECTION,
+                'fs_root': DATA_DIR,
+                'render_template': RENDER_TEMPLATE,
+            }
+        }
+    }
+}
+
+
+class TestMixedModuleStore(object):
+    '''Tests!'''
+    @classmethod
+    def setupClass(cls):
+        cls.connection = pymongo.connection.Connection(HOST, PORT)
+        cls.connection.drop_database(DB)
+        cls.fake_location = Location(['i4x', 'foo', 'bar', 'vertical', 'baz'])
+        cls.import_org, cls.import_course, cls.import_run = IMPORT_COURSEID.split('/')
+        # NOTE: Creating a single db for all the tests to save time.  This
+        # is ok only as long as none of the tests modify the db.
+        # If (when!) that changes, need to either reload the db, or load
+        # once and copy over to a tmp db for each test.
+        cls.store = cls.initdb()
+
+    @classmethod
+    def teardownClass(cls):
+        cls.destroy_db(cls.connection)
+
+    @staticmethod
+    def initdb():
+        # connect to the db
+        _options = {}
+        _options.update(OPTIONS)
+        store = MixedModuleStore(**_options)
+
+        import_from_xml(
+            store._get_modulestore_for_courseid(IMPORT_COURSEID),
+            DATA_DIR,
+            ['toy'],
+            target_location_namespace=Location(
+                'i4x',
+                TestMixedModuleStore.import_org,
+                TestMixedModuleStore.import_course,
+                'course',
+                TestMixedModuleStore.import_run
+            )
+        )
+
+        return store
+
+    @staticmethod
+    def destroy_db(connection):
+        # Destroy the test db.
+        connection.drop_database(DB)
+
+    def setUp(self):
+        # make a copy for convenience
+        self.connection = TestMixedModuleStore.connection
+
+    def tearDown(self):
+        pass
+
+    def test_get_modulestore_type(self):
+        """
+        Make sure we get back the store type we expect for given mappings
+        """
+        assert_equals(self.store.get_modulestore_type(XML_COURSEID1), XML_MODULESTORE_TYPE)
+        assert_equals(self.store.get_modulestore_type(XML_COURSEID2), XML_MODULESTORE_TYPE)
+        assert_equals(self.store.get_modulestore_type(IMPORT_COURSEID), MONGO_MODULESTORE_TYPE)
+        # try an unknown mapping, it should be the 'default' store
+        assert_equals(self.store.get_modulestore_type('foo/bar/2012_Fall'), MONGO_MODULESTORE_TYPE)
+
+    def test_has_item(self):
+        assert_true(self.store.has_item(
+            IMPORT_COURSEID, Location(['i4x', self.import_org, self.import_course, 'course', self.import_run])
+        ))
+        assert_true(self.store.has_item(
+            XML_COURSEID1, Location(['i4x', 'edX', 'toy', 'course', '2012_Fall'])
+        ))
+
+        # try negative cases
+        assert_false(self.store.has_item(
+            XML_COURSEID1, Location(['i4x', self.import_org, self.import_course, 'course', self.import_run])
+        ))
+        assert_false(self.store.has_item(
+            IMPORT_COURSEID, Location(['i4x', 'edX', 'toy', 'course', '2012_Fall'])
+        ))
+
+    def test_get_item(self):
+        with assert_raises(NotImplementedError):
+            self.store.get_item(self.fake_location)
+
+    def test_get_instance(self):
+        module = self.store.get_instance(
+            IMPORT_COURSEID, Location(['i4x', self.import_org, self.import_course, 'course', self.import_run])
+        )
+        assert_not_equals(module, None)
+
+        module = self.store.get_instance(
+            XML_COURSEID1, Location(['i4x', 'edX', 'toy', 'course', '2012_Fall'])
+        )
+        assert_not_equals(module, None)
+
+        # try negative cases
+        with assert_raises(ItemNotFoundError):
+            self.store.get_instance(
+                XML_COURSEID1, Location(['i4x', self.import_org, self.import_course, 'course', self.import_run])
+            )
+
+        with assert_raises(ItemNotFoundError):
+            self.store.get_instance(
+                IMPORT_COURSEID, Location(['i4x', 'edX', 'toy', 'course', '2012_Fall'])
+            )
+
+    def test_get_items(self):
+        modules = self.store.get_items(['i4x', None, None, 'course', None], IMPORT_COURSEID)
+        assert_equals(len(modules), 1)
+        assert_equals(modules[0].location.course, self.import_course)
+
+        modules = self.store.get_items(['i4x', None, None, 'course', None], XML_COURSEID1)
+        assert_equals(len(modules), 1)
+        assert_equals(modules[0].location.course, 'toy')
+
+        modules = self.store.get_items(['i4x', None, None, 'course', None], XML_COURSEID2)
+        assert_equals(len(modules), 1)
+        assert_equals(modules[0].location.course, 'simple')
+
+    def test_update_item(self):
+        with assert_raises(NotImplementedError):
+            self.store.update_item(self.fake_location, None)
+
+    def test_update_children(self):
+        with assert_raises(NotImplementedError):
+            self.store.update_children(self.fake_location, None)
+
+    def test_update_metadata(self):
+        with assert_raises(NotImplementedError):
+            self.store.update_metadata(self.fake_location, None)
+
+    def test_delete_item(self):
+        with assert_raises(NotImplementedError):
+            self.store.delete_item(self.fake_location)
+
+    def test_get_courses(self):
+        # we should have 3 total courses aggregated
+        courses = self.store.get_courses()
+        assert_equals(len(courses), 3)
+        course_ids = []
+        for course in courses:
+            course_ids.append(course.location.course_id)
+        assert_true(IMPORT_COURSEID in course_ids)
+        assert_true(XML_COURSEID1 in course_ids)
+        assert_true(XML_COURSEID2 in course_ids)
+
+    def test_get_course(self):
+        module = self.store.get_course(IMPORT_COURSEID)
+        assert_equals(module.location.course, self.import_course)
+
+        module = self.store.get_course(XML_COURSEID1)
+        assert_equals(module.location.course, 'toy')
+
+        module = self.store.get_course(XML_COURSEID2)
+        assert_equals(module.location.course, 'simple')
+
+    def test_get_parent_locations(self):
+        parents = self.store.get_parent_locations(
+            Location(['i4x', self.import_org, self.import_course, 'chapter', 'Overview']),
+            IMPORT_COURSEID
+        )
+        assert_equals(len(parents), 1)
+        assert_equals(Location(parents[0]).org, self.import_org)
+        assert_equals(Location(parents[0]).course, self.import_course)
+        assert_equals(Location(parents[0]).name, self.import_run)
+
+        parents = self.store.get_parent_locations(
+            Location(['i4x', 'edX', 'toy', 'chapter', 'Overview']),
+            XML_COURSEID1
+        )
+        assert_equals(len(parents), 1)
+        assert_equals(Location(parents[0]).org, 'edX')
+        assert_equals(Location(parents[0]).course, 'toy')
+        assert_equals(Location(parents[0]).name, '2012_Fall')
+
+    def test_set_modulestore_configuration(self):
+        config = {'foo': 'bar'}
+        self.store.set_modulestore_configuration(config)
+        assert_equals(
+            config,
+            self.store._get_modulestore_for_courseid(IMPORT_COURSEID).modulestore_configuration
+        )
+
+        assert_equals(
+            config,
+            self.store._get_modulestore_for_courseid(XML_COURSEID1).modulestore_configuration
+        )
+
+        assert_equals(
+            config,
+            self.store._get_modulestore_for_courseid(XML_COURSEID2).modulestore_configuration
+        )

common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py

@@ -1,18 +1,18 @@
-import pymongo
+from pprint import pprint
 
 from nose.tools import assert_equals, assert_raises, assert_not_equals, assert_false
-from pprint import pprint
+import pymongo
+from uuid import uuid4
 
 from xblock.core import Scope
 from xblock.runtime import KeyValueStore, InvalidScopeError
 
+from xmodule.tests import DATA_DIR
 from xmodule.modulestore import Location
 from xmodule.modulestore.mongo import MongoModuleStore, MongoKeyValueStore
 from xmodule.modulestore.xml_importer import import_from_xml
 
-from .test_modulestore import check_path_to_location
-from . import DATA_DIR
-from uuid import uuid4
+from xmodule.modulestore.tests.test_modulestore import check_path_to_location
 
 
 HOST = 'localhost'
@@ -45,8 +45,7 @@ class TestMongoModuleStore(object):
     @staticmethod
     def initdb():
         # connect to the db
-        store = MongoModuleStore(HOST, DB, COLLECTION, FS_ROOT, RENDER_TEMPLATE,
-            default_class=DEFAULT_CLASS)
+        store = MongoModuleStore(HOST, DB, COLLECTION, FS_ROOT, RENDER_TEMPLATE, default_class=DEFAULT_CLASS)
         # Explicitly list the courses to load (don't want the big one)
         courses = ['toy', 'simple']
         import_from_xml(store, DATA_DIR, courses)
@@ -71,6 +70,10 @@ class TestMongoModuleStore(object):
 
         pprint([Location(i['_id']).url() for i in ids])
 
+    def test_mongo_modulestore_type(self):
+        store = MongoModuleStore(HOST, DB, COLLECTION, FS_ROOT, RENDER_TEMPLATE, default_class=DEFAULT_CLASS)
+        assert_equals(store.get_modulestore_type('foo/bar/baz'), 'mongo')
+
     def test_get_courses(self):
         '''Make sure the course objects loaded properly'''
         courses = self.store.get_courses()
@@ -117,6 +120,7 @@ class TestMongoModuleStore(object):
                 '{0} is a template course'.format(course)
             )
 
+
 class TestMongoKeyValueStore(object):
 
     def setUp(self):

common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py

@@ -107,7 +107,7 @@ class SplitModuleCourseTests(SplitModuleTest):
     '''
 
     def test_get_courses(self):
-        courses = modulestore().get_courses('draft')
+        courses = modulestore().get_courses(branch='draft')
         # should have gotten 3 draft courses
         self.assertEqual(len(courses), 3, "Wrong number of courses")
         # check metadata -- NOTE no promised order
@@ -138,35 +138,40 @@ class SplitModuleCourseTests(SplitModuleTest):
 
     def test_branch_requests(self):
         # query w/ branch qualifier (both draft and published)
-        courses_published = modulestore().get_courses('published')
-        self.assertEqual(len(courses_published), 1, len(courses_published))
-        course = self.findByIdInResult(courses_published, "head23456")
-        self.assertIsNotNone(course, "published courses")
-        self.assertEqual(course.location.course_id, "wonderful")
-        self.assertEqual(str(course.location.version_guid), self.GUID_P,
-                         course.location.version_guid)
-        self.assertEqual(course.category, 'course', 'wrong category')
-        self.assertEqual(len(course.tabs), 4, "wrong number of tabs")
-        self.assertEqual(course.display_name, "The most wonderful course",
-                         course.display_name)
-        self.assertIsNone(course.advertised_start)
-        self.assertEqual(len(course.children), 0,
-                         "children")
+        def _verify_published_course(courses_published):
+            """ Helper function for verifying published course. """
+            self.assertEqual(len(courses_published), 1, len(courses_published))
+            course = self.findByIdInResult(courses_published, "head23456")
+            self.assertIsNotNone(course, "published courses")
+            self.assertEqual(course.location.course_id, "wonderful")
+            self.assertEqual(str(course.location.version_guid), self.GUID_P,
+                             course.location.version_guid)
+            self.assertEqual(course.category, 'course', 'wrong category')
+            self.assertEqual(len(course.tabs), 4, "wrong number of tabs")
+            self.assertEqual(course.display_name, "The most wonderful course",
+                             course.display_name)
+            self.assertIsNone(course.advertised_start)
+            self.assertEqual(len(course.children), 0,
+                             "children")
+
+        _verify_published_course(modulestore().get_courses(branch='published'))
+        # default for branch is 'published'.
+        _verify_published_course(modulestore().get_courses())
 
     def test_search_qualifiers(self):
         # query w/ search criteria
-        courses = modulestore().get_courses('draft', qualifiers={'org': 'testx'})
+        courses = modulestore().get_courses(branch='draft', qualifiers={'org': 'testx'})
         self.assertEqual(len(courses), 2)
         self.assertIsNotNone(self.findByIdInResult(courses, "head12345"))
         self.assertIsNotNone(self.findByIdInResult(courses, "head23456"))
 
         courses = modulestore().get_courses(
-            'draft',
+            branch='draft',
             qualifiers={'edited_on': {"$lt": datetime.datetime(2013, 3, 28, 15)}})
         self.assertEqual(len(courses), 2)
 
         courses = modulestore().get_courses(
-            'draft',
+            branch='draft',
             qualifiers={'org': 'testx', "prettyid": "test_course"})
         self.assertEqual(len(courses), 1)
         self.assertIsNotNone(self.findByIdInResult(courses, "head12345"))
@@ -182,6 +187,7 @@ class SplitModuleCourseTests(SplitModuleTest):
         self.assertEqual(course.category, 'course')
         self.assertEqual(len(course.tabs), 6)
         self.assertEqual(course.display_name, "The Ancient Greek Hero")
+        self.assertEqual(course.lms.graceperiod, datetime.timedelta(hours=2))
         self.assertIsNone(course.advertised_start)
         self.assertEqual(len(course.children), 0)
         self.assertEqual(course.definition_locator.definition_id, "head12345_11")
@@ -252,18 +258,19 @@ class SplitModuleItemTests(SplitModuleTest):
         '''
         has_item(BlockUsageLocator)
         '''
+        course_id = 'GreekHero'
         # positive tests of various forms
         locator = BlockUsageLocator(version_guid=self.GUID_D1, usage_id='head12345')
-        self.assertTrue(modulestore().has_item(locator),
+        self.assertTrue(modulestore().has_item(course_id, locator),
                         "couldn't find in %s" % self.GUID_D1)
 
         locator = BlockUsageLocator(course_id='GreekHero', usage_id='head12345', branch='draft')
         self.assertTrue(
-            modulestore().has_item(locator),
+            modulestore().has_item(locator.course_id, locator),
             "couldn't find in 12345"
         )
         self.assertTrue(
-            modulestore().has_item(BlockUsageLocator(
+            modulestore().has_item(locator.course_id, BlockUsageLocator(
                 course_id=locator.course_id,
                 branch='draft',
                 usage_id=locator.usage_id
@@ -271,7 +278,7 @@ class SplitModuleItemTests(SplitModuleTest):
             "couldn't find in draft 12345"
         )
         self.assertFalse(
-            modulestore().has_item(BlockUsageLocator(
+            modulestore().has_item(locator.course_id, BlockUsageLocator(
                 course_id=locator.course_id,
                 branch='published',
                 usage_id=locator.usage_id)),
@@ -279,40 +286,43 @@ class SplitModuleItemTests(SplitModuleTest):
         )
         locator.branch = 'draft'
         self.assertTrue(
-            modulestore().has_item(locator),
+            modulestore().has_item(locator.course_id, locator),
             "not found in draft 12345"
         )
 
         # not a course obj
         locator = BlockUsageLocator(course_id='GreekHero', usage_id='chapter1', branch='draft')
         self.assertTrue(
-            modulestore().has_item(locator),
+            modulestore().has_item(locator.course_id, locator),
             "couldn't find chapter1"
         )
 
         # in published course
         locator = BlockUsageLocator(course_id="wonderful", usage_id="head23456", branch='draft')
-        self.assertTrue(modulestore().has_item(BlockUsageLocator(course_id=locator.course_id,
-                                                                 usage_id=locator.usage_id,
-                                                                 branch='published')),
-                        "couldn't find in 23456")
+        self.assertTrue(
+            modulestore().has_item(
+                locator.course_id,
+                BlockUsageLocator(course_id=locator.course_id, usage_id=locator.usage_id, branch='published')
+            ), "couldn't find in 23456"
+        )
         locator.branch = 'published'
-        self.assertTrue(modulestore().has_item(locator), "couldn't find in 23456")
+        self.assertTrue(modulestore().has_item(course_id, locator), "couldn't find in 23456")
 
     def test_negative_has_item(self):
         # negative tests--not found
         # no such course or block
+        course_id = 'GreekHero'
         locator = BlockUsageLocator(course_id="doesnotexist", usage_id="head23456", branch='draft')
-        self.assertFalse(modulestore().has_item(locator))
+        self.assertFalse(modulestore().has_item(course_id, locator))
         locator = BlockUsageLocator(course_id="wonderful", usage_id="doesnotexist", branch='draft')
-        self.assertFalse(modulestore().has_item(locator))
+        self.assertFalse(modulestore().has_item(course_id, locator))
 
         # negative tests--insufficient specification
         self.assertRaises(InsufficientSpecificationError, BlockUsageLocator)
         self.assertRaises(InsufficientSpecificationError,
-                          modulestore().has_item, BlockUsageLocator(version_guid=self.GUID_D1))
+                          modulestore().has_item, None, BlockUsageLocator(version_guid=self.GUID_D1))
         self.assertRaises(InsufficientSpecificationError,
-                          modulestore().has_item, BlockUsageLocator(course_id='GreekHero'))
+                          modulestore().has_item, None, BlockUsageLocator(course_id='GreekHero'))
 
     def test_get_item(self):
         '''
@@ -322,21 +332,26 @@ class SplitModuleItemTests(SplitModuleTest):
         locator = BlockUsageLocator(version_guid=self.GUID_D1, usage_id='head12345')
         block = modulestore().get_item(locator)
         self.assertIsInstance(block, CourseDescriptor)
+        # get_instance just redirects to get_item, ignores course_id
+        self.assertIsInstance(modulestore().get_instance("course_id", locator), CourseDescriptor)
+
+        def verify_greek_hero(block):
+            self.assertEqual(block.location.course_id, "GreekHero")
+            self.assertEqual(len(block.tabs), 6, "wrong number of tabs")
+            self.assertEqual(block.display_name, "The Ancient Greek Hero")
+            self.assertEqual(block.advertised_start, "Fall 2013")
+            self.assertEqual(len(block.children), 3)
+            self.assertEqual(block.definition_locator.definition_id, "head12345_12")
+            # check dates and graders--forces loading of descriptor
+            self.assertEqual(block.edited_by, "testassist@edx.org")
+            self.assertDictEqual(
+                block.grade_cutoffs, {"Pass": 0.45},
+            )
 
         locator = BlockUsageLocator(course_id='GreekHero', usage_id='head12345', branch='draft')
-        block = modulestore().get_item(locator)
-        self.assertEqual(block.location.course_id, "GreekHero")
-        # look at this one in detail
-        self.assertEqual(len(block.tabs), 6, "wrong number of tabs")
-        self.assertEqual(block.display_name, "The Ancient Greek Hero")
-        self.assertEqual(block.advertised_start, "Fall 2013")
-        self.assertEqual(len(block.children), 3)
-        self.assertEqual(block.definition_locator.definition_id, "head12345_12")
-        # check dates and graders--forces loading of descriptor
-        self.assertEqual(block.edited_by, "testassist@edx.org")
-        self.assertDictEqual(
-            block.grade_cutoffs, {"Pass": 0.45},
-        )
+        verify_greek_hero(modulestore().get_item(locator))
+        # get_instance just redirects to get_item, ignores course_id
+        verify_greek_hero(modulestore().get_instance("course_id", locator))
 
         # try to look up other branches
         self.assertRaises(ItemNotFoundError,
@@ -415,22 +430,25 @@ class SplitModuleItemTests(SplitModuleTest):
         '''
         locator = CourseLocator(version_guid=self.GUID_D0)
         # get all modules
-        matches = modulestore().get_items(locator, {})
+        matches = modulestore().get_items(locator)
         self.assertEqual(len(matches), 6)
-        matches = modulestore().get_items(locator, {'category': 'chapter'})
+        matches = modulestore().get_items(locator, qualifiers={})
+        self.assertEqual(len(matches), 6)
+        matches = modulestore().get_items(locator, qualifiers={'category': 'chapter'})
         self.assertEqual(len(matches), 3)
-        matches = modulestore().get_items(locator, {'category': 'garbage'})
+        matches = modulestore().get_items(locator, qualifiers={'category': 'garbage'})
         self.assertEqual(len(matches), 0)
         matches = modulestore().get_items(
             locator,
+            qualifiers=
             {
                 'category': 'chapter',
-                'metadata': {'display_name': {'$regex': 'Hera'}}
+                'fields': {'display_name': {'$regex': 'Hera'}}
             }
         )
         self.assertEqual(len(matches), 2)
 
-        matches = modulestore().get_items(locator, {'children': 'chapter2'})
+        matches = modulestore().get_items(locator, qualifiers={'fields': {'children': 'chapter2'}})
         self.assertEqual(len(matches), 1)
         self.assertEqual(matches[0].location.usage_id, 'head12345')
 
@@ -438,8 +456,8 @@ class SplitModuleItemTests(SplitModuleTest):
         '''
         get_parent_locations(locator, [usage_id], [branch]): [BlockUsageLocator]
         '''
-        locator = CourseLocator(course_id="GreekHero", branch='draft')
-        parents = modulestore().get_parent_locations(locator, usage_id='chapter1')
+        locator = BlockUsageLocator(course_id="GreekHero", branch='draft', usage_id='chapter1')
+        parents = modulestore().get_parent_locations(locator)
         self.assertEqual(len(parents), 1)
         self.assertEqual(parents[0].usage_id, 'head12345')
         self.assertEqual(parents[0].course_id, "GreekHero")
@@ -447,7 +465,8 @@ class SplitModuleItemTests(SplitModuleTest):
         parents = modulestore().get_parent_locations(locator)
         self.assertEqual(len(parents), 1)
         self.assertEqual(parents[0].usage_id, 'head12345')
-        parents = modulestore().get_parent_locations(locator, usage_id='nosuchblock')
+        locator.usage_id = 'nosuchblock'
+        parents = modulestore().get_parent_locations(locator)
         self.assertEqual(len(parents), 0)
 
     def test_get_children(self):
@@ -493,8 +512,7 @@ class TestItemCrud(SplitModuleTest):
 
     def test_create_minimal_item(self):
         """
-        create_item(course_or_parent_locator, category, user, definition_locator=None, new_def_data=None,
-        metadata=None): new_desciptor
+        create_item(course_or_parent_locator, category, user, definition_locator=None, fields): new_desciptor
         """
         # grab link to course to ensure new versioning works
         locator = CourseLocator(course_id="GreekHero", branch='draft')
@@ -504,7 +522,7 @@ class TestItemCrud(SplitModuleTest):
         category = 'sequential'
         new_module = modulestore().create_item(
             locator, category, 'user123',
-            metadata={'display_name': 'new sequential'}
+            fields={'display_name': 'new sequential'}
         )
         # check that course version changed and course's previous is the other one
         self.assertEqual(new_module.location.course_id, "GreekHero")
@@ -539,7 +557,7 @@ class TestItemCrud(SplitModuleTest):
         category = 'chapter'
         new_module = modulestore().create_item(
             locator, category, 'user123',
-            metadata={'display_name': 'new chapter'},
+            fields={'display_name': 'new chapter'},
             definition_locator=DescriptionLocator("chapter12345_2")
         )
         # check that course version changed and course's previous is the other one
@@ -560,15 +578,13 @@ class TestItemCrud(SplitModuleTest):
         new_payload = "<problem>empty</problem>"
         new_module = modulestore().create_item(
             locator, category, 'anotheruser',
-            metadata={'display_name': 'problem 1'},
-            new_def_data=new_payload
+            fields={'display_name': 'problem 1', 'data': new_payload},
         )
         another_payload = "<problem>not empty</problem>"
         another_module = modulestore().create_item(
             locator, category, 'anotheruser',
-            metadata={'display_name': 'problem 2'},
+            fields={'display_name': 'problem 2', 'data': another_payload},
             definition_locator=DescriptionLocator("problem12345_3_1"),
-            new_def_data=another_payload
         )
         # check that course version changed and course's previous is the other one
         parent = modulestore().get_item(locator)
@@ -602,6 +618,7 @@ class TestItemCrud(SplitModuleTest):
         self.assertNotEqual(problem.max_attempts, 4, "Invalidates rest of test")
 
         problem.max_attempts = 4
+        problem.save()  # decache above setting into the kvs
         updated_problem = modulestore().update_item(problem, 'changeMaven')
         # check that course version changed and course's previous is the other one
         self.assertEqual(updated_problem.definition_locator.definition_id, pre_def_id)
@@ -637,6 +654,7 @@ class TestItemCrud(SplitModuleTest):
         # reorder children
         self.assertGreater(len(block.children), 0, "meaningless test")
         moved_child = block.children.pop()
+        block.save()  # decache model changes
         updated_problem = modulestore().update_item(block, 'childchanger')
         # check that course version changed and course's previous is the other one
         self.assertEqual(updated_problem.definition_locator.definition_id, pre_def_id)
@@ -646,6 +664,7 @@ class TestItemCrud(SplitModuleTest):
         locator.usage_id = "chapter1"
         other_block = modulestore().get_item(locator)
         other_block.children.append(moved_child)
+        other_block.save()  # decache model changes
         other_updated = modulestore().update_item(other_block, 'childchanger')
         self.assertIn(moved_child, other_updated.children)
 
@@ -659,6 +678,7 @@ class TestItemCrud(SplitModuleTest):
         pre_version_guid = block.location.version_guid
 
         block.grading_policy['GRADER'][0]['min_count'] = 13
+        block.save()  # decache model changes
         updated_block = modulestore().update_item(block, 'definition_changer')
 
         self.assertNotEqual(updated_block.definition_locator.definition_id, pre_def_id)
@@ -675,15 +695,13 @@ class TestItemCrud(SplitModuleTest):
         new_payload = "<problem>empty</problem>"
         modulestore().create_item(
             locator, category, 'test_update_manifold',
-            metadata={'display_name': 'problem 1'},
-            new_def_data=new_payload
+            fields={'display_name': 'problem 1', 'data': new_payload},
         )
         another_payload = "<problem>not empty</problem>"
         modulestore().create_item(
             locator, category, 'test_update_manifold',
-            metadata={'display_name': 'problem 2'},
+            fields={'display_name': 'problem 2', 'data': another_payload},
             definition_locator=DescriptionLocator("problem12345_3_1"),
-            new_def_data=another_payload
         )
         # pylint: disable=W0212
         modulestore()._clear_cache()
@@ -698,6 +716,7 @@ class TestItemCrud(SplitModuleTest):
         block.children = block.children[1:] + [block.children[0]]
         block.advertised_start = "Soon"
 
+        block.save()  # decache model changes
         updated_block = modulestore().update_item(block, "test_update_manifold")
         self.assertNotEqual(updated_block.definition_locator.definition_id, pre_def_id)
         self.assertNotEqual(updated_block.location.version_guid, pre_version_guid)
@@ -719,28 +738,28 @@ class TestItemCrud(SplitModuleTest):
         # delete a leaf
         problems = modulestore().get_items(reusable_location, {'category': 'problem'})
         locn_to_del = problems[0].location
-        new_course_loc = modulestore().delete_item(locn_to_del, 'deleting_user')
+        new_course_loc = modulestore().delete_item(locn_to_del, 'deleting_user', delete_children=True)
         deleted = BlockUsageLocator(course_id=reusable_location.course_id,
                                     branch=reusable_location.branch,
                                     usage_id=locn_to_del.usage_id)
-        self.assertFalse(modulestore().has_item(deleted))
-        self.assertRaises(VersionConflictError, modulestore().has_item, locn_to_del)
+        self.assertFalse(modulestore().has_item(reusable_location.course_id, deleted))
+        self.assertRaises(VersionConflictError, modulestore().has_item, reusable_location.course_id, locn_to_del)
         locator = BlockUsageLocator(
             version_guid=locn_to_del.version_guid,
             usage_id=locn_to_del.usage_id
         )
-        self.assertTrue(modulestore().has_item(locator))
+        self.assertTrue(modulestore().has_item(reusable_location.course_id, locator))
         self.assertNotEqual(new_course_loc.version_guid, course.location.version_guid)
 
         # delete a subtree
         nodes = modulestore().get_items(reusable_location, {'category': 'chapter'})
-        new_course_loc = modulestore().delete_item(nodes[0].location, 'deleting_user')
+        new_course_loc = modulestore().delete_item(nodes[0].location, 'deleting_user', delete_children=True)
         # check subtree
 
         def check_subtree(node):
             if node:
                 node_loc = node.location
-                self.assertFalse(modulestore().has_item(
+                self.assertFalse(modulestore().has_item(reusable_location.course_id,
                     BlockUsageLocator(
                         course_id=node_loc.course_id,
                         branch=node_loc.branch,
@@ -748,7 +767,7 @@ class TestItemCrud(SplitModuleTest):
                 locator = BlockUsageLocator(
                     version_guid=node.location.version_guid,
                     usage_id=node.location.usage_id)
-                self.assertTrue(modulestore().has_item(locator))
+                self.assertTrue(modulestore().has_item(reusable_location.course_id, locator))
                 if node.has_children:
                     for sub in node.get_children():
                         check_subtree(sub)
@@ -841,7 +860,7 @@ class TestCourseCreation(SplitModuleTest):
         # using new_draft.location will insert the chapter under the course root
         new_item = modulestore().create_item(
             new_draft.location, 'chapter', 'leech_master',
-            metadata={'display_name': 'new chapter'}
+            fields={'display_name': 'new chapter'}
         )
         new_draft_locator.version_guid = None
         new_index = modulestore().get_course_index_info(new_draft_locator)
@@ -859,7 +878,7 @@ class TestCourseCreation(SplitModuleTest):
         original_course = modulestore().get_course(original_locator)
         self.assertEqual(original_course.location.version_guid, original_index['versions']['draft'])
         self.assertFalse(
-            modulestore().has_item(BlockUsageLocator(
+            modulestore().has_item(new_draft_locator.course_id, BlockUsageLocator(
                 original_locator,
                 usage_id=new_item.location.usage_id
             ))
@@ -873,20 +892,18 @@ class TestCourseCreation(SplitModuleTest):
         original_locator = CourseLocator(course_id="contender", branch='draft')
         original = modulestore().get_course(original_locator)
         original_index = modulestore().get_course_index_info(original_locator)
-        data_payload = {}
-        metadata_payload = {}
+        fields = {}
         for field in original.fields:
             if field.scope == Scope.content and field.name != 'location':
-                data_payload[field.name] = getattr(original, field.name)
+                fields[field.name] = getattr(original, field.name)
             elif field.scope == Scope.settings:
-                metadata_payload[field.name] = getattr(original, field.name)
-        data_payload['grading_policy']['GRADE_CUTOFFS'] = {'A': .9, 'B': .8, 'C': .65}
-        metadata_payload['display_name'] = 'Derivative'
+                fields[field.name] = getattr(original, field.name)
+        fields['grading_policy']['GRADE_CUTOFFS'] = {'A': .9, 'B': .8, 'C': .65}
+        fields['display_name'] = 'Derivative'
         new_draft = modulestore().create_course(
             'leech', 'derivative', 'leech_master', id_root='counter',
             versions_dict={'draft': original_index['versions']['draft']},
-            course_data=data_payload,
-            metadata=metadata_payload
+            fields=fields
         )
         new_draft_locator = new_draft.location
         self.assertRegexpMatches(new_draft_locator.course_id, r'counter.*')
@@ -899,10 +916,10 @@ class TestCourseCreation(SplitModuleTest):
         self.assertGreaterEqual(new_index["edited_on"], pre_time)
         self.assertLessEqual(new_index["edited_on"], datetime.datetime.now(UTC))
         self.assertEqual(new_index['edited_by'], 'leech_master')
-        self.assertEqual(new_draft.display_name, metadata_payload['display_name'])
+        self.assertEqual(new_draft.display_name, fields['display_name'])
         self.assertDictEqual(
             new_draft.grading_policy['GRADE_CUTOFFS'],
-            data_payload['grading_policy']['GRADE_CUTOFFS']
+            fields['grading_policy']['GRADE_CUTOFFS']
         )
 
     def test_update_course_index(self):

common/lib/xmodule/xmodule/modulestore/tests/test_xml.py

@@ -1,12 +1,13 @@
 import os.path
 
+from nose.tools import assert_raises, assert_equals
+
 from xmodule.course_module import CourseDescriptor
 from xmodule.modulestore.xml import XMLModuleStore
-
-from nose.tools import assert_raises
+from xmodule.modulestore import XML_MODULESTORE_TYPE
 
 from .test_modulestore import check_path_to_location
-from . import DATA_DIR
+from xmodule.tests import DATA_DIR
 
 
 class TestXMLModuleStore(object):
@@ -19,6 +20,10 @@ class TestXMLModuleStore(object):
 
         check_path_to_location(modulestore)
 
+    def test_xml_modulestore_type(self):
+        store = XMLModuleStore(DATA_DIR, course_dirs=['toy', 'simple'])
+        assert_equals(store.get_modulestore_type('foo/bar/baz'), XML_MODULESTORE_TYPE)
+
     def test_unicode_chars_in_xml_content(self):
         # edX/full/6.002_Spring_2012 has non-ASCII chars, and during
         # uniquification of names, would raise a UnicodeError. It no longer does.

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -21,7 +21,7 @@ from xmodule.x_module import XModuleDescriptor, XMLParsingSystem
 
 from xmodule.html_module import HtmlDescriptor
 
-from . import ModuleStoreBase, Location
+from . import ModuleStoreBase, Location, XML_MODULESTORE_TYPE
 from .exceptions import ItemNotFoundError
 from .inheritance import compute_inherited_metadata
 
@@ -505,12 +505,12 @@ class XMLModuleStore(ModuleStoreBase):
         except KeyError:
             raise ItemNotFoundError(location)
 
-    def has_item(self, location):
+    def has_item(self, course_id, location):
         """
         Returns True if location exists in this ModuleStore.
         """
         location = Location(location)
-        return any(location in course_modules for course_modules in self.modules.values())
+        return location in self.modules[course_id]
 
     def get_item(self, location, depth=0):
         """
@@ -601,3 +601,10 @@ class XMLModuleStore(ModuleStoreBase):
             raise ItemNotFoundError("{0} not in {1}".format(location, course_id))
 
         return self.parent_trackers[course_id].parents(location)
+
+    def get_modulestore_type(self, course_id):
+        """
+        Returns a type which identifies which modulestore is servicing the given
+        course_id. The return can be either "xml" (for XML based courses) or "mongo" for MongoDB backed courses
+        """
+        return XML_MODULESTORE_TYPE

common/lib/xmodule/xmodule/modulestore/xml_importer.py

@@ -527,57 +527,3 @@ def perform_xlint(data_dir, course_dirs,
         print "This course can be imported successfully."
 
     return err_cnt
-
-
-#
-# UNSURE IF THIS IS UNUSED CODE - IF SO NEEDS TO BE PRUNED. TO BE INVESTIGATED.
-#
-def import_module_from_xml(modulestore, static_content_store, course_data_path, module, target_location_namespace=None, verbose=False):
-    # remap module to the new namespace
-    if target_location_namespace is not None:
-        # This looks a bit wonky as we need to also change the 'name' of the imported course to be what
-        # the caller passed in
-        if module.location.category != 'course':
-            module.location = module.location._replace(tag=target_location_namespace.tag, org=target_location_namespace.org,
-                                                       course=target_location_namespace.course)
-        else:
-            module.location = module.location._replace(tag=target_location_namespace.tag, org=target_location_namespace.org,
-                                                       course=target_location_namespace.course, name=target_location_namespace.name)
-
-        # then remap children pointers since they too will be re-namespaced
-        if module.has_children:
-            children_locs = module.children
-            new_locs = []
-            for child in children_locs:
-                child_loc = Location(child)
-                new_child_loc = child_loc._replace(tag=target_location_namespace.tag, org=target_location_namespace.org,
-                                                   course=target_location_namespace.course)
-
-                new_locs.append(new_child_loc.url())
-
-            module.children = new_locs
-
-    if hasattr(module, 'data'):
-        modulestore.update_item(module.location, module.data)
-
-    if module.has_children:
-        modulestore.update_children(module.location, module.children)
-
-    modulestore.update_metadata(module.location, own_metadata(module))
-
-
-def import_course_from_xml(modulestore, static_content_store, course_data_path, module, target_location_namespace=None, verbose=False):
-    # CDODGE: Is this unused code (along with import_module_from_xml)? I can't find any references to it. If so, then
-    # we need to delete this apparently duplicate code.
-
-    # cdodge: more hacks (what else). Seems like we have a problem when importing a course (like 6.002) which
-    # does not have any tabs defined in the policy file. The import goes fine and then displays fine in LMS,
-    # but if someone tries to add a new tab in the CMS, then the LMS barfs because it expects that -
-    # if there is *any* tabs - then there at least needs to be some predefined ones
-    if module.tabs is None or len(module.tabs) == 0:
-        module.tabs = [{"type": "courseware"},
-                       {"type": "course_info", "name": "Course Info"},
-                       {"type": "discussion", "name": "Discussion"},
-                       {"type": "wiki", "name": "Wiki"}]  # note, add 'progress' when we can support it on Edge
-
-    import_module_from_xml(modulestore, static_content_store, course_data_path, module, target_location_namespace, verbose=verbose)

common/lib/xmodule/xmodule/open_ended_grading_classes/combined_open_ended_modulev1.py

@@ -106,6 +106,11 @@ class CombinedOpenEndedV1Module():
         self.accept_file_upload = instance_state.get('accept_file_upload', ACCEPT_FILE_UPLOAD) in TRUE_DICT
         self.skip_basic_checks = instance_state.get('skip_spelling_checks', SKIP_BASIC_CHECKS) in TRUE_DICT
 
+        self.required_peer_grading = instance_state.get('required_peer_grading', 3)
+        self.peer_grader_count = instance_state.get('peer_grader_count', 3)
+        self.min_to_calibrate = instance_state.get('min_to_calibrate', 3)
+        self.max_to_calibrate = instance_state.get('max_to_calibrate', 6)
+
         due_date = instance_state.get('due', None)
 
         grace_period_string = instance_state.get('graceperiod', None)
@@ -131,6 +136,12 @@ class CombinedOpenEndedV1Module():
             'close_date': self.timeinfo.close_date,
             's3_interface': self.system.s3_interface,
             'skip_basic_checks': self.skip_basic_checks,
+            'control': {
+                'required_peer_grading': self.required_peer_grading,
+                'peer_grader_count': self.peer_grader_count,
+                'min_to_calibrate': self.min_to_calibrate,
+                'max_to_calibrate': self.max_to_calibrate,
+            }
         }
 
         self.task_xml = definition['task_xml']

common/lib/xmodule/xmodule/open_ended_grading_classes/open_ended_module.py

@@ -118,6 +118,7 @@ class OpenEndedModule(openendedchild.OpenEndedChild):
             'answer': self.answer,
             'problem_id': self.display_name,
             'skip_basic_checks': self.skip_basic_checks,
+            'control': json.dumps(self.control),
         })
         updated_grader_payload = json.dumps(parsed_grader_payload)
 

common/lib/xmodule/xmodule/open_ended_grading_classes/openendedchild.py

@@ -92,6 +92,7 @@ class OpenEndedChild(object):
         self.s3_interface = static_data['s3_interface']
         self.skip_basic_checks = static_data['skip_basic_checks']
         self._max_score = static_data['max_score']
+        self.control = static_data['control']
 
         # Used for progress / grading.  Currently get credit just for
         # completion (doesn't matter if you self-assessed correct/incorrect).

common/lib/xmodule/xmodule/tests/__init__.py

@@ -7,19 +7,24 @@ Run like this:
 
 """
 
-import unittest
+import json
 import os
+import unittest
+
 import fs
 import fs.osfs
-
 import numpy
-
-import json
+from mock import Mock
+from path import path
 
 import calc
-import xmodule
-from xmodule.x_module import ModuleSystem
-from mock import Mock
+from xmodule.x_module import ModuleSystem, XModuleDescriptor
+
+
+# Location of common test DATA directory
+# '../../../../edx-platform/common/test/data/'
+MODULE_DIR = path(__file__).dirname()
+DATA_DIR = path.joinpath(*MODULE_DIR.splitall()[:-4]) / 'test/data/'
 
 
 open_ended_grading_interface = {
@@ -67,7 +72,7 @@ class ModelsTest(unittest.TestCase):
         pass
 
     def test_load_class(self):
-        vc = xmodule.x_module.XModuleDescriptor.load_class('video')
+        vc = XModuleDescriptor.load_class('video')
         vc_str = "<class 'xmodule.video_module.VideoDescriptor'>"
         self.assertEqual(str(vc), vc_str)
 

common/lib/xmodule/xmodule/tests/test_combined_open_ended.py

@@ -1,31 +1,3 @@
-import json
-from mock import Mock, MagicMock, ANY
-import unittest
-
-from test_util_open_ended import MockQueryDict, DummyModulestore
-
-from xmodule.open_ended_grading_classes.openendedchild import OpenEndedChild
-from xmodule.open_ended_grading_classes.open_ended_module import OpenEndedModule
-from xmodule.open_ended_grading_classes.combined_open_ended_modulev1 import CombinedOpenEndedV1Module
-from xmodule.open_ended_grading_classes.grading_service_module import GradingServiceError
-from xmodule.combined_open_ended_module import CombinedOpenEndedModule
-from xmodule.modulestore import Location
-
-from lxml import etree
-import capa.xqueue_interface as xqueue_interface
-from datetime import datetime
-from pytz import UTC
-import logging
-
-log = logging.getLogger(__name__)
-
-from . import get_test_system
-
-ORG = 'edX'
-COURSE = 'open_ended'      # name of directory with course data
-
-import test_util_open_ended
-
 """
 Tests for the various pieces of the CombinedOpenEndedGrading system
 
@@ -34,6 +6,31 @@ OpenEndedModule
 
 """
 
+from datetime import datetime
+import json
+import logging
+import unittest
+
+from lxml import etree
+from mock import Mock, MagicMock, ANY
+from pytz import UTC
+
+from xmodule.open_ended_grading_classes.openendedchild import OpenEndedChild
+from xmodule.open_ended_grading_classes.open_ended_module import OpenEndedModule
+from xmodule.open_ended_grading_classes.combined_open_ended_modulev1 import CombinedOpenEndedV1Module
+from xmodule.open_ended_grading_classes.grading_service_module import GradingServiceError
+from xmodule.combined_open_ended_module import CombinedOpenEndedModule
+from xmodule.modulestore import Location
+from xmodule.tests import get_test_system, test_util_open_ended
+from xmodule.tests.test_util_open_ended import MockQueryDict, DummyModulestore
+import capa.xqueue_interface as xqueue_interface
+
+
+log = logging.getLogger(__name__)
+
+ORG = 'edX'
+COURSE = 'open_ended'      # name of directory with course data
+
 
 class OpenEndedChildTest(unittest.TestCase):
     """
@@ -64,6 +61,12 @@ class OpenEndedChildTest(unittest.TestCase):
         's3_interface': "",
         'open_ended_grading_interface': {},
         'skip_basic_checks': False,
+        'control': {
+            'required_peer_grading': 1,
+            'peer_grader_count': 1,
+            'min_to_calibrate': 3,
+            'max_to_calibrate': 6,
+            }
     }
     definition = Mock()
     descriptor = Mock()
@@ -180,6 +183,12 @@ class OpenEndedModuleTest(unittest.TestCase):
         's3_interface': test_util_open_ended.S3_INTERFACE,
         'open_ended_grading_interface': test_util_open_ended.OPEN_ENDED_GRADING_INTERFACE,
         'skip_basic_checks': False,
+        'control': {
+            'required_peer_grading': 1,
+            'peer_grader_count': 1,
+            'min_to_calibrate': 3,
+            'max_to_calibrate': 6,
+            }
     }
 
     oeparam = etree.XML('''

common/lib/xmodule/xmodule/tests/test_conditional.py

@@ -1,22 +1,21 @@
 
+from ast import literal_eval
 import json
 import unittest
+
 from fs.memoryfs import MemoryFS
-from ast import literal_eval
 from mock import Mock, patch
 
 from xmodule.error_module import NonStaffErrorDescriptor
 from xmodule.modulestore import Location
 from xmodule.modulestore.xml import ImportSystem, XMLModuleStore
 from xmodule.conditional_module import ConditionalModule
+from xmodule.tests import DATA_DIR, get_test_system
 
-from xmodule.tests.test_export import DATA_DIR
 
 ORG = 'test_org'
 COURSE = 'conditional'      # name of directory with course data
 
-from . import get_test_system
-
 
 class DummySystem(ImportSystem):
 

common/lib/xmodule/xmodule/tests/test_content.py

@@ -19,12 +19,18 @@ class ContentTest(unittest.TestCase):
 
         content = StaticContent('loc', 'name', 'content_type', 'data')
         self.assertIsNone(content.thumbnail_location)
+
+    def test_static_url_generation_from_courseid(self):
+        url = StaticContent.convert_legacy_static_url_with_course_id('images_course_image.jpg', 'foo/bar/bz')
+        self.assertEqual(url, '/c4x/foo/bar/asset/images_course_image.jpg')
+
     def test_generate_thumbnail_image(self):
         contentStore = ContentStore()
         content = Content(Location(u'c4x', u'mitX', u'800', u'asset', u'monsters__.jpg'), None)
         (thumbnail_content, thumbnail_file_location) = contentStore.generate_thumbnail(content)
         self.assertIsNone(thumbnail_content)
         self.assertEqual(Location(u'c4x', u'mitX', u'800', u'thumbnail', u'monsters__.jpg'), thumbnail_file_location)
+
     def test_compute_location(self):
         # We had a bug that __ got converted into a single _. Make sure that substitution of INVALID_CHARS (like space)
         # still happen.

common/lib/xmodule/xmodule/tests/test_export.py

@@ -2,28 +2,19 @@
 Tests of XML export
 """
 
-import unittest
-import pytz
-
 from datetime import datetime, timedelta, tzinfo
-from fs.osfs import OSFS
-from path import path
 from tempfile import mkdtemp
+import unittest
 import shutil
 
-from xmodule.modulestore.xml import XMLModuleStore
-from xmodule.modulestore.xml_exporter import EdxJSONEncoder
+import pytz
+from fs.osfs import OSFS
+from path import path
 
 from xmodule.modulestore import Location
-
-# from ~/mitx_all/mitx/common/lib/xmodule/xmodule/tests/
-# to   ~/mitx_all/mitx/common/test
-TEST_DIR = path(__file__).abspath().dirname()
-for i in range(4):
-    TEST_DIR = TEST_DIR.dirname()
-TEST_DIR = TEST_DIR / 'test'
-
-DATA_DIR = TEST_DIR / 'data'
+from xmodule.modulestore.xml import XMLModuleStore
+from xmodule.modulestore.xml_exporter import EdxJSONEncoder
+from xmodule.tests import DATA_DIR
 
 
 def strip_filenames(descriptor):

common/lib/xmodule/xmodule/tests/test_import.py

@@ -1,20 +1,21 @@
 # -*- coding: utf-8 -*-
 
+import datetime
 import unittest
-from fs.memoryfs import MemoryFS
 
+from fs.memoryfs import MemoryFS
 from lxml import etree
 from mock import Mock, patch
 
+from django.utils.timezone import UTC
+
 from xmodule.xml_module import is_pointer_tag
 from xmodule.modulestore import Location
 from xmodule.modulestore.xml import ImportSystem, XMLModuleStore
 from xmodule.modulestore.inheritance import compute_inherited_metadata
 from xmodule.fields import Date
+from xmodule.tests import DATA_DIR
 
-from .test_export import DATA_DIR
-import datetime
-from django.utils.timezone import UTC
 
 ORG = 'test_org'
 COURSE = 'test_course'

common/lib/xmodule/xmodule/tests/test_self_assessment.py

@@ -49,6 +49,12 @@ class SelfAssessmentTest(unittest.TestCase):
             's3_interface': test_util_open_ended.S3_INTERFACE,
             'open_ended_grading_interface': test_util_open_ended.OPEN_ENDED_GRADING_INTERFACE,
             'skip_basic_checks': False,
+            'control': {
+                'required_peer_grading': 1,
+                'peer_grader_count': 1,
+                'min_to_calibrate': 3,
+                'max_to_calibrate': 6,
+                }
         }
 
         self.module = SelfAssessmentModule(get_test_system(), self.location,

common/lib/xmodule/xmodule/tests/test_util_open_ended.py

@@ -1,7 +1,6 @@
-from .import get_test_system
 from xmodule.modulestore import Location
 from xmodule.modulestore.xml import XMLModuleStore
-from xmodule.tests.test_export import DATA_DIR
+from xmodule.tests import DATA_DIR, get_test_system
 
 OPEN_ENDED_GRADING_INTERFACE = {
     'url': 'blah/',
@@ -42,7 +41,7 @@ class DummyModulestore(object):
     def setup_modulestore(self, name):
         self.modulestore = XMLModuleStore(DATA_DIR, course_dirs=[name])
 
-    def get_course(self, name):
+    def get_course(self, _):
         """Get a test course by directory name.  If there's more than one, error."""
         courses = self.modulestore.get_courses()
         return courses[0]

common/lib/xmodule/xmodule/tests/test_video.py

@@ -15,6 +15,7 @@ the course, section, subsection, unit, etc.
 
 import unittest
 from . import LogicTest
+from lxml import etree
 from .import get_test_system
 from xmodule.modulestore import Location
 from xmodule.video_module import VideoDescriptor, _create_youtube_string
@@ -64,6 +65,32 @@ class VideoModuleTest(LogicTest):
                                   '1.25': '',
                                   '1.50': ''})
 
+    def test_parse_youtube_invalid(self):
+        """Ensure that ids that are invalid return an empty dict"""
+
+        # invalid id
+        youtube_str = 'thisisaninvalidid'
+        output = VideoDescriptor._parse_youtube(youtube_str)
+        self.assertEqual(output, {'0.75': '',
+                                  '1.00': '',
+                                  '1.25': '',
+                                  '1.50': ''})
+        # another invalid id
+        youtube_str = ',::,:,,'
+        output = VideoDescriptor._parse_youtube(youtube_str)
+        self.assertEqual(output, {'0.75': '',
+                                  '1.00': '',
+                                  '1.25': '',
+                                  '1.50': ''})
+
+        # and another one, partially invalid
+        youtube_str = '0.75_BAD!!!,1.0:AXdE34_U,1.25:KLHF9K_Y,1.5:VO3SxfeD,'
+        output = VideoDescriptor._parse_youtube(youtube_str)
+        self.assertEqual(output, {'0.75': '',
+                                  '1.00': 'AXdE34_U',
+                                  '1.25': 'KLHF9K_Y',
+                                  '1.50': 'VO3SxfeD'})
+
     def test_parse_youtube_key_format(self):
         """
         Make sure that inconsistent speed keys are parsed correctly.
@@ -263,6 +290,62 @@ class VideoDescriptorImportTestCase(unittest.TestCase):
             'data': ''
         })
 
+    def test_from_xml_double_quotes(self):
+        """
+        Make sure we can handle the double-quoted string format (which was used for exporting for
+        a few weeks).
+        """
+        module_system = DummySystem(load_error_modules=True)
+        xml_data ='''
+            <video display_name="&quot;display_name&quot;"
+                html5_sources="[&quot;source_1&quot;, &quot;source_2&quot;]"
+                show_captions="false"
+                source="&quot;http://download_video&quot;"
+                sub="&quot;html5_subtitles&quot;"
+                track="&quot;http://download_track&quot;"
+                youtube_id_0_75="&quot;OEoXaMPEzf65&quot;"
+                youtube_id_1_25="&quot;OEoXaMPEzf125&quot;"
+                youtube_id_1_5="&quot;OEoXaMPEzf15&quot;"
+                youtube_id_1_0="&quot;OEoXaMPEzf10&quot;"
+                />
+        '''
+        output = VideoDescriptor.from_xml(xml_data, module_system)
+        self.assert_attributes_equal(output, {
+            'youtube_id_0_75': 'OEoXaMPEzf65',
+            'youtube_id_1_0': 'OEoXaMPEzf10',
+            'youtube_id_1_25': 'OEoXaMPEzf125',
+            'youtube_id_1_5': 'OEoXaMPEzf15',
+            'show_captions': False,
+            'start_time': 0.0,
+            'end_time': 0.0,
+            'track': 'http://download_track',
+            'source': 'http://download_video',
+            'html5_sources': ["source_1", "source_2"],
+            'data': ''
+        })
+
+    def test_from_xml_double_quote_concatenated_youtube(self):
+        module_system = DummySystem(load_error_modules=True)
+        xml_data = '''
+            <video display_name="Test Video"
+                   youtube="1.0:&quot;p2Q6BrNhdh8&quot;,1.25:&quot;1EeWXzPdhSA&quot;">
+            </video>
+        '''
+        output = VideoDescriptor.from_xml(xml_data, module_system)
+        self.assert_attributes_equal(output, {
+            'youtube_id_0_75': '',
+            'youtube_id_1_0': 'p2Q6BrNhdh8',
+            'youtube_id_1_25': '1EeWXzPdhSA',
+            'youtube_id_1_5': '',
+            'show_captions': True,
+            'start_time': 0.0,
+            'end_time': 0.0,
+            'track': '',
+            'source': '',
+            'html5_sources': [],
+            'data': ''
+        })
+
     def test_old_video_format(self):
         """
         Test backwards compatibility with VideoModule's XML format.
@@ -344,7 +427,7 @@ class VideoExportTestCase(unittest.TestCase):
         desc.track = 'http://www.example.com/track'
         desc.html5_sources = ['http://www.example.com/source.mp4', 'http://www.example.com/source.ogg']
 
-        xml = desc.export_to_xml(None)  # We don't use the `resource_fs` parameter
+        xml = desc.definition_to_xml(None)  # We don't use the `resource_fs` parameter
         expected = dedent('''\
          <video url_name="SampleProblem1" start_time="0:00:01" youtube="0.75:izygArpw-Qo,1.00:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8" show_captions="false" end_time="0:01:00">
            <source src="http://www.example.com/source.mp4"/>
@@ -353,7 +436,7 @@ class VideoExportTestCase(unittest.TestCase):
          </video>
         ''')
 
-        self.assertEquals(expected, xml)
+        self.assertEquals(expected, etree.tostring(xml, pretty_print=True))
 
     def test_export_to_xml_empty_parameters(self):
         """Test XML export with defaults."""
@@ -361,7 +444,7 @@ class VideoExportTestCase(unittest.TestCase):
         location = Location(["i4x", "edX", "video", "default", "SampleProblem1"])
         desc = VideoDescriptor(module_system, {'location': location})
 
-        xml = desc.export_to_xml(None)
+        xml = desc.definition_to_xml(None)
         expected = '<video url_name="SampleProblem1"/>\n'
 
-        self.assertEquals(expected, xml)
+        self.assertEquals(expected, etree.tostring(xml, pretty_print=True))

common/lib/xmodule/xmodule/video_module.py

@@ -161,12 +161,7 @@ class VideoModule(VideoFields, XModule):
         return json.dumps({'position': self.position})
 
     def get_html(self):
-        if isinstance(modulestore(), MongoModuleStore):
-            caption_asset_path = StaticContent.get_base_url_path_for_course_assets(self.location) + '/subs_'
-        else:
-            # VS[compat]
-            # cdodge: filesystem static content support.
-            caption_asset_path = "/static/subs/"
+        caption_asset_path = "/static/subs/"
 
         get_ext = lambda filename: filename.rpartition('.')[-1]
         sources = {get_ext(src): src for src in self.html5_sources}
@@ -240,7 +235,7 @@ class VideoDescriptor(VideoFields, TabsEditingDescriptor, EmptyDataRawDescriptor
         video = cls(system, model_data)
         return video
 
-    def export_to_xml(self, resource_fs):
+    def definition_to_xml(self, resource_fs):
         """
         Returns an xml string representing this module.
         """
@@ -266,7 +261,7 @@ class VideoDescriptor(VideoFields, TabsEditingDescriptor, EmptyDataRawDescriptor
             if key in fields and fields[key].default == getattr(self, key):
                 continue
             if value:
-                xml.set(key, str(value))
+                xml.set(key, unicode(value))
 
         for source in self.html5_sources:
             ele = etree.Element('source')
@@ -277,7 +272,7 @@ class VideoDescriptor(VideoFields, TabsEditingDescriptor, EmptyDataRawDescriptor
             ele = etree.Element('track')
             ele.set('src', self.track)
             xml.append(ele)
-        return etree.tostring(xml, pretty_print=True)
+        return xml
 
     @staticmethod
     def _parse_youtube(data):
@@ -287,19 +282,20 @@ class VideoDescriptor(VideoFields, TabsEditingDescriptor, EmptyDataRawDescriptor
         XML-based courses.
         """
         ret = {'0.75': '', '1.00': '', '1.25': '', '1.50': ''}
-        if data == '':
-            return ret
+
         videos = data.split(',')
         for video in videos:
             pieces = video.split(':')
-            # HACK
-            # To elaborate somewhat: in many LMS tests, the keys for
-            # Youtube IDs are inconsistent. Sometimes a particular
-            # speed isn't present, and formatting is also inconsistent
-            # ('1.0' versus '1.00'). So it's necessary to either do
-            # something like this or update all the tests to work
-            # properly.
-            ret['%.2f' % float(pieces[0])] = pieces[1]
+            try:
+                speed = '%.2f' % float(pieces[0])  # normalize speed
+
+                # Handle the fact that youtube IDs got double-quoted for a period of time.
+                # Note: we pass in "VideoFields.youtube_id_1_0" so we deserialize as a String--
+                # it doesn't matter what the actual speed is for the purposes of deserializing.
+                youtube_id = VideoDescriptor._deserialize(VideoFields.youtube_id_1_0.name, pieces[1])
+                ret[speed] = youtube_id
+            except (ValueError, IndexError):
+                log.warning('Invalid YouTube ID: %s' % video)
         return ret
 
     @staticmethod
@@ -312,7 +308,6 @@ class VideoDescriptor(VideoFields, TabsEditingDescriptor, EmptyDataRawDescriptor
         model_data = {}
 
         conversions = {
-            'show_captions': json.loads,
             'start_time': VideoDescriptor._parse_time,
             'end_time': VideoDescriptor._parse_time
         }
@@ -351,10 +346,21 @@ class VideoDescriptor(VideoFields, TabsEditingDescriptor, EmptyDataRawDescriptor
                 #  Convert XML attrs into Python values.
                 if attr in conversions:
                     value = conversions[attr](value)
+                else:
+                # We export values with json.dumps (well, except for Strings, but
+                # for about a month we did it for Strings also).
+                    value = VideoDescriptor._deserialize(attr, value)
                 model_data[attr] = value
 
         return model_data
 
+    @classmethod
+    def _deserialize(cls, attr, value):
+        """
+        Handles deserializing values that may have been encoded with json.dumps.
+        """
+        return cls.get_map_for_field(attr).from_xml(value)
+
     @staticmethod
     def _parse_time(str_time):
         """Converts s in '12:34:45' format to seconds. If s is

common/lib/xmodule/xmodule/x_module.py

@@ -7,7 +7,7 @@ from lxml import etree
 from collections import namedtuple
 from pkg_resources import resource_listdir, resource_string, resource_isdir
 
-from xmodule.modulestore import inheritance, Location
+from xmodule.modulestore import Location
 from xmodule.modulestore.exceptions import ItemNotFoundError, InsufficientSpecificationError, InvalidLocationError
 
 from xblock.core import XBlock, Scope, String, Integer, Float, List, ModelType
@@ -173,11 +173,11 @@ class XModule(XModuleFields, HTMLSnippet, XBlock):
             # don't need to set category as it will automatically get from descriptor
         elif isinstance(self.location, Location):
             self.url_name = self.location.name
-            if not hasattr(self, 'category'):
+            if getattr(self, 'category', None) is None:
                 self.category = self.location.category
         elif isinstance(self.location, BlockUsageLocator):
             self.url_name = self.location.usage_id
-            if not hasattr(self, 'category'):
+            if getattr(self, 'category', None) is None:
                 raise InsufficientSpecificationError()
         else:
             raise InsufficientSpecificationError()
@@ -467,11 +467,11 @@ class XModuleDescriptor(XModuleFields, HTMLSnippet, ResourceTemplates, XBlock):
         self.system = self.runtime
         if isinstance(self.location, Location):
             self.url_name = self.location.name
-            if not hasattr(self, 'category'):
+            if getattr(self, 'category', None) is None:
                 self.category = self.location.category
         elif isinstance(self.location, BlockUsageLocator):
             self.url_name = self.location.usage_id
-            if not hasattr(self, 'category'):
+            if getattr(self, 'category', None) is None:
                 raise InsufficientSpecificationError()
         else:
             raise InsufficientSpecificationError()
@@ -557,75 +557,6 @@ class XModuleDescriptor(XModuleFields, HTMLSnippet, ResourceTemplates, XBlock):
         """
         return False
 
-    # ================================= JSON PARSING ===========================
-    @staticmethod
-    def load_from_json(json_data, system, default_class=None, parent_xblock=None):
-        """
-        This method instantiates the correct subclass of XModuleDescriptor based
-        on the contents of json_data. It does not persist it and can create one which
-        has no usage id.
-
-        parent_xblock is used to compute inherited metadata as well as to append the new xblock.
-
-        json_data:
-        - 'location' : must have this field
-        - 'category': the xmodule category (required or location must be a Location)
-        - 'metadata': a dict of locally set metadata (not inherited)
-        - 'children': a list of children's usage_ids w/in this course
-        - 'definition':
-        - '_id' (optional): the usage_id of this. Will generate one if not given one.
-        """
-        class_ = XModuleDescriptor.load_class(
-            json_data.get('category', json_data.get('location', {}).get('category')),
-            default_class
-        )
-        return class_.from_json(json_data, system, parent_xblock)
-
-    @classmethod
-    def from_json(cls, json_data, system, parent_xblock=None):
-        """
-        Creates an instance of this descriptor from the supplied json_data.
-        This may be overridden by subclasses
-
-        json_data: A json object with the keys 'definition' and 'metadata',
-            definition: A json object with the keys 'data' and 'children'
-                data: A json value
-                children: A list of edX Location urls
-            metadata: A json object with any keys
-
-        This json_data is transformed to model_data using the following rules:
-            1) The model data contains all of the fields from metadata
-            2) The model data contains the 'children' array
-            3) If 'definition.data' is a json object, model data contains all of its fields
-               Otherwise, it contains the single field 'data'
-            4) Any value later in this list overrides a value earlier in this list
-
-        json_data:
-        - 'category': the xmodule category (required)
-        - 'metadata': a dict of locally set metadata (not inherited)
-        - 'children': a list of children's usage_ids w/in this course
-        - 'definition':
-        - '_id' (optional): the usage_id of this. Will generate one if not given one.
-        """
-        usage_id = json_data.get('_id', None)
-        if not '_inherited_metadata' in json_data and parent_xblock is not None:
-            json_data['_inherited_metadata'] = parent_xblock.xblock_kvs.get_inherited_metadata().copy()
-            json_metadata = json_data.get('metadata', {})
-            for field in inheritance.INHERITABLE_METADATA:
-                if field in json_metadata:
-                    json_data['_inherited_metadata'][field] = json_metadata[field]
-
-        new_block = system.xblock_from_json(cls, usage_id, json_data)
-        if parent_xblock is not None:
-            children = parent_xblock.children
-            children.append(new_block)
-            # trigger setter method by using top level field access
-            parent_xblock.children = children
-            # decache pending children field settings (Note, truly persisting at this point would break b/c
-            # persistence assumes children is a list of ids not actual xblocks)
-            parent_xblock.save()
-        return new_block
-
     @classmethod
     def _translate(cls, key):
         'VS[compat]'
@@ -726,6 +657,17 @@ class XModuleDescriptor(XModuleFields, HTMLSnippet, ResourceTemplates, XBlock):
             )
         )
 
+    def iterfields(self):
+        """
+        A generator for iterate over the fields of this xblock (including the ones in namespaces).
+        Example usage: [field.name for field in module.iterfields()]
+        """
+        for field in self.fields:
+            yield field
+        for namespace in self.namespaces:
+            for field in getattr(self, namespace).fields:
+                yield field
+
     @property
     def non_editable_metadata_fields(self):
         """
@@ -736,6 +678,27 @@ class XModuleDescriptor(XModuleFields, HTMLSnippet, ResourceTemplates, XBlock):
         # We are not allowing editing of xblock tag and name fields at this time (for any component).
         return [XBlock.tags, XBlock.name]
 
+    def get_explicitly_set_fields_by_scope(self, scope=Scope.content):
+        """
+        Get a dictionary of the fields for the given scope which are set explicitly on this xblock. (Including
+        any set to None.)
+        """
+        if scope == Scope.settings and hasattr(self, '_inherited_metadata'):
+            inherited_metadata = getattr(self, '_inherited_metadata')
+            result = {}
+            for field in self.iterfields():
+                if (field.scope == scope and
+                        field.name in self._model_data and
+                        field.name not in inherited_metadata):
+                    result[field.name] = self._model_data[field.name]
+            return result
+        else:
+            result = {}
+            for field in self.iterfields():
+                if (field.scope == scope and field.name in self._model_data):
+                    result[field.name] = self._model_data[field.name]
+            return result
+
     @property
     def editable_metadata_fields(self):
         """

common/lib/xmodule/xmodule/xml_module.py

@@ -167,6 +167,11 @@ class XmlDescriptor(XModuleDescriptor):
 
     @classmethod
     def get_map_for_field(cls, attr):
+        """
+        Returns a serialize/deserialize AttrMap for the given field of a class.
+
+        Searches through fields defined by cls to find one named attr.
+        """
         for field in set(cls.fields + cls.lms.fields):
             if field.name == attr:
                 from_xml = lambda val: deserialize_field(field, val)

common/static/js/capa/spec/formula_equation_preview_spec.js

@@ -77,37 +77,33 @@ describe("Formula Equation Preview", function () {
     });
 
     describe('Ajax requests', function () {
-        it('has an initial request with the correct parameters', function () {
+        beforeEach(function () {
+            // This is common to all tests on ajax requests.
             formulaEquationPreview.enable();
 
-	    expect(MathJax.Hub.Queue).toHaveBeenCalled();
-	    // Do what Queue would've done--call the function.
-	    var args = MathJax.Hub.Queue.mostRecentCall.args;
-	    args[1].call(args[0]);
-
             // This part may be asynchronous, so wait.
             waitsFor(function () {
                 return Problem.inputAjax.wasCalled;
             }, "AJAX never called initially", 1000);
+        });
 
-	    runs(function () {
-		expect(Problem.inputAjax.callCount).toEqual(1);
+        it('has an initial request with the correct parameters', function () {
+            expect(Problem.inputAjax.callCount).toEqual(1);
 
-		// Use `.toEqual` rather than `.toHaveBeenCalledWith`
-		// since it supports `jasmine.any`.
-		expect(Problem.inputAjax.mostRecentCall.args).toEqual([
-                    "THE_URL",
-                    "THE_ID",
-                    "preview_formcalc",
-                    {formula: "prefilled_value",
-                     request_start: jasmine.any(Number)},
-                    jasmine.any(Function)
-		]);
-	    });
+            // Use `.toEqual` rather than `.toHaveBeenCalledWith`
+            // since it supports `jasmine.any`.
+            expect(Problem.inputAjax.mostRecentCall.args).toEqual([
+                "THE_URL",
+                "THE_ID",
+                "preview_formcalc",
+                {formula: "prefilled_value",
+                 request_start: jasmine.any(Number)},
+                jasmine.any(Function)
+            ]);
         });
 
         it('makes a request on user input', function () {
-            formulaEquationPreview.enable();
+            Problem.inputAjax.reset();
             $('#input_THE_ID').val('user_input').trigger('input');
 
             // This part is probably asynchronous
@@ -122,8 +118,7 @@ describe("Formula Equation Preview", function () {
         });
 
         it("shouldn't be requested for empty input", function () {
-            formulaEquationPreview.enable();
-	    MathJax.Hub.Queue.reset();
+            Problem.inputAjax.reset();
 
             // When we make an input of '',
             $('#input_THE_ID').val('').trigger('input');
@@ -142,9 +137,7 @@ describe("Formula Equation Preview", function () {
         });
 
         it('should limit the number of requests per second', function () {
-            formulaEquationPreview.enable();
-
-	    var minDelay = formulaEquationPreview.minDelay;
+            var minDelay = formulaEquationPreview.minDelay;
             var end = Date.now() + minDelay * 1.1;
             var step = 10;  // ms
 
@@ -179,23 +172,35 @@ describe("Formula Equation Preview", function () {
 
     describe("Visible results (icon and mathjax)", function () {
         it('should display a loading icon when requests are open', function () {
-            formulaEquationPreview.enable();
             var $img = $("img.loading");
             expect($img.css('visibility')).toEqual('hidden');
-
-            $("#input_THE_ID").val("different").trigger('input');
+            formulaEquationPreview.enable();
             expect($img.css('visibility')).toEqual('visible');
 
+            // This part could be asynchronous
+            waitsFor(function () {
+                return Problem.inputAjax.wasCalled;
+            }, "AJAX never called initially", 1000);
+
+            runs(function () {
+                expect($img.css('visibility')).toEqual('visible');
+
+                // Reset and send another request.
+                $img.css('visibility', 'hidden');
+                $("#input_THE_ID").val("different").trigger('input');
+
+                expect($img.css('visibility')).toEqual('visible');
+            });
+
             // Don't let it fail later.
             waitsFor(function () {
-		return Problem.inputAjax.wasCalled;
+                var args = Problem.inputAjax.mostRecentCall.args;
+                return args[3].formula == "different";
             });
         });
 
         it('should update MathJax and loading icon on callback', function () {
             formulaEquationPreview.enable();
-            $('#input_THE_ID').val('user_input').trigger('input');
-
             waitsFor(function () {
                 return Problem.inputAjax.wasCalled;
             }, "AJAX never called initially", 1000);
@@ -223,12 +228,44 @@ describe("Formula Equation Preview", function () {
             });
         });
 
+        it('finds alternatives if MathJax hasn\'t finished loading', function () {
+            formulaEquationPreview.enable();
+            $('#input_THE_ID').val('user_input').trigger('input');
+
+            waitsFor(function () {
+                return Problem.inputAjax.wasCalled;
+            }, "AJAX never called initially", 1000);
+
+            runs(function () {
+                var args = Problem.inputAjax.mostRecentCall.args;
+                var callback = args[4];
+
+                // Cannot find MathJax.
+                MathJax.Hub.getAllJax.andReturn([]);
+                spyOn(console, 'error');
+
+                callback({
+                    preview: 'THE_FORMULA',
+                    request_start: args[3].request_start
+                });
+
+                // Tests.
+                expect(console.error).toHaveBeenCalled();
+
+                // We should look in the preview div for the MathJax.
+                var previewElement = $("div")[0];
+                expect(previewElement.firstChild.data).toEqual("\\[THE_FORMULA\\]");
+
+                // Refresh the MathJax.
+                expect(MathJax.Hub.Queue).toHaveBeenCalledWith(
+                    ['Typeset', jasmine.any(Object), jasmine.any(Element)]
+                );
+            });
+        });
+
         it('should display errors from the server well', function () {
             var $img = $("img.loading");
             formulaEquationPreview.enable();
-	    MathJax.Hub.Queue.reset();
-            $("#input_THE_ID").val("different").trigger('input');
-
             waitsFor(function () {
                 return Problem.inputAjax.wasCalled;
             }, "AJAX never called initially", 1000);
@@ -263,16 +300,14 @@ describe("Formula Equation Preview", function () {
     describe('Multiple callbacks', function () {
         beforeEach(function () {
             formulaEquationPreview.enable();
-	    MathJax.Hub.Queue.reset();
-            $('#input_THE_ID').val('different').trigger('input');
 
-	    waitsFor(function () {
-		return Problem.inputAjax.wasCalled;
-	    });
+            waitsFor(function () {
+                return Problem.inputAjax.wasCalled;
+            });
 
-	    runs(function () {
-		$("#input_THE_ID").val("different2").trigger('input');
-	    });
+            runs(function () {
+                $('#input_THE_ID').val('different').trigger('input');
+            });
 
             waitsFor(function () {
                 return Problem.inputAjax.callCount > 1;

common/static/js/capa/src/formula_equation_preview.js

@@ -32,8 +32,7 @@ formulaEquationPreview.enable = function () {
 
             // Store the DOM/MathJax elements in which visible output occurs.
             $preview: $preview,
-            // Note: sometimes MathJax hasn't finished loading yet.
-            jax: MathJax.Hub.getAllJax($preview[0])[0],
+            jax: null,  // Fill this in later.
             $img: $preview.find("img.loading"),
 
             requestCallback: null  // Fill it in in a bit.
@@ -59,7 +58,7 @@ formulaEquationPreview.enable = function () {
 
         $this.on("input", initializeRequest);
         // send an initial
-        MathJax.Hub.Queue(this, initializeRequest);
+        initializeRequest.call(this);
     }
 
     /**
@@ -127,20 +126,26 @@ formulaEquationPreview.enable = function () {
 
         function display(latex) {
             // Load jax if it failed before.
+            var previewElement = inputData.$preview[0];
             if (!inputData.jax) {
-                results = MathJax.Hub.getAllJax(inputData.$preview[0]);
-                if (!results.length) {
-                    console.log("Unable to find MathJax to display");
-                    return;
-                }
-                inputData.jax = results[0];
+                inputData.jax = MathJax.Hub.getAllJax(previewElement)[0];
             }
 
-            // Set the text as the latex code, and then update the MathJax.
-            MathJax.Hub.Queue(
-                ['Text', inputData.jax, latex],
-                ['Reprocess', inputData.jax]
-            );
+            // MathJax might not be loaded yet (still).
+            if (inputData.jax) {
+                // Set the text as the latex code, and then update the MathJax.
+                MathJax.Hub.Queue(
+                    ['Text', inputData.jax, latex],
+                    ['Reprocess', inputData.jax]
+                );
+            }
+            else if (latex) {
+                console.error("Oops no mathjax for ", latex);
+                // Fall back to modifying the actual element.
+                var textNode = previewElement.childNodes[0];
+                textNode.data = "\\[" + latex + "\\]";
+                MathJax.Hub.Queue(["Typeset", MathJax.Hub, previewElement]);
+            }
         }
 
         if (response.error) {

common/test/data/splitmongo_json/definitions.json

@@ -2,7 +2,7 @@
     {
         "_id":"head12345_12",
         "category":"course",
-        "data":{
+        "fields":{
             "textbooks":[
                 
             ],
@@ -43,15 +43,17 @@
             },
             "wiki_slug":null
         },
-        "edited_by":"testassist@edx.org",
-        "edited_on":{"$date" : 1364481713238},
-        "previous_version":"head12345_11",
-        "original_version":"head12345_10"
+        "edit_info": {
+            "edited_by":"testassist@edx.org",
+            "edited_on":{"$date" : 1364481713238},
+            "previous_version":"head12345_11",
+            "original_version":"head12345_10"
+        }
     },
     {
         "_id":"head12345_11",
         "category":"course",
-        "data":{
+        "fields":{
             "textbooks":[
                 
             ],
@@ -92,15 +94,17 @@
             },
             "wiki_slug":null
         },
-        "edited_by":"testassist@edx.org",
-        "edited_on":{"$date" : 1364481713238},
-        "previous_version":"head12345_10",
-        "original_version":"head12345_10"
+        "edit_info": {
+            "edited_by":"testassist@edx.org",
+            "edited_on":{"$date" : 1364481713238},
+            "previous_version":"head12345_10",
+            "original_version":"head12345_10"
+        }
     },
     {
         "_id":"head12345_10",
         "category":"course",
-        "data":{
+        "fields":{
             "textbooks":[
                 
             ],
@@ -141,15 +145,17 @@
             },
             "wiki_slug":null
         },
-        "edited_by":"test@edx.org",
-        "edited_on":{"$date": 1364473713238},
-        "previous_version":null,
-        "original_version":"head12345_10"
+        "edit_info": {
+            "edited_by":"test@edx.org",
+            "edited_on":{"$date": 1364473713238},
+            "previous_version":null,
+            "original_version":"head12345_10"
+        }
     },
     {
         "_id":"head23456_1",
         "category":"course",
-        "data":{
+        "fields":{
             "textbooks":[
                 
             ],
@@ -190,15 +196,17 @@
             },
             "wiki_slug":null
         },
-        "edited_by":"test@edx.org",
-        "edited_on":{"$date": 1364481313238},
-        "previous_version":"head23456_0",
-        "original_version":"head23456_0"
+        "edit_info": {
+            "edited_by":"test@edx.org",
+            "edited_on":{"$date": 1364481313238},
+            "previous_version":"head23456_0",
+            "original_version":"head23456_0"
+        }
     },
     {
         "_id":"head23456_0",
         "category":"course",
-        "data":{
+        "fields":{
             "textbooks":[
                 
             ],
@@ -239,15 +247,17 @@
             },
             "wiki_slug":null
         },
-        "edited_by":"test@edx.org",
-        "edited_on":{"$date" : 1364481313238},
-        "previous_version":null,
-        "original_version":"head23456_0"
+        "edit_info": {
+            "edited_by":"test@edx.org",
+            "edited_on":{"$date" : 1364481313238},
+            "previous_version":null,
+            "original_version":"head23456_0"
+        }
     },
     {
         "_id":"head345679_1",
         "category":"course",
-        "data":{
+        "fields":{
             "textbooks":[
                 
             ],
@@ -281,54 +291,66 @@
             },
             "wiki_slug":null
         },
-        "edited_by":"test@edx.org",
-        "edited_on":{"$date" : 1364481313238},
-        "previous_version":null,
-        "original_version":"head23456_0"
+        "edit_info": {
+            "edited_by":"test@edx.org",
+            "edited_on":{"$date" : 1364481313238},
+            "previous_version":null,
+            "original_version":"head23456_0"
+        }
     },
     {
         "_id":"chapter12345_1",
         "category":"chapter",
-        "data":null,
-        "edited_by":"testassist@edx.org",
-        "edited_on":{"$date" : 1364483713238},
-        "previous_version":null,
-        "original_version":"chapter12345_1"
+        "fields":{},
+        "edit_info": {
+            "edited_by":"testassist@edx.org",
+            "edited_on":{"$date" : 1364483713238},
+            "previous_version":null,
+            "original_version":"chapter12345_1"
+        }
     },
     {
         "_id":"chapter12345_2",
         "category":"chapter",
-        "data":null,
-        "edited_by":"testassist@edx.org",
-        "edited_on":{"$date" : 1364483713238},
-        "previous_version":null,
-        "original_version":"chapter12345_2"
+        "fields":{},
+        "edit_info": {
+            "edited_by":"testassist@edx.org",
+            "edited_on":{"$date" : 1364483713238},
+            "previous_version":null,
+            "original_version":"chapter12345_2"
+        }
     },
     {
         "_id":"chapter12345_3",
         "category":"chapter",
-        "data":null,
-        "edited_by":"testassist@edx.org",
-        "edited_on":{"$date" : 1364483713238},
-        "previous_version":null,
-        "original_version":"chapter12345_3"
+        "fields":{},
+        "edit_info": {
+            "edited_by":"testassist@edx.org",
+            "edited_on":{"$date" : 1364483713238},
+            "previous_version":null,
+            "original_version":"chapter12345_3"
+        }
     },
     {
         "_id":"problem12345_3_1",
         "category":"problem",
-        "data":"",
-        "edited_by":"testassist@edx.org",
-        "edited_on":{"$date" : 1364483713238},
-        "previous_version":null,
-        "original_version":"problem12345_3_1"
+        "fields": {"data": ""},
+        "edit_info": {
+            "edited_by":"testassist@edx.org",
+            "edited_on":{"$date" : 1364483713238},
+            "previous_version":null,
+            "original_version":"problem12345_3_1"
+        }
     },
     {
         "_id":"problem12345_3_2",
         "category":"problem",
-        "data":"",
-        "edited_by":"testassist@edx.org",
-        "edited_on":{"$date" : 1364483713238},
-        "previous_version":null,
-        "original_version":"problem12345_3_2"
+        "fields": {"data": ""},
+        "edit_info": {
+            "edited_by":"testassist@edx.org",
+            "edited_on":{"$date" : 1364483713238},
+            "previous_version":null,
+            "original_version":"problem12345_3_2"
+        }
     }
 ]

common/test/data/splitmongo_json/structures.json

@@ -10,14 +10,14 @@
         },
         "blocks":{
             "head12345":{
-                "children":[
-                    "chapter1",
-                    "chapter2",
-                    "chapter3"
-                ],
                 "category":"course",
                 "definition":"head12345_12",
-                "metadata":{
+                "fields":{
+                    "children":[
+                        "chapter1",
+                        "chapter2",
+                        "chapter3"
+                    ],
                     "end":"2013-06-13T04:30",
                     "tabs":[
                         {
@@ -54,93 +54,105 @@
                     "advertised_start":"Fall 2013",
                     "display_name":"The Ancient Greek Hero"
                 },
-                "update_version":{ "$oid" : "1d00000000000000dddd0000" },
-                "previous_version":{ "$oid" : "1d00000000000000dddd1111" },
-                "edited_by":"testassist@edx.org",
-                "edited_on":{
-                    "$date":1364483713238
+                "edit_info": {
+                    "update_version":{ "$oid" : "1d00000000000000dddd0000" },
+                    "previous_version":{ "$oid" : "1d00000000000000dddd1111" },
+                    "edited_by":"testassist@edx.org",
+                    "edited_on":{
+                        "$date":1364483713238
+                    }
                 }
             },
             "chapter1":{
-                "children":[
-                    
-                ],
                 "category":"chapter",
                 "definition":"chapter12345_1",
-                "metadata":{
+                "fields":{
+                    "children":[
+                        
+                    ],
                     "display_name":"Hercules"
                 },
-                "update_version":{ "$oid" : "1d00000000000000dddd0000" },
-                "previous_version":null,
-                "edited_by":"testassist@edx.org",
-                "edited_on":{
-                    "$date":1364483713238
+                "edit_info": {
+                    "update_version":{ "$oid" : "1d00000000000000dddd0000" },
+                    "previous_version":null,
+                    "edited_by":"testassist@edx.org",
+                    "edited_on":{
+                        "$date":1364483713238
+                    }
                 }
             },
             "chapter2":{
-                "children":[
-                    
-                ],
                 "category":"chapter",
                 "definition":"chapter12345_2",
-                "metadata":{
+                "fields":{
+                    "children":[
+                        
+                    ],
                     "display_name":"Hera heckles Hercules"
                 },
-                "update_version":{ "$oid" : "1d00000000000000dddd0000" },
-                "previous_version":null,
-                "edited_by":"testassist@edx.org",
-                "edited_on":{
-                    "$date":1364483713238
+                "edit_info": {
+                    "update_version":{ "$oid" : "1d00000000000000dddd0000" },
+                    "previous_version":null,
+                    "edited_by":"testassist@edx.org",
+                    "edited_on":{
+                        "$date":1364483713238
+                    }
                 }
             },
             "chapter3":{
-                "children":[
-                    "problem1",
-                    "problem3_2"
-                ],
                 "category":"chapter",
                 "definition":"chapter12345_3",
-                "metadata":{
+                "fields":{
+                    "children":[
+                        "problem1",
+                        "problem3_2"
+                    ],
                     "display_name":"Hera cuckolds Zeus"
                 },
-                "update_version":{ "$oid" : "1d00000000000000dddd0000" },
-                "previous_version":null,
-                "edited_by":"testassist@edx.org",
-                "edited_on":{
-                    "$date":1364483713238
+                "edit_info": {
+                    "update_version":{ "$oid" : "1d00000000000000dddd0000" },
+                    "previous_version":null,
+                    "edited_by":"testassist@edx.org",
+                    "edited_on":{
+                        "$date":1364483713238
+                    }
                 }
             },
             "problem1":{
-                "children":[
-                    
-                ],
                 "category":"problem",
                 "definition":"problem12345_3_1",
-                "metadata":{
+                "fields":{
+                    "children":[
+                        
+                    ],
                     "display_name":"Problem 3.1",
                     "graceperiod":"4 hours 0 minutes 0 seconds"
                 },
-                "update_version":{ "$oid" : "1d00000000000000dddd0000" },
-                "previous_version":null,
-                "edited_by":"testassist@edx.org",
-                "edited_on":{
-                    "$date":1364483713238
+                "edit_info": {
+                    "update_version":{ "$oid" : "1d00000000000000dddd0000" },
+                    "previous_version":null,
+                    "edited_by":"testassist@edx.org",
+                    "edited_on":{
+                        "$date":1364483713238
+                    }
                 }
             },
             "problem3_2":{
-                "children":[
-                    
-                ],
                 "category":"problem",
                 "definition":"problem12345_3_2",
-                "metadata":{
+                "fields":{
+                    "children":[
+                        
+                    ],
                     "display_name":"Problem 3.2"
                 },
-                "update_version":{ "$oid" : "1d00000000000000dddd0000" },
-                "previous_version":null,
-                "edited_by":"testassist@edx.org",
-                "edited_on":{
-                    "$date":1364483713238
+                "edit_info": {
+                    "update_version":{ "$oid" : "1d00000000000000dddd0000" },
+                    "previous_version":null,
+                    "edited_by":"testassist@edx.org",
+                    "edited_on":{
+                        "$date":1364483713238
+                    }
                 }
             }
         }
@@ -156,12 +168,12 @@
         },
         "blocks":{
             "head12345":{
-                "children":[
-                    
-                ],
                 "category":"course",
                 "definition":"head12345_11",
-                "metadata":{
+                "fields":{
+                    "children":[
+                        
+                    ],
                     "end":"2013-04-13T04:30",
                     "tabs":[
                         {
@@ -198,11 +210,13 @@
                     "advertised_start":null,
                     "display_name":"The Ancient Greek Hero"
                 },
-                "update_version":{ "$oid" : "1d00000000000000dddd1111" },
-                "previous_version":{ "$oid" : "1d00000000000000dddd3333" },
-                "edited_by":"testassist@edx.org",
-                "edited_on":{
-                    "$date":1364481713238
+                "edit_info": {
+                    "update_version":{ "$oid" : "1d00000000000000dddd1111" },
+                    "previous_version":{ "$oid" : "1d00000000000000dddd3333" },
+                    "edited_by":"testassist@edx.org",
+                    "edited_on":{
+                        "$date":1364481713238
+                    }
                 }
             }
         }
@@ -218,12 +232,12 @@
         },
         "blocks":{
             "head12345":{
-                "children":[
-                    
-                ],
                 "category":"course",
                 "definition":"head12345_10",
-                "metadata":{
+                "fields":{
+                    "children":[
+                        
+                    ],
                     "end":null,
                     "tabs":[
                         {
@@ -250,11 +264,13 @@
                     "advertised_start":null,
                     "display_name":"The Ancient Greek Hero"
                 },
-                "update_version":{ "$oid" : "1d00000000000000dddd3333" },
-                "previous_version":null,
-                "edited_by":"test@edx.org",
-                "edited_on":{
-                    "$date":1364473713238
+                "edit_info": {
+                    "update_version":{ "$oid" : "1d00000000000000dddd3333" },
+                    "previous_version":null,
+                    "edited_by":"test@edx.org",
+                    "edited_on":{
+                        "$date":1364473713238
+                    }
                 }
             }
         }
@@ -270,12 +286,12 @@
         },
         "blocks":{
             "head23456":{
-                "children":[
-                    
-                ],
                 "category":"course",
                 "definition":"head23456_1",
-                "metadata":{
+                "fields":{
+                    "children":[
+                        
+                    ],
                     "end":null,
                     "tabs":[
                         {
@@ -302,11 +318,13 @@
                     "advertised_start":null,
                     "display_name":"The most wonderful course"
                 },
-                "update_version":{ "$oid" : "1d00000000000000dddd2222" },
-                "previous_version":{ "$oid" : "1d00000000000000dddd4444" },
-                "edited_by":"test@edx.org",
-                "edited_on":{
-                    "$date":1364481313238
+                "edit_info": {
+                    "update_version":{ "$oid" : "1d00000000000000dddd2222" },
+                    "previous_version":{ "$oid" : "1d00000000000000dddd4444" },
+                    "edited_by":"test@edx.org",
+                    "edited_on":{
+                        "$date":1364481313238
+                    }
                 }
                 
             }
@@ -323,12 +341,12 @@
         },
         "blocks":{
             "head23456":{
-                "children":[
-                    
-                ],
                 "category":"course",
                 "definition":"head23456_0",
-                "metadata":{
+                "fields":{
+                    "children":[
+                        
+                    ],
                     "end":null,
                     "tabs":[
                         {
@@ -355,11 +373,13 @@
                     "advertised_start":null,
                     "display_name":"A wonderful course"
                 },
-                "update_version":{ "$oid" : "1d00000000000000dddd4444" },
-                "previous_version":null,
-                "edited_by":"test@edx.org",
-                "edited_on":{
-                    "$date":1364480313238
+                "edit_info": {
+                    "update_version":{ "$oid" : "1d00000000000000dddd4444" },
+                    "previous_version":null,
+                    "edited_by":"test@edx.org",
+                    "edited_on":{
+                        "$date":1364480313238
+                    }
                 }
             }
         }
@@ -375,12 +395,12 @@
         },
         "blocks":{
             "head23456":{
-                "children":[
-                    
-                ],
                 "category":"course",
                 "definition":"head23456_1",
-                "metadata":{
+                "fields":{
+                    "children":[
+                        
+                    ],
                     "end":null,
                     "tabs":[
                         {
@@ -407,11 +427,13 @@
                     "advertised_start":null,
                     "display_name":"The most wonderful course"
                 },
-                "update_version":{ "$oid" : "1d00000000000000eeee0000" },
-                "previous_version":null,
-                "edited_by":"test@edx.org",
-                "edited_on":{
-                    "$date":1364481333238
+                "edit_info": {
+                    "update_version":{ "$oid" : "1d00000000000000eeee0000" },
+                    "previous_version":null,
+                    "edited_by":"test@edx.org",
+                    "edited_on":{
+                        "$date":1364481333238
+                    }
                 }
             }
         }
@@ -427,12 +449,12 @@
         },
         "blocks":{
             "head345679":{
-                "children":[
-                    
-                ],
                 "category":"course",
                 "definition":"head345679_1",
-                "metadata":{
+                "fields":{
+                    "children":[
+                        
+                    ],
                     "end":null,
                     "tabs":[
                         {
@@ -459,11 +481,13 @@
                     "advertised_start":null,
                     "display_name":"Yet another contender"
                 },
-                "update_version":{ "$oid" : "1d00000000000000dddd5555" },
-                "previous_version":null,
-                "edited_by":"test@guestx.edu",
-                "edited_on":{
-                    "$date":1364491313238
+                "edit_info": {
+                    "update_version":{ "$oid" : "1d00000000000000dddd5555" },
+                    "previous_version":null,
+                    "edited_by":"test@guestx.edu",
+                    "edited_on":{
+                        "$date":1364491313238
+                    }
                 }
             }
         }

docs/data/source/internal_data_formats/sql_schema.rst

@@ -347,7 +347,7 @@ There is an important split in demographic data gathered for the students who si
 
 `student_courseenrollment`
 ==========================
-A row in this table represents a student's enrollment for a particular course run. If they decide to unenroll in the course, we delete their entry in this table, but we still leave all their state in `courseware_studentmodule` untouched.
+A row in this table represents a student's enrollment for a particular course run. If they decide to unenroll in the course, we set `is_active` to `False`. We still leave all their state in `courseware_studentmodule` untouched, so they will not lose courseware state if they unenroll and reenroll.
 
 `id`
 ----
@@ -365,6 +365,13 @@ A row in this table represents a student's enrollment for a particular course ru
 ---------
   Datetime of enrollment, UTC.
 
+`is_active`
+-----------
+  Boolean indicating whether this enrollment is active. If an enrollment is not active, a student is not enrolled in that course. This lets us unenroll students without losing a record of what courses they were enrolled in previously. This was introduced in the 2013-08-20 release. Before this release, unenrolling a student simply deleted the row in `student_courseenrollment`.
+
+`mode`
+------
+  String indicating what kind of enrollment this was. The default is "honor" (honor certificate) and all enrollments prior to 2013-08-20 will be of that type. Other types being considered are "audit" and "verified_id".
 
 
 *******************

lms/djangoapps/analytics/basic.py

@@ -25,8 +25,10 @@ def enrolled_students_features(course_id, features):
         {'username': 'username3', 'first_name': 'firstname3'}
     ]
     """
-    students = User.objects.filter(courseenrollment__course_id=course_id)\
-        .order_by('username').select_related('profile')
+    students = User.objects.filter(
+        courseenrollment__course_id=course_id,
+        courseenrollment__is_active=1,
+    ).order_by('username').select_related('profile')
 
     def extract_student(student, features):
         """ convert student to dictionary """

lms/djangoapps/analytics/tests/test_basic.py

@@ -15,7 +15,8 @@ class TestAnalyticsBasic(TestCase):
     def setUp(self):
         self.course_id = 'some/robot/course/id'
         self.users = tuple(UserFactory() for _ in xrange(30))
-        self.ces = tuple(CourseEnrollment.objects.create(course_id=self.course_id, user=user) for user in self.users)
+        self.ces = tuple(CourseEnrollment.enroll(user, self.course_id)
+                         for user in self.users)
 
     def test_enrolled_students_features_username(self):
         self.assertIn('username', AVAILABLE_FEATURES)

lms/djangoapps/analytics/tests/test_distributions.py

@@ -19,10 +19,8 @@ class TestAnalyticsDistributions(TestCase):
             profile__year_of_birth=i + 1930
         ) for i in xrange(30)]
 
-        self.ces = [CourseEnrollment.objects.create(
-            course_id=self.course_id,
-            user=user
-        ) for user in self.users]
+        self.ces = [CourseEnrollment.enroll(user, self.course_id)
+                    for user in self.users]
 
     @raises(ValueError)
     def test_profile_distribution_bad_feature(self):
@@ -68,7 +66,8 @@ class TestAnalyticsDistributionsNoData(TestCase):
 
         self.users += self.nodata_users
 
-        self.ces = tuple(CourseEnrollment.objects.create(course_id=self.course_id, user=user) for user in self.users)
+        self.ces = tuple(CourseEnrollment.enroll(user, self.course_id)
+                         for user in self.users)
 
     def test_profile_distribution_easy_choice_nodata(self):
         feature = 'gender'