RESTful api for getting course listing and opening course in studio.

Pattern for how to do refactoring from locations to locators and from
old style urls to restful ones.

common/lib/xmodule/xmodule/modulestore/loc_mapper_store.py

@@ -56,7 +56,7 @@ class LocMapperStore(object):
         """
         Add a new entry to map this course_location to the new style CourseLocator.course_id. If course_id is not
         provided, it creates the default map of using org.course.name from the location (just like course_id) if
-        the location.cateogry = 'course'; otherwise, it uses org.course.
+        the location.category = 'course'; otherwise, it uses org.course.
 
         You can create more than one mapping to the
         same course_id target. In that case, the reverse translate will be arbitrary (no guarantee of which wins).

common/lib/xmodule/xmodule/modulestore/locator.py

@@ -14,6 +14,8 @@ from xmodule.modulestore.exceptions import InsufficientSpecificationError, OverS
 
 from .parsers import parse_url, parse_course_id, parse_block_ref
 from .parsers import BRANCH_PREFIX, BLOCK_PREFIX, URL_VERSION_PREFIX
+import re
+from xmodule.modulestore import Location
 
 log = logging.getLogger(__name__)
 
@@ -89,6 +91,59 @@ class Locator(object):
                                          (property_name, current, new))
         setattr(self, property_name, new)
 
+    @staticmethod
+    def to_locator_or_location(location):
+        """
+        Convert the given locator like thing to the appropriate type of object, or, if already
+        that type, just return it. Returns an old Location, BlockUsageLocator,
+        or DefinitionLocator.
+
+        :param location: can be a Location, Locator, string, tuple, list, or dict.
+        """
+        if isinstance(location, (Location, Locator)):
+            return location
+        if isinstance(location, basestring):
+            return Locator.parse_url(location)
+        if isinstance(location, (list, tuple)):
+            return Location(location)
+        if isinstance(location, dict) and 'name' in location:
+            return Location(location)
+        if isinstance(location, dict):
+            return BlockUsageLocator(**location)
+        raise ValueError(location)
+
+    URL_TAG_RE = re.compile(r'^(\w+)://')
+    @staticmethod
+    def parse_url(url):
+        """
+        Parse the url into one of the Locator types (must have a tag indicating type)
+        Return the new instance. Supports i4x, cvx, edx, defx
+
+        :param url: the url to parse
+        """
+        parsed = Locator.URL_TAG_RE.match(url)
+        if parsed is None:
+            raise ValueError(parsed)
+        parsed = parsed.group(1)
+        if parsed in ['i4x', 'c4x']:
+            return Location(url)
+        elif parsed == 'edx':
+            return BlockUsageLocator(url)
+        elif parsed == 'defx':
+            return DefinitionLocator(url)
+        return None
+
+    @classmethod
+    def as_object_id(cls, value):
+        """
+        Attempts to cast value as a bson.objectid.ObjectId.
+        If cast fails, raises ValueError
+        """
+        try:
+            return ObjectId(value)
+        except InvalidId:
+            raise ValueError('"%s" is not a valid version_guid' % value)
+
 
 class CourseLocator(Locator):
     """
@@ -208,18 +263,55 @@ class CourseLocator(Locator):
                              version_guid=self.version_guid,
                              branch=self.branch)
 
-    @classmethod
-    def as_object_id(cls, value):
+    OLD_COURSE_ID_RE = re.compile(r'^(?P<org>[^.]+)\.?(?P<old_course_id>.+)?\.(?P<run>[^.]+)$')
+    @property
+    def as_old_location_course_id(self):
         """
-        Attempts to cast value as a bson.objectid.ObjectId.
-        If cast fails, raises ValueError
+        The original Location type presented its course id as org/course/run. This function
+        assumes the course_id starts w/ org, has an arbitrarily long 'course' identifier, and then
+        ends w/ run all separated by periods.
+
+        If this object does not have a course_id, this function returns None.
         """
-        if isinstance(value, ObjectId):
-            return value
-        try:
-            return ObjectId(value)
-        except InvalidId:
-            raise ValueError('"%s" is not a valid version_guid' % value)
+        if self.course_id is None:
+            return None
+        parsed = self.OLD_COURSE_ID_RE.match(self.course_id)
+        # check whether there are 2 or > 2 'fields'
+        if parsed.group('old_course_id'):
+            return '/'.join(parsed.groups())
+        else:
+            return parsed.group('org') + '/' + parsed.group('run')
+
+    def _old_location_field_helper(self, field):
+        """
+        Parse course_id to get the old location field named field out
+        """
+        if self.course_id is None:
+            return None
+        parsed = self.OLD_COURSE_ID_RE.match(self.course_id)
+        return parsed.group(field)
+
+    @property
+    def as_old_location_org(self):
+        """
+        Presume the first part of the course_id is the org and return it.
+        """
+        return self._old_location_field_helper('org')
+
+    @property
+    def as_old_location_course(self):
+        """
+        Presume the middle part, if any, of the course_id is the old location scheme's
+        course id and return it.
+        """
+        return self._old_location_field_helper('old_course_id')
+
+    @property
+    def as_old_location_run(self):
+        """
+        Presume the last part of the course_id is the old location scheme's run and return it.
+        """
+        return self._old_location_field_helper('run')
 
     def init_from_url(self, url):
         """
@@ -230,7 +322,7 @@ class CourseLocator(Locator):
             url = url.url()
         if not isinstance(url, basestring):
             raise TypeError('%s is not an instance of basestring' % url)
-        parse = parse_url(url)
+        parse = parse_url(url, tag_optional=True)
         if not parse:
             raise ValueError('Could not parse "%s" as a url' % url)
         self._set_value(
@@ -349,7 +441,7 @@ class BlockUsageLocator(CourseLocator):
         """
         self._validate_args(url, version_guid, course_id)
         if url:
-            self.init_block_ref_from_url(url)
+            self.init_block_ref_from_str(url)
         if course_id:
             self.init_block_ref_from_course_id(course_id)
         if usage_id:
@@ -401,11 +493,18 @@ class BlockUsageLocator(CourseLocator):
                 raise ValueError('Could not parse "%s" as a block_ref' % block_ref)
             self.set_usage_id(parse['block'])
 
-    def init_block_ref_from_url(self, url):
-        if isinstance(url, Locator):
-            url = url.url()
-        parse = parse_url(url)
-        assert parse, 'Could not parse "%s" as a url' % url
+    def init_block_ref_from_str(self, value):
+        """
+        Create a block locator from the given string which may be a url or just the repr (no tag)
+        """
+        if hasattr(value, 'usage_id'):
+            self.init_block_ref(value.usage_id)
+            return
+        if not isinstance(value, basestring):
+            return None
+        parse = parse_url(value, tag_optional=True)
+        if parse is None:
+            raise ValueError('Could not parse "%s" as a url' % value)
         self._set_value(parse, 'block', lambda(new_block): self.set_usage_id(new_block))
 
     def init_block_ref_from_course_id(self, course_id):
@@ -429,8 +528,13 @@ class DefinitionLocator(Locator):
     Container for how to locate a description (the course-independent content).
     """
 
+    URL_RE = re.compile(r'^defx://' + URL_VERSION_PREFIX + '([^/]+)$', re.IGNORECASE)
     def __init__(self, definition_id):
-        self.definition_id = definition_id
+        if isinstance(definition_id, basestring):
+            regex_match = self.URL_RE.match(definition_id)
+            if regex_match is not None:
+                definition_id = self.as_object_id(regex_match.group(1))
+        self.definition_id = self.as_object_id(definition_id)
 
     def __unicode__(self):
         '''
@@ -442,9 +546,9 @@ class DefinitionLocator(Locator):
     def url(self):
         """
         Return a string containing the URL for this location.
-        url(self) returns something like this: 'edx://version/519665f6223ebd6980884f2b'
+        url(self) returns something like this: 'defx://version/519665f6223ebd6980884f2b'
         """
-        return 'edx://' + unicode(self)
+        return 'defx://' + unicode(self)
 
     def version(self):
         """

common/lib/xmodule/xmodule/modulestore/parsers.py

@@ -9,13 +9,14 @@ VERSION_PREFIX = "/version/"
 # Prefix for version when it begins the URL (no course ID).
 URL_VERSION_PREFIX = 'version/'
 
-URL_RE = re.compile(r'^edx://(.+)$', re.IGNORECASE)
+URL_RE = re.compile(r'^(edx://)?(.+)$', re.IGNORECASE)
 
 
-def parse_url(string):
+def parse_url(string, tag_optional=False):
     """
-    A url must begin with 'edx://' (case-insensitive match),
-    followed by either a version_guid or a course_id.
+    A url usually begins with 'edx://' (case-insensitive match),
+    followed by either a version_guid or a course_id. If tag_optional, then
+    the url does not have to start with the tag and edx will be assumed.
 
     Examples:
         'edx://version/0123FFFF'
@@ -36,7 +37,9 @@ def parse_url(string):
     match = URL_RE.match(string)
     if not match:
         return None
-    path = match.group(1)
+    if match.group(1) is None and not tag_optional:
+        return None
+    path = match.group(2)
     if path.startswith(URL_VERSION_PREFIX):
         return parse_guid(path[len(URL_VERSION_PREFIX):])
     return parse_course_id(path)

common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py

@@ -135,7 +135,7 @@ class SplitModuleCourseTests(SplitModuleTest):
         self.assertEqual(
             len(course.children), 3,
             "children")
-        self.assertEqual(course.definition_locator.definition_id, "head12345_12")
+        self.assertEqual(str(course.definition_locator.definition_id), "ad00000000000000dddd0000")
         # check dates and graders--forces loading of descriptor
         self.assertEqual(course.edited_by, "testassist@edx.org")
         self.assertEqual(str(course.previous_version), self.GUID_D1)
@@ -195,7 +195,7 @@ class SplitModuleCourseTests(SplitModuleTest):
         self.assertEqual(course.graceperiod, datetime.timedelta(hours=2))
         self.assertIsNone(course.advertised_start)
         self.assertEqual(len(course.children), 0)
-        self.assertEqual(course.definition_locator.definition_id, "head12345_11")
+        self.assertEqual(str(course.definition_locator.definition_id), "ad00000000000000dddd0001")
         # check dates and graders--forces loading of descriptor
         self.assertEqual(course.edited_by, "testassist@edx.org")
         self.assertDictEqual(course.grade_cutoffs, {"Pass": 0.55})
@@ -345,7 +345,7 @@ class SplitModuleItemTests(SplitModuleTest):
             self.assertEqual(block.display_name, "The Ancient Greek Hero")
             self.assertEqual(block.advertised_start, "Fall 2013")
             self.assertEqual(len(block.children), 3)
-            self.assertEqual(block.definition_locator.definition_id, "head12345_12")
+            self.assertEqual(str(block.definition_locator.definition_id), "ad00000000000000dddd0000")
             # check dates and graders--forces loading of descriptor
             self.assertEqual(block.edited_by, "testassist@edx.org")
             self.assertDictEqual(
@@ -375,7 +375,7 @@ class SplitModuleItemTests(SplitModuleTest):
         block = modulestore().get_item(locator)
         self.assertEqual(block.location.course_id, "GreekHero")
         self.assertEqual(block.category, 'chapter')
-        self.assertEqual(block.definition_locator.definition_id, "chapter12345_1")
+        self.assertEqual(str(block.definition_locator.definition_id), "cd00000000000000dddd0020")
         self.assertEqual(block.display_name, "Hercules")
         self.assertEqual(block.edited_by, "testassist@edx.org")
 
@@ -562,13 +562,13 @@ class TestItemCrud(SplitModuleTest):
         new_module = modulestore().create_item(
             locator, category, 'user123',
             fields={'display_name': 'new chapter'},
-            definition_locator=DefinitionLocator("chapter12345_2")
+            definition_locator=DefinitionLocator("cd00000000000000dddd0022")
         )
         # check that course version changed and course's previous is the other one
         self.assertNotEqual(new_module.location.version_guid, premod_course.location.version_guid)
         parent = modulestore().get_item(locator)
         self.assertIn(new_module.location.usage_id, parent.children)
-        self.assertEqual(new_module.definition_locator.definition_id, "chapter12345_2")
+        self.assertEqual(str(new_module.definition_locator.definition_id), "cd00000000000000dddd0022")
 
     def test_unique_naming(self):
         """
@@ -588,7 +588,7 @@ class TestItemCrud(SplitModuleTest):
         another_module = modulestore().create_item(
             locator, category, 'anotheruser',
             fields={'display_name': 'problem 2', 'data': another_payload},
-            definition_locator=DefinitionLocator("problem12345_3_1"),
+            definition_locator=DefinitionLocator("0d00000040000000dddd0031"),
         )
         # check that course version changed and course's previous is the other one
         parent = modulestore().get_item(locator)
@@ -605,7 +605,7 @@ class TestItemCrud(SplitModuleTest):
         self.assertLessEqual(new_history['edited_on'], datetime.datetime.now(UTC))
         self.assertGreaterEqual(new_history['edited_on'], premod_time)
         another_history = modulestore().get_definition_history_info(another_module.definition_locator)
-        self.assertEqual(another_history['previous_version'], 'problem12345_3_1')
+        self.assertEqual(str(another_history['previous_version']), '0d00000040000000dddd0031')
 
     def test_create_continue_version(self):
         """
@@ -789,7 +789,7 @@ class TestItemCrud(SplitModuleTest):
         modulestore().create_item(
             locator, category, 'test_update_manifold',
             fields={'display_name': 'problem 2', 'data': another_payload},
-            definition_locator=DefinitionLocator("problem12345_3_1"),
+            definition_locator=DefinitionLocator("0d00000040000000dddd0031"),
         )
         # pylint: disable=W0212
         modulestore()._clear_cache()

common/test/data/splitmongo_json/definitions.json

@@ -1,6 +1,6 @@
 [
     {
-        "_id":"head12345_12",
+        "_id": { "$oid" : "ad00000000000000dddd0000"},
         "category":"course",
         "fields":{
             "textbooks":[
@@ -46,12 +46,12 @@
         "edit_info": {
             "edited_by":"testassist@edx.org",
             "edited_on":{"$date" : 1364481713238},
-            "previous_version":"head12345_11",
-            "original_version":"head12345_10"
+            "previous_version":{ "$oid" : "ad00000000000000dddd0001"},
+            "original_version":{ "$oid" : "ad00000000000000dddd0010"}
         }
     },
     {
-        "_id":"head12345_11",
+        "_id":{ "$oid" : "ad00000000000000dddd0001"},
         "category":"course",
         "fields":{
             "textbooks":[
@@ -97,12 +97,12 @@
         "edit_info": {
             "edited_by":"testassist@edx.org",
             "edited_on":{"$date" : 1364481713238},
-            "previous_version":"head12345_10",
-            "original_version":"head12345_10"
+            "previous_version":{ "$oid" : "ad00000000000000dddd0010"},
+            "original_version":{ "$oid" : "ad00000000000000dddd0010"}
         }
     },
     {
-        "_id":"head12345_10",
+        "_id":{ "$oid" : "ad00000000000000dddd0010"},
         "category":"course",
         "fields":{
             "textbooks":[
@@ -149,11 +149,11 @@
             "edited_by":"test@edx.org",
             "edited_on":{"$date": 1364473713238},
             "previous_version":null,
-            "original_version":"head12345_10"
+            "original_version":{ "$oid" : "ad00000000000000dddd0010"}
         }
     },
     {
-        "_id":"head23456_1",
+        "_id":{ "$oid" : "ad00000000000000dddd0020"},
         "category":"course",
         "fields":{
             "textbooks":[
@@ -199,12 +199,12 @@
         "edit_info": {
             "edited_by":"test@edx.org",
             "edited_on":{"$date": 1364481313238},
-            "previous_version":"head23456_0",
-            "original_version":"head23456_0"
+            "previous_version":{ "$oid" : "2d00000000000000dddd0020"},
+            "original_version":{ "$oid" : "2d00000000000000dddd0020"}
         }
     },
     {
-        "_id":"head23456_0",
+        "_id":{ "$oid" : "2d00000000000000dddd0020"},
         "category":"course",
         "fields":{
             "textbooks":[
@@ -251,11 +251,11 @@
             "edited_by":"test@edx.org",
             "edited_on":{"$date" : 1364481313238},
             "previous_version":null,
-            "original_version":"head23456_0"
+            "original_version":{ "$oid" : "2d00000000000000dddd0020"}
         }
     },
     {
-        "_id":"head345679_1",
+        "_id":{ "$oid" : "3d00000000000000dddd0020"},
         "category":"course",
         "fields":{
             "textbooks":[
@@ -295,62 +295,62 @@
             "edited_by":"test@edx.org",
             "edited_on":{"$date" : 1364481313238},
             "previous_version":null,
-            "original_version":"head23456_0"
+            "original_version":{ "$oid" : "2d00000000000000dddd0020"}
         }
     },
     {
-        "_id":"chapter12345_1",
+        "_id":{ "$oid" : "cd00000000000000dddd0020"},
         "category":"chapter",
         "fields":{},
         "edit_info": {
             "edited_by":"testassist@edx.org",
             "edited_on":{"$date" : 1364483713238},
             "previous_version":null,
-            "original_version":"chapter12345_1"
+            "original_version":{ "$oid" : "cd00000000000000dddd0020"}
         }
     },
     {
-        "_id":"chapter12345_2",
+        "_id":{ "$oid" : "cd00000000000000dddd0022"},
         "category":"chapter",
         "fields":{},
         "edit_info": {
             "edited_by":"testassist@edx.org",
             "edited_on":{"$date" : 1364483713238},
             "previous_version":null,
-            "original_version":"chapter12345_2"
+            "original_version":{ "$oid" : "cd00000000000000dddd0022"}
         }
     },
     {
-        "_id":"chapter12345_3",
+        "_id":{ "$oid" : "cd00000000000000dddd0032"},
         "category":"chapter",
         "fields":{},
         "edit_info": {
             "edited_by":"testassist@edx.org",
             "edited_on":{"$date" : 1364483713238},
             "previous_version":null,
-            "original_version":"chapter12345_3"
+            "original_version":{ "$oid" : "cd00000000000000dddd0032"}
         }
     },
     {
-        "_id":"problem12345_3_1",
+        "_id":{ "$oid" : "0d00000040000000dddd0031"},
         "category":"problem",
         "fields": {"data": ""},
         "edit_info": {
             "edited_by":"testassist@edx.org",
             "edited_on":{"$date" : 1364483713238},
             "previous_version":null,
-            "original_version":"problem12345_3_1"
+            "original_version":{ "$oid" : "0d00000040000000dddd0031"}
         }
     },
     {
-        "_id":"problem12345_3_2",
+        "_id":{ "$oid" : "0d00000040000000dddd0032"},
         "category":"problem",
         "fields": {"data": ""},
         "edit_info": {
             "edited_by":"testassist@edx.org",
             "edited_on":{"$date" : 1364483713238},
             "previous_version":null,
-            "original_version":"problem12345_3_2"
+            "original_version":{ "$oid" : "0d00000040000000dddd0032"}
         }
     }
 ]

common/test/data/splitmongo_json/structures.json

@@ -11,7 +11,7 @@
         "blocks":{
             "head12345":{
                 "category":"course",
-                "definition":"head12345_12",
+                "definition":{ "$oid" : "ad00000000000000dddd0000"},
                 "fields":{
                     "children":[
                         "chapter1",
@@ -65,7 +65,7 @@
             },
             "chapter1":{
                 "category":"chapter",
-                "definition":"chapter12345_1",
+                "definition":{ "$oid" : "cd00000000000000dddd0020"},
                 "fields":{
                     "children":[
                         
@@ -83,7 +83,7 @@
             },
             "chapter2":{
                 "category":"chapter",
-                "definition":"chapter12345_2",
+                "definition":{ "$oid" : "cd00000000000000dddd0022"},
                 "fields":{
                     "children":[
                         
@@ -101,7 +101,7 @@
             },
             "chapter3":{
                 "category":"chapter",
-                "definition":"chapter12345_3",
+                "definition":{ "$oid" : "cd00000000000000dddd0032"},
                 "fields":{
                     "children":[
                         "problem1",
@@ -120,7 +120,7 @@
             },
             "problem1":{
                 "category":"problem",
-                "definition":"problem12345_3_1",
+                "definition":{ "$oid" : "0d00000040000000dddd0031"},
                 "fields":{
                     "children":[
                         
@@ -139,7 +139,7 @@
             },
             "problem3_2":{
                 "category":"problem",
-                "definition":"problem12345_3_2",
+                "definition":{ "$oid" : "0d00000040000000dddd0032"},
                 "fields":{
                     "children":[
                         
@@ -169,7 +169,7 @@
         "blocks":{
             "head12345":{
                 "category":"course",
-                "definition":"head12345_11",
+                "definition":{ "$oid" : "ad00000000000000dddd0001"},
                 "fields":{
                     "children":[
                         
@@ -233,7 +233,7 @@
         "blocks":{
             "head12345":{
                 "category":"course",
-                "definition":"head12345_10",
+                "definition":{ "$oid" : "ad00000000000000dddd0010"},
                 "fields":{
                     "children":[
                         
@@ -287,7 +287,7 @@
         "blocks":{
             "head23456":{
                 "category":"course",
-                "definition":"head23456_1",
+                "definition":{ "$oid" : "ad00000000000000dddd0020"},
                 "fields":{
                     "children":[
                         
@@ -342,7 +342,7 @@
         "blocks":{
             "head23456":{
                 "category":"course",
-                "definition":"head23456_0",
+                "definition":{ "$oid" : "2d00000000000000dddd0020"},
                 "fields":{
                     "children":[
                         
@@ -396,7 +396,7 @@
         "blocks":{
             "head23456":{
                 "category":"course",
-                "definition":"head23456_1",
+                "definition":{ "$oid" : "ad00000000000000dddd0020"},
                 "fields":{
                     "children":[
                         
@@ -450,7 +450,7 @@
         "blocks":{
             "head345679":{
                 "category":"course",
-                "definition":"head345679_1",
+                "definition":{ "$oid" : "3d00000000000000dddd0020"},
                 "fields":{
                     "children":[