Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Write restful service to find all orphans

Browse Source 
To help fix recent bugs re orphaned discussions and
to prototype a more restful json oriented api.
This commit is contained in:
Don Mitchell
2013-10-01 16:03:26 -04:00
parent 286ae9e73d
commit 6b9797522b
7 changed files with 285 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
CHANGELOG.rst
View File

@@ -9,6 +9,14 @@ Blades: When start time and end time are specified for a video, a visual range
will be shown on the time slider to highlight the place in the video that will
be played.
Studio: added restful interface for finding orphans in courses.
An orphan is an xblock to which no children relation points and whose type is not
in the set contentstore.views.item.DETACHED_CATEGORIES nor 'course'.
GET http://host/orphan/org.course returns json array of ids.
Requires course author access.
DELETE http://orphan/org.course deletes all the orphans in that course.
Requires is_staff access
Studio: Bug fix for text loss in Course Updates when the text exists
before the first tag.

58
cms/djangoapps/contentstore/tests/test_orphan.py Normal file
View File

@@ -0,0 +1,58 @@
"""
Test finding orphans via the view and django config
"""
import json
from contentstore.tests.utils import CourseTestCase
from xmodule.modulestore.django import editable_modulestore
from django.core.urlresolvers import reverse
class TestOrphan(CourseTestCase):
"""
Test finding orphans via view and django config
"""
def setUp(self):
super(TestOrphan, self).setUp()
runtime = self.course.runtime
self._create_item('chapter', 'Chapter1', {}, {'display_name': 'Chapter 1'}, 'course', self.course.location.name, runtime)
self._create_item('chapter', 'Chapter2', {}, {'display_name': 'Chapter 2'}, 'course', self.course.location.name, runtime)
self._create_item('chapter', 'OrphanChapter', {}, {'display_name': 'Orphan Chapter'}, None, None, runtime)
self._create_item('vertical', 'Vert1', {}, {'display_name': 'Vertical 1'}, 'chapter', 'Chapter1', runtime)
self._create_item('vertical', 'OrphanVert', {}, {'display_name': 'Orphan Vertical'}, None, None, runtime)
self._create_item('html', 'Html1', "<p>Goodbye</p>", {'display_name': 'Parented Html'}, 'vertical', 'Vert1', runtime)
self._create_item('html', 'OrphanHtml', "<p>Hello</p>", {'display_name': 'Orphan html'}, None, None, runtime)
self._create_item('static_tab', 'staticuno', "<p>tab</p>", {'display_name': 'Tab uno'}, None, None, runtime)
self._create_item('about', 'overview', "<p>overview</p>", {}, None, None, runtime)
self._create_item('course_info', 'updates', "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>", {}, None, None, runtime)
def _create_item(self, category, name, data, metadata, parent_category, parent_name, runtime):
location = self.course.location.replace(category=category, name=name)
editable_modulestore('direct').create_and_save_xmodule(location, data, metadata, runtime)
if parent_name:
# add child to parent in mongo
parent_location = self.course.location.replace(category=parent_category, name=parent_name)
parent = editable_modulestore('direct').get_item(parent_location)
parent.children.append(location.url())
editable_modulestore('direct').update_children(parent_location, parent.children)
def test_mongo_orphan(self):
"""
Test that old mongo finds the orphans
"""
orphans = json.loads(
self.client.get(
reverse(
'orphan',
kwargs={'course_id': '{}.{}'.format(self.course.location.org, self.course.location.course)}
),
HTTP_ACCEPT='application/json'
).content
)
self.assertEqual(len(orphans), 3, "Wrong # {}".format(orphans))
location = self.course.location.replace(category='chapter', name='OrphanChapter')
self.assertIn(location.url(), orphans)
location = self.course.location.replace(category='vertical', name='OrphanVert')
self.assertIn(location.url(), orphans)
location = self.course.location.replace(category='html', name='OrphanHtml')
self.assertIn(location.url(), orphans)

19
cms/djangoapps/contentstore/views/item.py
View File

@@ -21,7 +21,7 @@ from .access import has_access
from .helpers import _xmodule_recurse
from xmodule.x_module import XModuleDescriptor
__all__ = ['save_item', 'create_item', 'delete_item']
__all__ = ['save_item', 'create_item', 'delete_item', 'orphan']
log = logging.getLogger(__name__)
@@ -200,3 +200,20 @@ def delete_item(request):
modulestore('direct').update_children(parent.location, parent.children)
return JsonResponse()
@login_required
def orphan(request, course_id):
"""
View for handling orphan related requests. A get gets all of the current orphans.
DELETE, PUT and POST are meaningless for now.
An orphan is a block whose category is not in the DETACHED_CATEGORY list, is not the root, and is not reachable
from the root via children
:param request:
:param course_id: Locator syntax course_id
"""
# dhm: I'd add DELETE but I'm not sure what type of authentication/authorization we'd need
if request.method == 'GET':
return JsonResponse(modulestore().get_orphans(course_id, DETACHED_CATEGORIES, 'draft'))

2
cms/urls.py
View File

@@ -130,6 +130,8 @@ urlpatterns += patterns(
url(r'^login_post$', 'student.views.login_user', name='login_post'),
url(r'^logout$', 'student.views.logout_user', name='logout'),
url(r'^(?P<course_id>[^/]+)/orphan', 'contentstore.views.orphan', name='orphan')
)
# restful api

35
common/lib/xmodule/xmodule/modulestore/mongo/base.py
View File

@@ -34,6 +34,7 @@ from xblock.fields import Scope, ScopeIds
from xmodule.modulestore import ModuleStoreBase, Location, MONGO_MODULESTORE_TYPE
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.inheritance import own_metadata, InheritanceMixin, inherit_metadata, InheritanceKeyValueStore
import re
log = logging.getLogger(__name__)
@@ -697,7 +698,7 @@ class MongoModuleStore(ModuleStoreBase):
course.tabs = existing_tabs
# Save any changes to the course to the MongoKeyValueStore
course.save()
self.update_metadata(course.location, course.xblock_kvs._metadata)
self.update_metadata(course.location, course.get_explicitly_set_fields_by_scope(Scope.settings))
def fire_updated_modulestore_signal(self, course_id, location):
"""
@@ -854,6 +855,38 @@ class MongoModuleStore(ModuleStoreBase):
"""
return MONGO_MODULESTORE_TYPE
COURSE_ID_RE = re.compile(r'(?P<org>[^.]+)\.(?P<course_id>.+)')
def parse_course_id(self, course_id):
"""
Parse a Locator style course_id into a dict w/ the org and course_id
:param course_id: a string looking like 'org.course.id.part'
"""
match = self.COURSE_ID_RE.match(course_id)
if match is None:
raise ValueError(course_id)
return match.groupdict()
def get_orphans(self, course_id, detached_categories, _branch):
"""
Return a dict of all of the orphans in the course.
:param course_id:
"""
locator_dict = self.parse_course_id(course_id)
all_items = self.collection.find({
'_id.org': locator_dict['org'],
'_id.course': locator_dict['course_id'],
'_id.category': {'$nin': detached_categories}
})
all_reachable = set()
item_locs = set()
for item in all_items:
if item['_id']['category'] != 'course':
item_locs.add(Location(item['_id']).replace(revision=None).url())
all_reachable = all_reachable.union(item.get('definition', {}).get('children', []))
item_locs -= all_reachable
return list(item_locs)
def _create_new_field_data(self, category, location, definition_data, metadata):
"""
To instantiate a new xmodule which will be saved latter, set up the dbModel and kvs

18
common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
View File

@@ -421,6 +421,24 @@ class SplitMongoModuleStore(ModuleStoreBase):
items.append(BlockUsageLocator(url=locator.as_course_locator(), usage_id=parent_id))
return items
def get_orphans(self, course_id, detached_categories, branch):
"""
Return a dict of all of the orphans in the course.
:param course_id:
"""
course = self._lookup_course(CourseLocator(course_id=course_id, branch=branch))
items = set(course['structure']['blocks'].keys())
items.remove(course['structure']['root'])
for block_id, block_data in course['structure']['blocks'].iteritems():
items.difference_update(block_data.get('fields', {}).get('children', []))
if block_data['category'] in detached_categories:
items.discard(block_id)
return [
BlockUsageLocator(course_id=course_id, branch=branch, usage_id=block_id)
for block_id in items
]
def get_course_index_info(self, course_locator):
"""
The index records the initial creation of the indexed course and tracks the current version

147
common/lib/xmodule/xmodule/modulestore/tests/test_orphan.py Normal file
View File

@@ -0,0 +1,147 @@
import uuid
import mock
import unittest
import random
import datetime
from xmodule.modulestore.inheritance import InheritanceMixin
from xmodule.modulestore.mongo import MongoModuleStore
from xmodule.modulestore.split_mongo import SplitMongoModuleStore
from xmodule.modulestore import Location
from xmodule.fields import Date
from xmodule.modulestore.locator import BlockUsageLocator, CourseLocator
class TestOrphan(unittest.TestCase):
"""
Test the orphan finding code
"""
# Snippet of what would be in the django settings envs file
db_config = {
'host': 'localhost',
'db': 'test_xmodule',
}
modulestore_options = dict({
'default_class': 'xmodule.raw_module.RawDescriptor',
'fs_root': '',
'render_template': mock.Mock(return_value=""),
'xblock_mixins': (InheritanceMixin,)
}, **db_config)
split_course_id = 'test_org.test_course.runid'
def setUp(self):
self.modulestore_options['collection'] = 'modulestore{0}'.format(uuid.uuid4().hex)
self.userid = random.getrandbits(32)
super(TestOrphan, self).setUp()
self.split_mongo = SplitMongoModuleStore(
**self.modulestore_options
)
self.addCleanup(self.tearDownSplit)
self.old_mongo = MongoModuleStore(**self.modulestore_options)
self.addCleanup(self.tearDownMongo)
self.course_location = None
self._create_course()
def tearDownSplit(self):
split_db = self.split_mongo.db
split_db.drop_collection(split_db.course_index)
split_db.drop_collection(split_db.structures)
split_db.drop_collection(split_db.definitions)
split_db.connection.close()
def tearDownMongo(self):
split_db = self.split_mongo.db
# old_mongo doesn't give a db attr, but all of the dbs are the same
split_db.drop_collection(self.old_mongo.collection)
def _create_item(self, category, name, data, metadata, parent_category, parent_name, runtime):
location = Location('i4x', 'test_org', 'test_course', category, name)
self.old_mongo.create_and_save_xmodule(location, data, metadata, runtime)
if isinstance(data, basestring):
fields = {'data': data}
else:
fields = data.copy()
fields.update(metadata)
if parent_name:
# add child to parent in mongo
parent_location = Location('i4x', 'test_org', 'test_course', parent_category, parent_name)
parent = self.old_mongo.get_item(parent_location)
parent.children.append(location.url())
self.old_mongo.update_children(parent_location, parent.children)
# create pointer for split
course_or_parent_locator = BlockUsageLocator(
course_id=self.split_course_id,
branch='draft',
usage_id=parent_name
)
else:
course_or_parent_locator = CourseLocator(
course_id='test_org.test_course.runid',
branch='draft',
)
self.split_mongo.create_item(course_or_parent_locator, category, self.userid, usage_id=name, fields=fields)
def _create_course(self):
"""
* some detached items
* some attached children
* some orphans
"""
date_proxy = Date()
metadata = {
'start': date_proxy.to_json(datetime.datetime(2000, 3, 13, 4)),
'display_name': 'Migration test course',
}
data = {
'wiki_slug': 'test_course_slug'
}
fields = metadata.copy()
fields.update(data)
# split requires the course to be created separately from creating items
self.split_mongo.create_course(
'test_org', 'my course', self.userid, self.split_course_id, fields=fields, root_usage_id='runid'
)
self.course_location = Location('i4x', 'test_org', 'test_course', 'course', 'runid')
self.old_mongo.create_and_save_xmodule(self.course_location, data, metadata)
runtime = self.old_mongo.get_item(self.course_location).runtime
self._create_item('chapter', 'Chapter1', {}, {'display_name': 'Chapter 1'}, 'course', 'runid', runtime)
self._create_item('chapter', 'Chapter2', {}, {'display_name': 'Chapter 2'}, 'course', 'runid', runtime)
self._create_item('chapter', 'OrphanChapter', {}, {'display_name': 'Orphan Chapter'}, None, None, runtime)
self._create_item('vertical', 'Vert1', {}, {'display_name': 'Vertical 1'}, 'chapter', 'Chapter1', runtime)
self._create_item('vertical', 'OrphanVert', {}, {'display_name': 'Orphan Vertical'}, None, None, runtime)
self._create_item('html', 'Html1', "<p>Goodbye</p>", {'display_name': 'Parented Html'}, 'vertical', 'Vert1', runtime)
self._create_item('html', 'OrphanHtml', "<p>Hello</p>", {'display_name': 'Orphan html'}, None, None, runtime)
self._create_item('static_tab', 'staticuno', "<p>tab</p>", {'display_name': 'Tab uno'}, None, None, runtime)
self._create_item('about', 'overview', "<p>overview</p>", {}, None, None, runtime)
self._create_item('course_info', 'updates', "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>", {}, None, None, runtime)
def test_mongo_orphan(self):
"""
Test that old mongo finds the orphans
"""
orphans = self.old_mongo.get_orphans('test_org.test_course', ['static_tab', 'about', 'course_info'], None)
self.assertEqual(len(orphans), 3, "Wrong # {}".format(orphans))
location = self.course_location.replace(category='chapter', name='OrphanChapter')
self.assertIn(location.url(), orphans)
location = self.course_location.replace(category='vertical', name='OrphanVert')
self.assertIn(location.url(), orphans)
location = self.course_location.replace(category='html', name='OrphanHtml')
self.assertIn(location.url(), orphans)
def test_split_orphan(self):
"""
Test that old mongo finds the orphans
"""
orphans = self.split_mongo.get_orphans(self.split_course_id, ['static_tab', 'about', 'course_info'], 'draft')
self.assertEqual(len(orphans), 3, "Wrong # {}".format(orphans))
location = BlockUsageLocator(course_id=self.split_course_id, branch='draft', usage_id='OrphanChapter')
self.assertIn(location, orphans)
location = BlockUsageLocator(course_id=self.split_course_id, branch='draft', usage_id='OrphanVert')
self.assertIn(location, orphans)
location = BlockUsageLocator(course_id=self.split_course_id, branch='draft', usage_id='OrphanHtml')
self.assertIn(location, orphans)