Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3915 from edx/studio/uses-mixed-modulestore

Browse Source 
Enable mixed modulestore
This commit is contained in:
Nimisha Asthagiri
2014-06-26 18:52:58 -04:00
parent 580de5528a a9213509c3
commit b5d1ec089a
125 changed files with 2984 additions and 2190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
cms/djangoapps/contentstore/course_info_model.py
View File

@@ -28,16 +28,15 @@ from xmodule.html_module import CourseInfoModule
log = logging.getLogger(__name__)
def get_course_updates(location, provided_id):
def get_course_updates(location, provided_id, user_id):
"""
Retrieve the relevant course_info updates and unpack into the model which the client expects:
[{id : index, date : string, content : html string}]
"""
try:
course_updates = modulestore('direct').get_item(location)
course_updates = modulestore().get_item(location)
except ItemNotFoundError:
modulestore('direct').create_and_save_xmodule(location)
course_updates = modulestore('direct').get_item(location)
course_updates = modulestore().create_and_save_xmodule(location, user_id)
course_update_items = get_course_update_items(course_updates, provided_id)
return _get_visible_update(course_update_items)
@@ -50,10 +49,9 @@ def update_course_updates(location, update, passed_id=None, user=None):
into the html structure.
"""
try:
course_updates = modulestore('direct').get_item(location)
course_updates = modulestore().get_item(location)
except ItemNotFoundError:
modulestore('direct').create_and_save_xmodule(location)
course_updates = modulestore('direct').get_item(location)
course_updates = modulestore().create_and_save_xmodule(location, user.id)
course_update_items = list(reversed(get_course_update_items(course_updates)))
@@ -135,7 +133,7 @@ def delete_course_update(location, update, passed_id, user):
return HttpResponseBadRequest()
try:
course_updates = modulestore('direct').get_item(location)
course_updates = modulestore().get_item(location)
except ItemNotFoundError:
return HttpResponseBadRequest()
@@ -239,6 +237,6 @@ def save_course_update_items(location, course_updates, course_update_items, user
course_updates.data = _get_html(course_update_items)
# update db record
modulestore('direct').update_item(course_updates, user.id)
modulestore().update_item(course_updates, user.id)
return course_updates

6
cms/djangoapps/contentstore/features/video.py
View File

@@ -1,8 +1,8 @@
# pylint: disable=C0111
from lettuce import world, step
from contentstore.utils import get_modulestore
from selenium.webdriver.common.keys import Keys
from xmodule.modulestore.django import modulestore
VIDEO_BUTTONS = {
'CC': '.hide-subtitles',
@@ -137,9 +137,9 @@ def xml_only_video(step):
world.wait(1)
course = world.scenario_dict['COURSE']
store = get_modulestore(course.location)
store = modulestore()
parent_location = store.get_items(course.id, category='vertical', revision='draft')[0].location
parent_location = store.get_items(course.id, category='vertical')[0].location
youtube_id = 'ABCDEFG'
world.scenario_dict['YOUTUBE_ID'] = youtube_id

4
cms/djangoapps/contentstore/git_export_utils.py
View File

@@ -128,8 +128,8 @@ def export_to_git(course_id, repo, user='', rdir=None):
root_dir = os.path.dirname(rdirp)
course_dir = os.path.splitext(os.path.basename(rdirp))[0]
try:
export_to_xml(modulestore('direct'), contentstore(), course_id,
root_dir, course_dir, modulestore())
export_to_xml(modulestore(), contentstore(), course_id,
root_dir, course_dir)
except (EnvironmentError, AttributeError):
log.exception('Failed export to xml')
raise GitExportError(GitExportError.XML_EXPORT_FAIL)

24
cms/djangoapps/contentstore/management/commands/clone_course.py
View File

@@ -4,6 +4,7 @@ Script for cloning a course
from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.store_utilities import clone_course
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.mixed import store_bulk_write_operations_on_course
from xmodule.contentstore.django import contentstore
from student.roles import CourseInstructorRole, CourseStaffRole
from opaque_keys.edx.keys import CourseKey
@@ -35,19 +36,18 @@ class Command(BaseCommand):
source_course_id = self.course_key_from_arg(args[0])
dest_course_id = self.course_key_from_arg(args[1])
mstore = modulestore('direct')
mstore = modulestore()
cstore = contentstore()
mstore.ignore_write_events_on_courses.add(dest_course_id)
print("Cloning course {0} to {1}".format(source_course_id, dest_course_id))
if clone_course(mstore, cstore, source_course_id, dest_course_id):
print("copying User permissions...")
# purposely avoids auth.add_user b/c it doesn't have a caller to authorize
CourseInstructorRole(dest_course_id).add_users(
*CourseInstructorRole(source_course_id).users_with_role()
)
CourseStaffRole(dest_course_id).add_users(
*CourseStaffRole(source_course_id).users_with_role()
)
with store_bulk_write_operations_on_course(mstore, dest_course_id):
if clone_course(mstore, cstore, source_course_id, dest_course_id, None):
print("copying User permissions...")
# purposely avoids auth.add_user b/c it doesn't have a caller to authorize
CourseInstructorRole(dest_course_id).add_users(
*CourseInstructorRole(source_course_id).users_with_role()
)
CourseStaffRole(dest_course_id).add_users(
*CourseStaffRole(source_course_id).users_with_role()
)

7
cms/djangoapps/contentstore/management/commands/course_id_clash.py
View File

@@ -3,6 +3,7 @@ Script for finding all courses whose org/name pairs == other courses when ignori
"""
from django.core.management.base import BaseCommand
from xmodule.modulestore.django import modulestore
from xmodule.modulestore import MONGO_MODULESTORE_TYPE
#
@@ -10,12 +11,12 @@ from xmodule.modulestore.django import modulestore
#
class Command(BaseCommand):
"""
Script for finding all courses whose org/name pairs == other courses when ignoring case
Script for finding all courses in the Mongo Modulestore whose org/name pairs == other courses when ignoring case
"""
help = 'List all courses ids which may collide when ignoring case'
help = 'List all courses ids in the Mongo Modulestore which may collide when ignoring case'
def handle(self, *args, **options):
mstore = modulestore()
mstore = modulestore()._get_modulestore_by_type(MONGO_MODULESTORE_TYPE) # pylint: disable=protected-access
if hasattr(mstore, 'collection'):
map_fn = '''
function () {

2
cms/djangoapps/contentstore/management/commands/empty_asset_trashcan.py
View File

@@ -22,7 +22,7 @@ class Command(BaseCommand):
course_ids = [course_key]
else:
course_ids = [course.id for course in modulestore('direct').get_courses()]
course_ids = [course.id for course in modulestore().get_courses()]
if query_yes_no("Emptying trashcan. Confirm?", default="no"):
empty_asset_trashcan(course_ids)

2
cms/djangoapps/contentstore/management/commands/export.py
View File

@@ -35,4 +35,4 @@ class Command(BaseCommand):
root_dir = os.path.dirname(output_path)
course_dir = os.path.splitext(os.path.basename(output_path))[0]
export_to_xml(modulestore('direct'), contentstore(), course_key, root_dir, course_dir, modulestore())
export_to_xml(modulestore(), contentstore(), course_key, root_dir, course_dir)

4
cms/djangoapps/contentstore/management/commands/export_all_courses.py
View File

@@ -19,7 +19,7 @@ class Command(BaseCommand):
output_path = args[0]
cs = contentstore()
ms = modulestore('direct')
ms = modulestore()
root_dir = output_path
courses = ms.get_courses()
@@ -35,7 +35,7 @@ class Command(BaseCommand):
if 1:
try:
course_dir = course_id.replace('/', '...')
export_to_xml(ms, cs, course_id, root_dir, course_dir, modulestore())
export_to_xml(ms, cs, course_id, root_dir, course_dir)
except Exception as err:
print("="*30 + "> Oops, failed to export %s" % course_id)
print("Error:")

9
cms/djangoapps/contentstore/management/commands/import.py
View File

@@ -37,15 +37,10 @@ class Command(BaseCommand):
data=data_dir,
courses=course_dirs,
dis=do_import_static))
try:
mstore = modulestore('direct')
except KeyError:
self.stdout.write('Unable to load direct modulestore, trying '
'default\n')
mstore = modulestore('default')
mstore = modulestore()
_, course_items = import_from_xml(
mstore, data_dir, course_dirs, load_error_modules=False,
mstore, "**replace_user**", data_dir, course_dirs, load_error_modules=False,
static_content_store=contentstore(), verbose=True,
do_import_static=do_import_static,
create_new_course_if_not_present=True,

3
cms/djangoapps/contentstore/management/commands/tests/test_import.py
View File

@@ -8,16 +8,13 @@ import shutil
import tempfile
from django.core.management import call_command
from django.test.utils import override_settings
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from django_comment_common.utils import are_permissions_roles_seeded
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from opaque_keys.edx.locations import SlashSeparatedCourseKey
@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestImport(ModuleStoreTestCase):
"""
Unit tests for importing a course from command line

8
cms/djangoapps/contentstore/management/commands/tests/test_migrate_to_split.py
View File

@@ -5,9 +5,8 @@ import unittest
from django.contrib.auth.models import User
from django.core.management import CommandError, call_command
from django.test.utils import override_settings
from contentstore.management.commands.migrate_to_split import Command
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from xmodule.modulestore import SPLIT_MONGO_MODULESTORE_TYPE, REVISION_OPTION_PUBLISHED_ONLY
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.django import modulestore, clear_existing_modulestores
@@ -45,7 +44,6 @@ class TestArgParsing(unittest.TestCase):
@unittest.skip("Not fixing split mongo until we land this long branch")
@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestMigrateToSplit(ModuleStoreTestCase):
"""
Unit tests for migrating a course from old mongo to split mongo
@@ -58,7 +56,7 @@ class TestMigrateToSplit(ModuleStoreTestCase):
password = 'foo'
self.user = User.objects.create_user(uname, email, password)
self.course = CourseFactory()
self.addCleanup(ModuleStoreTestCase.drop_mongo_collections, 'split')
self.addCleanup(ModuleStoreTestCase.drop_mongo_collections, SPLIT_MONGO_MODULESTORE_TYPE)
self.addCleanup(clear_existing_modulestores)
def test_user_email(self):
@@ -86,6 +84,6 @@ class TestMigrateToSplit(ModuleStoreTestCase):
str(self.user.id),
"org.dept+name.run",
)
locator = CourseLocator(org="org.dept", offering="name.run", branch="published")
locator = CourseLocator(org="org.dept", offering="name.run", branch=REVISION_OPTION_PUBLISHED_ONLY)
course_from_split = modulestore('split').get_course(locator)
self.assertIsNotNone(course_from_split)

5
cms/djangoapps/contentstore/management/commands/tests/test_rollback_split_course.py
View File

@@ -7,9 +7,7 @@ from mock import patch
from django.contrib.auth.models import User
from django.core.management import CommandError, call_command
from django.test.utils import override_settings
from contentstore.management.commands.rollback_split_course import Command
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.persistent_factories import PersistentCourseFactory
from xmodule.modulestore.tests.factories import CourseFactory
@@ -39,7 +37,6 @@ class TestArgParsing(unittest.TestCase):
@unittest.skip("Not fixing split mongo until we land opaque-keys 0.9")
@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestRollbackSplitCourseNoOldMongo(ModuleStoreTestCase):
"""
Unit tests for rolling back a split-mongo course from command line,
@@ -58,7 +55,6 @@ class TestRollbackSplitCourseNoOldMongo(ModuleStoreTestCase):
@unittest.skip("Not fixing split mongo until we land opaque-keys 0.9")
@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestRollbackSplitCourseNoSplitMongo(ModuleStoreTestCase):
"""
Unit tests for rolling back a split-mongo course from command line,
@@ -77,7 +73,6 @@ class TestRollbackSplitCourseNoSplitMongo(ModuleStoreTestCase):
@unittest.skip("Not fixing split mongo until we land opaque-keys 0.9")
@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestRollbackSplitCourse(ModuleStoreTestCase):
"""
Unit tests for rolling back a split-mongo course from command line

8
cms/djangoapps/contentstore/tests/modulestore_config.py
View File

@@ -1,8 +0,0 @@
"""
Define test configuration for modulestores.
"""
from xmodule.modulestore.tests.django_utils import studio_store_config
from django.conf import settings
TEST_MODULESTORE = studio_store_config(settings.TEST_ROOT / "data")

401
cms/djangoapps/contentstore/tests/test_contentstore.py
View File

@@ -20,8 +20,6 @@ from django.contrib.auth.models import User
from django.test import TestCase
from django.test.utils import override_settings
from contentstore.utils import get_modulestore
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from contentstore.tests.utils import parse_json, AjaxEnabledTestClient
from contentstore.views.component import ADVANCED_COMPONENT_TYPES
@@ -29,7 +27,11 @@ from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import contentstore, _CONTENTSTORE
from xmodule.contentstore.utils import restore_asset_from_trashcan, empty_asset_trashcan
from xmodule.exceptions import NotFoundError, InvalidVersionError
from xmodule.modulestore import mongo
from xmodule.modulestore import (
mongo, MONGO_MODULESTORE_TYPE, PublishState,
REVISION_OPTION_PUBLISHED_ONLY, REVISION_OPTION_DRAFT_ONLY, KEY_REVISION_PUBLISHED, BRANCH_PUBLISHED_ONLY
)
from xmodule.modulestore.mixed import store_branch_setting
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.inheritance import own_metadata
@@ -74,16 +76,12 @@ def get_url(handler_name, key_value, key_name='usage_key_string', kwargs=None):
return reverse_url(handler_name, key_name, key_value, kwargs)
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
class ContentStoreToyCourseTest(ModuleStoreTestCase):
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class ContentStoreTestCase(ModuleStoreTestCase):
"""
Tests that rely on the toy courses.
TODO: refactor using CourseFactory so they do not.
Base class for Content Store Test Cases
"""
def setUp(self):
settings.MODULESTORE['default']['OPTIONS']['fs_root'] = path('common/test/data')
settings.MODULESTORE['direct']['OPTIONS']['fs_root'] = path('common/test/data')
uname = 'testuser'
email = 'test+courses@edx.org'
password = 'foo'
@@ -103,6 +101,12 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
self.client = AjaxEnabledTestClient()
self.client.login(username=uname, password=password)
class ContentStoreToyCourseTest(ContentStoreTestCase):
"""
Tests that rely on the toy courses.
TODO: refactor using CourseFactory so they do not.
"""
def tearDown(self):
MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'])
_CONTENTSTORE.clear()
@@ -119,8 +123,8 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
exactly the same -- for example, 'video' in
component_types should cause 'Video' to be present.
"""
store = modulestore('direct')
_, course_items = import_from_xml(store, 'common/test/data/', ['simple'])
store = modulestore()
_, course_items = import_from_xml(store, self.user.id, 'common/test/data/', ['simple'])
course = course_items[0]
course.advanced_modules = component_types
store.update_item(course, self.user.id)
@@ -147,8 +151,8 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
self.check_components_on_page(['word_cloud'], ['Word cloud'])
def test_malformed_edit_unit_request(self):
store = modulestore('direct')
_, course_items = import_from_xml(store, 'common/test/data/', ['simple'])
store = modulestore()
_, course_items = import_from_xml(store, self.user.id, 'common/test/data/', ['simple'])
# just pick one vertical
usage_key = course_items[0].id.make_usage_key('vertical', None)
@@ -158,7 +162,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
_test_no_locations(self, resp, status_code=400)
def check_edit_unit(self, test_course_name):
_, course_items = import_from_xml(modulestore('direct'), 'common/test/data/', [test_course_name])
_, course_items = import_from_xml(modulestore(), self.user.id, 'common/test/data/', [test_course_name])
items = modulestore().get_items(course_items[0].id, category='vertical')
self._check_verticals(items)
@@ -191,25 +195,22 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
Unfortunately, None = published for the revision field, so get_items() would return
both draft and non-draft copies.
'''
direct_store = modulestore('direct')
draft_store = modulestore('draft')
_, course_items = import_from_xml(direct_store, 'common/test/data/', ['simple'])
store = modulestore()
_, course_items = import_from_xml(store, self.user.id, 'common/test/data/', ['simple'])
course_key = course_items[0].id
html_usage_key = course_key.make_usage_key('html', 'test_html')
html_module_from_draft_store = draft_store.get_item(html_usage_key)
draft_store.convert_to_draft(html_module_from_draft_store.location)
html_module_from_draft_store = store.get_item(html_usage_key)
store.convert_to_draft(html_module_from_draft_store.location, self.user.id)
# Query get_items() and find the html item. This should just return back a single item (not 2).
direct_store_items = direct_store.get_items(course_key)
direct_store_items = store.get_items(course_key, revision=REVISION_OPTION_PUBLISHED_ONLY)
html_items_from_direct_store = [item for item in direct_store_items if (item.location == html_usage_key)]
self.assertEqual(len(html_items_from_direct_store), 1)
self.assertFalse(getattr(html_items_from_direct_store[0], 'is_draft', False))
# Fetch from the draft store. Note that even though we pass
# None in the revision field, the draft store will replace that with 'draft'.
draft_store_items = draft_store.get_items(course_key)
# Fetch from the draft store.
draft_store_items = store.get_items(course_key, revision=REVISION_OPTION_DRAFT_ONLY)
html_items_from_draft_store = [item for item in draft_store_items if (item.location == html_usage_key)]
self.assertEqual(len(html_items_from_draft_store), 1)
self.assertTrue(getattr(html_items_from_draft_store[0], 'is_draft', False))
@@ -221,9 +222,8 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
module as 'own-metadata' when publishing. Also verifies the metadata inheritance is
properly computed
'''
store = modulestore('direct')
draft_store = modulestore('draft')
import_from_xml(store, 'common/test/data/', ['simple'])
draft_store = modulestore()
import_from_xml(draft_store, self.user.id, 'common/test/data/', ['simple'])
course_key = SlashSeparatedCourseKey('edX', 'simple', '2012_Fall')
html_usage_key = course_key.make_usage_key('html', 'test_html')
@@ -233,7 +233,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
self.assertEqual(html_module.graceperiod, course.graceperiod)
self.assertNotIn('graceperiod', own_metadata(html_module))
draft_store.convert_to_draft(html_module.location)
draft_store.convert_to_draft(html_module.location, self.user.id)
# refetch to check metadata
html_module = draft_store.get_item(html_usage_key)
@@ -242,7 +242,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
self.assertNotIn('graceperiod', own_metadata(html_module))
# publish module
draft_store.publish(html_module.location, 0)
draft_store.publish(html_module.location, self.user.id)
# refetch to check metadata
html_module = draft_store.get_item(html_usage_key)
@@ -251,7 +251,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
self.assertNotIn('graceperiod', own_metadata(html_module))
# put back in draft and change metadata and see if it's now marked as 'own_metadata'
draft_store.convert_to_draft(html_module.location)
draft_store.convert_to_draft(html_module.location, self.user.id)
html_module = draft_store.get_item(html_usage_key)
new_graceperiod = timedelta(hours=1)
@@ -273,45 +273,46 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
self.assertEqual(html_module.graceperiod, new_graceperiod)
# republish
draft_store.publish(html_module.location, 0)
draft_store.publish(html_module.location, self.user.id)
# and re-read and verify 'own-metadata'
draft_store.convert_to_draft(html_module.location)
draft_store.convert_to_draft(html_module.location, self.user.id)
html_module = draft_store.get_item(html_usage_key)
self.assertIn('graceperiod', own_metadata(html_module))
self.assertEqual(html_module.graceperiod, new_graceperiod)
def test_get_depth_with_drafts(self):
import_from_xml(modulestore('direct'), 'common/test/data/', ['simple'])
store = modulestore()
import_from_xml(store, self.user.id, 'common/test/data/', ['simple'])
course_key = SlashSeparatedCourseKey('edX', 'simple', '2012_Fall')
course = modulestore('draft').get_course(course_key)
course = store.get_course(course_key)
# make sure no draft items have been returned
num_drafts = self._get_draft_counts(course)
self.assertEqual(num_drafts, 0)
problem_usage_key = course_key.make_usage_key('problem', 'ps01-simple')
problem = modulestore('draft').get_item(problem_usage_key)
problem = store.get_item(problem_usage_key)
# put into draft
modulestore('draft').convert_to_draft(problem.location)
store.convert_to_draft(problem.location, self.user.id)
# make sure we can query that item and verify that it is a draft
draft_problem = modulestore('draft').get_item(problem_usage_key)
draft_problem = store.get_item(problem_usage_key)
self.assertTrue(getattr(draft_problem, 'is_draft', False))
# now requery with depth
course = modulestore('draft').get_course(course_key)
course = store.get_course(course_key)
# make sure just one draft item have been returned
num_drafts = self._get_draft_counts(course)
self.assertEqual(num_drafts, 1)
def test_no_static_link_rewrites_on_import(self):
module_store = modulestore('direct')
_, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'])
module_store = modulestore()
_, course_items = import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'])
course = course_items[0]
handouts_usage_key = course.id.make_usage_key('course_info', 'handouts')
@@ -330,14 +331,14 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
</table_of_contents>
""").strip()
module_store = modulestore('direct')
import_from_xml(module_store, 'common/test/data/', ['toy'])
module_store = modulestore()
import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'])
course = module_store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
self.assertGreater(len(course.textbooks), 0)
def test_import_polls(self):
module_store = modulestore('direct')
_, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'])
module_store = modulestore()
_, course_items = import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'])
course_key = course_items[0].id
items = module_store.get_items(course_key, category='poll_question')
@@ -356,8 +357,8 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
"""
Tests the ajax callback to render an XModule
"""
direct_store = modulestore('direct')
_, course_items = import_from_xml(direct_store, 'common/test/data/', ['toy'])
direct_store = modulestore()
_, course_items = import_from_xml(direct_store, self.user.id, 'common/test/data/', ['toy'])
usage_key = course_items[0].id.make_usage_key('vertical', 'vertical_test')
# also try a custom response which will trigger the 'is this course in whitelist' logic
@@ -375,32 +376,32 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
self.assertContains(resp, 'edX+toy+2012_Fall+poll_question+T1_changemind_poll_foo_2')
def test_delete(self):
direct_store = modulestore('direct')
store = modulestore()
course = CourseFactory.create(org='edX', course='999', display_name='Robot Super Course')
chapterloc = ItemFactory.create(parent_location=course.location, display_name="Chapter").location
ItemFactory.create(parent_location=chapterloc, category='sequential', display_name="Sequential")
sequential_key = course.id.make_usage_key('sequential', 'Sequential')
sequential = direct_store.get_item(sequential_key)
sequential = store.get_item(sequential_key)
chapter_key = course.id.make_usage_key('chapter', 'Chapter')
chapter = direct_store.get_item(chapter_key)
chapter = store.get_item(chapter_key)
# make sure the parent points to the child object which is to be deleted
self.assertTrue(sequential.location in chapter.children)
self.client.delete(get_url('xblock_handler', sequential_key), {'recurse': True, 'all_versions': True})
self.client.delete(get_url('xblock_handler', sequential_key))
found = False
try:
direct_store.get_item(sequential_key)
store.get_item(sequential_key)
found = True
except ItemNotFoundError:
pass
self.assertFalse(found)
chapter = direct_store.get_item(chapter_key)
chapter = store.get_item(chapter_key)
# make sure the parent no longer points to the child object which was deleted
self.assertFalse(sequential.location in chapter.children)
@@ -410,8 +411,8 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
This test case verifies that a course can use specialized override for about data, e.g. /about/Fall_2012/effort.html
while there is a base definition in /about/effort.html
'''
module_store = modulestore('direct')
_, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'])
module_store = modulestore()
_, course_items = import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'])
course_key = course_items[0].id
effort = module_store.get_item(course_key.make_usage_key('about', 'effort'))
self.assertEqual(effort.data, '6 hours')
@@ -426,8 +427,8 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
'''
content_store = contentstore()
module_store = modulestore('direct')
import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store, verbose=True)
module_store = modulestore()
import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'], static_content_store=content_store, verbose=True)
course = module_store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
@@ -515,8 +516,8 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
content_store = contentstore()
trash_store = contentstore('trashcan')
module_store = modulestore('direct')
_, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store)
module_store = modulestore()
_, course_items = import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'], static_content_store=content_store)
# look up original (and thumbnail) in content store, should be there after import
location = AssetLocation.from_deprecated_string('/c4x/edX/toy/asset/sample_static.txt')
@@ -547,9 +548,9 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
Test that course info updates are imported and exported with all content fields ('data', 'items')
"""
content_store = contentstore()
module_store = modulestore('direct')
module_store = modulestore()
data_dir = "common/test/data/"
import_from_xml(module_store, data_dir, ['course_info_updates'],
import_from_xml(module_store, self.user.id, data_dir, ['course_info_updates'],
static_content_store=content_store, verbose=True)
course_id = SlashSeparatedCourseKey('edX', 'course_info_updates', '2014_T1')
@@ -636,25 +637,22 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
'run': '2013_Spring',
}
module_store = modulestore('direct')
draft_store = modulestore('draft')
_, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'])
module_store = modulestore()
_, course_items = import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'])
source_course_id = course_items[0].id
dest_course_id = _get_course_id(course_data)
# get a vertical (and components in it) to put into 'draft'
# get a vertical (and components in it) to put into DRAFT
# this is to assert that draft content is also cloned over
vertical = module_store.get_item(
source_course_id.make_usage_key('vertical', 'vertical_test'),
depth=1
)
draft_store.convert_to_draft(vertical.location)
for child in vertical.get_children():
draft_store.convert_to_draft(child.location)
module_store.convert_to_draft(vertical.location, self.user.id)
items = module_store.get_items(source_course_id, revision='draft')
items = module_store.get_items(source_course_id)
self.assertGreater(len(items), 0)
_create_course(self, dest_course_id, course_data)
@@ -662,25 +660,27 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
content_store = contentstore()
# now do the actual cloning
clone_course(module_store, content_store, source_course_id, dest_course_id)
clone_course(module_store, content_store, source_course_id, dest_course_id, self.user.id)
# first assert that all draft content got cloned as well
items = module_store.get_items(source_course_id, revision='draft')
self.assertGreater(len(items), 0)
clone_items = module_store.get_items(dest_course_id, revision='draft')
self.assertGreater(len(clone_items), 0)
self.assertEqual(len(items), len(clone_items))
draft_items = module_store.get_items(source_course_id, revision=REVISION_OPTION_DRAFT_ONLY)
self.assertGreater(len(draft_items), 0)
draft_clone_items = module_store.get_items(dest_course_id, revision=REVISION_OPTION_DRAFT_ONLY)
self.assertGreater(len(draft_clone_items), 0)
self.assertEqual(len(draft_items), len(draft_clone_items))
# now loop through all the units in the course and verify that the clone can render them, which
# means the objects are at least present
items = module_store.get_items(source_course_id, revision=None)
items = module_store.get_items(source_course_id)
self.assertGreater(len(items), 0)
clone_items = module_store.get_items(dest_course_id, revision=None)
clone_items = module_store.get_items(dest_course_id)
self.assertGreater(len(clone_items), 0)
for descriptor in items:
source_item = module_store.get_item(descriptor.location)
new_loc = descriptor.location.map_into_course(dest_course_id)
if descriptor.location.category == 'course':
new_loc = new_loc.replace(name=new_loc.run)
print "Checking {0} should now also be at {1}".format(descriptor.location, new_loc)
lookup_item = module_store.get_item(new_loc)
@@ -708,10 +708,10 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
'run': '2013_Spring'
}
module_store = modulestore('direct')
module_store = modulestore()
content_store = contentstore()
import_from_xml(module_store, 'common/test/data/', ['toy'])
import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'])
source_course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
dest_course_id = _get_course_id(course_data)
@@ -732,7 +732,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
_create_course(self, dest_course_id, course_data)
# do the actual cloning
clone_course(module_store, content_store, source_course_id, dest_course_id)
clone_course(module_store, content_store, source_course_id, dest_course_id, self.user.id)
# make sure that any non-portable links are rewritten during cloning
html_module = module_store.get_item(dest_course_id.make_usage_key('html', 'nonportable'))
@@ -740,16 +740,16 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
self.assertIn('/asset/foo.jpg', html_module.data)
def test_illegal_draft_crud_ops(self):
draft_store = modulestore('draft')
draft_store = modulestore()
course = CourseFactory.create(org='MITx', course='999', display_name='Robot Super Course')
location = course.id.make_usage_key('chapter', 'neuvo')
# Ensure draft mongo store does not create drafts for things that shouldn't be draft
newobject = draft_store.create_and_save_xmodule(location)
newobject = draft_store.create_and_save_xmodule(location, self.user.id)
self.assertFalse(getattr(newobject, 'is_draft', False))
with self.assertRaises(InvalidVersionError):
draft_store.convert_to_draft(location)
draft_store.convert_to_draft(location, self.user.id)
chapter = draft_store.get_item(location)
chapter.data = 'chapter data'
@@ -758,7 +758,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
self.assertFalse(getattr(newobject, 'is_draft', False))
with self.assertRaises(InvalidVersionError):
draft_store.unpublish(location)
draft_store.unpublish(location, self.user.id)
def test_bad_contentstore_request(self):
resp = self.client.get_html('http://localhost:8001/c4x/CDX/123123/asset/&images_circuits_Lab7Solution2.png')
@@ -766,10 +766,10 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
_test_no_locations(self, resp, 400)
def test_rewrite_nonportable_links_on_import(self):
module_store = modulestore('direct')
module_store = modulestore()
content_store = contentstore()
import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store)
import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'], static_content_store=content_store)
# first check a static asset link
course_key = SlashSeparatedCourseKey('edX', 'toy', 'run')
@@ -787,21 +787,17 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
This test will import a course, make a draft item, and delete it. This will also assert that the
draft content is also deleted
"""
module_store = modulestore('direct')
module_store = modulestore()
content_store = contentstore()
draft_store = modulestore('draft')
_, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store)
_, course_items = import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'], static_content_store=content_store)
course_id = course_items[0].id
# get a vertical (and components in it) to put into 'draft'
# get a vertical (and components in it) to put into DRAFT
vertical = module_store.get_item(course_id.make_usage_key('vertical', 'vertical_test'), depth=1)
draft_store.convert_to_draft(vertical.location)
for child in vertical.get_children():
draft_store.convert_to_draft(child.location)
module_store.convert_to_draft(vertical.location, self.user.id)
# delete the course
delete_course(module_store, content_store, course_id, commit=True)
@@ -827,53 +823,58 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
self.assertTrue(filesystem.exists(item.location.name + filename_suffix))
@mock.patch('xmodule.course_module.requests.get')
def test_export_course(self, mock_get):
def test_export_course_roundtrip(self, mock_get):
mock_get.return_value.text = dedent("""
<?xml version="1.0"?><table_of_contents>
<entry page="5" page_label="ii" name="Table of Contents"/>
</table_of_contents>
""").strip()
module_store = modulestore('direct')
draft_store = modulestore('draft')
module_store = modulestore()
content_store = contentstore()
import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store)
import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'], static_content_store=content_store)
course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
# get a vertical (and components in it) to copy into an orphan sub dag
vertical = module_store.get_item(course_id.make_usage_key('vertical', 'vertical_test'), depth=1)
# We had a bug where orphaned draft nodes caused export to fail. This is here to cover that case.
vertical.location = mongo.draft.as_draft(vertical.location.replace(name='no_references'))
vertical.location = vertical.location.replace(name='no_references')
draft_store.update_item(vertical, allow_not_found=True)
orphan_vertical = draft_store.get_item(vertical.location)
module_store.update_item(vertical, self.user.id, allow_not_found=True)
orphan_vertical = module_store.get_item(vertical.location)
self.assertEqual(orphan_vertical.location.name, 'no_references')
# get the original vertical (and components in it) to put into 'draft'
# get the original vertical (and components in it) to put into DRAFT
vertical = module_store.get_item(course_id.make_usage_key('vertical', 'vertical_test'), depth=1)
self.assertEqual(len(orphan_vertical.children), len(vertical.children))
draft_store.convert_to_draft(vertical.location)
for child in vertical.get_children():
draft_store.convert_to_draft(child.location)
draft_vertical = module_store.convert_to_draft(vertical.location, self.user.id)
self.assertEqual(module_store.compute_publish_state(draft_vertical), PublishState.draft)
root_dir = path(mkdtemp_clean())
# now create a new/different private (draft only) vertical
vertical.location = mongo.draft.as_draft(course_id.make_usage_key('vertical', 'a_private_vertical'))
draft_store.update_item(vertical, allow_not_found=True)
private_vertical = draft_store.get_item(vertical.location)
vertical = module_store.create_and_save_xmodule(vertical.location, self.user.id)
self.assertEqual(module_store.compute_publish_state(vertical), PublishState.private)
private_vertical = module_store.get_item(vertical.location)
vertical = None # blank out b/c i destructively manipulated its location 2 lines above
# add the new private to list of children
# now create a new/different published (no draft) vertical
public_vertical_location = course_id.make_usage_key('vertical', 'a_published_vertical')
module_store.create_and_save_xmodule(public_vertical_location, self.user.id)
public_vertical = module_store.publish(public_vertical_location, self.user.id)
self.assertEqual(module_store.compute_publish_state(public_vertical), PublishState.public)
# add the new private and new public to list of children
sequential = module_store.get_item(course_id.make_usage_key('sequential', 'vertical_sequential'))
private_location_no_draft = private_vertical.location.replace(revision=None)
private_location_no_draft = private_vertical.location.replace(revision=KEY_REVISION_PUBLISHED)
sequential.children.append(private_location_no_draft)
sequential.children.append(public_vertical_location)
module_store.update_item(sequential, self.user.id)
# read back the sequential, to make sure we have a pointer to
sequential = module_store.get_item(course_id.make_usage_key('sequential', 'vertical_sequential'))
self.assertIn(private_location_no_draft, sequential.children)
locked_asset_key = self._lock_an_asset(content_store, course_id)
@@ -884,7 +885,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
print 'Exporting to tempdir = {0}'.format(root_dir)
# export out to a tempdir
export_to_xml(module_store, content_store, course_id, root_dir, 'test_export', draft_modulestore=draft_store)
export_to_xml(module_store, content_store, course_id, root_dir, 'test_export')
# check for static tabs
self.verify_content_existence(module_store, root_dir, course_id, 'tabs', 'static_tab', '.html')
@@ -915,44 +916,51 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
delete_course(module_store, content_store, course_id, commit=True)
# reimport over old course
self.check_import(
module_store, root_dir, draft_store, content_store, course_id,
module_store, root_dir, content_store, course_id,
locked_asset_key, locked_asset_attrs
)
# import to different course id
self.check_import(
module_store, root_dir, draft_store, content_store, SlashSeparatedCourseKey('anotherX', 'anotherToy', 'Someday'),
module_store, root_dir, content_store, SlashSeparatedCourseKey('anotherX', 'anotherToy', 'Someday'),
locked_asset_key, locked_asset_attrs
)
shutil.rmtree(root_dir)
def check_import(self, module_store, root_dir, draft_store, content_store, course_id,
def check_import(self, module_store, root_dir, content_store, course_id,
locked_asset_key, locked_asset_attrs):
# reimport
import_from_xml(
module_store,
self.user.id,
root_dir,
['test_export'],
draft_store=draft_store,
static_content_store=content_store,
target_course_id=course_id,
)
items = module_store.get_items(course_id, category='vertical')
items = module_store.get_items(course_id, category='vertical', revision=REVISION_OPTION_PUBLISHED_ONLY)
self._check_verticals(items)
# verify that we have the content in the draft store as well
vertical = draft_store.get_item(
course_id.make_usage_key('vertical', 'vertical_test'),
depth=1
)
def verify_item_publish_state(item, publish_state):
if publish_state in (PublishState.private, PublishState.draft):
self.assertTrue(getattr(item, 'is_draft', False))
else:
self.assertFalse(getattr(item, 'is_draft', False))
self.assertEqual(module_store.compute_publish_state(item), publish_state)
self.assertTrue(getattr(vertical, 'is_draft', False))
def get_and_verify_item_publish_state(item_type, item_name, publish_state):
item = module_store.get_item(course_id.make_usage_key(item_type, item_name))
verify_item_publish_state(item, publish_state)
return item
# verify that the draft vertical is draft
vertical = get_and_verify_item_publish_state('vertical', 'vertical_test', PublishState.draft)
self.assertNotIn('index_in_children_list', vertical.xml_attributes)
self.assertNotIn('parent_sequential_url', vertical.xml_attributes)
for child in vertical.get_children():
self.assertTrue(getattr(child, 'is_draft', False))
verify_item_publish_state(child, PublishState.draft)
self.assertNotIn('index_in_children_list', child.xml_attributes)
if hasattr(child, 'data'):
self.assertNotIn('index_in_children_list', child.data)
@@ -960,19 +968,14 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
if hasattr(child, 'data'):
self.assertNotIn('parent_sequential_url', child.data)
# make sure that we don't have a sequential that is in draft mode
sequential = draft_store.get_item(
course_id.make_usage_key('sequential', 'vertical_sequential')
)
self.assertFalse(getattr(sequential, 'is_draft', False))
# make sure that we don't have a sequential that is not in draft mode
get_and_verify_item_publish_state('sequential', 'vertical_sequential', PublishState.public)
# verify that we have the private vertical
test_private_vertical = draft_store.get_item(
course_id.make_usage_key('vertical', 'a_private_vertical')
)
get_and_verify_item_publish_state('vertical', 'a_private_vertical', PublishState.private)
self.assertTrue(getattr(test_private_vertical, 'is_draft', False))
# verify that we have the public vertical
get_and_verify_item_publish_state('vertical', 'a_published_vertical', PublishState.public)
# make sure the textbook survived the export/import
course = module_store.get_course(course_id)
@@ -990,11 +993,10 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
self.assertEqual(value, new_attrs[key])
def test_export_course_with_metadata_only_video(self):
module_store = modulestore('direct')
draft_store = modulestore('draft')
module_store = modulestore()
content_store = contentstore()
import_from_xml(module_store, 'common/test/data/', ['toy'])
import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'])
course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
# create a new video module and add it as a child to a vertical
@@ -1013,7 +1015,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
print 'Exporting to tempdir = {0}'.format(root_dir)
# export out to a tempdir
export_to_xml(module_store, content_store, course_id, root_dir, 'test_export', draft_modulestore=draft_store)
export_to_xml(module_store, content_store, course_id, root_dir, 'test_export')
shutil.rmtree(root_dir)
@@ -1021,11 +1023,10 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
"""
Similar to `test_export_course_with_metadata_only_video`.
"""
module_store = modulestore('direct')
draft_store = modulestore('draft')
module_store = modulestore()
content_store = contentstore()
import_from_xml(module_store, 'common/test/data/', ['word_cloud'])
import_from_xml(module_store, self.user.id, 'common/test/data/', ['word_cloud'])
course_id = SlashSeparatedCourseKey('HarvardX', 'ER22x', '2013_Spring')
verticals = module_store.get_items(course_id, category='vertical')
@@ -1041,7 +1042,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
print 'Exporting to tempdir = {0}'.format(root_dir)
# export out to a tempdir
export_to_xml(module_store, content_store, course_id, root_dir, 'test_export', draft_modulestore=draft_store)
export_to_xml(module_store, content_store, course_id, root_dir, 'test_export')
shutil.rmtree(root_dir)
@@ -1050,11 +1051,10 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
Test that an empty `data` field is preserved through
export/import.
"""
module_store = modulestore('direct')
draft_store = modulestore('draft')
module_store = modulestore()
content_store = contentstore()
import_from_xml(module_store, 'common/test/data/', ['toy'])
import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'])
course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
verticals = module_store.get_items(course_id, category='vertical')
@@ -1070,10 +1070,10 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
# Export the course
root_dir = path(mkdtemp_clean())
export_to_xml(module_store, content_store, course_id, root_dir, 'test_roundtrip', draft_modulestore=draft_store)
export_to_xml(module_store, content_store, course_id, root_dir, 'test_roundtrip')
# Reimport and get the video back
import_from_xml(module_store, root_dir)
import_from_xml(module_store, self.user.id, root_dir)
imported_word_cloud = module_store.get_item(course_id.make_usage_key('word_cloud', 'untitled'))
# It should now contain empty data
@@ -1083,10 +1083,10 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
"""
Test that a course which has HTML that has style formatting is preserved in export/import
"""
module_store = modulestore('direct')
module_store = modulestore()
content_store = contentstore()
import_from_xml(module_store, 'common/test/data/', ['toy'])
import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'])
course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
@@ -1095,7 +1095,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
export_to_xml(module_store, content_store, course_id, root_dir, 'test_roundtrip')
# Reimport and get the video back
import_from_xml(module_store, root_dir)
import_from_xml(module_store, self.user.id, root_dir)
# get the sample HTML with styling information
html_module = module_store.get_item(course_id.make_usage_key('html', 'with_styling'))
@@ -1106,10 +1106,10 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
self.assertIn('<img src="/static/foo_bar.jpg" />', html_module.data)
def test_course_handouts_rewrites(self):
module_store = modulestore('direct')
module_store = modulestore()
# import a test course
_, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'])
_, course_items = import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'])
course_id = course_items[0].id
handouts_location = course_id.make_usage_key('course_info', 'handouts')
@@ -1124,35 +1124,43 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
self.assertContains(resp, '/c4x/edX/toy/asset/handouts_sample_handout.txt')
def test_prefetch_children(self):
module_store = modulestore('direct')
import_from_xml(module_store, 'common/test/data/', ['toy'])
mongo_store = modulestore()._get_modulestore_by_type(MONGO_MODULESTORE_TYPE)
import_from_xml(modulestore(), self.user.id, 'common/test/data/', ['toy'])
course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
wrapper = MongoCollectionFindWrapper(module_store.collection.find)
module_store.collection.find = wrapper.find
print module_store.metadata_inheritance_cache_subsystem
print module_store.request_cache
course = module_store.get_course(course_id, depth=2)
wrapper = MongoCollectionFindWrapper(mongo_store.collection.find)
mongo_store.collection.find = wrapper.find
# make sure we haven't done too many round trips to DB
# note we say 3 round trips here for 1) the course, and 2 & 3) for the chapters and sequentials
# Because we're querying from the top of the tree, we cache information needed for inheritance,
# so we don't need to make an extra query to compute it.
self.assertEqual(wrapper.counter, 3)
# set the branch to 'publish' in order to prevent extra lookups of draft versions
with store_branch_setting(mongo_store, BRANCH_PUBLISHED_ONLY):
course = mongo_store.get_course(course_id, depth=2)
# make sure we pre-fetched a known sequential which should be at depth=2
self.assertTrue(course_id.make_usage_key('sequential', 'vertical_sequential') in course.system.module_data)
# make sure we haven't done too many round trips to DB
# note we say 3 round trips here for 1) the course, and 2 & 3) for the chapters and sequentials
# Because we're querying from the top of the tree, we cache information needed for inheritance,
# so we don't need to make an extra query to compute it.
self.assertEqual(wrapper.counter, 3)
# make sure we pre-fetched a known sequential which should be at depth=2
self.assertTrue(course_id.make_usage_key('sequential', 'vertical_sequential') in course.system.module_data)
# make sure we don't have a specific vertical which should be at depth=3
self.assertFalse(course_id.make_usage_key('vertical', 'vertical_test') in course.system.module_data)
# Now, test with the branch set to draft. We should have one extra round trip call to check for
# the existence of the draft versions
wrapper.counter = 0
mongo_store.get_course(course_id, depth=2)
self.assertEqual(wrapper.counter, 4)
# make sure we don't have a specific vertical which should be at depth=3
self.assertFalse(course_id.make_usage_key('vertical', 'vertical_test') in course.system.module_data)
def test_export_course_without_content_store(self):
module_store = modulestore('direct')
module_store = modulestore()
content_store = contentstore()
# Create toy course
_, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'])
_, course_items = import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'])
course_id = course_items[0].id
root_dir = path(mkdtemp_clean())
@@ -1165,8 +1173,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
delete_course(module_store, content_store, course_id, commit=True)
import_from_xml(
module_store, root_dir, ['test_export_no_content_store'],
draft_store=None,
module_store, self.user.id, root_dir, ['test_export_no_content_store'],
static_content_store=None,
target_course_id=course_id
)
@@ -1190,8 +1197,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
_test_no_locations(self, resp)
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
class ContentStoreTest(ModuleStoreTestCase):
class ContentStoreTest(ContentStoreTestCase):
"""
Tests for the CMS ContentStore application.
"""
@@ -1536,7 +1542,7 @@ class ContentStoreTest(ModuleStoreTestCase):
self.assertEqual(resp.status_code, 200)
payload = parse_json(resp)
problem_loc = UsageKey.from_string(payload['locator'])
problem = get_modulestore(problem_loc).get_item(problem_loc)
problem = modulestore().get_item(problem_loc)
# should be a CapaDescriptor
self.assertIsInstance(problem, CapaDescriptor, "New problem is not a CapaDescriptor")
context = problem.get_context()
@@ -1557,7 +1563,7 @@ class ContentStoreTest(ModuleStoreTestCase):
self.assertEqual(resp.status_code, 200)
_test_no_locations(self, resp)
_, course_items = import_from_xml(modulestore('direct'), 'common/test/data/', ['simple'])
_, course_items = import_from_xml(modulestore(), self.user.id, 'common/test/data/', ['simple'])
course_key = course_items[0].id
resp = self._show_course_overview(course_key)
@@ -1609,11 +1615,11 @@ class ContentStoreTest(ModuleStoreTestCase):
delete_item(category='chapter', name='chapter_2')
def test_import_into_new_course_id(self):
module_store = modulestore('direct')
module_store = modulestore()
target_course_id = _get_course_id(self.course_data)
_create_course(self, target_course_id, self.course_data)
import_from_xml(module_store, 'common/test/data/', ['toy'], target_course_id=target_course_id)
import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'], target_course_id=target_course_id)
modules = module_store.get_items(target_course_id)
@@ -1634,7 +1640,7 @@ class ContentStoreTest(ModuleStoreTestCase):
self.assertEqual(course_module.pdf_textbooks[0]["chapters"][1]["url"], '/static/Chapter2.pdf')
def test_import_into_new_course_id_wiki_slug_renamespacing(self):
module_store = modulestore('direct')
module_store = modulestore()
# If reimporting into the same course do not change the wiki_slug.
target_course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
@@ -1650,7 +1656,7 @@ class ContentStoreTest(ModuleStoreTestCase):
course_module.save()
# Import a course with wiki_slug == location.course
import_from_xml(module_store, 'common/test/data/', ['toy'], target_course_id=target_course_id)
import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'], target_course_id=target_course_id)
course_module = module_store.get_course(target_course_id)
self.assertEquals(course_module.wiki_slug, 'toy')
@@ -1665,18 +1671,18 @@ class ContentStoreTest(ModuleStoreTestCase):
_create_course(self, target_course_id, course_data)
# Import a course with wiki_slug == location.course
import_from_xml(module_store, 'common/test/data/', ['toy'], target_course_id=target_course_id)
import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'], target_course_id=target_course_id)
course_module = module_store.get_course(target_course_id)
self.assertEquals(course_module.wiki_slug, 'MITx.999.2013_Spring')
# Now try importing a course with wiki_slug == '{0}.{1}.{2}'.format(location.org, location.course, location.run)
import_from_xml(module_store, 'common/test/data/', ['two_toys'], target_course_id=target_course_id)
import_from_xml(module_store, self.user.id, 'common/test/data/', ['two_toys'], target_course_id=target_course_id)
course_module = module_store.get_course(target_course_id)
self.assertEquals(course_module.wiki_slug, 'MITx.999.2013_Spring')
def test_import_metadata_with_attempts_empty_string(self):
module_store = modulestore('direct')
import_from_xml(module_store, 'common/test/data/', ['simple'])
module_store = modulestore()
import_from_xml(module_store, self.user.id, 'common/test/data/', ['simple'])
did_load_item = False
try:
course_key = SlashSeparatedCourseKey('edX', 'simple', 'problem')
@@ -1690,20 +1696,20 @@ class ContentStoreTest(ModuleStoreTestCase):
self.assertTrue(did_load_item)
def test_forum_id_generation(self):
module_store = modulestore('direct')
module_store = modulestore()
course = CourseFactory.create(org='edX', course='999', display_name='Robot Super Course')
new_component_location = course.id.make_usage_key('discussion', 'new_component')
# crate a new module and add it as a child to a vertical
module_store.create_and_save_xmodule(new_component_location)
module_store.create_and_save_xmodule(new_component_location, self.user.id)
new_discussion_item = module_store.get_item(new_component_location)
self.assertNotEquals(new_discussion_item.discussion_id, '$$GUID$$')
def test_metadata_inheritance(self):
module_store = modulestore('direct')
_, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'])
module_store = modulestore()
_, course_items = import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'])
course = course_items[0]
verticals = module_store.get_items(course.id, category='vertical')
@@ -1718,13 +1724,13 @@ class ContentStoreTest(ModuleStoreTestCase):
new_component_location = course.id.make_usage_key('html', 'new_component')
# crate a new module and add it as a child to a vertical
module_store.create_and_save_xmodule(new_component_location)
new_object = module_store.create_xmodule(new_component_location)
module_store.update_item(new_object, self.user.id, allow_not_found=True)
parent = verticals[0]
parent.children.append(new_component_location)
module_store.update_item(parent, self.user.id)
# flush the cache
module_store.refresh_cached_metadata_inheritance_tree(new_component_location.course_key)
new_module = module_store.get_item(new_component_location)
# check for grace period definition which should be defined at the course level
@@ -1741,7 +1747,6 @@ class ContentStoreTest(ModuleStoreTestCase):
module_store.update_item(new_module, self.user.id)
# flush the cache and refetch
module_store.refresh_cached_metadata_inheritance_tree(new_component_location.course_key)
new_module = module_store.get_item(new_component_location)
self.assertEqual(timedelta(1), new_module.graceperiod)
@@ -1759,7 +1764,7 @@ class ContentStoreTest(ModuleStoreTestCase):
self.assertGreaterEqual(len(course.checklists), 4)
# by fetching
module_store = modulestore('direct')
module_store = modulestore()
fetched_course = module_store.get_item(course.location)
fetched_item = module_store.get_item(vertical.location)
self.assertIsNotNone(fetched_course.start)
@@ -1771,13 +1776,14 @@ class ContentStoreTest(ModuleStoreTestCase):
def test_image_import(self):
"""Test backwards compatibilty of course image."""
module_store = modulestore('direct')
module_store = modulestore()
content_store = contentstore()
# Use conditional_and_poll, as it's got an image already
import_from_xml(
module_store,
self.user.id,
'common/test/data/',
['conditional_and_poll'],
static_content_store=content_store
@@ -1805,15 +1811,16 @@ class ContentStoreTest(ModuleStoreTestCase):
course_key = _get_course_id(self.course_data)
_create_course(self, course_key, self.course_data)
course_module = modulestore('direct').get_course(course_key)
course_module = modulestore().get_course(course_key)
self.assertEquals(course_module.wiki_slug, 'MITx.999.2013_Spring')
@override_settings(MODULESTORE=TEST_MODULESTORE)
class MetadataSaveTestCase(ModuleStoreTestCase):
class MetadataSaveTestCase(ContentStoreTestCase):
"""Test that metadata is correctly cached and decached."""
def setUp(self):
super(MetadataSaveTestCase, self).setUp()
course = CourseFactory.create(
org='edX', course='999', display_name='Robot Super Course')
@@ -1857,8 +1864,8 @@ class MetadataSaveTestCase(ModuleStoreTestCase):
delattr(self.video_descriptor, field_name)
self.assertNotIn('html5_sources', own_metadata(self.video_descriptor))
get_modulestore(location).update_item(self.video_descriptor, '**replace_user**')
module = get_modulestore(location).get_item(location)
modulestore().update_item(self.video_descriptor, self.user.id)
module = modulestore().get_item(location)
self.assertNotIn('html5_sources', own_metadata(module))

19
cms/djangoapps/contentstore/tests/test_course_listing.py
View File

@@ -16,7 +16,8 @@ from contentstore.tests.utils import AjaxEnabledTestClient
from student.tests.factories import UserFactory
from student.roles import CourseInstructorRole, CourseStaffRole, GlobalStaff, OrgStaffRole, OrgInstructorRole
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.tests.factories import CourseFactory, check_mongo_calls
from xmodule.modulestore import MONGO_MODULESTORE_TYPE
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from xmodule.modulestore.django import modulestore
from xmodule.error_module import ErrorDescriptor
@@ -197,6 +198,14 @@ class TestCourseListing(ModuleStoreTestCase):
self.assertGreaterEqual(iteration_over_courses_time_1.elapsed, iteration_over_groups_time_1.elapsed)
self.assertGreaterEqual(iteration_over_courses_time_2.elapsed, iteration_over_groups_time_2.elapsed)
# Now count the db queries
store = modulestore()._get_modulestore_by_type(MONGO_MODULESTORE_TYPE)
with check_mongo_calls(store.collection, USER_COURSES_COUNT):
courses_list = _accessible_courses_list_from_groups(self.request)
with check_mongo_calls(store.collection, 1):
courses_list = _accessible_courses_list(self.request)
def test_get_course_list_with_same_course_id(self):
"""
Test getting courses with same id but with different name case. Then try to delete one of them and
@@ -253,18 +262,20 @@ class TestCourseListing(ModuleStoreTestCase):
Create good courses, courses that won't load, and deleted courses which still have
roles. Test course listing.
"""
store = modulestore()._get_modulestore_by_type(MONGO_MODULESTORE_TYPE)
course_location = SlashSeparatedCourseKey('testOrg', 'testCourse', 'RunBabyRun')
self._create_course_with_access_groups(course_location, self.user)
course_location = SlashSeparatedCourseKey('testOrg', 'doomedCourse', 'RunBabyRun')
self._create_course_with_access_groups(course_location, self.user)
modulestore().delete_course(course_location)
store.delete_course(course_location)
course_location = SlashSeparatedCourseKey('testOrg', 'erroredCourse', 'RunBabyRun')
course = self._create_course_with_access_groups(course_location, self.user)
course_db_record = modulestore()._find_one(course.location)
course_db_record = store._find_one(course.location)
course_db_record.setdefault('metadata', {}).get('tabs', []).append({"type": "wiko", "name": "Wiki" })
modulestore().collection.update(
store.collection.update(
{'_id': course.location.to_deprecated_son()},
{'$set': {
'metadata.tabs': course_db_record['metadata']['tabs'],

13
cms/djangoapps/contentstore/tests/test_course_settings.py
View File

@@ -11,7 +11,7 @@ from django.test.utils import override_settings
from models.settings.course_details import (CourseDetails, CourseSettingsEncoder)
from models.settings.course_grading import CourseGradingModel
from contentstore.utils import get_modulestore, EXTRA_TAB_PANELS, reverse_course_url, reverse_usage_url
from contentstore.utils import EXTRA_TAB_PANELS, reverse_course_url, reverse_usage_url
from xmodule.modulestore.tests.factories import CourseFactory
from models.settings.course_metadata import CourseMetadata
@@ -335,7 +335,7 @@ class CourseGradingTest(CourseTestCase):
def test_update_section_grader_type(self):
# Get the descriptor and the section_grader_type and assert they are the default values
descriptor = get_modulestore(self.course.location).get_item(self.course.location)
descriptor = modulestore().get_item(self.course.location)
section_grader_type = CourseGradingModel.get_section_grader_type(self.course.location)
self.assertEqual('notgraded', section_grader_type['graderType'])
@@ -344,7 +344,7 @@ class CourseGradingTest(CourseTestCase):
# Change the default grader type to Homework, which should also mark the section as graded
CourseGradingModel.update_section_grader_type(self.course, 'Homework', self.user)
descriptor = get_modulestore(self.course.location).get_item(self.course.location)
descriptor = modulestore().get_item(self.course.location)
section_grader_type = CourseGradingModel.get_section_grader_type(self.course.location)
self.assertEqual('Homework', section_grader_type['graderType'])
@@ -353,7 +353,7 @@ class CourseGradingTest(CourseTestCase):
# Change the grader type back to notgraded, which should also unmark the section as graded
CourseGradingModel.update_section_grader_type(self.course, 'notgraded', self.user)
descriptor = get_modulestore(self.course.location).get_item(self.course.location)
descriptor = modulestore().get_item(self.course.location)
section_grader_type = CourseGradingModel.get_section_grader_type(self.course.location)
self.assertEqual('notgraded', section_grader_type['graderType'])
@@ -413,8 +413,7 @@ class CourseGradingTest(CourseTestCase):
Populate the course, grab a section, get the url for the assignment type access
"""
self.populate_course()
sequential_usage_key = self.course.id.make_usage_key("sequential", None)
sections = get_modulestore(self.course.id).get_items(sequential_usage_key)
sections = modulestore().get_items(self.course.id, category="sequential")
# see if test makes sense
self.assertGreater(len(sections), 0, "No sections found")
section = sections[0] # just take the first one
@@ -470,7 +469,7 @@ class CourseMetadataEditingTest(CourseTestCase):
)
self.update_check(test_model)
# try fresh fetch to ensure persistence
fresh = modulestore('direct').get_course(self.course.id)
fresh = modulestore().get_course(self.course.id)
test_model = CourseMetadata.fetch(fresh)
self.update_check(test_model)
# now change some of the existing metadata

81
cms/djangoapps/contentstore/tests/test_crud.py
View File

@@ -1,9 +1,11 @@
import unittest
from xmodule import templates
from xmodule.modulestore import SPLIT_MONGO_MODULESTORE_TYPE, BRANCH_NAME_DRAFT
from xmodule.modulestore.tests import persistent_factories
from xmodule.course_module import CourseDescriptor
from xmodule.modulestore.django import modulestore, loc_mapper, clear_existing_modulestores
from xmodule.modulestore.django import modulestore, clear_existing_modulestores, _MIXED_MODULESTORE, \
loc_mapper, _loc_singleton
from xmodule.seq_module import SequenceDescriptor
from xmodule.capa_module import CapaDescriptor
from opaque_keys.edx.locator import BlockUsageLocator, LocalId
@@ -19,8 +21,9 @@ class TemplateTests(unittest.TestCase):
def setUp(self):
clear_existing_modulestores() # redundant w/ cleanup but someone was getting errors
self.addCleanup(ModuleStoreTestCase.drop_mongo_collections, 'split')
self.addCleanup(ModuleStoreTestCase.drop_mongo_collections, SPLIT_MONGO_MODULESTORE_TYPE)
self.addCleanup(clear_existing_modulestores)
self.split_store = modulestore()._get_modulestore_by_type(SPLIT_MONGO_MODULESTORE_TYPE)
def test_get_templates(self):
found = templates.all_templates()
@@ -59,7 +62,7 @@ class TemplateTests(unittest.TestCase):
)
self.assertIsInstance(test_course, CourseDescriptor)
self.assertEqual(test_course.display_name, 'fun test course')
index_info = modulestore('split').get_course_index_info(test_course.id)
index_info = self.split_store.get_course_index_info(test_course.id)
self.assertEqual(index_info['org'], 'testx')
self.assertEqual(index_info['offering'], 'tempcourse')
@@ -67,7 +70,7 @@ class TemplateTests(unittest.TestCase):
parent_location=test_course.location)
self.assertIsInstance(test_chapter, SequenceDescriptor)
# refetch parent which should now point to child
test_course = modulestore('split').get_course(test_course.id.version_agnostic())
test_course = self.split_store.get_course(test_course.id.version_agnostic())
self.assertIn(test_chapter.location, test_course.children)
with self.assertRaises(DuplicateCourseError):
@@ -85,7 +88,7 @@ class TemplateTests(unittest.TestCase):
display_name='fun test course', user_id='testbot'
)
test_chapter = modulestore('split').create_xblock(
test_chapter = self.split_store.create_xblock(
test_course.system, 'chapter', {'display_name': 'chapter n'}, parent_xblock=test_course
)
self.assertIsInstance(test_chapter, SequenceDescriptor)
@@ -94,7 +97,7 @@ class TemplateTests(unittest.TestCase):
# test w/ a definition (e.g., a problem)
test_def_content = '<problem>boo</problem>'
test_problem = modulestore('split').create_xblock(
test_problem = self.split_store.create_xblock(
test_course.system, 'problem', {'data': test_def_content}, parent_xblock=test_chapter
)
self.assertIsInstance(test_problem, CapaDescriptor)
@@ -111,13 +114,13 @@ class TemplateTests(unittest.TestCase):
offering='tempcourse', org='testx',
display_name='fun test course', user_id='testbot'
)
test_chapter = modulestore('split').create_xblock(
test_chapter = self.split_store.create_xblock(
test_course.system, 'chapter', {'display_name': 'chapter n'}, parent_xblock=test_course
)
self.assertEqual(test_chapter.display_name, 'chapter n')
test_def_content = '<problem>boo</problem>'
# create child
new_block = modulestore('split').create_xblock(
new_block = self.split_store.create_xblock(
test_course.system,
'problem',
fields={
@@ -131,7 +134,7 @@ class TemplateTests(unittest.TestCase):
# better to pass in persisted parent over the subdag so
# subdag gets the parent pointer (otherwise 2 ops, persist dag, update parent children,
# persist parent
persisted_course = modulestore('split').persist_xblock_dag(test_course, 'testbot')
persisted_course = self.split_store.persist_xblock_dag(test_course, 'testbot')
self.assertEqual(len(persisted_course.children), 1)
persisted_chapter = persisted_course.get_children()[0]
self.assertEqual(persisted_chapter.category, 'chapter')
@@ -142,7 +145,7 @@ class TemplateTests(unittest.TestCase):
self.assertEqual(persisted_problem.data, test_def_content)
# update it
persisted_problem.display_name = 'altered problem'
persisted_problem = modulestore('split').persist_xblock_dag(persisted_problem, 'testbot')
persisted_problem = self.split_store.persist_xblock_dag(persisted_problem, 'testbot')
self.assertEqual(persisted_problem.display_name, 'altered problem')
def test_delete_course(self):
@@ -153,17 +156,17 @@ class TemplateTests(unittest.TestCase):
persistent_factories.ItemFactory.create(display_name='chapter 1',
parent_location=test_course.location)
id_locator = test_course.id.for_branch('draft')
id_locator = test_course.id.for_branch(BRANCH_NAME_DRAFT)
guid_locator = test_course.location.course_agnostic()
# verify it can be retrieved by id
self.assertIsInstance(modulestore('split').get_course(id_locator), CourseDescriptor)
self.assertIsInstance(self.split_store.get_course(id_locator), CourseDescriptor)
# and by guid
self.assertIsInstance(modulestore('split').get_item(guid_locator), CourseDescriptor)
modulestore('split').delete_course(id_locator)
self.assertIsInstance(self.split_store.get_item(guid_locator), CourseDescriptor)
self.split_store.delete_course(id_locator)
# test can no longer retrieve by id
self.assertRaises(ItemNotFoundError, modulestore('split').get_course, id_locator)
self.assertRaises(ItemNotFoundError, self.split_store.get_course, id_locator)
# but can by guid
self.assertIsInstance(modulestore('split').get_item(guid_locator), CourseDescriptor)
self.assertIsInstance(self.split_store.get_item(guid_locator), CourseDescriptor)
def test_block_generations(self):
"""
@@ -184,11 +187,11 @@ class TemplateTests(unittest.TestCase):
)
first_problem.max_attempts = 3
first_problem.save() # decache the above into the kvs
updated_problem = modulestore('split').update_item(first_problem, '**replace_user**')
updated_problem = self.split_store.update_item(first_problem, '**replace_user**')
self.assertIsNotNone(updated_problem.previous_version)
self.assertEqual(updated_problem.previous_version, first_problem.update_version)
self.assertNotEqual(updated_problem.update_version, first_problem.update_version)
updated_loc = modulestore('split').delete_item(updated_problem.location, 'testbot', delete_children=True)
updated_loc = self.split_store.delete_item(updated_problem.location, '**replace_user**', 'testbot')
second_problem = persistent_factories.ItemFactory.create(
display_name='problem 2',
@@ -200,45 +203,59 @@ class TemplateTests(unittest.TestCase):
)
# course root only updated 2x
version_history = modulestore('split').get_block_generations(test_course.location)
version_history = self.split_store.get_block_generations(test_course.location)
self.assertEqual(version_history.locator.version_guid, test_course.location.version_guid)
self.assertEqual(len(version_history.children), 1)
self.assertEqual(version_history.children[0].children, [])
self.assertEqual(version_history.children[0].locator.version_guid, chapter.location.version_guid)
# sub changed on add, add problem, delete problem, add problem in strict linear seq
version_history = modulestore('split').get_block_generations(sub.location)
version_history = self.split_store.get_block_generations(sub.location)
self.assertEqual(len(version_history.children), 1)
self.assertEqual(len(version_history.children[0].children), 1)
self.assertEqual(len(version_history.children[0].children[0].children), 1)
self.assertEqual(len(version_history.children[0].children[0].children[0].children), 0)
# first and second problem may show as same usage_id; so, need to ensure their histories are right
version_history = modulestore('split').get_block_generations(updated_problem.location)
version_history = self.split_store.get_block_generations(updated_problem.location)
self.assertEqual(version_history.locator.version_guid, first_problem.location.version_guid)
self.assertEqual(len(version_history.children), 1) # updated max_attempts
self.assertEqual(len(version_history.children[0].children), 0)
version_history = modulestore('split').get_block_generations(second_problem.location)
version_history = self.split_store.get_block_generations(second_problem.location)
self.assertNotEqual(version_history.locator.version_guid, first_problem.location.version_guid)
class SplitAndLocMapperTests(unittest.TestCase):
"""
Test injection of loc_mapper into Split
"""
def test_split_inject_loc_mapper(self):
"""
Test that creating a loc_mapper causes it to automatically attach to the split mongo store
Test loc_mapper created before split
"""
# ensure modulestore is not instantiated
self.assertIsNone(_MIXED_MODULESTORE)
# instantiate location mapper before split
mapper = loc_mapper()
# split must inject the location mapper itself since the mapper existed before it did
self.assertEqual(modulestore('split').loc_mapper, mapper)
# instantiate mixed modulestore and thus split
split_store = modulestore()._get_modulestore_by_type(SPLIT_MONGO_MODULESTORE_TYPE)
# split must inject the same location mapper object since the mapper existed before it did
self.assertEqual(split_store.loc_mapper, mapper)
def test_loc_inject_into_split(self):
"""
Test that creating a loc_mapper causes it to automatically attach to the split mongo store
Test split created before loc_mapper
"""
# force instantiation of split modulestore before there's a location mapper and verify
# it has no pointer to loc mapper
self.assertIsNone(modulestore('split').loc_mapper)
# force instantiation of location mapper which must inject itself into the split
mapper = loc_mapper()
self.assertEqual(modulestore('split').loc_mapper, mapper)
# ensure loc_mapper is not instantiated
self.assertIsNone(_loc_singleton)
# instantiate split before location mapper
split_store = modulestore()._get_modulestore_by_type(SPLIT_MONGO_MODULESTORE_TYPE)
# split must have instantiated loc_mapper
mapper = loc_mapper()
self.assertEqual(split_store.loc_mapper, mapper)

8
cms/djangoapps/contentstore/tests/test_export_git.py
View File

@@ -16,7 +16,7 @@ from .utils import CourseTestCase
import contentstore.git_export_utils as git_export_utils
from xmodule.contentstore.django import _CONTENTSTORE
from xmodule.modulestore.django import modulestore
from contentstore.utils import get_modulestore, reverse_course_url
from contentstore.utils import reverse_course_url
TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex
@@ -66,7 +66,7 @@ class TestExportGit(CourseTestCase):
Test failed course export response.
"""
self.course_module.giturl = 'foobar'
get_modulestore(self.course_module.location).update_item(self.course_module)
modulestore().update_item(self.course_module, '**replace_user**')
response = self.client.get('{}?action=push'.format(self.test_url))
self.assertIn('Export Failed:', response.content)
@@ -76,7 +76,7 @@ class TestExportGit(CourseTestCase):
Regression test for making sure errors are properly stringified
"""
self.course_module.giturl = 'foobar'
get_modulestore(self.course_module.location).update_item(self.course_module)
modulestore().update_item(self.course_module, '**replace_user**')
response = self.client.get('{}?action=push'.format(self.test_url))
self.assertNotIn('django.utils.functional.__proxy__', response.content)
@@ -99,7 +99,7 @@ class TestExportGit(CourseTestCase):
self.populate_course()
self.course_module.giturl = 'file://{}'.format(bare_repo_dir)
get_modulestore(self.course_module.location).update_item(self.course_module)
modulestore().update_item(self.course_module, '**replace_user**')
response = self.client.get('{}?action=push'.format(self.test_url))
self.assertIn('Export Succeeded', response.content)

3
cms/djangoapps/contentstore/tests/test_i18n.py
View File

@@ -1,14 +1,11 @@
from unittest import skip
from django.contrib.auth.models import User
from django.test.utils import override_settings
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from contentstore.tests.utils import AjaxEnabledTestClient
@override_settings(MODULESTORE=TEST_MODULESTORE)
class InternationalizationTest(ModuleStoreTestCase):
"""
Tests to validate Internationalization.

30
cms/djangoapps/contentstore/tests/test_import.py
View File

@@ -13,13 +13,11 @@ import copy
from django.contrib.auth.models import User
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore
from opaque_keys.edx.locations import SlashSeparatedCourseKey, AssetLocation
from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import _CONTENTSTORE
from xmodule.exceptions import NotFoundError
@@ -30,7 +28,7 @@ TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class ContentStoreImportTest(ModuleStoreTestCase):
"""
Tests that rely on the toy and test_import_course courses.
@@ -38,8 +36,6 @@ class ContentStoreImportTest(ModuleStoreTestCase):
"""
def setUp(self):
settings.MODULESTORE['default']['OPTIONS']['fs_root'] = path('common/test/data')
settings.MODULESTORE['direct']['OPTIONS']['fs_root'] = path('common/test/data')
uname = 'testuser'
email = 'test+courses@edx.org'
password = 'foo'
@@ -69,9 +65,10 @@ class ContentStoreImportTest(ModuleStoreTestCase):
(for do_import_static=False behavior).
'''
content_store = contentstore()
module_store = modulestore('direct')
module_store = modulestore()
import_from_xml(
module_store,
'**replace_user**',
'common/test/data/',
['test_import_course'],
static_content_store=content_store,
@@ -91,6 +88,7 @@ class ContentStoreImportTest(ModuleStoreTestCase):
module_store, __, course = self.load_test_import_course()
__, course_items = import_from_xml(
module_store,
'**replace_user**',
'common/test/data',
['test_import_course_2'],
target_course_id=course.id,
@@ -102,10 +100,11 @@ class ContentStoreImportTest(ModuleStoreTestCase):
"""
# Test that importing course with unicode 'id' and 'display name' doesn't give UnicodeEncodeError
"""
module_store = modulestore('direct')
module_store = modulestore()
course_id = SlashSeparatedCourseKey(u'Юникода', u'unicode_course', u'échantillon')
import_from_xml(
module_store,
'**replace_user**',
'common/test/data/',
['2014_Uni'],
target_course_id=course_id
@@ -150,8 +149,8 @@ class ContentStoreImportTest(ModuleStoreTestCase):
'''
content_store = contentstore()
module_store = modulestore('direct')
import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store, do_import_static=False, verbose=True)
module_store = modulestore()
import_from_xml(module_store, '**replace_user**', 'common/test/data/', ['toy'], static_content_store=content_store, do_import_static=False, verbose=True)
course = module_store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
@@ -161,8 +160,8 @@ class ContentStoreImportTest(ModuleStoreTestCase):
self.assertEqual(count, 0)
def test_no_static_link_rewrites_on_import(self):
module_store = modulestore('direct')
_, courses = import_from_xml(module_store, 'common/test/data/', ['toy'], do_import_static=False, verbose=True)
module_store = modulestore()
_, courses = import_from_xml(module_store, '**replace_user**', 'common/test/data/', ['toy'], do_import_static=False, verbose=True)
course_key = courses[0].id
handouts = module_store.get_item(course_key.make_usage_key('course_info', 'handouts'))
@@ -177,10 +176,11 @@ class ContentStoreImportTest(ModuleStoreTestCase):
self.assertEqual(course.tabs[2]['name'], 'Syllabus')
def test_rewrite_reference_list(self):
module_store = modulestore('direct')
module_store = modulestore()
target_course_id = SlashSeparatedCourseKey('testX', 'conditional_copy', 'copy_run')
import_from_xml(
module_store,
'**replace_user**',
'common/test/data/',
['conditional'],
target_course_id=target_course_id
@@ -206,10 +206,11 @@ class ContentStoreImportTest(ModuleStoreTestCase):
)
def test_rewrite_reference(self):
module_store = modulestore('direct')
module_store = modulestore()
target_course_id = SlashSeparatedCourseKey('testX', 'peergrading_copy', 'copy_run')
import_from_xml(
module_store,
'**replace_user**',
'common/test/data/',
['open_ended'],
target_course_id=target_course_id
@@ -224,10 +225,11 @@ class ContentStoreImportTest(ModuleStoreTestCase):
)
def test_rewrite_reference_value_dict(self):
module_store = modulestore('direct')
module_store = modulestore()
target_course_id = SlashSeparatedCourseKey('testX', 'split_test_copy', 'copy_run')
import_from_xml(
module_store,
'**replace_user**',
'common/test/data/',
['split_test_module'],
target_course_id=target_course_id

14
cms/djangoapps/contentstore/tests/test_import_draft_order.py
View File

@@ -1,24 +1,18 @@
from django.test.utils import override_settings
from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.django import modulestore
from contentstore.tests.modulestore_config import TEST_MODULESTORE
# This test is in the CMS module because the test configuration to use a draft
# modulestore is dependent on django.
@override_settings(MODULESTORE=TEST_MODULESTORE)
class DraftReorderTestCase(ModuleStoreTestCase):
def test_order(self):
store = modulestore('direct')
draft_store = modulestore('default')
_, course_items = import_from_xml(store, 'common/test/data/', ['import_draft_order'], draft_store=draft_store)
store = modulestore()
_, course_items = import_from_xml(store, '**replace_user**', 'common/test/data/', ['import_draft_order'])
course_key = course_items[0].id
sequential = draft_store.get_item(course_key.make_usage_key('sequential', '0f4f7649b10141b0bdc9922dcf94515a'))
sequential = store.get_item(course_key.make_usage_key('sequential', '0f4f7649b10141b0bdc9922dcf94515a'))
verticals = sequential.children
# The order that files are read in from the file system is not guaranteed (cannot rely on
@@ -39,7 +33,7 @@ class DraftReorderTestCase(ModuleStoreTestCase):
self.assertEqual(course_key.make_usage_key('vertical', 'c'), verticals[6])
# Now also test that the verticals in a second sequential are correct.
sequential = draft_store.get_item(course_key.make_usage_key('sequential', 'secondseq'))
sequential = store.get_item(course_key.make_usage_key('sequential', 'secondseq'))
verticals = sequential.children
# 'asecond' and 'zsecond' are drafts with 'index_in_children_list' 0 and 2, respectively.
# 'secondsubsection' is a public vertical.

13
cms/djangoapps/contentstore/tests/test_import_pure_xblock.py
View File

@@ -2,8 +2,6 @@
Integration tests for importing courses containing pure XBlocks.
"""
from django.test.utils import override_settings
from xblock.core import XBlock
from xblock.fields import String
@@ -11,7 +9,6 @@ from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.mongo.draft import as_draft
from contentstore.tests.modulestore_config import TEST_MODULESTORE
class StubXBlock(XBlock):
@@ -29,12 +26,10 @@ class StubXBlock(XBlock):
test_field = String(default="default")
@override_settings(MODULESTORE=TEST_MODULESTORE)
class XBlockImportTest(ModuleStoreTestCase):
def setUp(self):
self.store = modulestore('direct')
self.draft_store = modulestore('default')
self.store = modulestore()
@XBlock.register_temp_plugin(StubXBlock)
def test_import_public(self):
@@ -67,8 +62,7 @@ class XBlockImportTest(ModuleStoreTestCase):
"""
_, courses = import_from_xml(
self.store, 'common/test/data', [course_dir],
draft_store=self.draft_store
self.store, '**replace_user**', 'common/test/data', [course_dir]
)
xblock_location = courses[0].id.make_usage_key('stubxblock', 'xblock_test')
@@ -81,6 +75,7 @@ class XBlockImportTest(ModuleStoreTestCase):
self.assertEqual(xblock.test_field, expected_field_val)
if has_draft:
draft_xblock = self.draft_store.get_item(xblock_location)
draft_xblock = self.store.get_item(xblock_location)
self.assertTrue(getattr(draft_xblock, 'is_draft', False))
self.assertTrue(isinstance(draft_xblock, StubXBlock))
self.assertEqual(draft_xblock.test_field, expected_field_val)

6
cms/djangoapps/contentstore/tests/test_orphan.py
View File

@@ -32,8 +32,10 @@ class TestOrphan(CourseTestCase):
def _create_item(self, category, name, data, metadata, parent_category, parent_name, runtime):
location = self.course.location.replace(category=category, name=name)
store = modulestore('direct')
store.create_and_save_xmodule(location, data, metadata, runtime)
store = modulestore()
store.create_and_save_xmodule(
location, self.user.id, definition_data=data, metadata=metadata, runtime=runtime
)
if parent_name:
# add child to parent in mongo
parent_location = self.course.location.replace(category=parent_category, name=parent_name)

3
cms/djangoapps/contentstore/tests/test_permissions.py
View File

@@ -3,11 +3,9 @@ Test CRUD for authorization.
"""
import copy
from django.test.utils import override_settings
from django.contrib.auth.models import User
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from contentstore.tests.utils import AjaxEnabledTestClient
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from contentstore.utils import reverse_url, reverse_course_url
@@ -16,7 +14,6 @@ from contentstore.views.access import has_course_access
from student import auth
@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestCourseAccess(ModuleStoreTestCase):
"""
Course-based access (as opposed to access of a non-course xblock)

5
cms/djangoapps/contentstore/tests/test_transcripts_utils.py
View File

@@ -21,7 +21,6 @@ from xmodule.exceptions import NotFoundError
from xmodule.contentstore.django import contentstore, _CONTENTSTORE
from xmodule.video_module import transcripts_utils
from contentstore.tests.modulestore_config import TEST_MODULESTORE
TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex
@@ -76,7 +75,7 @@ class TestGenerateSubs(unittest.TestCase):
)
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class TestSaveSubsToStore(ModuleStoreTestCase):
"""Tests for `save_subs_to_store` function."""
@@ -156,7 +155,7 @@ class TestSaveSubsToStore(ModuleStoreTestCase):
_CONTENTSTORE.clear()
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class TestDownloadYoutubeSubs(ModuleStoreTestCase):
"""Tests for `download_youtube_subs` function."""

2
cms/djangoapps/contentstore/tests/tests.py
View File

@@ -15,14 +15,12 @@ from contentstore.tests.utils import parse_json, user, registration, AjaxEnabled
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from contentstore.tests.test_course_settings import CourseTestCase
from xmodule.modulestore.tests.factories import CourseFactory
from contentstore.tests.modulestore_config import TEST_MODULESTORE
import datetime
from pytz import UTC
from freezegun import freeze_time
@override_settings(MODULESTORE=TEST_MODULESTORE)
class ContentStoreTestCase(ModuleStoreTestCase):
def _login(self, email, password):
"""

8
cms/djangoapps/contentstore/tests/utils.py
View File

@@ -6,12 +6,10 @@ import json
from django.contrib.auth.models import User
from django.test.client import Client
from django.test.utils import override_settings
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from contentstore.utils import get_modulestore
from student.models import Registration
@@ -58,8 +56,6 @@ class AjaxEnabledTestClient(Client):
return self.get(path, data or {}, follow, HTTP_ACCEPT="application/json", **extra)
@override_settings(MODULESTORE=TEST_MODULESTORE)
class CourseTestCase(ModuleStoreTestCase):
def setUp(self):
"""
@@ -91,7 +87,7 @@ class CourseTestCase(ModuleStoreTestCase):
number='999',
display_name='Robot Super Course',
)
self.store = get_modulestore(self.course.location)
self.store = modulestore()
def create_non_staff_authed_user_client(self, authenticate=True):
"""

68
cms/djangoapps/contentstore/utils.py
View File

@@ -11,7 +11,7 @@ from django.core.urlresolvers import reverse
from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import contentstore
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.mixed import store_bulk_write_operations_on_course
from opaque_keys.edx.locations import SlashSeparatedCourseKey, Location
from xmodule.modulestore.store_utilities import delete_course
from student.roles import CourseInstructorRole, CourseStaffRole
@@ -30,31 +30,22 @@ def delete_course_and_groups(course_id, commit=False):
This deletes the courseware associated with a course_id as well as cleaning update_item
the various user table stuff (groups, permissions, etc.)
"""
module_store = modulestore('direct')
module_store = modulestore()
content_store = contentstore()
module_store.ignore_write_events_on_courses.add(course_id)
with store_bulk_write_operations_on_course(module_store, course_id):
if delete_course(module_store, content_store, course_id, commit):
if delete_course(module_store, content_store, course_id, commit):
print 'removing User permissions from course....'
# in the django layer, we need to remove all the user permissions groups associated with this course
if commit:
try:
staff_role = CourseStaffRole(course_id)
staff_role.remove_users(*staff_role.users_with_role())
instructor_role = CourseInstructorRole(course_id)
instructor_role.remove_users(*instructor_role.users_with_role())
except Exception as err:
log.error("Error in deleting course groups for {0}: {1}".format(course_id, err))
def get_modulestore(category_or_location):
"""
This function no longer does anything more than just calling `modulestore()`. It used
to select 'direct' v 'draft' based on the category.
"""
return modulestore()
print 'removing User permissions from course....'
# in the django layer, we need to remove all the user permissions groups associated with this course
if commit:
try:
staff_role = CourseStaffRole(course_id)
staff_role.remove_users(*staff_role.users_with_role())
instructor_role = CourseInstructorRole(course_id)
instructor_role.remove_users(*instructor_role.users_with_role())
except Exception as err:
log.error("Error in deleting course groups for {0}: {1}".format(course_id, err))
def get_lms_link_for_item(location, preview=False):
@@ -124,35 +115,18 @@ def course_image_url(course):
return path
class PublishState(object):
"""
The publish state for a given xblock-- either 'draft', 'private', or 'public'.
Currently in CMS, an xblock can only be in 'draft' or 'private' if it is at or below the Unit level.
"""
draft = 'draft'
private = 'private'
public = 'public'
def compute_publish_state(xblock):
"""
Returns whether this xblock is 'draft', 'public', or 'private'.
Returns whether this xblock is draft, public, or private.
'draft' content is in the process of being edited, but still has a previous
version visible in the LMS
'public' content is locked and visible in the LMS
'private' content is editable and not visible in the LMS
Returns:
PublishState.draft - content is in the process of being edited, but still has a previous
version deployed to LMS
PublishState.public - content is locked and deployed to LMS
PublishState.private - content is editable and not deployed to LMS
"""
if getattr(xblock, 'is_draft', False):
try:
modulestore('direct').get_item(xblock.location)
return PublishState.draft
except ItemNotFoundError:
return PublishState.private
else:
return PublishState.public
return modulestore().compute_publish_state(xblock)
def add_extra_panel_tab(tab_type, course):

6
cms/djangoapps/contentstore/views/checklist.py
View File

@@ -11,7 +11,7 @@ from django.http import HttpResponseNotFound
from django.core.exceptions import PermissionDenied
from opaque_keys.edx.keys import CourseKey
from xmodule.modulestore.django import modulestore
from contentstore.utils import get_modulestore, reverse_course_url
from contentstore.utils import reverse_course_url
from .access import has_course_access
from xmodule.course_module import CourseDescriptor
@@ -47,7 +47,7 @@ def checklists_handler(request, course_key_string, checklist_index=None):
# from the template.
if not course_module.checklists:
course_module.checklists = CourseDescriptor.checklists.default
get_modulestore(course_module.location).update_item(course_module, request.user.id)
modulestore().update_item(course_module, request.user.id)
expanded_checklists = expand_all_action_urls(course_module)
if json_request:
@@ -76,7 +76,7 @@ def checklists_handler(request, course_key_string, checklist_index=None):
# not default
course_module.checklists = course_module.checklists
course_module.save()
get_modulestore(course_module.location).update_item(course_module, request.user.id)
modulestore().update_item(course_module, request.user.id)
expanded_checklist = expand_checklist_action_url(course_module, persisted_checklist)
return JsonResponse(localize_checklist_text(expanded_checklist))
else:

9
cms/djangoapps/contentstore/views/component.py
View File

@@ -13,6 +13,7 @@ from edxmako.shortcuts import render_to_response
from util.date_utils import get_default_time_display
from xmodule.modulestore.django import modulestore
from xmodule.modulestore import PublishState
from xblock.core import XBlock
from xblock.django.request import webob_to_django_response, django_to_webob_request
@@ -21,7 +22,7 @@ from xblock.fields import Scope
from xblock.plugin import PluginMissingError
from xblock.runtime import Mixologist
from contentstore.utils import get_lms_link_for_item, compute_publish_state, PublishState, get_modulestore
from contentstore.utils import get_lms_link_for_item, compute_publish_state
from contentstore.views.helpers import get_parent_xblock
from models.settings.course_grading import CourseGradingModel
@@ -413,7 +414,7 @@ def _get_item_in_course(request, usage_key):
raise PermissionDenied()
course = modulestore().get_course(course_key)
item = get_modulestore(usage_key).get_item(usage_key, depth=1)
item = modulestore().get_item(usage_key, depth=1)
lms_link = get_lms_link_for_item(usage_key)
return course, item, lms_link
@@ -436,7 +437,7 @@ def component_handler(request, usage_key_string, handler, suffix=''):
usage_key = UsageKey.from_string(usage_key_string)
descriptor = get_modulestore(usage_key).get_item(usage_key)
descriptor = modulestore().get_item(usage_key)
# Let the module handle the AJAX
req = django_to_webob_request(request)
@@ -449,6 +450,6 @@ def component_handler(request, usage_key_string, handler, suffix=''):
# unintentional update to handle any side effects of handle call; so, request user didn't author
# the change
get_modulestore(usage_key).update_item(descriptor, None)
modulestore().update_item(descriptor, None)
return webob_to_django_response(resp)

28
cms/djangoapps/contentstore/views/course.py
View File

@@ -31,7 +31,6 @@ from contentstore.utils import (
get_lms_link_for_item,
add_extra_panel_tab,
remove_extra_panel_tab,
get_modulestore,
reverse_course_url
)
from models.settings.course_details import CourseDetails, CourseSettingsEncoder
@@ -165,7 +164,7 @@ def _accessible_courses_list(request):
"""
List all courses available to the logged in user by iterating through all the courses
"""
courses = modulestore('direct').get_courses()
courses = modulestore().get_courses()
# filter out courses that we don't have access to
def course_filter(course):
@@ -202,14 +201,15 @@ def _accessible_courses_list_from_groups(request):
if course_key is None:
# If the course_access does not have a course_id, it's an org-based role, so we fall back
raise AccessListFallback
try:
course = modulestore('direct').get_course(course_key)
except ItemNotFoundError:
# If a user has access to a course that doesn't exist, don't do anything with that course
pass
if course is not None and not isinstance(course, ErrorDescriptor):
# ignore deleted or errored courses
courses_list[course_key] = course
if course_key not in courses_list:
try:
course = modulestore().get_course(course_key)
except ItemNotFoundError:
# If a user has access to a course that doesn't exist, don't do anything with that course
pass
if course is not None and not isinstance(course, ErrorDescriptor):
# ignore deleted or errored courses
courses_list[course_key] = course
return courses_list.values()
@@ -333,7 +333,7 @@ def create_new_course(request):
fields.update(metadata)
# Creating the course raises InvalidLocationError if an existing course with this org/name is found
new_course = modulestore('direct').create_course(
new_course = modulestore().create_course(
course_key.org,
course_key.offering,
fields=fields,
@@ -440,7 +440,7 @@ def course_info_update_handler(request, course_key_string, provided_id=None):
raise PermissionDenied()
if request.method == 'GET':
course_updates = get_course_updates(usage_key, provided_id)
course_updates = get_course_updates(usage_key, provided_id, request.user.id)
if isinstance(course_updates, dict) and course_updates.get('error'):
return JsonResponse(course_updates, course_updates.get('status', 400))
else:
@@ -739,7 +739,7 @@ def textbooks_list_handler(request, course_key_string):
"""
course_key = CourseKey.from_string(course_key_string)
course = _get_course_module(course_key, request.user)
store = get_modulestore(course.location)
store = modulestore()
if not "application/json" in request.META.get('HTTP_ACCEPT', 'text/html'):
# return HTML page
@@ -814,7 +814,7 @@ def textbooks_detail_handler(request, course_key_string, textbook_id):
"""
course_key = CourseKey.from_string(course_key_string)
course_module = _get_course_module(course_key, request.user)
store = get_modulestore(course_module.location)
store = modulestore()
matching_id = [tb for tb in course_module.pdf_textbooks
if unicode(tb.get("id")) == unicode(textbook_id)]
if matching_id:

24
cms/djangoapps/contentstore/views/helpers.py
View File

@@ -39,34 +39,16 @@ def render_from_lms(template_name, dictionary, context=None, namespace='main'):
return render_to_string(template_name, dictionary, context, namespace="lms." + namespace)
def _xmodule_recurse(item, action, ignore_exception=()):
"""
Recursively apply provided action on item and its children
ignore_exception (Exception Object): A optional argument; when passed ignores the corresponding
exception raised during xmodule recursion,
"""
for child in item.get_children():
_xmodule_recurse(child, action, ignore_exception)
try:
return action(item)
except ignore_exception:
return
def get_parent_xblock(xblock):
"""
Returns the xblock that is the parent of the specified xblock, or None if it has no parent.
"""
locator = xblock.location
parent_locations = modulestore().get_parent_locations(locator,)
parent_location = modulestore().get_parent_location(locator)
if len(parent_locations) == 0:
if parent_location is None:
return None
elif len(parent_locations) > 1:
logging.error('Multiple parents have been found for %s', unicode(locator))
return modulestore().get_item(parent_locations[0])
return modulestore().get_item(parent_location)
def is_unit(xblock):

12
cms/djangoapps/contentstore/views/import_export.py
View File

@@ -217,13 +217,13 @@ def import_handler(request, course_key_string):
shutil.move(dirpath / fname, course_dir)
_module_store, course_items = import_from_xml(
modulestore('direct'),
modulestore(),
request.user.id,
settings.GITHUB_REPO_ROOT,
[course_subdir],
load_error_modules=False,
static_content_store=contentstore(),
target_course_id=course_key,
draft_store=modulestore()
)
new_location = course_items[0].location
@@ -322,7 +322,7 @@ def export_handler(request, course_key_string):
root_dir = path(mkdtemp())
try:
export_to_xml(modulestore('direct'), contentstore(), course_module.id, root_dir, name, modulestore())
export_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name)
logging.debug('tar file being generated at {0}'.format(export_file.name))
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
@@ -334,10 +334,10 @@ def export_handler(request, course_key_string):
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_locs = modulestore().get_parent_locations(failed_item.location)
parent_loc = modulestore().get_parent_location(failed_item.location)
if len(parent_locs) > 0:
parent = modulestore().get_item(parent_locs[0])
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
if parent.location.category == 'vertical':
unit = parent
except: # pylint: disable=bare-except

172
cms/djangoapps/contentstore/views/item.py
View File

@@ -20,20 +20,19 @@ from xblock.fields import Scope
from xblock.fragment import Fragment
import xmodule
from xmodule.tabs import StaticTab, CourseTabList
from xmodule.modulestore import PublishState, REVISION_OPTION_ALL
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.draft import DIRECT_ONLY_CATEGORIES
from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError, DuplicateItemError
from xmodule.modulestore.inheritance import own_metadata
from xmodule.x_module import PREVIEW_VIEWS, STUDIO_VIEW
from util.json_request import expect_json, JsonResponse
from util.string_utils import str_to_bool
from ..utils import get_modulestore
from .access import has_course_access
from .helpers import _xmodule_recurse, xblock_has_own_studio_page
from contentstore.utils import compute_publish_state, PublishState
from xmodule.modulestore.draft import DIRECT_ONLY_CATEGORIES
from .helpers import xblock_has_own_studio_page
from contentstore.utils import compute_publish_state
from contentstore.views.preview import get_preview_fragment
from edxmako.shortcuts import render_to_string
from models.settings.course_grading import CourseGradingModel
@@ -72,8 +71,7 @@ def xblock_handler(request, usage_key_string):
The restful handler for xblock requests.
DELETE
json: delete this xblock instance from the course. Supports query parameters "recurse" to delete
all children and "all_versions" to delete from all (mongo) versions.
json: delete this xblock instance from the course.
GET
json: returns representation of the xblock (locator id, data, and metadata).
if ?fields=graderType, it returns the graderType for the unit instead of the above.
@@ -115,19 +113,17 @@ def xblock_handler(request, usage_key_string):
# right now can't combine output of this w/ output of _get_module_info, but worthy goal
return JsonResponse(CourseGradingModel.get_section_grader_type(usage_key))
# TODO: pass fields to _get_module_info and only return those
rsp = _get_module_info(usage_key)
rsp = _get_module_info(usage_key, request.user)
return JsonResponse(rsp)
else:
return HttpResponse(status=406)
elif request.method == 'DELETE':
delete_children = str_to_bool(request.REQUEST.get('recurse', 'False'))
delete_all_versions = str_to_bool(request.REQUEST.get('all_versions', 'False'))
return _delete_item_at_location(usage_key, delete_children, delete_all_versions, request.user)
_delete_item(usage_key, request.user)
return JsonResponse()
else: # Since we have a usage_key, we are updating an existing xblock.
return _save_item(
request,
request.user,
usage_key,
data=request.json.get('data'),
children=request.json.get('children'),
@@ -177,7 +173,7 @@ def xblock_view_handler(request, usage_key_string, view_name):
accept_header = request.META.get('HTTP_ACCEPT', 'application/json')
if 'application/json' in accept_header:
store = get_modulestore(usage_key)
store = modulestore()
xblock = store.get_item(usage_key)
is_read_only = _is_xblock_read_only(xblock)
container_views = ['container_preview', 'reorderable_container_child_preview']
@@ -264,14 +260,14 @@ def _is_xblock_read_only(xblock):
return component_publish_state == PublishState.public
def _save_item(request, usage_key, data=None, children=None, metadata=None, nullout=None,
def _save_item(user, usage_key, data=None, children=None, metadata=None, nullout=None,
grader_type=None, publish=None):
"""
Saves xblock w/ its fields. Has special processing for grader_type, publish, and nullout and Nones in metadata.
nullout means to truly set the field to None whereas nones in metadata mean to unset them (so they revert
to default).
"""
store = get_modulestore(usage_key)
store = modulestore()
try:
existing_item = store.get_item(usage_key)
@@ -279,8 +275,7 @@ def _save_item(request, usage_key, data=None, children=None, metadata=None, null
if usage_key.category in CREATE_IF_NOT_FOUND:
# New module at this location, for pages that are not pre-created.
# Used for course info handouts.
store.create_and_save_xmodule(usage_key)
existing_item = store.get_item(usage_key)
existing_item = store.create_and_save_xmodule(usage_key, user.id)
else:
raise
except InvalidLocationError:
@@ -292,19 +287,17 @@ def _save_item(request, usage_key, data=None, children=None, metadata=None, null
if publish:
if publish == 'make_private':
_xmodule_recurse(
existing_item,
lambda i: modulestore().unpublish(i.location),
ignore_exception=ItemNotFoundError
)
try:
store.unpublish(existing_item.location, user.id),
except ItemNotFoundError:
pass
elif publish == 'create_draft':
# This recursively clones the existing item location to a draft location (the draft is
# implicit, because modulestore is a Draft modulestore)
_xmodule_recurse(
existing_item,
lambda i: modulestore().convert_to_draft(i.location),
ignore_exception=DuplicateItemError
)
try:
# This recursively clones the item subtree and marks the copies as draft
store.convert_to_draft(existing_item.location, user.id)
except DuplicateItemError:
pass
if data:
# TODO Allow any scope.content fields not just "data" (exactly like the get below this)
@@ -346,10 +339,20 @@ def _save_item(request, usage_key, data=None, children=None, metadata=None, null
field.write_to(existing_item, value)
if callable(getattr(existing_item, "editor_saved", None)):
existing_item.editor_saved(request.user, old_metadata, old_content)
existing_item.editor_saved(user, old_metadata, old_content)
# commit to datastore
store.update_item(existing_item, request.user.id)
store.update_item(existing_item, user.id)
# for static tabs, their containing course also records their display name
if usage_key.category == 'static_tab':
course = store.get_course(usage_key.course_key)
# find the course's reference to this tab and update the name.
static_tab = CourseTabList.get_tab_by_slug(course.tabs, usage_key.name)
# only update if changed
if static_tab and static_tab['name'] != existing_item.display_name:
static_tab['name'] = existing_item.display_name
store.update_item(course, user.id)
result = {
'id': unicode(usage_key),
@@ -358,23 +361,12 @@ def _save_item(request, usage_key, data=None, children=None, metadata=None, null
}
if grader_type is not None:
result.update(CourseGradingModel.update_section_grader_type(existing_item, grader_type, request.user))
result.update(CourseGradingModel.update_section_grader_type(existing_item, grader_type, user))
# Make public after updating the xblock, in case the caller asked
# for both an update and a publish.
if publish and publish == 'make_public':
def _publish(block):
# This is super gross, but prevents us from publishing something that
# we shouldn't. Ideally, all modulestores would have a consistant
# interface for publishing. However, as of now, only the DraftMongoModulestore
# does, so we have to check for the attribute explicitly.
store = get_modulestore(block.location)
store.publish(block.location, request.user.id)
_xmodule_recurse(
existing_item,
_publish
)
modulestore().publish(existing_item.location, user.id)
# Note that children aren't being returned until we have a use case.
return JsonResponse(result)
@@ -392,7 +384,8 @@ def _create_item(request):
if not has_course_access(request.user, usage_key.course_key):
raise PermissionDenied()
parent = get_modulestore(category).get_item(usage_key)
store = modulestore()
parent = store.get_item(usage_key)
dest_usage_key = usage_key.replace(category=category, name=uuid4().hex)
# get the metadata, display_name, and definition from the request
@@ -410,17 +403,31 @@ def _create_item(request):
if display_name is not None:
metadata['display_name'] = display_name
get_modulestore(category).create_and_save_xmodule(
store.create_and_save_xmodule(
dest_usage_key,
request.user.id,
definition_data=data,
metadata=metadata,
system=parent.runtime,
runtime=parent.runtime,
)
# VS[compat] cdodge: This is a hack because static_tabs also have references from the course module, so
# if we add one then we need to also add it to the policy information (i.e. metadata)
# we should remove this once we can break this reference from the course to static tabs
if category == 'static_tab':
course = store.get_course(dest_usage_key.course_key)
course.tabs.append(
StaticTab(
name=display_name,
url_slug=dest_usage_key.name,
)
)
store.update_item(course, request.user.id)
# TODO replace w/ nicer accessor
if not 'detached' in parent.runtime.load_block_type(category)._class_tags:
parent.children.append(dest_usage_key)
get_modulestore(parent.location).update_item(parent, request.user.id)
store.update_item(parent, request.user.id)
return JsonResponse({"locator": unicode(dest_usage_key), "courseKey": unicode(dest_usage_key.course_key)})
@@ -429,7 +436,7 @@ def _duplicate_item(parent_usage_key, duplicate_source_usage_key, display_name=N
"""
Duplicate an existing xblock as a child of the supplied parent_usage_key.
"""
store = get_modulestore(duplicate_source_usage_key)
store = modulestore()
source_item = store.get_item(duplicate_source_usage_key)
# Change the blockID to be unique.
dest_usage_key = duplicate_source_usage_key.replace(name=uuid4().hex)
@@ -445,14 +452,14 @@ def _duplicate_item(parent_usage_key, duplicate_source_usage_key, display_name=N
else:
duplicate_metadata['display_name'] = _("Duplicate of '{0}'").format(source_item.display_name)
get_modulestore(category).create_and_save_xmodule(
dest_module = store.create_and_save_xmodule(
dest_usage_key,
definition_data=source_item.data if hasattr(source_item, 'data') else None,
user.id,
definition_data=source_item.get_explicitly_set_fields_by_scope(Scope.content),
metadata=duplicate_metadata,
system=source_item.runtime,
runtime=source_item.runtime,
)
dest_module = get_modulestore(category).get_item(dest_usage_key)
# Children are not automatically copied over (and not all xblocks have a 'children' attribute).
# Because DAGs are not fully supported, we need to actually duplicate each child as well.
if source_item.has_children:
@@ -460,10 +467,10 @@ def _duplicate_item(parent_usage_key, duplicate_source_usage_key, display_name=N
for child in source_item.children:
dupe = _duplicate_item(dest_usage_key, child, user=user)
dest_module.children.append(dupe)
get_modulestore(dest_usage_key).update_item(dest_module, user.id if user else None)
store.update_item(dest_module, user.id if user else None)
if not 'detached' in source_item.runtime.load_block_type(category)._class_tags:
parent = get_modulestore(parent_usage_key).get_item(parent_usage_key)
parent = store.get_item(parent_usage_key)
# If source was already a child of the parent, add duplicate immediately afterward.
# Otherwise, add child to end.
if duplicate_source_usage_key in parent.children:
@@ -471,36 +478,28 @@ def _duplicate_item(parent_usage_key, duplicate_source_usage_key, display_name=N
parent.children.insert(source_index + 1, dest_usage_key)
else:
parent.children.append(dest_usage_key)
get_modulestore(parent_usage_key).update_item(parent, user.id if user else None)
store.update_item(parent, user.id if user else None)
return dest_usage_key
def _delete_item_at_location(item_usage_key, delete_children=False, delete_all_versions=False, user=None):
def _delete_item(usage_key, user):
"""
Deletes the item at with the given Location.
It is assumed that course permissions have already been checked.
Deletes an existing xblock with the given usage_key.
If the xblock is a Static Tab, removes it from course.tabs as well.
"""
store = get_modulestore(item_usage_key)
store = modulestore()
item = store.get_item(item_usage_key)
# VS[compat] cdodge: This is a hack because static_tabs also have references from the course module, so
# if we add one then we need to also add it to the policy information (i.e. metadata)
# we should remove this once we can break this reference from the course to static tabs
if usage_key.category == 'static_tab':
course = store.get_course(usage_key.course_key)
existing_tabs = course.tabs or []
course.tabs = [tab for tab in existing_tabs if tab.get('url_slug') != usage_key.name]
store.update_item(course, user.id)
if delete_children:
_xmodule_recurse(item, lambda i: store.delete_item(i.location, delete_all_versions=delete_all_versions))
else:
store.delete_item(item.location, delete_all_versions=delete_all_versions)
# cdodge: we need to remove our parent's pointer to us so that it is no longer dangling
if delete_all_versions:
parent_locs = modulestore('direct').get_parent_locations(item_usage_key)
for parent_loc in parent_locs:
parent = modulestore('direct').get_item(parent_loc)
parent.children.remove(item_usage_key)
modulestore('direct').update_item(parent, user.id if user else None)
return JsonResponse()
store.delete_item(usage_key, user.id)
# pylint: disable=W0613
@@ -522,29 +521,30 @@ def orphan_handler(request, course_key_string):
raise PermissionDenied()
if request.method == 'DELETE':
if request.user.is_staff:
items = modulestore().get_orphans(course_usage_key)
store = modulestore()
items = store.get_orphans(course_usage_key)
for itemloc in items:
# get_orphans returns the deprecated string format
# get_orphans returns the deprecated string format w/o revision
usage_key = course_usage_key.make_usage_key_from_deprecated_string(itemloc)
modulestore().delete_item(usage_key, delete_all_versions=True)
# need to delete all versions
store.delete_item(usage_key, request.user.id, revision=REVISION_OPTION_ALL)
return JsonResponse({'deleted': items})
else:
raise PermissionDenied()
def _get_module_info(usage_key, rewrite_static_links=True):
def _get_module_info(usage_key, user, rewrite_static_links=True):
"""
metadata, data, id representation of a leaf module fetcher.
:param usage_key: A UsageKey
"""
store = get_modulestore(usage_key)
store = modulestore()
try:
module = store.get_item(usage_key)
except ItemNotFoundError:
if usage_key.category in CREATE_IF_NOT_FOUND:
# Create a new one for certain categories only. Used for course info handouts.
store.create_and_save_xmodule(usage_key)
module = store.get_item(usage_key)
module = store.create_and_save_xmodule(usage_key, user.id)
else:
raise

10
cms/djangoapps/contentstore/views/tabs.py
View File

@@ -117,7 +117,7 @@ def reorder_tabs_handler(course_item, request):
# persist the new order of the tabs
course_item.tabs = new_tab_list
modulestore('direct').update_item(course_item, request.user.id)
modulestore().update_item(course_item, request.user.id)
return JsonResponse()
@@ -140,7 +140,7 @@ def edit_tab_handler(course_item, request):
if 'is_hidden' in request.json:
# set the is_hidden attribute on the requested tab
tab.is_hidden = request.json['is_hidden']
modulestore('direct').update_item(course_item, request.user.id)
modulestore().update_item(course_item, request.user.id)
else:
raise NotImplementedError('Unsupported request to edit tab: {0}'.format(request.json))
@@ -163,7 +163,7 @@ def get_tab_by_locator(tab_list, usage_key_string):
Look for a tab with the specified locator. Returns the first matching tab.
"""
tab_location = UsageKey.from_string(usage_key_string)
item = modulestore('direct').get_item(tab_location)
item = modulestore().get_item(tab_location)
static_tab = StaticTab(
name=item.display_name,
url_slug=item.location.name,
@@ -192,7 +192,7 @@ def primitive_delete(course, num):
# Note for future implementations: if you delete a static_tab, then Chris Dodge
# points out that there's other stuff to delete beyond this element.
# This code happens to not delete static_tab so it doesn't come up.
modulestore('direct').update_item(course, '**replace_user**')
modulestore().update_item(course, '**replace_user**')
def primitive_insert(course, num, tab_type, name):
@@ -201,5 +201,5 @@ def primitive_insert(course, num, tab_type, name):
new_tab = CourseTab.from_json({u'type': unicode(tab_type), u'name': unicode(name)})
tabs = course.tabs
tabs.insert(num, new_tab)
modulestore('direct').update_item(course, '**replace_user**')
modulestore().update_item(course, '**replace_user**')

6
cms/djangoapps/contentstore/views/tests/test_assets.py
View File

@@ -48,9 +48,10 @@ class BasicAssetsTestCase(AssetsTestCase):
self.assertEquals(path, '/static/my_file_name.jpg')
def test_pdf_asset(self):
module_store = modulestore('direct')
module_store = modulestore()
_, course_items = import_from_xml(
module_store,
'**replace_user**',
'common/test/data/',
['toy'],
static_content_store=contentstore(),
@@ -191,9 +192,10 @@ class LockAssetTestCase(AssetsTestCase):
return json.loads(resp.content)
# Load the toy course.
module_store = modulestore('direct')
module_store = modulestore()
_, course_items = import_from_xml(
module_store,
'**replace_user**',
'common/test/data/',
['toy'],
static_content_store=contentstore(),

9
cms/djangoapps/contentstore/views/tests/test_checklists.py
View File

@@ -1,7 +1,8 @@
""" Unit tests for checklist methods in views.py. """
from contentstore.utils import get_modulestore, reverse_course_url
from contentstore.utils import reverse_course_url
from contentstore.views.checklist import expand_checklist_action_url
from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.django import modulestore
import json
from contentstore.tests.utils import CourseTestCase
@@ -21,8 +22,7 @@ class ChecklistTestCase(CourseTestCase):
def get_persisted_checklists(self):
""" Returns the checklists as persisted in the modulestore. """
modulestore = get_modulestore(self.course.location)
return modulestore.get_item(self.course.location).checklists
return modulestore().get_item(self.course.location).checklists
def compare_checklists(self, persisted, request):
"""
@@ -54,8 +54,7 @@ class ChecklistTestCase(CourseTestCase):
self.course.checklists = None
# Save the changed `checklists` to the underlying KeyValueStore before updating the modulestore
self.course.save()
modulestore = get_modulestore(self.course.location)
modulestore.update_item(self.course, self.user.id)
modulestore().update_item(self.course, self.user.id)
self.assertEqual(self.get_persisted_checklists(), None)
response = self.client.get(self.checklists_url)
self.assertEqual(payload, response.content)

47
cms/djangoapps/contentstore/views/tests/test_container_page.py
View File

@@ -3,8 +3,9 @@ Unit tests for the container page.
"""
import re
from contentstore.utils import compute_publish_state, PublishState
from contentstore.utils import compute_publish_state
from contentstore.views.tests.utils import StudioPageTestCase
from xmodule.modulestore import PublishState
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.factories import ItemFactory
@@ -29,6 +30,7 @@ class ContainerPageTestCase(StudioPageTestCase):
category='vertical', display_name='Child Vertical')
self.video = ItemFactory.create(parent_location=self.child_vertical.location,
category="video", display_name="My Video")
self.store = modulestore()
def test_container_html(self):
self._test_html_content(
@@ -49,12 +51,12 @@ class ContainerPageTestCase(StudioPageTestCase):
Create the scenario of an xblock with children (non-vertical) on the container page.
This should create a container page that is a child of another container page.
"""
published_container = ItemFactory.create(
draft_container = ItemFactory.create(
parent_location=self.child_container.location,
category="wrapper", display_name="Wrapper"
)
ItemFactory.create(
parent_location=published_container.location,
parent_location=draft_container.location,
category="html", display_name="Child HTML"
)
@@ -63,7 +65,7 @@ class ContainerPageTestCase(StudioPageTestCase):
xblock,
expected_section_tag=(
'<section class="wrapper-xblock level-page is-hidden studio-xblock-wrapper" '
'data-locator="{0}" data-course-key="{0.course_key}">'.format(published_container.location)
'data-locator="{0}" data-course-key="{0.course_key}">'.format(draft_container.location)
),
expected_breadcrumbs=(
r'<a href="/unit/{unit}"\s*'
@@ -77,13 +79,12 @@ class ContainerPageTestCase(StudioPageTestCase):
)
)
# Test the published version of the container
test_container_html(published_container)
# Test the draft version of the container
test_container_html(draft_container)
# Now make the unit and its children into a draft and validate the container again
modulestore('draft').convert_to_draft(self.vertical.location)
modulestore('draft').convert_to_draft(self.child_vertical.location)
draft_container = modulestore('draft').convert_to_draft(published_container.location)
# Now publish the unit and validate again
self.store.publish(self.vertical.location, self.user.id)
draft_container = self.store.get_item(draft_container.location)
test_container_html(draft_container)
def _test_html_content(self, xblock, expected_section_tag, expected_breadcrumbs):
@@ -98,7 +99,6 @@ class ContainerPageTestCase(StudioPageTestCase):
self.assertRegexpMatches(html, expected_breadcrumbs)
# Verify the link that allows users to change publish status.
expected_message = None
if publish_state == PublishState.public:
expected_message = 'you need to edit unit <a href="/unit/{}">Unit</a> as a draft.'
else:
@@ -110,25 +110,25 @@ class ContainerPageTestCase(StudioPageTestCase):
"""
Verify that a public xblock's container preview returns the expected HTML.
"""
self.validate_preview_html(self.vertical, self.container_view,
published_unit = self.store.publish(self.vertical.location, self.user.id)
published_child_container = self.store.get_item(self.child_container.location)
published_child_vertical = self.store.get_item(self.child_vertical.location)
self.validate_preview_html(published_unit, self.container_view,
can_edit=False, can_reorder=False, can_add=False)
self.validate_preview_html(self.child_container, self.container_view,
self.validate_preview_html(published_child_container, self.container_view,
can_edit=False, can_reorder=False, can_add=False)
self.validate_preview_html(self.child_vertical, self.reorderable_child_view,
self.validate_preview_html(published_child_vertical, self.reorderable_child_view,
can_edit=False, can_reorder=False, can_add=False)
def test_draft_container_preview_html(self):
"""
Verify that a draft xblock's container preview returns the expected HTML.
"""
draft_unit = modulestore('draft').convert_to_draft(self.vertical.location)
draft_child_container = modulestore('draft').convert_to_draft(self.child_container.location)
draft_child_vertical = modulestore('draft').convert_to_draft(self.child_vertical.location)
self.validate_preview_html(draft_unit, self.container_view,
self.validate_preview_html(self.vertical, self.container_view,
can_edit=True, can_reorder=True, can_add=True)
self.validate_preview_html(draft_child_container, self.container_view,
self.validate_preview_html(self.child_container, self.container_view,
can_edit=True, can_reorder=True, can_add=True)
self.validate_preview_html(draft_child_vertical, self.reorderable_child_view,
self.validate_preview_html(self.child_vertical, self.reorderable_child_view,
can_edit=True, can_reorder=True, can_add=True)
def test_public_child_container_preview_html(self):
@@ -137,7 +137,8 @@ class ContainerPageTestCase(StudioPageTestCase):
"""
empty_child_container = ItemFactory.create(parent_location=self.vertical.location,
category='split_test', display_name='Split Test')
self.validate_preview_html(empty_child_container, self.reorderable_child_view,
published_empty_child_container = self.store.publish(empty_child_container.location, '**replace_user**')
self.validate_preview_html(published_empty_child_container, self.reorderable_child_view,
can_reorder=False, can_edit=False, can_add=False)
def test_draft_child_container_preview_html(self):
@@ -146,7 +147,5 @@ class ContainerPageTestCase(StudioPageTestCase):
"""
empty_child_container = ItemFactory.create(parent_location=self.vertical.location,
category='split_test', display_name='Split Test')
modulestore('draft').convert_to_draft(self.vertical.location)
draft_empty_child_container = modulestore('draft').convert_to_draft(empty_child_container.location)
self.validate_preview_html(draft_empty_child_container, self.reorderable_child_view,
self.validate_preview_html(empty_child_container, self.reorderable_child_view,
can_reorder=True, can_edit=True, can_add=False)

19
cms/djangoapps/contentstore/views/tests/test_course_updates.py
View File

@@ -129,13 +129,12 @@ class CourseUpdateTest(CourseTestCase):
'''
# get the updates and populate 'data' field with some data.
location = self.course.id.make_usage_key('course_info', 'updates')
modulestore('direct').create_and_save_xmodule(location)
course_updates = modulestore('direct').get_item(location)
course_updates = modulestore().create_and_save_xmodule(location, self.user.id)
update_date = u"January 23, 2014"
update_content = u"Hello world!"
update_data = u"<ol><li><h2>" + update_date + "</h2>" + update_content + "</li></ol>"
course_updates.data = update_data
modulestore('direct').update_item(course_updates, self.user.id)
modulestore().update_item(course_updates, self.user.id)
# test getting all updates list
course_update_url = self.create_update_url()
@@ -155,7 +154,7 @@ class CourseUpdateTest(CourseTestCase):
# test that while updating it converts old data (with string format in 'data' field)
# to new data (with list format in 'items' field) and respectively updates 'data' field.
course_updates = modulestore('direct').get_item(location)
course_updates = modulestore().get_item(location)
self.assertEqual(course_updates.items, [])
# now try to update first update item
update_content = 'Testing'
@@ -164,20 +163,20 @@ class CourseUpdateTest(CourseTestCase):
course_update_url + '1', payload, HTTP_X_HTTP_METHOD_OVERRIDE="PUT", REQUEST_METHOD="POST"
)
self.assertHTMLEqual(update_content, json.loads(resp.content)['content'])
course_updates = modulestore('direct').get_item(location)
course_updates = modulestore().get_item(location)
self.assertEqual(course_updates.items, [{u'date': update_date, u'content': update_content, u'id': 1}])
# course_updates 'data' field should update accordingly
update_data = u"<section><article><h2>{date}</h2>{content}</article></section>".format(date=update_date, content=update_content)
self.assertEqual(course_updates.data, update_data)
# test delete course update item (soft delete)
course_updates = modulestore('direct').get_item(location)
course_updates = modulestore().get_item(location)
self.assertEqual(course_updates.items, [{u'date': update_date, u'content': update_content, u'id': 1}])
# now try to delete first update item
resp = self.client.delete(course_update_url + '1')
self.assertEqual(json.loads(resp.content), [])
# confirm that course update is soft deleted ('status' flag set to 'deleted') in db
course_updates = modulestore('direct').get_item(location)
course_updates = modulestore().get_item(location)
self.assertEqual(course_updates.items,
[{u'date': update_date, u'content': update_content, u'id': 1, u'status': 'deleted'}])
@@ -204,10 +203,10 @@ class CourseUpdateTest(CourseTestCase):
'''Test trying to add to a saved course_update which is not an ol.'''
# get the updates and set to something wrong
location = self.course.id.make_usage_key('course_info', 'updates')
modulestore('direct').create_and_save_xmodule(location)
course_updates = modulestore('direct').get_item(location)
modulestore().create_and_save_xmodule(location, self.user.id)
course_updates = modulestore().get_item(location)
course_updates.data = 'bad news'
modulestore('direct').update_item(course_updates, self.user.id)
modulestore().update_item(course_updates, self.user.id)
init_content = '<iframe width="560" height="315" src="http://www.youtube.com/embed/RocY-Jd93XU" frameborder="0">'
content = init_content + '</iframe>'

3
cms/djangoapps/contentstore/views/tests/test_import_export.py
View File

@@ -294,7 +294,8 @@ class ExportTestCase(CourseTestCase):
"""
Export failure.
"""
ItemFactory.create(parent_location=self.course.location, category='aawefawef')
fake_xblock = ItemFactory.create(parent_location=self.course.location, category='aawefawef')
self.store.publish(fake_xblock.location, self.user.id)
self._verify_export_failure(u'/unit/location:MITx+999+Robot_Super_Course+course+Robot_Super_Course')
def test_export_failure_subsection_level(self):

167
cms/djangoapps/contentstore/views/tests/test_item.py
View File

@@ -17,12 +17,12 @@ from contentstore.utils import reverse_usage_url
from contentstore.views.component import component_handler, get_component_templates
from contentstore.tests.utils import CourseTestCase
from contentstore.utils import compute_publish_state, PublishState
from student.tests.factories import UserFactory
from xmodule.capa_module import CapaDescriptor
from xmodule.modulestore import PublishState
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.x_module import STUDIO_VIEW, STUDENT_VIEW
from xblock.exceptions import NoSuchHandlerError
from opaque_keys.edx.keys import UsageKey
from opaque_keys.edx.locations import Location
from xmodule.partitions.partitions import Group, UserPartition
@@ -35,14 +35,16 @@ class ItemTest(CourseTestCase):
self.course_key = self.course.id
self.usage_key = self.course.location
self.store = modulestore()
@staticmethod
def get_item_from_modulestore(usage_key, draft=False):
def get_item_from_modulestore(self, usage_key, verify_is_draft=False):
"""
Get the item referenced by the UsageKey from the modulestore
"""
store = modulestore('draft') if draft else modulestore('direct')
return store.get_item(usage_key)
item = self.store.get_item(usage_key)
if verify_is_draft:
self.assertTrue(getattr(item, 'is_draft', False))
return item
def response_usage_key(self, response):
"""
@@ -218,9 +220,7 @@ class TestCreateItem(ItemTest):
boilerplate=template_id
)
prob_usage_key = self.response_usage_key(resp)
problem = self.get_item_from_modulestore(prob_usage_key, True)
# ensure it's draft
self.assertTrue(problem.is_draft)
problem = self.get_item_from_modulestore(prob_usage_key, verify_is_draft=True)
# check against the template
template = CapaDescriptor.get_template(template_id)
self.assertEqual(problem.data, template['data'])
@@ -278,8 +278,8 @@ class TestDuplicateItem(ItemTest):
self.assertTrue(check_equality(source_usage_key, usage_key), "Duplicated item differs from original")
def check_equality(source_usage_key, duplicate_usage_key):
original_item = self.get_item_from_modulestore(source_usage_key, draft=True)
duplicated_item = self.get_item_from_modulestore(duplicate_usage_key, draft=True)
original_item = self.get_item_from_modulestore(source_usage_key)
duplicated_item = self.get_item_from_modulestore(duplicate_usage_key)
self.assertNotEqual(
original_item.location,
@@ -354,7 +354,7 @@ class TestDuplicateItem(ItemTest):
"""
def verify_name(source_usage_key, parent_usage_key, expected_name, display_name=None):
usage_key = self._duplicate_item(parent_usage_key, source_usage_key, display_name)
duplicated_item = self.get_item_from_modulestore(usage_key, draft=True)
duplicated_item = self.get_item_from_modulestore(usage_key)
self.assertEqual(duplicated_item.display_name, expected_name)
return usage_key
@@ -407,6 +407,18 @@ class TestEditItem(ItemTest):
self.course_update_url = reverse_usage_url("xblock_handler", self.usage_key)
def verify_publish_state(self, usage_key, expected_publish_state):
"""
Helper method that gets the item from the module store and verifies that the publish state is as expected.
Returns the item corresponding to the given usage_key.
"""
item = self.get_item_from_modulestore(
usage_key,
(expected_publish_state == PublishState.private) or (expected_publish_state == PublishState.draft)
)
self.assertEqual(expected_publish_state, self.store.compute_publish_state(item))
return item
def test_delete_field(self):
"""
Sending null in for a field 'deletes' it
@@ -415,26 +427,26 @@ class TestEditItem(ItemTest):
self.problem_update_url,
data={'metadata': {'rerandomize': 'onreset'}}
)
problem = self.get_item_from_modulestore(self.problem_usage_key, True)
problem = self.get_item_from_modulestore(self.problem_usage_key, verify_is_draft=True)
self.assertEqual(problem.rerandomize, 'onreset')
self.client.ajax_post(
self.problem_update_url,
data={'metadata': {'rerandomize': None}}
)
problem = self.get_item_from_modulestore(self.problem_usage_key, True)
problem = self.get_item_from_modulestore(self.problem_usage_key, verify_is_draft=True)
self.assertEqual(problem.rerandomize, 'never')
def test_null_field(self):
"""
Sending null in for a field 'deletes' it
"""
problem = self.get_item_from_modulestore(self.problem_usage_key, True)
problem = self.get_item_from_modulestore(self.problem_usage_key, verify_is_draft=True)
self.assertIsNotNone(problem.markdown)
self.client.ajax_post(
self.problem_update_url,
data={'nullout': ['markdown']}
)
problem = self.get_item_from_modulestore(self.problem_usage_key, True)
problem = self.get_item_from_modulestore(self.problem_usage_key, verify_is_draft=True)
self.assertIsNone(problem.markdown)
def test_date_fields(self):
@@ -515,13 +527,12 @@ class TestEditItem(ItemTest):
def test_make_public(self):
""" Test making a private problem public (publishing it). """
# When the problem is first created, it is only in draft (because of its category).
with self.assertRaises(ItemNotFoundError):
self.get_item_from_modulestore(self.problem_usage_key, False)
self.verify_publish_state(self.problem_usage_key, PublishState.private)
self.client.ajax_post(
self.problem_update_url,
data={'publish': 'make_public'}
)
self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))
self.verify_publish_state(self.problem_usage_key, PublishState.public)
def test_make_private(self):
""" Test making a public problem private (un-publishing it). """
@@ -530,14 +541,14 @@ class TestEditItem(ItemTest):
self.problem_update_url,
data={'publish': 'make_public'}
)
self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))
self.verify_publish_state(self.problem_usage_key, PublishState.public)
# Now make it private
self.client.ajax_post(
self.problem_update_url,
data={'publish': 'make_private'}
)
with self.assertRaises(ItemNotFoundError):
self.get_item_from_modulestore(self.problem_usage_key, False)
self.verify_publish_state(self.problem_usage_key, PublishState.private)
def test_make_draft(self):
""" Test creating a draft version of a public problem. """
@@ -546,21 +557,23 @@ class TestEditItem(ItemTest):
self.problem_update_url,
data={'publish': 'make_public'}
)
self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))
published = self.verify_publish_state(self.problem_usage_key, PublishState.public)
# Now make it draft, which means both versions will exist.
self.client.ajax_post(
self.problem_update_url,
data={'publish': 'create_draft'}
)
self.verify_publish_state(self.problem_usage_key, PublishState.draft)
# Update the draft version and check that published is different.
self.client.ajax_post(
self.problem_update_url,
data={'metadata': {'due': '2077-10-10T04:00Z'}}
)
published = self.get_item_from_modulestore(self.problem_usage_key, False)
updated_draft = self.get_item_from_modulestore(self.problem_usage_key, verify_is_draft=True)
self.assertEqual(updated_draft.due, datetime(2077, 10, 10, 4, 0, tzinfo=UTC))
self.assertIsNone(published.due)
draft = self.get_item_from_modulestore(self.problem_usage_key, True)
self.assertEqual(draft.due, datetime(2077, 10, 10, 4, 0, tzinfo=UTC))
def test_make_public_with_update(self):
""" Update a problem and make it public at the same time. """
@@ -571,7 +584,7 @@ class TestEditItem(ItemTest):
'publish': 'make_public'
}
)
published = self.get_item_from_modulestore(self.problem_usage_key, False)
published = self.get_item_from_modulestore(self.problem_usage_key)
self.assertEqual(published.due, datetime(2077, 10, 10, 4, 0, tzinfo=UTC))
def test_make_private_with_update(self):
@@ -581,6 +594,9 @@ class TestEditItem(ItemTest):
self.problem_update_url,
data={'publish': 'make_public'}
)
self.verify_publish_state(self.problem_usage_key, PublishState.public)
# Make problem private and update.
self.client.ajax_post(
self.problem_update_url,
data={
@@ -588,9 +604,7 @@ class TestEditItem(ItemTest):
'publish': 'make_private'
}
)
with self.assertRaises(ItemNotFoundError):
self.get_item_from_modulestore(self.problem_usage_key, False)
draft = self.get_item_from_modulestore(self.problem_usage_key, True)
draft = self.verify_publish_state(self.problem_usage_key, PublishState.private)
self.assertEqual(draft.due, datetime(2077, 10, 10, 4, 0, tzinfo=UTC))
def test_create_draft_with_update(self):
@@ -600,7 +614,8 @@ class TestEditItem(ItemTest):
self.problem_update_url,
data={'publish': 'make_public'}
)
self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))
published = self.verify_publish_state(self.problem_usage_key, PublishState.public)
# Now make it draft, which means both versions will exist.
self.client.ajax_post(
self.problem_update_url,
@@ -609,10 +624,9 @@ class TestEditItem(ItemTest):
'publish': 'create_draft'
}
)
published = self.get_item_from_modulestore(self.problem_usage_key, False)
self.assertIsNone(published.due)
draft = self.get_item_from_modulestore(self.problem_usage_key, True)
draft = self.get_item_from_modulestore(self.problem_usage_key, verify_is_draft=True)
self.assertEqual(draft.due, datetime(2077, 10, 10, 4, 0, tzinfo=UTC))
self.assertIsNone(published.due)
def test_create_draft_with_multiple_requests(self):
"""
@@ -623,7 +637,8 @@ class TestEditItem(ItemTest):
self.problem_update_url,
data={'publish': 'make_public'}
)
self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))
self.verify_publish_state(self.problem_usage_key, PublishState.public)
# Now make it draft, which means both versions will exist.
self.client.ajax_post(
self.problem_update_url,
@@ -631,9 +646,7 @@ class TestEditItem(ItemTest):
'publish': 'create_draft'
}
)
self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))
draft_1 = self.get_item_from_modulestore(self.problem_usage_key, True)
self.assertIsNotNone(draft_1)
draft_1 = self.verify_publish_state(self.problem_usage_key, PublishState.draft)
# Now check that when a user sends request to create a draft when there is already a draft version then
# user gets that already created draft instead of getting 'DuplicateItemError' exception.
@@ -643,7 +656,7 @@ class TestEditItem(ItemTest):
'publish': 'create_draft'
}
)
draft_2 = self.get_item_from_modulestore(self.problem_usage_key, True)
draft_2 = self.verify_publish_state(self.problem_usage_key, PublishState.draft)
self.assertIsNotNone(draft_2)
self.assertEqual(draft_1, draft_2)
@@ -656,9 +669,9 @@ class TestEditItem(ItemTest):
self.problem_update_url,
data={'publish': 'make_public'}
)
self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))
self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key))
# Now make it private, and check that its published version not exists
# Now make it private, and check that its version is private
resp = self.client.ajax_post(
self.problem_update_url,
data={
@@ -666,10 +679,7 @@ class TestEditItem(ItemTest):
}
)
self.assertEqual(resp.status_code, 200)
with self.assertRaises(ItemNotFoundError):
self.get_item_from_modulestore(self.problem_usage_key, False)
draft_1 = self.get_item_from_modulestore(self.problem_usage_key, True)
self.assertIsNotNone(draft_1)
draft_1 = self.verify_publish_state(self.problem_usage_key, PublishState.private)
# Now check that when a user sends request to make it private when it already is private then
# user gets that private version instead of getting 'ItemNotFoundError' exception.
@@ -680,24 +690,21 @@ class TestEditItem(ItemTest):
}
)
self.assertEqual(resp.status_code, 200)
with self.assertRaises(ItemNotFoundError):
self.get_item_from_modulestore(self.problem_usage_key, False)
draft_2 = self.get_item_from_modulestore(self.problem_usage_key, True)
self.assertIsNotNone(draft_2)
draft_2 = self.verify_publish_state(self.problem_usage_key, PublishState.private)
self.assertEqual(draft_1, draft_2)
def test_published_and_draft_contents_with_update(self):
""" Create a draft and publish it then modify the draft and check that published content is not modified """
# Make problem public.
resp = self.client.ajax_post(
self.client.ajax_post(
self.problem_update_url,
data={'publish': 'make_public'}
)
self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))
published = self.verify_publish_state(self.problem_usage_key, PublishState.public)
# Now make a draft
resp = self.client.ajax_post(
self.client.ajax_post(
self.problem_update_url,
data={
'id': unicode(self.problem_usage_key),
@@ -708,8 +715,7 @@ class TestEditItem(ItemTest):
)
# Both published and draft content should be different
published = self.get_item_from_modulestore(self.problem_usage_key, False)
draft = self.get_item_from_modulestore(self.problem_usage_key, True)
draft = self.get_item_from_modulestore(self.problem_usage_key, verify_is_draft=True)
self.assertNotEqual(draft.data, published.data)
# Get problem by 'xblock_handler'
@@ -723,8 +729,7 @@ class TestEditItem(ItemTest):
self.assertEqual(resp.status_code, 200)
# Both published and draft content should still be different
published = self.get_item_from_modulestore(self.problem_usage_key, False)
draft = self.get_item_from_modulestore(self.problem_usage_key, True)
draft = self.get_item_from_modulestore(self.problem_usage_key, verify_is_draft=True)
self.assertNotEqual(draft.data, published.data)
def test_publish_states_of_nested_xblocks(self):
@@ -739,10 +744,8 @@ class TestEditItem(ItemTest):
# The unit and its children should be private initially
unit_update_url = reverse_usage_url('xblock_handler', unit_usage_key)
unit = self.get_item_from_modulestore(unit_usage_key, True)
html = self.get_item_from_modulestore(html_usage_key, True)
self.assertEqual(compute_publish_state(unit), PublishState.private)
self.assertEqual(compute_publish_state(html), PublishState.private)
self.verify_publish_state(unit_usage_key, PublishState.private)
self.verify_publish_state(html_usage_key, PublishState.private)
# Make the unit public and verify that the problem is also made public
resp = self.client.ajax_post(
@@ -750,10 +753,8 @@ class TestEditItem(ItemTest):
data={'publish': 'make_public'}
)
self.assertEqual(resp.status_code, 200)
unit = self.get_item_from_modulestore(unit_usage_key, True)
html = self.get_item_from_modulestore(html_usage_key, True)
self.assertEqual(compute_publish_state(unit), PublishState.public)
self.assertEqual(compute_publish_state(html), PublishState.public)
self.verify_publish_state(unit_usage_key, PublishState.public)
self.verify_publish_state(html_usage_key, PublishState.public)
# Make a draft for the unit and verify that the problem also has a draft
resp = self.client.ajax_post(
@@ -765,10 +766,8 @@ class TestEditItem(ItemTest):
}
)
self.assertEqual(resp.status_code, 200)
unit = self.get_item_from_modulestore(unit_usage_key, True)
html = self.get_item_from_modulestore(html_usage_key, True)
self.assertEqual(compute_publish_state(unit), PublishState.draft)
self.assertEqual(compute_publish_state(html), PublishState.draft)
self.verify_publish_state(unit_usage_key, PublishState.draft)
self.verify_publish_state(html_usage_key, PublishState.draft)
class TestEditSplitModule(ItemTest):
@@ -792,6 +791,9 @@ class TestEditSplitModule(ItemTest):
resp = self.create_xblock(category='split_test', parent_usage_key=root_usage_key)
self.split_test_usage_key = self.response_usage_key(resp)
self.split_test_update_url = reverse_usage_url("xblock_handler", self.split_test_usage_key)
self.request_factory = RequestFactory()
self.request = self.request_factory.get('/dummy-url')
self.request.user = self.user
def _update_partition_id(self, partition_id):
"""
@@ -808,7 +810,7 @@ class TestEditSplitModule(ItemTest):
)
# Verify the partition_id was saved.
split_test = self.get_item_from_modulestore(self.split_test_usage_key, True)
split_test = self.get_item_from_modulestore(self.split_test_usage_key, verify_is_draft=True)
self.assertEqual(partition_id, split_test.user_partition_id)
return split_test
@@ -825,7 +827,7 @@ class TestEditSplitModule(ItemTest):
Test that verticals are created for the experiment groups when
a spit test module is edited.
"""
split_test = self.get_item_from_modulestore(self.split_test_usage_key, True)
split_test = self.get_item_from_modulestore(self.split_test_usage_key, verify_is_draft=True)
# Initially, no user_partition_id is set, and the split_test has no children.
self.assertEqual(-1, split_test.user_partition_id)
self.assertEqual(0, len(split_test.children))
@@ -835,8 +837,8 @@ class TestEditSplitModule(ItemTest):
# Verify that child verticals have been set to match the groups
self.assertEqual(2, len(split_test.children))
vertical_0 = self.get_item_from_modulestore(split_test.children[0], True)
vertical_1 = self.get_item_from_modulestore(split_test.children[1], True)
vertical_0 = self.get_item_from_modulestore(split_test.children[0], verify_is_draft=True)
vertical_1 = self.get_item_from_modulestore(split_test.children[1], verify_is_draft=True)
self.assertEqual("vertical", vertical_0.category)
self.assertEqual("vertical", vertical_1.category)
self.assertEqual("alpha", vertical_0.display_name)
@@ -863,9 +865,9 @@ class TestEditSplitModule(ItemTest):
self.assertEqual(5, len(split_test.children))
self.assertEqual(initial_vertical_0_location, split_test.children[0])
self.assertEqual(initial_vertical_1_location, split_test.children[1])
vertical_0 = self.get_item_from_modulestore(split_test.children[2], True)
vertical_1 = self.get_item_from_modulestore(split_test.children[3], True)
vertical_2 = self.get_item_from_modulestore(split_test.children[4], True)
vertical_0 = self.get_item_from_modulestore(split_test.children[2], verify_is_draft=True)
vertical_1 = self.get_item_from_modulestore(split_test.children[3], verify_is_draft=True)
vertical_2 = self.get_item_from_modulestore(split_test.children[4], verify_is_draft=True)
# Verify that the group_id_to child mapping is correct.
self.assertEqual(3, len(split_test.group_id_to_child))
@@ -968,14 +970,14 @@ class TestEditSplitModule(ItemTest):
self.assertEqual(2, len(group_id_to_child))
# Call add_missing_groups method to add the missing group.
split_test.add_missing_groups(None)
split_test.add_missing_groups(self.request)
split_test = self._assert_children(3)
self.assertNotEqual(group_id_to_child, split_test.group_id_to_child)
group_id_to_child = split_test.group_id_to_child
self.assertEqual(split_test.children[2], group_id_to_child["2"])
# Call add_missing_groups again -- it should be a no-op.
split_test.add_missing_groups(None)
split_test.add_missing_groups(self.request)
split_test = self._assert_children(3)
self.assertEqual(group_id_to_child, split_test.group_id_to_child)
@@ -985,11 +987,14 @@ class TestComponentHandler(TestCase):
def setUp(self):
self.request_factory = RequestFactory()
patcher = patch('contentstore.views.component.get_modulestore')
self.get_modulestore = patcher.start()
patcher = patch('contentstore.views.component.modulestore')
self.modulestore = patcher.start()
self.addCleanup(patcher.stop)
self.descriptor = self.get_modulestore.return_value.get_item.return_value
# component_handler calls modulestore.get_item to get the descriptor of the requested xBlock.
# Here, we mock the return value of modulestore.get_item so it can be used to mock the handler
# of the xBlock descriptor.
self.descriptor = self.modulestore.return_value.get_item.return_value
self.usage_key_string = unicode(
Location('dummy_org', 'dummy_course', 'dummy_run', 'dummy_category', 'dummy_name')
@@ -1001,7 +1006,7 @@ class TestComponentHandler(TestCase):
self.request.user = self.user
def test_invalid_handler(self):
self.descriptor.handle.side_effect = Http404
self.descriptor.handle.side_effect = NoSuchHandlerError
with self.assertRaises(Http404):
component_handler(self.request, self.usage_key_string, 'invalid_handler')

3
cms/djangoapps/contentstore/views/tests/test_transcripts.py
View File

@@ -21,12 +21,11 @@ from xmodule.exceptions import NotFoundError
from opaque_keys.edx.keys import UsageKey
from xmodule.video_module import transcripts_utils
from contentstore.tests.modulestore_config import TEST_MODULESTORE
TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class Basetranscripts(CourseTestCase):
"""Base test class for transcripts tests."""

16
cms/djangoapps/contentstore/views/tests/test_unit_page.py
View File

@@ -19,11 +19,13 @@ class UnitPageTestCase(StudioPageTestCase):
category='vertical', display_name='Unit')
self.video = ItemFactory.create(parent_location=self.vertical.location,
category="video", display_name="My Video")
self.store = modulestore()
def test_public_unit_page_html(self):
"""
Verify that an xblock returns the expected HTML for a public unit page.
"""
html = self.get_page_html(self.vertical)
self.validate_html_for_add_buttons(html)
@@ -31,14 +33,14 @@ class UnitPageTestCase(StudioPageTestCase):
"""
Verify that an xblock returns the expected HTML for a draft unit page.
"""
draft_unit = modulestore('draft').convert_to_draft(self.vertical.location)
html = self.get_page_html(draft_unit)
html = self.get_page_html(self.vertical)
self.validate_html_for_add_buttons(html)
def test_public_component_preview_html(self):
"""
Verify that a public xblock's preview returns the expected HTML.
"""
published_video = self.store.publish(self.video.location, '**replace_user**')
self.validate_preview_html(self.video, STUDENT_VIEW,
can_edit=True, can_reorder=True, can_add=False)
@@ -46,9 +48,7 @@ class UnitPageTestCase(StudioPageTestCase):
"""
Verify that a draft xblock's preview returns the expected HTML.
"""
modulestore('draft').convert_to_draft(self.vertical.location)
draft_video = modulestore('draft').convert_to_draft(self.video.location)
self.validate_preview_html(draft_video, STUDENT_VIEW,
self.validate_preview_html(self.video, STUDENT_VIEW,
can_edit=True, can_reorder=True, can_add=False)
def test_public_child_container_preview_html(self):
@@ -60,7 +60,8 @@ class UnitPageTestCase(StudioPageTestCase):
category='split_test', display_name='Split Test')
ItemFactory.create(parent_location=child_container.location,
category='html', display_name='grandchild')
self.validate_preview_html(child_container, STUDENT_VIEW,
published_child_container = self.store.publish(child_container.location, '**replace_user**')
self.validate_preview_html(published_child_container, STUDENT_VIEW,
can_reorder=True, can_edit=True, can_add=False)
def test_draft_child_container_preview_html(self):
@@ -72,7 +73,6 @@ class UnitPageTestCase(StudioPageTestCase):
category='split_test', display_name='Split Test')
ItemFactory.create(parent_location=child_container.location,
category='html', display_name='grandchild')
modulestore('draft').convert_to_draft(self.vertical.location)
draft_child_container = modulestore('draft').get_item(child_container.location)
draft_child_container = self.store.get_item(child_container.location)
self.validate_preview_html(draft_child_container, STUDENT_VIEW,
can_reorder=True, can_edit=True, can_add=False)

22
cms/djangoapps/models/settings/course_details.py
View File

@@ -6,7 +6,7 @@ from json.encoder import JSONEncoder
from opaque_keys.edx.locations import Location
from xmodule.modulestore.exceptions import ItemNotFoundError
from contentstore.utils import get_modulestore, course_image_url
from contentstore.utils import course_image_url
from models.settings import course_grading
from xmodule.fields import Date
from xmodule.modulestore.django import modulestore
@@ -34,7 +34,7 @@ class CourseDetails(object):
"""
Fetch the course details for the given course from persistence and return a CourseDetails model.
"""
descriptor = modulestore('direct').get_course(course_key)
descriptor = modulestore().get_course(course_key)
course_details = cls(course_key.org, course_key.course, course_key.run)
course_details.start_date = descriptor.start
@@ -46,31 +46,31 @@ class CourseDetails(object):
temploc = course_key.make_usage_key('about', 'syllabus')
try:
course_details.syllabus = get_modulestore(temploc).get_item(temploc).data
course_details.syllabus = modulestore().get_item(temploc).data
except ItemNotFoundError:
pass
temploc = course_key.make_usage_key('about', 'short_description')
try:
course_details.short_description = get_modulestore(temploc).get_item(temploc).data
course_details.short_description = modulestore().get_item(temploc).data
except ItemNotFoundError:
pass
temploc = course_key.make_usage_key('about', 'overview')
try:
course_details.overview = get_modulestore(temploc).get_item(temploc).data
course_details.overview = modulestore().get_item(temploc).data
except ItemNotFoundError:
pass
temploc = course_key.make_usage_key('about', 'effort')
try:
course_details.effort = get_modulestore(temploc).get_item(temploc).data
course_details.effort = modulestore().get_item(temploc).data
except ItemNotFoundError:
pass
temploc = course_key.make_usage_key('about', 'video')
try:
raw_video = get_modulestore(temploc).get_item(temploc).data
raw_video = modulestore().get_item(temploc).data
course_details.intro_video = CourseDetails.parse_video_tag(raw_video)
except ItemNotFoundError:
pass
@@ -84,14 +84,14 @@ class CourseDetails(object):
delete the about item.
"""
temploc = course_key.make_usage_key('about', about_key)
store = get_modulestore(temploc)
store = modulestore()
if data is None:
store.delete_item(temploc)
store.delete_item(temploc, user.id)
else:
try:
about_item = store.get_item(temploc)
except ItemNotFoundError:
about_item = store.create_xmodule(temploc, system=course.runtime)
about_item = store.create_xmodule(temploc, runtime=course.runtime)
about_item.data = data
store.update_item(about_item, user.id)
@@ -100,7 +100,7 @@ class CourseDetails(object):
"""
Decode the json into CourseDetails and save any changed attrs to the db
"""
module_store = modulestore('direct')
module_store = modulestore()
descriptor = module_store.get_course(course_key)
dirty = False

32
cms/djangoapps/models/settings/course_grading.py
View File

@@ -21,7 +21,7 @@ class CourseGradingModel(object):
"""
Fetch the course grading policy for the given course from persistence and return a CourseGradingModel.
"""
descriptor = modulestore('direct').get_course(course_key)
descriptor = modulestore().get_course(course_key)
model = cls(descriptor)
return model
@@ -31,7 +31,7 @@ class CourseGradingModel(object):
Fetch the course's nth grader
Returns an empty dict if there's no such grader.
"""
descriptor = modulestore('direct').get_course(course_key)
descriptor = modulestore().get_course(course_key)
index = int(index)
if len(descriptor.raw_grader) > index:
return CourseGradingModel.jsonize_grader(index, descriptor.raw_grader[index])
@@ -52,14 +52,14 @@ class CourseGradingModel(object):
Decode the json into CourseGradingModel and save any changes. Returns the modified model.
Probably not the usual path for updates as it's too coarse grained.
"""
descriptor = modulestore('direct').get_course(course_key)
descriptor = modulestore().get_course(course_key)
graders_parsed = [CourseGradingModel.parse_grader(jsonele) for jsonele in jsondict['graders']]
descriptor.raw_grader = graders_parsed
descriptor.grade_cutoffs = jsondict['grade_cutoffs']
modulestore('direct').update_item(descriptor, user.id)
modulestore().update_item(descriptor, user.id)
CourseGradingModel.update_grace_period_from_json(course_key, jsondict['grace_period'], user)
@@ -71,7 +71,7 @@ class CourseGradingModel(object):
Create or update the grader of the given type (string key) for the given course. Returns the modified
grader which is a full model on the client but not on the server (just a dict)
"""
descriptor = modulestore('direct').get_course(course_key)
descriptor = modulestore().get_course(course_key)
# parse removes the id; so, grab it before parse
index = int(grader.get('id', len(descriptor.raw_grader)))
@@ -82,7 +82,7 @@ class CourseGradingModel(object):
else:
descriptor.raw_grader.append(grader)
modulestore('direct').update_item(descriptor, user.id)
modulestore().update_item(descriptor, user.id)
return CourseGradingModel.jsonize_grader(index, descriptor.raw_grader[index])
@@ -92,10 +92,10 @@ class CourseGradingModel(object):
Create or update the grade cutoffs for the given course. Returns sent in cutoffs (ie., no extra
db fetch).
"""
descriptor = modulestore('direct').get_course(course_key)
descriptor = modulestore().get_course(course_key)
descriptor.grade_cutoffs = cutoffs
modulestore('direct').update_item(descriptor, user.id)
modulestore().update_item(descriptor, user.id)
return cutoffs
@@ -106,7 +106,7 @@ class CourseGradingModel(object):
grace_period entry in an enclosing dict. It is also safe to call this method with a value of
None for graceperiodjson.
"""
descriptor = modulestore('direct').get_course(course_key)
descriptor = modulestore().get_course(course_key)
# Before a graceperiod has ever been created, it will be None (once it has been
# created, it cannot be set back to None).
@@ -117,14 +117,14 @@ class CourseGradingModel(object):
grace_timedelta = timedelta(**graceperiodjson)
descriptor.graceperiod = grace_timedelta
modulestore('direct').update_item(descriptor, user.id)
modulestore().update_item(descriptor, user.id)
@staticmethod
def delete_grader(course_key, index, user):
"""
Delete the grader of the given type from the given course.
"""
descriptor = modulestore('direct').get_course(course_key)
descriptor = modulestore().get_course(course_key)
index = int(index)
if index < len(descriptor.raw_grader):
@@ -132,22 +132,22 @@ class CourseGradingModel(object):
# force propagation to definition
descriptor.raw_grader = descriptor.raw_grader
modulestore('direct').update_item(descriptor, user.id)
modulestore().update_item(descriptor, user.id)
@staticmethod
def delete_grace_period(course_key, user):
"""
Delete the course's grace period.
"""
descriptor = modulestore('direct').get_course(course_key)
descriptor = modulestore().get_course(course_key)
del descriptor.graceperiod
modulestore('direct').update_item(descriptor, user.id)
modulestore().update_item(descriptor, user.id)
@staticmethod
def get_section_grader_type(location):
descriptor = modulestore('direct').get_item(location)
descriptor = modulestore().get_item(location)
return {
"graderType": descriptor.format if descriptor.format is not None else 'notgraded',
"location": unicode(location),
@@ -162,7 +162,7 @@ class CourseGradingModel(object):
del descriptor.format
del descriptor.graded
modulestore('direct').update_item(descriptor, user.id)
modulestore().update_item(descriptor, user.id)
return {'graderType': grader_type}
@staticmethod

6
cms/djangoapps/models/settings/course_metadata.py
View File

@@ -1,9 +1,7 @@
from xblock.fields import Scope
from contentstore.utils import get_modulestore
from xmodule.modulestore.django import modulestore
from django.utils.translation import ugettext as _
class CourseMetadata(object):
'''
For CRUD operations on metadata fields which do not have specific editors
@@ -86,6 +84,6 @@ class CourseMetadata(object):
setattr(descriptor, key, value)
if len(key_values) > 0:
get_modulestore(descriptor.location).update_item(descriptor, user.id if user else None)
modulestore().update_item(descriptor, user.id if user else None)
return cls.fetch(descriptor)

36
cms/envs/acceptance.py
View File

@@ -34,35 +34,17 @@ logging.getLogger('track.middleware').setLevel(logging.CRITICAL)
# use, so these are not set up for TOS and PRIVACY
logging.getLogger('edxmako.shortcuts').setLevel(logging.ERROR)
DOC_STORE_CONFIG = {
'host': 'localhost',
'db': 'acceptance_xmodule',
'collection': 'acceptance_modulestore_%s' % seed(),
}
MODULESTORE_OPTIONS = {
'default_class': 'xmodule.raw_module.RawDescriptor',
'fs_root': TEST_ROOT / "data",
'render_template': 'edxmako.shortcuts.render_to_string',
}
MODULESTORE = {
'default': {
'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': MODULESTORE_OPTIONS
update_module_store_settings(
MODULESTORE,
doc_store_settings={
'db': 'acceptance_xmodule',
'collection': 'acceptance_modulestore_%s' % seed(),
},
'direct': {
'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': MODULESTORE_OPTIONS
},
'draft': {
'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': MODULESTORE_OPTIONS
module_store_options={
'default_class': 'xmodule.raw_module.RawDescriptor',
'fs_root': TEST_ROOT / "data",
}
}
)
CONTENTSTORE = {
'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',

2
cms/envs/aws.py
View File

@@ -229,7 +229,7 @@ if AWS_SECRET_ACCESS_KEY == "":
AWS_SECRET_ACCESS_KEY = None
DATABASES = AUTH_TOKENS['DATABASES']
MODULESTORE = AUTH_TOKENS['MODULESTORE']
MODULESTORE = convert_module_store_setting_if_needed(AUTH_TOKENS['MODULESTORE'])
CONTENTSTORE = AUTH_TOKENS['CONTENTSTORE']
DOC_STORE_CONFIG = AUTH_TOKENS['DOC_STORE_CONFIG']
# Datadog for events!

88
cms/envs/bok_choy.auth.json
View File

@@ -48,55 +48,47 @@
},
"MODULESTORE": {
"default": {
"DOC_STORE_CONFIG": {
"collection": "modulestore",
"db": "test",
"host": [
"localhost"
],
"password": "password",
"port": 27017,
"user": "edxapp"
},
"ENGINE": "xmodule.modulestore.mongo.DraftMongoModuleStore",
"ENGINE": "xmodule.modulestore.mixed.MixedModuleStore",
"OPTIONS": {
"collection": "modulestore",
"db": "test",
"default_class": "xmodule.hidden_module.HiddenDescriptor",
"fs_root": "** OVERRIDDEN **",
"host": [
"localhost"
],
"password": "password",
"port": 27017,
"render_template": "edxmako.shortcuts.render_to_string",
"user": "edxapp"
}
},
"direct": {
"DOC_STORE_CONFIG": {
"collection": "modulestore",
"db": "test",
"host": [
"localhost"
],
"password": "password",
"port": 27017,
"user": "edxapp"
},
"ENGINE": "xmodule.modulestore.mongo.MongoModuleStore",
"OPTIONS": {
"collection": "modulestore",
"db": "test",
"default_class": "xmodule.hidden_module.HiddenDescriptor",
"fs_root": "** OVERRIDDEN **",
"host": [
"localhost"
],
"password": "password",
"port": 27017,
"render_template": "edxmako.shortcuts.render_to_string",
"user": "edxapp"
"mappings": {},
"reference_type": "Location",
"stores": [
{
"NAME": "draft",
"DOC_STORE_CONFIG": {
"collection": "modulestore",
"db": "test",
"host": [
"localhost"
],
"password": "password",
"port": 27017,
"user": "edxapp"
},
"ENGINE": "xmodule.modulestore.mongo.DraftMongoModuleStore",
"OPTIONS": {
"collection": "modulestore",
"db": "test",
"default_class": "xmodule.hidden_module.HiddenDescriptor",
"fs_root": "** OVERRIDDEN **",
"host": [
"localhost"
],
"password": "password",
"port": 27017,
"render_template": "edxmako.shortcuts.render_to_string",
"user": "edxapp"
}
},
{
"NAME": "xml",
"ENGINE": "xmodule.modulestore.xml.XMLModuleStore",
"OPTIONS": {
"data_dir": "** OVERRIDDEN **",
"default_class": "xmodule.hidden_module.HiddenDescriptor"
}
}
]
}
}
},

13
cms/envs/bok_choy.py
View File

@@ -27,9 +27,16 @@ TEST_ROOT = CONFIG_ROOT.dirname().dirname() / "test_root" # pylint: disable=E11
GITHUB_REPO_ROOT = (TEST_ROOT / "data").abspath()
LOG_DIR = (TEST_ROOT / "log").abspath()
# Configure Mongo modulestore to use the test folder within the repo
for store in ["default", "direct"]:
MODULESTORE[store]['OPTIONS']['fs_root'] = (TEST_ROOT / "data").abspath() # pylint: disable=E1120
# Configure modulestore to use the test folder within the repo
update_module_store_settings(
MODULESTORE,
module_store_options={
'fs_root': (TEST_ROOT / "data").abspath(), # pylint: disable=E1120
},
xml_store_options={
'data_dir': (TEST_ROOT / "data").abspath(),
},
)
# Enable django-pipeline and staticfiles
STATIC_ROOT = (TEST_ROOT / "staticfiles").abspath()

7
cms/envs/common.py
View File

@@ -27,10 +27,12 @@ Longer TODO:
import imp
import sys
import lms.envs.common
# Although this module itself may not use these imported variables, other dependent modules may.
from lms.envs.common import (
USE_TZ, TECH_SUPPORT_EMAIL, PLATFORM_NAME, BUGS_EMAIL, DOC_STORE_CONFIG, ALL_LANGUAGES, WIKI_ENABLED
USE_TZ, TECH_SUPPORT_EMAIL, PLATFORM_NAME, BUGS_EMAIL, DOC_STORE_CONFIG, ALL_LANGUAGES, WIKI_ENABLED, MODULESTORE
)
from path import path
from lms.envs.modulestore_settings import *
from lms.lib.xblock.mixin import LmsBlockMixin
from dealer.git import git
@@ -244,6 +246,9 @@ XBLOCK_MIXINS = (LmsBlockMixin, InheritanceMixin, XModuleMixin)
# xblocks can be added via advanced settings
XBLOCK_SELECT_FUNCTION = prefer_xmodules
############################ Modulestore Configuration ################################
MODULESTORE_BRANCH = 'draft'
############################ DJANGO_BUILTINS ################################
# Change DEBUG/TEMPLATE_DEBUG in your environment settings files, not here
DEBUG = False

29
cms/envs/dev.py
View File

@@ -19,30 +19,13 @@ LOGGING = get_logger_config(ENV_ROOT / "log",
dev_env=True,
debug=True)
modulestore_options = {
'default_class': 'xmodule.raw_module.RawDescriptor',
'fs_root': GITHUB_REPO_ROOT,
'render_template': 'edxmako.shortcuts.render_to_string',
}
MODULESTORE = {
'default': {
'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': modulestore_options
},
'direct': {
'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': modulestore_options
},
'split': {
'ENGINE': 'xmodule.modulestore.split_mongo.SplitMongoModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': modulestore_options
update_module_store_settings(
MODULESTORE,
module_store_options={
'default_class': 'xmodule.raw_module.RawDescriptor',
'fs_root': GITHUB_REPO_ROOT,
}
}
)
# cdodge: This is the specifier for the MongoDB (using GridFS) backed static content store
# This is for static content for courseware, not system static content (e.g. javascript, css, edX branding, etc)

54
cms/envs/test.py
View File

@@ -16,6 +16,7 @@ from .common import *
import os
from path import path
from warnings import filterwarnings
from uuid import uuid4
# import settings from LMS for consistent behavior with CMS
from lms.envs.test import (WIKI_ENABLED, PLATFORM_NAME, SITE_NAME)
@@ -58,40 +59,29 @@ STATICFILES_DIRS += [
if os.path.isdir(COMMON_TEST_DATA_ROOT / course_dir)
]
DOC_STORE_CONFIG = {
'host': 'localhost',
'db': 'test_xmodule',
'collection': 'test_modulestore',
}
MODULESTORE_OPTIONS = {
'default_class': 'xmodule.raw_module.RawDescriptor',
'fs_root': TEST_ROOT / "data",
'render_template': 'edxmako.shortcuts.render_to_string',
}
MODULESTORE = {
'default': {
'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': MODULESTORE_OPTIONS
},
'direct': {
'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': MODULESTORE_OPTIONS
},
'draft': {
'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': MODULESTORE_OPTIONS
},
'split': {
# Add split as another store for testing
MODULESTORE['default']['OPTIONS']['stores'].append(
{
'NAME': 'split',
'ENGINE': 'xmodule.modulestore.split_mongo.SplitMongoModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': MODULESTORE_OPTIONS
}
}
'OPTIONS': {
'render_template': 'edxmako.shortcuts.render_to_string',
}
},
)
# Update module store settings per defaults for tests
update_module_store_settings(
MODULESTORE,
module_store_options={
'default_class': 'xmodule.raw_module.RawDescriptor',
'fs_root': TEST_ROOT / "data",
},
doc_store_settings={
'db': 'test_xmodule',
'collection': 'test_modulestore{0}'.format(uuid4().hex[:5]),
},
)
CONTENTSTORE = {
'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',

2
cms/static/coffee/src/views/unit.coffee
View File

@@ -159,7 +159,7 @@ define ["jquery", "jquery.ui", "gettext", "backbone",
@wait(true)
$.ajax({
type: 'DELETE',
url: @model.url() + "?" + $.param({recurse: true})
url: @model.url()
}).success(=>
analytics.track "Deleted Draft",

2
cms/static/js/base.js
View File

@@ -284,7 +284,7 @@ function _deleteItem($el, type) {
$.ajax({
type: 'DELETE',
url: ModuleUtils.getUpdateUrl(locator) +'?'+ $.param({recurse: true, all_versions: true}),
url: ModuleUtils.getUpdateUrl(locator),
success: function () {
$el.remove();
deleting.hide();

3
cms/static/js/views/pages/container.js
View File

@@ -161,8 +161,7 @@ define(["jquery", "underscore", "gettext", "js/views/baseview", "js/views/contai
return $.ajax({
type: 'DELETE',
url: self.getURLRoot() + "/" +
xblockElement.data('locator') + "?" +
$.param({recurse: true, all_versions: false})
xblockElement.data('locator')
}).success(_.bind(self.onDelete, self, xblockElement));
});
});

2
cms/templates/container.html
View File

@@ -2,7 +2,7 @@
<%!
import json
from contentstore.utils import PublishState
from xmodule.modulestore import PublishState
from contentstore.views.helpers import xblock_studio_url, EDITING_TEMPLATES
from django.utils.translation import ugettext as _
%>

13
common/djangoapps/contentserver/tests/test.py
View File

@@ -17,8 +17,7 @@ from student.models import CourseEnrollment
from xmodule.contentstore.django import contentstore, _CONTENTSTORE
from xmodule.modulestore.django import modulestore
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from xmodule.modulestore.tests.django_utils import (studio_store_config,
ModuleStoreTestCase)
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.xml_importer import import_from_xml
log = logging.getLogger(__name__)
@@ -26,10 +25,8 @@ log = logging.getLogger(__name__)
TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex
TEST_MODULESTORE = studio_store_config(settings.TEST_ROOT / "data")
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class ContentStoreToyCourseTest(ModuleStoreTestCase):
"""
Tests that use the toy course.
@@ -39,16 +36,12 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
"""
Create user and login.
"""
settings.MODULESTORE['default']['OPTIONS']['fs_root'] = path('common/test/data')
settings.MODULESTORE['direct']['OPTIONS']['fs_root'] = path('common/test/data')
self.client = Client()
self.contentstore = contentstore()
self.course_key = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
import_from_xml(modulestore('direct'), 'common/test/data/', ['toy'],
import_from_xml(modulestore(), '**replace_user**', 'common/test/data/', ['toy'],
static_content_store=self.contentstore, verbose=True)
# A locked asset

4
common/djangoapps/external_auth/tests/test_shib.py
View File

@@ -18,7 +18,7 @@ from django.utils.importlib import import_module
from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, mixed_store_config
from xmodule.modulestore.inheritance import own_metadata
from xmodule.modulestore.django import editable_modulestore
from xmodule.modulestore.django import modulestore
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from external_auth.models import ExternalAuthMap
@@ -80,7 +80,7 @@ class ShibSPTest(ModuleStoreTestCase):
request_factory = RequestFactory()
def setUp(self):
self.store = editable_modulestore()
self.store = modulestore()
@unittest.skipUnless(settings.FEATURES.get('AUTH_USE_SHIB'), "AUTH_USE_SHIB not set")
def test_exception_shib_login(self):

4
common/djangoapps/heartbeat/tests/test_heartbeat.py
View File

@@ -9,9 +9,9 @@ import mock
from django.test.utils import override_settings
from django.conf import settings
from django.test.testcases import TestCase
from xmodule.modulestore.tests.django_utils import mongo_store_config
from xmodule.modulestore.tests.django_utils import draft_mongo_store_config
TEST_MODULESTORE = mongo_store_config(settings.TEST_ROOT / "data")
TEST_MODULESTORE = draft_mongo_store_config(settings.TEST_ROOT / "data")
@override_settings(MODULESTORE=TEST_MODULESTORE)
class HeartbeatTestCase(TestCase):

22
common/djangoapps/student/migrations/0036_access_roles_orgless.py
View File

@@ -24,27 +24,11 @@ class Migration(DataMigration):
"""
Converts group table entries for write access and beta_test roles to course access roles table.
"""
def get_modulestore(ms_type, key):
"""
Find the modulestore of the given type trying the key first
"""
try:
store = modulestore(key)
if isinstance(store, MixedModuleStore):
store = store.modulestores[key]
if store.get_modulestore_type(None) == ms_type:
return store
else:
return None
except KeyError:
return None
# Note: Remember to use orm['appname.ModelName'] rather than "from appname.models..."
loc_map_collection = loc_mapper().location_map
xml_ms = get_modulestore(XML_MODULESTORE_TYPE, 'xml')
mongo_ms = get_modulestore(MONGO_MODULESTORE_TYPE, 'default')
if mongo_ms is None:
mongo_ms = get_modulestore(MONGO_MODULESTORE_TYPE, 'direct')
mixed_ms = modulestore()
xml_ms = mixed_ms._get_modulestore_by_type(XML_MODULESTORE_TYPE)
mongo_ms = mixed_ms._get_modulestore_by_type(MONGO_MODULESTORE_TYPE)
query = Q(name__startswith='staff') | Q(name__startswith='instructor') | Q(name__startswith='beta_testers')
for group in orm['auth.Group'].objects.filter(query).exclude(name__contains="/").all():

21
common/djangoapps/student/tests/test_course_listing.py
View File

@@ -6,7 +6,8 @@ from mock import patch, Mock
from student.tests.factories import UserFactory
from student.roles import GlobalStaff
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, studio_store_config
from xmodule.modulestore import MONGO_MODULESTORE_TYPE
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from xmodule.modulestore.django import modulestore
@@ -14,13 +15,8 @@ from xmodule.error_module import ErrorDescriptor
from django.test.client import Client
from student.models import CourseEnrollment
from student.views import get_course_enrollment_pairs
from django.conf import settings
from django.test.utils import override_settings
TEST_MODULESTORE = studio_store_config(settings.TEST_ROOT / "data")
@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestCourseListing(ModuleStoreTestCase):
"""
Unit tests for getting the list of courses for a logged in user
@@ -44,8 +40,7 @@ class TestCourseListing(ModuleStoreTestCase):
course = CourseFactory.create(
org=course_location.org,
number=course_location.course,
run=course_location.run,
modulestore=modulestore('direct'),
run=course_location.run
)
CourseEnrollment.enroll(self.student, course.id)
@@ -84,7 +79,7 @@ class TestCourseListing(ModuleStoreTestCase):
self._create_course_with_access_groups(course_key)
with patch('xmodule.modulestore.mongo.base.MongoKeyValueStore', Mock(side_effect=Exception)):
self.assertIsInstance(modulestore('direct').get_course(course_key), ErrorDescriptor)
self.assertIsInstance(modulestore().get_course(course_key), ErrorDescriptor)
# get courses through iterating all courses
courses_list = list(get_course_enrollment_pairs(self.student, None, []))
@@ -95,18 +90,20 @@ class TestCourseListing(ModuleStoreTestCase):
Create good courses, courses that won't load, and deleted courses which still have
roles. Test course listing.
"""
mongo_store = modulestore()._get_modulestore_by_type(MONGO_MODULESTORE_TYPE)
good_location = SlashSeparatedCourseKey('testOrg', 'testCourse', 'RunBabyRun')
self._create_course_with_access_groups(good_location)
course_location = SlashSeparatedCourseKey('testOrg', 'doomedCourse', 'RunBabyRun')
self._create_course_with_access_groups(course_location)
modulestore('direct').delete_course(course_location)
mongo_store.delete_course(course_location)
course_location = SlashSeparatedCourseKey('testOrg', 'erroredCourse', 'RunBabyRun')
course = self._create_course_with_access_groups(course_location)
course_db_record = modulestore('direct')._find_one(course.location)
course_db_record = mongo_store._find_one(course.location)
course_db_record.setdefault('metadata', {}).get('tabs', []).append({"type": "wiko", "name": "Wiki" })
modulestore('direct').collection.update(
mongo_store.collection.update(
{'_id': course.location.to_deprecated_son()},
{'$set': {
'metadata.tabs': course_db_record['metadata']['tabs'],

4
common/djangoapps/student/tests/test_login.py
View File

@@ -17,7 +17,7 @@ from student.views import _parse_course_id_from_string, _get_course_enrollment_d
from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, mixed_store_config
from xmodule.modulestore.inheritance import own_metadata
from xmodule.modulestore.django import editable_modulestore
from xmodule.modulestore.django import modulestore
from external_auth.models import ExternalAuthMap
from opaque_keys.edx.locations import SlashSeparatedCourseKey
@@ -289,7 +289,7 @@ class ExternalAuthShibTest(ModuleStoreTestCase):
Tests how login_user() interacts with ExternalAuth, in particular Shib
"""
def setUp(self):
self.store = editable_modulestore()
self.store = modulestore()
self.course = CourseFactory.create(org='Stanford', number='456', display_name='NO SHIB')
self.shib_course = CourseFactory.create(org='Stanford', number='123', display_name='Shib Only')
self.shib_course.enrollment_domain = 'shib:https://idp.stanford.edu/'

3
common/djangoapps/terrain/browser.py
View File

@@ -19,6 +19,7 @@ from json import dumps
from pymongo import MongoClient
import xmodule.modulestore.django
from xmodule.contentstore.django import _CONTENTSTORE
from xmodule.modulestore import MONGO_MODULESTORE_TYPE
# There is an import issue when using django-staticfiles with lettuce
# Lettuce assumes that we are using django.contrib.staticfiles,
@@ -189,7 +190,7 @@ def reset_databases(scenario):
mongo.drop_database(settings.CONTENTSTORE['DOC_STORE_CONFIG']['db'])
_CONTENTSTORE.clear()
modulestore = xmodule.modulestore.django.editable_modulestore()
modulestore = xmodule.modulestore.django.modulestore()._get_modulestore_by_type(MONGO_MODULESTORE_TYPE)
modulestore.collection.drop()
xmodule.modulestore.django.clear_existing_modulestores()

6
common/djangoapps/terrain/course_helpers.py
View File

@@ -5,7 +5,8 @@ import urllib
from lettuce import world
from django.contrib.auth.models import User, Group
from student.models import CourseEnrollment
from xmodule.modulestore.django import editable_modulestore
from xmodule.modulestore.django import modulestore
from xmodule.modulestore import MONGO_MODULESTORE_TYPE
from xmodule.contentstore.django import contentstore
@@ -71,5 +72,6 @@ def clear_courses():
# (though it shouldn't), do this manually
# from the bash shell to drop it:
# $ mongo test_xmodule --eval "db.dropDatabase()"
editable_modulestore().collection.drop()
store = modulestore()._get_modulestore_by_type(MONGO_MODULESTORE_TYPE)
store.collection.drop()
contentstore().fs_files.drop()

2
common/lib/xmodule/xmodule/contentstore/content.py
View File

@@ -48,7 +48,7 @@ class StaticContent(object):
- course_key: the course that this asset belongs to
- path: is the name of the static asset
- revision: is the object's revision information
- is_tumbnail: is whether or not we want the thumbnail version of this
- is_thumbnail: is whether or not we want the thumbnail version of this
asset
"""
path = path.replace('/', '_')

1
common/lib/xmodule/xmodule/contentstore/mongo.py
View File

@@ -12,7 +12,6 @@ from xmodule.exceptions import NotFoundError
from fs.osfs import OSFS
import os
import json
import bson.son
from bson.son import SON
from opaque_keys.edx.locations import AssetLocation

2
common/lib/xmodule/xmodule/error_module.py
View File

@@ -13,7 +13,7 @@ from xmodule.x_module import XModule, XModuleDescriptor
from xmodule.errortracker import exc_info_to_str
from xblock.fields import String, Scope, ScopeIds
from xblock.field_data import DictFieldData
from xmodule.modulestore.xml_exporter import EdxJSONEncoder
from xmodule.modulestore import EdxJSONEncoder
log = logging.getLogger(__name__)

122
common/lib/xmodule/xmodule/modulestore/__init__.py
View File

@@ -5,6 +5,8 @@ that are stored in a database an accessible using their Location as an identifie
import logging
import re
import json
import datetime
from collections import namedtuple, defaultdict
import collections
@@ -20,15 +22,60 @@ from opaque_keys import InvalidKeyError
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from xblock.runtime import Mixologist
from xblock.core import XBlock
import datetime
log = logging.getLogger('edx.modulestore')
# Modulestore Types
SPLIT_MONGO_MODULESTORE_TYPE = 'split'
MONGO_MODULESTORE_TYPE = 'mongo'
XML_MODULESTORE_TYPE = 'xml'
# Key Revision constants to use for Location and Usage Keys
# Note: These values are persisted in the database, so should not be changed without migrations
KEY_REVISION_DRAFT = 'draft'
KEY_REVISION_PUBLISHED = None
# Revision constants to use for Module Store operations
# Note: These values are passed into store APIs and only used at run time
# both DRAFT and PUBLISHED versions are queried, with preference to DRAFT versions
REVISION_OPTION_DRAFT_PREFERRED = 'rev-opt-draft-preferred'
# only DRAFT versions are queried and no PUBLISHED versions
REVISION_OPTION_DRAFT_ONLY = 'rev-opt-draft-only'
# # only PUBLISHED versions are queried and no DRAFT versions
REVISION_OPTION_PUBLISHED_ONLY = 'rev-opt-published-only'
# all revisions are queried
REVISION_OPTION_ALL = 'rev-opt-all'
# Branch constants to use for stores, such as Mongo, that have only 2 branches: DRAFT and PUBLISHED
# Note: These values are taken from server configuration settings, so should not be changed without alerting DevOps
BRANCH_DRAFT_PREFERRED = 'draft'
BRANCH_PUBLISHED_ONLY = 'published'
# Branch constants to use for stores, such as Split, that have named branches
BRANCH_NAME_DRAFT = 'draft'
BRANCH_NAME_PUBLISHED = 'published'
class PublishState(object):
"""
The publish state for a given xblock-- either 'draft', 'private', or 'public'.
Currently in CMS, an xblock can only be in 'draft' or 'private' if it is at or below the Unit level.
"""
draft = 'draft'
private = 'private'
public = 'public'
class ModuleStoreRead(object):
"""
An abstract interface for a database backend that stores XModuleDescriptor
@@ -184,11 +231,9 @@ class ModuleStoreRead(object):
pass
@abstractmethod
def get_parent_locations(self, location):
'''Find all locations that are the parents of this location in this
def get_parent_location(self, location, **kwargs):
'''Find the location that is the parent of this location in this
course. Needed for path_to_location().
returns an iterable of things that can be passed to Location.
'''
pass
@@ -245,11 +290,14 @@ class ModuleStoreWrite(ModuleStoreRead):
@abstractmethod
def delete_item(self, location, user_id=None, **kwargs):
"""
Delete an item from persistence. Pass the user's unique id which the persistent store
Delete an item and its subtree from persistence. Remove the item from any parents (Note, does not
affect parents from other branches or logical branches; thus, in old mongo, deleting something
whose parent cannot be draft, deletes it from both but deleting a component under a draft vertical
only deletes it from the draft.
Pass the user's unique id which the persistent store
should save with the update if it has that ability.
:param delete_all_versions: removes both the draft and published version of this item from
the course if using draft and old mongo. Split may or may not implement this.
:param force: fork the structure and don't update the course draftVersion if there's a version
conflict (only applicable to version tracking and conflict detecting persistence stores)
@@ -346,7 +394,7 @@ class ModuleStoreReadBase(ModuleStoreRead):
def has_course(self, course_id, ignore_case=False):
"""
Look for a specific course id. Returns whether it exists.
Returns the course_id of the course if it was found, else None
Args:
course_id (CourseKey):
ignore_case (boolean): some modulestores are case-insensitive. Use this flag
@@ -355,12 +403,18 @@ class ModuleStoreReadBase(ModuleStoreRead):
# linear search through list
assert(isinstance(course_id, CourseKey))
if ignore_case:
return any(
(c.id.org.lower() == course_id.org.lower() and c.id.offering.lower() == course_id.offering.lower())
for c in self.get_courses()
return next(
(
c.id for c in self.get_courses()
if c.id.org.lower() == course_id.org.lower() and c.id.offering.lower() == course_id.offering.lower()
),
None
)
else:
return any(c.id == course_id for c in self.get_courses())
return next(
(c.id for c in self.get_courses() if c.id == course_id),
None
)
def heartbeat(self):
"""
@@ -411,13 +465,12 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
"""
raise NotImplementedError
def delete_item(self, location, user_id=None, delete_all_versions=False, delete_children=False, force=False):
def delete_item(self, location, user_id=None, force=False):
"""
Delete an item from persistence. Pass the user's unique id which the persistent store
should save with the update if it has that ability.
:param delete_all_versions: removes both the draft and published version of this item from
the course if using draft and old mongo. Split may or may not implement this.
:param user_id: ID of the user deleting the item
:param force: fork the structure and don't update the course draftVersion if there's a version
conflict (only applicable to version tracking and conflict detecting persistence stores)
@@ -426,6 +479,21 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
"""
raise NotImplementedError
def create_and_save_xmodule(self, location, user_id, definition_data=None, metadata=None, runtime=None, fields={}):
"""
Create the new xmodule and save it.
:param location: a Location--must have a category
:param user_id: ID of the user creating and saving the xmodule
:param definition_data: can be empty. The initial definition_data for the kvs
:param metadata: can be empty, the initial metadata for the kvs
:param runtime: if you already have an xblock from the course, the xblock.runtime value
:param fields: a dictionary of field names and values for the new xmodule
"""
new_object = self.create_xmodule(location, definition_data, metadata, runtime, fields)
self.update_item(new_object, user_id, allow_not_found=True)
return new_object
def only_xmodules(identifier, entry_points):
"""Only use entry_points that are supplied by the xmodule package"""
@@ -441,3 +509,25 @@ def prefer_xmodules(identifier, entry_points):
return default_select(identifier, from_xmodule)
else:
return default_select(identifier, entry_points)
class EdxJSONEncoder(json.JSONEncoder):
"""
Custom JSONEncoder that handles `Location` and `datetime.datetime` objects.
`Location`s are encoded as their url string form, and `datetime`s as
ISO date strings
"""
def default(self, obj):
if isinstance(obj, Location):
return obj.to_deprecated_string()
elif isinstance(obj, datetime.datetime):
if obj.tzinfo is not None:
if obj.utcoffset() is None:
return obj.isoformat() + 'Z'
else:
return obj.isoformat()
else:
return obj.isoformat()
else:
return super(EdxJSONEncoder, self).default(obj)

175
common/lib/xmodule/xmodule/modulestore/django.py
View File

@@ -5,16 +5,18 @@ Passes settings.MODULESTORE as kwargs to MongoModuleStore
"""
from __future__ import absolute_import
from importlib import import_module
import re
from importlib import import_module
from django.conf import settings
from django.core.cache import get_cache, InvalidCacheBackendError
import django.utils
import re
import threading
from xmodule.modulestore.loc_mapper_store import LocMapperStore
from xmodule.util.django import get_current_request_hostname
import xmodule.modulestore # pylint: disable=unused-import
# We may not always have the request_cache module available
try:
@@ -23,10 +25,6 @@ try:
except ImportError:
HAS_REQUEST_CACHE = False
_MODULESTORES = {}
FUNCTION_KEYS = ['render_template']
def load_function(path):
"""
@@ -48,6 +46,7 @@ def create_modulestore_instance(engine, doc_store_config, options, i18n_service=
_options = {}
_options.update(options)
FUNCTION_KEYS = ['render_template']
for key in FUNCTION_KEYS:
if key in _options and isinstance(_options[key], basestring):
_options[key] = load_function(_options[key])
@@ -69,59 +68,50 @@ def create_modulestore_instance(engine, doc_store_config, options, i18n_service=
xblock_select=getattr(settings, 'XBLOCK_SELECT_FUNCTION', None),
doc_store_config=doc_store_config,
i18n_service=i18n_service or ModuleI18nService(),
branch_setting_func=_get_modulestore_branch_setting,
**_options
)
def get_default_store_name_for_current_request():
# A singleton instance of the Mixed Modulestore
_MIXED_MODULESTORE = None
def modulestore():
"""
This method will return the appropriate default store mapping for the current Django request,
else 'default' which is the system default
Returns the Mixed modulestore
"""
store_name = 'default'
# see what request we are currently processing - if any at all - and get hostname for the request
hostname = get_current_request_hostname()
# get mapping information which is defined in configurations
mappings = getattr(settings, 'HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS', None)
# compare hostname against the regex expressions set of mappings
# which will tell us which store name to use
if hostname and mappings:
for key in mappings.keys():
if re.match(key, hostname):
store_name = mappings[key]
return store_name
return store_name
def modulestore(name=None):
"""
This returns an instance of a modulestore of given name. This will wither return an existing
modulestore or create a new one
"""
if not name:
# If caller did not specify name then we should
# determine what should be the default
name = get_default_store_name_for_current_request()
if name not in _MODULESTORES:
_MODULESTORES[name] = create_modulestore_instance(
settings.MODULESTORE[name]['ENGINE'],
settings.MODULESTORE[name].get('DOC_STORE_CONFIG', {}),
settings.MODULESTORE[name].get('OPTIONS', {})
global _MIXED_MODULESTORE # pylint: disable=global-statement
if _MIXED_MODULESTORE is None:
_MIXED_MODULESTORE = create_modulestore_instance(
settings.MODULESTORE['default']['ENGINE'],
settings.MODULESTORE['default'].get('DOC_STORE_CONFIG', {}),
settings.MODULESTORE['default'].get('OPTIONS', {})
)
# inject loc_mapper into newly created modulestore if it needs it
if name == 'split' and _loc_singleton is not None:
_MODULESTORES['split'].loc_mapper = _loc_singleton
return _MODULESTORES[name]
return _MIXED_MODULESTORE
def clear_existing_modulestores():
"""
Clear the existing modulestore instances, causing
them to be re-created when accessed again.
This is useful for flushing state between unit tests.
"""
global _MIXED_MODULESTORE, _loc_singleton # pylint: disable=global-statement
_MIXED_MODULESTORE = None
# pylint: disable=W0603
cache = getattr(_loc_singleton, "cache", None)
if cache:
cache.clear()
_loc_singleton = None
# singleton instance of the loc_mapper
_loc_singleton = None
def loc_mapper():
"""
Get the loc mapper which bidirectionally maps Locations to Locators. Used like modulestore() as
@@ -137,58 +127,10 @@ def loc_mapper():
loc_cache = get_cache('default')
# instantiate
_loc_singleton = LocMapperStore(loc_cache, **settings.DOC_STORE_CONFIG)
# inject into split mongo modulestore
if 'split' in _MODULESTORES:
_MODULESTORES['split'].loc_mapper = _loc_singleton
return _loc_singleton
def clear_existing_modulestores():
"""
Clear the existing modulestore instances, causing
them to be re-created when accessed again.
This is useful for flushing state between unit tests.
"""
_MODULESTORES.clear()
# pylint: disable=W0603
global _loc_singleton
cache = getattr(_loc_singleton, "cache", None)
if cache:
cache.clear()
_loc_singleton = None
def editable_modulestore(name='default'):
"""
Retrieve a modulestore that we can modify.
This is useful for tests that need to insert test
data into the modulestore.
Currently, only Mongo-backed modulestores can be modified.
Returns `None` if no editable modulestore is available.
"""
# Try to retrieve the ModuleStore
# Depending on the settings, this may or may not
# be editable.
store = modulestore(name)
# If this is a `MixedModuleStore`, then we will need
# to retrieve the actual Mongo instance.
# We assume that the default is Mongo.
if hasattr(store, 'modulestores'):
store = store.modulestores['default']
# At this point, we either have the ability to create
# items in the store, or we do not.
if hasattr(store, 'create_course'):
return store
else:
return None
class ModuleI18nService(object):
"""
Implement the XBlock runtime "i18n" service.
@@ -217,3 +159,40 @@ class ModuleI18nService(object):
# then Cale was a liar.
from util.date_utils import strftime_localized
return strftime_localized(*args, **kwargs)
# thread local cache
_THREAD_CACHE = threading.local()
def _get_modulestore_branch_setting():
"""
Returns the branch setting for the module store from the current Django request if configured,
else returns the branch value from the configuration settings if set,
else returns None
The value of the branch setting is cached in a thread-local variable so it is not repeatedly recomputed
"""
def get_branch_setting():
"""
Finds and returns the branch setting based on the Django request and the configuration settings
"""
branch = None
hostname = get_current_request_hostname()
if hostname:
# get mapping information which is defined in configurations
mappings = getattr(settings, 'HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS', None)
# compare hostname against the regex expressions set of mappings which will tell us which branch to use
if mappings:
for key in mappings.iterkeys():
if re.match(key, hostname):
return mappings[key]
if branch is None:
branch = getattr(settings, 'MODULESTORE_BRANCH', None)
return branch
# cache the branch setting for this thread so we don't have to recompute it each time
if not hasattr(_THREAD_CACHE, 'branch_setting'):
_THREAD_CACHE.branch_setting = get_branch_setting()
return _THREAD_CACHE.branch_setting

19
common/lib/xmodule/xmodule/modulestore/exceptions.py
View File

@@ -27,6 +27,15 @@ class NoPathToItem(Exception):
pass
class ReferentialIntegrityError(Exception):
"""
An incorrect pointer to an object exists. For example, 2 parents point to the same child, an
xblock points to a nonexistent child (which probably raises ItemNotFoundError instead depending
on context).
"""
pass
class DuplicateItemError(Exception):
"""
Attempted to create an item which already exists.
@@ -66,3 +75,13 @@ class DuplicateCourseError(Exception):
super(DuplicateCourseError, self).__init__()
self.course_id = course_id
self.existing_entry = existing_entry
class InvalidBranchSetting(Exception):
"""
Raised when the process' branch setting did not match the required setting for the attempted operation on a store.
"""
def __init__(self, expected_setting, actual_setting):
super(InvalidBranchSetting, self).__init__()
self.expected_setting = expected_setting
self.actual_setting = actual_setting

7
common/lib/xmodule/xmodule/modulestore/loc_mapper_store.py
View File

@@ -7,6 +7,7 @@ import pymongo
import bson.son
import urllib
from xmodule.modulestore import BRANCH_NAME_PUBLISHED, BRANCH_NAME_DRAFT
from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError
from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator
from opaque_keys.edx.locations import SlashSeparatedCourseKey
@@ -53,8 +54,8 @@ class LocMapperStore(object):
self.cache = cache
# location_map functions
def create_map_entry(self, course_key, org=None, offering=None, draft_branch='draft', prod_branch='published',
block_map=None):
def create_map_entry(self, course_key, org=None, offering=None,
draft_branch=BRANCH_NAME_DRAFT, prod_branch=BRANCH_NAME_PUBLISHED, block_map=None):
"""
Add a new entry to map this SlashSeparatedCourseKey to the new style CourseLocator.org & offering. If
org and offering are not provided, it defaults them based on course_key.
@@ -244,7 +245,7 @@ class LocMapperStore(object):
for old_name, cat_to_usage in entry['block_map'].iteritems():
for category, block_id in cat_to_usage.iteritems():
# cache all entries and then figure out if we have the one we want
# Always return revision=None because the
# Always return revision=KEY_REVISION_PUBLISHED because the
# old draft module store wraps locations as draft before
# trying to access things.
location = old_course_id.make_usage_key(

324
common/lib/xmodule/xmodule/modulestore/mixed.py
View File

@@ -7,13 +7,13 @@ In this way, courses can be served up both - say - XMLModuleStore or MongoModule
import logging
from uuid import uuid4
from contextlib import contextmanager
from opaque_keys import InvalidKeyError
from . import ModuleStoreWriteBase
from xmodule.modulestore import PublishState
from xmodule.modulestore.django import create_modulestore_instance, loc_mapper
from opaque_keys.edx.locations import Location
from xmodule.modulestore import XML_MODULESTORE_TYPE
from opaque_keys.edx.locator import CourseLocator, Locator, BlockUsageLocator
from opaque_keys.edx.locator import CourseLocator, BlockUsageLocator
from xmodule.modulestore.exceptions import ItemNotFoundError
from opaque_keys.edx.keys import CourseKey, UsageKey
from xmodule.modulestore.mongo.base import MongoModuleStore
@@ -21,6 +21,7 @@ from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore
from opaque_keys.edx.locations import SlashSeparatedCourseKey
import itertools
log = logging.getLogger(__name__)
@@ -31,11 +32,11 @@ class MixedModuleStore(ModuleStoreWriteBase):
def __init__(self, mappings, stores, i18n_service=None, **kwargs):
"""
Initialize a MixedModuleStore. Here we look into our passed in kwargs which should be a
collection of other modulestore configuration informations
collection of other modulestore configuration information
"""
super(MixedModuleStore, self).__init__(**kwargs)
self.modulestores = {}
self.modulestores = []
self.mappings = {}
for course_id, store_name in mappings.iteritems():
@@ -48,51 +49,92 @@ class MixedModuleStore(ModuleStoreWriteBase):
log.exception("Invalid MixedModuleStore configuration. Unable to parse course_id %r", course_id)
continue
if 'default' not in stores:
raise Exception('Missing a default modulestore in the MixedModuleStore __init__ method.')
for key, store in stores.iteritems():
is_xml = 'XMLModuleStore' in store['ENGINE']
for store_settings in stores:
key = store_settings['NAME']
is_xml = 'XMLModuleStore' in store_settings['ENGINE']
if is_xml:
# restrict xml to only load courses in mapping
store['OPTIONS']['course_ids'] = [
store_settings['OPTIONS']['course_ids'] = [
course_key.to_deprecated_string()
for course_key, store_key in self.mappings.iteritems()
if store_key == key
]
self.modulestores[key] = create_modulestore_instance(
store['ENGINE'],
# XMLModuleStore's don't have doc store configs
store.get('DOC_STORE_CONFIG', {}),
store['OPTIONS'],
store = create_modulestore_instance(
store_settings['ENGINE'],
store_settings.get('DOC_STORE_CONFIG', {}),
store_settings.get('OPTIONS', {}),
i18n_service=i18n_service,
)
if key == 'split':
store.loc_mapper = loc_mapper()
# replace all named pointers to the store into actual pointers
for course_key, store_name in self.mappings.iteritems():
if store_name == key:
self.mappings[course_key] = store
self.modulestores.append(store)
def _get_modulestore_for_courseid(self, course_id):
def _clean_course_id_for_mapping(self, course_id):
"""
In order for mapping to work, the course_id must be minimal--no version, no branch--
as we never store one version or one branch in one ms and another in another ms.
:param course_id: the CourseKey
"""
if hasattr(course_id, 'version_agnostic'):
course_id = course_id.version_agnostic()
if hasattr(course_id, 'branch_agnostic'):
course_id = course_id.branch_agnostic()
return course_id
def _get_modulestore_for_courseid(self, course_id=None):
"""
For a given course_id, look in the mapping table and see if it has been pinned
to a particular modulestore
"""
# TODO when this becomes a router capable of handling more than one r/w backend
# we'll need to generalize this to handle mappings from old Locations w/o full
# course_id in much the same way as loc_mapper().translate_location does.
mapping = self.mappings.get(course_id, 'default')
return self.modulestores[mapping]
def has_item(self, usage_key):
If course_id is None, returns the first (ordered) store as the default
"""
if course_id is not None:
course_id = self._clean_course_id_for_mapping(course_id)
mapping = self.mappings.get(course_id, None)
if mapping is not None:
return mapping
else:
for store in self.modulestores:
if isinstance(course_id, store.reference_type) and store.has_course(course_id):
self.mappings[course_id] = store
return store
# return the first store, as the default
return self.modulestores[0]
def _get_modulestore_by_type(self, modulestore_type):
"""
This method should only really be used by tests and migration scripts when necessary.
Returns the module store as requested by type. The type can be:
SPLIT_MONGO_MODULESTORE_TYPE
MONGO_MODULESTORE_TYPE
XML_MODULESTORE_TYPE
"""
for store in self.modulestores:
if store.get_modulestore_type() == modulestore_type:
return store
return None
def has_item(self, usage_key, **kwargs):
"""
Does the course include the xblock who's id is reference?
"""
store = self._get_modulestore_for_courseid(usage_key.course_key)
return store.has_item(usage_key)
return store.has_item(usage_key, **kwargs)
def get_item(self, usage_key, depth=0):
def get_item(self, usage_key, depth=0, **kwargs):
"""
This method is explicitly not implemented as we need a course_id to disambiguate
We should be able to fix this when the data-model rearchitecting is done
"""
store = self._get_modulestore_for_courseid(usage_key.course_key)
return store.get_item(usage_key, depth)
return store.get_item(usage_key, depth, **kwargs)
def get_items(self, course_key, settings=None, content=None, **kwargs):
"""
@@ -126,36 +168,38 @@ class MixedModuleStore(ModuleStoreWriteBase):
def get_courses(self):
'''
Returns a list containing the top level XModuleDescriptors of the courses
in this modulestore.
Returns a list containing the top level XModuleDescriptors of the courses in this modulestore.
'''
# order the modulestores and ensure no dupes (default may be a dupe of a named store)
# remove 'draft' as we know it's a functional dupe of 'direct' (ugly hardcoding)
stores = set([value for key, value in self.modulestores.iteritems() if key != 'draft'])
stores = sorted(stores, cmp=_compare_stores)
courses = {} # a dictionary of course keys to course objects
# first populate with the ones in mappings as the mapping override discovery
for course_id, store in self.mappings.iteritems():
course = store.get_course(course_id)
# check if the course is not None - possible if the mappings file is outdated
# TODO - log an error if the course is None, but move it to an initialization method to keep it less noisy
if course is not None:
courses[course_id] = store.get_course(course_id)
has_locators = any(issubclass(CourseLocator, store.reference_type) for store in self.modulestores)
for store in self.modulestores:
courses = {} # a dictionary of stringified course locations to course objects
has_locators = any(issubclass(CourseLocator, store.reference_type) for store in stores)
for store in stores:
store_courses = store.get_courses()
# filter out ones which were fetched from earlier stores but locations may not be ==
for course in store_courses:
course_location = unicode(course.location)
if course_location not in courses:
if has_locators and isinstance(course.location, Location):
for course in store.get_courses():
course_id = self._clean_course_id_for_mapping(course.id)
if course_id not in courses:
if has_locators and isinstance(course_id, SlashSeparatedCourseKey):
# see if a locator version of course is in the result
try:
# if there's no existing mapping, then the course can't have been in split
course_locator = loc_mapper().translate_location(
course.location,
add_entry_if_missing=False
)
if unicode(course_locator) not in courses:
courses[course_location] = course
course_locator = loc_mapper().translate_location_to_course_locator(course_id)
if course_locator in courses:
continue
except ItemNotFoundError:
courses[course_location] = course
else:
courses[course_location] = course
# if there's no existing mapping, then the course can't have been in split
pass
# course is indeed unique. save it in result
courses[course_id] = course
return courses.values()
@@ -175,11 +219,13 @@ class MixedModuleStore(ModuleStoreWriteBase):
def has_course(self, course_id, ignore_case=False):
"""
returns whether the course exists
returns the course_id of the course if it was found, else None
Note: we return the course_id instead of a boolean here since the found course may have
a different id than the given course_id when ignore_case is True.
Args:
* course_id (CourseKey)
* ignore_case (bool): Tf True, do a case insensitive search. If
* ignore_case (bool): If True, do a case insensitive search. If
False, do a case sensitive search
"""
assert(isinstance(course_id, CourseKey))
@@ -192,14 +238,17 @@ class MixedModuleStore(ModuleStoreWriteBase):
"""
assert(isinstance(course_key, CourseKey))
store = self._get_modulestore_for_courseid(course_key)
return store.delete_course(course_key, user_id)
if hasattr(store, 'delete_course'):
return store.delete_course(course_key, user_id)
else:
raise NotImplementedError(u"Cannot delete a course on store {}".format(store))
def get_parent_locations(self, location):
def get_parent_location(self, location, **kwargs):
"""
returns the parent locations for a given location
"""
store = self._get_modulestore_for_courseid(location.course_key)
return store.get_parent_locations(location)
return store.get_parent_location(location, **kwargs)
def get_modulestore_type(self, course_id):
"""
@@ -209,7 +258,7 @@ class MixedModuleStore(ModuleStoreWriteBase):
"mongo" for old-style MongoDB backed courses,
"split" for new-style split MongoDB backed courses.
"""
return self._get_modulestore_for_courseid(course_id).get_modulestore_type(course_id)
return self._get_modulestore_for_courseid(course_id).get_modulestore_type()
def get_orphans(self, course_key):
"""
@@ -226,11 +275,11 @@ class MixedModuleStore(ModuleStoreWriteBase):
course_dir where course loading failed.
"""
errs = {}
for store in self.modulestores.values():
for store in self.modulestores:
errs.update(store.get_errored_courses())
return errs
def create_course(self, org, offering, user_id=None, fields=None, store_name='default', **kwargs):
def create_course(self, org, offering, user_id=None, fields=None, **kwargs):
"""
Creates and returns the course.
@@ -239,15 +288,14 @@ class MixedModuleStore(ModuleStoreWriteBase):
offering (str): the name of the course offering
user_id: id of the user creating the course
fields (dict): Fields to set on the course at initialization
store_name (str): the name of the modulestore that we will create this course within
kwargs: Any optional arguments understood by a subset of modulestores to customize instantiation
Returns: a CourseDescriptor
"""
store = self.modulestores[store_name]
store = self._get_modulestore_for_courseid(None)
if not hasattr(store, 'create_course'):
raise NotImplementedError(u"Cannot create a course on store %s" % store_name)
raise NotImplementedError(u"Cannot create a course on store {}".format(store))
return store.create_course(org, offering, user_id, fields, **kwargs)
@@ -273,7 +321,7 @@ class MixedModuleStore(ModuleStoreWriteBase):
if location is None:
location = course_id.make_usage_key(category, block_id)
# do the actual creation
xblock = store.create_and_save_xmodule(location, **kwargs)
xblock = self.create_and_save_xmodule(location, user_id, **kwargs)
# don't forget to attach to parent
if parent_loc is not None and not 'detached' in xblock._class_tags:
parent = store.get_item(parent_loc)
@@ -300,28 +348,39 @@ class MixedModuleStore(ModuleStoreWriteBase):
Update the xblock persisted to be the same as the given for all types of fields
(content, children, and metadata) attribute the change to the given user.
"""
course_id = xblock.scope_ids.usage_id.course_key
store = self._get_modulestore_for_courseid(course_id)
return store.update_item(xblock, user_id)
store = self._verify_modulestore_support(xblock.location, 'update_item')
return store.update_item(xblock, user_id, allow_not_found)
def delete_item(self, location, user_id=None, **kwargs):
"""
Delete the given item from persistence. kwargs allow modulestore specific parameters.
"""
course_id = location.course_key
store = self._get_modulestore_for_courseid(course_id)
return store.delete_item(location, user_id=user_id, **kwargs)
store = self._verify_modulestore_support(location, 'delete_item')
store.delete_item(location, user_id=user_id, **kwargs)
def close_all_connections(self):
"""
Close all db connections
"""
for mstore in self.modulestores.itervalues():
for mstore in self.modulestores:
if hasattr(mstore, 'database'):
mstore.database.connection.close()
elif hasattr(mstore, 'db'):
mstore.db.connection.close()
def create_xmodule(self, location, definition_data=None, metadata=None, runtime=None, fields={}):
"""
Create the new xmodule but don't save it. Returns the new module.
:param location: a Location--must have a category
:param definition_data: can be empty. The initial definition_data for the kvs
:param metadata: can be empty, the initial metadata for the kvs
:param runtime: if you already have an xblock from the course, the xblock.runtime value
:param fields: a dictionary of field names and values for the new xmodule
"""
store = self._verify_modulestore_support(location, 'create_xmodule')
return store.create_xmodule(location, definition_data, metadata, runtime, fields)
def get_courses_for_wiki(self, wiki_slug):
"""
Return the list of courses which use this wiki_slug
@@ -329,7 +388,7 @@ class MixedModuleStore(ModuleStoreWriteBase):
:return: list of course locations
"""
courses = []
for modulestore in self.modulestores.values():
for modulestore in self.modulestores:
courses.extend(modulestore.get_courses_for_wiki(wiki_slug))
return courses
@@ -345,29 +404,110 @@ class MixedModuleStore(ModuleStoreWriteBase):
)
)
def compute_publish_state(self, xblock):
"""
Returns whether this xblock is draft, public, or private.
def _compare_stores(left, right):
"""
Order stores via precedence: if a course is found in an earlier store, it shadows the later store.
xml stores take precedence b/c they only contain hardcoded mappings, then Locator-based ones,
then others. Locators before Locations because if some courses may be in both,
the ones in the Locator-based stores shadow the others.
"""
if left.get_modulestore_type(None) == XML_MODULESTORE_TYPE:
if right.get_modulestore_type(None) == XML_MODULESTORE_TYPE:
return 0
Returns:
PublishState.draft - content is in the process of being edited, but still has a previous
version deployed to LMS
PublishState.public - content is locked and deployed to LMS
PublishState.private - content is editable and not deployed to LMS
"""
course_id = xblock.scope_ids.usage_id.course_key
store = self._get_modulestore_for_courseid(course_id)
if hasattr(store, 'compute_publish_state'):
return store.compute_publish_state(xblock)
elif hasattr(store, 'publish'):
raise NotImplementedError(u"Cannot compute_publish_state on store {}".format(store))
else:
return -1
elif right.get_modulestore_type(None) == XML_MODULESTORE_TYPE:
return 1
# read-only store; so, everything's public
return PublishState.public
if issubclass(left.reference_type, Locator):
if issubclass(right.reference_type, Locator):
return 0
def publish(self, location, user_id):
"""
Save a current draft to the underlying modulestore
Returns the newly published item.
"""
store = self._verify_modulestore_support(location, 'publish')
return store.publish(location, user_id)
def unpublish(self, location, user_id):
"""
Save a current draft to the underlying modulestore
Returns the newly unpublished item.
"""
store = self._verify_modulestore_support(location, 'unpublish')
return store.unpublish(location, user_id)
def convert_to_draft(self, location, user_id):
"""
Create a copy of the source and mark its revision as draft.
Note: This method is to support the Mongo Modulestore and may be deprecated.
:param source: the location of the source (its revision must be None)
"""
store = self._verify_modulestore_support(location, 'convert_to_draft')
return store.convert_to_draft(location, user_id)
def _verify_modulestore_support(self, location, method):
"""
Finds and returns the store that contains the course for the given location, and verifying
that the store supports the given method.
Raises NotImplementedError if the found store does not support the given method.
"""
course_id = location.course_key
store = self._get_modulestore_for_courseid(course_id)
if hasattr(store, method):
return store
else:
return -1
elif issubclass(right.reference_type, Locator):
return 1
raise NotImplementedError(u"Cannot call {} on store {}".format(method, store))
return 0
@contextmanager
def store_branch_setting(store, branch_setting):
"""
A context manager for temporarily setting a store's branch value
Note: to be effective, the store must be a direct pointer to the underlying store;
not the intermediary Mixed store.
"""
assert not isinstance(store, MixedModuleStore)
try:
previous_branch_setting_func = store.branch_setting_func
store.branch_setting_func = lambda: branch_setting
yield
finally:
store.branch_setting_func = previous_branch_setting_func
@contextmanager
def store_bulk_write_operations_on_course(store, course_id):
"""
A context manager for notifying the store of bulk write events.
In the case of Mongo, it temporarily disables refreshing the metadata inheritance tree
until the bulk operation is completed.
The store can be either the Mixed modulestore or a direct pointer to the underlying store.
"""
# TODO
# Make this multi-process-safe if future operations need it.
# Right now, only Import Course, Clone Course, and Delete Course use this, so
# it's ok if the cached metadata in the memcache is invalid when another
# request comes in for the same course.
# if the caller passed in the mixed modulestore, get a direct pointer to the underlying store
if hasattr(store, '_get_modulestore_by_course_id'):
store = store._get_modulestore_by_course_id(course_id)
try:
if hasattr(store, 'begin_bulk_write_operation_on_course'):
store.begin_bulk_write_operation_on_course(course_id)
yield
finally:
if hasattr(store, 'begin_bulk_write_operation_on_course'):
store.end_bulk_write_operation_on_course(course_id)

276
common/lib/xmodule/xmodule/modulestore/mongo/base.py
View File

@@ -33,11 +33,14 @@ from xblock.runtime import KvsFieldData
from xblock.exceptions import InvalidScopeError
from xblock.fields import Scope, ScopeIds, Reference, ReferenceList, ReferenceValueDict
from xmodule.modulestore import ModuleStoreWriteBase, MONGO_MODULESTORE_TYPE
from xmodule.modulestore import (
ModuleStoreWriteBase, MONGO_MODULESTORE_TYPE,
REVISION_OPTION_PUBLISHED_ONLY, REVISION_OPTION_DRAFT_PREFERRED,
KEY_REVISION_DRAFT, KEY_REVISION_PUBLISHED
)
from opaque_keys.edx.locations import Location
from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError
from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError, ReferentialIntegrityError
from xmodule.modulestore.inheritance import own_metadata, InheritanceMixin, inherit_metadata, InheritanceKeyValueStore
from xmodule.tabs import StaticTab, CourseTabList
from xblock.core import XBlock
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from xmodule.exceptions import HeartbeatFailure
@@ -45,6 +48,16 @@ from xmodule.exceptions import HeartbeatFailure
log = logging.getLogger(__name__)
# Things w/ these categories should never be marked as version=DRAFT
DIRECT_ONLY_CATEGORIES = ['course', 'chapter', 'sequential', 'about', 'static_tab', 'course_info']
# sort order that returns DRAFT items first
SORT_REVISION_FAVOR_DRAFT = ('_id.revision', pymongo.DESCENDING)
# sort order that returns PUBLISHED items first
SORT_REVISION_FAVOR_PUBLISHED = ('_id.revision', pymongo.ASCENDING)
class InvalidWriteError(Exception):
"""
Raised to indicate that writing to a particular key
@@ -116,8 +129,6 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
"""
A system that has a cache of module json that it will use to load modules
from, with a backup of calling to the underlying modulestore for more data
TODO (cdodge) when the 'split module store' work has been completed we can remove all
references to metadata_inheritance_tree
"""
def __init__(self, modulestore, course_key, module_data, default_class, cached_metadata, **kwargs):
"""
@@ -188,7 +199,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
if isinstance(data, basestring):
data = {'data': data}
mixed_class = self.mixologist.mix(class_)
if data is not None:
if data: # empty or None means no work
data = self._convert_reference_fields_to_keys(mixed_class, location.course_key, data)
metadata = self._convert_reference_fields_to_keys(mixed_class, location.course_key, metadata)
kvs = MongoKeyValueStore(
@@ -203,7 +214,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
if self.cached_metadata is not None:
# parent container pointers don't differentiate between draft and non-draft
# so when we do the lookup, we should do so with a non-draft location
non_draft_loc = location.replace(revision=None)
non_draft_loc = as_published(location)
# Convert the serialized fields values in self.cached_metadata
# to python values
@@ -285,6 +296,23 @@ def location_to_query(location, wildcard=True, tag='i4x'):
return query
def as_draft(location):
"""
Returns the Location that is the draft for `location`
If the location is in the DIRECT_ONLY_CATEGORIES, returns itself
"""
if location.category in DIRECT_ONLY_CATEGORIES:
return location
return location.replace(revision=KEY_REVISION_DRAFT)
def as_published(location):
"""
Returns the Location that is the published version for `location`
"""
return location.replace(revision=KEY_REVISION_PUBLISHED)
class MongoModuleStore(ModuleStoreWriteBase):
"""
A Mongodb backed ModuleStore
@@ -342,8 +370,25 @@ class MongoModuleStore(ModuleStoreWriteBase):
self.render_template = render_template
self.i18n_service = i18n_service
# performance optimization to prevent updating the meta-data inheritance tree during
# bulk write operations
self.ignore_write_events_on_courses = set()
def begin_bulk_write_operation_on_course(self, course_id):
"""
Prevent updating the meta-data inheritance cache for the given course
"""
self.ignore_write_events_on_courses.add(course_id)
def end_bulk_write_operation_on_course(self, course_id):
"""
Restart updating the meta-data inheritance cache for the given course.
Refresh the meta-data inheritance cache now since it was temporarily disabled.
"""
if course_id in self.ignore_write_events_on_courses:
self.ignore_write_events_on_courses.remove(course_id)
self.refresh_cached_metadata_inheritance_tree(course_id)
def _compute_metadata_inheritance_tree(self, course_id):
'''
TODO (cdodge) This method can be deleted when the 'split module store' work has been completed
@@ -378,8 +423,8 @@ class MongoModuleStore(ModuleStoreWriteBase):
# now go through the results and order them by the location url
for result in resultset:
# manually pick it apart b/c the db has tag and we want revision = None regardless
location = Location._from_deprecated_son(result['_id'], course_id.run).replace(revision=None)
# manually pick it apart b/c the db has tag and we want as_published revision regardless
location = as_published(Location._from_deprecated_son(result['_id'], course_id.run))
location_url = location.to_deprecated_string()
if location_url in results_by_url:
@@ -388,7 +433,8 @@ class MongoModuleStore(ModuleStoreWriteBase):
additional_children = result.get('definition', {}).get('children', [])
total_children = existing_children + additional_children
results_by_url[location_url].setdefault('definition', {})['children'] = total_children
results_by_url[location_url] = result
else:
results_by_url[location_url] = result
if location.category == 'course':
root = location_url
@@ -421,7 +467,7 @@ class MongoModuleStore(ModuleStoreWriteBase):
def _get_cached_metadata_inheritance_tree(self, course_id, force_refresh=False):
'''
TODO (cdodge) This method can be deleted when the 'split module store' work has been completed
Compute the metadata inheritance for the course.
'''
tree = {}
@@ -434,7 +480,10 @@ class MongoModuleStore(ModuleStoreWriteBase):
if self.metadata_inheritance_cache_subsystem is not None:
tree = self.metadata_inheritance_cache_subsystem.get(unicode(course_id), {})
else:
logging.warning('Running MongoModuleStore without a metadata_inheritance_cache_subsystem. This is OK in localdev and testing environment. Not OK in production.')
logging.warning(
'Running MongoModuleStore without a metadata_inheritance_cache_subsystem. This is \
OK in localdev and testing environment. Not OK in production.'
)
if not tree:
# if not in subsystem, or we are on force refresh, then we have to compute
@@ -465,6 +514,7 @@ class MongoModuleStore(ModuleStoreWriteBase):
a runtime may mean that some objects report old values for inherited data.
"""
if course_id not in self.ignore_write_events_on_courses:
# below is done for side effects when runtime is None
cached_metadata = self._get_cached_metadata_inheritance_tree(course_id, force_refresh=True)
if runtime:
runtime.cached_metadata = cached_metadata
@@ -598,14 +648,12 @@ class MongoModuleStore(ModuleStoreWriteBase):
return [course for course in base_list if not isinstance(course, ErrorDescriptor)]
def _find_one(self, location):
'''Look for a given location in the collection. If revision is not
specified, returns the latest. If the item is not present, raise
'''Look for a given location in the collection. If the item is not present, raise
ItemNotFoundError.
'''
assert isinstance(location, Location)
item = self.collection.find_one(
{'_id': location.to_deprecated_son()},
sort=[('revision', pymongo.ASCENDING)],
{'_id': location.to_deprecated_son()}
)
if item is None:
raise ItemNotFoundError(location)
@@ -624,7 +672,9 @@ class MongoModuleStore(ModuleStoreWriteBase):
def has_course(self, course_key, ignore_case=False):
"""
Is the given course in this modulestore
Returns the course_id of the course if it was found, else None
Note: we return the course_id instead of a boolean here since the found course may have
a different id than the given course_id when ignore_case is True.
If ignore_case is True, do a case insensitive search,
otherwise, do a case sensitive search
@@ -638,7 +688,11 @@ class MongoModuleStore(ModuleStoreWriteBase):
course_query[key] = re.compile(r"(?i)^{}$".format(course_query[key]))
else:
course_query = {'_id': location.to_deprecated_son()}
return self.collection.find_one(course_query, fields={'_id': True}) is not None
course = self.collection.find_one(course_query, fields={'_id': True})
if course:
return SlashSeparatedCourseKey(course['_id']['org'], course['_id']['course'], course['_id']['name'])
else:
return None
def has_item(self, usage_key):
"""
@@ -680,7 +734,17 @@ class MongoModuleStore(ModuleStoreWriteBase):
('_id.course', course_id.course),
])
def get_items(self, course_id, settings=None, content=None, revision=None, **kwargs):
@staticmethod
def _id_dict_to_son(id_dict):
"""
Generate the partial key to look up items relative to a given course
"""
return SON([
(key, id_dict[key])
for key in ('tag', 'org', 'course', 'category', 'name', 'revision')
])
def get_items(self, course_id, settings=None, content=None, key_revision=KEY_REVISION_PUBLISHED, **kwargs):
"""
Returns:
list of XModuleDescriptor instances for the matching items within the course with
@@ -698,11 +762,11 @@ class MongoModuleStore(ModuleStoreWriteBase):
and rules as kwargs below
content (dict): fields to look for which have content scope. Follows same syntax and
rules as kwargs below.
revision (str): the revision of the items you're looking for. (only 'draft' makes sense for
this modulestore. If you don't provide a revision, it won't retrieve any drafts. If you
say 'draft', it will only return drafts. If you want one of each matching xblock but
preferring draft to published, call this same method on the draft modulestore w/o a
revision qualifier.)
key_revision (str): the revision of the items you're looking for.
KEY_REVISION_DRAFT - only returns drafts
KEY_REVISION_PUBLISHED (equates to None) - only returns published
If you want one of each matching xblock but preferring draft to published, call this same method
on the draft modulestore with REVISION_OPTION_DRAFT_PREFERRED.
kwargs (key=value): what to look for within the course.
Common qualifiers are ``category`` or any field name. if the target field is a list,
then it searches for the given value in the list not list equivalence.
@@ -712,7 +776,7 @@ class MongoModuleStore(ModuleStoreWriteBase):
update_version info.
"""
query = self._course_key_to_son(course_id)
query['_id.revision'] = revision
query['_id.revision'] = key_revision
for field in ['category', 'name']:
if field in kwargs:
query['_id.' + field] = kwargs.pop(field)
@@ -727,7 +791,7 @@ class MongoModuleStore(ModuleStoreWriteBase):
query.update(kwargs)
items = self.collection.find(
query,
sort=[('_id.revision', pymongo.ASCENDING)],
sort=[SORT_REVISION_FAVOR_DRAFT],
)
modules = self._load_items(course_id, list(items))
@@ -768,7 +832,7 @@ class MongoModuleStore(ModuleStoreWriteBase):
]))
location = course_id.make_usage_key('course', course_id.run)
course = self.create_and_save_xmodule(location, fields=fields, **kwargs)
course = self.create_and_save_xmodule(location, user_id, fields=fields, **kwargs)
# clone a default 'about' overview module as well
about_location = location.replace(
@@ -778,8 +842,9 @@ class MongoModuleStore(ModuleStoreWriteBase):
overview_template = AboutDescriptor.get_template('overview.yaml')
self.create_and_save_xmodule(
about_location,
system=course.system,
definition_data=overview_template.get('data')
user_id,
definition_data=overview_template.get('data'),
runtime=course.system
)
return course
@@ -793,14 +858,15 @@ class MongoModuleStore(ModuleStoreWriteBase):
course_query = self._course_key_to_son(course_key)
self.collection.remove(course_query, multi=True)
def create_xmodule(self, location, definition_data=None, metadata=None, system=None, fields={}):
def create_xmodule(self, location, definition_data=None, metadata=None, runtime=None, fields={}):
"""
Create the new xmodule but don't save it. Returns the new module.
:param location: a Location--must have a category
:param definition_data: can be empty. The initial definition_data for the kvs
:param metadata: can be empty, the initial metadata for the kvs
:param system: if you already have an xblock from the course, the xblock.runtime value
:param runtime: if you already have an xblock from the course, the xblock.runtime value
:param fields: a dictionary of field names and values for the new xmodule
"""
# differs from split mongo in that I believe most of this logic should be above the persistence
# layer but added it here to enable quick conversion. I'll need to reconcile these.
@@ -810,12 +876,12 @@ class MongoModuleStore(ModuleStoreWriteBase):
if definition_data is None:
definition_data = {}
if system is None:
if runtime is None:
services = {}
if self.i18n_service:
services["i18n"] = self.i18n_service
system = CachingDescriptorSystem(
runtime = CachingDescriptorSystem(
modulestore=self,
module_data={},
course_key=location.course_key,
@@ -828,9 +894,9 @@ class MongoModuleStore(ModuleStoreWriteBase):
select=self.xblock_select,
services=services,
)
xblock_class = system.load_block_type(location.category)
xblock_class = runtime.load_block_type(location.category)
dbmodel = self._create_new_field_data(location.category, location, definition_data, metadata)
xmodule = system.construct_xblock_from_class(
xmodule = runtime.construct_xblock_from_class(
xblock_class,
# We're loading a descriptor, so student_id is meaningless
# We also don't have separate notions of definition and usage ids yet,
@@ -845,42 +911,6 @@ class MongoModuleStore(ModuleStoreWriteBase):
xmodule.save()
return xmodule
def create_and_save_xmodule(self, location, definition_data=None, metadata=None, system=None,
fields={}, user_id=None):
"""
Create the new xmodule and save it. Does not return the new module because if the caller
will insert it as a child, it's inherited metadata will completely change. The difference
between this and just doing create_xmodule and update_item is this ensures static_tabs get
pointed to by the course.
:param location: a Location--must have a category
:param definition_data: can be empty. The initial definition_data for the kvs
:param metadata: can be empty, the initial metadata for the kvs
:param system: if you already have an xblock from the course, the xblock.runtime value
:param user_id: the user that created the xblock
"""
# differs from split mongo in that I believe most of this logic should be above the persistence
# layer but added it here to enable quick conversion. I'll need to reconcile these.
new_object = self.create_xmodule(location, definition_data, metadata, system, fields)
location = new_object.scope_ids.usage_id
self.update_item(new_object, allow_not_found=True, user_id=user_id)
# VS[compat] cdodge: This is a hack because static_tabs also have references from the course module, so
# if we add one then we need to also add it to the policy information (i.e. metadata)
# we should remove this once we can break this reference from the course to static tabs
# TODO move this special casing to app tier (similar to attaching new element to parent)
if location.category == 'static_tab':
course = self._get_course_for_item(location)
course.tabs.append(
StaticTab(
name=new_object.display_name,
url_slug=new_object.scope_ids.usage_id.name,
)
)
self.update_item(course, user_id=user_id)
return new_object
def _get_course_for_item(self, location, depth=0):
'''
for a given Xmodule, return the course that it belongs to
@@ -917,9 +947,14 @@ class MongoModuleStore(ModuleStoreWriteBase):
user_id: who made the change (ignored for now by this modulestore)
allow_not_found: whether to create a new object if one didn't already exist or give an error
force: force is meaningless for this modulestore
isPublish: an internal parameter that indicates whether this update is due to a Publish operation, and
thus whether the item's published information should be updated.
"""
try:
definition_data = self._convert_reference_fields_to_strings(xblock, xblock.get_explicitly_set_fields_by_scope())
definition_data = self._convert_reference_fields_to_strings(
xblock,
xblock.get_explicitly_set_fields_by_scope()
)
payload = {
'definition.data': definition_data,
'metadata': self._convert_reference_fields_to_strings(xblock, own_metadata(xblock)),
@@ -937,15 +972,6 @@ class MongoModuleStore(ModuleStoreWriteBase):
children = self._convert_reference_fields_to_strings(xblock, {'children': xblock.children})
payload.update({'definition.children': children['children']})
self._update_single_item(xblock.scope_ids.usage_id, payload)
# for static tabs, their containing course also records their display name
if xblock.scope_ids.block_type == 'static_tab':
course = self._get_course_for_item(xblock.scope_ids.usage_id)
# find the course's reference to this tab and update the name.
static_tab = CourseTabList.get_tab_by_slug(course.tabs, xblock.scope_ids.usage_id.name)
# only update if changed
if static_tab and static_tab['name'] != xblock.display_name:
static_tab['name'] = xblock.display_name
self.update_item(course, user_id)
# recompute (and update) the metadata inheritance tree which is cached
self.refresh_cached_metadata_inheritance_tree(xblock.scope_ids.usage_id.course_key, xblock.runtime)
@@ -954,6 +980,8 @@ class MongoModuleStore(ModuleStoreWriteBase):
if not allow_not_found:
raise
return xblock
def _convert_reference_fields_to_strings(self, xblock, jsonfields):
"""
Find all fields of type reference and convert the payload from UsageKeys to deprecated strings
@@ -975,56 +1003,72 @@ class MongoModuleStore(ModuleStoreWriteBase):
value[key] = subvalue.to_deprecated_string()
return jsonfields
# pylint: disable=unused-argument
def delete_item(self, location, **kwargs):
"""
Delete an item from this modulestore.
def get_parent_location(self, location, revision=REVISION_OPTION_PUBLISHED_ONLY, **kwargs):
'''
Find the location that is the parent of this location in this course.
Returns: version agnostic location (revision always None) as per the rest of mongo.
Args:
location (UsageKey)
"""
# pylint: enable=unused-argument
# VS[compat] cdodge: This is a hack because static_tabs also have references from the course module, so
# if we add one then we need to also add it to the policy information (i.e. metadata)
# we should remove this once we can break this reference from the course to static tabs
if location.category == 'static_tab':
item = self.get_item(location)
course = self._get_course_for_item(item.scope_ids.usage_id)
existing_tabs = course.tabs or []
course.tabs = [tab for tab in existing_tabs if tab.get('url_slug') != location.name]
self.update_item(course, '**replace_user**')
# Must include this to avoid the django debug toolbar (which defines the deprecated "safe=False")
# from overriding our default value set in the init method.
self.collection.remove({'_id': location.to_deprecated_son()}, safe=self.collection.safe)
# recompute (and update) the metadata inheritance tree which is cached
self.refresh_cached_metadata_inheritance_tree(location.course_key)
def get_parent_locations(self, location):
'''Find all locations that are the parents of this location in this
course. Needed for path_to_location().
revision:
REVISION_OPTION_PUBLISHED_ONLY - return only the PUBLISHED parent if it exists, else returns None
REVISION_OPTION_DRAFT_PREFERRED - return either the DRAFT or PUBLISHED parent,
preferring DRAFT, if parent(s) exists,
else returns None
'''
assert location.revision is None
assert revision == REVISION_OPTION_PUBLISHED_ONLY or revision == REVISION_OPTION_DRAFT_PREFERRED
# create a query with tag, org, course, and the children field set to the given location
query = self._course_key_to_son(location.course_key)
query['definition.children'] = location.to_deprecated_string()
items = self.collection.find(query, {'_id': True})
return [
location.course_key.make_usage_key(i['_id']['category'], i['_id']['name'])
for i in items
]
def get_modulestore_type(self, course_id):
# if only looking for the PUBLISHED parent, set the revision in the query to None
if revision == REVISION_OPTION_PUBLISHED_ONLY:
query['_id.revision'] = KEY_REVISION_PUBLISHED
# query the collection, sorting by DRAFT first
parents = self.collection.find(query, {'_id': True}, sort=[SORT_REVISION_FAVOR_DRAFT])
if parents.count() == 0:
# no parents were found
return None
if revision == REVISION_OPTION_PUBLISHED_ONLY:
if parents.count() > 1:
# should never have multiple PUBLISHED parents
raise ReferentialIntegrityError(
u"{} parents claim {}".format(parents.count(), location)
)
else:
# return the single PUBLISHED parent
return Location._from_deprecated_son(parents[0]['_id'], location.course_key.run)
else:
# there could be 2 different parents if
# (1) the draft item was moved or
# (2) the parent itself has 2 versions: DRAFT and PUBLISHED
# since we sorted by SORT_REVISION_FAVOR_DRAFT, the 0'th parent is the one we want
found_id = parents[0]['_id']
# don't disclose revision outside modulestore
return as_published(Location._from_deprecated_son(found_id, location.course_key.run))
def get_modulestore_type(self, course_key=None):
"""
Returns an enumeration-like type reflecting the type of this modulestore
The return can be one of:
"xml" (for XML based courses),
"mongo" for old-style MongoDB backed courses,
"split" for new-style split MongoDB backed courses.
Args:
course_key: just for signature compatibility
"""
return MONGO_MODULESTORE_TYPE
def get_orphans(self, course_key):
"""
Return an array all of the locations (deprecated string format) for orphans in the course.
Return an array of all of the locations (deprecated string format) for orphans in the course.
"""
detached_categories = [name for name, __ in XBlock.load_tagged_classes("detached")]
query = self._course_key_to_son(course_key)
@@ -1036,7 +1080,7 @@ class MongoModuleStore(ModuleStoreWriteBase):
if item['_id']['category'] != 'course':
# It would be nice to change this method to return UsageKeys instead of the deprecated string.
item_locs.add(
Location._from_deprecated_son(item['_id'], course_key.run).replace(revision=None).to_deprecated_string()
as_published(Location._from_deprecated_son(item['_id'], course_key.run)).to_deprecated_string()
)
all_reachable = all_reachable.union(item.get('definition', {}).get('children', []))
item_locs -= all_reachable
@@ -1054,7 +1098,7 @@ class MongoModuleStore(ModuleStoreWriteBase):
def _create_new_field_data(self, _category, _location, definition_data, metadata):
"""
To instantiate a new xmodule which will be saved latter, set up the dbModel and kvs
To instantiate a new xmodule which will be saved later, set up the dbModel and kvs
"""
kvs = MongoKeyValueStore(
definition_data,

692
common/lib/xmodule/xmodule/modulestore/mongo/draft.py
View File

@@ -1,89 +1,208 @@
"""
A ModuleStore that knows about a special version 'draft'. Modules
marked as 'draft' are read in preference to modules without the 'draft'
A ModuleStore that knows about a special version DRAFT. Modules
marked as DRAFT are read in preference to modules without the DRAFT
version by this ModuleStore (so, access to i4x://org/course/cat/name
returns the i4x://org/course/cat/name@draft object if that exists,
and otherwise returns i4x://org/course/cat/name).
"""
from datetime import datetime
import pymongo
from pytz import UTC
from xmodule.exceptions import InvalidVersionError
from xmodule.modulestore.exceptions import ItemNotFoundError, DuplicateItemError
from xmodule.modulestore.mongo.base import MongoModuleStore
from xmodule.modulestore import (
PublishState,
REVISION_OPTION_DRAFT_PREFERRED, REVISION_OPTION_DRAFT_ONLY, REVISION_OPTION_PUBLISHED_ONLY, REVISION_OPTION_ALL,
KEY_REVISION_PUBLISHED, KEY_REVISION_DRAFT, BRANCH_PUBLISHED_ONLY, BRANCH_DRAFT_PREFERRED
)
from xmodule.modulestore.exceptions import ItemNotFoundError, DuplicateItemError, InvalidBranchSetting
from xmodule.modulestore.mongo.base import (
MongoModuleStore, as_draft, as_published,
DIRECT_ONLY_CATEGORIES, SORT_REVISION_FAVOR_DRAFT,
)
from opaque_keys.edx.locations import Location
DRAFT = 'draft'
# Things w/ these categories should never be marked as version='draft'
DIRECT_ONLY_CATEGORIES = ['course', 'chapter', 'sequential', 'about', 'static_tab', 'course_info']
def as_draft(location):
"""
Returns the Location that is the draft for `location`
"""
return location.replace(revision=DRAFT)
def as_published(location):
"""
Returns the Location that is the published version for `location`
"""
return location.replace(revision=None)
def wrap_draft(item):
"""
Sets `item.is_draft` to `True` if the item is a
draft, and `False` otherwise. Sets the item's location to the
non-draft location in either case
Cleans the item's location and sets the `is_draft` attribute if needed.
Sets `item.is_draft` to `True` if the item is DRAFT, and `False` otherwise.
Sets the item's location to the non-draft location in either case.
"""
setattr(item, 'is_draft', item.location.revision == DRAFT)
item.location = item.location.replace(revision=None)
setattr(item, 'is_draft', item.location.revision == KEY_REVISION_DRAFT)
item.location = item.location.replace(revision=KEY_REVISION_PUBLISHED)
return item
class DraftModuleStore(MongoModuleStore):
"""
This mixin modifies a modulestore to give it draft semantics.
That is, edits made to units are stored to locations that have the revision DRAFT,
and when reads are made, they first read with revision DRAFT, and then fall back
Edits made to units are stored to locations that have the revision DRAFT.
Reads are first read with revision DRAFT, and then fall back
to the baseline revision only if DRAFT doesn't exist.
This module also includes functionality to promote DRAFT modules (and optionally
their children) to published modules.
This module also includes functionality to promote DRAFT modules (and their children)
to published modules.
"""
def get_item(self, usage_key, depth=0):
def __init__(self, *args, **kwargs):
"""
Args:
branch_setting_func: a function that returns the branch setting to use for this store's operations
"""
super(DraftModuleStore, self).__init__(*args, **kwargs)
self.branch_setting_func = kwargs.pop('branch_setting_func', lambda: BRANCH_PUBLISHED_ONLY)
def get_item(self, usage_key, depth=0, revision=None):
"""
Returns an XModuleDescriptor instance for the item at usage_key.
If usage_key.revision is None, returns the item with the most
recent revision
If any segment of the usage_key is None except revision, raises
Args:
usage_key: A :class:`.UsageKey` instance
depth (int): An argument that some module stores may use to prefetch
descendants of the queried modules for more efficient results later
in the request. The depth is counted in the number of calls to
get_children() to cache. None indicates to cache all descendants.
revision:
REVISION_OPTION_PUBLISHED_ONLY - returns only the published item.
REVISION_OPTION_DRAFT_ONLY - returns only the draft item.
None - uses the branch setting as follows:
if branch setting is BRANCH_PUBLISHED_ONLY, returns only the published item.
if branch setting is BRANCH_DRAFT_PREFERRED, returns either draft or published item,
preferring draft.
Note: If the item is in DIRECT_ONLY_CATEGORIES, then returns only the PUBLISHED
version regardless of the revision.
Raises:
xmodule.modulestore.exceptions.InsufficientSpecificationError
if any segment of the usage_key is None except revision
If no object is found at that usage_key, raises
xmodule.modulestore.exceptions.ItemNotFoundError
usage_key: A :class:`.UsageKey` instance
depth (int): An argument that some module stores may use to prefetch
descendents of the queried modules for more efficient results later
in the request. The depth is counted in the number of calls to
get_children() to cache. None indicates to cache all descendents
xmodule.modulestore.exceptions.ItemNotFoundError if no object
is found at that usage_key
"""
if usage_key.category not in DIRECT_ONLY_CATEGORIES:
try:
return wrap_draft(super(DraftModuleStore, self).get_item(as_draft(usage_key), depth=depth))
except ItemNotFoundError:
return wrap_draft(super(DraftModuleStore, self).get_item(usage_key, depth=depth))
else:
return super(DraftModuleStore, self).get_item(usage_key, depth=depth)
def get_published():
return wrap_draft(super(DraftModuleStore, self).get_item(usage_key, depth=depth))
def create_xmodule(self, location, definition_data=None, metadata=None, system=None, fields={}):
def get_draft():
return wrap_draft(super(DraftModuleStore, self).get_item(as_draft(usage_key), depth=depth))
# return the published version if REVISION_OPTION_PUBLISHED_ONLY is requested
if revision == REVISION_OPTION_PUBLISHED_ONLY:
return get_published()
# if the item is direct-only, there can only be a published version
elif usage_key.category in DIRECT_ONLY_CATEGORIES:
return get_published()
# return the draft version (without any fallback to PUBLISHED) if DRAFT-ONLY is requested
elif revision == REVISION_OPTION_DRAFT_ONLY:
return get_draft()
elif self.branch_setting_func() == BRANCH_PUBLISHED_ONLY:
return get_published()
else:
# could use a single query wildcarding revision and sorting by revision. would need to
# use prefix form of to_deprecated_son
try:
# first check for a draft version
return get_draft()
except ItemNotFoundError:
# otherwise, fall back to the published version
return get_published()
def has_item(self, usage_key, revision=None):
"""
Returns True if location exists in this ModuleStore.
Args:
revision:
REVISION_OPTION_PUBLISHED_ONLY - checks for the published item only
REVISION_OPTION_DRAFT_ONLY - checks for the draft item only
None - uses the branch setting, as follows:
if branch setting is BRANCH_PUBLISHED_ONLY, checks for the published item only
if branch setting is BRANCH_DRAFT_PREFERRED, checks whether draft or published item exists
"""
def has_published():
return super(DraftModuleStore, self).has_item(usage_key)
def has_draft():
return super(DraftModuleStore, self).has_item(as_draft(usage_key))
if revision == REVISION_OPTION_DRAFT_ONLY:
return has_draft()
elif revision == REVISION_OPTION_PUBLISHED_ONLY or self.branch_setting_func() == BRANCH_PUBLISHED_ONLY:
return has_published()
else:
key = usage_key.to_deprecated_son(prefix='_id.')
del key['_id.revision']
return self.collection.find(key).count() > 0
def _get_raw_parent_locations(self, location, key_revision):
"""
Get the parents but don't unset the revision in their locations.
Intended for internal use but not restricted.
Args:
location (UsageKey): assumes the location's revision is None; so, uses revision keyword solely
key_revision:
KEY_REVISION_DRAFT - return only the draft parent
KEY_REVISION_PUBLISHED - return only the published parent
REVISION_OPTION_ALL - return both draft and published parents
"""
_verify_revision_is_published(location)
# create a query to find all items in the course that have the given location listed as a child
query = self._course_key_to_son(location.course_key)
query['definition.children'] = location.to_deprecated_string()
# find all the items that satisfy the query
parents = self.collection.find(query, {'_id': True}, sort=[SORT_REVISION_FAVOR_DRAFT])
# return only the parent(s) that satisfy the request
return [
Location._from_deprecated_son(parent['_id'], location.course_key.run)
for parent in parents
if (
# return all versions of the parent if revision is REVISION_OPTION_ALL
key_revision == REVISION_OPTION_ALL or
# return this parent if it's direct-only, regardless of which revision is requested
parent['_id']['category'] in DIRECT_ONLY_CATEGORIES or
# return this parent only if its revision matches the requested one
parent['_id']['revision'] == key_revision
)
]
def get_parent_location(self, location, revision=None, **kwargs):
'''
Returns the given location's parent location in this course.
Returns: version agnostic locations (revision always None) as per the rest of mongo.
Args:
revision:
None - uses the branch setting for the revision
REVISION_OPTION_PUBLISHED_ONLY - return only the PUBLISHED parent if it exists, else returns None
REVISION_OPTION_DRAFT_PREFERRED - return either the DRAFT or PUBLISHED parent, preferring DRAFT, if parent(s) exists,
else returns None
If the draft has a different parent than the published, it returns only
the draft's parent. Because parents don't record their children's revisions, this
is actually a potentially fragile deduction based on parent type. If the parent type
is not DIRECT_ONLY, then the parent revision must be DRAFT.
Only xml_exporter currently uses this argument. Others should avoid it.
'''
if revision is None:
revision = REVISION_OPTION_PUBLISHED_ONLY \
if self.branch_setting_func() == BRANCH_PUBLISHED_ONLY \
else REVISION_OPTION_DRAFT_PREFERRED
return super(DraftModuleStore, self).get_parent_location(location, revision, **kwargs)
def create_xmodule(self, location, definition_data=None, metadata=None, runtime=None, fields={}):
"""
Create the new xmodule but don't save it. Returns the new module with a draft locator if
the category allows drafts. If the category does not allow drafts, just creates a published module.
@@ -91,108 +210,284 @@ class DraftModuleStore(MongoModuleStore):
:param location: a Location--must have a category
:param definition_data: can be empty. The initial definition_data for the kvs
:param metadata: can be empty, the initial metadata for the kvs
:param system: if you already have an xmodule from the course, the xmodule.system value
:param runtime: if you already have an xmodule from the course, the xmodule.runtime value
:param fields: a dictionary of field names and values for the new xmodule
"""
self._verify_branch_setting(BRANCH_DRAFT_PREFERRED)
if location.category not in DIRECT_ONLY_CATEGORIES:
location = as_draft(location)
return super(DraftModuleStore, self).create_xmodule(location, definition_data, metadata, system, fields)
return wrap_draft(
super(DraftModuleStore, self).create_xmodule(location, definition_data, metadata, runtime, fields)
)
def get_items(self, course_key, settings=None, content=None, revision=None, **kwargs):
"""
Performance Note: This is generally a costly operation, but useful for wildcard searches.
Returns:
list of XModuleDescriptor instances for the matching items within the course with
the given course_key
NOTE: don't use this to look for courses
as the course_key is required. Use get_courses.
NOTE: don't use this to look for courses as the course_key is required. Use get_courses instead.
Args:
course_key (CourseKey): the course identifier
settings: not used
content: not used
revision (str): the revision of the items you're looking for. Only 'draft' makes sense for
this modulestore. None implies get one of either the draft or published for each
matching xblock preferring the draft if it exists.
revision:
REVISION_OPTION_PUBLISHED_ONLY - returns only Published items
REVISION_OPTION_DRAFT_ONLY - returns only Draft items
None - uses the branch setting, as follows:
if the branch setting is BRANCH_PUBLISHED_ONLY,
returns only Published items
if the branch setting is BRANCH_DRAFT_PREFERRED,
returns either Draft or Published, preferring Draft items.
kwargs (key=value): what to look for within the course.
Common qualifiers are ``category`` or any field name. if the target field is a list,
then it searches for the given value in the list not list equivalence.
Substring matching pass a regex object.
``name`` is another commonly provided key (Location based stores)
"""
draft_items = [
wrap_draft(item) for item in
super(DraftModuleStore, self).get_items(course_key, revision='draft', **kwargs)
]
if revision == 'draft':
# the user only wants the drafts not everything w/ preference for draft
return draft_items
draft_items_locations = {item.location for item in draft_items}
non_draft_items = [
item for item in
super(DraftModuleStore, self).get_items(course_key, revision=None, **kwargs)
# filter out items that are not already in draft
if item.location not in draft_items_locations
]
return draft_items + non_draft_items
def base_get_items(key_revision):
return super(DraftModuleStore, self).get_items(course_key, key_revision=key_revision, **kwargs)
def convert_to_draft(self, source_location):
def draft_items():
return [wrap_draft(item) for item in base_get_items(KEY_REVISION_DRAFT)]
def published_items(draft_items):
# filters out items that are not already in draft_items
draft_items_locations = {item.location for item in draft_items}
return [
item for item in
base_get_items(KEY_REVISION_PUBLISHED)
if item.location not in draft_items_locations
]
if revision == REVISION_OPTION_DRAFT_ONLY:
return draft_items()
elif revision == REVISION_OPTION_PUBLISHED_ONLY or self.branch_setting_func() == BRANCH_PUBLISHED_ONLY:
return published_items([])
else:
draft_items = draft_items()
return draft_items + published_items(draft_items)
def convert_to_draft(self, location, user_id):
"""
Create a copy of the source and mark its revision as draft.
Copy the subtree rooted at source_location and mark the copies as draft.
:param source: the location of the source (its revision must be None)
Args:
location: the location of the source (its revision must be None)
user_id: the ID of the user doing the operation
Raises:
InvalidVersionError: if the source can not be made into a draft
ItemNotFoundError: if the source does not exist
DuplicateItemError: if the source or any of its descendants already has a draft copy
"""
if source_location.category in DIRECT_ONLY_CATEGORIES:
raise InvalidVersionError(source_location)
original = self.collection.find_one({'_id': source_location.to_deprecated_son()})
if not original:
raise ItemNotFoundError(source_location)
draft_location = as_draft(source_location)
original['_id'] = draft_location.to_deprecated_son()
try:
self.collection.insert(original)
except pymongo.errors.DuplicateKeyError:
raise DuplicateItemError(original['_id'])
# delegating to internal b/c we don't want any public user to use the kwargs on the internal
return self._convert_to_draft(location, user_id)
self.refresh_cached_metadata_inheritance_tree(draft_location.course_key)
def _convert_to_draft(self, location, user_id, delete_published=False, ignore_if_draft=False):
"""
Internal method with additional internal parameters to convert a subtree to draft.
return wrap_draft(self._load_items(source_location.course_key, [original])[0])
Args:
location: the location of the source (its revision must be KEY_REVISION_PUBLISHED)
user_id: the ID of the user doing the operation
delete_published (Boolean): intended for use by unpublish
ignore_if_draft(Boolean): for internal use only as part of depth first change
Raises:
InvalidVersionError: if the source can not be made into a draft
ItemNotFoundError: if the source does not exist
DuplicateItemError: if the source or any of its descendants already has a draft copy
"""
# verify input conditions
self._verify_branch_setting(BRANCH_DRAFT_PREFERRED)
_verify_revision_is_published(location)
# ensure we are not creating a DRAFT of an item that is direct-only
if location.category in DIRECT_ONLY_CATEGORIES:
raise InvalidVersionError(location)
def convert_item(item, to_be_deleted):
"""
Convert the subtree
"""
# collect the children's ids for future processing
next_tier = []
for child in item.get('definition', {}).get('children', []):
child_loc = Location.from_deprecated_string(child)
next_tier.append(child_loc.to_deprecated_son())
# insert a new DRAFT version of the item
item['_id']['revision'] = KEY_REVISION_DRAFT
# ensure keys are in fixed and right order before inserting
item['_id'] = self._id_dict_to_son(item['_id'])
try:
self.collection.insert(item)
except pymongo.errors.DuplicateKeyError:
# prevent re-creation of DRAFT versions, unless explicitly requested to ignore
if not ignore_if_draft:
raise DuplicateItemError(item['_id'], self, 'collection')
# delete the old PUBLISHED version if requested
if delete_published:
item['_id']['revision'] = KEY_REVISION_PUBLISHED
to_be_deleted.append(item['_id'])
return next_tier
# convert the subtree using the original item as the root
self._breadth_first(convert_item, [location])
# return the new draft item (does another fetch)
# get_item will wrap_draft so don't call it here (otherwise, it would override the is_draft attribute)
return self.get_item(location)
def update_item(self, xblock, user_id=None, allow_not_found=False, force=False, isPublish=False):
"""
See superclass doc.
In addition to the superclass's behavior, this method converts the unit to draft if it's not
already draft.
direct-only and not already draft.
"""
self._verify_branch_setting(BRANCH_DRAFT_PREFERRED)
# if the xblock is direct-only, update the PUBLISHED version
if xblock.location.category in DIRECT_ONLY_CATEGORIES:
return super(DraftModuleStore, self).update_item(xblock, user_id, allow_not_found)
draft_loc = as_draft(xblock.location)
try:
if not self.has_item(draft_loc):
self.convert_to_draft(xblock.location)
except ItemNotFoundError:
if not allow_not_found:
raise
if not super(DraftModuleStore, self).has_item(draft_loc):
try:
# ignore any descendants which are already draft
self._convert_to_draft(xblock.location, user_id, ignore_if_draft=True)
except ItemNotFoundError as exception:
# ignore the exception only if allow_not_found is True and
# the item that wasn't found is the one that was passed in
# we make this extra location check so we do not hide errors when converting any children to draft
if not (allow_not_found and exception.args[0] == xblock.location):
raise
xblock.location = draft_loc
super(DraftModuleStore, self).update_item(xblock, user_id, allow_not_found, isPublish)
# don't allow locations to truly represent themselves as draft outside of this file
xblock.location = as_published(xblock.location)
return wrap_draft(xblock)
def delete_item(self, location, delete_all_versions=False, **kwargs):
def delete_item(self, location, user_id, revision=None, **kwargs):
"""
Delete an item from this modulestore
Delete an item from this modulestore.
The method determines which revisions to delete. It disconnects and deletes the subtree.
In general, it assumes deletes only occur on drafts except for direct_only. The only exceptions
are internal calls like deleting orphans (during publishing as well as from delete_orphan view).
To indicate that all versions should be deleted, pass the keyword revision=REVISION_OPTION_ALL.
location: Something that can be passed to Location
* Deleting a DIRECT_ONLY_CATEGORIES block, deletes both draft and published children and removes from parent.
* Deleting a specific version of block whose parent is of DIRECT_ONLY_CATEGORIES, only removes it from parent if
the other version of the block does not exist. Deletes only children of same version.
* Other deletions remove from parent of same version and subtree of same version
Args:
location: UsageKey of the item to be deleted
user_id: id of the user deleting the item
revision:
None - deletes the item and its subtree, and updates the parents per description above
REVISION_OPTION_PUBLISHED_ONLY - removes only Published versions
REVISION_OPTION_ALL - removes both Draft and Published parents
currently only provided by contentstore.views.item.orphan_handler
"""
if location.category in DIRECT_ONLY_CATEGORIES:
return super(DraftModuleStore, self).delete_item(as_published(location))
self._verify_branch_setting(BRANCH_DRAFT_PREFERRED)
_verify_revision_is_published(location)
super(DraftModuleStore, self).delete_item(as_draft(location))
if delete_all_versions:
super(DraftModuleStore, self).delete_item(as_published(location))
is_item_direct_only = location.category in DIRECT_ONLY_CATEGORIES
if is_item_direct_only or revision == REVISION_OPTION_PUBLISHED_ONLY:
parent_revision = KEY_REVISION_PUBLISHED
elif revision == REVISION_OPTION_ALL:
parent_revision = REVISION_OPTION_ALL
else:
parent_revision = KEY_REVISION_DRAFT
return
# remove subtree from its parent
parent_locations = self._get_raw_parent_locations(location, key_revision=parent_revision)
# there could be 2 parents if
# Case 1: the draft item moved from one parent to another
# Case 2: revision==REVISION_OPTION_ALL and the single parent has 2 versions: draft and published
for parent_location in parent_locations:
# don't remove from direct_only parent if other versions of this still exists
if not is_item_direct_only and parent_location.category in DIRECT_ONLY_CATEGORIES:
# see if other version of root exists
alt_location = location.replace(
revision=KEY_REVISION_PUBLISHED if location.revision == KEY_REVISION_DRAFT else KEY_REVISION_DRAFT
)
if super(DraftModuleStore, self).has_item(alt_location):
continue
parent_block = super(DraftModuleStore, self).get_item(parent_location)
parent_block.children.remove(location)
parent_block.location = parent_location # ensure the location is with the correct revision
self.update_item(parent_block, user_id)
if is_item_direct_only or revision == REVISION_OPTION_ALL:
as_functions = [as_draft, as_published]
elif revision == REVISION_OPTION_PUBLISHED_ONLY:
as_functions = [as_published]
else:
as_functions = [as_draft]
self._delete_subtree(location, as_functions)
def _delete_subtree(self, location, as_functions):
"""
Internal method for deleting all of the subtree whose revisions match the as_functions
"""
course_key = location.course_key
def _delete_item(current_entry, to_be_deleted):
"""
Depth first deletion of nodes
"""
to_be_deleted.append(self._id_dict_to_son(current_entry['_id']))
next_tier = []
for child_loc in current_entry.get('definition', {}).get('children', []):
child_loc = course_key.make_usage_key_from_deprecated_string(child_loc)
for rev_func in as_functions:
current_loc = rev_func(child_loc)
current_son = current_loc.to_deprecated_son()
next_tier.append(current_son)
return next_tier
first_tier = [as_func(location) for as_func in as_functions]
self._breadth_first(_delete_item, first_tier)
def _breadth_first(self, function, root_usages):
"""
Get the root_usage from the db and do a depth first scan. Call the function on each. The
function should return a list of SON for any next tier items to process and should
add the SON for any items to delete to the to_be_deleted array.
At the end, it mass deletes the to_be_deleted items and refreshes the cached metadata inheritance
tree.
:param function: a function taking (item, to_be_deleted) and returning [SON] for next_tier invocation
:param root_usages: the usage keys for the root items (ensure they have the right revision set)
"""
if len(root_usages) == 0:
return
to_be_deleted = []
def _internal(tier):
next_tier = []
tier_items = self.collection.find({'_id': {'$in': tier}})
for current_entry in tier_items:
next_tier.extend(function(current_entry, to_be_deleted))
if len(next_tier) > 0:
_internal(next_tier)
_internal([root_usage.to_deprecated_son() for root_usage in root_usages])
self.collection.remove({'_id': {'$in': to_be_deleted}}, safe=self.collection.safe)
# recompute (and update) the metadata inheritance tree which is cached
self.refresh_cached_metadata_inheritance_tree(root_usages[0].course_key)
def has_changes(self, location):
"""
@@ -218,40 +513,84 @@ class DraftModuleStore(MongoModuleStore):
else:
return True
def publish(self, location, published_by_id):
def publish(self, location, user_id):
"""
Save a current draft to the underlying modulestore
"""
if location.category in DIRECT_ONLY_CATEGORIES:
# ignore noop attempt to publish something that can't be draft.
# ignoring v raising exception b/c bok choy tests always pass make_public which calls publish
return
try:
original_published = super(DraftModuleStore, self).get_item(location)
except ItemNotFoundError:
original_published = None
Publish the subtree rooted at location to the live course and remove the drafts.
Such publishing may cause the deletion of previously published but subsequently deleted
child trees. Overwrites any existing published xblocks from the subtree.
draft = self.get_item(location)
Treats the publishing of non-draftable items as merely a subtree selection from
which to descend.
if draft.has_children:
if original_published is not None:
# see if children were deleted. 2 reasons for children lists to differ:
# 1) child deleted
# 2) child moved
for child in original_published.children:
if child not in draft.children:
rents = self.get_parent_locations(child)
if (len(rents) == 1 and rents[0] == location): # the 1 is this original_published
self.delete_item(child, True)
super(DraftModuleStore, self).update_item(draft, published_by_id, isPublish=True)
self.delete_item(location)
Raises:
ItemNotFoundError: if any of the draft subtree nodes aren't found
"""
# NOTE: cannot easily use self._breadth_first b/c need to get pub'd and draft as pairs
# (could do it by having 2 breadth first scans, the first to just get all published children
# and the second to do the publishing on the drafts looking for the published in the cached
# list of published ones.)
to_be_deleted = []
def unpublish(self, location):
def _internal_depth_first(item_location):
"""
Depth first publishing from the given location
"""
item = self.get_item(item_location)
# publish the children first
if item.has_children:
for child_loc in item.children:
_internal_depth_first(child_loc)
if item_location.category in DIRECT_ONLY_CATEGORIES or not getattr(item, 'is_draft', False):
# ignore noop attempt to publish something that can't be or isn't currently draft
return
# try to find the originally PUBLISHED version, if it exists
try:
original_published = super(DraftModuleStore, self).get_item(item_location)
except ItemNotFoundError:
original_published = None
# if the category of this item allows having children
if item.has_children:
if original_published is not None:
# see if previously published children were deleted. 2 reasons for children lists to differ:
# Case 1: child deleted
# Case 2: child moved
for orig_child in original_published.children:
if orig_child not in item.children:
published_parent = self.get_parent_location(orig_child)
if published_parent == item_location:
# Case 1: child was deleted in draft parent item
# So, delete published version of the child now that we're publishing the draft parent
self._delete_subtree(item_location, [as_published])
else:
# Case 2: child was moved to a new draft parent item
# So, do not delete the child. It will be published when the new parent is published.
pass
super(DraftModuleStore, self).update_item(item, user_id, isPublish=True)
to_be_deleted.append(as_draft(item_location).to_deprecated_son())
# verify input conditions
self._verify_branch_setting(BRANCH_DRAFT_PREFERRED)
_verify_revision_is_published(location)
_internal_depth_first(location)
if len(to_be_deleted) > 0:
self.collection.remove({'_id': {'$in': to_be_deleted}})
return self.get_item(as_published(location))
def unpublish(self, location, user_id):
"""
Turn the published version into a draft, removing the published version
Turn the published version into a draft, removing the published version.
NOTE: unlike publish, this gives an error if called above the draftable level as it's intended
to remove things from the published version
"""
self.convert_to_draft(location)
super(DraftModuleStore, self).delete_item(location)
self._verify_branch_setting(BRANCH_DRAFT_PREFERRED)
return self._convert_to_draft(location, user_id, delete_published=True)
def _query_children_for_cache_children(self, course_key, items):
# first get non-draft in a round-trip
@@ -261,27 +600,70 @@ class DraftModuleStore(MongoModuleStore):
for non_draft in to_process_non_drafts:
to_process_dict[Location._from_deprecated_son(non_draft["_id"], course_key.run)] = non_draft
# now query all draft content in another round-trip
query = {
'_id': {'$in': [
as_draft(course_key.make_usage_key_from_deprecated_string(item)).to_deprecated_son() for item in items
]}
}
to_process_drafts = list(self.collection.find(query))
if self.branch_setting_func() == BRANCH_DRAFT_PREFERRED:
# now query all draft content in another round-trip
query = []
for item in items:
item_usage_key = course_key.make_usage_key_from_deprecated_string(item)
if item_usage_key.category not in DIRECT_ONLY_CATEGORIES:
query.append(as_draft(item_usage_key).to_deprecated_son())
if query:
query = {'_id': {'$in': query}}
to_process_drafts = list(self.collection.find(query))
# now we have to go through all drafts and replace the non-draft
# with the draft. This is because the semantics of the DraftStore is to
# always return the draft - if available
for draft in to_process_drafts:
draft_loc = Location._from_deprecated_son(draft["_id"], course_key.run)
draft_as_non_draft_loc = draft_loc.replace(revision=None)
# now we have to go through all drafts and replace the non-draft
# with the draft. This is because the semantics of the DraftStore is to
# always return the draft - if available
for draft in to_process_drafts:
draft_loc = Location._from_deprecated_son(draft["_id"], course_key.run)
draft_as_non_draft_loc = as_published(draft_loc)
# does non-draft exist in the collection
# if so, replace it
if draft_as_non_draft_loc in to_process_dict:
to_process_dict[draft_as_non_draft_loc] = draft
# does non-draft exist in the collection
# if so, replace it
if draft_as_non_draft_loc in to_process_dict:
to_process_dict[draft_as_non_draft_loc] = draft
# convert the dict - which is used for look ups - back into a list
queried_children = to_process_dict.values()
return queried_children
def compute_publish_state(self, xblock):
"""
Returns whether this xblock is draft, public, or private.
Returns:
PublishState.draft - content is in the process of being edited, but still has a previous
version deployed to LMS
PublishState.public - content is locked and deployed to LMS
PublishState.private - content is editable and not deployed to LMS
"""
if getattr(xblock, 'is_draft', False):
published_xblock_location = as_published(xblock.location)
published_item = self.collection.find_one(
{'_id': published_xblock_location.to_deprecated_son()}
)
if published_item is None:
return PublishState.private
else:
return PublishState.draft
else:
return PublishState.public
def _verify_branch_setting(self, expected_branch_setting):
"""
Raises an exception if the current branch setting does not match the expected branch setting.
"""
actual_branch_setting = self.branch_setting_func()
if actual_branch_setting != expected_branch_setting:
raise InvalidBranchSetting(
expected_setting=expected_branch_setting,
actual_setting=actual_branch_setting
)
def _verify_revision_is_published(location):
"""
Asserts that the revision set on the given location is KEY_REVISION_PUBLISHED
"""
assert location.revision == KEY_REVISION_PUBLISHED

10
common/lib/xmodule/xmodule/modulestore/search.py
View File

@@ -1,4 +1,3 @@
from itertools import repeat
from .exceptions import (ItemNotFoundError, NoPathToItem)
@@ -53,11 +52,8 @@ def path_to_location(modulestore, usage_key):
while len(queue) > 0:
(next_usage, path) = queue.pop() # Takes from the end
# get_parent_locations should raise ItemNotFoundError if location
# isn't found so we don't have to do it explicitly. Call this
# first to make sure the location is there (even if it's a course, and
# we would otherwise immediately exit).
parents = modulestore.get_parent_locations(next_usage)
# get_parent_location raises ItemNotFoundError if location isn't found
parent = modulestore.get_parent_location(next_usage)
# print 'Processing loc={0}, path={1}'.format(next_usage, path)
if next_usage.definition_key.block_type == "course":
@@ -67,7 +63,7 @@ def path_to_location(modulestore, usage_key):
# otherwise, add parent locations at the end
newpath = (next_usage, path)
queue.extend(zip(parents, repeat(newpath)))
queue.append((parent, newpath))
# If we're here, there is no path
return None

137
common/lib/xmodule/xmodule/modulestore/split_migrator.py
View File

@@ -7,6 +7,7 @@ In general, it's strategy is to treat the other modulestores as read-only and to
manipulate storage but use existing api's.
'''
from xblock.fields import Reference, ReferenceList, ReferenceValueDict
from xmodule.modulestore import BRANCH_NAME_DRAFT, BRANCH_NAME_PUBLISHED, REVISION_OPTION_DRAFT_ONLY
class SplitMigrator(object):
@@ -65,8 +66,8 @@ class SplitMigrator(object):
# iterate over published course elements. Wildcarding rather than descending b/c some elements are orphaned (e.g.,
# course about pages, conditionals)
for module in self.direct_modulestore.get_items(course_key):
# don't copy the course again. No drafts should get here but check
if module.location != old_course_loc and not getattr(module, 'is_draft', False):
# don't copy the course again. No drafts should get here
if module.location != old_course_loc:
# create split_xblock using split.create_item
# where block_id is computed by translate_location_to_locator
new_locator = self.loc_mapper.translate_location(
@@ -81,10 +82,10 @@ class SplitMigrator(object):
fields=self._get_json_fields_translate_references(module, course_key, True),
continue_version=True
)
# after done w/ published items, add version for 'draft' pointing to the published structure
# after done w/ published items, add version for DRAFT pointing to the published structure
index_info = self.split_modulestore.get_course_index_info(course_version_locator)
versions = index_info['versions']
versions['draft'] = versions['published']
versions[BRANCH_NAME_DRAFT] = versions[BRANCH_NAME_PUBLISHED]
self.split_modulestore.update_course_index(index_info)
# clean up orphans in published version: in old mongo, parents pointed to the union of their published and draft
@@ -97,63 +98,62 @@ class SplitMigrator(object):
"""
# each true update below will trigger a new version of the structure. We may want to just have one new version
# but that's for a later date.
new_draft_course_loc = published_course_key.for_branch('draft')
new_draft_course_loc = published_course_key.for_branch(BRANCH_NAME_DRAFT)
# to prevent race conditions of grandchilden being added before their parents and thus having no parent to
# add to
awaiting_adoption = {}
for module in self.draft_modulestore.get_items(course_key):
if getattr(module, 'is_draft', False):
new_locator = self.loc_mapper.translate_location(
module.location, False, add_entry_if_missing=True
)
if self.split_modulestore.has_item(new_locator):
# was in 'direct' so draft is a new version
split_module = self.split_modulestore.get_item(new_locator)
# need to remove any no-longer-explicitly-set values and add/update any now set values.
for name, field in split_module.fields.iteritems():
if field.is_set_on(split_module) and not module.fields[name].is_set_on(module):
field.delete_from(split_module)
for name, field in module.fields.iteritems():
# draft children will insert themselves and the others are here already; so, don't do it 2x
if name != 'children' and field.is_set_on(module):
field.write_to(split_module, field.read_from(module))
for module in self.draft_modulestore.get_items(course_key, revision=REVISION_OPTION_DRAFT_ONLY):
new_locator = self.loc_mapper.translate_location(
module.location, False, add_entry_if_missing=True
)
if self.split_modulestore.has_item(new_locator):
# was in 'direct' so draft is a new version
split_module = self.split_modulestore.get_item(new_locator)
# need to remove any no-longer-explicitly-set values and add/update any now set values.
for name, field in split_module.fields.iteritems():
if field.is_set_on(split_module) and not module.fields[name].is_set_on(module):
field.delete_from(split_module)
for field, value in self._get_fields_translate_references(module, course_key, True).iteritems():
# draft children will insert themselves and the others are here already; so, don't do it 2x
if field.name != 'children':
field.write_to(split_module, value)
_new_module = self.split_modulestore.update_item(split_module, user.id)
else:
# only a draft version (aka, 'private'). parent needs updated too.
# create a new course version just in case the current head is also the prod head
_new_module = self.split_modulestore.create_item(
new_draft_course_loc, module.category, user.id,
block_id=new_locator.block_id,
fields=self._get_json_fields_translate_references(module, course_key, True)
)
awaiting_adoption[module.location] = new_locator
for draft_location, new_locator in awaiting_adoption.iteritems():
for parent_loc in self.draft_modulestore.get_parent_locations(draft_location):
old_parent = self.draft_modulestore.get_item(parent_loc)
new_parent = self.split_modulestore.get_item(
self.loc_mapper.translate_location(old_parent.location, False)
_new_module = self.split_modulestore.update_item(split_module, user.id)
else:
# only a draft version (aka, 'private'). parent needs updated too.
# create a new course version just in case the current head is also the prod head
_new_module = self.split_modulestore.create_item(
new_draft_course_loc, module.category, user.id,
block_id=new_locator.block_id,
fields=self._get_json_fields_translate_references(module, course_key, True)
)
# this only occurs if the parent was also awaiting adoption
if any(new_locator == child.version_agnostic() for child in new_parent.children):
break
# find index for module: new_parent may be missing quite a few of old_parent's children
new_parent_cursor = 0
for old_child_loc in old_parent.children:
if old_child_loc == draft_location:
break
sibling_loc = self.loc_mapper.translate_location(old_child_loc, False)
# sibling may move cursor
for idx in range(new_parent_cursor, len(new_parent.children)):
if new_parent.children[idx].version_agnostic() == sibling_loc:
new_parent_cursor = idx + 1
break
new_parent.children.insert(new_parent_cursor, new_locator)
new_parent = self.split_modulestore.update_item(new_parent, user.id)
awaiting_adoption[module.location] = new_locator
for draft_location, new_locator in awaiting_adoption.iteritems():
parent_loc = self.draft_modulestore.get_parent_location(draft_location)
old_parent = self.draft_modulestore.get_item(parent_loc)
new_parent = self.split_modulestore.get_item(
self.loc_mapper.translate_location(old_parent.location, False)
)
# this only occurs if the parent was also awaiting adoption: skip this one, go to next
if any(new_locator == child.version_agnostic() for child in new_parent.children):
continue
# find index for module: new_parent may be missing quite a few of old_parent's children
new_parent_cursor = 0
for old_child_loc in old_parent.children:
if old_child_loc == draft_location:
break # moved cursor enough, insert it here
sibling_loc = self.loc_mapper.translate_location(old_child_loc, False)
# sibling may move cursor
for idx in range(new_parent_cursor, len(new_parent.children)):
if new_parent.children[idx].version_agnostic() == sibling_loc:
new_parent_cursor = idx + 1
break # skipped sibs enough, pick back up scan
new_parent.children.insert(new_parent_cursor, new_locator)
new_parent = self.split_modulestore.update_item(new_parent, user.id)
def _get_json_fields_translate_references(self, xblock, old_course_id, published):
"""
Return the json repr for explicitly set fields but convert all references to their block_id's
Return the json repr for explicitly set fields but convert all references to their Locators
"""
def get_translation(location):
"""
@@ -180,3 +180,34 @@ class SplitMigrator(object):
result[field_name] = field.read_json(xblock)
return result
def _get_fields_translate_references(self, xblock, old_course_id, published):
"""
Return a dictionary of field: value pairs for explicitly set fields
but convert all references to their BlockUsageLocators
"""
def get_translation(location):
"""
Convert the location and add to loc mapper
"""
return self.loc_mapper.translate_location(location, published, add_entry_if_missing=True)
result = {}
for field_name, field in xblock.fields.iteritems():
if field.is_set_on(xblock):
field_value = getattr(xblock, field_name)
if isinstance(field, Reference) and field_value is not None:
result[field] = get_translation(field_value)
elif isinstance(field, ReferenceList):
result[field] = [
get_translation(ele) for ele in field_value
]
elif isinstance(field, ReferenceValueDict):
result[field] = {
key: get_translation(subvalue)
for key, subvalue in field_value.iteritems()
}
else:
result[field] = field_value
return result

169
common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
View File

@@ -60,7 +60,9 @@ from opaque_keys.edx.locator import (
)
from xmodule.modulestore.exceptions import InsufficientSpecificationError, VersionConflictError, DuplicateItemError, \
DuplicateCourseError
from xmodule.modulestore import inheritance, ModuleStoreWriteBase, SPLIT_MONGO_MODULESTORE_TYPE
from xmodule.modulestore import (
inheritance, ModuleStoreWriteBase, SPLIT_MONGO_MODULESTORE_TYPE, BRANCH_NAME_DRAFT, BRANCH_NAME_PUBLISHED
)
from ..exceptions import ItemNotFoundError
from .definition_lazy_loader import DefinitionLazyLoader
@@ -72,6 +74,7 @@ from xblock.core import XBlock
from xmodule.modulestore.loc_mapper_store import LocMapperStore
from xmodule.error_module import ErrorDescriptor
log = logging.getLogger(__name__)
#==============================================================================
# Documentation is at
@@ -293,7 +296,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
}
return envelope
def get_courses(self, branch='draft', qualifiers=None):
def get_courses(self, branch=BRANCH_NAME_DRAFT, qualifiers=None):
'''
Returns a list of course descriptors matching any given qualifiers.
@@ -301,29 +304,29 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
legal query for mongo to use against the active_versions collection.
Note, this is to find the current head of the named branch type
(e.g., 'draft'). To get specific versions via guid use get_course.
(e.g., BRANCH_NAME_DRAFT). To get specific versions via guid use get_course.
:param branch: the branch for which to return courses. Default value is 'draft'.
:param qualifiers: a optional dict restricting which elements should match
:param branch: the branch for which to return courses. Default value is BRANCH_NAME_DRAFT.
:param qualifiers: an optional dict restricting which elements should match
'''
if qualifiers is None:
qualifiers = {}
qualifiers.update({"versions.{}".format(branch): {"$exists": True}})
matching = self.db_connection.find_matching_course_indexes(qualifiers)
matching_indexes = self.db_connection.find_matching_course_indexes(qualifiers)
# collect ids and then query for those
version_guids = []
id_version_map = {}
for structure in matching:
version_guid = structure['versions'][branch]
for course_index in matching_indexes:
version_guid = course_index['versions'][branch]
version_guids.append(version_guid)
id_version_map[version_guid] = structure
id_version_map[version_guid] = course_index
course_entries = self.db_connection.find_matching_structures({'_id': {'$in': version_guids}})
matching_structures = self.db_connection.find_matching_structures({'_id': {'$in': version_guids}})
# get the block for the course element (s/b the root)
# get the blocks for each course index (s/b the root)
result = []
for entry in course_entries:
for entry in matching_structures:
course_info = id_version_map[entry['_id']]
envelope = {
'org': course_info['org'],
@@ -351,16 +354,20 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
'''
Does this course exist in this modulestore. This method does not verify that the branch &/or
version in the course_id exists. Use get_course_index_info to check that.
Returns the course_id of the course if it was found, else None
Note: we return the course_id instead of a boolean here since the found course may have
a different id than the given course_id when ignore_case is True.
'''
assert(isinstance(course_id, CourseLocator))
course_entry = self.db_connection.get_course_index(course_id, ignore_case)
return course_entry is not None
course_index = self.db_connection.get_course_index(course_id, ignore_case)
return CourseLocator(course_index['org'], course_index['offering'], course_id.branch) if course_index else None
def has_item(self, usage_key):
"""
Returns True if location exists in its course. Returns false if
Returns True if usage_key exists in its course. Returns false if
the course or the block w/in the course do not exist for the given version.
raises InsufficientSpecificationError if the locator does not id a block
raises InsufficientSpecificationError if the usage_key does not id a block
"""
if usage_key.block_id is None:
raise InsufficientSpecificationError(usage_key)
@@ -378,9 +385,9 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
:param usage_key: the block to check
:return: True if the draft and published versions differ
"""
draft = self.get_item(usage_key.for_branch("draft"))
draft = self.get_item(usage_key.for_branch(BRANCH_NAME_DRAFT))
try:
published = self.get_item(usage_key.for_branch("published"))
published = self.get_item(usage_key.for_branch(BRANCH_NAME_PUBLISHED))
except ItemNotFoundError:
return True
@@ -400,16 +407,17 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
items = self._load_items(course, [usage_key.block_id], depth, lazy=True)
if len(items) == 0:
raise ItemNotFoundError(usage_key)
elif len(items) > 1:
log.debug("Found more than one item for '{}'".format(usage_key))
return items[0]
def get_items(self, course_locator, settings=None, content=None, **kwargs):
"""
Returns:
list of XModuleDescriptor instances for the matching items within the course with
the given course_id
the given course_locator
NOTE: don't use this to look for courses
as the course_id is required. Use get_courses.
NOTE: don't use this to look for courses as the course_locator is required. Use get_courses.
Args:
course_locator (CourseLocator): the course identifier
@@ -420,7 +428,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
kwargs (key=value): what to look for within the course.
Common qualifiers are ``category`` or any field name. if the target field is a list,
then it searches for the given value in the list not list equivalence.
Substring matching pass a regex object.
For substring matching pass a regex object.
For split,
you can search by ``edited_by``, ``edited_on`` providing a function testing limits.
"""
@@ -464,25 +472,23 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
else:
return []
def get_parent_locations(self, locator):
def get_parent_location(self, locator, **kwargs):
'''
Return the locations (Locators w/ block_ids) for the parents of this location in this
Return the location (Locators w/ block_ids) for the parent of this location in this
course. Could use get_items(location, {'children': block_id}) but this is slightly faster.
NOTE: the locator must contain the block_id, and this code does not actually ensure block_id exists
:param locator: BlockUsageLocator restricting search scope
:param course_id: ignored. Only included for API compatibility. Specify the course_id within the locator.
'''
course = self._lookup_course(locator)
items = self._get_parents_from_structure(locator.block_id, course['structure'])
return [
BlockUsageLocator.make_relative(
parent_id = self._get_parent_from_structure(locator.block_id, course['structure'])
if parent_id is None:
return None
return BlockUsageLocator.make_relative(
locator,
block_type=course['structure']['blocks'][parent_id].get('category'),
block_id=LocMapperStore.decode_key_from_mongo(parent_id),
)
for parent_id in items
]
)
def get_orphans(self, course_key):
"""
@@ -574,7 +580,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
# TODO if depth is significant, it may make sense to get all that have the same original_version
# and reconstruct the subtree from version_guid
next_entries = self.db_connection.find_matching_structures({'previous_version' : version_guid})
next_entries = self.db_connection.find_matching_structures({'previous_version': version_guid})
# must only scan cursor's once
next_versions = [struct for struct in next_entries]
result = {version_guid: [CourseLocator(version_guid=struct['_id']) for struct in next_versions]}
@@ -661,7 +667,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
new_id = ObjectId()
document = {
'_id': new_id,
"category" : category,
"category": category,
"fields": new_def_data,
"edit_info": {
"edited_by": user_id,
@@ -866,7 +872,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
def create_course(
self, org, offering, user_id, fields=None,
master_branch='draft', versions_dict=None, root_category='course',
master_branch=BRANCH_NAME_DRAFT, versions_dict=None, root_category='course',
root_block_id='course', **kwargs
):
"""
@@ -899,10 +905,10 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
when
loaded.
master_branch: the tag (key) for the version name in the dict which is the 'draft' version. Not the actual
master_branch: the tag (key) for the version name in the dict which is the DRAFT version. Not the actual
version guid, but what to call it.
versions_dict: the starting version ids where the keys are the tags such as 'draft' and 'published'
versions_dict: the starting version ids where the keys are the tags such as DRAFT and REVISION_OPTION_PUBLISHED_ONLY
and the values are structure guids. If provided, the new course will reuse this version (unless you also
provide any fields overrides, see above). if not provided, will create a mostly empty course
structure with just a category course root xblock.
@@ -1263,17 +1269,19 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
orphans = set()
destination_blocks = destination_structure['blocks']
for subtree_root in subtree_list:
# find the parents and put root in the right sequence
parents = self._get_parents_from_structure(subtree_root.block_id, source_structure)
if not all(parent in destination_blocks for parent in parents):
raise ItemNotFoundError(parents)
for parent_loc in parents:
orphans.update(
self._sync_children(
source_structure['blocks'][parent_loc],
destination_blocks[parent_loc],
subtree_root.block_id
))
if subtree_root.block_id != source_structure['root']:
# find the parents and put root in the right sequence
parent = self._get_parent_from_structure(subtree_root.block_id, source_structure)
if parent is not None: # may be a detached category xblock
if not parent in destination_blocks:
raise ItemNotFoundError(parent)
orphans.update(
self._sync_children(
source_structure['blocks'][parent],
destination_blocks[parent],
subtree_root.block_id
)
)
# update/create the subtree and its children in destination (skipping blacklist)
orphans.update(
self._publish_subdag(
@@ -1289,6 +1297,10 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
self.db_connection.insert_structure(destination_structure)
self._update_head(index_entry, destination_course.branch, destination_structure['_id'])
def unpublish(self, location, user_id):
published_location = location.replace(branch=REVISION_OPTION_PUBLISHED_ONLY)
self.delete_item(published_location, user_id)
def update_course_index(self, updated_index_entry):
"""
Change the given course's index entry.
@@ -1299,7 +1311,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
"""
self.db_connection.update_course_index(updated_index_entry)
def delete_item(self, usage_locator, user_id, delete_all_versions=False, delete_children=False, force=False):
def delete_item(self, usage_locator, user_id, force=False):
"""
Delete the block or tree rooted at block (if delete_children) and any references w/in the course to the block
from a new version of the course structure.
@@ -1322,15 +1334,13 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
new_structure = self._version_structure(original_structure, user_id)
new_blocks = new_structure['blocks']
new_id = new_structure['_id']
parents = self.get_parent_locations(usage_locator)
for parent in parents:
encoded_block_id = LocMapperStore.encode_key_for_mongo(parent.block_id)
parent_block = new_blocks[encoded_block_id]
parent_block['fields']['children'].remove(usage_locator.block_id)
parent_block['edit_info']['edited_on'] = datetime.datetime.now(UTC)
parent_block['edit_info']['edited_by'] = user_id
parent_block['edit_info']['previous_version'] = parent_block['edit_info']['update_version']
parent_block['edit_info']['update_version'] = new_id
encoded_block_id = self._get_parent_from_structure(usage_locator.block_id, original_structure)
parent_block = new_blocks[encoded_block_id]
parent_block['fields']['children'].remove(usage_locator.block_id)
parent_block['edit_info']['edited_on'] = datetime.datetime.now(UTC)
parent_block['edit_info']['edited_by'] = user_id
parent_block['edit_info']['previous_version'] = parent_block['edit_info']['update_version']
parent_block['edit_info']['update_version'] = new_id
def remove_subtree(block_id):
"""
@@ -1340,10 +1350,8 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
for child in new_blocks[encoded_block_id]['fields'].get('children', []):
remove_subtree(child)
del new_blocks[encoded_block_id]
if delete_children:
remove_subtree(usage_locator.block_id)
else:
del new_blocks[LocMapperStore.encode_key_for_mongo(usage_locator.block_id)]
remove_subtree(usage_locator.block_id)
# update index if appropriate and structures
self.db_connection.insert_structure(new_structure)
@@ -1446,13 +1454,16 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
else:
return DefinitionLocator(definition['category'], definition['_id'])
def get_modulestore_type(self, course_id):
def get_modulestore_type(self, course_key=None):
"""
Returns an enumeration-like type reflecting the type of this modulestore
The return can be one of:
"xml" (for XML based courses),
"mongo" for old-style MongoDB backed courses,
"split" for new-style split MongoDB backed courses.
Args:
course_key: just for signature compatibility
"""
return SPLIT_MONGO_MODULESTORE_TYPE
@@ -1647,18 +1658,16 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
'schema_version': self.SCHEMA_VERSION,
}
def _get_parents_from_structure(self, block_id, structure):
def _get_parent_from_structure(self, block_id, structure):
"""
Given a structure, find all of block_id's parents in that structure. Note returns
Given a structure, find block_id's parent in that structure. Note returns
the encoded format for parent
"""
items = []
for parent_id, value in structure['blocks'].iteritems():
for child_id in value['fields'].get('children', []):
if block_id == child_id:
items.append(parent_id)
return items
return parent_id
return None
def _sync_children(self, source_parent, destination_parent, new_child):
"""
@@ -1730,7 +1739,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
"""
Delete the orphan and any of its descendants which no longer have parents.
"""
if not self._get_parents_from_structure(orphan, structure):
if self._get_parent_from_structure(orphan, structure) is None:
encoded_block_id = LocMapperStore.encode_key_for_mongo(orphan)
for child in structure['blocks'][encoded_block_id]['fields'].get('children', []):
self._delete_if_true_orphan(child, structure)
@@ -1789,3 +1798,25 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
Check that the db is reachable.
"""
return {SPLIT_MONGO_MODULESTORE_TYPE: self.db_connection.heartbeat()}
def compute_publish_state(self, xblock):
"""
Returns whether this xblock is draft, public, or private.
Returns:
PublishState.draft - content is in the process of being edited, but still has a previous
version deployed to LMS
PublishState.public - content is locked and deployed to LMS
PublishState.private - content is editable and not deployed to LMS
"""
# TODO implement
raise NotImplementedError()
def convert_to_draft(self, location, user_id):
"""
Create a copy of the source and mark its revision as draft.
:param source: the location of the source (its revision must be None)
"""
# This is a no-op in Split since a draft version of the data always remains
pass

34
common/lib/xmodule/xmodule/modulestore/store_utilities.py
View File

@@ -2,6 +2,7 @@ import re
import logging
from xmodule.contentstore.content import StaticContent
from xmodule.modulestore import REVISION_OPTION_PUBLISHED_ONLY, REVISION_OPTION_DRAFT_ONLY
def _prefix_only_url_replace_regex(prefix):
@@ -87,10 +88,12 @@ def rewrite_nonportable_content_links(source_course_id, dest_course_id, text):
return text
def _clone_modules(modulestore, modules, source_course_id, dest_course_id):
def _clone_modules(modulestore, modules, source_course_id, dest_course_id, user_id):
for module in modules:
original_loc = module.location
module.location = module.location.map_into_course(dest_course_id)
if module.location.category == 'course':
module.location = module.location.replace(name=module.location.run)
print "Cloning module {0} to {1}....".format(original_loc, module.location)
@@ -108,10 +111,10 @@ def _clone_modules(modulestore, modules, source_course_id, dest_course_id):
module.children = new_children
modulestore.update_item(module, '**replace_user**')
modulestore.update_item(module, user_id, allow_not_found=True)
def clone_course(modulestore, contentstore, source_course_id, dest_course_id):
def clone_course(modulestore, contentstore, source_course_id, dest_course_id, user_id):
# check to see if the dest_location exists as an empty course
# we need an empty course because the app layers manage the permissions and users
if not modulestore.has_course(dest_course_id):
@@ -120,29 +123,26 @@ def clone_course(modulestore, contentstore, source_course_id, dest_course_id):
# verify that the dest_location really is an empty course, which means only one with an optional 'overview'
dest_modules = modulestore.get_items(dest_course_id)
basically_empty = True
for module in dest_modules:
if module.location.category == 'course' or (module.location.category == 'about'
and module.location.name == 'overview'):
if module.location.category == 'course' or (
module.location.category == 'about' and module.location.name == 'overview'
):
continue
basically_empty = False
break
if not basically_empty:
raise Exception("Course at destination {0} is not an empty course. You can only clone into an empty course. Aborting...".format(dest_location))
# only course and about overview allowed
raise Exception("Course at destination {0} is not an empty course. You can only clone into an empty course. Aborting...".format(dest_course_id))
# check to see if the source course is actually there
if not modulestore.has_course(source_course_id):
raise Exception("Cannot find a course at {0}. Aborting".format(source_course_id))
# Get all modules under this namespace which is (tag, org, course) tuple
modules = modulestore.get_items(source_course_id, revision=REVISION_OPTION_PUBLISHED_ONLY)
_clone_modules(modulestore, modules, source_course_id, dest_course_id, user_id)
course_location = dest_course_id.make_usage_key('course', dest_course_id.run)
modulestore.publish(course_location, user_id)
modules = modulestore.get_items(source_course_id, revision=None)
_clone_modules(modulestore, modules, source_course_id, dest_course_id)
modules = modulestore.get_items(source_course_id, revision='draft')
_clone_modules(modulestore, modules, source_course_id, dest_course_id)
modules = modulestore.get_items(source_course_id, revision=REVISION_OPTION_DRAFT_ONLY)
_clone_modules(modulestore, modules, source_course_id, dest_course_id, user_id)
# now iterate through all of the assets and clone them
# first the thumbnails

98
common/lib/xmodule/xmodule/modulestore/tests/django_utils.py
View File

@@ -5,7 +5,8 @@ Modulestore configuration for test cases.
from uuid import uuid4
from django.test import TestCase
from xmodule.modulestore.django import (
editable_modulestore, clear_existing_modulestores, loc_mapper)
modulestore, clear_existing_modulestores, loc_mapper)
from xmodule.modulestore import MONGO_MODULESTORE_TYPE
from xmodule.contentstore.django import contentstore
@@ -25,7 +26,7 @@ def mixed_store_config(data_dir, mappings):
where 'xml' and 'default' are the two options provided by this configuration,
mapping (respectively) to XML-backed and Mongo-backed modulestores..
"""
mongo_config = mongo_store_config(data_dir)
draft_mongo_config = draft_mongo_store_config(data_dir)
xml_config = xml_store_config(data_dir)
store = {
@@ -33,40 +34,13 @@ def mixed_store_config(data_dir, mappings):
'ENGINE': 'xmodule.modulestore.mixed.MixedModuleStore',
'OPTIONS': {
'mappings': mappings,
'stores': {
'default': mongo_config['default'],
'xml': xml_config['default']
}
'stores': [
draft_mongo_config['default'],
xml_config['default']
]
}
}
}
store['direct'] = store['default']
return store
def mongo_store_config(data_dir):
"""
Defines default module store using MongoModuleStore.
Use of this config requires mongo to be running.
"""
store = {
'default': {
'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
'DOC_STORE_CONFIG': {
'host': 'localhost',
'db': 'test_xmodule',
'collection': 'modulestore{0}'.format(uuid4().hex[:5]),
},
'OPTIONS': {
'default_class': 'xmodule.raw_module.RawDescriptor',
'fs_root': data_dir,
'render_template': 'edxmako.shortcuts.render_to_string'
}
}
}
store['direct'] = store['default']
return store
@@ -83,6 +57,7 @@ def draft_mongo_store_config(data_dir):
store = {
'default': {
'NAME': 'draft',
'ENGINE': 'xmodule.modulestore.mongo.draft.DraftModuleStore',
'DOC_STORE_CONFIG': {
'host': 'localhost',
@@ -93,7 +68,6 @@ def draft_mongo_store_config(data_dir):
}
}
store['direct'] = store['default']
return store
@@ -103,6 +77,7 @@ def xml_store_config(data_dir):
"""
store = {
'default': {
'NAME': 'xml',
'ENGINE': 'xmodule.modulestore.xml.XMLModuleStore',
'OPTIONS': {
'data_dir': data_dir,
@@ -111,48 +86,6 @@ def xml_store_config(data_dir):
}
}
store['direct'] = store['default']
return store
def studio_store_config(data_dir):
"""
Defines modulestore structure used by Studio tests.
"""
store_config = {
'host': 'localhost',
'db': 'test_xmodule',
'collection': 'modulestore{0}'.format(uuid4().hex[:5]),
}
options = {
'default_class': 'xmodule.raw_module.RawDescriptor',
'fs_root': data_dir,
'render_template': 'edxmako.shortcuts.render_to_string',
}
store = {
'default': {
'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
'DOC_STORE_CONFIG': store_config,
'OPTIONS': options
},
'direct': {
'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
'DOC_STORE_CONFIG': store_config,
'OPTIONS': options
},
'draft': {
'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
'DOC_STORE_CONFIG': store_config,
'OPTIONS': options
},
'split': {
'ENGINE': 'xmodule.modulestore.split_mongo.SplitMongoModuleStore',
'DOC_STORE_CONFIG': store_config,
'OPTIONS': options
}
}
return store
@@ -203,20 +136,19 @@ class ModuleStoreTestCase(TestCase):
'course' is an instance of CourseDescriptor for which we want
to update metadata.
"""
store = editable_modulestore()
store = modulestore()
store.update_item(course, '**replace_user**')
updated_course = store.get_course(course.id)
return updated_course
@staticmethod
def drop_mongo_collections(store_name='default'):
def drop_mongo_collections(modulestore_type=MONGO_MODULESTORE_TYPE):
"""
If using a Mongo-backed modulestore & contentstore, drop the collections.
"""
# This will return the mongo-backed modulestore
# even if we're using a mixed modulestore
store = editable_modulestore(store_name)
store = modulestore()
if hasattr(store, '_get_modulestore_by_type'):
store = store._get_modulestore_by_type(modulestore_type) # pylint: disable=W0212
if hasattr(store, 'collection'):
connection = store.collection.database.connection
store.collection.drop()
@@ -268,7 +200,7 @@ class ModuleStoreTestCase(TestCase):
def _pre_setup(self):
"""
Flush the ModuleStore before each test.
Flush the ModuleStore.
"""
# Flush the Mongo modulestore

61
common/lib/xmodule/xmodule/modulestore/tests/factories.py
View File

@@ -5,6 +5,10 @@ from uuid import uuid4
from xmodule.modulestore import prefer_xmodules
from opaque_keys.edx.locations import Location
from xblock.core import XBlock
from xmodule.tabs import StaticTab
from decorator import contextmanager
from mock import Mock, patch
from nose.tools import assert_less_equal
class Dummy(object):
@@ -23,10 +27,8 @@ class XModuleFactory(Factory):
@lazy_attribute
def modulestore(self):
# Delayed import so that we only depend on django if the caller
# hasn't provided their own modulestore
from xmodule.modulestore.django import editable_modulestore
return editable_modulestore('direct')
from xmodule.modulestore.django import modulestore
return modulestore()
class CourseFactory(XModuleFactory):
@@ -63,7 +65,7 @@ class CourseFactory(XModuleFactory):
# Save the attributes we just set
new_course.save()
# Update the data in the mongo datastore
store.update_item(new_course)
store.update_item(new_course, '**replace_user**')
return new_course
@@ -141,6 +143,7 @@ class ItemFactory(XModuleFactory):
display_name = kwargs.pop('display_name', None)
metadata = kwargs.pop('metadata', {})
location = kwargs.pop('location')
user_id = kwargs.pop('user_id', 999)
assert isinstance(location, Location)
assert location != parent_location
@@ -162,7 +165,8 @@ class ItemFactory(XModuleFactory):
# replace the display name with an optional parameter passed in from the caller
if display_name is not None:
metadata['display_name'] = display_name
store.create_and_save_xmodule(location, metadata=metadata, definition_data=data)
runtime = parent.runtime if parent else None
store.create_and_save_xmodule(location, user_id, metadata=metadata, definition_data=data, runtime=runtime)
module = store.get_item(location)
@@ -171,10 +175,53 @@ class ItemFactory(XModuleFactory):
# Save the attributes we just set
module.save()
store.update_item(module)
store.update_item(module, '**replace_user**')
if 'detached' not in module._class_tags:
parent.children.append(location)
store.update_item(parent, '**replace_user**')
# VS[compat] cdodge: This is a hack because static_tabs also have references from the course module, so
# if we add one then we need to also add it to the policy information (i.e. metadata)
# we should remove this once we can break this reference from the course to static tabs
if category == 'static_tab':
course = store.get_course(location.course_key)
course.tabs.append(
StaticTab(
name=display_name,
url_slug=location.name,
)
)
store.update_item(course, '**replace_user**')
return store.get_item(location)
@contextmanager
def check_mongo_calls(mongo_store, max_finds=0, max_sends=None):
"""
Instruments the given store to count the number of calls to find (incl find_one) and the number
of calls to send_message which is for insert, update, and remove (if you provide max_sends). At the
end of the with statement, it compares the counts to the max_finds and max_sends using a simple
assertLessEqual.
:param mongo_store: the MongoModulestore or subclass to watch
:param max_finds: the maximum number of find calls to allow
:param max_sends: If none, don't instrument the send calls. If non-none, count and compare to
the given int value.
"""
try:
find_wrap = Mock(wraps=mongo_store.collection.find)
wrap_patch = patch.object(mongo_store.collection, 'find', find_wrap)
wrap_patch.start()
if max_sends:
sends_wrap = Mock(wraps=mongo_store.database.connection._send_message)
sends_patch = patch.object(mongo_store.database.connection, '_send_message', sends_wrap)
sends_patch.start()
yield
finally:
wrap_patch.stop()
if max_sends:
sends_patch.stop()
assert_less_equal(sends_wrap.call_count, max_sends)
assert_less_equal(find_wrap.call_count, max_finds)

7
common/lib/xmodule/xmodule/modulestore/tests/persistent_factories.py
View File

@@ -1,3 +1,4 @@
from xmodule.modulestore import SPLIT_MONGO_MODULESTORE_TYPE, BRANCH_NAME_DRAFT
from xmodule.course_module import CourseDescriptor
from xmodule.x_module import XModuleDescriptor
import factory
@@ -14,7 +15,7 @@ class SplitFactory(factory.Factory):
# Delayed import so that we only depend on django if the caller
# hasn't provided their own modulestore
from xmodule.modulestore.django import modulestore
return modulestore('split')
return modulestore()._get_modulestore_by_type(SPLIT_MONGO_MODULESTORE_TYPE)
class PersistentCourseFactory(SplitFactory):
@@ -24,7 +25,7 @@ class PersistentCourseFactory(SplitFactory):
keywords: any xblock field plus (note, the below are filtered out; so, if they
become legitimate xblock fields, they won't be settable via this factory)
* org: defaults to textX
* master_branch: (optional) defaults to 'draft'
* master_branch: (optional) defaults to BRANCH_NAME_DRAFT
* user_id: (optional) defaults to 'test_user'
* display_name (xblock field): will default to 'Robot Super Course' unless provided
"""
@@ -33,7 +34,7 @@ class PersistentCourseFactory(SplitFactory):
# pylint: disable=W0613
@classmethod
def _create(cls, target_class, offering='999', org='testX', user_id='test_user',
master_branch='draft', **kwargs):
master_branch=BRANCH_NAME_DRAFT, **kwargs):
modulestore = kwargs.pop('modulestore')
root_block_id = kwargs.pop('root_block_id', 'course')

45
common/lib/xmodule/xmodule/modulestore/tests/test_location_mapper.py
View File

@@ -5,6 +5,7 @@ import unittest
import uuid
from opaque_keys.edx.locations import Location
from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator
from xmodule.modulestore import BRANCH_NAME_PUBLISHED, BRANCH_NAME_DRAFT, KEY_REVISION_PUBLISHED
from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError
from xmodule.modulestore.loc_mapper_store import LocMapperStore
from mock import Mock
@@ -62,8 +63,8 @@ class TestLocationMapper(LocMapperSetupSansDjango):
self.assertIsNotNone(entry, "Didn't find entry")
self.assertEqual(entry['org'], org)
self.assertEqual(entry['offering'], '{}.{}'.format(course1, run))
self.assertEqual(entry['draft_branch'], 'draft')
self.assertEqual(entry['prod_branch'], 'published')
self.assertEqual(entry['draft_branch'], BRANCH_NAME_DRAFT)
self.assertEqual(entry['prod_branch'], BRANCH_NAME_PUBLISHED)
self.assertEqual(entry['block_map'], {})
course2 = 'quux_course'
@@ -123,7 +124,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
"""
prob_locator = loc_mapper().translate_location(
location,
published=(branch == 'published'),
published=(branch == BRANCH_NAME_PUBLISHED),
add_entry_if_missing=add_entry
)
self.assertEqual(prob_locator.org, org)
@@ -133,7 +134,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
course_locator = loc_mapper().translate_location_to_course_locator(
location.course_key,
published=(branch == 'published'),
published=(branch == BRANCH_NAME_PUBLISHED),
)
self.assertEqual(course_locator.org, org)
self.assertEqual(course_locator.offering, offering)
@@ -168,7 +169,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
)
test_problem_locn = Location(org, course, run, 'problem', 'abc123')
self.translate_n_check(test_problem_locn, new_style_org, new_style_offering, 'problem2', 'published')
self.translate_n_check(test_problem_locn, new_style_org, new_style_offering, 'problem2', BRANCH_NAME_PUBLISHED)
# look for non-existent problem
with self.assertRaises(ItemNotFoundError):
loc_mapper().translate_location(
@@ -183,7 +184,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
test_no_cat_locn = test_no_cat_locn.replace(name='def456')
self.translate_n_check(
test_no_cat_locn, new_style_org, new_style_offering, 'problem4', 'published'
test_no_cat_locn, new_style_org, new_style_offering, 'problem4', BRANCH_NAME_PUBLISHED
)
# add a distractor course (note that abc123 has a different translation in this one)
@@ -202,12 +203,12 @@ class TestLocationMapper(LocMapperSetupSansDjango):
)
# test that old translation still works
self.translate_n_check(
test_problem_locn, new_style_org, new_style_offering, 'problem2', 'published'
test_problem_locn, new_style_org, new_style_offering, 'problem2', BRANCH_NAME_PUBLISHED
)
# and new returns new id
self.translate_n_check(
test_problem_locn.replace(run=run), test_delta_new_org, test_delta_new_offering,
'problem3', 'published'
'problem3', BRANCH_NAME_PUBLISHED
)
def test_translate_location_dwim(self):
@@ -221,11 +222,11 @@ class TestLocationMapper(LocMapperSetupSansDjango):
problem_name = 'abc123abc123abc123abc123abc123f9'
location = Location(org, course, run, 'problem', problem_name)
new_offering = '{}.{}'.format(course, run)
self.translate_n_check(location, org, new_offering, 'problemabc', 'published', True)
self.translate_n_check(location, org, new_offering, 'problemabc', BRANCH_NAME_PUBLISHED, True)
# create an entry w/o a guid name
other_location = Location(org, course, run, 'chapter', 'intro')
self.translate_n_check(other_location, org, new_offering, 'intro', 'published', True)
self.translate_n_check(other_location, org, new_offering, 'intro', BRANCH_NAME_PUBLISHED, True)
# add a distractor course
delta_new_org = '{}.geek_dept'.format(org)
@@ -237,7 +238,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
delta_new_org, delta_new_offering,
block_map={problem_name: {'problem': 'problem3'}}
)
self.translate_n_check(location, org, new_offering, 'problemabc', 'published', True)
self.translate_n_check(location, org, new_offering, 'problemabc', BRANCH_NAME_PUBLISHED, True)
# add a new one to both courses (ensure name doesn't have same beginning)
new_prob_name = uuid.uuid4().hex
@@ -245,10 +246,10 @@ class TestLocationMapper(LocMapperSetupSansDjango):
new_prob_name = uuid.uuid4().hex
new_prob_locn = location.replace(name=new_prob_name)
new_usage_id = 'problem{}'.format(new_prob_name[:3])
self.translate_n_check(new_prob_locn, org, new_offering, new_usage_id, 'published', True)
self.translate_n_check(new_prob_locn, org, new_offering, new_usage_id, BRANCH_NAME_PUBLISHED, True)
new_prob_locn = new_prob_locn.replace(run=run)
self.translate_n_check(
new_prob_locn, delta_new_org, delta_new_offering, new_usage_id, 'published', True
new_prob_locn, delta_new_org, delta_new_offering, new_usage_id, BRANCH_NAME_PUBLISHED, True
)
def test_translate_locator(self):
@@ -263,7 +264,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
new_style_offering = '{}.{}'.format(course, run)
prob_course_key = CourseLocator(
org=new_style_org, offering=new_style_offering,
branch='published',
branch=BRANCH_NAME_PUBLISHED,
)
prob_locator = BlockUsageLocator(
prob_course_key,
@@ -285,22 +286,22 @@ class TestLocationMapper(LocMapperSetupSansDjango):
# only one course matches
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
# default branch
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', None))
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', KEY_REVISION_PUBLISHED))
# test get_course keyword
prob_location = loc_mapper().translate_locator_to_location(prob_locator, get_course=True)
self.assertEqual(prob_location, SlashSeparatedCourseKey(org, course, run))
# explicit branch
prob_locator = prob_locator.for_branch('draft')
prob_locator = prob_locator.for_branch(BRANCH_NAME_DRAFT)
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
# Even though the problem was set as draft, we always return revision=None to work
# Even though the problem was set as draft, we always return revision= KEY_REVISION_PUBLISHED to work
# with old mongo/draft modulestores.
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', None))
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', KEY_REVISION_PUBLISHED))
prob_locator = BlockUsageLocator(
prob_course_key.for_branch('production'),
block_type='problem', block_id='problem2'
)
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', None))
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', KEY_REVISION_PUBLISHED))
# same for chapter except chapter cannot be draft in old system
chap_locator = BlockUsageLocator(
prob_course_key.for_branch('production'),
@@ -309,7 +310,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
chap_location = loc_mapper().translate_locator_to_location(chap_locator)
self.assertEqual(chap_location, Location(org, course, run, 'chapter', '48f23a10395384929234'))
# explicit branch
chap_locator = chap_locator.for_branch('draft')
chap_locator = chap_locator.for_branch(BRANCH_NAME_DRAFT)
chap_location = loc_mapper().translate_locator_to_location(chap_locator)
self.assertEqual(chap_location, Location(org, course, run, 'chapter', '48f23a10395384929234'))
chap_locator = BlockUsageLocator(
@@ -320,7 +321,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
# look for non-existent problem
prob_locator2 = BlockUsageLocator(
prob_course_key.for_branch('draft'),
prob_course_key.for_branch(BRANCH_NAME_DRAFT),
block_type='problem', block_id='problem3'
)
prob_location = loc_mapper().translate_locator_to_location(prob_locator2)
@@ -335,7 +336,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
block_map={'abc123': {'problem': 'problem3'}}
)
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', None))
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', KEY_REVISION_PUBLISHED))
def test_special_chars(self):
"""

255
common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py
View File

@@ -3,10 +3,14 @@ from uuid import uuid4
import ddt
from mock import patch, Mock
from importlib import import_module
from collections import namedtuple
from xmodule.tests import DATA_DIR
from opaque_keys.edx.locations import Location
from xmodule.modulestore import MONGO_MODULESTORE_TYPE, SPLIT_MONGO_MODULESTORE_TYPE, XML_MODULESTORE_TYPE
from xmodule.modulestore import (
MONGO_MODULESTORE_TYPE, SPLIT_MONGO_MODULESTORE_TYPE, XML_MODULESTORE_TYPE,
REVISION_OPTION_DRAFT_PREFERRED, REVISION_OPTION_PUBLISHED_ONLY, BRANCH_DRAFT_PREFERRED
)
from xmodule.modulestore.exceptions import ItemNotFoundError
from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator
@@ -15,7 +19,6 @@ from xmodule.modulestore.tests.test_location_mapper import LocMapperSetupSansDja
# before importing the module
from django.conf import settings
from opaque_keys.edx.locations import SlashSeparatedCourseKey
import bson.son
if not settings.configured:
settings.configure()
from xmodule.modulestore.mixed import MixedModuleStore
@@ -55,32 +58,29 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
XML_COURSEID1: 'xml',
XML_COURSEID2: 'xml',
BAD_COURSE_ID: 'xml',
MONGO_COURSEID: 'default',
},
'stores': {
'xml': {
'stores': [
{
'NAME': 'draft',
'ENGINE': 'xmodule.modulestore.mongo.draft.DraftModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': modulestore_options
},
{
'NAME': 'split',
'ENGINE': 'xmodule.modulestore.split_mongo.SplitMongoModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': modulestore_options
},
{
'NAME': 'xml',
'ENGINE': 'xmodule.modulestore.xml.XMLModuleStore',
'OPTIONS': {
'data_dir': DATA_DIR,
'default_class': 'xmodule.hidden_module.HiddenDescriptor',
}
},
'direct': {
'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': modulestore_options
},
'draft': {
'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': modulestore_options
},
'split': {
'ENGINE': 'xmodule.modulestore.split_mongo.SplitMongoModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': modulestore_options
}
}
]
}
def _compareIgnoreVersion(self, loc1, loc2, msg=None):
@@ -115,9 +115,11 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
self.addTypeEqualityFunc(BlockUsageLocator, '_compareIgnoreVersion')
self.addTypeEqualityFunc(CourseLocator, '_compareIgnoreVersion')
# define attrs which get set in initdb to quell pylint
self.import_chapter_location = self.store = self.fake_location = self.xml_chapter_location = None
self.writable_chapter_location = self.store = self.fake_location = self.xml_chapter_location = None
self.course_locations = []
self.user_id = 0
# pylint: disable=invalid-name
def _create_course(self, default, course_key):
"""
@@ -127,19 +129,72 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
offering = course_key.offering.replace('/', '.')
else:
offering = course_key.offering
course = self.store.create_course(course_key.org, offering, store_name=default)
category = self.import_chapter_location.category
block_id = self.import_chapter_location.name
course = self.store.create_course(course_key.org, offering)
category = self.writable_chapter_location.category
block_id = self.writable_chapter_location.name
chapter = self.store.create_item(
# don't use course_location as it may not be the repr
course.location, category, location=self.import_chapter_location, block_id=block_id
course.location, category, location=self.writable_chapter_location, block_id=block_id
)
if isinstance(course.id, CourseLocator):
self.course_locations[self.MONGO_COURSEID] = course.location.version_agnostic()
self.import_chapter_location = chapter.location.version_agnostic()
self.writable_chapter_location = chapter.location.version_agnostic()
else:
self.assertEqual(course.id, course_key)
self.assertEqual(chapter.location, self.import_chapter_location)
self.assertEqual(chapter.location, self.writable_chapter_location)
self.course = course
def _create_block_hierarchy(self):
"""
Creates a hierarchy of blocks for testing
Each block is assigned as a field of the class and can be easily accessed
"""
BlockInfo = namedtuple('BlockInfo', 'field_name, category, display_name, sub_tree')
trees = [
BlockInfo(
'chapter_x', 'chapter', 'Chapter_x', [
BlockInfo(
'sequential_x1', 'sequential', 'Sequential_x1', [
BlockInfo(
'vertical_x1a', 'vertical', 'Vertical_x1a', [
BlockInfo('problem_x1a_1', 'problem', 'Problem_x1a_1', []),
BlockInfo('problem_x1a_2', 'problem', 'Problem_x1a_2', []),
BlockInfo('problem_x1a_3', 'problem', 'Problem_x1a_3', []),
]
)
]
)
]
),
BlockInfo(
'chapter_y', 'chapter', 'Chapter_y', [
BlockInfo(
'sequential_y1', 'sequential', 'Sequential_y1', [
BlockInfo(
'vertical_y1a', 'vertical', 'Vertical_y1a', [
BlockInfo('problem_y1a_1', 'problem', 'Problem_y1a_1', []),
BlockInfo('problem_y1a_2', 'problem', 'Problem_y1a_2', []),
BlockInfo('problem_y1a_3', 'problem', 'Problem_y1a_3', []),
]
)
]
)
]
)
]
def create_sub_tree(parent, block_info):
block = self.store.create_item(parent.location, category=block_info.category, block_id=block_info.display_name)
for tree in block_info.sub_tree:
create_sub_tree(block, tree)
# reload the block to update its children field
block = self.store.get_item(block.location)
setattr(self, block_info.field_name, block)
for tree in trees:
create_sub_tree(self.course, tree)
def _course_key_from_string(self, string):
"""
@@ -152,7 +207,12 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
Initialize the database and create one test course in it
"""
# set the default modulestore
self.options['stores']['default'] = self.options['stores'][default]
store_configs = self.options['stores']
for index in range(len(store_configs)):
if store_configs[index]['NAME'] == default:
if index > 0:
store_configs[index], store_configs[0] = store_configs[0], store_configs[index]
break
self.store = MixedModuleStore(**self.options)
self.addCleanup(self.store.close_all_connections)
@@ -167,19 +227,20 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
for course_id, course_key in self.course_locations.iteritems() # pylint: disable=maybe-no-member
}
self.fake_location = Location('foo', 'bar', 'slowly', 'vertical', 'baz')
self.import_chapter_location = self.course_locations[self.MONGO_COURSEID].replace(
self.writable_chapter_location = self.course_locations[self.MONGO_COURSEID].replace(
category='chapter', name='Overview'
)
self.xml_chapter_location = self.course_locations[self.XML_COURSEID1].replace(
category='chapter', name='Overview'
)
# get Locators and set up the loc mapper if app is Locator based
if default == 'split':
self.fake_location = loc_mapper().translate_location(self.fake_location)
self._create_course(default, self.course_locations[self.MONGO_COURSEID].course_key)
@ddt.data('direct', 'split')
@ddt.data('draft', 'split')
def test_get_modulestore_type(self, default_ms):
"""
Make sure we get back the store type we expect for given mappings
@@ -191,7 +252,7 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
self.assertEqual(self.store.get_modulestore_type(
self._course_key_from_string(self.XML_COURSEID2)), XML_MODULESTORE_TYPE
)
mongo_ms_type = MONGO_MODULESTORE_TYPE if default_ms == 'direct' else SPLIT_MONGO_MODULESTORE_TYPE
mongo_ms_type = MONGO_MODULESTORE_TYPE if default_ms == 'draft' else SPLIT_MONGO_MODULESTORE_TYPE
self.assertEqual(self.store.get_modulestore_type(
self._course_key_from_string(self.MONGO_COURSEID)), mongo_ms_type
)
@@ -200,7 +261,7 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
SlashSeparatedCourseKey('foo', 'bar', '2012_Fall')), mongo_ms_type
)
@ddt.data('direct', 'split')
@ddt.data('draft', 'split')
def test_has_item(self, default_ms):
self.initdb(default_ms)
for course_locn in self.course_locations.itervalues(): # pylint: disable=maybe-no-member
@@ -212,7 +273,7 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
))
self.assertFalse(self.store.has_item(self.fake_location))
@ddt.data('direct', 'split')
@ddt.data('draft', 'split')
def test_get_item(self, default_ms):
self.initdb(default_ms)
for course_locn in self.course_locations.itervalues(): # pylint: disable=maybe-no-member
@@ -226,7 +287,7 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
with self.assertRaises(ItemNotFoundError):
self.store.get_item(self.fake_location)
@ddt.data('direct', 'split')
@ddt.data('draft', 'split')
def test_get_items(self, default_ms):
self.initdb(default_ms)
for course_locn in self.course_locations.itervalues(): # pylint: disable=maybe-no-member
@@ -236,7 +297,7 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
self.assertEqual(len(modules), 1)
self.assertEqual(modules[0].location, course_locn)
@ddt.data('direct', 'split')
@ddt.data('draft', 'split')
def test_update_item(self, default_ms):
"""
Update should fail for r/o dbs and succeed for r/w ones
@@ -246,7 +307,7 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
# if following raised, then the test is really a noop, change it
self.assertFalse(course.show_calculator, "Default changed making test meaningless")
course.show_calculator = True
with self.assertRaises(AttributeError): # ensure it doesn't allow writing
with self.assertRaises(NotImplementedError): # ensure it doesn't allow writing
self.store.update_item(course, None)
# now do it for a r/w db
course = self.store.get_course(self.course_locations[self.MONGO_COURSEID].course_key)
@@ -257,21 +318,21 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
course = self.store.get_course(self.course_locations[self.MONGO_COURSEID].course_key)
self.assertTrue(course.show_calculator)
@ddt.data('direct', 'split')
@ddt.data('draft', 'split')
def test_delete_item(self, default_ms):
"""
Delete should reject on r/o db and work on r/w one
"""
self.initdb(default_ms)
# r/o try deleting the course (is here to ensure it can't be deleted)
with self.assertRaises(AttributeError):
self.store.delete_item(self.xml_chapter_location)
self.store.delete_item(self.import_chapter_location, '**replace_user**')
with self.assertRaises(NotImplementedError):
self.store.delete_item(self.xml_chapter_location, 13)
self.store.delete_item(self.writable_chapter_location, 9)
# verify it's gone
with self.assertRaises(ItemNotFoundError):
self.store.get_item(self.import_chapter_location)
self.store.get_item(self.writable_chapter_location)
@ddt.data('direct', 'split')
@ddt.data('draft', 'split')
def test_get_courses(self, default_ms):
self.initdb(default_ms)
# we should have 3 total courses across all stores
@@ -290,8 +351,9 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
"""
Test that the xml modulestore only loaded the courses from the maps.
"""
self.initdb('direct')
courses = self.store.modulestores['xml'].get_courses()
self.initdb('draft')
xml_store = self.store._get_modulestore_by_type(XML_MODULESTORE_TYPE)
courses = xml_store.get_courses()
self.assertEqual(len(courses), 2)
course_ids = [course.id for course in courses]
self.assertIn(self.course_locations[self.XML_COURSEID1].course_key, course_ids)
@@ -303,11 +365,13 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
"""
Test that the xml modulestore doesn't allow write ops.
"""
self.initdb('direct')
with self.assertRaises(NotImplementedError):
self.store.create_course("org", "course/run", store_name='xml')
self.initdb('draft')
xml_store = self.store._get_modulestore_by_type(XML_MODULESTORE_TYPE)
# the important thing is not which exception it raises but that it raises an exception
with self.assertRaises(AttributeError):
xml_store.create_course("org", "course/run", 999)
@ddt.data('direct', 'split')
@ddt.data('draft', 'split')
def test_get_course(self, default_ms):
self.initdb(default_ms)
for course_location in self.course_locations.itervalues(): # pylint: disable=maybe-no-member
@@ -316,18 +380,90 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
self.assertIsNotNone(course)
self.assertEqual(course.id, course_location.course_key)
@ddt.data('direct', 'split')
@ddt.data('draft', 'split')
def test_get_parent_locations(self, default_ms):
self.initdb(default_ms)
parents = self.store.get_parent_locations(self.import_chapter_location)
self.assertEqual(len(parents), 1)
self.assertEqual(parents[0], self.course_locations[self.MONGO_COURSEID])
parent = self.store.get_parent_location(self.writable_chapter_location)
self.assertEqual(parent, self.course_locations[self.MONGO_COURSEID])
parents = self.store.get_parent_locations(self.xml_chapter_location)
self.assertEqual(len(parents), 1)
self.assertEqual(parents[0], self.course_locations[self.XML_COURSEID1])
parent = self.store.get_parent_location(self.xml_chapter_location)
self.assertEqual(parent, self.course_locations[self.XML_COURSEID1])
@ddt.data('direct', 'split')
def verify_get_parent_locations_results(self, expected_results):
# expected_results should be a list of (child, parent, revision)
for test in expected_results:
self.assertEqual(
test[1].location if test[1] else None,
self.store.get_parent_location(test[0].location, revision=test[2])
)
@ddt.data('draft')
def test_get_parent_locations_moved_child(self, default_ms):
self.initdb(default_ms)
self._create_block_hierarchy()
# publish the course
self.store.publish(self.course.location, self.user_id)
# make drafts of verticals
self.store.convert_to_draft(self.vertical_x1a.location, self.user_id)
self.store.convert_to_draft(self.vertical_y1a.location, self.user_id)
# move child problem_x1a_1 to vertical_y1a
child_to_move = self.problem_x1a_1
old_parent = self.vertical_x1a
new_parent = self.vertical_y1a
old_parent.children.remove(child_to_move.location)
new_parent.children.append(child_to_move.location)
self.store.update_item(old_parent, self.user_id)
self.store.update_item(new_parent, self.user_id)
self.verify_get_parent_locations_results([
(child_to_move, new_parent, None),
(child_to_move, new_parent, REVISION_OPTION_DRAFT_PREFERRED),
(child_to_move, old_parent, REVISION_OPTION_PUBLISHED_ONLY),
])
# publish the course again
self.store.publish(self.course.location, self.user_id)
self.verify_get_parent_locations_results([
(child_to_move, new_parent, None),
(child_to_move, new_parent, REVISION_OPTION_DRAFT_PREFERRED),
(child_to_move, new_parent, REVISION_OPTION_PUBLISHED_ONLY),
])
@ddt.data('draft')
def test_get_parent_locations_deleted_child(self, default_ms):
self.initdb(default_ms)
self._create_block_hierarchy()
# publish the course
self.store.publish(self.course.location, self.user_id)
# make draft of vertical
self.store.convert_to_draft(self.vertical_y1a.location, self.user_id)
# delete child problem_y1a_1
child_to_delete = self.problem_y1a_1
old_parent = self.vertical_y1a
self.store.delete_item(child_to_delete.location, self.user_id)
self.verify_get_parent_locations_results([
(child_to_delete, old_parent, None),
# Note: The following could be an unexpected result, but we want to avoid an extra database call
(child_to_delete, old_parent, REVISION_OPTION_DRAFT_PREFERRED),
(child_to_delete, old_parent, REVISION_OPTION_PUBLISHED_ONLY),
])
# publish the course again
self.store.publish(self.course.location, self.user_id)
self.verify_get_parent_locations_results([
(child_to_delete, None, None),
(child_to_delete, None, REVISION_OPTION_DRAFT_PREFERRED),
(child_to_delete, None, REVISION_OPTION_PUBLISHED_ONLY),
])
@ddt.data('draft', 'split')
def test_get_orphans(self, default_ms):
self.initdb(default_ms)
# create an orphan
@@ -339,7 +475,7 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
else:
self.assertEqual(found_orphans, [orphan.location.to_deprecated_string()])
@ddt.data('direct')
@ddt.data('draft')
def test_create_item_from_parent_location(self, default_ms):
"""
Test a code path missed by the above: passing an old-style location as parent but no
@@ -350,7 +486,7 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
orphans = self.store.get_orphans(self.course_locations[self.MONGO_COURSEID].course_key)
self.assertEqual(len(orphans), 0, "unexpected orphans: {}".format(orphans))
@ddt.data('direct')
@ddt.data('draft')
def test_get_courses_for_wiki(self, default_ms):
"""
Test the get_courses_for_wiki method
@@ -393,5 +529,6 @@ def create_modulestore_instance(engine, doc_store_config, options, i18n_service=
return class_(
doc_store_config=doc_store_config,
branch_setting_func=lambda: BRANCH_DRAFT_PREFERRED,
**options
)

4
common/lib/xmodule/xmodule/modulestore/tests/test_modulestore.py
View File

@@ -63,8 +63,10 @@ def check_has_course_method(modulestore, locator, locator_key_fields):
]
for changes in locator_case_changes:
search_locator = locator.replace(**changes)
# if ignore_case is true, the course would be found with a different-cased course locator.
# if ignore_case is false, the course should NOT found given an incorrectly-cased locator.
assert_equals(
modulestore.has_course(search_locator, ignore_case),
modulestore.has_course(search_locator, ignore_case) is not None,
ignore_case,
error_message.format(search_locator, ignore_case)
)

127
common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py
View File

@@ -20,10 +20,9 @@ from xblock.plugin import Plugin
from xmodule.tests import DATA_DIR
from opaque_keys.edx.locations import Location
from xmodule.modulestore import MONGO_MODULESTORE_TYPE
from xmodule.modulestore import MONGO_MODULESTORE_TYPE, BRANCH_DRAFT_PREFERRED
from xmodule.modulestore.mongo import MongoModuleStore, MongoKeyValueStore
from xmodule.modulestore.draft import DraftModuleStore
from xmodule.modulestore.mongo.draft import as_draft
from opaque_keys.edx.locations import SlashSeparatedCourseKey, AssetLocation
from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.xml_importer import import_from_xml, perform_xlint
@@ -34,6 +33,8 @@ from nose.tools import assert_in
from xmodule.exceptions import NotFoundError
from git.test.lib.asserts import assert_not_none
from xmodule.x_module import XModuleMixin
from xmodule.modulestore.mongo.base import as_draft
from xmodule.modulestore.tests.factories import check_mongo_calls
log = logging.getLogger(__name__)
@@ -47,7 +48,7 @@ DEFAULT_CLASS = 'xmodule.raw_module.RawDescriptor'
RENDER_TEMPLATE = lambda t_n, d, ctx = None, nsp = 'main': ''
class ReferenceTestXBlock(XBlock):
class ReferenceTestXBlock(XBlock, XModuleMixin):
"""
Test xblock type to test the reference field types
"""
@@ -76,7 +77,7 @@ class TestMongoModuleStore(unittest.TestCase):
# is ok only as long as none of the tests modify the db.
# If (when!) that changes, need to either reload the db, or load
# once and copy over to a tmp db for each test.
cls.store, cls.content_store, cls.draft_store = cls.initdb()
cls.content_store, cls.draft_store = cls.initdb()
@classmethod
def teardownClass(cls):
@@ -93,22 +94,29 @@ class TestMongoModuleStore(unittest.TestCase):
'db': DB,
'collection': COLLECTION,
}
store = MongoModuleStore(
doc_store_config, FS_ROOT, RENDER_TEMPLATE, default_class=DEFAULT_CLASS,
xblock_mixins=(XModuleMixin,)
)
# since MongoModuleStore and MongoContentStore are basically assumed to be together, create this class
# as well
content_store = MongoContentStore(HOST, DB)
#
# Also test draft store imports
#
draft_store = DraftModuleStore(doc_store_config, FS_ROOT, RENDER_TEMPLATE, default_class=DEFAULT_CLASS)
import_from_xml(store, DATA_DIR, TestMongoModuleStore.courses, draft_store=draft_store, static_content_store=content_store)
draft_store = DraftModuleStore(
doc_store_config, FS_ROOT, RENDER_TEMPLATE,
default_class=DEFAULT_CLASS,
branch_setting_func=lambda: BRANCH_DRAFT_PREFERRED
)
import_from_xml(
draft_store,
999,
DATA_DIR,
TestMongoModuleStore.courses,
static_content_store=content_store
)
# also test a course with no importing of static content
import_from_xml(
store,
draft_store,
999,
DATA_DIR,
['test_import_course'],
static_content_store=content_store,
@@ -116,7 +124,7 @@ class TestMongoModuleStore(unittest.TestCase):
verbose=True
)
return store, content_store, draft_store
return content_store, draft_store
@staticmethod
def destroy_db(connection):
@@ -140,11 +148,11 @@ class TestMongoModuleStore(unittest.TestCase):
{'host': HOST, 'db': DB, 'collection': COLLECTION},
FS_ROOT, RENDER_TEMPLATE, default_class=DEFAULT_CLASS
)
assert_equals(store.get_modulestore_type('foo/bar/baz'), MONGO_MODULESTORE_TYPE)
assert_equals(store.get_modulestore_type(''), MONGO_MODULESTORE_TYPE)
def test_get_courses(self):
'''Make sure the course objects loaded properly'''
courses = self.store.get_courses()
courses = self.draft_store.get_courses()
assert_equals(len(courses), 5)
course_ids = [course.id for course in courses]
for course_key in [
@@ -157,14 +165,14 @@ class TestMongoModuleStore(unittest.TestCase):
]
]:
assert_in(course_key, course_ids)
course = self.store.get_course(course_key)
course = self.draft_store.get_course(course_key)
assert_not_none(course)
assert_true(self.store.has_course(course_key))
assert_true(self.draft_store.has_course(course_key))
mix_cased = SlashSeparatedCourseKey(
course_key.org.upper(), course_key.course.upper(), course_key.run.lower()
)
assert_false(self.store.has_course(mix_cased))
assert_true(self.store.has_course(mix_cased, ignore_case=True))
assert_false(self.draft_store.has_course(mix_cased))
assert_true(self.draft_store.has_course(mix_cased, ignore_case=True))
def test_no_such_course(self):
"""
@@ -178,26 +186,26 @@ class TestMongoModuleStore(unittest.TestCase):
['NO_SUCH_COURSE', 'Test_iMport_courSe', '2012_Fall'],
]
]:
course = self.store.get_course(course_key)
course = self.draft_store.get_course(course_key)
assert_is_none(course)
assert_false(self.store.has_course(course_key))
assert_false(self.draft_store.has_course(course_key))
mix_cased = SlashSeparatedCourseKey(
course_key.org.lower(), course_key.course.upper(), course_key.run.upper()
)
assert_false(self.store.has_course(mix_cased))
assert_false(self.store.has_course(mix_cased, ignore_case=True))
assert_false(self.draft_store.has_course(mix_cased))
assert_false(self.draft_store.has_course(mix_cased, ignore_case=True))
def test_loads(self):
assert_not_none(
self.store.get_item(Location('edX', 'toy', '2012_Fall', 'course', '2012_Fall'))
self.draft_store.get_item(Location('edX', 'toy', '2012_Fall', 'course', '2012_Fall'))
)
assert_not_none(
self.store.get_item(Location('edX', 'simple', '2012_Fall', 'course', '2012_Fall')),
self.draft_store.get_item(Location('edX', 'simple', '2012_Fall', 'course', '2012_Fall')),
)
assert_not_none(
self.store.get_item(Location('edX', 'toy', '2012_Fall', 'video', 'Welcome')),
self.draft_store.get_item(Location('edX', 'toy', '2012_Fall', 'video', 'Welcome')),
)
def test_unicode_loads(self):
@@ -205,36 +213,37 @@ class TestMongoModuleStore(unittest.TestCase):
Test that getting items from the test_unicode course works
"""
assert_not_none(
self.store.get_item(Location('edX', 'test_unicode', '2012_Fall', 'course', '2012_Fall')),
self.draft_store.get_item(Location('edX', 'test_unicode', '2012_Fall', 'course', '2012_Fall')),
)
# All items with ascii-only filenames should load properly.
assert_not_none(
self.store.get_item(Location('edX', 'test_unicode', '2012_Fall', 'video', 'Welcome')),
self.draft_store.get_item(Location('edX', 'test_unicode', '2012_Fall', 'video', 'Welcome')),
)
assert_not_none(
self.store.get_item(Location('edX', 'test_unicode', '2012_Fall', 'video', 'Welcome')),
self.draft_store.get_item(Location('edX', 'test_unicode', '2012_Fall', 'video', 'Welcome')),
)
assert_not_none(
self.store.get_item(Location('edX', 'test_unicode', '2012_Fall', 'chapter', 'Overview')),
self.draft_store.get_item(Location('edX', 'test_unicode', '2012_Fall', 'chapter', 'Overview')),
)
def test_find_one(self):
assert_not_none(
self.store._find_one(Location('edX', 'toy', '2012_Fall', 'course', '2012_Fall')),
self.draft_store._find_one(Location('edX', 'toy', '2012_Fall', 'course', '2012_Fall')),
)
assert_not_none(
self.store._find_one(Location('edX', 'simple', '2012_Fall', 'course', '2012_Fall')),
self.draft_store._find_one(Location('edX', 'simple', '2012_Fall', 'course', '2012_Fall')),
)
assert_not_none(
self.store._find_one(Location('edX', 'toy', '2012_Fall', 'video', 'Welcome')),
self.draft_store._find_one(Location('edX', 'toy', '2012_Fall', 'video', 'Welcome')),
)
def test_path_to_location(self):
'''Make sure that path_to_location works'''
check_path_to_location(self.store)
with check_mongo_calls(self.draft_store, 9):
check_path_to_location(self.draft_store)
def test_xlinter(self):
'''
@@ -244,7 +253,7 @@ class TestMongoModuleStore(unittest.TestCase):
assert_not_equals(perform_xlint(DATA_DIR, ['toy']), 0)
def test_get_courses_has_no_templates(self):
courses = self.store.get_courses()
courses = self.draft_store.get_courses()
for course in courses:
assert_false(
course.location.org == 'edx' and course.location.course == 'templates',
@@ -259,7 +268,7 @@ class TestMongoModuleStore(unittest.TestCase):
Assumes the information is desired for courses[4] ('toy' course).
"""
course = self.store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
course = self.draft_store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
return course.tabs[index]['name']
# There was a bug where model.save was not getting called after the static tab name
@@ -332,34 +341,34 @@ class TestMongoModuleStore(unittest.TestCase):
Test the get_courses_for_wiki method
"""
for course_number in self.courses:
course_locations = self.store.get_courses_for_wiki(course_number)
course_locations = self.draft_store.get_courses_for_wiki(course_number)
assert_equals(len(course_locations), 1)
assert_equals(Location('edX', course_number, '2012_Fall', 'course', '2012_Fall'), course_locations[0])
course_locations = self.store.get_courses_for_wiki('no_such_wiki')
course_locations = self.draft_store.get_courses_for_wiki('no_such_wiki')
assert_equals(len(course_locations), 0)
# set toy course to share the wiki with simple course
toy_course = self.store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
toy_course = self.draft_store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
toy_course.wiki_slug = 'simple'
self.store.update_item(toy_course)
self.draft_store.update_item(toy_course)
# now toy_course should not be retrievable with old wiki_slug
course_locations = self.store.get_courses_for_wiki('toy')
course_locations = self.draft_store.get_courses_for_wiki('toy')
assert_equals(len(course_locations), 0)
# but there should be two courses with wiki_slug 'simple'
course_locations = self.store.get_courses_for_wiki('simple')
course_locations = self.draft_store.get_courses_for_wiki('simple')
assert_equals(len(course_locations), 2)
for course_number in ['toy', 'simple']:
assert_in(Location('edX', course_number, '2012_Fall', 'course', '2012_Fall'), course_locations)
# configure simple course to use unique wiki_slug.
simple_course = self.store.get_course(SlashSeparatedCourseKey('edX', 'simple', '2012_Fall'))
simple_course = self.draft_store.get_course(SlashSeparatedCourseKey('edX', 'simple', '2012_Fall'))
simple_course.wiki_slug = 'edX.simple.2012_Fall'
self.store.update_item(simple_course)
self.draft_store.update_item(simple_course)
# it should be retrievable with its new wiki_slug
course_locations = self.store.get_courses_for_wiki('edX.simple.2012_Fall')
course_locations = self.draft_store.get_courses_for_wiki('edX.simple.2012_Fall')
assert_equals(len(course_locations), 1)
assert_in(Location('edX', 'simple', '2012_Fall', 'course', '2012_Fall'), course_locations)
@@ -371,13 +380,15 @@ class TestMongoModuleStore(unittest.TestCase):
course_key = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
def setup_test():
course = self.store.get_course(course_key)
course = self.draft_store.get_course(course_key)
# can't use item factory as it depends on django settings
p1ele = self.store.create_and_save_xmodule(course.id.make_usage_key('problem', 'p1'))
p2ele = self.store.create_and_save_xmodule(course.id.make_usage_key('problem', 'p2'))
p1ele = self.draft_store.create_and_save_xmodule(
course.id.make_usage_key('problem', 'p1'), 99, runtime=course.runtime)
p2ele = self.draft_store.create_and_save_xmodule(
course.id.make_usage_key('problem', 'p2'), 99, runtime=course.runtime)
self.refloc = course.id.make_usage_key('ref_test', 'ref_test')
self.store.create_and_save_xmodule(
self.refloc, fields={
self.draft_store.create_and_save_xmodule(
self.refloc, 99, runtime=course.runtime, fields={
'reference_link': p1ele.location,
'reference_list': [p1ele.location, p2ele.location],
'reference_dict': {'p1': p1ele.location, 'p2': p2ele.location},
@@ -390,10 +401,10 @@ class TestMongoModuleStore(unittest.TestCase):
for child in xblock.children:
assert_is_instance(child, Location)
course = self.store.get_course(course_key)
course = self.draft_store.get_course(course_key)
check_children(course)
refele = self.store.get_item(self.refloc)
refele = self.draft_store.get_item(self.refloc)
check_children(refele)
assert_is_instance(refele.reference_link, Location)
assert_greater(len(refele.reference_list), 0)
@@ -405,7 +416,7 @@ class TestMongoModuleStore(unittest.TestCase):
def check_mongo_fields():
def get_item(location):
return self.store._find_one(location)
return self.draft_store._find_one(as_draft(location))
def check_children(payload):
for child in payload['definition']['children']:
@@ -438,7 +449,7 @@ class TestMongoModuleStore(unittest.TestCase):
root_dir = path(mkdtemp())
try:
export_to_xml(self.store, self.content_store, course_key, root_dir, 'test_export')
export_to_xml(self.draft_store, self.content_store, course_key, root_dir, 'test_export')
assert_true(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
assert_true(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())
finally:
@@ -449,12 +460,12 @@ class TestMongoModuleStore(unittest.TestCase):
Make sure that if a non-default image path is specified that we
don't export it to the static default location
"""
course = self.store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
course = self.draft_store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
assert_true(course.course_image, 'just_a_test.jpg')
root_dir = path(mkdtemp())
try:
export_to_xml(self.store, self.content_store, course.id, root_dir, 'test_export')
export_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export')
assert_true(path(root_dir / 'test_export/static/just_a_test.jpg').isfile())
assert_false(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
finally:
@@ -465,10 +476,10 @@ class TestMongoModuleStore(unittest.TestCase):
Make sure we elegantly passover our code when there isn't a static
image
"""
course = self.store.get_course(SlashSeparatedCourseKey('edX', 'simple_with_draft', '2012_Fall'))
course = self.draft_store.get_course(SlashSeparatedCourseKey('edX', 'simple_with_draft', '2012_Fall'))
root_dir = path(mkdtemp())
try:
export_to_xml(self.store, self.content_store, course.id, root_dir, 'test_export')
export_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export')
assert_false(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
assert_false(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())
finally:
@@ -570,7 +581,7 @@ class TestMongoModuleStore(unittest.TestCase):
location = Location('edX', 'migration', '2012_Fall', 'html', 'test_html')
published_date = datetime(1970, 1, 1, tzinfo=UTC)
published_by = 123
self.store._update_single_item(
self.draft_store._update_single_item(
as_draft(location),
{
'definition.data': {},

114
common/lib/xmodule/xmodule/modulestore/tests/test_publish.py
View File

@@ -3,6 +3,7 @@ Test the publish code (mostly testing that publishing doesn't result in orphans)
"""
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.tests.test_split_w_old_mongo import SplitWMongoCourseBoostrapper
from xmodule.modulestore.tests.factories import check_mongo_calls
class TestPublish(SplitWMongoCourseBoostrapper):
@@ -14,53 +15,48 @@ class TestPublish(SplitWMongoCourseBoostrapper):
Create the course, publish all verticals
* some detached items
"""
super(TestPublish, self)._create_course(split=False)
# There should be 12 inserts and 11 updates (max_sends)
# Should be 1 to verify course unique, 11 parent fetches,
# and n per _create_item where n is the size of the course tree non-leaf nodes
# for inheritance computation (which is 7*4 + sum(1..4) = 38) (max_finds)
with check_mongo_calls(self.draft_mongo, 70, 27):
with check_mongo_calls(self.old_mongo, 70, 27):
super(TestPublish, self)._create_course(split=False)
self._create_item('chapter', 'Chapter1', {}, {'display_name': 'Chapter 1'}, 'course', 'runid', split=False)
self._create_item('chapter', 'Chapter2', {}, {'display_name': 'Chapter 2'}, 'course', 'runid', split=False)
self._create_item('vertical', 'Vert1', {}, {'display_name': 'Vertical 1'}, 'chapter', 'Chapter1', split=False)
self._create_item('vertical', 'Vert2', {}, {'display_name': 'Vertical 2'}, 'chapter', 'Chapter1', split=False)
self._create_item('html', 'Html1', "<p>Goodbye</p>", {'display_name': 'Parented Html'}, 'vertical', 'Vert1', split=False)
self._create_item(
'discussion', 'Discussion1',
"discussion discussion_category=\"Lecture 1\" discussion_id=\"a08bfd89b2aa40fa81f2c650a9332846\" discussion_target=\"Lecture 1\"/>\n",
{
"discussion_category": "Lecture 1",
"discussion_target": "Lecture 1",
"display_name": "Lecture 1 Discussion",
"discussion_id": "a08bfd89b2aa40fa81f2c650a9332846"
},
'vertical', 'Vert1',
split=False
)
self._create_item('html', 'Html2', "<p>Hellow</p>", {'display_name': 'Hollow Html'}, 'vertical', 'Vert1', split=False)
self._create_item(
'discussion', 'Discussion2',
"discussion discussion_category=\"Lecture 2\" discussion_id=\"b08bfd89b2aa40fa81f2c650a9332846\" discussion_target=\"Lecture 2\"/>\n",
{
"discussion_category": "Lecture 2",
"discussion_target": "Lecture 2",
"display_name": "Lecture 2 Discussion",
"discussion_id": "b08bfd89b2aa40fa81f2c650a9332846"
},
'vertical', 'Vert2',
split=False
)
self._create_item('static_tab', 'staticuno', "<p>tab</p>", {'display_name': 'Tab uno'}, None, None, split=False)
self._create_item('about', 'overview', "<p>overview</p>", {}, None, None, split=False)
self._create_item('course_info', 'updates', "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>", {}, None, None, split=False)
self._create_item('chapter', 'Chapter1', {}, {'display_name': 'Chapter 1'}, 'course', 'runid', split=False)
self._create_item('chapter', 'Chapter2', {}, {'display_name': 'Chapter 2'}, 'course', 'runid', split=False)
self._create_item('vertical', 'Vert1', {}, {'display_name': 'Vertical 1'}, 'chapter', 'Chapter1', split=False)
self._create_item('vertical', 'Vert2', {}, {'display_name': 'Vertical 2'}, 'chapter', 'Chapter1', split=False)
self._create_item('html', 'Html1', "<p>Goodbye</p>", {'display_name': 'Parented Html'}, 'vertical', 'Vert1', split=False)
self._create_item(
'discussion', 'Discussion1',
"discussion discussion_category=\"Lecture 1\" discussion_id=\"a08bfd89b2aa40fa81f2c650a9332846\" discussion_target=\"Lecture 1\"/>\n",
{
"discussion_category": "Lecture 1",
"discussion_target": "Lecture 1",
"display_name": "Lecture 1 Discussion",
"discussion_id": "a08bfd89b2aa40fa81f2c650a9332846"
},
'vertical', 'Vert1',
split=False
)
self._create_item('html', 'Html2', "<p>Hellow</p>", {'display_name': 'Hollow Html'}, 'vertical', 'Vert1', split=False)
self._create_item(
'discussion', 'Discussion2',
"discussion discussion_category=\"Lecture 2\" discussion_id=\"b08bfd89b2aa40fa81f2c650a9332846\" discussion_target=\"Lecture 2\"/>\n",
{
"discussion_category": "Lecture 2",
"discussion_target": "Lecture 2",
"display_name": "Lecture 2 Discussion",
"discussion_id": "b08bfd89b2aa40fa81f2c650a9332846"
},
'vertical', 'Vert2',
split=False
)
self._create_item('static_tab', 'staticuno', "<p>tab</p>", {'display_name': 'Tab uno'}, None, None, split=False)
self._create_item('about', 'overview', "<p>overview</p>", {}, None, None, split=False)
self._create_item('course_info', 'updates', "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>", {}, None, None, split=False)
def _xmodule_recurse(self, item, action):
"""
Applies action depth-first down tree and to item last.
A copy of cms.djangoapps.contentstore.views.helpers._xmodule_recurse to reproduce its use and behavior
outside of django.
"""
for child in item.get_children():
self._xmodule_recurse(child, action)
action(item)
def test_publish_draft_delete(self):
"""
@@ -69,29 +65,31 @@ class TestPublish(SplitWMongoCourseBoostrapper):
"""
location = self.old_course_key.make_usage_key('vertical', name='Vert1')
item = self.draft_mongo.get_item(location, 2)
self._xmodule_recurse(
item,
lambda i: self.draft_mongo.publish(i.location, self.userid)
)
# Vert1 has 3 children; so, publishes 4 nodes which may mean 4 inserts & 1 bulk remove
# 25-June-2014 find calls are 19. Probably due to inheritance recomputation?
with check_mongo_calls(self.draft_mongo, 19, 5):
self.draft_mongo.publish(item.location, self.userid)
# verify status
item = self.draft_mongo.get_item(location, 0)
self.assertFalse(getattr(item, 'is_draft', False), "Item was published. Draft should not exist")
# however, children are still draft, but I'm not sure that's by design
# convert back to draft
self.draft_mongo.convert_to_draft(location)
self.draft_mongo.convert_to_draft(location, self.userid)
# both draft and published should exist
draft_vert = self.draft_mongo.get_item(location, 0)
self.assertTrue(getattr(draft_vert, 'is_draft', False), "Item was converted to draft but doesn't say so")
item = self.old_mongo.get_item(location, 0)
self.assertFalse(getattr(item, 'is_draft', False), "Published item doesn't say so")
# delete the discussion (which oddly is not in draft mode)
# delete the draft version of the discussion
location = self.old_course_key.make_usage_key('discussion', name='Discussion1')
self.draft_mongo.delete_item(location)
# remove pointer from draft vertical (verify presence first to ensure process is valid)
self.assertIn(location, draft_vert.children)
draft_vert.children.remove(location)
self.draft_mongo.delete_item(location, self.userid)
draft_vert = self.draft_mongo.get_item(draft_vert.location, 0)
# remove pointer from draft vertical (still there b/c not refetching vert)
self.assertNotIn(location, draft_vert.children)
# move the other child
other_child_loc = self.old_course_key.make_usage_key('html', name='Html2')
draft_vert.children.remove(other_child_loc)
@@ -100,12 +98,10 @@ class TestPublish(SplitWMongoCourseBoostrapper):
self.draft_mongo.update_item(draft_vert, self.userid)
self.draft_mongo.update_item(other_vert, self.userid)
# publish
self._xmodule_recurse(
draft_vert,
lambda i: self.draft_mongo.publish(i.location, self.userid)
)
self.draft_mongo.publish(draft_vert.location, self.userid)
item = self.old_mongo.get_item(draft_vert.location, 0)
self.assertNotIn(location, item.children)
self.assertIsNone(self.draft_mongo.get_parent_location(location))
with self.assertRaises(ItemNotFoundError):
self.draft_mongo.get_item(location)
self.assertNotIn(other_child_loc, item.children)

6
common/lib/xmodule/xmodule/modulestore/tests/test_split_migrator.py
View File

@@ -5,9 +5,9 @@ Tests for split_migrator
import uuid
import random
import mock
from xmodule.modulestore import KEY_REVISION_PUBLISHED
from xmodule.modulestore.loc_mapper_store import LocMapperStore
from xmodule.modulestore.split_migrator import SplitMigrator
from xmodule.modulestore.mongo import draft
from xmodule.modulestore.tests import test_location_mapper
from xmodule.modulestore.tests.test_split_w_old_mongo import SplitWMongoCourseBoostrapper
from xblock.fields import Reference, ReferenceList, ReferenceValueDict
@@ -177,7 +177,9 @@ class TestMigration(SplitWMongoCourseBoostrapper):
# check that locations match
self.assertEqual(
presplit_dag_root.location,
self.loc_mapper.translate_locator_to_location(split_dag_root.location).replace(revision=None)
self.loc_mapper.translate_locator_to_location(split_dag_root.location).replace(
revision=KEY_REVISION_PUBLISHED
)
)
# compare all fields but children
for name, field in presplit_dag_root.fields.iteritems():

172
common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py
View File

@@ -11,6 +11,7 @@ import random
from xblock.fields import Scope
from xmodule.course_module import CourseDescriptor
from xmodule.modulestore import BRANCH_NAME_PUBLISHED, BRANCH_NAME_DRAFT
from xmodule.modulestore.exceptions import (InsufficientSpecificationError, ItemNotFoundError, VersionConflictError,
DuplicateItemError, DuplicateCourseError)
from opaque_keys.edx.locator import CourseLocator, BlockUsageLocator, VersionTree, LocalId
@@ -477,11 +478,11 @@ class SplitModuleTest(unittest.TestCase):
course = split_store.persist_xblock_dag(course, revision['user_id'])
# publish "testx.wonderful"
to_publish = BlockUsageLocator(
CourseLocator(org="testx", offering="wonderful", branch="draft"),
CourseLocator(org="testx", offering="wonderful", branch=BRANCH_NAME_DRAFT),
block_type='course',
block_id="head23456"
)
destination = CourseLocator(org="testx", offering="wonderful", branch="published")
destination = CourseLocator(org="testx", offering="wonderful", branch=BRANCH_NAME_PUBLISHED)
split_store.xblock_publish("test@edx.org", to_publish, destination, [to_publish], None)
def tearDown(self):
@@ -511,7 +512,7 @@ class TestHasChildrenAtDepth(SplitModuleTest):
def test_has_children_at_depth(self):
course_locator = CourseLocator(
org='testx', offering='GreekHero', branch='draft'
org='testx', offering='GreekHero', branch=BRANCH_NAME_DRAFT
)
block_locator = BlockUsageLocator(
course_locator, 'course', 'head12345'
@@ -548,7 +549,7 @@ class SplitModuleCourseTests(SplitModuleTest):
'''
def test_get_courses(self):
courses = modulestore().get_courses(branch='draft')
courses = modulestore().get_courses(branch=BRANCH_NAME_DRAFT)
# should have gotten 3 draft courses
self.assertEqual(len(courses), 3, "Wrong number of courses")
# check metadata -- NOTE no promised order
@@ -586,11 +587,11 @@ class SplitModuleCourseTests(SplitModuleTest):
self.assertEqual(len(course.children), 0,
"children")
_verify_published_course(modulestore().get_courses(branch='published'))
_verify_published_course(modulestore().get_courses(branch=BRANCH_NAME_PUBLISHED))
def test_search_qualifiers(self):
# query w/ search criteria
courses = modulestore().get_courses(branch='draft', qualifiers={'org': 'testx'})
courses = modulestore().get_courses(branch=BRANCH_NAME_DRAFT, qualifiers={'org': 'testx'})
self.assertEqual(len(courses), 2)
self.assertIsNotNone(self.findByIdInResult(courses, "head12345"))
self.assertIsNotNone(self.findByIdInResult(courses, "head23456"))
@@ -602,7 +603,7 @@ class SplitModuleCourseTests(SplitModuleTest):
check_has_course_method(
modulestore(),
CourseLocator(org='testx', offering='wonderful', branch="draft"),
CourseLocator(org='testx', offering='wonderful', branch=BRANCH_NAME_DRAFT),
locator_key_fields=['org', 'offering']
)
@@ -610,7 +611,7 @@ class SplitModuleCourseTests(SplitModuleTest):
'''
Test the various calling forms for get_course
'''
locator = CourseLocator(org='testx', offering='GreekHero', branch="draft")
locator = CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_DRAFT)
head_course = modulestore().get_course(locator)
self.assertNotEqual(head_course.location.version_guid, head_course.previous_version)
locator = CourseLocator(version_guid=head_course.previous_version)
@@ -628,7 +629,7 @@ class SplitModuleCourseTests(SplitModuleTest):
self.assertEqual(course.edited_by, "testassist@edx.org")
self.assertDictEqual(course.grade_cutoffs, {"Pass": 0.55})
locator = CourseLocator(org='testx', offering='GreekHero', branch='draft')
locator = CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_DRAFT)
course = modulestore().get_course(locator)
self.assertEqual(course.location.course_key.org, "testx")
self.assertEqual(course.location.course_key.offering, "GreekHero")
@@ -641,11 +642,11 @@ class SplitModuleCourseTests(SplitModuleTest):
self.assertEqual(course.edited_by, "testassist@edx.org")
self.assertDictEqual(course.grade_cutoffs, {"Pass": 0.45})
locator = CourseLocator(org='testx', offering='wonderful', branch='published')
locator = CourseLocator(org='testx', offering='wonderful', branch=BRANCH_NAME_PUBLISHED)
course = modulestore().get_course(locator)
published_version = course.location.version_guid
locator = CourseLocator(org='testx', offering='wonderful', branch='draft')
locator = CourseLocator(org='testx', offering='wonderful', branch=BRANCH_NAME_DRAFT)
course = modulestore().get_course(locator)
self.assertNotEqual(course.location.version_guid, published_version)
@@ -654,15 +655,15 @@ class SplitModuleCourseTests(SplitModuleTest):
with self.assertRaises(InsufficientSpecificationError):
modulestore().get_course(CourseLocator(org='edu', offering='meh.blah'))
with self.assertRaises(ItemNotFoundError):
modulestore().get_course(CourseLocator(org='edu', offering='nosuchthing', branch='draft'))
modulestore().get_course(CourseLocator(org='edu', offering='nosuchthing', branch=BRANCH_NAME_DRAFT))
with self.assertRaises(ItemNotFoundError):
modulestore().get_course(CourseLocator(org='testx', offering='GreekHero', branch='published'))
modulestore().get_course(CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_PUBLISHED))
def test_cache(self):
"""
Test that the mechanics of caching work.
"""
locator = CourseLocator(org='testx', offering='GreekHero', branch='draft')
locator = CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_DRAFT)
course = modulestore().get_course(locator)
block_map = modulestore().cache_items(
course.system, [child.block_id for child in course.children], course.id, depth=3
@@ -674,7 +675,7 @@ class SplitModuleCourseTests(SplitModuleTest):
"""
get_course_successors(course_locator, version_history_depth=1)
"""
locator = CourseLocator(org='testx', offering='GreekHero', branch='draft')
locator = CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_DRAFT)
course = modulestore().get_course(locator)
versions = [course.location.version_guid, course.previous_version]
locator = CourseLocator(version_guid=course.previous_version)
@@ -712,7 +713,7 @@ class SplitModuleItemTests(SplitModuleTest):
'''
org = 'testx'
offering = 'GreekHero'
course_locator = CourseLocator(org=org, offering=offering, branch='draft')
course_locator = CourseLocator(org=org, offering=offering, branch=BRANCH_NAME_DRAFT)
course = modulestore().get_course(course_locator)
previous_version = course.previous_version
# positive tests of various forms
@@ -728,7 +729,7 @@ class SplitModuleItemTests(SplitModuleTest):
self.assertFalse(
modulestore().has_item(
BlockUsageLocator(
locator.course_key.for_branch('published'),
locator.course_key.for_branch(BRANCH_NAME_PUBLISHED),
block_type=locator.block_type,
block_id=locator.block_id
)
@@ -745,25 +746,25 @@ class SplitModuleItemTests(SplitModuleTest):
# in published course
locator = BlockUsageLocator(
CourseLocator(org="testx", offering="wonderful", branch='draft'),
CourseLocator(org="testx", offering="wonderful", branch=BRANCH_NAME_DRAFT),
block_type="course",
block_id="head23456"
)
self.assertTrue(
modulestore().has_item(locator.for_branch("published"))
modulestore().has_item(locator.for_branch(BRANCH_NAME_PUBLISHED))
)
def test_negative_has_item(self):
# negative tests--not found
# no such course or block
locator = BlockUsageLocator(
CourseLocator(org="foo", offering="doesnotexist", branch='draft'),
CourseLocator(org="foo", offering="doesnotexist", branch=BRANCH_NAME_DRAFT),
block_type="course",
block_id="head23456"
)
self.assertFalse(modulestore().has_item(locator))
locator = BlockUsageLocator(
CourseLocator(org="testx", offering="wonderful", branch='draft'),
CourseLocator(org="testx", offering="wonderful", branch=BRANCH_NAME_DRAFT),
block_type="vertical",
block_id="doesnotexist"
)
@@ -773,7 +774,7 @@ class SplitModuleItemTests(SplitModuleTest):
'''
get_item(blocklocator)
'''
hero_locator = CourseLocator(org="testx", offering="GreekHero", branch='draft')
hero_locator = CourseLocator(org="testx", offering="GreekHero", branch=BRANCH_NAME_DRAFT)
course = modulestore().get_course(hero_locator)
previous_version = course.previous_version
@@ -803,14 +804,14 @@ class SplitModuleItemTests(SplitModuleTest):
# try to look up other branches
with self.assertRaises(ItemNotFoundError):
modulestore().get_item(course.location.for_branch("published"))
modulestore().get_item(course.location.for_branch(BRANCH_NAME_PUBLISHED))
def test_has_changes(self):
"""
Tests that has_changes() only returns true when changes are present
"""
draft_course = CourseLocator(org='testx', offering='GreekHero', branch='draft')
published_course = CourseLocator(org='testx', offering='GreekHero', branch='published')
draft_course = CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_DRAFT)
published_course = CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_PUBLISHED)
head = draft_course.make_usage_key('course', 'head12345')
dummy_user = 'testUser'
@@ -834,7 +835,7 @@ class SplitModuleItemTests(SplitModuleTest):
def test_get_non_root(self):
# not a course obj
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='GreekHero', branch='draft'), 'chapter', 'chapter1'
CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_DRAFT), 'chapter', 'chapter1'
)
block = modulestore().get_item(locator)
self.assertEqual(block.location.org, "testx")
@@ -845,7 +846,7 @@ class SplitModuleItemTests(SplitModuleTest):
# in published course
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='wonderful', branch='published'), 'course', 'head23456'
CourseLocator(org='testx', offering='wonderful', branch=BRANCH_NAME_PUBLISHED), 'course', 'head23456'
)
self.assertIsInstance(
modulestore().get_item(locator),
@@ -855,12 +856,12 @@ class SplitModuleItemTests(SplitModuleTest):
# negative tests--not found
# no such course or block
locator = BlockUsageLocator(
CourseLocator(org='doesnotexist', offering='doesnotexist', branch='draft'), 'course', 'head23456'
CourseLocator(org='doesnotexist', offering='doesnotexist', branch=BRANCH_NAME_DRAFT), 'course', 'head23456'
)
with self.assertRaises(ItemNotFoundError):
modulestore().get_item(locator)
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='wonderful', branch='draft'), 'html', 'doesnotexist'
CourseLocator(org='testx', offering='wonderful', branch=BRANCH_NAME_DRAFT), 'html', 'doesnotexist'
)
with self.assertRaises(ItemNotFoundError):
modulestore().get_item(locator)
@@ -894,7 +895,7 @@ class SplitModuleItemTests(SplitModuleTest):
'''
get_items(locator, qualifiers, [branch])
'''
locator = CourseLocator(org='testx', offering='GreekHero', branch='draft')
locator = CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_DRAFT)
# get all modules
matches = modulestore().get_items(locator)
self.assertEqual(len(matches), 6)
@@ -917,31 +918,30 @@ class SplitModuleItemTests(SplitModuleTest):
def test_get_parents(self):
'''
get_parent_locations(locator): [BlockUsageLocator]
get_parent_location(locator): BlockUsageLocator
'''
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='GreekHero', branch='draft'),
CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_DRAFT),
'chapter', block_id='chapter1'
)
parents = modulestore().get_parent_locations(locator)
self.assertEqual(len(parents), 1)
self.assertEqual(parents[0].block_id, 'head12345')
self.assertEqual(parents[0].org, "testx")
self.assertEqual(parents[0].offering, "GreekHero")
parent = modulestore().get_parent_location(locator)
self.assertIsNotNone(parent)
self.assertEqual(parent.block_id, 'head12345')
self.assertEqual(parent.org, "testx")
self.assertEqual(parent.offering, "GreekHero")
locator = locator.course_key.make_usage_key('Chapter', 'chapter2')
parents = modulestore().get_parent_locations(locator)
self.assertEqual(len(parents), 1)
self.assertEqual(parents[0].block_id, 'head12345')
parent = modulestore().get_parent_location(locator)
self.assertEqual(parent.block_id, 'head12345')
locator = locator.course_key.make_usage_key('garbage', 'nosuchblock')
parents = modulestore().get_parent_locations(locator)
self.assertEqual(len(parents), 0)
parent = modulestore().get_parent_location(locator)
self.assertIsNone(parent)
def test_get_children(self):
"""
Test the existing get_children method on xdescriptors
"""
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='GreekHero', branch='draft'), 'course', 'head12345'
CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_DRAFT), 'course', 'head12345'
)
block = modulestore().get_item(locator)
children = block.get_children()
@@ -988,7 +988,7 @@ class TestItemCrud(SplitModuleTest):
create_item(course_or_parent_locator, category, user, definition_locator=None, fields): new_desciptor
"""
# grab link to course to ensure new versioning works
locator = CourseLocator(org='testx', offering='GreekHero', branch='draft')
locator = CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_DRAFT)
premod_course = modulestore().get_course(locator)
premod_history = modulestore().get_course_history_info(premod_course.location)
# add minimal one w/o a parent
@@ -1024,13 +1024,13 @@ class TestItemCrud(SplitModuleTest):
Test create_item w/ specifying the parent of the new item
"""
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='GreekHero', branch='draft'),
CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_DRAFT),
'chapter', block_id='chapter2'
)
original = modulestore().get_item(locator)
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='wonderful', branch='draft'), 'course', 'head23456'
CourseLocator(org='testx', offering='wonderful', branch=BRANCH_NAME_DRAFT), 'course', 'head23456'
)
premod_course = modulestore().get_course(locator.course_key)
category = 'chapter'
@@ -1053,13 +1053,13 @@ class TestItemCrud(SplitModuleTest):
Actually, this tries to test all create_item features not tested above.
"""
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='GreekHero', branch='draft'),
CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_DRAFT),
'problem', block_id='problem1'
)
original = modulestore().get_item(locator)
locator = BlockUsageLocator(
CourseLocator(org='guestx', offering='contender', branch='draft'), 'course', 'head345679'
CourseLocator(org='guestx', offering='contender', branch=BRANCH_NAME_DRAFT), 'course', 'head345679'
)
category = 'problem'
new_payload = "<problem>empty</problem>"
@@ -1092,7 +1092,7 @@ class TestItemCrud(SplitModuleTest):
"""
Check that using odd characters in block id don't break ability to add and retrieve block.
"""
course_key = CourseLocator(org='guestx', offering='contender', branch='draft')
course_key = CourseLocator(org='guestx', offering='contender', branch=BRANCH_NAME_DRAFT)
parent_locator = BlockUsageLocator(course_key, 'course', block_id="head345679")
chapter_locator = BlockUsageLocator(course_key, 'chapter', block_id="foo.bar_-~:0")
modulestore().create_item(
@@ -1201,7 +1201,7 @@ class TestItemCrud(SplitModuleTest):
test updating an items metadata ensuring the definition doesn't version but the course does if it should
"""
locator = BlockUsageLocator(
CourseLocator(org="testx", offering="GreekHero", branch='draft'),
CourseLocator(org="testx", offering="GreekHero", branch=BRANCH_NAME_DRAFT),
'problem', block_id="problem3_2"
)
problem = modulestore().get_item(locator)
@@ -1235,7 +1235,7 @@ class TestItemCrud(SplitModuleTest):
test updating an item's children ensuring the definition doesn't version but the course does if it should
"""
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='GreekHero', branch='draft'), 'chapter', 'chapter3'
CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_DRAFT), 'chapter', 'chapter3'
)
block = modulestore().get_item(locator)
pre_def_id = block.definition_locator.definition_id
@@ -1262,7 +1262,7 @@ class TestItemCrud(SplitModuleTest):
test updating an item's definition: ensure it gets versioned as well as the course getting versioned
"""
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='GreekHero', branch='draft'), 'course', 'head12345'
CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_DRAFT), 'course', 'head12345'
)
block = modulestore().get_item(locator)
pre_def_id = block.definition_locator.definition_id
@@ -1281,13 +1281,13 @@ class TestItemCrud(SplitModuleTest):
Test updating metadata, children, and definition in a single call ensuring all the versioning occurs
"""
locator = BlockUsageLocator(
CourseLocator('testx', 'GreekHero', branch='draft'),
CourseLocator('testx', 'GreekHero', branch=BRANCH_NAME_DRAFT),
'problem', block_id='problem1'
)
original = modulestore().get_item(locator)
# first add 2 children to the course for the update to manipulate
locator = BlockUsageLocator(
CourseLocator('guestx', 'contender', branch='draft'),
CourseLocator('guestx', 'contender', branch=BRANCH_NAME_DRAFT),
'course', block_id="head345679"
)
category = 'problem'
@@ -1327,12 +1327,12 @@ class TestItemCrud(SplitModuleTest):
course = self.create_course_for_deletion()
with self.assertRaises(ValueError):
modulestore().delete_item(course.location, 'deleting_user')
reusable_location = course.id.version_agnostic().for_branch('draft')
reusable_location = course.id.version_agnostic().for_branch(BRANCH_NAME_DRAFT)
# delete a leaf
problems = modulestore().get_items(reusable_location, category='problem')
locn_to_del = problems[0].location
new_course_loc = modulestore().delete_item(locn_to_del, 'deleting_user', delete_children=False)
new_course_loc = modulestore().delete_item(locn_to_del, 'deleting_user')
deleted = locn_to_del.version_agnostic()
self.assertFalse(modulestore().has_item(deleted))
with self.assertRaises(VersionConflictError):
@@ -1343,7 +1343,7 @@ class TestItemCrud(SplitModuleTest):
# delete a subtree
nodes = modulestore().get_items(reusable_location, category='chapter')
new_course_loc = modulestore().delete_item(nodes[0].location, 'deleting_user', delete_children=True)
new_course_loc = modulestore().delete_item(nodes[0].location, 'deleting_user')
# check subtree
def check_subtree(node):
@@ -1366,7 +1366,7 @@ class TestItemCrud(SplitModuleTest):
Create a course we can delete
"""
course = modulestore().create_course('nihilx', 'deletion', 'deleting_user')
root = course.location.version_agnostic().for_branch('draft')
root = course.location.version_agnostic().for_branch(BRANCH_NAME_DRAFT)
for _ in range(4):
self.create_subtree_for_deletion(root, ['chapter', 'vertical', 'problem'])
return modulestore().get_item(root)
@@ -1400,7 +1400,7 @@ class TestCourseCreation(SplitModuleTest):
self.assertEqual(index_info['edited_by'], 'create_user')
# check structure info
structure_info = modulestore().get_course_history_info(new_locator)
self.assertEqual(structure_info['original_version'], index_info['versions']['draft'])
self.assertEqual(structure_info['original_version'], index_info['versions'][BRANCH_NAME_DRAFT])
self.assertIsNone(structure_info['previous_version'])
self.assertEqual(structure_info['edited_by'], 'create_user')
# check the returned course object
@@ -1417,7 +1417,7 @@ class TestCourseCreation(SplitModuleTest):
"""
Test making a course which points to an existing draft and published but not making any changes to either.
"""
original_locator = CourseLocator(org='testx', offering='wonderful', branch='draft')
original_locator = CourseLocator(org='testx', offering='wonderful', branch=BRANCH_NAME_DRAFT)
original_index = modulestore().get_course_index_info(original_locator)
new_draft = modulestore().create_course(
'best', 'leech', 'leech_master',
@@ -1426,15 +1426,15 @@ class TestCourseCreation(SplitModuleTest):
self.assertRegexpMatches(new_draft_locator.org, 'best')
# the edited_by and other meta fields on the new course will be the original author not this one
self.assertEqual(new_draft.edited_by, 'test@edx.org')
self.assertEqual(new_draft_locator.version_guid, original_index['versions']['draft'])
self.assertEqual(new_draft_locator.version_guid, original_index['versions'][BRANCH_NAME_DRAFT])
# however the edited_by and other meta fields on course_index will be this one
new_index = modulestore().get_course_index_info(new_draft_locator)
self.assertEqual(new_index['edited_by'], 'leech_master')
new_published_locator = new_draft_locator.course_key.for_branch("published")
new_published_locator = new_draft_locator.course_key.for_branch(BRANCH_NAME_PUBLISHED)
new_published = modulestore().get_course(new_published_locator)
self.assertEqual(new_published.edited_by, 'test@edx.org')
self.assertEqual(new_published.location.version_guid, original_index['versions']['published'])
self.assertEqual(new_published.location.version_guid, original_index['versions'][BRANCH_NAME_PUBLISHED])
# changing this course will not change the original course
# using new_draft.location will insert the chapter under the course root
@@ -1444,22 +1444,22 @@ class TestCourseCreation(SplitModuleTest):
)
new_draft_locator = new_draft_locator.course_key.version_agnostic()
new_index = modulestore().get_course_index_info(new_draft_locator)
self.assertNotEqual(new_index['versions']['draft'], original_index['versions']['draft'])
self.assertNotEqual(new_index['versions'][BRANCH_NAME_DRAFT], original_index['versions'][BRANCH_NAME_DRAFT])
new_draft = modulestore().get_course(new_draft_locator)
self.assertEqual(new_item.edited_by, 'leech_master')
self.assertNotEqual(new_item.location.version_guid, original_index['versions']['draft'])
self.assertNotEqual(new_draft.location.version_guid, original_index['versions']['draft'])
self.assertNotEqual(new_item.location.version_guid, original_index['versions'][BRANCH_NAME_DRAFT])
self.assertNotEqual(new_draft.location.version_guid, original_index['versions'][BRANCH_NAME_DRAFT])
structure_info = modulestore().get_course_history_info(new_draft_locator)
self.assertEqual(structure_info['edited_by'], 'leech_master')
original_course = modulestore().get_course(original_locator)
self.assertEqual(original_course.location.version_guid, original_index['versions']['draft'])
self.assertEqual(original_course.location.version_guid, original_index['versions'][BRANCH_NAME_DRAFT])
def test_derived_course(self):
"""
Create a new course which overrides metadata and course_data
"""
original_locator = CourseLocator(org='guestx', offering='contender', branch='draft')
original_locator = CourseLocator(org='guestx', offering='contender', branch=BRANCH_NAME_DRAFT)
original = modulestore().get_course(original_locator)
original_index = modulestore().get_course_index_info(original_locator)
fields = {}
@@ -1477,14 +1477,14 @@ class TestCourseCreation(SplitModuleTest):
fields['display_name'] = 'Derivative'
new_draft = modulestore().create_course(
'counter', 'leech', 'leech_master',
versions_dict={'draft': original_index['versions']['draft']},
versions_dict={BRANCH_NAME_DRAFT: original_index['versions'][BRANCH_NAME_DRAFT]},
fields=fields
)
new_draft_locator = new_draft.location
self.assertRegexpMatches(new_draft_locator.org, 'counter')
# the edited_by and other meta fields on the new course will be the original author not this one
self.assertEqual(new_draft.edited_by, 'leech_master')
self.assertNotEqual(new_draft_locator.version_guid, original_index['versions']['draft'])
self.assertNotEqual(new_draft_locator.version_guid, original_index['versions'][BRANCH_NAME_DRAFT])
# however the edited_by and other meta fields on course_index will be this one
new_index = modulestore().get_course_index_info(new_draft_locator)
self.assertEqual(new_index['edited_by'], 'leech_master')
@@ -1500,22 +1500,22 @@ class TestCourseCreation(SplitModuleTest):
it's not clear how you'd find them again or associate them w/ existing student history since
we use course_key so many places as immutable.
"""
locator = CourseLocator(org='testx', offering='GreekHero', branch='draft')
locator = CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_DRAFT)
course_info = modulestore().get_course_index_info(locator)
# an allowed but not necessarily recommended way to revert the draft version
head_course = modulestore().get_course(locator)
versions = course_info['versions']
versions['draft'] = head_course.previous_version
versions[BRANCH_NAME_DRAFT] = head_course.previous_version
modulestore().update_course_index(course_info)
course = modulestore().get_course(locator)
self.assertEqual(course.location.version_guid, versions['draft'])
self.assertEqual(course.location.version_guid, versions[BRANCH_NAME_DRAFT])
# an allowed but not recommended way to publish a course
versions['published'] = versions['draft']
versions[BRANCH_NAME_PUBLISHED] = versions[BRANCH_NAME_DRAFT]
modulestore().update_course_index(course_info)
course = modulestore().get_course(locator.for_branch("published"))
self.assertEqual(course.location.version_guid, versions['draft'])
course = modulestore().get_course(locator.for_branch(BRANCH_NAME_PUBLISHED))
self.assertEqual(course.location.version_guid, versions[BRANCH_NAME_DRAFT])
def test_create_with_root(self):
"""
@@ -1559,13 +1559,13 @@ class TestInheritance(SplitModuleTest):
# Note, not testing value where defined (course) b/c there's no
# defined accessor for it on CourseDescriptor.
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='GreekHero', branch='draft'), 'problem', 'problem3_2'
CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_DRAFT), 'problem', 'problem3_2'
)
node = modulestore().get_item(locator)
# inherited
self.assertEqual(node.graceperiod, datetime.timedelta(hours=2))
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='GreekHero', branch='draft'), 'problem', 'problem1'
CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_DRAFT), 'problem', 'problem1'
)
node = modulestore().get_item(locator)
# overridden
@@ -1587,8 +1587,8 @@ class TestPublish(SplitModuleTest):
"""
Test the standard patterns: publish to new branch, revise and publish
"""
source_course = CourseLocator(org='testx', offering='GreekHero', branch='draft')
dest_course = CourseLocator(org='testx', offering='GreekHero', branch="published")
source_course = CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_DRAFT)
dest_course = CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_PUBLISHED)
head = source_course.make_usage_key('course', "head12345")
chapter1 = source_course.make_usage_key('chapter', 'chapter1')
chapter2 = source_course.make_usage_key('chapter', 'chapter2')
@@ -1614,7 +1614,7 @@ class TestPublish(SplitModuleTest):
# check that it is in the published course and that its parent is the chapter
pub_module = modulestore().get_item(new_module.location.map_into_course(dest_course))
self.assertEqual(
modulestore().get_parent_locations(pub_module.location)[0].block_id, chapter1.block_id
modulestore().get_parent_location(pub_module.location).block_id, chapter1.block_id
)
# ensure intentionally orphaned blocks work (e.g., course_info)
new_module = modulestore().create_item(
@@ -1633,16 +1633,16 @@ class TestPublish(SplitModuleTest):
"""
Test the exceptions which preclude successful publication
"""
source_course = CourseLocator(org='testx', offering='GreekHero', branch='draft')
source_course = CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_DRAFT)
# destination does not exist
destination_course = CourseLocator(org='fake', offering='Unknown', branch="published")
destination_course = CourseLocator(org='fake', offering='Unknown', branch=BRANCH_NAME_PUBLISHED)
head = source_course.make_usage_key('course', "head12345")
chapter3 = source_course.make_usage_key('chapter', 'chapter3')
problem1 = source_course.make_usage_key('problem', 'problem1')
with self.assertRaises(ItemNotFoundError):
modulestore().xblock_publish(self.user, source_course, destination_course, [chapter3], None)
# publishing into a new branch w/o publishing the root
destination_course = CourseLocator(org='testx', offering='GreekHero', branch="published")
destination_course = CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_PUBLISHED)
with self.assertRaises(ItemNotFoundError):
modulestore().xblock_publish(self.user, source_course, destination_course, [chapter3], None)
# publishing a subdag w/o the parent already in course
@@ -1654,8 +1654,8 @@ class TestPublish(SplitModuleTest):
"""
Test publishing moves and deletes.
"""
source_course = CourseLocator(org='testx', offering='GreekHero', branch='draft')
dest_course = CourseLocator(org='testx', offering='GreekHero', branch="published")
source_course = CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_DRAFT)
dest_course = CourseLocator(org='testx', offering='GreekHero', branch=BRANCH_NAME_PUBLISHED)
head = source_course.make_usage_key('course', "head12345")
chapter2 = source_course.make_usage_key('chapter', 'chapter2')
problem1 = source_course.make_usage_key('problem', 'problem1')

10
common/lib/xmodule/xmodule/modulestore/tests/test_split_w_old_mongo.py
View File

@@ -9,6 +9,8 @@ from opaque_keys.edx.locator import CourseLocator, BlockUsageLocator
from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore
from xmodule.modulestore.mongo import MongoModuleStore, DraftMongoModuleStore
from xmodule.modulestore.mongo.draft import DIRECT_ONLY_CATEGORIES
from xmodule.modulestore import BRANCH_DRAFT_PREFERRED, BRANCH_NAME_DRAFT
from mock import Mock
class SplitWMongoCourseBoostrapper(unittest.TestCase):
@@ -39,7 +41,7 @@ class SplitWMongoCourseBoostrapper(unittest.TestCase):
'xblock_mixins': (InheritanceMixin,)
}
split_course_key = CourseLocator('test_org', 'test_course.runid', branch='draft')
split_course_key = CourseLocator('test_org', 'test_course.runid', branch=BRANCH_NAME_DRAFT)
def setUp(self):
self.db_config['collection'] = 'modulestore{0}'.format(uuid.uuid4().hex[:5])
@@ -53,7 +55,9 @@ class SplitWMongoCourseBoostrapper(unittest.TestCase):
self.addCleanup(self.split_mongo.db.connection.close)
self.addCleanup(self.tear_down_split)
self.old_mongo = MongoModuleStore(self.db_config, **self.modulestore_options)
self.draft_mongo = DraftMongoModuleStore(self.db_config, **self.modulestore_options)
self.draft_mongo = DraftMongoModuleStore(
self.db_config, branch_setting_func=lambda: BRANCH_DRAFT_PREFERRED, **self.modulestore_options
)
self.addCleanup(self.tear_down_mongo)
self.old_course_key = None
self.runtime = None
@@ -86,7 +90,7 @@ class SplitWMongoCourseBoostrapper(unittest.TestCase):
mongo = self.old_mongo
else:
mongo = self.draft_mongo
mongo.create_and_save_xmodule(location, data, metadata, self.runtime)
mongo.create_and_save_xmodule(location, self.userid, definition_data=data, metadata=metadata, runtime=self.runtime)
if isinstance(data, basestring):
fields = {'data': data}
else:

2
common/lib/xmodule/xmodule/modulestore/tests/test_xml.py
View File

@@ -43,7 +43,7 @@ class TestXMLModuleStore(unittest.TestCase):
def test_xml_modulestore_type(self):
store = XMLModuleStore(DATA_DIR, course_dirs=['toy', 'simple'])
self.assertEqual(store.get_modulestore_type('foo/bar/baz'), XML_MODULESTORE_TYPE)
self.assertEqual(store.get_modulestore_type(), XML_MODULESTORE_TYPE)
def test_unicode_chars_in_xml_content(self):
# edX/full/6.002_Spring_2012 has non-ASCII chars, and during

11
common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py
View File

@@ -8,7 +8,7 @@ from xblock.runtime import Runtime, KvsFieldData, DictKeyValueStore
from xmodule.x_module import XModuleMixin
from opaque_keys.edx.locations import Location
from xmodule.modulestore.inheritance import InheritanceMixin
from xmodule.modulestore.xml_importer import import_module
from xmodule.modulestore.xml_importer import _import_module_and_update_references
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from xmodule.tests import DATA_DIR
from uuid import uuid4
@@ -141,9 +141,10 @@ class RemapNamespaceTest(ModuleStoreNoSettings):
# Move to different runtime w/ different course id
target_location_namespace = SlashSeparatedCourseKey("org", "course", "run")
new_version = import_module(
new_version = _import_module_and_update_references(
self.xblock,
modulestore(),
999,
self.xblock.location.course_key,
target_location_namespace,
do_import_static=False
@@ -177,9 +178,10 @@ class RemapNamespaceTest(ModuleStoreNoSettings):
# Remap the namespace
target_location_namespace = Location("org", "course", "run", "category", "stubxblock")
new_version = import_module(
new_version = _import_module_and_update_references(
self.xblock,
modulestore(),
999,
self.xblock.location.course_key,
target_location_namespace.course_key,
do_import_static=False
@@ -208,9 +210,10 @@ class RemapNamespaceTest(ModuleStoreNoSettings):
# Remap the namespace
target_location_namespace = Location("org", "course", "run", "category", "stubxblock")
new_version = import_module(
new_version = _import_module_and_update_references(
self.xblock,
modulestore(),
999,
self.xblock.location.course_key,
target_location_namespace.course_key,
do_import_static=False

32
common/lib/xmodule/xmodule/modulestore/xml.py
View File

@@ -19,6 +19,7 @@ from xmodule.errortracker import make_error_tracker, exc_info_to_str
from xmodule.mako_module import MakoDescriptorSystem
from xmodule.x_module import XMLParsingSystem, policy_key
from xmodule.modulestore.xml_exporter import DEFAULT_CONTENT_FIELDS
from xmodule.modulestore import REVISION_OPTION_PUBLISHED_ONLY
from xmodule.tabs import CourseTabList
from opaque_keys.edx.keys import UsageKey
from opaque_keys.edx.locations import SlashSeparatedCourseKey
@@ -67,6 +68,7 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
# cdodge: adding the course_id as passed in for later reference rather than having to recomine the org/course/url_name
self.course_id = course_id
self.load_error_modules = load_error_modules
self.modulestore = xmlstore
def process_xml(xml):
"""Takes an xml string, and returns a XBlock created from
@@ -332,7 +334,7 @@ class ParentTracker(object):
"""
Init
"""
# location -> set(parents). Not using defaultdict because we care about the empty case.
# location -> parent. Not using defaultdict because we care about the empty case.
self._parents = dict()
def add_parent(self, child, parent):
@@ -341,8 +343,7 @@ class ParentTracker(object):
child and parent must be :class:`.Location` instances.
"""
setp = self._parents.setdefault(child, set())
setp.add(parent)
self._parents[child] = parent
def is_known(self, child):
"""
@@ -353,13 +354,13 @@ class ParentTracker(object):
def make_known(self, location):
"""Tell the parent tracker about an object, without registering any
parents for it. Used for the top level course descriptor locations."""
self._parents.setdefault(location, set())
self._parents.setdefault(location, None)
def parents(self, child):
def parent(self, child):
"""
Return a list of the parents of this child. If not is_known(child), will throw a KeyError
Return the parent of this child. If not is_known(child), will throw a KeyError
"""
return list(self._parents[child])
return self._parents[child]
class XMLModuleStore(ModuleStoreReadBase):
@@ -409,6 +410,9 @@ class XMLModuleStore(ModuleStoreReadBase):
self.i18n_service = i18n_service
# The XML Module Store is a read-only store and only handles published content
self.branch_setting_func = lambda: REVISION_OPTION_PUBLISHED_ONLY
# If we are specifically asked for missing courses, that should
# be an error. If we are asked for "all" courses, find the ones
# that have a course.xml. We sort the dirs in alpha order so we always
@@ -785,25 +789,25 @@ class XMLModuleStore(ModuleStoreReadBase):
# here just to quell the abstractmethod. someone could write the impl if needed
raise NotImplementedError
def get_parent_locations(self, location):
'''Find all locations that are the parents of this location in this
def get_parent_location(self, location, **kwargs):
'''Find the location that is the parent of this location in this
course. Needed for path_to_location().
returns an iterable of things that can be passed to Location. This may
be empty if there are no parents.
'''
if not self.parent_trackers[location.course_key].is_known(location):
raise ItemNotFoundError("{0} not in {1}".format(location, location.course_key))
return self.parent_trackers[location.course_key].parents(location)
return self.parent_trackers[location.course_key].parent(location)
def get_modulestore_type(self, course_id):
def get_modulestore_type(self, course_key=None):
"""
Returns an enumeration-like type reflecting the type of this modulestore
The return can be one of:
"xml" (for XML based courses),
"mongo" for old-style MongoDB backed courses,
"split" for new-style split MongoDB backed courses.
Args:
course_key: just for signature compatibility
"""
return XML_MODULESTORE_TYPE

74
common/lib/xmodule/xmodule/modulestore/xml_exporter.py
View File

@@ -7,15 +7,18 @@ import lxml.etree
from xblock.fields import Scope
from xmodule.contentstore.content import StaticContent
from xmodule.exceptions import NotFoundError
from opaque_keys.edx.locations import Location
from xmodule.modulestore import (
EdxJSONEncoder, BRANCH_PUBLISHED_ONLY, REVISION_OPTION_DRAFT_PREFERRED, REVISION_OPTION_DRAFT_ONLY
)
from xmodule.modulestore.inheritance import own_metadata
from xmodule.modulestore.mixed import store_branch_setting
from fs.osfs import OSFS
from json import dumps
import json
import datetime
import os
from path import path
import shutil
from xmodule.modulestore.mongo.base import DIRECT_ONLY_CATEGORIES
DRAFT_DIR = "drafts"
PUBLISHED_DIR = "published"
@@ -25,29 +28,7 @@ EXPORT_VERSION_KEY = "export_format"
DEFAULT_CONTENT_FIELDS = ['metadata', 'data']
class EdxJSONEncoder(json.JSONEncoder):
"""
Custom JSONEncoder that handles `Location` and `datetime.datetime` objects.
`Location`s are encoded as their url string form, and `datetime`s as
ISO date strings
"""
def default(self, obj):
if isinstance(obj, Location):
return obj.to_deprecated_string()
elif isinstance(obj, datetime.datetime):
if obj.tzinfo is not None:
if obj.utcoffset() is None:
return obj.isoformat() + 'Z'
else:
return obj.isoformat()
else:
return obj.isoformat()
else:
return super(EdxJSONEncoder, self).default(obj)
def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir, draft_modulestore=None):
def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir):
"""
Export all modules from `modulestore` and content from `contentstore` as xml to `root_dir`.
@@ -56,8 +37,6 @@ def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir, d
`course_key`: The `CourseKey` of the `CourseModuleDescriptor` to export
`root_dir`: The directory to write the exported xml to
`course_dir`: The name of the directory inside `root_dir` to write the course content to
`draft_modulestore`: An optional `DraftModuleStore` that contains draft content, which will be exported
alongside the public content in the course.
"""
course = modulestore.get_course(course_key)
@@ -66,7 +45,10 @@ def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir, d
export_fs = course.runtime.export_fs = fsm.makeopendir(course_dir)
root = lxml.etree.Element('unknown')
course.add_xml_to_node(root)
# export only the published content
with store_branch_setting(course.runtime.modulestore, BRANCH_PUBLISHED_ONLY):
course.add_xml_to_node(root)
with export_fs.open('course.xml', 'w') as course_xml:
lxml.etree.ElementTree(root).write(course_xml)
@@ -121,26 +103,26 @@ def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir, d
policy = {'course/' + course.location.name: own_metadata(course)}
course_policy.write(dumps(policy, cls=EdxJSONEncoder))
# export draft content
# NOTE: this code assumes that verticals are the top most draftable container
# should we change the application, then this assumption will no longer
# be valid
if draft_modulestore is not None:
draft_verticals = draft_modulestore.get_items(course_key, category='vertical', revision='draft')
if len(draft_verticals) > 0:
draft_course_dir = export_fs.makeopendir(DRAFT_DIR)
for draft_vertical in draft_verticals:
parent_locs = draft_modulestore.get_parent_locations(draft_vertical.location)
# Don't try to export orphaned items.
if len(parent_locs) > 0:
logging.debug('parent_locs = {0}'.format(parent_locs))
draft_vertical.xml_attributes['parent_sequential_url'] = parent_locs[0].to_deprecated_string()
sequential = modulestore.get_item(parent_locs[0])
# should we change the application, then this assumption will no longer be valid
# NOTE: we need to explicitly implement the logic for setting the vertical's parent
# and index here since the XML modulestore cannot load draft modules
draft_verticals = modulestore.get_items(course_key, category='vertical', revision=REVISION_OPTION_DRAFT_ONLY)
if len(draft_verticals) > 0:
draft_course_dir = export_fs.makeopendir(DRAFT_DIR)
for draft_vertical in draft_verticals:
parent_loc = modulestore.get_parent_location(draft_vertical.location, revision=REVISION_OPTION_DRAFT_PREFERRED)
# Don't try to export orphaned items.
if parent_loc is not None:
logging.debug('parent_loc = {0}'.format(parent_loc))
if parent_loc.category in DIRECT_ONLY_CATEGORIES:
draft_vertical.xml_attributes['parent_sequential_url'] = parent_loc.to_deprecated_string()
sequential = modulestore.get_item(parent_loc)
index = sequential.children.index(draft_vertical.location)
draft_vertical.xml_attributes['index_in_children_list'] = str(index)
draft_vertical.runtime.export_fs = draft_course_dir
node = lxml.etree.Element('unknown')
draft_vertical.add_xml_to_node(node)
draft_vertical.runtime.export_fs = draft_course_dir
node = lxml.etree.Element('unknown')
draft_vertical.add_xml_to_node(node)
def _export_field_content(xblock_item, item_dir):
@@ -205,7 +187,7 @@ def convert_between_versions(source_dir, target_dir):
shutil.copytree(published_dir, copy_root)
# If there is a "draft" branch, copy it. All other branches are ignored.
# If there is a DRAFT branch, copy it. All other branches are ignored.
copy_drafts()
def copy_drafts():

107
common/lib/xmodule/xmodule/modulestore/xml_importer.py
View File

@@ -10,12 +10,14 @@ from xmodule.x_module import XModuleDescriptor
from opaque_keys.edx.keys import UsageKey
from xblock.fields import Scope, Reference, ReferenceList, ReferenceValueDict
from xmodule.contentstore.content import StaticContent
from xmodule.modulestore.mixed import store_bulk_write_operations_on_course
from .inheritance import own_metadata
from xmodule.errortracker import make_error_tracker
from .store_utilities import rewrite_nonportable_content_links
import xblock
from xmodule.tabs import CourseTabList
from xmodule.modulestore.exceptions import InvalidLocationError
from xmodule.modulestore import KEY_REVISION_PUBLISHED, KEY_REVISION_DRAFT
log = logging.getLogger(__name__)
@@ -107,10 +109,10 @@ def import_static_content(
def import_from_xml(
store, data_dir, course_dirs=None,
store, user_id, data_dir, course_dirs=None,
default_class='xmodule.raw_module.RawDescriptor',
load_error_modules=True, static_content_store=None,
target_course_id=None, verbose=False, draft_store=None,
target_course_id=None, verbose=False,
do_import_static=True, create_new_course_if_not_present=False):
"""
Import the specified xml data_dir into the "store" modulestore,
@@ -171,18 +173,13 @@ def import_from_xml(
store.create_course(dest_course_id.org, dest_course_id.offering)
except InvalidLocationError:
# course w/ same org and course exists
log.debug(
"Skipping import of course with id, {0},"
"since it collides with an existing one".format(dest_course_id)
)
continue
try:
# turn off all write signalling while importing as this
# is a high volume operation on stores that need it
if hasattr(store, 'ignore_write_events_on_courses'):
store.ignore_write_events_on_courses.add(dest_course_id)
log.debug(
"Skipping import of course with id, {0},"
"since it collides with an existing one".format(dest_course_id)
)
continue
with store_bulk_write_operations_on_course(store, dest_course_id):
course_data_path = None
if verbose:
@@ -209,8 +206,8 @@ def import_from_xml(
log.debug('course data_dir={0}'.format(module.data_dir))
course = import_module(
module, store,
course = _import_module_and_update_references(
module, store, user_id,
course_key,
dest_course_id,
do_import_static=do_import_static
@@ -249,9 +246,12 @@ def import_from_xml(
if course.tabs is None or len(course.tabs) == 0:
CourseTabList.initialize_default(course)
store.update_item(course)
store.update_item(course, user_id)
course_items.append(course)
break
# TODO: shouldn't this raise an exception if course wasn't found?
# then import all the static content
if static_content_store is not None and do_import_static:
@@ -284,7 +284,7 @@ def import_from_xml(
dest_course_id, subpath=simport, verbose=verbose
)
# finally loop through all the modules
# now loop through all the modules
for module in xml_module_store.modules[course_key].itervalues():
if module.scope_ids.block_type == 'course':
# we've already saved the course module up at the top
@@ -296,41 +296,36 @@ def import_from_xml(
loc=module.location
))
import_module(
_import_module_and_update_references(
module, store,
user_id,
course_key,
dest_course_id,
do_import_static=do_import_static,
system=course.runtime
runtime=course.runtime
)
# now import any 'draft' items
if draft_store is not None:
import_course_draft(
xml_module_store,
store,
draft_store,
course_data_path,
static_content_store,
course_key,
dest_course_id,
course.runtime
)
# finally, publish the course
store.publish(course.location, user_id)
finally:
# turn back on all write signalling on stores that need it
if (hasattr(store, 'ignore_write_events_on_courses') and
dest_course_id in store.ignore_write_events_on_courses):
store.ignore_write_events_on_courses.remove(dest_course_id)
store.refresh_cached_metadata_inheritance_tree(dest_course_id)
# now import any DRAFT items
_import_course_draft(
xml_module_store,
store,
user_id,
course_data_path,
course_key,
dest_course_id,
course.runtime
)
return xml_module_store, course_items
def import_module(
module, store,
def _import_module_and_update_references(
module, store, user_id,
source_course_id, dest_course_id,
do_import_static=True, system=None):
do_import_static=True, runtime=None):
logging.debug(u'processing import of module {}...'.format(module.location.to_deprecated_string()))
@@ -347,7 +342,7 @@ def import_module(
new_usage_key = module.scope_ids.usage_id.map_into_course(dest_course_id)
if new_usage_key.category == 'course':
new_usage_key = new_usage_key.replace(name=dest_course_id.run)
new_module = store.create_xmodule(new_usage_key, system=system)
new_module = store.create_xmodule(new_usage_key, runtime=runtime)
def _convert_reference_fields_to_new_namespace(reference):
"""
@@ -391,14 +386,19 @@ def import_module(
setattr(new_module, field_name, value)
else:
setattr(new_module, field_name, getattr(module, field_name))
store.update_item(new_module, '**replace_user**', allow_not_found=True)
store.update_item(new_module, user_id, allow_not_found=True)
return new_module
def import_course_draft(
xml_module_store, store, draft_store, course_data_path,
static_content_store, source_course_id,
target_course_id, mongo_runtime):
def _import_course_draft(
xml_module_store,
store,
user_id,
course_data_path,
source_course_id,
target_course_id,
mongo_runtime
):
'''
This will import all the content inside of the 'drafts' folder, if it exists
NOTE: This is not a full course import, basically in our current
@@ -502,17 +502,17 @@ def import_course_draft(
course_key = descriptor.location.course_key
try:
def _import_module(module):
# Update the module's location to "draft" revision
# Update the module's location to DRAFT revision
# We need to call this method (instead of updating the location directly)
# to ensure that pure XBlock field data is updated correctly.
_update_module_location(module, module.location.replace(revision='draft'))
_update_module_location(module, module.location.replace(revision=KEY_REVISION_DRAFT))
# make sure our parent has us in its list of children
# this is to make sure private only verticals show up
# in the list of children since they would have been
# filtered out from the non-draft store export
if module.location.category == 'vertical':
non_draft_location = module.location.replace(revision=None)
non_draft_location = module.location.replace(revision=KEY_REVISION_PUBLISHED)
sequential_url = module.xml_attributes['parent_sequential_url']
index = int(module.xml_attributes['index_in_children_list'])
@@ -525,12 +525,13 @@ def import_course_draft(
if non_draft_location not in sequential.children:
sequential.children.insert(index, non_draft_location)
store.update_item(sequential, '**replace_user**')
store.update_item(sequential, user_id)
import_module(
module, draft_store,
_import_module_and_update_references(
module, store, user_id,
source_course_id,
target_course_id, system=mongo_runtime
target_course_id,
runtime=mongo_runtime,
)
for child in module.get_children():
_import_module(child)

9
common/lib/xmodule/xmodule/split_test_module.py
View File

@@ -420,7 +420,7 @@ class SplitTestDescriptor(SplitTestFields, SequenceDescriptor, StudioEditableDes
if selected_partition is not None:
self.group_id_mapping = {} # pylint: disable=attribute-defined-outside-init
for group in selected_partition.groups:
self._create_vertical_for_group(group)
self._create_vertical_for_group(group, user.id)
# Don't need to call update_item in the modulestore because the caller of this method will do it.
else:
# If children referenced in group_id_to_child have been deleted, remove them from the map.
@@ -553,7 +553,7 @@ class SplitTestDescriptor(SplitTestFields, SequenceDescriptor, StudioEditableDes
for group in user_partition.groups:
str_group_id = unicode(group.id)
if str_group_id not in self.group_id_to_child:
self._create_vertical_for_group(group)
self._create_vertical_for_group(group, request.user.id)
changed = True
if changed:
@@ -561,7 +561,7 @@ class SplitTestDescriptor(SplitTestFields, SequenceDescriptor, StudioEditableDes
self.system.modulestore.update_item(self, None)
return Response()
def _create_vertical_for_group(self, group):
def _create_vertical_for_group(self, group, user_id):
"""
Creates a vertical to associate with the group.
@@ -576,9 +576,10 @@ class SplitTestDescriptor(SplitTestFields, SequenceDescriptor, StudioEditableDes
metadata = {'display_name': group.name}
modulestore.create_and_save_xmodule(
dest_usage_key,
user_id,
definition_data=None,
metadata=metadata,
system=self.system,
runtime=self.system,
)
self.children.append(dest_usage_key) # pylint: disable=no-member
self.group_id_to_child[unicode(group.id)] = dest_usage_key

3
common/lib/xmodule/xmodule/tests/test_export.py
View File

@@ -23,9 +23,10 @@ from xblock.fields import String, Scope, Integer
from xblock.test.tools import blocks_are_equivalent
from opaque_keys.edx.locations import Location
from xmodule.modulestore import EdxJSONEncoder
from xmodule.modulestore.xml import XMLModuleStore
from xmodule.modulestore.xml_exporter import (
EdxJSONEncoder, convert_between_versions, get_version
convert_between_versions, get_version
)
from xmodule.tests import DATA_DIR
from xmodule.tests.helpers import directories_equal

2
common/lib/xmodule/xmodule/xml_module.py
View File

@@ -8,7 +8,7 @@ from lxml import etree
from xblock.fields import Dict, Scope, ScopeIds
from xmodule.x_module import XModuleDescriptor
from xmodule.modulestore.inheritance import own_metadata, InheritanceKeyValueStore
from xmodule.modulestore.xml_exporter import EdxJSONEncoder
from xmodule.modulestore import EdxJSONEncoder
from xblock.runtime import KvsFieldData
log = logging.getLogger(__name__)

6
lms/djangoapps/branding/tests.py
View File

@@ -10,7 +10,7 @@ from django.test.utils import override_settings
from django.test.client import RequestFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.django import editable_modulestore
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.factories import CourseFactory
from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE
import student.views
@@ -28,12 +28,12 @@ class AnonymousIndexPageTest(ModuleStoreTestCase):
Tests that anonymous users can access the '/' page, Need courses with start date
"""
def setUp(self):
self.store = editable_modulestore()
self.store = modulestore()
self.factory = RequestFactory()
self.course = CourseFactory.create()
self.course.days_early_for_beta = 5
self.course.enrollment_start = datetime.datetime.now(UTC) + datetime.timedelta(days=3)
self.store.update_item(self.course)
self.store.update_item(self.course, '**replace_user**')
@override_settings(FEATURES=FEATURES_WITH_STARTDATE)
def test_none_user_index_access_with_startdate_fails(self):

5
lms/djangoapps/courseware/features/annotatable.py
View File

@@ -3,7 +3,7 @@ import textwrap
from lettuce import world, steps
from nose.tools import assert_in, assert_equals, assert_true
from common import i_am_registered_for_the_course, visit_scenario_item
from common import i_am_registered_for_the_course, visit_scenario_item, publish
DATA_TEMPLATE = textwrap.dedent("""\
<annotatable>
@@ -79,6 +79,8 @@ class AnnotatableSteps(object):
data=DATA_TEMPLATE.format("\n".join(ANNOTATION_TEMPLATE.format(i) for i in xrange(count)))
)
publish(world.scenario_dict['ANNOTATION_VERTICAL'].location)
self.annotations_count = count
def view_component(self, step):
@@ -123,6 +125,7 @@ class AnnotatableSteps(object):
)
)
)
publish(world.scenario_dict['ANNOTATION_VERTICAL'].location)
def click_reply(self, step, problem):
r"""I click "Reply to annotation" on passage (?P<problem>\d+)$"""

Some files were not shown because too many files have changed in this diff Show More