Andal.LND/edx-platform
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #2302 from edx/db/django-command-migrate-to-split

Browse Source 
Django command for migrating courses to split-mongo (and deleting)
This commit is contained in:
David Baumgold
2014-01-30 06:51:27 -08:00
parent 31593699e5 dd627edae4
commit 08fb49506a
13 changed files with 386 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
common/lib/xmodule/xmodule/modulestore/django.py
View File

@@ -152,6 +152,9 @@ def clear_existing_modulestores():
_MODULESTORES.clear()
# pylint: disable=W0603
global _loc_singleton
cache = getattr(_loc_singleton, "cache", None)
if cache:
cache.clear()
_loc_singleton = None

21
common/lib/xmodule/xmodule/modulestore/loc_mapper_store.py
View File

@@ -93,7 +93,7 @@ class LocMapperStore(object):
package_id = "{0.org}.{0.course}".format(course_location)
# very like _interpret_location_id but w/o the _id
location_id = self._construct_location_son(
course_location.org, course_location.course,
course_location.org, course_location.course,
course_location.name if course_location.category == 'course' else None
)
@@ -219,6 +219,11 @@ class LocMapperStore(object):
return None
result = None
for candidate in maps:
if get_course and 'name' in candidate['_id']:
candidate_id = candidate['_id']
return Location(
'i4x', candidate_id['org'], candidate_id['course'], 'course', candidate_id['name']
)
old_course_id = self._generate_location_course_id(candidate['_id'])
for old_name, cat_to_usage in candidate['block_map'].iteritems():
for category, block_id in cat_to_usage.iteritems():
@@ -240,7 +245,7 @@ class LocMapperStore(object):
candidate['course_id'], branch=candidate['draft_branch'], block_id=block_id
)
self._cache_location_map_entry(old_course_id, location, published_locator, draft_locator)
if get_course and category == 'course':
result = location
elif not get_course and block_id == locator.block_id:
@@ -261,8 +266,6 @@ class LocMapperStore(object):
return cached
location_id = self._interpret_location_course_id(old_style_course_id, location)
if old_style_course_id is None:
old_style_course_id = self._generate_location_course_id(location_id)
maps = self.location_map.find(location_id)
maps = list(maps)
@@ -320,10 +323,10 @@ class LocMapperStore(object):
return {'_id': self._construct_location_son(location.org, location.course, location.name)}
else:
return bson.son.SON([('_id.org', location.org), ('_id.course', location.course)])
def _generate_location_course_id(self, entry_id):
"""
Generate a Location course_id for the given entry's id
Generate a Location course_id for the given entry's id.
"""
# strip id envelope if any
entry_id = entry_id.get('_id', entry_id)
@@ -334,7 +337,7 @@ class LocMapperStore(object):
return '{0[_id.org]}/{0[_id.course]}'.format(entry_id)
else:
return '{0[org]}/{0[course]}'.format(entry_id)
def _construct_location_son(self, org, course, name=None):
"""
Construct the SON needed to repr the location for either a query or an insertion
@@ -401,6 +404,8 @@ class LocMapperStore(object):
"""
Get the course Locator for this old course id
"""
if not old_course_id:
return None
entry = self.cache.get(old_course_id)
if entry is not None:
if published:
@@ -425,6 +430,8 @@ class LocMapperStore(object):
"""
For quick lookup of courses
"""
if not old_course_id:
return
self.cache.set(old_course_id, (published_course_locator, draft_course_locator))
def _cache_location_map_entry(self, old_course_id, location, published_usage, draft_usage):

22
common/lib/xmodule/xmodule/modulestore/split_migrator.py
View File

@@ -23,7 +23,7 @@ class SplitMigrator(object):
self.draft_modulestore = draft_modulestore
self.loc_mapper = loc_mapper
def migrate_mongo_course(self, course_location, user_id, new_package_id=None):
def migrate_mongo_course(self, course_location, user, new_package_id=None):
"""
Create a new course in split_mongo representing the published and draft versions of the course from the
original mongo store. And return the new_package_id (which the caller can also get by calling
@@ -32,7 +32,7 @@ class SplitMigrator(object):
If the new course already exists, this raises DuplicateItemError
:param course_location: a Location whose category is 'course' and points to the course
:param user_id: the user whose action is causing this migration
:param user: the user whose action is causing this migration
:param new_package_id: (optional) the Locator.package_id for the new course. Defaults to
whatever translate_location_to_locator returns
"""
@@ -48,18 +48,18 @@ class SplitMigrator(object):
new_course_root_locator = self.loc_mapper.translate_location(old_course_id, course_location)
new_course = self.split_modulestore.create_course(
course_location.org, original_course.display_name,
user_id, id_root=new_package_id,
user.id, id_root=new_package_id,
fields=self._get_json_fields_translate_children(original_course, old_course_id, True),
root_block_id=new_course_root_locator.block_id,
master_branch=new_course_root_locator.branch
)
self._copy_published_modules_to_course(new_course, course_location, old_course_id, user_id)
self._add_draft_modules_to_course(new_package_id, old_course_id, course_location, user_id)
self._copy_published_modules_to_course(new_course, course_location, old_course_id, user)
self._add_draft_modules_to_course(new_package_id, old_course_id, course_location, user)
return new_package_id
def _copy_published_modules_to_course(self, new_course, old_course_loc, old_course_id, user_id):
def _copy_published_modules_to_course(self, new_course, old_course_loc, old_course_id, user):
"""
Copy all of the modules from the 'direct' version of the course to the new split course.
"""
@@ -79,7 +79,7 @@ class SplitMigrator(object):
old_course_id, module.location, True, add_entry_if_missing=True
)
_new_module = self.split_modulestore.create_item(
course_version_locator, module.category, user_id,
course_version_locator, module.category, user.id,
block_id=new_locator.block_id,
fields=self._get_json_fields_translate_children(module, old_course_id, True),
continue_version=True
@@ -94,7 +94,7 @@ class SplitMigrator(object):
# children which meant some pointers were to non-existent locations in 'direct'
self.split_modulestore.internal_clean_children(course_version_locator)
def _add_draft_modules_to_course(self, new_package_id, old_course_id, old_course_loc, user_id):
def _add_draft_modules_to_course(self, new_package_id, old_course_id, old_course_loc, user):
"""
update each draft. Create any which don't exist in published and attach to their parents.
"""
@@ -124,12 +124,12 @@ class SplitMigrator(object):
if name != 'children' and field.is_set_on(module):
field.write_to(split_module, field.read_from(module))
_new_module = self.split_modulestore.update_item(split_module, user_id)
_new_module = self.split_modulestore.update_item(split_module, user.id)
else:
# only a draft version (aka, 'private'). parent needs updated too.
# create a new course version just in case the current head is also the prod head
_new_module = self.split_modulestore.create_item(
new_draft_course_loc, module.category, user_id,
new_draft_course_loc, module.category, user.id,
block_id=new_locator.block_id,
fields=self._get_json_fields_translate_children(module, old_course_id, True)
)
@@ -156,7 +156,7 @@ class SplitMigrator(object):
new_parent_cursor = idx + 1
break
new_parent.children.insert(new_parent_cursor, new_block_id)
new_parent = self.split_modulestore.update_item(new_parent, user_id)
new_parent = self.split_modulestore.update_item(new_parent, user.id)
def _get_json_fields_translate_children(self, xblock, old_course_id, published):
"""

1
common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
View File

@@ -1284,6 +1284,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
if index is None:
raise ItemNotFoundError(package_id)
# this is the only real delete in the system. should it do something else?
log.info("deleting course from split-mongo: %s", package_id)
self.db_connection.delete_course_index(index['_id'])
def get_errored_courses(self):

7
common/lib/xmodule/xmodule/modulestore/tests/django_utils.py
View File

@@ -4,8 +4,8 @@ Modulestore configuration for test cases.
from uuid import uuid4
from django.test import TestCase
from xmodule.modulestore.django import editable_modulestore, \
clear_existing_modulestores
from xmodule.modulestore.django import (
editable_modulestore, clear_existing_modulestores, loc_mapper)
from xmodule.contentstore.django import contentstore
@@ -225,6 +225,9 @@ class ModuleStoreTestCase(TestCase):
if contentstore().fs_files:
db = contentstore().fs_files.database
db.connection.drop_database(db)
location_mapper = loc_mapper()
if location_mapper.db:
location_mapper.location_map.drop()
@classmethod
def setUpClass(cls):

8
common/lib/xmodule/xmodule/modulestore/tests/test_location_mapper.py
View File

@@ -80,8 +80,8 @@ class TestLocationMapper(unittest.TestCase):
Request translation, check package_id, block_id, and branch
"""
prob_locator = loc_mapper().translate_location(
old_style_course_id,
location,
old_style_course_id,
location,
published= (branch=='published'),
add_entry_if_missing=add_entry
)
@@ -114,7 +114,7 @@ class TestLocationMapper(unittest.TestCase):
new_style_package_id = '{}.geek_dept.{}.baz_run'.format(org, course)
block_map = {
'abc123': {'problem': 'problem2'},
'abc123': {'problem': 'problem2'},
'def456': {'problem': 'problem4'},
'ghi789': {'problem': 'problem7'},
}
@@ -139,7 +139,7 @@ class TestLocationMapper(unittest.TestCase):
# add a distractor course (note that abc123 has a different translation in this one)
distractor_block_map = {
'abc123': {'problem': 'problem3'},
'abc123': {'problem': 'problem3'},
'def456': {'problem': 'problem4'},
'ghi789': {'problem': 'problem7'},
}

3
common/lib/xmodule/xmodule/modulestore/tests/test_split_migrator.py
View File

@@ -271,7 +271,8 @@ class TestMigration(unittest.TestCase):
self.compare_dags(presplit, pre_child, split_child, published)
def test_migrator(self):
self.migrator.migrate_mongo_course(self.course_location, random.getrandbits(32))
user = mock.Mock(id=1)
self.migrator.migrate_mongo_course(self.course_location, user)
# now compare the migrated to the original course
self.compare_courses(self.old_mongo, True)
self.compare_courses(self.draft_mongo, False)