From a9695828ab2bdd27923f0b2197df4371a92f0ab4 Mon Sep 17 00:00:00 2001 From: Don Mitchell Date: Mon, 14 Jul 2014 11:43:56 -0400 Subject: [PATCH 1/2] Remove LocMapperStore LMS-2918 --- .../lib/xmodule/xmodule/modulestore/django.py | 31 +- .../xmodule/modulestore/loc_mapper_store.py | 570 ------------------ .../xmodule/modulestore/split_migrator.py | 4 +- .../xmodule/modulestore/tests/django_utils.py | 9 +- .../modulestore/tests/test_location_mapper.py | 432 ------------- docs/en_us/developers/source/modulestore.rst | 4 - mongo_indexes.md | 8 - 7 files changed, 4 insertions(+), 1054 deletions(-) delete mode 100644 common/lib/xmodule/xmodule/modulestore/loc_mapper_store.py delete mode 100644 common/lib/xmodule/xmodule/modulestore/tests/test_location_mapper.py diff --git a/common/lib/xmodule/xmodule/modulestore/django.py b/common/lib/xmodule/xmodule/modulestore/django.py index 973833f4f8..cd69fb1c7a 100644 --- a/common/lib/xmodule/xmodule/modulestore/django.py +++ b/common/lib/xmodule/xmodule/modulestore/django.py @@ -14,7 +14,6 @@ import django.utils import re import threading -from xmodule.modulestore.loc_mapper_store import LocMapperStore from xmodule.util.django import get_current_request_hostname import xmodule.modulestore # pylint: disable=unused-import from xmodule.contentstore.django import contentstore @@ -102,36 +101,8 @@ def clear_existing_modulestores(): This is useful for flushing state between unit tests. """ - global _MIXED_MODULESTORE, _loc_singleton # pylint: disable=global-statement + global _MIXED_MODULESTORE # pylint: disable=global-statement _MIXED_MODULESTORE = None - # pylint: disable=W0603 - cache = getattr(_loc_singleton, "cache", None) - if cache: - cache.clear() - _loc_singleton = None - - -# singleton instance of the loc_mapper -_loc_singleton = None - - -def loc_mapper(): - """ - Get the loc mapper which bidirectionally maps Locations to Locators. Used like modulestore() as - a singleton accessor. - """ - # pylint: disable=W0603 - global _loc_singleton - # pylint: disable=W0212 - if _loc_singleton is None: - try: - loc_cache = get_cache('loc_cache') - except InvalidCacheBackendError: - loc_cache = get_cache('default') - # instantiate - _loc_singleton = LocMapperStore(loc_cache, **settings.DOC_STORE_CONFIG) - - return _loc_singleton class ModuleI18nService(object): diff --git a/common/lib/xmodule/xmodule/modulestore/loc_mapper_store.py b/common/lib/xmodule/xmodule/modulestore/loc_mapper_store.py deleted file mode 100644 index 012971e5f7..0000000000 --- a/common/lib/xmodule/xmodule/modulestore/loc_mapper_store.py +++ /dev/null @@ -1,570 +0,0 @@ -''' -Method for converting among our differing Location/Locator whatever reprs -''' -from random import randint -import re -import pymongo -import bson.son -import urllib - -from xmodule.modulestore import ModuleStoreEnum -from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError -from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator -from opaque_keys.edx.locations import SlashSeparatedCourseKey -from opaque_keys.edx.keys import CourseKey - - -class LocMapperStore(object): - ''' - This store persists mappings among the addressing schemes. At this time, it's between the old i4x Location - tuples and the split mongo Course and Block Locator schemes. - - edX has used several different addressing schemes. The original ones were organically created based on - immediate needs and were overly restrictive esp wrt course ids. These were slightly extended to support - some types of blocks may need to have draft states during editing to keep live courses from seeing the wip. - A later refactoring generalized course ids to enable governance and more complex naming, branch naming with - anything able to be in any branch. - - The expectation is that the configuration will have this use the same store as whatever is the default - or dominant store, but that's not a requirement. This store creates its own connection. - ''' - - SCHEMA_VERSION = 1 - def __init__( - self, cache, host, db, collection, port=27017, user=None, password=None, - **kwargs - ): - ''' - Constructor - ''' - self.db = pymongo.database.Database( - pymongo.MongoClient( - host=host, - port=port, - tz_aware=True, - document_class=bson.son.SON, - **kwargs - ), - db - ) - if user is not None and password is not None: - self.db.authenticate(user, password) - - self.location_map = self.db[collection + '.location_map'] - self.location_map.write_concern = {'w': 1} - self.cache = cache - - # location_map functions - def create_map_entry(self, course_key, org=None, course=None, run=None, - draft_branch=ModuleStoreEnum.BranchName.draft, - prod_branch=ModuleStoreEnum.BranchName.published, - block_map=None): - """ - Add a new entry to map this SlashSeparatedCourseKey to the new style CourseLocator.org & course & run. If - org and course and run are not provided, it defaults them based on course_key. - - WARNING: Exactly 1 CourseLocator key should index a given SlashSeparatedCourseKey. - We provide no mechanism to enforce this assertion. - - NOTE: if there's already an entry w the given course_key, this may either overwrite that entry or - throw an error depending on how mongo is configured. - - :param course_key (SlashSeparatedCourseKey): a SlashSeparatedCourseKey - :param org (string): the CourseLocator style org - :param course (string): the CourseLocator course number - :param run (string): the CourseLocator run of this course - :param draft_branch: the branch name to assign for drafts. This is hardcoded because old mongo had - a fixed notion that there was 2 and only 2 versions for modules: draft and production. The old mongo - did not, however, require that a draft version exist. The new one, however, does require a draft to - exist. - :param prod_branch: the branch name to assign for the production (live) copy. In old mongo, every course - had to have a production version (whereas new split mongo does not require that until the author's ready - to publish). - :param block_map: an optional map to specify preferred names for blocks where the keys are the - Location block names and the values are the BlockUsageLocator.block_id. - - Returns: - :class:`CourseLocator` representing the new id for the course - - Raises: - ValueError if one and only one of org and course and run is provided. Provide all of them or none of them. - """ - if org is None and course is None and run is None: - assert(isinstance(course_key, CourseKey)) - org = course_key.org - course = course_key.course - run = course_key.run - elif org is None or course is None or run is None: - raise ValueError( - u"Either supply org, course and run or none of them. Not just some of them: {}, {}, {}".format(org, course, run) - ) - - # very like _interpret_location_id but using mongo subdoc lookup (more performant) - course_son = self._construct_course_son(course_key) - - self.location_map.insert({ - '_id': course_son, - 'org': org, - 'course': course, - 'run': run, - 'draft_branch': draft_branch, - 'prod_branch': prod_branch, - 'block_map': block_map or {}, - 'schema': self.SCHEMA_VERSION, - }) - - return CourseLocator(org, course, run) - - def translate_location(self, location, published=True, - add_entry_if_missing=True, passed_block_id=None): - """ - Translate the given module location to a Locator. - - The rationale for auto adding entries was that there should be a reasonable default translation - if the code just trips into this w/o creating translations. - - Will raise ItemNotFoundError if there's no mapping and add_entry_if_missing is False. - - :param location: a Location pointing to a module - :param published: a boolean to indicate whether the caller wants the draft or published branch. - :param add_entry_if_missing: a boolean as to whether to raise ItemNotFoundError or to create an entry if - the course - or block is not found in the map. - :param passed_block_id: what block_id to assign and save if none is found - (only if add_entry_if_missing) - - NOTE: unlike old mongo, draft branches contain the whole course; so, it applies to all category - of locations including course. - """ - course_son = self._interpret_location_course_id(location.course_key) - - cached_value = self._get_locator_from_cache(location, published) - if cached_value: - return cached_value - - entry = self.location_map.find_one(course_son) - if entry is None: - if add_entry_if_missing: - # create a new map - self.create_map_entry(location.course_key) - entry = self.location_map.find_one(course_son) - else: - raise ItemNotFoundError(location) - else: - entry = self._migrate_if_necessary([entry])[0] - - block_id = entry['block_map'].get(self.encode_key_for_mongo(location.name)) - category = location.category - if block_id is None: - if add_entry_if_missing: - block_id = self._add_to_block_map( - location, course_son, entry['block_map'], passed_block_id - ) - else: - raise ItemNotFoundError(location) - else: - # jump_to_id uses a None category. - if category is None: - if len(block_id) == 1: - # unique match (most common case) - category = block_id.keys()[0] - block_id = block_id.values()[0] - else: - raise InvalidLocationError() - elif category in block_id: - block_id = block_id[category] - elif add_entry_if_missing: - block_id = self._add_to_block_map(location, course_son, entry['block_map']) - else: - raise ItemNotFoundError(location) - - prod_course_locator = CourseLocator( - org=entry['org'], - course=entry['course'], - run=entry['run'], - branch=entry['prod_branch'] - ) - published_usage = BlockUsageLocator( - prod_course_locator, - block_type=category, - block_id=block_id - ) - draft_usage = BlockUsageLocator( - prod_course_locator.for_branch(entry['draft_branch']), - block_type=category, - block_id=block_id - ) - if published: - result = published_usage - else: - result = draft_usage - - self._cache_location_map_entry(location, published_usage, draft_usage) - return result - - def translate_locator_to_location(self, locator, get_course=False): - """ - Returns an old style Location for the given Locator if there's an appropriate entry in the - mapping collection. Note, it requires that the course was previously mapped (a side effect of - translate_location or explicitly via create_map_entry) and - the block's block_id was previously stored in the - map (a side effect of translate_location or via add|update_block_location). - - If there are no matches, it returns None. - - Args: - locator: a BlockUsageLocator to translate - get_course: rather than finding the map for this locator, returns the CourseKey - for the mapped course. - """ - if get_course: - cached_value = self._get_course_location_from_cache( - # if locator is already a course_key it won't have course_key attr - getattr(locator, 'course_key', locator) - ) - else: - cached_value = self._get_location_from_cache(locator) - if cached_value: - return cached_value - - # migrate any records which don't have the org and course and run fields as - # this won't be able to find what it wants. (only needs to be run once ever per db, - # I'm not sure how to control that, but I'm putting some check here for once per launch) - if not getattr(self, 'offering_migrated', False): - obsolete = self.location_map.find( - {'org': {"$exists": False}, "offering": {"$exists": False}, } - ) - self._migrate_if_necessary(obsolete) - setattr(self, 'offering_migrated', True) - - entry = self.location_map.find_one(bson.son.SON([ - ('org', locator.org), - ('course', locator.course), - ('run', locator.run), - ])) - - # look for one which maps to this block block_id - if entry is None: - return None - old_course_id = self._generate_location_course_id(entry['_id']) - if get_course: - return old_course_id - - for old_name, cat_to_usage in entry['block_map'].iteritems(): - for category, block_id in cat_to_usage.iteritems(): - # cache all entries and then figure out if we have the one we want - # Always return revision=MongoRevisionKey.published because the - # old draft module store wraps locations as draft before - # trying to access things. - location = old_course_id.make_usage_key( - category, - self.decode_key_from_mongo(old_name) - ) - - entry_org = "org" - entry_course = "course" - entry_run = "run" - - published_locator = BlockUsageLocator( - CourseLocator( - org=entry[entry_org], - course=entry[entry_course], - run=entry[entry_run], - branch=entry['prod_branch'] - ), - block_type=category, - block_id=block_id - ) - draft_locator = BlockUsageLocator( - CourseLocator( - org=entry[entry_org], course=entry[entry_course], run=entry[entry_run], - branch=entry['draft_branch'] - ), - block_type=category, - block_id=block_id - ) - self._cache_location_map_entry(location, published_locator, draft_locator) - - if block_id == locator.block_id: - return location - - return None - - def translate_location_to_course_locator(self, course_key, published=True): - """ - Used when you only need the CourseLocator and not a full BlockUsageLocator. Probably only - useful for get_items which wildcards name or category. - - :param course_key: a CourseKey - :param published: a boolean representing whether or not we should return the published or draft version - - Returns a Courselocator - """ - cached = self._get_course_locator_from_cache(course_key, published) - if cached: - return cached - - course_son = self._interpret_location_course_id(course_key) - - entry = self.location_map.find_one(course_son) - if entry is None: - raise ItemNotFoundError(course_key) - - published_course_locator = CourseLocator( - org=entry['org'], course=entry['course'], run=entry['run'], branch=entry['prod_branch'] - ) - draft_course_locator = CourseLocator( - org=entry['org'], course=entry['course'], run=entry['run'], branch=entry['draft_branch'] - ) - self._cache_course_locator(course_key, published_course_locator, draft_course_locator) - if published: - return published_course_locator - else: - return draft_course_locator - - def _add_to_block_map(self, location, course_son, block_map, block_id=None): - '''add the given location to the block_map and persist it''' - if block_id is None: - if self._block_id_is_guid(location.name): - # This makes the ids more meaningful with a small probability of name collision. - # The downside is that if there's more than one course mapped to from the same org/course root - # the block ids will likely be out of sync and collide from an id perspective. HOWEVER, - # if there are few == org/course roots or their content is unrelated, this will work well. - block_id = self._verify_uniqueness(location.category + location.name[:3], block_map) - else: - # if 2 different category locations had same name, then they'll collide. Make the later - # mapped ones unique - block_id = self._verify_uniqueness(location.name, block_map) - encoded_location_name = self.encode_key_for_mongo(location.name) - block_map.setdefault(encoded_location_name, {})[location.category] = block_id - self.location_map.update(course_son, {'$set': {'block_map': block_map}}) - return block_id - - def _interpret_location_course_id(self, course_key): - """ - Take a CourseKey and return a SON for querying the mapping table. - - :param course_key: a CourseKey object for a course. - """ - return {'_id': self._construct_course_son(course_key)} - - def _generate_location_course_id(self, entry_id): - """ - Generate a CourseKey for the given entry's id. - """ - return SlashSeparatedCourseKey(entry_id['org'], entry_id['course'], entry_id['name']) - - def _construct_course_son(self, course_key): - """ - Construct the SON needed to repr the course_key for either a query or an insertion - """ - assert(isinstance(course_key, CourseKey)) - return bson.son.SON([ - ('org', course_key.org), - ('course', course_key.course), - ('name', course_key.run) - ]) - - def _block_id_is_guid(self, name): - """ - Does the given name look like it's a guid? - """ - return len(name) == 32 and re.search(r'[^0-9A-Fa-f]', name) is None - - def _verify_uniqueness(self, name, block_map): - ''' - Verify that the name doesn't occur elsewhere in block_map. If it does, keep adding to it until - it's unique. - ''' - for targets in block_map.itervalues(): - if isinstance(targets, dict): - for values in targets.itervalues(): - if values == name: - name += str(randint(0, 9)) - return self._verify_uniqueness(name, block_map) - - elif targets == name: - name += str(randint(0, 9)) - return self._verify_uniqueness(name, block_map) - return name - - @staticmethod - def encode_key_for_mongo(fieldname): - """ - Fieldnames in mongo cannot have periods nor dollar signs. So encode them. - :param fieldname: an atomic field name. Note, don't pass structured paths as it will flatten them - """ - for char in [".", "$"]: - fieldname = fieldname.replace(char, '%{:02x}'.format(ord(char))) - return fieldname - - @staticmethod - def decode_key_from_mongo(fieldname): - """ - The inverse of encode_key_for_mongo - :param fieldname: with period and dollar escaped - """ - return urllib.unquote(fieldname) - - def _get_locator_from_cache(self, location, published): - """ - See if the location x published pair is in the cache. If so, return the mapped locator. - """ - entry = self.cache.get(u'{}+{}'.format(location.course_key, location)) - if entry is not None: - if published: - return entry[0] - else: - return entry[1] - return None - - def _get_course_locator_from_cache(self, old_course_id, published): - """ - Get the course Locator for this old course id - """ - if not old_course_id: - return None - entry = self.cache.get(unicode(old_course_id)) - if entry is not None: - if published: - return entry[0].course_key - else: - return entry[1].course_key - - def _get_location_from_cache(self, locator): - """ - See if the locator is in the cache. If so, return the mapped location. - """ - return self.cache.get(unicode(locator)) - - def _get_course_location_from_cache(self, course_key): - """ - See if the course_key is in the cache. If so, return the mapped location to the - course root. - """ - cache_key = self._course_key_cache_string(course_key) - return self.cache.get(cache_key) - - def _course_key_cache_string(self, course_key): - """ - Return the string used to cache the course key - """ - return u'{0.org}+{0.course}+{0.run}'.format(course_key) - - def _cache_course_locator(self, old_course_id, published_course_locator, draft_course_locator): - """ - For quick lookup of courses - """ - if not old_course_id: - return - self.cache.set(unicode(old_course_id), (published_course_locator, draft_course_locator)) - - def _cache_location_map_entry(self, location, published_usage, draft_usage): - """ - Cache the mapping from location to the draft and published Locators in entry. - Also caches the inverse. If the location is category=='course', it caches it for - the get_course query - """ - setmany = {} - if location.category == 'course': - setmany[self._course_key_cache_string(published_usage)] = location.course_key - setmany[unicode(published_usage)] = location - setmany[unicode(draft_usage)] = location - setmany[unicode(location)] = (published_usage, draft_usage) - setmany[unicode(location.course_key)] = (published_usage, draft_usage) - self.cache.set_many(setmany) - - def delete_course_mapping(self, course_key): - """ - Remove provided course location from loc_mapper and cache. - - :param course_key: a CourseKey for the course we wish to delete - """ - self.location_map.remove(self._interpret_location_course_id(course_key)) - - # Remove the location of course (draft and published) from cache - cached_key = self.cache.get(unicode(course_key)) - if cached_key: - delete_keys = [] - published_locator = unicode(cached_key[0].course_key) - course_location = self._course_location_from_cache(published_locator) - delete_keys.append(self._course_key_cache_string(course_key)) - delete_keys.append(published_locator) - delete_keys.append(unicode(cached_key[1].course_key)) - delete_keys.append(unicode(course_location)) - delete_keys.append(unicode(course_key)) - self.cache.delete_many(delete_keys) - - def _migrate_if_necessary(self, entries): - """ - Run the entries through any applicable schema updates and return the updated entries - """ - entries = [ - self._migrate[entry.get('schema', 0)](self, entry) - for entry in entries - ] - return entries - - def _entry_id_to_son(self, entry_id): - return bson.son.SON([ - ('org', entry_id['org']), - ('course', entry_id['course']), - ('name', entry_id['name']) - ]) - - def _delete_cache_location_map_entry(self, old_course_id, location, published_usage, draft_usage): - """ - Remove the location of course (draft and published) from cache - """ - delete_keys = [] - if location.category == 'course': - delete_keys.append(self._course_key_cache_string(published_usage.course_key)) - - delete_keys.append(unicode(published_usage)) - delete_keys.append(unicode(draft_usage)) - delete_keys.append(u'{}+{}'.format(old_course_id, location.to_deprecated_string())) - delete_keys.append(old_course_id) - self.cache.delete_many(delete_keys) - - def _migrate_top(self, entry, updated=False): - """ - Current version, so a no data change until next update. But since it's the top - it's responsible for persisting the record if it changed. - """ - if updated: - entry['schema'] = self.SCHEMA_VERSION - entry_id = self._entry_id_to_son(entry['_id']) - self.location_map.update({'_id': entry_id}, entry) - - return entry - - def _migrate_0(self, entry): - """ - If entry had an '_id' without a run, remove the whole record. - - Add fields: schema, org, course, run - Remove: course_id, lower_course_id - :param entry: - """ - if 'name' not in entry['_id']: - entry_id = entry['_id'] - entry_id = bson.son.SON([ - ('org', entry_id['org']), - ('course', entry_id['course']), - ]) - self.location_map.remove({'_id': entry_id}) - return None - - # add schema, org, course, run, etc, remove old fields - entry['schema'] = 0 - entry.pop('course_id', None) - entry.pop('lower_course_id', None) - old_course_id = SlashSeparatedCourseKey(entry['_id']['org'], entry['_id']['course'], entry['_id']['name']) - entry['org'] = old_course_id.org - entry['course'] = old_course_id.course - entry['run'] = old_course_id.run - return self._migrate_1(entry, True) - - # insert new migrations just before _migrate_top. _migrate_top sets the schema version and - # saves the record - _migrate = [_migrate_0, _migrate_top] diff --git a/common/lib/xmodule/xmodule/modulestore/split_migrator.py b/common/lib/xmodule/xmodule/modulestore/split_migrator.py index 4b485d1bda..bf8929d760 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_migrator.py +++ b/common/lib/xmodule/xmodule/modulestore/split_migrator.py @@ -173,7 +173,7 @@ class SplitMigrator(object): """ def get_translation(location): """ - Convert the location and add to loc mapper + Convert the location """ return new_course_key.make_usage_key( location.category, @@ -207,7 +207,7 @@ class SplitMigrator(object): """ def get_translation(location): """ - Convert the location and add to loc mapper + Convert the location """ return new_course_key.make_usage_key( location.category, diff --git a/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py b/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py index 03a4c02165..071de046c3 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py @@ -5,10 +5,8 @@ Modulestore configuration for test cases. from uuid import uuid4 from django.test import TestCase from django.contrib.auth.models import User -from xmodule.modulestore.django import ( - modulestore, clear_existing_modulestores, loc_mapper -) from xmodule.contentstore.django import _CONTENTSTORE +from xmodule.modulestore.django import modulestore, clear_existing_modulestores from xmodule.modulestore import ModuleStoreEnum @@ -197,11 +195,6 @@ class ModuleStoreTestCase(TestCase): module_store._drop_database() # pylint: disable=protected-access _CONTENTSTORE.clear() - location_mapper = loc_mapper() - if location_mapper.db: - location_mapper.location_map.drop() - location_mapper.db.connection.close() - @classmethod def setUpClass(cls): """ diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_location_mapper.py b/common/lib/xmodule/xmodule/modulestore/tests/test_location_mapper.py deleted file mode 100644 index 4b32b23887..0000000000 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_location_mapper.py +++ /dev/null @@ -1,432 +0,0 @@ -""" -Test the loc mapper store -""" -import unittest -import uuid -from opaque_keys.edx.locations import Location -from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator -from xmodule.modulestore import ModuleStoreEnum -from xmodule.modulestore.mongo.base import MongoRevisionKey -from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError -from xmodule.modulestore.loc_mapper_store import LocMapperStore -from mock import Mock -from opaque_keys.edx.locations import SlashSeparatedCourseKey -import bson.son - - -class LocMapperSetupSansDjango(unittest.TestCase): - """ - Create and destroy a loc mapper for each test - """ - loc_store = None - def setUp(self): - modulestore_options = { - 'host': 'localhost', - 'db': 'test_xmodule', - 'collection': 'modulestore{0}'.format(uuid.uuid4().hex[:5]), - } - - cache_standin = TrivialCache() - self.instrumented_cache = Mock(spec=cache_standin, wraps=cache_standin) - # pylint: disable=W0142 - LocMapperSetupSansDjango.loc_store = LocMapperStore(self.instrumented_cache, **modulestore_options) - - def tearDown(self): - dbref = TestLocationMapper.loc_store.db - dbref.drop_collection(TestLocationMapper.loc_store.location_map) - dbref.connection.close() - self.loc_store = None - - -class TestLocationMapper(LocMapperSetupSansDjango): - """ - Test the location to locator mapper - """ - @unittest.skip("getting rid of loc_mapper") - def test_create_map(self): - def _construct_course_son(org, course, run): - """ - Make a lookup son - """ - return bson.son.SON([ - ('org', org), - ('course', course), - ('name', run) - ]) - - org = 'foo_org' - course1 = 'bar_course' - run = 'baz_run' - loc_mapper().create_map_entry(SlashSeparatedCourseKey(org, course1, run)) - # pylint: disable=protected-access - entry = loc_mapper().location_map.find_one({ - '_id': _construct_course_son(org, course1, run) - }) - self.assertIsNotNone(entry, "Didn't find entry") - self.assertEqual(entry['org'], org) - self.assertEqual(entry['offering'], '{}.{}'.format(course1, run)) - self.assertEqual(entry['draft_branch'], ModuleStoreEnum.BranchName.draft) - self.assertEqual(entry['prod_branch'], ModuleStoreEnum.BranchName.published) - self.assertEqual(entry['block_map'], {}) - - course2 = 'quux_course' - # oldname: {category: newname} - block_map = {'abc123': {'problem': 'problem2'}} - loc_mapper().create_map_entry( - SlashSeparatedCourseKey(org, course2, run), - 'foo_org.geek_dept', - 'quux_course.baz_run', - 'wip', - 'live', - block_map) - entry = loc_mapper().location_map.find_one({ - '_id': _construct_course_son(org, course2, run) - }) - self.assertIsNotNone(entry, "Didn't find entry") - self.assertEqual(entry['org'], 'foo_org.geek_dept') - self.assertEqual(entry['offering'], '{}.{}'.format(course2, run)) - self.assertEqual(entry['draft_branch'], 'wip') - self.assertEqual(entry['prod_branch'], 'live') - self.assertEqual(entry['block_map'], block_map) - - @unittest.skip("getting rid of loc_mapper") - def test_delete_course_map(self): - """ - Test that course location is properly remove from loc_mapper and cache when course is deleted - """ - org = u'foo_org' - course = u'bar_course' - run = u'baz_run' - course_location = SlashSeparatedCourseKey(org, course, run) - loc_mapper().create_map_entry(course_location) - # pylint: disable=protected-access - entry = loc_mapper().location_map.find_one({ - '_id': loc_mapper()._construct_course_son(course_location) - }) - self.assertIsNotNone(entry, 'Entry not found in loc_mapper') - self.assertEqual(entry['offering'], u'{1}.{2}'.format(org, course, run)) - - # now delete course location from loc_mapper and cache and test that course location no longer - # exists in loca_mapper and cache - loc_mapper().delete_course_mapping(course_location) - # pylint: disable=protected-access - entry = loc_mapper().location_map.find_one({ - '_id': loc_mapper()._construct_course_son(course_location) - }) - self.assertIsNone(entry, 'Entry found in loc_mapper') - # pylint: disable=protected-access - cached_value = loc_mapper()._get_location_from_cache(course_location.make_usage_key('course', run)) - self.assertIsNone(cached_value, 'course_locator found in cache') - # pylint: disable=protected-access - cached_value = loc_mapper()._get_course_location_from_cache(course_location) - self.assertIsNone(cached_value, 'Entry found in cache') - - @unittest.skip("getting rid of loc_mapper") - def translate_n_check(self, location, org, offering, block_id, branch, add_entry=False): - """ - Request translation, check org, offering, block_id, and branch - """ - prob_locator = loc_mapper().translate_location( - location, - published=(branch == ModuleStoreEnum.BranchName.published), - add_entry_if_missing=add_entry - ) - self.assertEqual(prob_locator.org, org) - self.assertEqual(prob_locator.offering, offering) - self.assertEqual(prob_locator.block_id, block_id) - self.assertEqual(prob_locator.branch, branch) - - course_locator = loc_mapper().translate_location_to_course_locator( - location.course_key, - published=(branch == ModuleStoreEnum.BranchName.published), - ) - self.assertEqual(course_locator.org, org) - self.assertEqual(course_locator.offering, offering) - self.assertEqual(course_locator.branch, branch) - - @unittest.skip("getting rid of loc_mapper") - def test_translate_location_read_only(self): - """ - Test the variants of translate_location which don't create entries, just decode - """ - # lookup before there are any maps - org = 'foo_org' - course = 'bar_course' - run = 'baz_run' - slash_course_key = SlashSeparatedCourseKey(org, course, run) - with self.assertRaises(ItemNotFoundError): - _ = loc_mapper().translate_location( - Location(org, course, run, 'problem', 'abc123'), - add_entry_if_missing=False - ) - - new_style_org = '{}.geek_dept'.format(org) - new_style_offering = '.{}.{}'.format(course, run) - block_map = { - 'abc123': {'problem': 'problem2', 'vertical': 'vertical2'}, - 'def456': {'problem': 'problem4'}, - 'ghi789': {'problem': 'problem7'}, - } - loc_mapper().create_map_entry( - slash_course_key, - new_style_org, new_style_offering, - block_map=block_map - ) - test_problem_locn = Location(org, course, run, 'problem', 'abc123') - - self.translate_n_check(test_problem_locn, new_style_org, new_style_offering, 'problem2', - ModuleStoreEnum.BranchName.published) - # look for non-existent problem - with self.assertRaises(ItemNotFoundError): - loc_mapper().translate_location( - Location(org, course, run, 'problem', '1def23'), - add_entry_if_missing=False - ) - test_no_cat_locn = test_problem_locn.replace(category=None) - with self.assertRaises(InvalidLocationError): - loc_mapper().translate_location( - slash_course_key.make_usage_key(None, 'abc123'), test_no_cat_locn, False, False - ) - test_no_cat_locn = test_no_cat_locn.replace(name='def456') - - self.translate_n_check( - test_no_cat_locn, new_style_org, new_style_offering, 'problem4', ModuleStoreEnum.BranchName.published - ) - - # add a distractor course (note that abc123 has a different translation in this one) - distractor_block_map = { - 'abc123': {'problem': 'problem3'}, - 'def456': {'problem': 'problem4'}, - 'ghi789': {'problem': 'problem7'}, - } - run = 'delta_run' - test_delta_new_org = '{}.geek_dept'.format(org) - test_delta_new_offering = '{}.{}'.format(course, run) - loc_mapper().create_map_entry( - SlashSeparatedCourseKey(org, course, run), - test_delta_new_org, test_delta_new_offering, - block_map=distractor_block_map - ) - # test that old translation still works - self.translate_n_check( - test_problem_locn, new_style_org, new_style_offering, 'problem2', ModuleStoreEnum.BranchName.published - ) - # and new returns new id - self.translate_n_check( - test_problem_locn.replace(run=run), test_delta_new_org, test_delta_new_offering, - 'problem3', ModuleStoreEnum.BranchName.published - ) - - @unittest.skip("getting rid of loc_mapper") - def test_translate_location_dwim(self): - """ - Test the location translation mechanisms which try to do-what-i-mean by creating new - entries for never seen queries. - """ - org = 'foo_org' - course = 'bar_course' - run = 'baz_run' - problem_name = 'abc123abc123abc123abc123abc123f9' - location = Location(org, course, run, 'problem', problem_name) - new_offering = '{}.{}'.format(course, run) - self.translate_n_check(location, org, new_offering, 'problemabc', ModuleStoreEnum.BranchName.published, True) - - # create an entry w/o a guid name - other_location = Location(org, course, run, 'chapter', 'intro') - self.translate_n_check(other_location, org, new_offering, 'intro', ModuleStoreEnum.BranchName.published, True) - - # add a distractor course - delta_new_org = '{}.geek_dept'.format(org) - run = 'delta_run' - delta_new_offering = '{}.{}'.format(course, run) - delta_course_locn = SlashSeparatedCourseKey(org, course, run) - loc_mapper().create_map_entry( - delta_course_locn, - delta_new_org, delta_new_offering, - block_map={problem_name: {'problem': 'problem3'}} - ) - self.translate_n_check(location, org, new_offering, 'problemabc', ModuleStoreEnum.BranchName.published, True) - - # add a new one to both courses (ensure name doesn't have same beginning) - new_prob_name = uuid.uuid4().hex - while new_prob_name.startswith('abc'): - new_prob_name = uuid.uuid4().hex - new_prob_locn = location.replace(name=new_prob_name) - new_usage_id = 'problem{}'.format(new_prob_name[:3]) - self.translate_n_check(new_prob_locn, org, new_offering, new_usage_id, ModuleStoreEnum.BranchName.published, True) - new_prob_locn = new_prob_locn.replace(run=run) - self.translate_n_check( - new_prob_locn, delta_new_org, delta_new_offering, new_usage_id, ModuleStoreEnum.BranchName.published, True - ) - - @unittest.skip("getting rid of loc_mapper") - def test_translate_locator(self): - """ - tests translate_locator_to_location(BlockUsageLocator) - """ - # lookup for non-existent course - org = 'foo_org' - course = 'bar_course' - run = 'baz_run' - new_style_org = '{}.geek_dept'.format(org) - new_style_offering = '{}.{}'.format(course, run) - prob_course_key = CourseLocator( - org=new_style_org, offering=new_style_offering, - branch=ModuleStoreEnum.BranchName.published, - ) - prob_locator = BlockUsageLocator( - prob_course_key, - block_type='problem', - block_id='problem2', - ) - prob_location = loc_mapper().translate_locator_to_location(prob_locator) - self.assertIsNone(prob_location, 'found entry in empty map table') - - loc_mapper().create_map_entry( - SlashSeparatedCourseKey(org, course, run), - new_style_org, new_style_offering, - block_map={ - 'abc123': {'problem': 'problem2'}, - '48f23a10395384929234': {'chapter': 'chapter48f'}, - 'baz_run': {'course': 'root'}, - } - ) - # only one course matches - prob_location = loc_mapper().translate_locator_to_location(prob_locator) - # default branch - self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', MongoRevisionKey.published)) - # test get_course keyword - prob_location = loc_mapper().translate_locator_to_location(prob_locator, get_course=True) - self.assertEqual(prob_location, SlashSeparatedCourseKey(org, course, run)) - # explicit branch - prob_locator = prob_locator.for_branch(ModuleStoreEnum.BranchName.draft) - prob_location = loc_mapper().translate_locator_to_location(prob_locator) - # Even though the problem was set as draft, we always return revision= MongoRevisionKey.published to work - # with old mongo/draft modulestores. - self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', MongoRevisionKey.published)) - prob_locator = BlockUsageLocator( - prob_course_key.for_branch('production'), - block_type='problem', block_id='problem2' - ) - prob_location = loc_mapper().translate_locator_to_location(prob_locator) - self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', MongoRevisionKey.published)) - # same for chapter except chapter cannot be draft in old system - chap_locator = BlockUsageLocator( - prob_course_key.for_branch('production'), - block_type='chapter', block_id='chapter48f', - ) - chap_location = loc_mapper().translate_locator_to_location(chap_locator) - self.assertEqual(chap_location, Location(org, course, run, 'chapter', '48f23a10395384929234')) - # explicit branch - chap_locator = chap_locator.for_branch(ModuleStoreEnum.BranchName.draft) - chap_location = loc_mapper().translate_locator_to_location(chap_locator) - self.assertEqual(chap_location, Location(org, course, run, 'chapter', '48f23a10395384929234')) - chap_locator = BlockUsageLocator( - prob_course_key.for_branch('production'), block_type='chapter', block_id='chapter48f' - ) - chap_location = loc_mapper().translate_locator_to_location(chap_locator) - self.assertEqual(chap_location, Location(org, course, run, 'chapter', '48f23a10395384929234')) - - # look for non-existent problem - prob_locator2 = BlockUsageLocator( - prob_course_key.for_branch(ModuleStoreEnum.BranchName.draft), - block_type='problem', block_id='problem3' - ) - prob_location = loc_mapper().translate_locator_to_location(prob_locator2) - self.assertIsNone(prob_location, 'Found non-existent problem') - - # add a distractor course - delta_run = 'delta_run' - new_style_offering = '{}.{}'.format(course, delta_run) - loc_mapper().create_map_entry( - SlashSeparatedCourseKey(org, course, delta_run), - new_style_org, new_style_offering, - block_map={'abc123': {'problem': 'problem3'}} - ) - prob_location = loc_mapper().translate_locator_to_location(prob_locator) - self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', MongoRevisionKey.published)) - - @unittest.skip("getting rid of loc_mapper") - def test_special_chars(self): - """ - Test locations which have special characters - """ - # afaik, location.check_list prevents $ in all fields - org = 'foo.org.edu' - course = 'bar.course-4' - name = 'baz.run_4-3' - location = Location(org, course, name, 'course', name) - prob_locator = loc_mapper().translate_location( - location, - add_entry_if_missing=True - ) - reverted_location = loc_mapper().translate_locator_to_location(prob_locator) - self.assertEqual(location, reverted_location) - - @unittest.skip("getting rid of loc_mapper") - def test_name_collision(self): - """ - Test dwim translation when the old name was not unique - """ - org = "myorg" - course = "another_course" - name = "running_again" - course_location = Location(org, course, name, 'course', name) - course_xlate = loc_mapper().translate_location(course_location, add_entry_if_missing=True) - self.assertEqual(course_location, loc_mapper().translate_locator_to_location(course_xlate)) - eponymous_block = course_location.replace(category='chapter') - chapter_xlate = loc_mapper().translate_location(eponymous_block, add_entry_if_missing=True) - self.assertEqual(course_location, loc_mapper().translate_locator_to_location(course_xlate)) - self.assertEqual(eponymous_block, loc_mapper().translate_locator_to_location(chapter_xlate)) - # and a non-existent one w/o add - eponymous_block = course_location.replace(category='problem') - with self.assertRaises(ItemNotFoundError): - chapter_xlate = loc_mapper().translate_location(eponymous_block, add_entry_if_missing=False) - - -#================================== -# functions to mock existing services -def loc_mapper(): - """ - Mocks the global location mapper. - """ - return LocMapperSetupSansDjango.loc_store - - -def render_to_template_mock(*_args): - """ - Mocks the mako render_to_template w/ a noop - """ - - -class TrivialCache(object): - """ - A trivial cache impl - """ - def __init__(self): - self.cache = {} - - def get(self, key, default=None): - """ - Mock the .get - """ - return self.cache.get(key, default) - - def set_many(self, entries): - """ - mock set_many - """ - self.cache.update(entries) - - def set(self, key, entry): - """ - mock set - """ - self.cache[key] = entry - - def delete_many(self, entries): - """ - mock delete_many - """ - for entry in entries: - del self.cache[entry] diff --git a/docs/en_us/developers/source/modulestore.rst b/docs/en_us/developers/source/modulestore.rst index 3fe9a50c16..a9ac4b9190 100644 --- a/docs/en_us/developers/source/modulestore.rst +++ b/docs/en_us/developers/source/modulestore.rst @@ -33,10 +33,6 @@ Modulestore Helpers These packages provide utilities for easier use of modulestores, and migrating data between modulestores. -.. automodule:: xmodule.modulestore.loc_mapper_store - :members: - :show-inheritance: - .. automodule:: xmodule.modulestore.search :members: :show-inheritance: diff --git a/mongo_indexes.md b/mongo_indexes.md index 639743d7ed..0764f82afe 100644 --- a/mongo_indexes.md +++ b/mongo_indexes.md @@ -12,14 +12,6 @@ db.collection_name ``` as in ```db.location_map.ensureIndex({'course_id': 1}{background: true})``` -location_map: -============= - -``` -ensureIndex({'org': 1, 'offering': 1}) -ensureIndex({'schema': 1}) -``` - fs.files: ========= From 9f2fa7e1052b08ef2969099ae7276f1e3f177e37 Mon Sep 17 00:00:00 2001 From: Don Mitchell Date: Mon, 14 Jul 2014 15:03:45 -0400 Subject: [PATCH 2/2] sql migrations cannot use loc mapper --- .../student/migrations/0035_access_roles.py | 97 +++++++++++++------ .../migrations/0036_access_roles_orgless.py | 73 +++++++++++--- 2 files changed, 127 insertions(+), 43 deletions(-) diff --git a/common/djangoapps/student/migrations/0035_access_roles.py b/common/djangoapps/student/migrations/0035_access_roles.py index 3dc6d7d214..4b22a7e5cd 100644 --- a/common/djangoapps/student/migrations/0035_access_roles.py +++ b/common/djangoapps/student/migrations/0035_access_roles.py @@ -1,8 +1,5 @@ # -*- coding: utf-8 -*- -from south.db import db from south.v2 import DataMigration -from django.db import models -from xmodule.modulestore.django import loc_mapper import re from opaque_keys.edx.locations import SlashSeparatedCourseKey from opaque_keys import InvalidKeyError @@ -10,6 +7,10 @@ import bson.son import logging from django.db.models.query_utils import Q from django.db.utils import IntegrityError +from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.mixed import MixedModuleStore +import itertools log = logging.getLogger(__name__) @@ -25,8 +26,20 @@ class Migration(DataMigration): """ Converts group table entries for write access and beta_test roles to course access roles table. """ + store = modulestore() + if isinstance(store, MixedModuleStore): + self.mongostore = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo) + self.xmlstore = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.xml) + elif store.get_modulestore_type() == ModuleStoreEnum.Type.mongo: + self.mongostore = store + self.xmlstore = None + elif store.get_modulestore_type() == ModuleStoreEnum.Type.xml: + self.mongostore = None + self.xmlstore = store + else: + return + # Note: Remember to use orm['appname.ModelName'] rather than "from appname.models..." - loc_map_collection = loc_mapper().location_map # b/c the Groups table had several entries for each course, we need to ensure we process each unique # course only once. The below datastructures help ensure that. hold = {} # key of course_id_strings with array of group objects. Should only be org scoped entries @@ -64,21 +77,27 @@ class Migration(DataMigration): course_key = SlashSeparatedCourseKey.from_deprecated_string(course_id_string) # course_key is the downcased version, get the normal cased one. loc_mapper() has no # methods taking downcased SSCK; so, need to do it manually here - correct_course_key = self._map_downcased_ssck(course_key, loc_map_collection) + correct_course_key = self._map_downcased_ssck(course_key) if correct_course_key is not None: _migrate_users(correct_course_key, role, course_key.org) except InvalidKeyError: - entry = loc_map_collection.find_one({ - 'course_id': re.compile(r'^{}$'.format(course_id_string), re.IGNORECASE) - }) - if entry is None: + # old dotted format, try permutations + parts = course_id_string.split('.') + if len(parts) < 3: hold.setdefault(course_id_string, []).append(group) - else: - correct_course_key = SlashSeparatedCourseKey(*entry['_id'].values()) - if 'lower_id' in entry: - _migrate_users(correct_course_key, role, entry['lower_id']['org']) + elif len(parts) == 3: + course_key = SlashSeparatedCourseKey(*parts) + correct_course_key = self._map_downcased_ssck(course_key) + if correct_course_key is None: + hold.setdefault(course_id_string, []).append(group) else: - _migrate_users(correct_course_key, role, entry['_id']['org'].lower()) + _migrate_users(correct_course_key, role, course_key.org) + else: + correct_course_key = self.divide_parts_find_key(parts) + if correct_course_key is None: + hold.setdefault(course_id_string, []).append(group) + else: + _migrate_users(correct_course_key, role, course_key.org) # see if any in hold were missed above for held_auth_scope, groups in hold.iteritems(): @@ -99,28 +118,50 @@ class Migration(DataMigration): # don't silently skip unexpected roles log.warn("Didn't convert roles %s", [group.name for group in groups]) + def divide_parts_find_key(self, parts): + """ + Look for all possible org/course/run patterns from a possibly dotted source + """ + for org_stop, course_stop in itertools.combinations(range(1, len(parts)), 2): + org = '.'.join(parts[:org_stop]) + course = '.'.join(parts[org_stop:course_stop]) + run = '.'.join(parts[course_stop:]) + course_key = SlashSeparatedCourseKey(org, course, run) + correct_course_key = self._map_downcased_ssck(course_key) + if correct_course_key is not None: + return correct_course_key + return None + def backwards(self, orm): - "Write your backwards methods here." + "Removes the new table." # Since this migration is non-destructive (monotonically adds information), I'm not sure what # the semantic of backwards should be other than perhaps clearing the table. orm['student.courseaccessrole'].objects.all().delete() - def _map_downcased_ssck(self, downcased_ssck, loc_map_collection): + def _map_downcased_ssck(self, downcased_ssck): """ Get the normal cased version of this downcased slash sep course key """ - # given the regex, the son may be an overkill - course_son = bson.son.SON([ - ('_id.org', re.compile(r'^{}$'.format(downcased_ssck.org), re.IGNORECASE)), - ('_id.course', re.compile(r'^{}$'.format(downcased_ssck.course), re.IGNORECASE)), - ('_id.name', re.compile(r'^{}$'.format(downcased_ssck.run), re.IGNORECASE)), - ]) - entry = loc_map_collection.find_one(course_son) - if entry: - idpart = entry['_id'] - return SlashSeparatedCourseKey(idpart['org'], idpart['course'], idpart['name']) - else: - return None + if self.mongostore is not None: + course_son = bson.son.SON([ + ('_id.tag', 'i4x'), + ('_id.org', re.compile(r'^{}$'.format(downcased_ssck.org), re.IGNORECASE)), + ('_id.course', re.compile(r'^{}$'.format(downcased_ssck.course), re.IGNORECASE)), + ('_id.category', 'course'), + ('_id.name', re.compile(r'^{}$'.format(downcased_ssck.run), re.IGNORECASE)), + ]) + entry = self.mongostore.collection.find_one(course_son) + if entry: + idpart = entry['_id'] + return SlashSeparatedCourseKey(idpart['org'], idpart['course'], idpart['name']) + if self.xmlstore is not None: + for course in self.xmlstore.get_courses(): + if ( + course.id.org.lower() == downcased_ssck.org and course.id.course.lower() == downcased_ssck.course + and course.id.run.lower() == downcased_ssck.run + ): + return course.id + return None models = { diff --git a/common/djangoapps/student/migrations/0036_access_roles_orgless.py b/common/djangoapps/student/migrations/0036_access_roles_orgless.py index 7ae056d403..f57bdb5f2b 100644 --- a/common/djangoapps/student/migrations/0036_access_roles_orgless.py +++ b/common/djangoapps/student/migrations/0036_access_roles_orgless.py @@ -1,14 +1,15 @@ # -*- coding: utf-8 -*- from south.v2 import DataMigration -from xmodule.modulestore.django import loc_mapper, modulestore +from xmodule.modulestore.django import modulestore import re from opaque_keys.edx.locations import SlashSeparatedCourseKey -from opaque_keys import InvalidKeyError import logging from django.db.models.query_utils import Q from django.db.utils import IntegrityError from xmodule.modulestore import ModuleStoreEnum +import bson.son from xmodule.modulestore.mixed import MixedModuleStore +import itertools log = logging.getLogger(__name__) @@ -24,11 +25,18 @@ class Migration(DataMigration): """ Converts group table entries for write access and beta_test roles to course access roles table. """ - # Note: Remember to use orm['appname.ModelName'] rather than "from appname.models..." - loc_map_collection = loc_mapper().location_map - mixed_ms = modulestore() - xml_ms = mixed_ms._get_modulestore_by_type(ModuleStoreEnum.Type.xml) - mongo_ms = mixed_ms._get_modulestore_by_type(ModuleStoreEnum.Type.mongo) + store = modulestore() + if isinstance(store, MixedModuleStore): + self.mongostore = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo) + self.xmlstore = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.xml) + elif store.get_modulestore_type() == ModuleStoreEnum.Type.mongo: + self.mongostore = store + self.xmlstore = None + elif store.get_modulestore_type() == ModuleStoreEnum.Type.xml: + self.mongostore = None + self.xmlstore = store + else: + return query = Q(name__startswith='staff') | Q(name__startswith='instructor') | Q(name__startswith='beta_testers') for group in orm['auth.Group'].objects.filter(query).exclude(name__contains="/").all(): @@ -59,10 +67,7 @@ class Migration(DataMigration): role = parsed_entry.group('role_id') course_id_string = parsed_entry.group('course_id_string') # if it's a full course_id w/ dots, ignore it - entry = loc_map_collection.find_one({ - 'course_id': re.compile(r'^{}$'.format(course_id_string), re.IGNORECASE) - }) - if entry is None: + if u'/' not in course_id_string and not self.dotted_course(course_id_string): # check new table to see if it's been added as org permission if not orm['student.courseaccessrole'].objects.filter( role=role, @@ -70,14 +75,14 @@ class Migration(DataMigration): ).exists(): # old auth was of form role_coursenum. Grant access to all such courses wildcarding org and run # look in xml for matching courses - if xml_ms is not None: - for course in xml_ms.get_courses(): + if self.xmlstore is not None: + for course in self.xmlstore.get_courses(): if course_id_string == course.id.course.lower(): _migrate_users(course.id, role) - if mongo_ms is not None: + if self.mongostore is not None: mongo_query = re.compile(ur'^{}$'.format(course_id_string), re.IGNORECASE) - for mongo_entry in mongo_ms.collection.find( + for mongo_entry in self.mongostore.collection.find( {"_id.category": "course", "_id.course": mongo_query}, fields=["_id"] ): mongo_id_dict = mongo_entry['_id'] @@ -86,6 +91,44 @@ class Migration(DataMigration): ) _migrate_users(course_key, role) + def dotted_course(self, parts): + """ + Look for all possible org/course/run patterns from a possibly dotted source + """ + for org_stop, course_stop in itertools.combinations(range(1, len(parts)), 2): + org = '.'.join(parts[:org_stop]) + course = '.'.join(parts[org_stop:course_stop]) + run = '.'.join(parts[course_stop:]) + course_key = SlashSeparatedCourseKey(org, course, run) + correct_course_key = self._map_downcased_ssck(course_key) + if correct_course_key is not None: + return correct_course_key + return False + + def _map_downcased_ssck(self, downcased_ssck): + """ + Get the normal cased version of this downcased slash sep course key + """ + if self.mongostore is not None: + course_son = bson.son.SON([ + ('_id.tag', 'i4x'), + ('_id.org', re.compile(r'^{}$'.format(downcased_ssck.org), re.IGNORECASE)), + ('_id.course', re.compile(r'^{}$'.format(downcased_ssck.course), re.IGNORECASE)), + ('_id.category', 'course'), + ('_id.name', re.compile(r'^{}$'.format(downcased_ssck.run), re.IGNORECASE)), + ]) + entry = self.mongostore.collection.find_one(course_son) + if entry: + idpart = entry['_id'] + return SlashSeparatedCourseKey(idpart['org'], idpart['course'], idpart['name']) + if self.xmlstore is not None: + for course in self.xmlstore.get_courses(): + if ( + course.id.org.lower() == downcased_ssck.org and course.id.course.lower() == downcased_ssck.course + and course.id.run.lower() == downcased_ssck.run + ): + return course.id + return None def backwards(self, orm): "No obvious way to reverse just this migration, but reversing 0035 will reverse this."