Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #4413 from edx/remove-loc-mapper

Browse Source 
Remove LocMapperStore
This commit is contained in:
Don Mitchell
2014-07-16 11:09:24 -04:00
parent a71919efa7 9f2fa7e105
commit 347b660623
9 changed files with 131 additions and 1097 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
common/djangoapps/student/migrations/0035_access_roles.py
View File

@@ -1,8 +1,5 @@
# -*- coding: utf-8 -*-
from south.db import db
from south.v2 import DataMigration
from django.db import models
from xmodule.modulestore.django import loc_mapper
import re
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from opaque_keys import InvalidKeyError
@@ -10,6 +7,10 @@ import bson.son
import logging
from django.db.models.query_utils import Q
from django.db.utils import IntegrityError
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.mixed import MixedModuleStore
import itertools
log = logging.getLogger(__name__)
@@ -25,8 +26,20 @@ class Migration(DataMigration):
"""
Converts group table entries for write access and beta_test roles to course access roles table.
"""
store = modulestore()
if isinstance(store, MixedModuleStore):
self.mongostore = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo)
self.xmlstore = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.xml)
elif store.get_modulestore_type() == ModuleStoreEnum.Type.mongo:
self.mongostore = store
self.xmlstore = None
elif store.get_modulestore_type() == ModuleStoreEnum.Type.xml:
self.mongostore = None
self.xmlstore = store
else:
return
# Note: Remember to use orm['appname.ModelName'] rather than "from appname.models..."
loc_map_collection = loc_mapper().location_map
# b/c the Groups table had several entries for each course, we need to ensure we process each unique
# course only once. The below datastructures help ensure that.
hold = {} # key of course_id_strings with array of group objects. Should only be org scoped entries
@@ -64,21 +77,27 @@ class Migration(DataMigration):
course_key = SlashSeparatedCourseKey.from_deprecated_string(course_id_string)
# course_key is the downcased version, get the normal cased one. loc_mapper() has no
# methods taking downcased SSCK; so, need to do it manually here
correct_course_key = self._map_downcased_ssck(course_key, loc_map_collection)
correct_course_key = self._map_downcased_ssck(course_key)
if correct_course_key is not None:
_migrate_users(correct_course_key, role, course_key.org)
except InvalidKeyError:
entry = loc_map_collection.find_one({
'course_id': re.compile(r'^{}$'.format(course_id_string), re.IGNORECASE)
})
if entry is None:
# old dotted format, try permutations
parts = course_id_string.split('.')
if len(parts) < 3:
hold.setdefault(course_id_string, []).append(group)
else:
correct_course_key = SlashSeparatedCourseKey(*entry['_id'].values())
if 'lower_id' in entry:
_migrate_users(correct_course_key, role, entry['lower_id']['org'])
elif len(parts) == 3:
course_key = SlashSeparatedCourseKey(*parts)
correct_course_key = self._map_downcased_ssck(course_key)
if correct_course_key is None:
hold.setdefault(course_id_string, []).append(group)
else:
_migrate_users(correct_course_key, role, entry['_id']['org'].lower())
_migrate_users(correct_course_key, role, course_key.org)
else:
correct_course_key = self.divide_parts_find_key(parts)
if correct_course_key is None:
hold.setdefault(course_id_string, []).append(group)
else:
_migrate_users(correct_course_key, role, course_key.org)
# see if any in hold were missed above
for held_auth_scope, groups in hold.iteritems():
@@ -99,28 +118,50 @@ class Migration(DataMigration):
# don't silently skip unexpected roles
log.warn("Didn't convert roles %s", [group.name for group in groups])
def divide_parts_find_key(self, parts):
"""
Look for all possible org/course/run patterns from a possibly dotted source
"""
for org_stop, course_stop in itertools.combinations(range(1, len(parts)), 2):
org = '.'.join(parts[:org_stop])
course = '.'.join(parts[org_stop:course_stop])
run = '.'.join(parts[course_stop:])
course_key = SlashSeparatedCourseKey(org, course, run)
correct_course_key = self._map_downcased_ssck(course_key)
if correct_course_key is not None:
return correct_course_key
return None
def backwards(self, orm):
"Write your backwards methods here."
"Removes the new table."
# Since this migration is non-destructive (monotonically adds information), I'm not sure what
# the semantic of backwards should be other than perhaps clearing the table.
orm['student.courseaccessrole'].objects.all().delete()
def _map_downcased_ssck(self, downcased_ssck, loc_map_collection):
def _map_downcased_ssck(self, downcased_ssck):
"""
Get the normal cased version of this downcased slash sep course key
"""
# given the regex, the son may be an overkill
course_son = bson.son.SON([
('_id.org', re.compile(r'^{}$'.format(downcased_ssck.org), re.IGNORECASE)),
('_id.course', re.compile(r'^{}$'.format(downcased_ssck.course), re.IGNORECASE)),
('_id.name', re.compile(r'^{}$'.format(downcased_ssck.run), re.IGNORECASE)),
])
entry = loc_map_collection.find_one(course_son)
if entry:
idpart = entry['_id']
return SlashSeparatedCourseKey(idpart['org'], idpart['course'], idpart['name'])
else:
return None
if self.mongostore is not None:
course_son = bson.son.SON([
('_id.tag', 'i4x'),
('_id.org', re.compile(r'^{}$'.format(downcased_ssck.org), re.IGNORECASE)),
('_id.course', re.compile(r'^{}$'.format(downcased_ssck.course), re.IGNORECASE)),
('_id.category', 'course'),
('_id.name', re.compile(r'^{}$'.format(downcased_ssck.run), re.IGNORECASE)),
])
entry = self.mongostore.collection.find_one(course_son)
if entry:
idpart = entry['_id']
return SlashSeparatedCourseKey(idpart['org'], idpart['course'], idpart['name'])
if self.xmlstore is not None:
for course in self.xmlstore.get_courses():
if (
course.id.org.lower() == downcased_ssck.org and course.id.course.lower() == downcased_ssck.course
and course.id.run.lower() == downcased_ssck.run
):
return course.id
return None
models = {

73
common/djangoapps/student/migrations/0036_access_roles_orgless.py
View File

@@ -1,14 +1,15 @@
# -*- coding: utf-8 -*-
from south.v2 import DataMigration
from xmodule.modulestore.django import loc_mapper, modulestore
from xmodule.modulestore.django import modulestore
import re
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from opaque_keys import InvalidKeyError
import logging
from django.db.models.query_utils import Q
from django.db.utils import IntegrityError
from xmodule.modulestore import ModuleStoreEnum
import bson.son
from xmodule.modulestore.mixed import MixedModuleStore
import itertools
log = logging.getLogger(__name__)
@@ -24,11 +25,18 @@ class Migration(DataMigration):
"""
Converts group table entries for write access and beta_test roles to course access roles table.
"""
# Note: Remember to use orm['appname.ModelName'] rather than "from appname.models..."
loc_map_collection = loc_mapper().location_map
mixed_ms = modulestore()
xml_ms = mixed_ms._get_modulestore_by_type(ModuleStoreEnum.Type.xml)
mongo_ms = mixed_ms._get_modulestore_by_type(ModuleStoreEnum.Type.mongo)
store = modulestore()
if isinstance(store, MixedModuleStore):
self.mongostore = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo)
self.xmlstore = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.xml)
elif store.get_modulestore_type() == ModuleStoreEnum.Type.mongo:
self.mongostore = store
self.xmlstore = None
elif store.get_modulestore_type() == ModuleStoreEnum.Type.xml:
self.mongostore = None
self.xmlstore = store
else:
return
query = Q(name__startswith='staff') | Q(name__startswith='instructor') | Q(name__startswith='beta_testers')
for group in orm['auth.Group'].objects.filter(query).exclude(name__contains="/").all():
@@ -59,10 +67,7 @@ class Migration(DataMigration):
role = parsed_entry.group('role_id')
course_id_string = parsed_entry.group('course_id_string')
# if it's a full course_id w/ dots, ignore it
entry = loc_map_collection.find_one({
'course_id': re.compile(r'^{}$'.format(course_id_string), re.IGNORECASE)
})
if entry is None:
if u'/' not in course_id_string and not self.dotted_course(course_id_string):
# check new table to see if it's been added as org permission
if not orm['student.courseaccessrole'].objects.filter(
role=role,
@@ -70,14 +75,14 @@ class Migration(DataMigration):
).exists():
# old auth was of form role_coursenum. Grant access to all such courses wildcarding org and run
# look in xml for matching courses
if xml_ms is not None:
for course in xml_ms.get_courses():
if self.xmlstore is not None:
for course in self.xmlstore.get_courses():
if course_id_string == course.id.course.lower():
_migrate_users(course.id, role)
if mongo_ms is not None:
if self.mongostore is not None:
mongo_query = re.compile(ur'^{}$'.format(course_id_string), re.IGNORECASE)
for mongo_entry in mongo_ms.collection.find(
for mongo_entry in self.mongostore.collection.find(
{"_id.category": "course", "_id.course": mongo_query}, fields=["_id"]
):
mongo_id_dict = mongo_entry['_id']
@@ -86,6 +91,44 @@ class Migration(DataMigration):
)
_migrate_users(course_key, role)
def dotted_course(self, parts):
"""
Look for all possible org/course/run patterns from a possibly dotted source
"""
for org_stop, course_stop in itertools.combinations(range(1, len(parts)), 2):
org = '.'.join(parts[:org_stop])
course = '.'.join(parts[org_stop:course_stop])
run = '.'.join(parts[course_stop:])
course_key = SlashSeparatedCourseKey(org, course, run)
correct_course_key = self._map_downcased_ssck(course_key)
if correct_course_key is not None:
return correct_course_key
return False
def _map_downcased_ssck(self, downcased_ssck):
"""
Get the normal cased version of this downcased slash sep course key
"""
if self.mongostore is not None:
course_son = bson.son.SON([
('_id.tag', 'i4x'),
('_id.org', re.compile(r'^{}$'.format(downcased_ssck.org), re.IGNORECASE)),
('_id.course', re.compile(r'^{}$'.format(downcased_ssck.course), re.IGNORECASE)),
('_id.category', 'course'),
('_id.name', re.compile(r'^{}$'.format(downcased_ssck.run), re.IGNORECASE)),
])
entry = self.mongostore.collection.find_one(course_son)
if entry:
idpart = entry['_id']
return SlashSeparatedCourseKey(idpart['org'], idpart['course'], idpart['name'])
if self.xmlstore is not None:
for course in self.xmlstore.get_courses():
if (
course.id.org.lower() == downcased_ssck.org and course.id.course.lower() == downcased_ssck.course
and course.id.run.lower() == downcased_ssck.run
):
return course.id
return None
def backwards(self, orm):
"No obvious way to reverse just this migration, but reversing 0035 will reverse this."

31
common/lib/xmodule/xmodule/modulestore/django.py
View File

@@ -14,7 +14,6 @@ import django.utils
import re
import threading
from xmodule.modulestore.loc_mapper_store import LocMapperStore
from xmodule.util.django import get_current_request_hostname
import xmodule.modulestore # pylint: disable=unused-import
from xmodule.contentstore.django import contentstore
@@ -102,36 +101,8 @@ def clear_existing_modulestores():
This is useful for flushing state between unit tests.
"""
global _MIXED_MODULESTORE, _loc_singleton # pylint: disable=global-statement
global _MIXED_MODULESTORE # pylint: disable=global-statement
_MIXED_MODULESTORE = None
# pylint: disable=W0603
cache = getattr(_loc_singleton, "cache", None)
if cache:
cache.clear()
_loc_singleton = None
# singleton instance of the loc_mapper
_loc_singleton = None
def loc_mapper():
"""
Get the loc mapper which bidirectionally maps Locations to Locators. Used like modulestore() as
a singleton accessor.
"""
# pylint: disable=W0603
global _loc_singleton
# pylint: disable=W0212
if _loc_singleton is None:
try:
loc_cache = get_cache('loc_cache')
except InvalidCacheBackendError:
loc_cache = get_cache('default')
# instantiate
_loc_singleton = LocMapperStore(loc_cache, **settings.DOC_STORE_CONFIG)
return _loc_singleton
class ModuleI18nService(object):

570
common/lib/xmodule/xmodule/modulestore/loc_mapper_store.py
View File

@@ -1,570 +0,0 @@
'''
Method for converting among our differing Location/Locator whatever reprs
'''
from random import randint
import re
import pymongo
import bson.son
import urllib
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError
from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from opaque_keys.edx.keys import CourseKey
class LocMapperStore(object):
'''
This store persists mappings among the addressing schemes. At this time, it's between the old i4x Location
tuples and the split mongo Course and Block Locator schemes.
edX has used several different addressing schemes. The original ones were organically created based on
immediate needs and were overly restrictive esp wrt course ids. These were slightly extended to support
some types of blocks may need to have draft states during editing to keep live courses from seeing the wip.
A later refactoring generalized course ids to enable governance and more complex naming, branch naming with
anything able to be in any branch.
The expectation is that the configuration will have this use the same store as whatever is the default
or dominant store, but that's not a requirement. This store creates its own connection.
'''
SCHEMA_VERSION = 1
def __init__(
self, cache, host, db, collection, port=27017, user=None, password=None,
**kwargs
):
'''
Constructor
'''
self.db = pymongo.database.Database(
pymongo.MongoClient(
host=host,
port=port,
tz_aware=True,
document_class=bson.son.SON,
**kwargs
),
db
)
if user is not None and password is not None:
self.db.authenticate(user, password)
self.location_map = self.db[collection + '.location_map']
self.location_map.write_concern = {'w': 1}
self.cache = cache
# location_map functions
def create_map_entry(self, course_key, org=None, course=None, run=None,
draft_branch=ModuleStoreEnum.BranchName.draft,
prod_branch=ModuleStoreEnum.BranchName.published,
block_map=None):
"""
Add a new entry to map this SlashSeparatedCourseKey to the new style CourseLocator.org & course & run. If
org and course and run are not provided, it defaults them based on course_key.
WARNING: Exactly 1 CourseLocator key should index a given SlashSeparatedCourseKey.
We provide no mechanism to enforce this assertion.
NOTE: if there's already an entry w the given course_key, this may either overwrite that entry or
throw an error depending on how mongo is configured.
:param course_key (SlashSeparatedCourseKey): a SlashSeparatedCourseKey
:param org (string): the CourseLocator style org
:param course (string): the CourseLocator course number
:param run (string): the CourseLocator run of this course
:param draft_branch: the branch name to assign for drafts. This is hardcoded because old mongo had
a fixed notion that there was 2 and only 2 versions for modules: draft and production. The old mongo
did not, however, require that a draft version exist. The new one, however, does require a draft to
exist.
:param prod_branch: the branch name to assign for the production (live) copy. In old mongo, every course
had to have a production version (whereas new split mongo does not require that until the author's ready
to publish).
:param block_map: an optional map to specify preferred names for blocks where the keys are the
Location block names and the values are the BlockUsageLocator.block_id.
Returns:
:class:`CourseLocator` representing the new id for the course
Raises:
ValueError if one and only one of org and course and run is provided. Provide all of them or none of them.
"""
if org is None and course is None and run is None:
assert(isinstance(course_key, CourseKey))
org = course_key.org
course = course_key.course
run = course_key.run
elif org is None or course is None or run is None:
raise ValueError(
u"Either supply org, course and run or none of them. Not just some of them: {}, {}, {}".format(org, course, run)
)
# very like _interpret_location_id but using mongo subdoc lookup (more performant)
course_son = self._construct_course_son(course_key)
self.location_map.insert({
'_id': course_son,
'org': org,
'course': course,
'run': run,
'draft_branch': draft_branch,
'prod_branch': prod_branch,
'block_map': block_map or {},
'schema': self.SCHEMA_VERSION,
})
return CourseLocator(org, course, run)
def translate_location(self, location, published=True,
add_entry_if_missing=True, passed_block_id=None):
"""
Translate the given module location to a Locator.
The rationale for auto adding entries was that there should be a reasonable default translation
if the code just trips into this w/o creating translations.
Will raise ItemNotFoundError if there's no mapping and add_entry_if_missing is False.
:param location: a Location pointing to a module
:param published: a boolean to indicate whether the caller wants the draft or published branch.
:param add_entry_if_missing: a boolean as to whether to raise ItemNotFoundError or to create an entry if
the course
or block is not found in the map.
:param passed_block_id: what block_id to assign and save if none is found
(only if add_entry_if_missing)
NOTE: unlike old mongo, draft branches contain the whole course; so, it applies to all category
of locations including course.
"""
course_son = self._interpret_location_course_id(location.course_key)
cached_value = self._get_locator_from_cache(location, published)
if cached_value:
return cached_value
entry = self.location_map.find_one(course_son)
if entry is None:
if add_entry_if_missing:
# create a new map
self.create_map_entry(location.course_key)
entry = self.location_map.find_one(course_son)
else:
raise ItemNotFoundError(location)
else:
entry = self._migrate_if_necessary([entry])[0]
block_id = entry['block_map'].get(self.encode_key_for_mongo(location.name))
category = location.category
if block_id is None:
if add_entry_if_missing:
block_id = self._add_to_block_map(
location, course_son, entry['block_map'], passed_block_id
)
else:
raise ItemNotFoundError(location)
else:
# jump_to_id uses a None category.
if category is None:
if len(block_id) == 1:
# unique match (most common case)
category = block_id.keys()[0]
block_id = block_id.values()[0]
else:
raise InvalidLocationError()
elif category in block_id:
block_id = block_id[category]
elif add_entry_if_missing:
block_id = self._add_to_block_map(location, course_son, entry['block_map'])
else:
raise ItemNotFoundError(location)
prod_course_locator = CourseLocator(
org=entry['org'],
course=entry['course'],
run=entry['run'],
branch=entry['prod_branch']
)
published_usage = BlockUsageLocator(
prod_course_locator,
block_type=category,
block_id=block_id
)
draft_usage = BlockUsageLocator(
prod_course_locator.for_branch(entry['draft_branch']),
block_type=category,
block_id=block_id
)
if published:
result = published_usage
else:
result = draft_usage
self._cache_location_map_entry(location, published_usage, draft_usage)
return result
def translate_locator_to_location(self, locator, get_course=False):
"""
Returns an old style Location for the given Locator if there's an appropriate entry in the
mapping collection. Note, it requires that the course was previously mapped (a side effect of
translate_location or explicitly via create_map_entry) and
the block's block_id was previously stored in the
map (a side effect of translate_location or via add|update_block_location).
If there are no matches, it returns None.
Args:
locator: a BlockUsageLocator to translate
get_course: rather than finding the map for this locator, returns the CourseKey
for the mapped course.
"""
if get_course:
cached_value = self._get_course_location_from_cache(
# if locator is already a course_key it won't have course_key attr
getattr(locator, 'course_key', locator)
)
else:
cached_value = self._get_location_from_cache(locator)
if cached_value:
return cached_value
# migrate any records which don't have the org and course and run fields as
# this won't be able to find what it wants. (only needs to be run once ever per db,
# I'm not sure how to control that, but I'm putting some check here for once per launch)
if not getattr(self, 'offering_migrated', False):
obsolete = self.location_map.find(
{'org': {"$exists": False}, "offering": {"$exists": False}, }
)
self._migrate_if_necessary(obsolete)
setattr(self, 'offering_migrated', True)
entry = self.location_map.find_one(bson.son.SON([
('org', locator.org),
('course', locator.course),
('run', locator.run),
]))
# look for one which maps to this block block_id
if entry is None:
return None
old_course_id = self._generate_location_course_id(entry['_id'])
if get_course:
return old_course_id
for old_name, cat_to_usage in entry['block_map'].iteritems():
for category, block_id in cat_to_usage.iteritems():
# cache all entries and then figure out if we have the one we want
# Always return revision=MongoRevisionKey.published because the
# old draft module store wraps locations as draft before
# trying to access things.
location = old_course_id.make_usage_key(
category,
self.decode_key_from_mongo(old_name)
)
entry_org = "org"
entry_course = "course"
entry_run = "run"
published_locator = BlockUsageLocator(
CourseLocator(
org=entry[entry_org],
course=entry[entry_course],
run=entry[entry_run],
branch=entry['prod_branch']
),
block_type=category,
block_id=block_id
)
draft_locator = BlockUsageLocator(
CourseLocator(
org=entry[entry_org], course=entry[entry_course], run=entry[entry_run],
branch=entry['draft_branch']
),
block_type=category,
block_id=block_id
)
self._cache_location_map_entry(location, published_locator, draft_locator)
if block_id == locator.block_id:
return location
return None
def translate_location_to_course_locator(self, course_key, published=True):
"""
Used when you only need the CourseLocator and not a full BlockUsageLocator. Probably only
useful for get_items which wildcards name or category.
:param course_key: a CourseKey
:param published: a boolean representing whether or not we should return the published or draft version
Returns a Courselocator
"""
cached = self._get_course_locator_from_cache(course_key, published)
if cached:
return cached
course_son = self._interpret_location_course_id(course_key)
entry = self.location_map.find_one(course_son)
if entry is None:
raise ItemNotFoundError(course_key)
published_course_locator = CourseLocator(
org=entry['org'], course=entry['course'], run=entry['run'], branch=entry['prod_branch']
)
draft_course_locator = CourseLocator(
org=entry['org'], course=entry['course'], run=entry['run'], branch=entry['draft_branch']
)
self._cache_course_locator(course_key, published_course_locator, draft_course_locator)
if published:
return published_course_locator
else:
return draft_course_locator
def _add_to_block_map(self, location, course_son, block_map, block_id=None):
'''add the given location to the block_map and persist it'''
if block_id is None:
if self._block_id_is_guid(location.name):
# This makes the ids more meaningful with a small probability of name collision.
# The downside is that if there's more than one course mapped to from the same org/course root
# the block ids will likely be out of sync and collide from an id perspective. HOWEVER,
# if there are few == org/course roots or their content is unrelated, this will work well.
block_id = self._verify_uniqueness(location.category + location.name[:3], block_map)
else:
# if 2 different category locations had same name, then they'll collide. Make the later
# mapped ones unique
block_id = self._verify_uniqueness(location.name, block_map)
encoded_location_name = self.encode_key_for_mongo(location.name)
block_map.setdefault(encoded_location_name, {})[location.category] = block_id
self.location_map.update(course_son, {'$set': {'block_map': block_map}})
return block_id
def _interpret_location_course_id(self, course_key):
"""
Take a CourseKey and return a SON for querying the mapping table.
:param course_key: a CourseKey object for a course.
"""
return {'_id': self._construct_course_son(course_key)}
def _generate_location_course_id(self, entry_id):
"""
Generate a CourseKey for the given entry's id.
"""
return SlashSeparatedCourseKey(entry_id['org'], entry_id['course'], entry_id['name'])
def _construct_course_son(self, course_key):
"""
Construct the SON needed to repr the course_key for either a query or an insertion
"""
assert(isinstance(course_key, CourseKey))
return bson.son.SON([
('org', course_key.org),
('course', course_key.course),
('name', course_key.run)
])
def _block_id_is_guid(self, name):
"""
Does the given name look like it's a guid?
"""
return len(name) == 32 and re.search(r'[^0-9A-Fa-f]', name) is None
def _verify_uniqueness(self, name, block_map):
'''
Verify that the name doesn't occur elsewhere in block_map. If it does, keep adding to it until
it's unique.
'''
for targets in block_map.itervalues():
if isinstance(targets, dict):
for values in targets.itervalues():
if values == name:
name += str(randint(0, 9))
return self._verify_uniqueness(name, block_map)
elif targets == name:
name += str(randint(0, 9))
return self._verify_uniqueness(name, block_map)
return name
@staticmethod
def encode_key_for_mongo(fieldname):
"""
Fieldnames in mongo cannot have periods nor dollar signs. So encode them.
:param fieldname: an atomic field name. Note, don't pass structured paths as it will flatten them
"""
for char in [".", "$"]:
fieldname = fieldname.replace(char, '%{:02x}'.format(ord(char)))
return fieldname
@staticmethod
def decode_key_from_mongo(fieldname):
"""
The inverse of encode_key_for_mongo
:param fieldname: with period and dollar escaped
"""
return urllib.unquote(fieldname)
def _get_locator_from_cache(self, location, published):
"""
See if the location x published pair is in the cache. If so, return the mapped locator.
"""
entry = self.cache.get(u'{}+{}'.format(location.course_key, location))
if entry is not None:
if published:
return entry[0]
else:
return entry[1]
return None
def _get_course_locator_from_cache(self, old_course_id, published):
"""
Get the course Locator for this old course id
"""
if not old_course_id:
return None
entry = self.cache.get(unicode(old_course_id))
if entry is not None:
if published:
return entry[0].course_key
else:
return entry[1].course_key
def _get_location_from_cache(self, locator):
"""
See if the locator is in the cache. If so, return the mapped location.
"""
return self.cache.get(unicode(locator))
def _get_course_location_from_cache(self, course_key):
"""
See if the course_key is in the cache. If so, return the mapped location to the
course root.
"""
cache_key = self._course_key_cache_string(course_key)
return self.cache.get(cache_key)
def _course_key_cache_string(self, course_key):
"""
Return the string used to cache the course key
"""
return u'{0.org}+{0.course}+{0.run}'.format(course_key)
def _cache_course_locator(self, old_course_id, published_course_locator, draft_course_locator):
"""
For quick lookup of courses
"""
if not old_course_id:
return
self.cache.set(unicode(old_course_id), (published_course_locator, draft_course_locator))
def _cache_location_map_entry(self, location, published_usage, draft_usage):
"""
Cache the mapping from location to the draft and published Locators in entry.
Also caches the inverse. If the location is category=='course', it caches it for
the get_course query
"""
setmany = {}
if location.category == 'course':
setmany[self._course_key_cache_string(published_usage)] = location.course_key
setmany[unicode(published_usage)] = location
setmany[unicode(draft_usage)] = location
setmany[unicode(location)] = (published_usage, draft_usage)
setmany[unicode(location.course_key)] = (published_usage, draft_usage)
self.cache.set_many(setmany)
def delete_course_mapping(self, course_key):
"""
Remove provided course location from loc_mapper and cache.
:param course_key: a CourseKey for the course we wish to delete
"""
self.location_map.remove(self._interpret_location_course_id(course_key))
# Remove the location of course (draft and published) from cache
cached_key = self.cache.get(unicode(course_key))
if cached_key:
delete_keys = []
published_locator = unicode(cached_key[0].course_key)
course_location = self._course_location_from_cache(published_locator)
delete_keys.append(self._course_key_cache_string(course_key))
delete_keys.append(published_locator)
delete_keys.append(unicode(cached_key[1].course_key))
delete_keys.append(unicode(course_location))
delete_keys.append(unicode(course_key))
self.cache.delete_many(delete_keys)
def _migrate_if_necessary(self, entries):
"""
Run the entries through any applicable schema updates and return the updated entries
"""
entries = [
self._migrate[entry.get('schema', 0)](self, entry)
for entry in entries
]
return entries
def _entry_id_to_son(self, entry_id):
return bson.son.SON([
('org', entry_id['org']),
('course', entry_id['course']),
('name', entry_id['name'])
])
def _delete_cache_location_map_entry(self, old_course_id, location, published_usage, draft_usage):
"""
Remove the location of course (draft and published) from cache
"""
delete_keys = []
if location.category == 'course':
delete_keys.append(self._course_key_cache_string(published_usage.course_key))
delete_keys.append(unicode(published_usage))
delete_keys.append(unicode(draft_usage))
delete_keys.append(u'{}+{}'.format(old_course_id, location.to_deprecated_string()))
delete_keys.append(old_course_id)
self.cache.delete_many(delete_keys)
def _migrate_top(self, entry, updated=False):
"""
Current version, so a no data change until next update. But since it's the top
it's responsible for persisting the record if it changed.
"""
if updated:
entry['schema'] = self.SCHEMA_VERSION
entry_id = self._entry_id_to_son(entry['_id'])
self.location_map.update({'_id': entry_id}, entry)
return entry
def _migrate_0(self, entry):
"""
If entry had an '_id' without a run, remove the whole record.
Add fields: schema, org, course, run
Remove: course_id, lower_course_id
:param entry:
"""
if 'name' not in entry['_id']:
entry_id = entry['_id']
entry_id = bson.son.SON([
('org', entry_id['org']),
('course', entry_id['course']),
])
self.location_map.remove({'_id': entry_id})
return None
# add schema, org, course, run, etc, remove old fields
entry['schema'] = 0
entry.pop('course_id', None)
entry.pop('lower_course_id', None)
old_course_id = SlashSeparatedCourseKey(entry['_id']['org'], entry['_id']['course'], entry['_id']['name'])
entry['org'] = old_course_id.org
entry['course'] = old_course_id.course
entry['run'] = old_course_id.run
return self._migrate_1(entry, True)
# insert new migrations just before _migrate_top. _migrate_top sets the schema version and
# saves the record
_migrate = [_migrate_0, _migrate_top]

4
common/lib/xmodule/xmodule/modulestore/split_migrator.py
View File

@@ -173,7 +173,7 @@ class SplitMigrator(object):
"""
def get_translation(location):
"""
Convert the location and add to loc mapper
Convert the location
"""
return new_course_key.make_usage_key(
location.category,
@@ -207,7 +207,7 @@ class SplitMigrator(object):
"""
def get_translation(location):
"""
Convert the location and add to loc mapper
Convert the location
"""
return new_course_key.make_usage_key(
location.category,

9
common/lib/xmodule/xmodule/modulestore/tests/django_utils.py
View File

@@ -5,10 +5,8 @@ Modulestore configuration for test cases.
from uuid import uuid4
from django.test import TestCase
from django.contrib.auth.models import User
from xmodule.modulestore.django import (
modulestore, clear_existing_modulestores, loc_mapper
)
from xmodule.contentstore.django import _CONTENTSTORE
from xmodule.modulestore.django import modulestore, clear_existing_modulestores
from xmodule.modulestore import ModuleStoreEnum
@@ -197,11 +195,6 @@ class ModuleStoreTestCase(TestCase):
module_store._drop_database() # pylint: disable=protected-access
_CONTENTSTORE.clear()
location_mapper = loc_mapper()
if location_mapper.db:
location_mapper.location_map.drop()
location_mapper.db.connection.close()
@classmethod
def setUpClass(cls):
"""

432
common/lib/xmodule/xmodule/modulestore/tests/test_location_mapper.py
View File

@@ -1,432 +0,0 @@
"""
Test the loc mapper store
"""
import unittest
import uuid
from opaque_keys.edx.locations import Location
from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.mongo.base import MongoRevisionKey
from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError
from xmodule.modulestore.loc_mapper_store import LocMapperStore
from mock import Mock
from opaque_keys.edx.locations import SlashSeparatedCourseKey
import bson.son
class LocMapperSetupSansDjango(unittest.TestCase):
"""
Create and destroy a loc mapper for each test
"""
loc_store = None
def setUp(self):
modulestore_options = {
'host': 'localhost',
'db': 'test_xmodule',
'collection': 'modulestore{0}'.format(uuid.uuid4().hex[:5]),
}
cache_standin = TrivialCache()
self.instrumented_cache = Mock(spec=cache_standin, wraps=cache_standin)
# pylint: disable=W0142
LocMapperSetupSansDjango.loc_store = LocMapperStore(self.instrumented_cache, **modulestore_options)
def tearDown(self):
dbref = TestLocationMapper.loc_store.db
dbref.drop_collection(TestLocationMapper.loc_store.location_map)
dbref.connection.close()
self.loc_store = None
class TestLocationMapper(LocMapperSetupSansDjango):
"""
Test the location to locator mapper
"""
@unittest.skip("getting rid of loc_mapper")
def test_create_map(self):
def _construct_course_son(org, course, run):
"""
Make a lookup son
"""
return bson.son.SON([
('org', org),
('course', course),
('name', run)
])
org = 'foo_org'
course1 = 'bar_course'
run = 'baz_run'
loc_mapper().create_map_entry(SlashSeparatedCourseKey(org, course1, run))
# pylint: disable=protected-access
entry = loc_mapper().location_map.find_one({
'_id': _construct_course_son(org, course1, run)
})
self.assertIsNotNone(entry, "Didn't find entry")
self.assertEqual(entry['org'], org)
self.assertEqual(entry['offering'], '{}.{}'.format(course1, run))
self.assertEqual(entry['draft_branch'], ModuleStoreEnum.BranchName.draft)
self.assertEqual(entry['prod_branch'], ModuleStoreEnum.BranchName.published)
self.assertEqual(entry['block_map'], {})
course2 = 'quux_course'
# oldname: {category: newname}
block_map = {'abc123': {'problem': 'problem2'}}
loc_mapper().create_map_entry(
SlashSeparatedCourseKey(org, course2, run),
'foo_org.geek_dept',
'quux_course.baz_run',
'wip',
'live',
block_map)
entry = loc_mapper().location_map.find_one({
'_id': _construct_course_son(org, course2, run)
})
self.assertIsNotNone(entry, "Didn't find entry")
self.assertEqual(entry['org'], 'foo_org.geek_dept')
self.assertEqual(entry['offering'], '{}.{}'.format(course2, run))
self.assertEqual(entry['draft_branch'], 'wip')
self.assertEqual(entry['prod_branch'], 'live')
self.assertEqual(entry['block_map'], block_map)
@unittest.skip("getting rid of loc_mapper")
def test_delete_course_map(self):
"""
Test that course location is properly remove from loc_mapper and cache when course is deleted
"""
org = u'foo_org'
course = u'bar_course'
run = u'baz_run'
course_location = SlashSeparatedCourseKey(org, course, run)
loc_mapper().create_map_entry(course_location)
# pylint: disable=protected-access
entry = loc_mapper().location_map.find_one({
'_id': loc_mapper()._construct_course_son(course_location)
})
self.assertIsNotNone(entry, 'Entry not found in loc_mapper')
self.assertEqual(entry['offering'], u'{1}.{2}'.format(org, course, run))
# now delete course location from loc_mapper and cache and test that course location no longer
# exists in loca_mapper and cache
loc_mapper().delete_course_mapping(course_location)
# pylint: disable=protected-access
entry = loc_mapper().location_map.find_one({
'_id': loc_mapper()._construct_course_son(course_location)
})
self.assertIsNone(entry, 'Entry found in loc_mapper')
# pylint: disable=protected-access
cached_value = loc_mapper()._get_location_from_cache(course_location.make_usage_key('course', run))
self.assertIsNone(cached_value, 'course_locator found in cache')
# pylint: disable=protected-access
cached_value = loc_mapper()._get_course_location_from_cache(course_location)
self.assertIsNone(cached_value, 'Entry found in cache')
@unittest.skip("getting rid of loc_mapper")
def translate_n_check(self, location, org, offering, block_id, branch, add_entry=False):
"""
Request translation, check org, offering, block_id, and branch
"""
prob_locator = loc_mapper().translate_location(
location,
published=(branch == ModuleStoreEnum.BranchName.published),
add_entry_if_missing=add_entry
)
self.assertEqual(prob_locator.org, org)
self.assertEqual(prob_locator.offering, offering)
self.assertEqual(prob_locator.block_id, block_id)
self.assertEqual(prob_locator.branch, branch)
course_locator = loc_mapper().translate_location_to_course_locator(
location.course_key,
published=(branch == ModuleStoreEnum.BranchName.published),
)
self.assertEqual(course_locator.org, org)
self.assertEqual(course_locator.offering, offering)
self.assertEqual(course_locator.branch, branch)
@unittest.skip("getting rid of loc_mapper")
def test_translate_location_read_only(self):
"""
Test the variants of translate_location which don't create entries, just decode
"""
# lookup before there are any maps
org = 'foo_org'
course = 'bar_course'
run = 'baz_run'
slash_course_key = SlashSeparatedCourseKey(org, course, run)
with self.assertRaises(ItemNotFoundError):
_ = loc_mapper().translate_location(
Location(org, course, run, 'problem', 'abc123'),
add_entry_if_missing=False
)
new_style_org = '{}.geek_dept'.format(org)
new_style_offering = '.{}.{}'.format(course, run)
block_map = {
'abc123': {'problem': 'problem2', 'vertical': 'vertical2'},
'def456': {'problem': 'problem4'},
'ghi789': {'problem': 'problem7'},
}
loc_mapper().create_map_entry(
slash_course_key,
new_style_org, new_style_offering,
block_map=block_map
)
test_problem_locn = Location(org, course, run, 'problem', 'abc123')
self.translate_n_check(test_problem_locn, new_style_org, new_style_offering, 'problem2',
ModuleStoreEnum.BranchName.published)
# look for non-existent problem
with self.assertRaises(ItemNotFoundError):
loc_mapper().translate_location(
Location(org, course, run, 'problem', '1def23'),
add_entry_if_missing=False
)
test_no_cat_locn = test_problem_locn.replace(category=None)
with self.assertRaises(InvalidLocationError):
loc_mapper().translate_location(
slash_course_key.make_usage_key(None, 'abc123'), test_no_cat_locn, False, False
)
test_no_cat_locn = test_no_cat_locn.replace(name='def456')
self.translate_n_check(
test_no_cat_locn, new_style_org, new_style_offering, 'problem4', ModuleStoreEnum.BranchName.published
)
# add a distractor course (note that abc123 has a different translation in this one)
distractor_block_map = {
'abc123': {'problem': 'problem3'},
'def456': {'problem': 'problem4'},
'ghi789': {'problem': 'problem7'},
}
run = 'delta_run'
test_delta_new_org = '{}.geek_dept'.format(org)
test_delta_new_offering = '{}.{}'.format(course, run)
loc_mapper().create_map_entry(
SlashSeparatedCourseKey(org, course, run),
test_delta_new_org, test_delta_new_offering,
block_map=distractor_block_map
)
# test that old translation still works
self.translate_n_check(
test_problem_locn, new_style_org, new_style_offering, 'problem2', ModuleStoreEnum.BranchName.published
)
# and new returns new id
self.translate_n_check(
test_problem_locn.replace(run=run), test_delta_new_org, test_delta_new_offering,
'problem3', ModuleStoreEnum.BranchName.published
)
@unittest.skip("getting rid of loc_mapper")
def test_translate_location_dwim(self):
"""
Test the location translation mechanisms which try to do-what-i-mean by creating new
entries for never seen queries.
"""
org = 'foo_org'
course = 'bar_course'
run = 'baz_run'
problem_name = 'abc123abc123abc123abc123abc123f9'
location = Location(org, course, run, 'problem', problem_name)
new_offering = '{}.{}'.format(course, run)
self.translate_n_check(location, org, new_offering, 'problemabc', ModuleStoreEnum.BranchName.published, True)
# create an entry w/o a guid name
other_location = Location(org, course, run, 'chapter', 'intro')
self.translate_n_check(other_location, org, new_offering, 'intro', ModuleStoreEnum.BranchName.published, True)
# add a distractor course
delta_new_org = '{}.geek_dept'.format(org)
run = 'delta_run'
delta_new_offering = '{}.{}'.format(course, run)
delta_course_locn = SlashSeparatedCourseKey(org, course, run)
loc_mapper().create_map_entry(
delta_course_locn,
delta_new_org, delta_new_offering,
block_map={problem_name: {'problem': 'problem3'}}
)
self.translate_n_check(location, org, new_offering, 'problemabc', ModuleStoreEnum.BranchName.published, True)
# add a new one to both courses (ensure name doesn't have same beginning)
new_prob_name = uuid.uuid4().hex
while new_prob_name.startswith('abc'):
new_prob_name = uuid.uuid4().hex
new_prob_locn = location.replace(name=new_prob_name)
new_usage_id = 'problem{}'.format(new_prob_name[:3])
self.translate_n_check(new_prob_locn, org, new_offering, new_usage_id, ModuleStoreEnum.BranchName.published, True)
new_prob_locn = new_prob_locn.replace(run=run)
self.translate_n_check(
new_prob_locn, delta_new_org, delta_new_offering, new_usage_id, ModuleStoreEnum.BranchName.published, True
)
@unittest.skip("getting rid of loc_mapper")
def test_translate_locator(self):
"""
tests translate_locator_to_location(BlockUsageLocator)
"""
# lookup for non-existent course
org = 'foo_org'
course = 'bar_course'
run = 'baz_run'
new_style_org = '{}.geek_dept'.format(org)
new_style_offering = '{}.{}'.format(course, run)
prob_course_key = CourseLocator(
org=new_style_org, offering=new_style_offering,
branch=ModuleStoreEnum.BranchName.published,
)
prob_locator = BlockUsageLocator(
prob_course_key,
block_type='problem',
block_id='problem2',
)
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
self.assertIsNone(prob_location, 'found entry in empty map table')
loc_mapper().create_map_entry(
SlashSeparatedCourseKey(org, course, run),
new_style_org, new_style_offering,
block_map={
'abc123': {'problem': 'problem2'},
'48f23a10395384929234': {'chapter': 'chapter48f'},
'baz_run': {'course': 'root'},
}
)
# only one course matches
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
# default branch
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', MongoRevisionKey.published))
# test get_course keyword
prob_location = loc_mapper().translate_locator_to_location(prob_locator, get_course=True)
self.assertEqual(prob_location, SlashSeparatedCourseKey(org, course, run))
# explicit branch
prob_locator = prob_locator.for_branch(ModuleStoreEnum.BranchName.draft)
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
# Even though the problem was set as draft, we always return revision= MongoRevisionKey.published to work
# with old mongo/draft modulestores.
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', MongoRevisionKey.published))
prob_locator = BlockUsageLocator(
prob_course_key.for_branch('production'),
block_type='problem', block_id='problem2'
)
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', MongoRevisionKey.published))
# same for chapter except chapter cannot be draft in old system
chap_locator = BlockUsageLocator(
prob_course_key.for_branch('production'),
block_type='chapter', block_id='chapter48f',
)
chap_location = loc_mapper().translate_locator_to_location(chap_locator)
self.assertEqual(chap_location, Location(org, course, run, 'chapter', '48f23a10395384929234'))
# explicit branch
chap_locator = chap_locator.for_branch(ModuleStoreEnum.BranchName.draft)
chap_location = loc_mapper().translate_locator_to_location(chap_locator)
self.assertEqual(chap_location, Location(org, course, run, 'chapter', '48f23a10395384929234'))
chap_locator = BlockUsageLocator(
prob_course_key.for_branch('production'), block_type='chapter', block_id='chapter48f'
)
chap_location = loc_mapper().translate_locator_to_location(chap_locator)
self.assertEqual(chap_location, Location(org, course, run, 'chapter', '48f23a10395384929234'))
# look for non-existent problem
prob_locator2 = BlockUsageLocator(
prob_course_key.for_branch(ModuleStoreEnum.BranchName.draft),
block_type='problem', block_id='problem3'
)
prob_location = loc_mapper().translate_locator_to_location(prob_locator2)
self.assertIsNone(prob_location, 'Found non-existent problem')
# add a distractor course
delta_run = 'delta_run'
new_style_offering = '{}.{}'.format(course, delta_run)
loc_mapper().create_map_entry(
SlashSeparatedCourseKey(org, course, delta_run),
new_style_org, new_style_offering,
block_map={'abc123': {'problem': 'problem3'}}
)
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', MongoRevisionKey.published))
@unittest.skip("getting rid of loc_mapper")
def test_special_chars(self):
"""
Test locations which have special characters
"""
# afaik, location.check_list prevents $ in all fields
org = 'foo.org.edu'
course = 'bar.course-4'
name = 'baz.run_4-3'
location = Location(org, course, name, 'course', name)
prob_locator = loc_mapper().translate_location(
location,
add_entry_if_missing=True
)
reverted_location = loc_mapper().translate_locator_to_location(prob_locator)
self.assertEqual(location, reverted_location)
@unittest.skip("getting rid of loc_mapper")
def test_name_collision(self):
"""
Test dwim translation when the old name was not unique
"""
org = "myorg"
course = "another_course"
name = "running_again"
course_location = Location(org, course, name, 'course', name)
course_xlate = loc_mapper().translate_location(course_location, add_entry_if_missing=True)
self.assertEqual(course_location, loc_mapper().translate_locator_to_location(course_xlate))
eponymous_block = course_location.replace(category='chapter')
chapter_xlate = loc_mapper().translate_location(eponymous_block, add_entry_if_missing=True)
self.assertEqual(course_location, loc_mapper().translate_locator_to_location(course_xlate))
self.assertEqual(eponymous_block, loc_mapper().translate_locator_to_location(chapter_xlate))
# and a non-existent one w/o add
eponymous_block = course_location.replace(category='problem')
with self.assertRaises(ItemNotFoundError):
chapter_xlate = loc_mapper().translate_location(eponymous_block, add_entry_if_missing=False)
#==================================
# functions to mock existing services
def loc_mapper():
"""
Mocks the global location mapper.
"""
return LocMapperSetupSansDjango.loc_store
def render_to_template_mock(*_args):
"""
Mocks the mako render_to_template w/ a noop
"""
class TrivialCache(object):
"""
A trivial cache impl
"""
def __init__(self):
self.cache = {}
def get(self, key, default=None):
"""
Mock the .get
"""
return self.cache.get(key, default)
def set_many(self, entries):
"""
mock set_many
"""
self.cache.update(entries)
def set(self, key, entry):
"""
mock set
"""
self.cache[key] = entry
def delete_many(self, entries):
"""
mock delete_many
"""
for entry in entries:
del self.cache[entry]

4
docs/en_us/developers/source/modulestore.rst
View File

@@ -33,10 +33,6 @@ Modulestore Helpers
These packages provide utilities for easier use of modulestores,
and migrating data between modulestores.
.. automodule:: xmodule.modulestore.loc_mapper_store
:members:
:show-inheritance:
.. automodule:: xmodule.modulestore.search
:members:
:show-inheritance:

8
mongo_indexes.md
View File

@@ -12,14 +12,6 @@ db.collection_name
```
as in ```db.location_map.ensureIndex({'course_id': 1}{background: true})```
location_map:
=============
```
ensureIndex({'org': 1, 'offering': 1})
ensureIndex({'schema': 1})
```
fs.files:
=========