Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use mongo indices for all queries

Browse Source 
STUD-1039
This commit is contained in:
Don Mitchell
2013-12-10 12:15:16 -05:00
parent 985417fcf6
commit f4e69275ee
5 changed files with 79 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
cms/djangoapps/contentstore/views/course.py
View File

@@ -116,7 +116,8 @@ def course_listing(request):
"""
List all courses available to the logged in user
"""
courses = modulestore('direct').get_items(['i4x', None, None, 'course', None])
# there's an index on category which will be used if none of its antecedents are set
courses = modulestore('direct').get_items([None, None, None, 'course', None])
# filter out courses that we don't have access too
def course_filter(course):

64
common/lib/xmodule/xmodule/modulestore/loc_mapper_store.py
View File

@@ -4,10 +4,10 @@ Method for converting among our differing Location/Locator whatever reprs
from random import randint
import re
import pymongo
import bson.son
from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError, DuplicateItemError
from xmodule.modulestore.locator import BlockUsageLocator
from xmodule.modulestore.mongo import draft
from xmodule.modulestore import Location
import urllib
@@ -91,9 +91,10 @@ class LocMapperStore(object):
else:
course_id = "{0.org}.{0.course}".format(course_location)
# very like _interpret_location_id but w/o the _id
location_id = {'org': course_location.org, 'course': course_location.course}
if course_location.category == 'course':
location_id['name'] = course_location.name
location_id = self._construct_location_son(
course_location.org, course_location.course,
course_location.name if course_location.category == 'course' else None
)
self.location_map.insert({
'_id': location_id,
@@ -128,20 +129,25 @@ class LocMapperStore(object):
"""
location_id = self._interpret_location_course_id(old_style_course_id, location)
maps = self.location_map.find(location_id).sort('_id.name', pymongo.ASCENDING)
if maps.count() == 0:
maps = self.location_map.find(location_id)
maps = list(maps)
if len(maps) == 0:
if add_entry_if_missing:
# create a new map
course_location = location.replace(category='course', name=location_id['_id.name'])
course_location = location.replace(category='course', name=location_id['_id']['name'])
self.create_map_entry(course_location)
entry = self.location_map.find_one(location_id)
else:
raise ItemNotFoundError()
elif maps.count() > 1:
# if more than one, prefer the one w/o a name if that exists. Otherwise, choose the first (alphabetically)
elif len(maps) == 1:
entry = maps[0]
else:
# find entry w/o name, if any; otherwise, pick arbitrary
entry = maps[0]
for item in maps:
if 'name' not in item['_id']:
entry = item
break
if published:
branch = entry['prod_branch']
@@ -242,11 +248,11 @@ class LocMapperStore(object):
location_id = self._interpret_location_course_id(old_course_id, location)
maps = self.location_map.find(location_id)
if maps.count() == 0:
raise ItemNotFoundError()
# turn maps from cursor to list
map_list = list(maps)
if len(map_list) == 0:
raise ItemNotFoundError()
encoded_location_name = self._encode_for_mongo(location.name)
# check whether there's already a usage_id for this location (and it agrees w/ any passed in or found)
for map_entry in map_list:
@@ -279,7 +285,10 @@ class LocMapperStore(object):
)
map_entry['block_map'].setdefault(encoded_location_name, {})[location.category] = computed_usage_id
self.location_map.update({'_id': map_entry['_id']}, {'$set': {'block_map': map_entry['block_map']}})
self.location_map.update(
{'_id': self._construct_location_son(**map_entry['_id'])},
{'$set': {'block_map': map_entry['block_map']}}
)
return computed_usage_id
@@ -317,7 +326,10 @@ class LocMapperStore(object):
if location.category in map_entry['block_map'].setdefault(encoded_location_name, {}):
map_entry['block_map'][encoded_location_name][location.category] = usage_id
self.location_map.update({'_id': map_entry['_id']}, {'$set': {'block_map': map_entry['block_map']}})
self.location_map.update(
{'_id': self._construct_location_son(**map_entry['_id'])},
{'$set': {'block_map': map_entry['block_map']}}
)
return usage_id
@@ -338,7 +350,10 @@ class LocMapperStore(object):
del map_entry['block_map'][encoded_location_name]
else:
del map_entry['block_map'][encoded_location_name][location.category]
self.location_map.update({'_id': map_entry['_id']}, {'$set': {'block_map': map_entry['block_map']}})
self.location_map.update(
{'_id': self._construct_location_son(**map_entry['_id'])},
{'$set': {'block_map': map_entry['block_map']}}
)
def _add_to_block_map(self, location, location_id, block_map):
'''add the given location to the block_map and persist it'''
@@ -357,7 +372,7 @@ class LocMapperStore(object):
def _interpret_location_course_id(self, course_id, location):
"""
Take the old style course id (org/course/run) and return a dict for querying the mapping table.
Take the old style course id (org/course/run) and return a dict w/ a SON for querying the mapping table.
If the course_id is empty, it uses location, but this may result in an inadequate id.
:param course_id: old style 'org/course/run' id from Location.course_id where Location.category = 'course'
@@ -367,12 +382,21 @@ class LocMapperStore(object):
if course_id:
# re doesn't allow ?P<_id.org> and ilk
matched = re.match(r'([^/]+)/([^/]+)/([^/]+)', course_id)
return dict(zip(['_id.org', '_id.course', '_id.name'], matched.groups()))
return {'_id': self._construct_location_son(*matched.groups())}
location_id = {'_id.org': location.org, '_id.course': location.course}
if location.category == 'course':
location_id['_id.name'] = location.name
return location_id
return {'_id': self._construct_location_son(location.org, location.course, location.name)}
else:
return bson.son.SON([('_id.org', location.org), ('_id.course', location.course)])
def _construct_location_son(self, org, course, name=None):
"""
Construct the SON needed to repr the location for either a query or an insertion
"""
if name:
return bson.son.SON([('org', org), ('course', course), ('name', name)])
else:
return bson.son.SON([('org', org), ('course', course)])
def _block_id_is_guid(self, name):
"""

10
common/lib/xmodule/xmodule/modulestore/tests/test_location_mapper.py
View File

@@ -36,8 +36,9 @@ class TestLocationMapper(unittest.TestCase):
org = 'foo_org'
course = 'bar_course'
loc_mapper().create_map_entry(Location('i4x', org, course, 'course', 'baz_run'))
# pylint: disable=protected-access
entry = loc_mapper().location_map.find_one({
'_id': {'org': org, 'course': course, 'name': 'baz_run'}
'_id': loc_mapper()._construct_location_son(org, course, 'baz_run')
})
self.assertIsNotNone(entry, "Didn't find entry")
self.assertEqual(entry['course_id'], '{}.{}.baz_run'.format(org, course))
@@ -48,8 +49,9 @@ class TestLocationMapper(unittest.TestCase):
# ensure create_entry does the right thing when not given a course (creates org/course
# rather than org/course/run course_id)
loc_mapper().create_map_entry(Location('i4x', org, course, 'vertical', 'baz_vert'))
# find the one which has no name
entry = loc_mapper().location_map.find_one({
'_id': {'org': org, 'course': course}
'_id' : loc_mapper()._construct_location_son(org, course, None)
})
self.assertIsNotNone(entry, "Didn't find entry")
self.assertEqual(entry['course_id'], '{}.{}'.format(org, course))
@@ -63,9 +65,7 @@ class TestLocationMapper(unittest.TestCase):
'wip',
'live',
block_map)
entry = loc_mapper().location_map.find_one({
'_id': {'org': org, 'course': course}
})
entry = loc_mapper().location_map.find_one({'_id.org': org, '_id.course': course})
self.assertIsNotNone(entry, "Didn't find entry")
self.assertEqual(entry['course_id'], 'foo_org.geek_dept.quux_course.baz_run')
self.assertEqual(entry['draft_branch'], 'wip')

9
lms/djangoapps/courseware/roles.py
View File

@@ -137,6 +137,11 @@ class CourseRole(GroupBasedRole):
A named role in a particular course
"""
def __init__(self, role, location, course_context=None):
"""
Location may be either a Location, a string, dict, or tuple which Location will accept
in its constructor, or a CourseLocator. Handle all these giving some preference to
the preferred naming.
"""
# TODO: figure out how to make the group name generation lazy so it doesn't force the
# loc mapping?
if not hasattr(location, 'course_id'):
@@ -153,14 +158,14 @@ class CourseRole(GroupBasedRole):
# pylint: disable=no-member
if isinstance(location, Location):
# least preferred legacy role_course format
groupnames.append('{0}_{1}'.format(role, location.course))
try:
locator = loc_mapper().translate_location(location.course_id, location, False, False)
groupnames.append('{0}_{1}'.format(role, locator.course_id))
except (InvalidLocationError, ItemNotFoundError):
# if it's never been mapped, the auth won't be via the Locator syntax
pass
# least preferred legacy role_course format
groupnames.append('{0}_{1}'.format(role, location.course))
elif isinstance(location, CourseLocator):
# handle old Location syntax
old_location = loc_mapper().translate_locator_to_location(location, get_course=True)

21
mongo_indexes.md Normal file
View File

@@ -0,0 +1,21 @@
These are the indexes each mongo db should have in order to perform well.
Each section states the collection name and then the indexes. To create an index,
you'll typically either use the mongohq type web interface or a standard terminal console.
If a terminal, this assumes you've logged in and gotten to the mongo prompt
```
mongo mydatabasename
```
If using the terminal, to add an index to a collection, you'll need to prefix ```ensureIndex``` with
```
db.collection_name
```
as in ```db.location_map.ensureIndex({'course_id': 1}{background: true})```
location_map:
=============
```
ensureIndex({​'_id.org': 1, '_id.course': 1})
ensureIndex({​'course_id': 1})
```