Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sql migrations cannot use loc mapper

Browse Source
This commit is contained in:
Don Mitchell
2014-07-14 15:03:45 -04:00
parent a9695828ab
commit 9f2fa7e105
2 changed files with 127 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
common/djangoapps/student/migrations/0035_access_roles.py
View File

@@ -1,8 +1,5 @@
# -*- coding: utf-8 -*-
from south.db import db
from south.v2 import DataMigration
from django.db import models
from xmodule.modulestore.django import loc_mapper
import re
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from opaque_keys import InvalidKeyError
@@ -10,6 +7,10 @@ import bson.son
import logging
from django.db.models.query_utils import Q
from django.db.utils import IntegrityError
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.mixed import MixedModuleStore
import itertools
log = logging.getLogger(__name__)
@@ -25,8 +26,20 @@ class Migration(DataMigration):
"""
Converts group table entries for write access and beta_test roles to course access roles table.
"""
store = modulestore()
if isinstance(store, MixedModuleStore):
self.mongostore = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo)
self.xmlstore = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.xml)
elif store.get_modulestore_type() == ModuleStoreEnum.Type.mongo:
self.mongostore = store
self.xmlstore = None
elif store.get_modulestore_type() == ModuleStoreEnum.Type.xml:
self.mongostore = None
self.xmlstore = store
else:
return
# Note: Remember to use orm['appname.ModelName'] rather than "from appname.models..."
loc_map_collection = loc_mapper().location_map
# b/c the Groups table had several entries for each course, we need to ensure we process each unique
# course only once. The below datastructures help ensure that.
hold = {} # key of course_id_strings with array of group objects. Should only be org scoped entries
@@ -64,21 +77,27 @@ class Migration(DataMigration):
course_key = SlashSeparatedCourseKey.from_deprecated_string(course_id_string)
# course_key is the downcased version, get the normal cased one. loc_mapper() has no
# methods taking downcased SSCK; so, need to do it manually here
correct_course_key = self._map_downcased_ssck(course_key, loc_map_collection)
correct_course_key = self._map_downcased_ssck(course_key)
if correct_course_key is not None:
_migrate_users(correct_course_key, role, course_key.org)
except InvalidKeyError:
entry = loc_map_collection.find_one({
'course_id': re.compile(r'^{}$'.format(course_id_string), re.IGNORECASE)
})
if entry is None:
# old dotted format, try permutations
parts = course_id_string.split('.')
if len(parts) < 3:
hold.setdefault(course_id_string, []).append(group)
else:
correct_course_key = SlashSeparatedCourseKey(*entry['_id'].values())
if 'lower_id' in entry:
_migrate_users(correct_course_key, role, entry['lower_id']['org'])
elif len(parts) == 3:
course_key = SlashSeparatedCourseKey(*parts)
correct_course_key = self._map_downcased_ssck(course_key)
if correct_course_key is None:
hold.setdefault(course_id_string, []).append(group)
else:
_migrate_users(correct_course_key, role, entry['_id']['org'].lower())
_migrate_users(correct_course_key, role, course_key.org)
else:
correct_course_key = self.divide_parts_find_key(parts)
if correct_course_key is None:
hold.setdefault(course_id_string, []).append(group)
else:
_migrate_users(correct_course_key, role, course_key.org)
# see if any in hold were missed above
for held_auth_scope, groups in hold.iteritems():
@@ -99,28 +118,50 @@ class Migration(DataMigration):
# don't silently skip unexpected roles
log.warn("Didn't convert roles %s", [group.name for group in groups])
def divide_parts_find_key(self, parts):
"""
Look for all possible org/course/run patterns from a possibly dotted source
"""
for org_stop, course_stop in itertools.combinations(range(1, len(parts)), 2):
org = '.'.join(parts[:org_stop])
course = '.'.join(parts[org_stop:course_stop])
run = '.'.join(parts[course_stop:])
course_key = SlashSeparatedCourseKey(org, course, run)
correct_course_key = self._map_downcased_ssck(course_key)
if correct_course_key is not None:
return correct_course_key
return None
def backwards(self, orm):
"Write your backwards methods here."
"Removes the new table."
# Since this migration is non-destructive (monotonically adds information), I'm not sure what
# the semantic of backwards should be other than perhaps clearing the table.
orm['student.courseaccessrole'].objects.all().delete()
def _map_downcased_ssck(self, downcased_ssck, loc_map_collection):
def _map_downcased_ssck(self, downcased_ssck):
"""
Get the normal cased version of this downcased slash sep course key
"""
# given the regex, the son may be an overkill
course_son = bson.son.SON([
('_id.org', re.compile(r'^{}$'.format(downcased_ssck.org), re.IGNORECASE)),
('_id.course', re.compile(r'^{}$'.format(downcased_ssck.course), re.IGNORECASE)),
('_id.name', re.compile(r'^{}$'.format(downcased_ssck.run), re.IGNORECASE)),
])
entry = loc_map_collection.find_one(course_son)
if entry:
idpart = entry['_id']
return SlashSeparatedCourseKey(idpart['org'], idpart['course'], idpart['name'])
else:
return None
if self.mongostore is not None:
course_son = bson.son.SON([
('_id.tag', 'i4x'),
('_id.org', re.compile(r'^{}$'.format(downcased_ssck.org), re.IGNORECASE)),
('_id.course', re.compile(r'^{}$'.format(downcased_ssck.course), re.IGNORECASE)),
('_id.category', 'course'),
('_id.name', re.compile(r'^{}$'.format(downcased_ssck.run), re.IGNORECASE)),
])
entry = self.mongostore.collection.find_one(course_son)
if entry:
idpart = entry['_id']
return SlashSeparatedCourseKey(idpart['org'], idpart['course'], idpart['name'])
if self.xmlstore is not None:
for course in self.xmlstore.get_courses():
if (
course.id.org.lower() == downcased_ssck.org and course.id.course.lower() == downcased_ssck.course
and course.id.run.lower() == downcased_ssck.run
):
return course.id
return None
models = {

73
common/djangoapps/student/migrations/0036_access_roles_orgless.py
View File

@@ -1,14 +1,15 @@
# -*- coding: utf-8 -*-
from south.v2 import DataMigration
from xmodule.modulestore.django import loc_mapper, modulestore
from xmodule.modulestore.django import modulestore
import re
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from opaque_keys import InvalidKeyError
import logging
from django.db.models.query_utils import Q
from django.db.utils import IntegrityError
from xmodule.modulestore import ModuleStoreEnum
import bson.son
from xmodule.modulestore.mixed import MixedModuleStore
import itertools
log = logging.getLogger(__name__)
@@ -24,11 +25,18 @@ class Migration(DataMigration):
"""
Converts group table entries for write access and beta_test roles to course access roles table.
"""
# Note: Remember to use orm['appname.ModelName'] rather than "from appname.models..."
loc_map_collection = loc_mapper().location_map
mixed_ms = modulestore()
xml_ms = mixed_ms._get_modulestore_by_type(ModuleStoreEnum.Type.xml)
mongo_ms = mixed_ms._get_modulestore_by_type(ModuleStoreEnum.Type.mongo)
store = modulestore()
if isinstance(store, MixedModuleStore):
self.mongostore = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo)
self.xmlstore = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.xml)
elif store.get_modulestore_type() == ModuleStoreEnum.Type.mongo:
self.mongostore = store
self.xmlstore = None
elif store.get_modulestore_type() == ModuleStoreEnum.Type.xml:
self.mongostore = None
self.xmlstore = store
else:
return
query = Q(name__startswith='staff') | Q(name__startswith='instructor') | Q(name__startswith='beta_testers')
for group in orm['auth.Group'].objects.filter(query).exclude(name__contains="/").all():
@@ -59,10 +67,7 @@ class Migration(DataMigration):
role = parsed_entry.group('role_id')
course_id_string = parsed_entry.group('course_id_string')
# if it's a full course_id w/ dots, ignore it
entry = loc_map_collection.find_one({
'course_id': re.compile(r'^{}$'.format(course_id_string), re.IGNORECASE)
})
if entry is None:
if u'/' not in course_id_string and not self.dotted_course(course_id_string):
# check new table to see if it's been added as org permission
if not orm['student.courseaccessrole'].objects.filter(
role=role,
@@ -70,14 +75,14 @@ class Migration(DataMigration):
).exists():
# old auth was of form role_coursenum. Grant access to all such courses wildcarding org and run
# look in xml for matching courses
if xml_ms is not None:
for course in xml_ms.get_courses():
if self.xmlstore is not None:
for course in self.xmlstore.get_courses():
if course_id_string == course.id.course.lower():
_migrate_users(course.id, role)
if mongo_ms is not None:
if self.mongostore is not None:
mongo_query = re.compile(ur'^{}$'.format(course_id_string), re.IGNORECASE)
for mongo_entry in mongo_ms.collection.find(
for mongo_entry in self.mongostore.collection.find(
{"_id.category": "course", "_id.course": mongo_query}, fields=["_id"]
):
mongo_id_dict = mongo_entry['_id']
@@ -86,6 +91,44 @@ class Migration(DataMigration):
)
_migrate_users(course_key, role)
def dotted_course(self, parts):
"""
Look for all possible org/course/run patterns from a possibly dotted source
"""
for org_stop, course_stop in itertools.combinations(range(1, len(parts)), 2):
org = '.'.join(parts[:org_stop])
course = '.'.join(parts[org_stop:course_stop])
run = '.'.join(parts[course_stop:])
course_key = SlashSeparatedCourseKey(org, course, run)
correct_course_key = self._map_downcased_ssck(course_key)
if correct_course_key is not None:
return correct_course_key
return False
def _map_downcased_ssck(self, downcased_ssck):
"""
Get the normal cased version of this downcased slash sep course key
"""
if self.mongostore is not None:
course_son = bson.son.SON([
('_id.tag', 'i4x'),
('_id.org', re.compile(r'^{}$'.format(downcased_ssck.org), re.IGNORECASE)),
('_id.course', re.compile(r'^{}$'.format(downcased_ssck.course), re.IGNORECASE)),
('_id.category', 'course'),
('_id.name', re.compile(r'^{}$'.format(downcased_ssck.run), re.IGNORECASE)),
])
entry = self.mongostore.collection.find_one(course_son)
if entry:
idpart = entry['_id']
return SlashSeparatedCourseKey(idpart['org'], idpart['course'], idpart['name'])
if self.xmlstore is not None:
for course in self.xmlstore.get_courses():
if (
course.id.org.lower() == downcased_ssck.org and course.id.course.lower() == downcased_ssck.course
and course.id.run.lower() == downcased_ssck.run
):
return course.id
return None
def backwards(self, orm):
"No obvious way to reverse just this migration, but reversing 0035 will reverse this."