Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make course ids and usage ids opaque to LMS and Studio [partial commit]

Browse Source 
This commit updates common/lib/xmodule.

These keys are now objects with a limited interface, and the particular
internal representation is managed by the data storage layer (the
modulestore).

For the LMS, there should be no outward-facing changes to the system.
The keys are, for now, a change to internal representation only. For
Studio, the new serialized form of the keys is used in urls, to allow
for further migration in the future.

Co-Author: Andy Armstrong <andya@edx.org>
Co-Author: Christina Roberts <christina@edx.org>
Co-Author: David Baumgold <db@edx.org>
Co-Author: Diana Huang <dkh@edx.org>
Co-Author: Don Mitchell <dmitchell@edx.org>
Co-Author: Julia Hansbrough <julia@edx.org>
Co-Author: Nimisha Asthagiri <nasthagiri@edx.org>
Co-Author: Sarina Canelake <sarina@edx.org>

[LMS-2370]
This commit is contained in:
Calen Pennington
2014-04-30 10:17:45 -04:00
parent 7852906ce0
commit d654798856
79 changed files with 3792 additions and 4055 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
common/lib/xmodule/setup.py
View File

@@ -67,5 +67,17 @@ setup(
'console_scripts': [
'xmodule_assets = xmodule.static_content:main',
],
'course_key': [
'slashes = xmodule.modulestore.locations:SlashSeparatedCourseKey',
'course-locator = xmodule.modulestore.locator:CourseLocator',
],
'usage_key': [
'location = xmodule.modulestore.locations:Location',
'edx = xmodule.modulestore.locator:BlockUsageLocator',
],
'asset_key': [
'asset-location = xmodule.modulestore.locations:AssetLocation',
'edx = xmodule.modulestore.locator:BlockUsageLocator',
],
},
)

2
common/lib/xmodule/xmodule/abtest_module.py
View File

@@ -67,7 +67,7 @@ class ABTestModule(ABTestFields, XModule):
def get_child_descriptors(self):
active_locations = set(self.group_content[self.group])
return [desc for desc in self.descriptor.get_children() if desc.location.url() in active_locations]
return [desc for desc in self.descriptor.get_children() if desc.location.to_deprecated_string() in active_locations]
def displayable_items(self):
# Most modules return "self" as the displayable_item. We never display ourself

24
common/lib/xmodule/xmodule/capa_base.py
View File

@@ -207,7 +207,7 @@ class CapaMixin(CapaFields):
# Need the problem location in openendedresponse to send out. Adding
# it to the system here seems like the least clunky way to get it
# there.
self.runtime.set('location', self.location.url())
self.runtime.set('location', self.location.to_deprecated_string())
try:
# TODO (vshnayder): move as much as possible of this work and error
@@ -225,7 +225,7 @@ class CapaMixin(CapaFields):
except Exception as err: # pylint: disable=broad-except
msg = u'cannot create LoncapaProblem {loc}: {err}'.format(
loc=self.location.url(), err=err)
loc=self.location.to_deprecated_string(), err=err)
# TODO (vshnayder): do modules need error handlers too?
# We shouldn't be switching on DEBUG.
if self.runtime.DEBUG:
@@ -239,7 +239,7 @@ class CapaMixin(CapaFields):
# create a dummy problem with error message instead of failing
problem_text = (u'<problem><text><span class="inline-error">'
u'Problem {url} has an error:</span>{msg}</text></problem>'.format(
url=self.location.url(),
url=self.location.to_deprecated_string(),
msg=msg)
)
self.lcp = self.new_lcp(self.get_state_for_lcp(), text=problem_text)
@@ -259,7 +259,7 @@ class CapaMixin(CapaFields):
self.seed = 1
elif self.rerandomize == "per_student" and hasattr(self.runtime, 'seed'):
# see comment on randomization_bin
self.seed = randomization_bin(self.runtime.seed, self.location.url)
self.seed = randomization_bin(self.runtime.seed, unicode(self.location).encode('utf-8'))
else:
self.seed = struct.unpack('i', os.urandom(4))[0]
@@ -370,7 +370,7 @@ class CapaMixin(CapaFields):
progress = self.get_progress()
return self.runtime.render_template('problem_ajax.html', {
'element_id': self.location.html_id(),
'id': self.id,
'id': self.location.to_deprecated_string(),
'ajax_url': self.runtime.ajax_url,
'progress_status': Progress.to_js_status_str(progress),
'progress_detail': Progress.to_js_detail_str(progress),
@@ -510,7 +510,7 @@ class CapaMixin(CapaFields):
msg = (
u'[courseware.capa.capa_module] <font size="+1" color="red">'
u'Failed to generate HTML for problem {url}</font>'.format(
url=cgi.escape(self.location.url()))
url=cgi.escape(self.location.to_deprecated_string()))
)
msg += u'<p>Error:</p><p><pre>{msg}</pre></p>'.format(msg=cgi.escape(err.message))
msg += u'<p><pre>{tb}</pre></p>'.format(tb=cgi.escape(traceback.format_exc()))
@@ -598,7 +598,7 @@ class CapaMixin(CapaFields):
context = {
'problem': content,
'id': self.id,
'id': self.location.to_deprecated_string(),
'check_button': check_button,
'check_button_checking': check_button_checking,
'reset_button': self.should_show_reset_button(),
@@ -763,7 +763,7 @@ class CapaMixin(CapaFields):
Returns the answers: {'answers' : answers}
"""
event_info = dict()
event_info['problem_id'] = self.location.url()
event_info['problem_id'] = self.location.to_deprecated_string()
self.track_function_unmask('showanswer', event_info)
if not self.answer_available():
raise NotFoundError('Answer is not available')
@@ -906,7 +906,7 @@ class CapaMixin(CapaFields):
"""
event_info = dict()
event_info['state'] = self.lcp.get_state()
event_info['problem_id'] = self.location.url()
event_info['problem_id'] = self.location.to_deprecated_string()
answers = self.make_dict_of_responses(data)
answers_without_files = convert_files_to_filenames(answers)
@@ -1218,7 +1218,7 @@ class CapaMixin(CapaFields):
Returns the error messages for exceptions occurring while performing
the rescoring, rather than throwing them.
"""
event_info = {'state': self.lcp.get_state(), 'problem_id': self.location.url()}
event_info = {'state': self.lcp.get_state(), 'problem_id': self.location.to_deprecated_string()}
_ = self.runtime.service(self, "i18n").ugettext
@@ -1293,7 +1293,7 @@ class CapaMixin(CapaFields):
"""
event_info = dict()
event_info['state'] = self.lcp.get_state()
event_info['problem_id'] = self.location.url()
event_info['problem_id'] = self.location.to_deprecated_string()
answers = self.make_dict_of_responses(data)
event_info['answers'] = answers
@@ -1346,7 +1346,7 @@ class CapaMixin(CapaFields):
"""
event_info = dict()
event_info['old_state'] = self.lcp.get_state()
event_info['problem_id'] = self.location.url()
event_info['problem_id'] = self.location.to_deprecated_string()
_ = self.runtime.service(self, "i18n").ugettext
if self.closed():

28
common/lib/xmodule/xmodule/conditional_module.py
View File

@@ -9,7 +9,6 @@ from lxml import etree
from pkg_resources import resource_string
from xmodule.x_module import XModule
from xmodule.modulestore import Location
from xmodule.seq_module import SequenceDescriptor
from xblock.fields import Scope, ReferenceList
from xmodule.modulestore.exceptions import ItemNotFoundError
@@ -144,7 +143,6 @@ class ConditionalModule(ConditionalFields, XModule):
return self.system.render_template('conditional_ajax.html', {
'element_id': self.location.html_id(),
'id': self.id,
'ajax_url': self.system.ajax_url,
'depends': ';'.join(self.required_html_ids)
})
@@ -199,20 +197,14 @@ class ConditionalDescriptor(ConditionalFields, SequenceDescriptor):
# substitution can be done.
if not self.sources_list:
if 'sources' in self.xml_attributes and isinstance(self.xml_attributes['sources'], basestring):
sources = ConditionalDescriptor.parse_sources(self.xml_attributes)
self.sources_list = sources
self.sources_list = ConditionalDescriptor.parse_sources(self.xml_attributes)
@staticmethod
def parse_sources(xml_element):
""" Parse xml_element 'sources' attr and return a list of location strings. """
result = []
sources = xml_element.get('sources')
if sources:
locations = [location.strip() for location in sources.split(';')]
for location in locations:
if Location.is_valid(location): # Check valid location url.
result.append(location)
return result
return [location.strip() for location in sources.split(';')]
def get_required_module_descriptors(self):
"""Returns a list of XModuleDescriptor instances upon
@@ -221,7 +213,7 @@ class ConditionalDescriptor(ConditionalFields, SequenceDescriptor):
descriptors = []
for location in self.sources_list:
try:
descriptor = self.system.load_item(Location(location))
descriptor = self.system.load_item(location)
descriptors.append(descriptor)
except ItemNotFoundError:
msg = "Invalid module by location."
@@ -238,7 +230,7 @@ class ConditionalDescriptor(ConditionalFields, SequenceDescriptor):
if child.tag == 'show':
locations = ConditionalDescriptor.parse_sources(child)
for location in locations:
children.append(Location(location))
children.append(location)
show_tag_list.append(location)
else:
try:
@@ -251,22 +243,18 @@ class ConditionalDescriptor(ConditionalFields, SequenceDescriptor):
return {'show_tag_list': show_tag_list}, children
def definition_to_xml(self, resource_fs):
def to_string(string_list):
""" Convert List of strings to a single string with "; " as the separator. """
return "; ".join(string_list)
xml_object = etree.Element(self._tag_name)
for child in self.get_children():
location = str(child.location)
if location not in self.show_tag_list:
if child.location not in self.show_tag_list:
self.runtime.add_block_as_child_node(child, xml_object)
if self.show_tag_list:
show_str = u'<{tag_name} sources="{sources}" />'.format(
tag_name='show', sources=to_string(self.show_tag_list))
tag_name='show', sources=';'.join(location.to_deprecated_string() for location in self.show_tag_list))
xml_object.append(etree.fromstring(show_str))
# Overwrite the original sources attribute with the value from sources_list, as
# Locations may have been changed to Locators.
self.xml_attributes['sources'] = to_string(self.sources_list)
stringified_sources_list = map(lambda loc: loc.to_deprecated_string(), self.sources_list)
self.xml_attributes['sources'] = ';'.join(stringified_sources_list)
return xml_object

105
common/lib/xmodule/xmodule/contentstore/content.py
View File

@@ -1,3 +1,5 @@
import bson.son
import re
XASSET_LOCATION_TAG = 'c4x'
XASSET_SRCREF_PREFIX = 'xasset:'
@@ -8,7 +10,7 @@ import logging
import StringIO
from urlparse import urlparse, urlunparse
from xmodule.modulestore import Location
from xmodule.modulestore.locations import AssetLocation, SlashSeparatedCourseKey
from .django import contentstore
from PIL import Image
@@ -22,7 +24,7 @@ class StaticContent(object):
self._data = data
self.length = length
self.last_modified_at = last_modified_at
self.thumbnail_location = Location(thumbnail_location) if thumbnail_location is not None else None
self.thumbnail_location = thumbnail_location
# optional information about where this file was imported from. This is needed to support import/export
# cycles
self.import_path = import_path
@@ -39,44 +41,48 @@ class StaticContent(object):
extension=XASSET_THUMBNAIL_TAIL_NAME,)
@staticmethod
def compute_location(org, course, name, revision=None, is_thumbnail=False):
name = name.replace('/', '_')
return Location([XASSET_LOCATION_TAG, org, course, 'asset' if not is_thumbnail else 'thumbnail',
Location.clean_keeping_underscores(name), revision])
def compute_location(course_key, path, revision=None, is_thumbnail=False):
"""
Constructs a location object for static content.
- course_key: the course that this asset belongs to
- path: is the name of the static asset
- revision: is the object's revision information
- is_tumbnail: is whether or not we want the thumbnail version of this
asset
"""
path = path.replace('/', '_')
return AssetLocation(
course_key.org, course_key.course, course_key.run,
'asset' if not is_thumbnail else 'thumbnail',
AssetLocation.clean_keeping_underscores(path),
revision
)
def get_id(self):
return StaticContent.get_id_from_location(self.location)
def get_url_path(self):
return StaticContent.get_url_path_from_location(self.location)
return self.location.to_deprecated_string()
@property
def data(self):
return self._data
@staticmethod
def get_url_path_from_location(location):
if location is not None:
return u"/{tag}/{org}/{course}/{category}/{name}".format(**location.dict())
else:
return None
ASSET_URL_RE = re.compile(r"""
/?c4x/
(?P<org>[^/]+)/
(?P<course>[^/]+)/
(?P<category>[^/]+)/
(?P<name>[^/]+)
""", re.VERBOSE | re.IGNORECASE)
@staticmethod
def is_c4x_path(path_string):
"""
Returns a boolean if a path is believed to be a c4x link based on the leading element
"""
return path_string.startswith(u'/{0}/'.format(XASSET_LOCATION_TAG))
@staticmethod
def renamespace_c4x_path(path_string, target_location):
"""
Returns an updated string which incorporates a new org/course in order to remap an asset path
to a new namespace
"""
location = StaticContent.get_location_from_path(path_string)
location = location.replace(org=target_location.org, course=target_location.course)
return StaticContent.get_url_path_from_location(location)
return StaticContent.ASSET_URL_RE.match(path_string) is not None
@staticmethod
def get_static_path_from_location(location):
@@ -88,28 +94,35 @@ class StaticContent(object):
the actual /c4x/... path which the client needs to reference static content
"""
if location is not None:
return u"/static/{name}".format(**location.dict())
return u"/static/{name}".format(name=location.name)
else:
return None
@staticmethod
def get_base_url_path_for_course_assets(loc):
if loc is not None:
return u"/c4x/{org}/{course}/asset".format(**loc.dict())
def get_base_url_path_for_course_assets(course_key):
if course_key is None:
return None
assert(isinstance(course_key, SlashSeparatedCourseKey))
return course_key.make_asset_key('asset', '').to_deprecated_string()
@staticmethod
def get_id_from_location(location):
return {'tag': location.tag, 'org': location.org, 'course': location.course,
'category': location.category, 'name': location.name,
'revision': location.revision}
"""
Get the doc store's primary key repr for this location
"""
return bson.son.SON([
('tag', 'c4x'), ('org', location.org), ('course', location.course),
('category', location.category), ('name', location.name),
('revision', location.revision),
])
@staticmethod
def get_location_from_path(path):
# remove leading / character if it is there one
if path.startswith('/'):
path = path[1:]
return Location(path.split('/'))
"""
Generate an AssetKey for the given path (old c4x/org/course/asset/name syntax)
"""
return AssetLocation.from_deprecated_string(path)
@staticmethod
def convert_legacy_static_url_with_course_id(path, course_id):
@@ -117,12 +130,10 @@ class StaticContent(object):
Returns a path to a piece of static content when we are provided with a filepath and
a course_id
"""
# Generate url of urlparse.path component
scheme, netloc, orig_path, params, query, fragment = urlparse(path)
course_id_dict = Location.parse_course_id(course_id)
loc = StaticContent.compute_location(course_id_dict['org'], course_id_dict['course'], orig_path)
loc_url = StaticContent.get_url_path_from_location(loc)
loc = StaticContent.compute_location(course_id, orig_path)
loc_url = loc.to_deprecated_string()
# Reconstruct with new path
return urlunparse((scheme, netloc, loc_url, params, query, fragment))
@@ -167,7 +178,7 @@ class ContentStore(object):
def find(self, filename):
raise NotImplementedError
def get_all_content_for_course(self, location, start=0, maxresults=-1, sort=None):
def get_all_content_for_course(self, course_key, start=0, maxresults=-1, sort=None):
'''
Returns a list of static assets for a course, followed by the total number of assets.
By default all assets are returned, but start and maxresults can be provided to limit the query.
@@ -192,13 +203,21 @@ class ContentStore(object):
'''
raise NotImplementedError
def delete_all_course_assets(self, course_key):
"""
Delete all of the assets which use this course_key as an identifier
:param course_key:
"""
raise NotImplementedError
def generate_thumbnail(self, content, tempfile_path=None):
thumbnail_content = None
# use a naming convention to associate originals with the thumbnail
thumbnail_name = StaticContent.generate_thumbnail_name(content.location.name)
thumbnail_file_location = StaticContent.compute_location(content.location.org, content.location.course,
thumbnail_name, is_thumbnail=True)
thumbnail_file_location = StaticContent.compute_location(
content.location.course_key, thumbnail_name, is_thumbnail=True
)
# if we're uploading an image, then let's generate a thumbnail so that we can
# serve it up when needed without having to rescale on the fly

103
common/lib/xmodule/xmodule/contentstore/mongo.py
View File

@@ -2,8 +2,7 @@ import pymongo
import gridfs
from gridfs.errors import NoFile
from xmodule.modulestore import Location
from xmodule.modulestore.mongo.base import location_to_query
from xmodule.modulestore.mongo.base import location_to_query, MongoModuleStore, location_to_son
from xmodule.contentstore.content import XASSET_LOCATION_TAG
import logging
@@ -13,6 +12,8 @@ from xmodule.exceptions import NotFoundError
from fs.osfs import OSFS
import os
import json
import bson.son
from xmodule.modulestore.locations import AssetLocation
class MongoContentStore(ContentStore):
@@ -28,6 +29,7 @@ class MongoContentStore(ContentStore):
pymongo.MongoClient(
host=host,
port=port,
document_class=bson.son.SON,
**kwargs
),
db
@@ -46,8 +48,10 @@ class MongoContentStore(ContentStore):
# Seems like with the GridFS we can't update existing ID's we have to do a delete/add pair
self.delete(content_id)
thumbnail_location = content.thumbnail_location.to_deprecated_list_repr() if content.thumbnail_location else None
with self.fs.new_file(_id=content_id, filename=content.get_url_path(), content_type=content.content_type,
displayname=content.name, thumbnail_location=content.thumbnail_location,
displayname=content.name,
thumbnail_location=thumbnail_location,
import_path=content.import_path,
# getattr b/c caching may mean some pickled instances don't have attr
locked=getattr(content, 'locked', False)) as fp:
@@ -62,23 +66,40 @@ class MongoContentStore(ContentStore):
def delete(self, content_id):
if self.fs.exists({"_id": content_id}):
self.fs.delete(content_id)
assert not self.fs.exists({"_id": content_id})
def find(self, location, throw_on_not_found=True, as_stream=False):
content_id = StaticContent.get_id_from_location(location)
# Use slow attr based lookup b/c we weren't careful to control the key order in _id before
content_id = {u'_id.{}'.format(key): value for key, value in content_id.iteritems()}
fs_pointer = self.fs_files.find_one(content_id, fields={'_id': 1})
if fs_pointer is None:
if throw_on_not_found:
raise NotFoundError()
else:
return None
content_id = fs_pointer['_id']
try:
if as_stream:
fp = self.fs.get(content_id)
thumbnail_location = getattr(fp, 'thumbnail_location', None)
if thumbnail_location:
thumbnail_location = location.course_key.make_asset_key('thumbnail', thumbnail_location[4])
return StaticContentStream(
location, fp.displayname, fp.content_type, fp, last_modified_at=fp.uploadDate,
thumbnail_location=getattr(fp, 'thumbnail_location', None),
thumbnail_location=thumbnail_location,
import_path=getattr(fp, 'import_path', None),
length=fp.length, locked=getattr(fp, 'locked', False)
)
else:
with self.fs.get(content_id) as fp:
thumbnail_location = getattr(fp, 'thumbnail_location', None)
if thumbnail_location:
thumbnail_location = location.course_key.make_asset_key('thumbnail', thumbnail_location[4])
return StaticContent(
location, fp.displayname, fp.content_type, fp.read(), last_modified_at=fp.uploadDate,
thumbnail_location=getattr(fp, 'thumbnail_location', None),
thumbnail_location=thumbnail_location,
import_path=getattr(fp, 'import_path', None),
length=fp.length, locked=getattr(fp, 'locked', False)
)
@@ -90,8 +111,12 @@ class MongoContentStore(ContentStore):
def get_stream(self, location):
content_id = StaticContent.get_id_from_location(location)
# use slow attr based lookup because we weren't careful to control the key order in _id before
content_id = {u'_id.{}'.format(key): value for key, value in content_id.iteritems()}
fs_pointer = self.fs_files.find_one(content_id, fields={'_id': 1})
try:
handle = self.fs.get(content_id)
handle = self.fs.get(fs_pointer['_id'])
except NoFile:
raise NotFoundError()
@@ -100,7 +125,7 @@ class MongoContentStore(ContentStore):
def close_stream(self, handle):
try:
handle.close()
except Exception:
except Exception: # pylint: disable=broad-except
pass
def export(self, location, output_directory):
@@ -117,21 +142,22 @@ class MongoContentStore(ContentStore):
with disk_fs.open(content.name, 'wb') as asset_file:
asset_file.write(content.data)
def export_all_for_course(self, course_location, output_directory, assets_policy_file):
def export_all_for_course(self, course_key, output_directory, assets_policy_file):
"""
Export all of this course's assets to the output_directory. Export all of the assets'
attributes to the policy file.
:param course_location: the Location of type 'course'
:param output_directory: the directory under which to put all the asset files
:param assets_policy_file: the filename for the policy file which should be in the same
directory as the other policy files.
Args:
course_key (CourseKey): the :class:`CourseKey` identifying the course
output_directory: the directory under which to put all the asset files
assets_policy_file: the filename for the policy file which should be in the same
directory as the other policy files.
"""
policy = {}
assets, __ = self.get_all_content_for_course(course_location)
assets, __ = self.get_all_content_for_course(course_key)
for asset in assets:
asset_location = Location(asset['_id'])
asset_location = AssetLocation._from_deprecated_son(asset['_id'], course_key.run) # pylint: disable=protected-access
self.export(asset_location, output_directory)
for attr, value in asset.iteritems():
if attr not in ['_id', 'md5', 'uploadDate', 'length', 'chunkSize']:
@@ -140,15 +166,15 @@ class MongoContentStore(ContentStore):
with open(assets_policy_file, 'w') as f:
json.dump(policy, f)
def get_all_content_thumbnails_for_course(self, location):
return self._get_all_content_for_course(location, get_thumbnails=True)[0]
def get_all_content_thumbnails_for_course(self, course_key):
return self._get_all_content_for_course(course_key, get_thumbnails=True)[0]
def get_all_content_for_course(self, location, start=0, maxresults=-1, sort=None):
def get_all_content_for_course(self, course_key, start=0, maxresults=-1, sort=None):
return self._get_all_content_for_course(
location, start=start, maxresults=maxresults, get_thumbnails=False, sort=sort
course_key, start=start, maxresults=maxresults, get_thumbnails=False, sort=sort
)
def _get_all_content_for_course(self, location, get_thumbnails=False, start=0, maxresults=-1, sort=None):
def _get_all_content_for_course(self, course_key, get_thumbnails=False, start=0, maxresults=-1, sort=None):
'''
Returns a list of all static assets for a course. The return format is a list of dictionary elements. Example:
@@ -168,20 +194,22 @@ class MongoContentStore(ContentStore):
]
'''
course_filter = Location(XASSET_LOCATION_TAG, category="asset" if not get_thumbnails else "thumbnail",
course=location.course, org=location.org)
course_filter = course_key.make_asset_key(
"asset" if not get_thumbnails else "thumbnail",
None
)
# 'borrow' the function 'location_to_query' from the Mongo modulestore implementation
if maxresults > 0:
items = self.fs_files.find(
location_to_query(course_filter),
location_to_query(course_filter, wildcard=True, tag=XASSET_LOCATION_TAG),
skip=start, limit=maxresults, sort=sort
)
else:
items = self.fs_files.find(location_to_query(course_filter), sort=sort)
items = self.fs_files.find(location_to_query(course_filter, wildcard=True, tag=XASSET_LOCATION_TAG), sort=sort)
count = items.count()
return list(items), count
def set_attr(self, location, attr, value=True):
def set_attr(self, asset_key, attr, value=True):
"""
Add/set the given attr on the asset at the given location. Does not allow overwriting gridFS built in
attrs such as _id, md5, uploadDate, length. Value can be any type which pymongo accepts.
@@ -191,11 +219,11 @@ class MongoContentStore(ContentStore):
Raises NotFoundError if no such item exists
Raises AttributeError is attr is one of the build in attrs.
:param location: a c4x asset location
:param asset_key: an AssetKey
:param attr: which attribute to set
:param value: the value to set it to (any type pymongo accepts such as datetime, number, string)
"""
self.set_attrs(location, {attr: value})
self.set_attrs(asset_key, {attr: value})
def get_attr(self, location, attr, default=None):
"""
@@ -216,15 +244,13 @@ class MongoContentStore(ContentStore):
:param location: a c4x asset location
"""
# raises exception if location is not fully specified
Location.ensure_fully_specified(location)
for attr in attr_dict.iterkeys():
if attr in ['_id', 'md5', 'uploadDate', 'length']:
raise AttributeError("{} is a protected attribute.".format(attr))
item = self.fs_files.find_one(location_to_query(location))
if item is None:
raise NotFoundError()
self.fs_files.update({"_id": item["_id"]}, {"$set": attr_dict})
asset_db_key = {'_id': location_to_son(location, tag=XASSET_LOCATION_TAG)}
if self.fs_files.find(asset_db_key).count() == 0:
raise NotFoundError(asset_db_key)
self.fs_files.update(asset_db_key, {"$set": attr_dict})
def get_attrs(self, location):
"""
@@ -236,7 +262,18 @@ class MongoContentStore(ContentStore):
:param location: a c4x asset location
"""
item = self.fs_files.find_one(location_to_query(location))
item = self.fs_files.find_one({'_id': location_to_son(location, tag=XASSET_LOCATION_TAG)})
if item is None:
raise NotFoundError()
return item
def delete_all_course_assets(self, course_key):
"""
Delete all assets identified via this course_key. Dangerous operation which may remove assets
referenced by other runs or other courses.
:param course_key:
"""
course_query = MongoModuleStore._course_key_to_son(course_key, tag=XASSET_LOCATION_TAG) # pylint: disable=protected-access
matching_assets = self.fs_files.find(course_query)
for asset in matching_assets:
self.fs.delete(asset['_id'])

13
common/lib/xmodule/xmodule/contentstore/utils.py
View File

@@ -1,4 +1,3 @@
from xmodule.modulestore import Location
from xmodule.contentstore.content import StaticContent
from .django import contentstore
@@ -13,18 +12,14 @@ def empty_asset_trashcan(course_locs):
# first delete all of the thumbnails
thumbs = store.get_all_content_thumbnails_for_course(course_loc)
for thumb in thumbs:
thumb_loc = Location(thumb["_id"])
id = StaticContent.get_id_from_location(thumb_loc)
print "Deleting {0}...".format(id)
store.delete(id)
print "Deleting {0}...".format(thumb)
store.delete(thumb['_id'])
# then delete all of the assets
assets, __ = store.get_all_content_for_course(course_loc)
for asset in assets:
asset_loc = Location(asset["_id"])
id = StaticContent.get_id_from_location(asset_loc)
print "Deleting {0}...".format(id)
store.delete(id)
print "Deleting {0}...".format(asset)
store.delete(asset['_id'])
def restore_asset_from_trashcan(location):

26
common/lib/xmodule/xmodule/course_module.py
View File

@@ -438,7 +438,7 @@ class CourseDescriptor(CourseFields, SequenceDescriptor):
if isinstance(self.location, Location):
self.wiki_slug = self.location.course
elif isinstance(self.location, CourseLocator):
self.wiki_slug = self.location.package_id or self.display_name
self.wiki_slug = self.id.offering or self.display_name
if self.due_date_display_format is None and self.show_timezone is False:
# For existing courses with show_timezone set to False (and no due_date_display_format specified),
@@ -810,32 +810,10 @@ class CourseDescriptor(CourseFields, SequenceDescriptor):
def make_id(org, course, url_name):
return '/'.join([org, course, url_name])
@staticmethod
def id_to_location(course_id):
'''Convert the given course_id (org/course/name) to a location object.
Throws ValueError if course_id is of the wrong format.
'''
course_id_dict = Location.parse_course_id(course_id)
course_id_dict['tag'] = 'i4x'
course_id_dict['category'] = 'course'
return Location(course_id_dict)
@staticmethod
def location_to_id(location):
'''Convert a location of a course to a course_id. If location category
is not "course", raise a ValueError.
location: something that can be passed to Location
'''
loc = Location(location)
if loc.category != "course":
raise ValueError("{0} is not a course location".format(loc))
return "/".join([loc.org, loc.course, loc.name])
@property
def id(self):
"""Return the course_id for this course"""
return self.location_to_id(self.location)
return self.location.course_key
@property
def start_date_text(self):

2
common/lib/xmodule/xmodule/error_module.py
View File

@@ -11,7 +11,6 @@ import sys
from lxml import etree
from xmodule.x_module import XModule, XModuleDescriptor
from xmodule.errortracker import exc_info_to_str
from xmodule.modulestore import Location
from xblock.fields import String, Scope, ScopeIds
from xblock.field_data import DictFieldData
@@ -81,7 +80,6 @@ class ErrorDescriptor(ErrorFields, XModuleDescriptor):
@classmethod
def _construct(cls, system, contents, error_msg, location):
location = Location(location)
if error_msg is None:
# this string is not marked for translation because we don't have

2
common/lib/xmodule/xmodule/foldit_module.py
View File

@@ -108,7 +108,7 @@ class FolditModule(FolditFields, XModule):
from foldit.models import Score
if courses is None:
courses = [self.location.course_id]
courses = [self.location.course_key]
leaders = [(leader['username'], leader['score']) for leader in Score.get_tops_n(10, course_list=courses)]
leaders.sort(key=lambda x: -x[1])

2
common/lib/xmodule/xmodule/html_module.py
View File

@@ -121,7 +121,7 @@ class HtmlDescriptor(HtmlFields, XmlDescriptor, EditingDescriptor):
# Add some specific HTML rendering context when editing HTML modules where we pass
# the root /c4x/ url for assets. This allows client-side substitutions to occur.
_context.update({
'base_asset_url': StaticContent.get_base_url_path_for_course_assets(self.location) + '/',
'base_asset_url': StaticContent.get_base_url_path_for_course_assets(self.location.course_key),
'enable_latex_compiler': self.use_latex_compiler,
'editor': self.editor
})

6
common/lib/xmodule/xmodule/lti_module.py
View File

@@ -347,9 +347,7 @@ class LTIModule(LTIFields, XModule):
"""
Return course by course id.
"""
course_location = CourseDescriptor.id_to_location(self.course_id)
course = self.descriptor.runtime.modulestore.get_item(course_location)
return course
return self.descriptor.runtime.modulestore.get_course(self.course_id)
@property
def context_id(self):
@@ -359,7 +357,7 @@ class LTIModule(LTIFields, XModule):
context_id is an opaque identifier that uniquely identifies the context (e.g., a course)
that contains the link being launched.
"""
return self.course_id
return self.course_id.to_deprecated_string()
@property
def role(self):

542
common/lib/xmodule/xmodule/modulestore/__init__.py
View File

@@ -6,7 +6,7 @@ that are stored in a database an accessible using their Location as an identifie
import logging
import re
from collections import namedtuple
from collections import namedtuple, defaultdict
import collections
from abc import ABCMeta, abstractmethod
@@ -14,8 +14,13 @@ from xblock.plugin import default_select
from .exceptions import InvalidLocationError, InsufficientSpecificationError
from xmodule.errortracker import make_error_tracker
from xmodule.modulestore.keys import CourseKey, UsageKey
from xmodule.modulestore.locations import Location # For import backwards compatibility
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from xblock.runtime import Mixologist
from xblock.core import XBlock
import datetime
log = logging.getLogger('edx.modulestore')
@@ -23,286 +28,6 @@ SPLIT_MONGO_MODULESTORE_TYPE = 'split'
MONGO_MODULESTORE_TYPE = 'mongo'
XML_MODULESTORE_TYPE = 'xml'
URL_RE = re.compile("""
(?P<tag>[^:]+)://?
(?P<org>[^/]+)/
(?P<course>[^/]+)/
(?P<category>[^/]+)/
(?P<name>[^@]+)
(@(?P<revision>[^/]+))?
""", re.VERBOSE)
# TODO (cpennington): We should decide whether we want to expand the
# list of valid characters in a location
INVALID_CHARS = re.compile(r"[^\w.%-]", re.UNICODE)
# Names are allowed to have colons.
INVALID_CHARS_NAME = re.compile(r"[^\w.:%-]", re.UNICODE)
# html ids can contain word chars and dashes
INVALID_HTML_CHARS = re.compile(r"[^\w-]", re.UNICODE)
_LocationBase = namedtuple('LocationBase', 'tag org course category name revision')
def _check_location_part(val, regexp):
"""
Check that `regexp` doesn't match inside `val`. If it does, raise an exception
Args:
val (string): The value to check
regexp (re.RegexObject): The regular expression specifying invalid characters
Raises:
InvalidLocationError: Raised if any invalid character is found in `val`
"""
if val is not None and regexp.search(val) is not None:
raise InvalidLocationError("Invalid characters in {!r}.".format(val))
class Location(_LocationBase):
'''
Encodes a location.
Locations representations of URLs of the
form {tag}://{org}/{course}/{category}/{name}[@{revision}]
However, they can also be represented as dictionaries (specifying each component),
tuples or lists (specified in order), or as strings of the url
'''
__slots__ = ()
@staticmethod
def _clean(value, invalid):
"""
invalid should be a compiled regexp of chars to replace with '_'
"""
return re.sub('_+', '_', invalid.sub('_', value))
@staticmethod
def clean(value):
"""
Return value, made into a form legal for locations
"""
return Location._clean(value, INVALID_CHARS)
@staticmethod
def clean_keeping_underscores(value):
"""
Return value, replacing INVALID_CHARS, but not collapsing multiple '_' chars.
This for cleaning asset names, as the YouTube ID's may have underscores in them, and we need the
transcript asset name to match. In the future we may want to change the behavior of _clean.
"""
return INVALID_CHARS.sub('_', value)
@staticmethod
def clean_for_url_name(value):
"""
Convert value into a format valid for location names (allows colons).
"""
return Location._clean(value, INVALID_CHARS_NAME)
@staticmethod
def clean_for_html(value):
"""
Convert a string into a form that's safe for use in html ids, classes, urls, etc.
Replaces all INVALID_HTML_CHARS with '_', collapses multiple '_' chars
"""
return Location._clean(value, INVALID_HTML_CHARS)
@staticmethod
def is_valid(value):
'''
Check if the value is a valid location, in any acceptable format.
'''
try:
Location(value)
except InvalidLocationError:
return False
return True
@staticmethod
def ensure_fully_specified(location):
'''Make sure location is valid, and fully specified. Raises
InvalidLocationError or InsufficientSpecificationError if not.
returns a Location object corresponding to location.
'''
loc = Location(location)
for key, val in loc.dict().iteritems():
if key != 'revision' and val is None:
raise InsufficientSpecificationError(location)
return loc
def __new__(_cls, loc_or_tag=None, org=None, course=None, category=None,
name=None, revision=None):
"""
Create a new location that is a clone of the specifed one.
location - Can be any of the following types:
string: should be of the form
{tag}://{org}/{course}/{category}/{name}[@{revision}]
list: should be of the form [tag, org, course, category, name, revision]
dict: should be of the form {
'tag': tag,
'org': org,
'course': course,
'category': category,
'name': name,
'revision': revision,
}
Location: another Location object
In both the dict and list forms, the revision is optional, and can be
ommitted.
Components must be composed of alphanumeric characters, or the
characters '_', '-', and '.'. The name component is additionally allowed to have ':',
which is interpreted specially for xml storage.
Components may be set to None, which may be interpreted in some contexts
to mean wildcard selection.
"""
if (org is None and course is None and category is None and name is None and revision is None):
location = loc_or_tag
else:
location = (loc_or_tag, org, course, category, name, revision)
if location is None:
return _LocationBase.__new__(_cls, *([None] * 6))
def check_dict(dict_):
# Order matters, so flatten out into a list
keys = ['tag', 'org', 'course', 'category', 'name', 'revision']
list_ = [dict_[k] for k in keys]
check_list(list_)
def check_list(list_):
list_ = list(list_)
for val in list_[:4] + [list_[5]]:
_check_location_part(val, INVALID_CHARS)
# names allow colons
_check_location_part(list_[4], INVALID_CHARS_NAME)
if isinstance(location, Location):
return location
elif isinstance(location, basestring):
match = URL_RE.match(location)
if match is None:
log.debug(u"location %r doesn't match URL", location)
raise InvalidLocationError(location)
groups = match.groupdict()
check_dict(groups)
return _LocationBase.__new__(_cls, **groups)
elif isinstance(location, (list, tuple)):
if len(location) not in (5, 6):
log.debug(u'location has wrong length')
raise InvalidLocationError(location)
if len(location) == 5:
args = tuple(location) + (None,)
else:
args = tuple(location)
check_list(args)
return _LocationBase.__new__(_cls, *args)
elif isinstance(location, dict):
kwargs = dict(location)
kwargs.setdefault('revision', None)
check_dict(kwargs)
return _LocationBase.__new__(_cls, **kwargs)
else:
raise InvalidLocationError(location)
def url(self):
"""
Return a string containing the URL for this location
"""
url = u"{0.tag}://{0.org}/{0.course}/{0.category}/{0.name}".format(self)
if self.revision:
url += u"@{rev}".format(rev=self.revision) # pylint: disable=E1101
return url
def html_id(self):
"""
Return a string with a version of the location that is safe for use in
html id attributes
"""
id_string = u"-".join(v for v in self.list() if v is not None)
return Location.clean_for_html(id_string)
def dict(self):
"""
Return an OrderedDict of this locations keys and values. The order is
tag, org, course, category, name, revision
"""
return self._asdict()
def list(self):
return list(self)
def __str__(self):
return str(self.url().encode("utf-8"))
def __unicode__(self):
return self.url()
def __repr__(self):
return "Location%s" % repr(tuple(self))
@property
def course_id(self):
"""
Return the ID of the Course that this item belongs to by looking
at the location URL hierachy.
Throws an InvalidLocationError is this location does not represent a course.
"""
if self.category != 'course':
raise InvalidLocationError(u'Cannot call course_id for {0} because it is not of category course'.format(self))
return "/".join([self.org, self.course, self.name])
COURSE_ID_RE = re.compile("""
(?P<org>[^/]+)/
(?P<course>[^/]+)/
(?P<name>.*)
""", re.VERBOSE)
@staticmethod
def parse_course_id(course_id):
"""
Given a org/course/name course_id, return a dict of {"org": org, "course": course, "name": name}
If the course_id is not of the right format, raise ValueError
"""
match = Location.COURSE_ID_RE.match(course_id)
if match is None:
raise ValueError("{} is not of form ORG/COURSE/NAME".format(course_id))
return match.groupdict()
def _replace(self, **kwargs):
"""
Return a new :class:`Location` with values replaced
by the values specified in `**kwargs`
"""
for name, value in kwargs.iteritems():
if name == 'name':
_check_location_part(value, INVALID_CHARS_NAME)
else:
_check_location_part(value, INVALID_CHARS)
# namedtuple is an old-style class, so don't use super
return _LocationBase._replace(self, **kwargs)
def replace(self, **kwargs):
'''
Expose a public method for replacing location elements
'''
return self._replace(**kwargs)
class ModuleStoreRead(object):
"""
@@ -313,14 +38,14 @@ class ModuleStoreRead(object):
__metaclass__ = ABCMeta
@abstractmethod
def has_item(self, course_id, location):
def has_item(self, usage_key):
"""
Returns True if location exists in this ModuleStore.
Returns True if usage_key exists in this ModuleStore.
"""
pass
@abstractmethod
def get_item(self, location, depth=0):
def get_item(self, usage_key, depth=0):
"""
Returns an XModuleDescriptor instance for the item at location.
@@ -330,7 +55,7 @@ class ModuleStoreRead(object):
If no object is found at that location, raises
xmodule.modulestore.exceptions.ItemNotFoundError
location: Something that can be passed to Location
usage_key: A :class:`.UsageKey` subclass instance
depth (int): An argument that some module stores may use to prefetch
descendents of the queried modules for more efficient results later
@@ -340,23 +65,16 @@ class ModuleStoreRead(object):
pass
@abstractmethod
def get_instance(self, course_id, location, depth=0):
"""
Get an instance of this location, with policy for course_id applied.
TODO (vshnayder): this may want to live outside the modulestore eventually
"""
pass
@abstractmethod
def get_item_errors(self, location):
def get_course_errors(self, course_key):
"""
Return a list of (msg, exception-or-None) errors that the modulestore
encountered when loading the item at location.
location : something that can be passed to Location
encountered when loading the course at course_id.
Raises the same exceptions as get_item if the location isn't found or
isn't fully specified.
Args:
course_key (:class:`.CourseKey`): The course to check for errors
"""
pass
@@ -376,6 +94,68 @@ class ModuleStoreRead(object):
"""
pass
def _block_matches(self, fields_or_xblock, qualifiers):
'''
Return True or False depending on whether the field value (block contents)
matches the qualifiers as per get_items. Note, only finds directly set not
inherited nor default value matches.
For substring matching pass a regex object.
for arbitrary function comparison such as date time comparison, pass
the function as in start=lambda x: x < datetime.datetime(2014, 1, 1, 0, tzinfo=pytz.UTC)
Args:
fields_or_xblock (dict or XBlock): either the json blob (from the db or get_explicitly_set_fields)
or the xblock.fields() value or the XBlock from which to get those values
qualifiers (dict): field: searchvalue pairs.
'''
if isinstance(fields_or_xblock, XBlock):
fields = fields_or_xblock.fields
xblock = fields_or_xblock
is_xblock = True
else:
fields = fields_or_xblock
is_xblock = False
def _is_set_on(key):
"""
Is this key set in fields? (return tuple of boolean and value). A helper which can
handle fields either being the json doc or xblock fields. Is inner function to restrict
use and to access local vars.
"""
if key not in fields:
return False, None
field = fields[key]
if is_xblock:
return field.is_set_on(fields_or_xblock), getattr(xblock, key)
else:
return True, field
for key, criteria in qualifiers.iteritems():
is_set, value = _is_set_on(key)
if not is_set:
return False
if not self._value_matches(value, criteria):
return False
return True
def _value_matches(self, target, criteria):
'''
helper for _block_matches: does the target (field value) match the criteria?
If target is a list, do any of the list elements meet the criteria
If the criteria is a regex, does the target match it?
If the criteria is a function, does invoking it on the target yield something truthy?
Otherwise, is the target == criteria
'''
if isinstance(target, list):
return any(self._value_matches(ele, criteria) for ele in target)
elif isinstance(criteria, re._pattern_type):
return criteria.search(target) is not None
elif callable(criteria):
return criteria(target)
else:
return criteria == target
@abstractmethod
def get_courses(self):
'''
@@ -385,14 +165,26 @@ class ModuleStoreRead(object):
pass
@abstractmethod
def get_course(self, course_id):
def get_course(self, course_id, depth=None):
'''
Look for a specific course id. Returns the course descriptor, or None if not found.
Look for a specific course by its id (:class:`CourseKey`).
Returns the course descriptor, or None if not found.
'''
pass
@abstractmethod
def get_parent_locations(self, location, course_id):
def has_course(self, course_id, ignore_case=False):
'''
Look for a specific course id. Returns whether it exists.
Args:
course_id (CourseKey):
ignore_case (boolean): some modulestores are case-insensitive. Use this flag
to search for whether a potentially conflicting course exists in that case.
'''
pass
@abstractmethod
def get_parent_locations(self, location):
'''Find all locations that are the parents of this location in this
course. Needed for path_to_location().
@@ -401,7 +193,7 @@ class ModuleStoreRead(object):
pass
@abstractmethod
def get_orphans(self, course_location, branch):
def get_orphans(self, course_key):
"""
Get all of the xblocks in the given course which have no parents and are not of types which are
usually orphaned. NOTE: may include xblocks which still have references via xblocks which don't
@@ -445,7 +237,7 @@ class ModuleStoreWrite(ModuleStoreRead):
:param force: fork the structure and don't update the course draftVersion if there's a version
conflict (only applicable to version tracking and conflict detecting persistence stores)
:raises VersionConflictError: if package_id and version_guid given and the current
:raises VersionConflictError: if org, offering, and version_guid given and the current
version head != version_guid and force is not True. (only applicable to version tracking stores)
"""
pass
@@ -461,11 +253,39 @@ class ModuleStoreWrite(ModuleStoreRead):
:param force: fork the structure and don't update the course draftVersion if there's a version
conflict (only applicable to version tracking and conflict detecting persistence stores)
:raises VersionConflictError: if package_id and version_guid given and the current
:raises VersionConflictError: if org, offering, and version_guid given and the current
version head != version_guid and force is not True. (only applicable to version tracking stores)
"""
pass
@abstractmethod
def create_course(self, org, offering, user_id=None, fields=None, **kwargs):
"""
Creates and returns the course.
Args:
org (str): the organization that owns the course
offering (str): the name of the course offering
user_id: id of the user creating the course
fields (dict): Fields to set on the course at initialization
kwargs: Any optional arguments understood by a subset of modulestores to customize instantiation
Returns: a CourseDescriptor
"""
pass
@abstractmethod
def delete_course(self, course_key, user_id=None):
"""
Deletes the course. It may be a soft or hard delete. It may or may not remove the xblock definitions
depending on the persistence layer and how tightly bound the xblocks are to the course.
Args:
course_key (CourseKey): which course to delete
user_id: id of the user deleting the course
"""
pass
class ModuleStoreReadBase(ModuleStoreRead):
'''
@@ -477,7 +297,7 @@ class ModuleStoreReadBase(ModuleStoreRead):
self,
doc_store_config=None, # ignore if passed up
metadata_inheritance_cache_subsystem=None, request_cache=None,
modulestore_update_signal=None, xblock_mixins=(), xblock_select=None,
xblock_mixins=(), xblock_select=None,
# temporary parms to enable backward compatibility. remove once all envs migrated
db=None, collection=None, host=None, port=None, tz_aware=True, user=None, password=None,
# allow lower level init args to pass harmlessly
@@ -486,38 +306,22 @@ class ModuleStoreReadBase(ModuleStoreRead):
'''
Set up the error-tracking logic.
'''
self._location_errors = {} # location -> ErrorLog
self._course_errors = defaultdict(make_error_tracker) # location -> ErrorLog
self.metadata_inheritance_cache_subsystem = metadata_inheritance_cache_subsystem
self.modulestore_update_signal = modulestore_update_signal
self.request_cache = request_cache
self.xblock_mixins = xblock_mixins
self.xblock_select = xblock_select
def _get_errorlog(self, location):
def get_course_errors(self, course_key):
"""
If we already have an errorlog for this location, return it. Otherwise,
create one.
"""
location = Location(location)
if location not in self._location_errors:
self._location_errors[location] = make_error_tracker()
return self._location_errors[location]
def get_item_errors(self, location):
"""
Return list of errors for this location, if any. Raise the same
errors as get_item if location isn't present.
NOTE: For now, the only items that track errors are CourseDescriptors in
the xml datastore. This will return an empty list for all other items
and datastores.
Return list of errors for this :class:`.CourseKey`, if any. Raise the same
errors as get_item if course_key isn't present.
"""
# check that item is present and raise the promised exceptions if needed
# TODO (vshnayder): post-launch, make errors properties of items
# self.get_item(location)
errorlog = self._get_errorlog(location)
return errorlog.errors
assert(isinstance(course_key, CourseKey))
return self._course_errors[course_key].errors
def get_errored_courses(self):
"""
@@ -528,42 +332,36 @@ class ModuleStoreReadBase(ModuleStoreRead):
"""
return {}
def get_course(self, course_id):
"""Default impl--linear search through course list"""
for c in self.get_courses():
if c.id == course_id:
return c
def get_course(self, course_id, depth=None):
"""
See ModuleStoreRead.get_course
Default impl--linear search through course list
"""
assert(isinstance(course_id, CourseKey))
for course in self.get_courses():
if course.id == course_id:
return course
return None
def update_item(self, xblock, user_id=None, allow_not_found=False, force=False):
def has_course(self, course_id, ignore_case=False):
"""
Update the given xblock's persisted repr. Pass the user's unique id which the persistent store
should save with the update if it has that ability.
:param allow_not_found: whether this method should raise an exception if the given xblock
has not been persisted before.
:param force: fork the structure and don't update the course draftVersion if there's a version
conflict (only applicable to version tracking and conflict detecting persistence stores)
:raises VersionConflictError: if package_id and version_guid given and the current
version head != version_guid and force is not True. (only applicable to version tracking stores)
Look for a specific course id. Returns whether it exists.
Args:
course_id (CourseKey):
ignore_case (boolean): some modulestores are case-insensitive. Use this flag
to search for whether a potentially conflicting course exists in that case.
"""
raise NotImplementedError
# linear search through list
assert(isinstance(course_id, CourseKey))
if ignore_case:
return any(
(c.id.org.lower() == course_id.org.lower() and c.id.offering.lower() == course_id.offering.lower())
for c in self.get_courses()
)
else:
return any(c.id == course_id for c in self.get_courses())
def delete_item(self, location, user_id=None, delete_all_versions=False, delete_children=False, force=False):
"""
Delete an item from persistence. Pass the user's unique id which the persistent store
should save with the update if it has that ability.
:param delete_all_versions: removes both the draft and published version of this item from
the course if using draft and old mongo. Split may or may not implement this.
:param force: fork the structure and don't update the course draftVersion if there's a version
conflict (only applicable to version tracking and conflict detecting persistence stores)
:raises VersionConflictError: if package_id and version_guid given and the current
version head != version_guid and force is not True. (only applicable to version tracking stores)
"""
raise NotImplementedError
class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
'''
@@ -592,6 +390,36 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
result[field.scope][field_name] = value
return result
def update_item(self, xblock, user_id=None, allow_not_found=False, force=False):
"""
Update the given xblock's persisted repr. Pass the user's unique id which the persistent store
should save with the update if it has that ability.
:param allow_not_found: whether this method should raise an exception if the given xblock
has not been persisted before.
:param force: fork the structure and don't update the course draftVersion if there's a version
conflict (only applicable to version tracking and conflict detecting persistence stores)
:raises VersionConflictError: if org, offering, and version_guid given and the current
version head != version_guid and force is not True. (only applicable to version tracking stores)
"""
raise NotImplementedError
def delete_item(self, location, user_id=None, delete_all_versions=False, delete_children=False, force=False):
"""
Delete an item from persistence. Pass the user's unique id which the persistent store
should save with the update if it has that ability.
:param delete_all_versions: removes both the draft and published version of this item from
the course if using draft and old mongo. Split may or may not implement this.
:param force: fork the structure and don't update the course draftVersion if there's a version
conflict (only applicable to version tracking and conflict detecting persistence stores)
:raises VersionConflictError: if org, offering, and version_guid given and the current
version head != version_guid and force is not True. (only applicable to version tracking stores)
"""
raise NotImplementedError
def only_xmodules(identifier, entry_points):
"""Only use entry_points that are supplied by the xmodule package"""

1
common/lib/xmodule/xmodule/modulestore/django.py
View File

@@ -66,7 +66,6 @@ def create_modulestore_instance(engine, doc_store_config, options, i18n_service=
return class_(
metadata_inheritance_cache_subsystem=metadata_inheritance_cache,
request_cache=request_cache,
modulestore_update_signal=Signal(providing_args=['modulestore', 'course_id', 'location']),
xblock_mixins=getattr(settings, 'XBLOCK_MIXINS', ()),
xblock_select=getattr(settings, 'XBLOCK_SELECT_FUNCTION', None),
doc_store_config=doc_store_config,

7
common/lib/xmodule/xmodule/modulestore/exceptions.py
View File

@@ -37,6 +37,13 @@ class DuplicateItemError(Exception):
self.store = store
self.collection = collection
def __str__(self, *args, **kwargs):
"""
Print info about what's duplicated
"""
return '{0.store}[{0.collection}] already has {0.element_id}'.format(
self, Exception.__str__(self, *args, **kwargs)
)
class VersionConflictError(Exception):
"""

495
common/lib/xmodule/xmodule/modulestore/loc_mapper_store.py
View File

@@ -5,11 +5,12 @@ from random import randint
import re
import pymongo
import bson.son
import urllib
from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError
from xmodule.modulestore.locator import BlockUsageLocator, CourseLocator
from xmodule.modulestore import Location
import urllib
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from xmodule.modulestore.keys import UsageKey
class LocMapperStore(object):
@@ -27,6 +28,7 @@ class LocMapperStore(object):
or dominant store, but that's not a requirement. This store creates its own connection.
'''
SCHEMA_VERSION = 1
def __init__(
self, cache, host, db, collection, port=27017, user=None, password=None,
**kwargs
@@ -39,6 +41,7 @@ class LocMapperStore(object):
host=host,
port=port,
tz_aware=True,
document_class=bson.son.SON,
**kwargs
),
db
@@ -51,31 +54,21 @@ class LocMapperStore(object):
self.cache = cache
# location_map functions
def create_map_entry(self, course_location, package_id=None, draft_branch='draft', prod_branch='published',
def create_map_entry(self, course_key, org=None, offering=None, draft_branch='draft', prod_branch='published',
block_map=None):
"""
Add a new entry to map this course_location to the new style CourseLocator.package_id. If package_id is not
provided, it creates the default map of using org.course.name from the location if
the location.category = 'course'; otherwise, it uses org.course.
Add a new entry to map this SlashSeparatedCourseKey to the new style CourseLocator.org & offering. If
org and offering are not provided, it defaults them based on course_key.
You can create more than one mapping to the
same package_id target. In that case, the reverse translate will be arbitrary (no guarantee of which wins).
The use
case for more than one mapping is to map both org/course/run and org/course to the same new package_id thus
making a default for org/course. When querying for just org/course, the translator will prefer any entry
which does not have a name in the _id; otherwise, it will return an arbitrary match.
WARNING: Exactly 1 CourseLocator key should index a given SlashSeparatedCourseKey.
We provide no mechanism to enforce this assertion.
Note: the opposite is not true. That is, it never makes sense to use 2 different CourseLocator.package_id
keys to index the same old Locator org/course/.. pattern. There's no checking to ensure you don't do this.
NOTE: if there's already an entry w the given course_location, this may either overwrite that entry or
NOTE: if there's already an entry w the given course_key, this may either overwrite that entry or
throw an error depending on how mongo is configured.
:param course_location: a Location preferably whose category is 'course'. Unlike the other
map methods, this one doesn't take the old-style course_id. It should be called with
a course location not a block location; however, if called w/ a non-course Location, it creates
a "default" map for the org/course pair to a new package_id.
:param package_id: the CourseLocator style package_id
:param course_key (SlashSeparatedCourseKey): a SlashSeparatedCourseKey
:param org (string): the CourseLocator style org
:param offering (string): the CourseLocator offering
:param draft_branch: the branch name to assign for drafts. This is hardcoded because old mongo had
a fixed notion that there was 2 and only 2 versions for modules: draft and production. The old mongo
did not, however, require that a draft version exist. The new one, however, does require a draft to
@@ -85,54 +78,49 @@ class LocMapperStore(object):
to publish).
:param block_map: an optional map to specify preferred names for blocks where the keys are the
Location block names and the values are the BlockUsageLocator.block_id.
Returns:
:class:`CourseLocator` representing the new id for the course
Raises:
ValueError if one and only one of org and offering is provided. Provide either both or neither.
"""
if package_id is None:
if course_location.category == 'course':
package_id = u"{0.org}.{0.course}.{0.name}".format(course_location)
else:
package_id = u"{0.org}.{0.course}".format(course_location)
# very like _interpret_location_id but w/o the _id
location_id = self._construct_location_son(
course_location.org, course_location.course,
course_location.name if course_location.category == 'course' else None
)
# create location id with lower case
location_id_lower = self._construct_lower_location_son(
course_location.org, course_location.course,
course_location.name if course_location.category == 'course' else None
)
if org is None and offering is None:
assert(isinstance(course_key, SlashSeparatedCourseKey))
org = course_key.org
offering = u"{0.course}.{0.run}".format(course_key)
elif org is None or offering is None:
raise ValueError(
u"Either supply both org and offering or neither. Not just one: {}, {}".format(org, offering)
)
try:
self.location_map.insert({
'_id': location_id,
'lower_id': location_id_lower,
'course_id': package_id,
'lower_course_id': package_id.lower(),
'draft_branch': draft_branch,
'prod_branch': prod_branch,
'block_map': block_map or {},
})
except pymongo.errors.DuplicateKeyError:
# update old entry with 'lower_id' and 'lower_course_id'
location_update = {'lower_id': location_id_lower, 'lower_course_id': package_id.lower()}
self.location_map.update({'_id': location_id}, {'$set': location_update})
# very like _interpret_location_id but using mongo subdoc lookup (more performant)
course_son = self._construct_course_son(course_key)
return package_id
self.location_map.insert({
'_id': course_son,
'org': org,
'lower_org': org.lower(),
'offering': offering,
'lower_offering': offering.lower(),
'draft_branch': draft_branch,
'prod_branch': prod_branch,
'block_map': block_map or {},
'schema': self.SCHEMA_VERSION,
})
def translate_location(self, old_style_course_id, location, published=True,
return CourseLocator(org, offering)
def translate_location(self, location, published=True,
add_entry_if_missing=True, passed_block_id=None):
"""
Translate the given module location to a Locator. If the mapping has the run id in it, then you
should provide old_style_course_id with that run id in it to disambiguate the mapping if there exists more
than one entry in the mapping table for the org.course.
Translate the given module location to a Locator.
The rationale for auto adding entries was that there should be a reasonable default translation
if the code just trips into this w/o creating translations. The downfall is that ambiguous course
locations may generate conflicting block_ids.
if the code just trips into this w/o creating translations.
Will raise ItemNotFoundError if there's no mapping and add_entry_if_missing is False.
:param old_style_course_id: the course_id used in old mongo not the new one (optional, will use location)
:param location: a Location pointing to a module
:param published: a boolean to indicate whether the caller wants the draft or published branch.
:param add_entry_if_missing: a boolean as to whether to raise ItemNotFoundError or to create an entry if
@@ -144,43 +132,32 @@ class LocMapperStore(object):
NOTE: unlike old mongo, draft branches contain the whole course; so, it applies to all category
of locations including course.
"""
location_id = self._interpret_location_course_id(old_style_course_id, location)
if old_style_course_id is None:
old_style_course_id = self._generate_location_course_id(location_id)
course_son = self._interpret_location_course_id(location.course_key)
cached_value = self._get_locator_from_cache(old_style_course_id, location, published)
cached_value = self._get_locator_from_cache(location, published)
if cached_value:
return cached_value
maps = self.location_map.find(location_id)
maps = list(maps)
if len(maps) == 0:
entry = self.location_map.find_one(course_son)
if entry is None:
if add_entry_if_missing:
# create a new map
course_location = location.replace(category='course', name=location_id['_id']['name'])
self.create_map_entry(course_location)
entry = self.location_map.find_one(location_id)
self.create_map_entry(location.course_key)
entry = self.location_map.find_one(course_son)
else:
raise ItemNotFoundError(location)
elif len(maps) == 1:
entry = maps[0]
else:
# find entry w/o name, if any; otherwise, pick arbitrary
entry = maps[0]
for item in maps:
if 'name' not in item['_id']:
entry = item
break
entry = self._migrate_if_necessary([entry])[0]
block_id = entry['block_map'].get(self.encode_key_for_mongo(location.name))
if block_id is None:
if add_entry_if_missing:
block_id = self._add_to_block_map(
location, location_id, entry['block_map'], passed_block_id
location, course_son, entry['block_map'], passed_block_id
)
else:
raise ItemNotFoundError(location)
elif isinstance(block_id, dict):
else:
# jump_to_id uses a None category.
if location.category is None:
if len(block_id) == 1:
@@ -191,22 +168,29 @@ class LocMapperStore(object):
elif location.category in block_id:
block_id = block_id[location.category]
elif add_entry_if_missing:
block_id = self._add_to_block_map(location, location_id, entry['block_map'])
block_id = self._add_to_block_map(location, course_son, entry['block_map'])
else:
raise ItemNotFoundError(location)
else:
raise InvalidLocationError()
prod_course_locator = CourseLocator(
org=entry['org'],
offering=entry['offering'],
branch=entry['prod_branch']
)
published_usage = BlockUsageLocator(
package_id=entry['course_id'], branch=entry['prod_branch'], block_id=block_id)
prod_course_locator,
block_id=block_id
)
draft_usage = BlockUsageLocator(
package_id=entry['course_id'], branch=entry['draft_branch'], block_id=block_id)
prod_course_locator.for_branch(entry['draft_branch']),
block_id=block_id
)
if published:
result = published_usage
else:
result = draft_usage
self._cache_location_map_entry(old_style_course_id, location, published_usage, draft_usage)
self._cache_location_map_entry(location, published_usage, draft_usage)
return result
def translate_locator_to_location(self, locator, get_course=False, lower_only=False):
@@ -217,18 +201,22 @@ class LocMapperStore(object):
the block's block_id was previously stored in the
map (a side effect of translate_location or via add|update_block_location).
If get_course, then rather than finding the map for this locator, it finds the 'course' root
for the mapped course.
If there are no matches, it returns None.
If there's more than one location to locator mapping to the same package_id, it looks for the first
one with a mapping for the block block_id and picks that arbitrary course location.
:param locator: a BlockUsageLocator
Args:
locator: a BlockUsageLocator to translate
get_course: rather than finding the map for this locator, returns the CourseKey
for the mapped course.
lower_only: (obsolete?) the locator's fields are lowercased and not the actual case
for the identifier (e.g., came from a sql db which lowercases all ids). Find the actual
case Location for the desired object
"""
if get_course:
cached_value = self._get_course_location_from_cache(locator.package_id, lower_only)
cached_value = self._get_course_location_from_cache(
# if locator is already a course_key it won't have course_key attr
getattr(locator, 'course_key', locator),
lower_only
)
else:
cached_value = self._get_location_from_cache(locator)
if cached_value:
@@ -237,90 +225,118 @@ class LocMapperStore(object):
# This does not require that the course exist in any modulestore
# only that it has a mapping entry.
if lower_only:
maps = self.location_map.find({'lower_course_id': locator.package_id.lower()})
else:
maps = self.location_map.find({'course_id': locator.package_id})
# look for one which maps to this block block_id
if maps.count() == 0:
return None
result = None
for candidate in maps:
if get_course and 'name' in candidate['_id']:
candidate_id = candidate['_id']
return Location(
'i4x', candidate_id['org'], candidate_id['course'], 'course', candidate_id['name']
# migrate any records which don't have the lower_org and lower_offering fields as
# this won't be able to find what it wants. (only needs to be run once ever per db,
# I'm not sure how to control that, but I'm putting some check here for once per launch)
if not getattr(self, 'lower_offering_migrated', False):
obsolete = self.location_map.find(
{'lower_org': {"$exists": False}, "lower_offering": {"$exists": False}, }
)
old_course_id = self._generate_location_course_id(candidate['_id'])
for old_name, cat_to_usage in candidate['block_map'].iteritems():
for category, block_id in cat_to_usage.iteritems():
# cache all entries and then figure out if we have the one we want
# Always return revision=None because the
# old draft module store wraps locations as draft before
# trying to access things.
location = Location(
'i4x',
candidate['_id']['org'],
candidate['_id']['course'],
category,
self.decode_key_from_mongo(old_name),
None)
self._migrate_if_necessary(obsolete)
setattr(self, 'lower_offering_migrated', True)
if lower_only:
candidate_key = "lower_course_id"
else:
candidate_key = "course_id"
entry = self.location_map.find_one(bson.son.SON([
('lower_org', locator.org.lower()),
('lower_offering', locator.offering.lower()),
]))
else:
# migrate any records which don't have the lower_org and lower_offering fields as
# this won't be able to find what it wants. (only needs to be run once ever per db,
# I'm not sure how to control that, but I'm putting some check here for once per launch)
if not getattr(self, 'offering_migrated', False):
obsolete = self.location_map.find(
{'org': {"$exists": False}, "offering": {"$exists": False}, }
)
self._migrate_if_necessary(obsolete)
setattr(self, 'offering_migrated', True)
published_locator = BlockUsageLocator(
candidate[candidate_key], branch=candidate['prod_branch'], block_id=block_id
)
draft_locator = BlockUsageLocator(
candidate[candidate_key], branch=candidate['draft_branch'], block_id=block_id
)
self._cache_location_map_entry(old_course_id, location, published_locator, draft_locator)
entry = self.location_map.find_one(bson.son.SON([
('org', locator.org),
('lower_offering', locator.offering),
]))
# look for one which maps to this block block_id
if entry is None:
return None
old_course_id = self._generate_location_course_id(entry['_id'])
if get_course:
return old_course_id
for old_name, cat_to_usage in entry['block_map'].iteritems():
for category, block_id in cat_to_usage.iteritems():
# cache all entries and then figure out if we have the one we want
# Always return revision=None because the
# old draft module store wraps locations as draft before
# trying to access things.
location = old_course_id.make_usage_key(
category,
self.decode_key_from_mongo(old_name)
)
if lower_only:
entry_org = "lower_org"
entry_offering = "lower_offering"
else:
entry_org = "org"
entry_offering = "offering"
published_locator = BlockUsageLocator(
CourseLocator(
org=entry[entry_org], offering=entry[entry_offering],
branch=entry['prod_branch']
),
block_id=block_id
)
draft_locator = BlockUsageLocator(
CourseLocator(
org=entry[entry_org], offering=entry[entry_offering],
branch=entry['draft_branch']
),
block_id=block_id
)
self._cache_location_map_entry(location, published_locator, draft_locator)
if block_id == locator.block_id:
return location
if get_course and category == 'course':
result = location
elif not get_course and block_id == locator.block_id:
result = location
if result is not None:
return result
return None
def translate_location_to_course_locator(self, old_style_course_id, location, published=True, lower_only=False):
def translate_location_to_course_locator(self, course_key, published=True):
"""
Used when you only need the CourseLocator and not a full BlockUsageLocator. Probably only
useful for get_items which wildcards name or category.
:param course_id: old style course id
:param course_key: a CourseKey or a UsageKey
:param published: a boolean representing whether or not we should return the published or draft version
Returns a Courselocator
"""
cached = self._get_course_locator_from_cache(old_style_course_id, published)
if isinstance(course_key, UsageKey):
course_key = course_key.course_key
cached = self._get_course_locator_from_cache(course_key, published)
if cached:
return cached
location_id = self._interpret_location_course_id(old_style_course_id, location, lower_only)
course_son = self._interpret_location_course_id(course_key)
maps = self.location_map.find(location_id)
maps = list(maps)
if len(maps) == 0:
raise ItemNotFoundError(location)
elif len(maps) == 1:
entry = maps[0]
else:
# find entry w/o name, if any; otherwise, pick arbitrary
entry = maps[0]
for item in maps:
if 'name' not in item['_id']:
entry = item
break
published_course_locator = CourseLocator(package_id=entry['course_id'], branch=entry['prod_branch'])
draft_course_locator = CourseLocator(package_id=entry['course_id'], branch=entry['draft_branch'])
self._cache_course_locator(old_style_course_id, published_course_locator, draft_course_locator)
entry = self.location_map.find_one(course_son)
if entry is None:
raise ItemNotFoundError(course_key)
published_course_locator = CourseLocator(
org=entry['org'], offering=entry['offering'], branch=entry['prod_branch']
)
draft_course_locator = CourseLocator(
org=entry['org'], offering=entry['offering'], branch=entry['draft_branch']
)
self._cache_course_locator(course_key, published_course_locator, draft_course_locator)
if published:
return published_course_locator
else:
return draft_course_locator
def _add_to_block_map(self, location, location_id, block_map, block_id=None):
def _add_to_block_map(self, location, course_son, block_map, block_id=None):
'''add the given location to the block_map and persist it'''
if block_id is None:
if self._block_id_is_guid(location.name):
@@ -335,63 +351,33 @@ class LocMapperStore(object):
block_id = self._verify_uniqueness(location.name, block_map)
encoded_location_name = self.encode_key_for_mongo(location.name)
block_map.setdefault(encoded_location_name, {})[location.category] = block_id
self.location_map.update(location_id, {'$set': {'block_map': block_map}})
self.location_map.update(course_son, {'$set': {'block_map': block_map}})
return block_id
def _interpret_location_course_id(self, course_id, location, lower_only=False):
def _interpret_location_course_id(self, course_key):
"""
Take the old style course id (org/course/run) and return a dict w/ a SON for querying the mapping table.
If the course_id is empty, it uses location, but this may result in an inadequate id.
Take a CourseKey and return a SON for querying the mapping table.
:param course_id: old style 'org/course/run' id from Location.course_id where Location.category = 'course'
:param location: a Location object which may be to a module or a course. Provides partial info
if course_id is omitted.
:param course_key: a CourseKey object for a course.
"""
if course_id:
# re doesn't allow ?P<_id.org> and ilk
matched = re.match(r'([^/]+)/([^/]+)/([^/]+)', course_id)
if lower_only:
return {'lower_id': self._construct_lower_location_son(*matched.groups())}
return {'_id': self._construct_location_son(*matched.groups())}
if location.category == 'course':
if lower_only:
return {'lower_id': self._construct_lower_location_son(location.org, location.course, location.name)}
return {'_id': self._construct_location_son(location.org, location.course, location.name)}
else:
return bson.son.SON([('_id.org', location.org), ('_id.course', location.course)])
return {'_id': self._construct_course_son(course_key)}
def _generate_location_course_id(self, entry_id):
"""
Generate a Location course_id for the given entry's id.
Generate a CourseKey for the given entry's id.
"""
# strip id envelope if any
entry_id = entry_id.get('_id', entry_id)
if entry_id.get('name', False):
return u'{0[org]}/{0[course]}/{0[name]}'.format(entry_id)
elif entry_id.get('_id.org', False):
# the odd format one
return u'{0[_id.org]}/{0[_id.course]}'.format(entry_id)
else:
return u'{0[org]}/{0[course]}'.format(entry_id)
return SlashSeparatedCourseKey(entry_id['org'], entry_id['course'], entry_id['name'])
def _construct_location_son(self, org, course, name=None):
def _construct_course_son(self, course_key):
"""
Construct the SON needed to repr the location for either a query or an insertion
Construct the SON needed to repr the course_key for either a query or an insertion
"""
if name:
return bson.son.SON([('org', org), ('course', course), ('name', name)])
else:
return bson.son.SON([('org', org), ('course', course)])
def _construct_lower_location_son(self, org, course, name=None):
"""
Construct the SON needed to represent the location with lower case
"""
if name is not None:
name = name.lower()
return self._construct_location_son(org.lower(), course.lower(), name)
assert(isinstance(course_key, SlashSeparatedCourseKey))
return bson.son.SON([
('org', course_key.org),
('course', course_key.course),
('name', course_key.run)
])
def _block_id_is_guid(self, name):
"""
@@ -434,11 +420,11 @@ class LocMapperStore(object):
"""
return urllib.unquote(fieldname)
def _get_locator_from_cache(self, old_course_id, location, published):
def _get_locator_from_cache(self, location, published):
"""
See if the location x published pair is in the cache. If so, return the mapped locator.
"""
entry = self.cache.get(u'{}+{}'.format(old_course_id, location.url()))
entry = self.cache.get(u'{}+{}'.format(location.course_key, location))
if entry is not None:
if published:
return entry[0]
@@ -452,12 +438,12 @@ class LocMapperStore(object):
"""
if not old_course_id:
return None
entry = self.cache.get(old_course_id)
entry = self.cache.get(unicode(old_course_id))
if entry is not None:
if published:
return entry[0].as_course_locator()
return entry[0].course_key
else:
return entry[1].as_course_locator()
return entry[1].course_key
def _get_location_from_cache(self, locator):
"""
@@ -483,9 +469,9 @@ class LocMapperStore(object):
"""
if not old_course_id:
return
self.cache.set(old_course_id, (published_course_locator, draft_course_locator))
self.cache.set(unicode(old_course_id), (published_course_locator, draft_course_locator))
def _cache_location_map_entry(self, old_course_id, location, published_usage, draft_usage):
def _cache_location_map_entry(self, location, published_usage, draft_usage):
"""
Cache the mapping from location to the draft and published Locators in entry.
Also caches the inverse. If the location is category=='course', it caches it for
@@ -497,25 +483,48 @@ class LocMapperStore(object):
setmany[u'courseIdLower+{}'.format(published_usage.package_id.lower())] = location
setmany[unicode(published_usage)] = location
setmany[unicode(draft_usage)] = location
setmany[u'{}+{}'.format(old_course_id, location.url())] = (published_usage, draft_usage)
setmany[old_course_id] = (published_usage, draft_usage)
setmany[unicode(location)] = (published_usage, draft_usage)
setmany[unicode(location.course_key)] = (published_usage, draft_usage)
self.cache.set_many(setmany)
def delete_course_mapping(self, course_location):
def delete_course_mapping(self, course_key):
"""
Remove provided course location from loc_mapper and cache.
:param course_location: a Location whose category is 'course'.
:param course_key: a CourseKey for the course we wish to delete
"""
course_locator = self.translate_location(course_location.course_id, course_location)
course_locator_draft = self.translate_location(
course_location.course_id, course_location, published=False
)
self.location_map.remove(self._interpret_location_course_id(course_key))
self.location_map.remove({'course_id': course_locator.package_id})
self._delete_cache_location_map_entry(
course_location.course_id, course_location, course_locator, course_locator_draft
)
# Remove the location of course (draft and published) from cache
cached_key = self.cache.get(unicode(course_key))
if cached_key:
delete_keys = []
published_locator = unicode(cached_key[0].course_key)
course_location = self._course_location_from_cache(published_locator)
delete_keys.append(u'courseId+{}'.format(published_locator))
delete_keys.append(u'courseIdLower+{}'.format(unicode(cached_key[0].course_key).lower()))
delete_keys.append(published_locator)
delete_keys.append(unicode(cached_key[1].course_key))
delete_keys.append(unicode(course_location))
delete_keys.append(unicode(course_key))
self.cache.delete_many(delete_keys)
def _migrate_if_necessary(self, entries):
"""
Run the entries through any applicable schema updates and return the updated entries
"""
entries = [
self._migrate[entry.get('schema', 0)](self, entry)
for entry in entries
]
return entries
def _entry_id_to_son(self, entry_id):
return bson.son.SON([
('org', entry_id['org']),
('course', entry_id['course']),
('name', entry_id['name'])
])
def _delete_cache_location_map_entry(self, old_course_id, location, published_usage, draft_usage):
"""
@@ -528,6 +537,50 @@ class LocMapperStore(object):
delete_keys.append(unicode(published_usage))
delete_keys.append(unicode(draft_usage))
delete_keys.append(u'{}+{}'.format(old_course_id, location.url()))
delete_keys.append(u'{}+{}'.format(old_course_id, location.to_deprecated_string()))
delete_keys.append(old_course_id)
self.cache.delete_many(delete_keys)
def _migrate_top(self, entry, updated=False):
"""
Current version, so a no data change until next update. But since it's the top
it's responsible for persisting the record if it changed.
"""
if updated:
entry['schema'] = self.SCHEMA_VERSION
entry_id = self._entry_id_to_son(entry['_id'])
self.location_map.update({'_id': entry_id}, entry)
return entry
def _migrate_0(self, entry):
"""
If entry had an '_id' without a run, remove the whole record.
Add fields: schema, org, offering, lower_org, and lower_offering
Remove: course_id, lower_course_id
:param entry:
"""
if 'name' not in entry['_id']:
entry_id = entry['_id']
entry_id = bson.son.SON([
('org', entry_id['org']),
('course', entry_id['course']),
])
self.location_map.remove({'_id': entry_id})
return None
# add schema, org, offering, etc, remove old fields
entry['schema'] = 0
entry.pop('course_id', None)
entry.pop('lower_course_id', None)
old_course_id = SlashSeparatedCourseKey(entry['_id']['org'], entry['_id']['course'], entry['_id']['name'])
entry['org'] = old_course_id.org
entry['lower_org'] = old_course_id.org.lower()
entry['offering'] = old_course_id.offering.replace('/', '+')
entry['lower_offering'] = entry['offering'].lower()
return self._migrate_1(entry, True)
# insert new migrations just before _migrate_top. _migrate_top sets the schema version and
# saves the record
_migrate = [_migrate_0, _migrate_top]

732
common/lib/xmodule/xmodule/modulestore/locator.py
View File

@@ -5,17 +5,23 @@ Identifier for course resources.
from __future__ import absolute_import
import logging
import inspect
from abc import ABCMeta, abstractmethod
import re
from abc import abstractmethod
from bson.objectid import ObjectId
from bson.errors import InvalidId
from xmodule.modulestore.exceptions import InsufficientSpecificationError, OverSpecificationError
from opaque_keys import OpaqueKey, InvalidKeyError
from .parsers import parse_url, parse_package_id, parse_block_ref
from .parsers import BRANCH_PREFIX, BLOCK_PREFIX, VERSION_PREFIX
import re
from xmodule.modulestore import Location
from xmodule.modulestore.keys import CourseKey, UsageKey
from xmodule.modulestore.parsers import (
parse_url,
parse_block_ref,
BRANCH_PREFIX,
BLOCK_PREFIX,
VERSION_PREFIX,
ALLOWED_ID_RE)
log = logging.getLogger(__name__)
@@ -32,41 +38,21 @@ class LocalId(object):
return "localid_{}".format(self.block_id or id(self))
class Locator(object):
class Locator(OpaqueKey):
"""
A locator is like a URL, it refers to a course resource.
Locator is an abstract base class: do not instantiate
"""
__metaclass__ = ABCMeta
@abstractmethod
def url(self):
"""
Return a string containing the URL for this location. Raises
InsufficientSpecificationError if the instance doesn't have a
InvalidKeyError if the instance doesn't have a
complete enough specification to generate a url
"""
raise InsufficientSpecificationError()
def __eq__(self, other):
return self.__dict__ == other.__dict__
def __hash__(self):
"""
Hash on contents.
"""
return hash(unicode(self))
def __repr__(self):
'''
repr(self) returns something like this: CourseLocator("mit.eecs.6002x")
'''
classname = self.__class__.__name__
if classname.find('.') != -1:
classname = classname.split['.'][-1]
return '%s("%s")' % (classname, unicode(self))
raise NotImplementedError()
def __str__(self):
'''
@@ -74,73 +60,14 @@ class Locator(object):
'''
return unicode(self).encode('utf-8')
def __unicode__(self):
'''
unicode(self) returns something like this: "mit.eecs.6002x"
'''
return unicode(self).encode('utf-8')
@abstractmethod
def version(self):
"""
Returns the ObjectId referencing this specific location.
Raises InsufficientSpecificationError if the instance
Raises InvalidKeyError if the instance
doesn't have a complete enough specification.
"""
raise InsufficientSpecificationError()
def set_property(self, property_name, new):
"""
Initialize property to new value.
If property has already been initialized to a different value, raise an exception.
"""
current = getattr(self, property_name)
if current and current != new:
raise OverSpecificationError('%s cannot be both %s and %s' %
(property_name, current, new))
setattr(self, property_name, new)
@staticmethod
def to_locator_or_location(location):
"""
Convert the given locator like thing to the appropriate type of object, or, if already
that type, just return it. Returns an old Location, BlockUsageLocator,
or DefinitionLocator.
:param location: can be a Location, Locator, string, tuple, list, or dict.
"""
if isinstance(location, (Location, Locator)):
return location
if isinstance(location, basestring):
return Locator.parse_url(location)
if isinstance(location, (list, tuple)):
return Location(location)
if isinstance(location, dict) and 'name' in location:
return Location(location)
if isinstance(location, dict):
return BlockUsageLocator(**location)
raise ValueError(location)
URL_TAG_RE = re.compile(r'^(\w+)://')
@staticmethod
def parse_url(url):
"""
Parse the url into one of the Locator types (must have a tag indicating type)
Return the new instance. Supports i4x, cvx, edx, defx
:param url: the url to parse
"""
parsed = Locator.URL_TAG_RE.match(url)
if parsed is None:
raise ValueError(parsed)
parsed = parsed.group(1)
if parsed in ['i4x', 'c4x']:
return Location(url)
elif parsed == 'edx':
return BlockUsageLocator(url)
elif parsed == 'defx':
return DefinitionLocator(url)
return None
raise NotImplementedError()
@classmethod
def as_object_id(cls, value):
@@ -154,60 +81,203 @@ class Locator(object):
raise ValueError('"%s" is not a valid version_guid' % value)
class CourseLocator(Locator):
class BlockLocatorBase(Locator):
# Token separating org from offering
ORG_SEPARATOR = '+'
def version(self):
"""
Returns the ObjectId referencing this specific location.
"""
return self.version_guid
def url(self):
"""
Return a string containing the URL for this location.
"""
return self.NAMESPACE_SEPARATOR.join([self.CANONICAL_NAMESPACE, self._to_string()])
@classmethod
def _parse_url(cls, url):
"""
url must be a string beginning with 'edx:' and containing
either a valid version_guid or org & offering (with optional branch), or both.
"""
if not isinstance(url, basestring):
raise TypeError('%s is not an instance of basestring' % url)
parse = parse_url(url)
if not parse:
raise InvalidKeyError(cls, url)
if parse['version_guid']:
parse['version_guid'] = cls.as_object_id(parse['version_guid'])
return parse
@property
def package_id(self):
if self.org and self.offering:
return u'{}{}{}'.format(self.org, self.ORG_SEPARATOR, self.offering)
else:
return None
class CourseLocator(BlockLocatorBase, CourseKey):
"""
Examples of valid CourseLocator specifications:
CourseLocator(version_guid=ObjectId('519665f6223ebd6980884f2b'))
CourseLocator(package_id='mit.eecs.6002x')
CourseLocator(package_id='mit.eecs.6002x/branch/published')
CourseLocator(package_id='mit.eecs.6002x', branch='published')
CourseLocator(url='edx://version/519665f6223ebd6980884f2b')
CourseLocator(url='edx://mit.eecs.6002x')
CourseLocator(url='edx://mit.eecs.6002x/branch/published')
CourseLocator(url='edx://mit.eecs.6002x/branch/published/version/519665f6223ebd6980884f2b')
CourseLocator(org='mit.eecs', offering='6.002x')
CourseLocator(org='mit.eecs', offering='6002x', branch = 'published')
CourseLocator.from_string('edx:version/519665f6223ebd6980884f2b')
CourseLocator.from_string('version/519665f6223ebd6980884f2b')
CourseLocator.from_string('edx:mit.eecs+6002x')
CourseLocator.from_string('mit.eecs+6002x')
CourseLocator.from_string('edx:mit.eecs+6002x/branch/published')
CourseLocator.from_string('edx:mit.eecs+6002x/branch/published/version/519665f6223ebd6980884f2b')
CourseLocator.from_string('mit.eecs+6002x/branch/published/version/519665f6223ebd6980884f2b')
Should have at lease a specific package_id (id for the course as if it were a project w/
Should have at least a specific org & offering (id for the course as if it were a project w/
versions) with optional 'branch',
or version_guid (which points to a specific version). Can contain both in which case
the persistence layer may raise exceptions if the given version != the current such version
of the course.
"""
CANONICAL_NAMESPACE = 'course-locator'
KEY_FIELDS = ('org', 'offering', 'branch', 'version_guid')
# Default values
version_guid = None
package_id = None
branch = None
# stubs to fake out the abstractproperty class instrospection and allow treatment as attrs in instances
org = None
offering = None
def __init__(self, url=None, version_guid=None, package_id=None, branch=None):
def __init__(self, org=None, offering=None, branch=None, version_guid=None):
"""
Construct a CourseLocator
Caller may provide url (but no other parameters).
Caller may provide version_guid (but no other parameters).
Caller may provide package_id (optionally provide branch).
Resulting CourseLocator will have either a version_guid property
or a package_id (with optional branch) property, or both.
version_guid must be an instance of bson.objectid.ObjectId or None
url, package_id, and branch must be strings or None
Args:
version_guid (string or ObjectId): optional unique id for the version
org, offering (string): the standard definition. Optional only if version_guid given
branch (string): the branch such as 'draft', 'published', 'staged', 'beta'
"""
self._validate_args(url, version_guid, package_id)
if url:
self.init_from_url(url)
if version_guid:
self.init_from_version_guid(version_guid)
if package_id or branch:
self.init_from_package_id(package_id, branch)
if self.version_guid is None and self.package_id is None:
raise ValueError("Either version_guid or package_id should be set: {}".format(url))
version_guid = self.as_object_id(version_guid)
def __unicode__(self):
if not all(field is None or ALLOWED_ID_RE.match(field) for field in [org, offering, branch]):
raise InvalidKeyError(self.__class__, [org, offering, branch])
super(CourseLocator, self).__init__(
org=org,
offering=offering,
branch=branch,
version_guid=version_guid
)
if self.version_guid is None and self.org is None and self.offering is None:
raise InvalidKeyError(self.__class__, "Either version_guid or org and offering should be set")
@classmethod
def _from_string(cls, serialized):
"""
Return a CourseLocator parsing the given serialized string
:param serialized: matches the string to a CourseLocator
"""
kwargs = cls._parse_url(serialized)
try:
return cls(**{key: kwargs.get(key) for key in cls.KEY_FIELDS})
except ValueError:
raise InvalidKeyError(cls, "Either version_guid or org and offering should be set: {}".format(serialized))
def is_fully_specified(self):
"""
Returns True if either version_guid is specified, or org+offering+branch
are specified.
This should always return True, since this should be validated in the constructor.
"""
return (
self.version_guid is not None or
(self.org is not None and self.offering is not None and self.branch is not None)
)
def html_id(self):
"""
Generate a discussion group id based on course
To make compatible with old Location object functionality. I don't believe this behavior fits at this
place, but I have no way to override. We should clearly define the purpose and restrictions of this
(e.g., I'm assuming periods are fine).
"""
return unicode(self)
def make_usage_key(self, block_type, block_id):
return BlockUsageLocator(
course_key=self,
block_id=block_id
)
def make_asset_key(self, asset_type, path):
raise NotImplementedError()
def version_agnostic(self):
"""
We don't care if the locator's version is not the current head; so, avoid version conflict
by reducing info.
Returns a copy of itself without any version info.
:raises: ValueError if the block locator has no org & offering
"""
return CourseLocator(
org=self.org,
offering=self.offering,
branch=self.branch,
version_guid=None
)
def course_agnostic(self):
"""
We only care about the locator's version not its course.
Returns a copy of itself without any course info.
:raises: ValueError if the block locator has no version_guid
"""
return CourseLocator(
org=None,
offering=None,
branch=None,
version_guid=self.version_guid
)
def for_branch(self, branch):
"""
Return a new CourseLocator for another branch of the same course (also version agnostic)
"""
if self.org is None:
raise InvalidKeyError(self.__class__, "Branches must have full course ids not just versions")
return CourseLocator(
org=self.org,
offering=self.offering,
branch=branch,
version_guid=None
)
def for_version(self, version_guid):
"""
Return a new CourseLocator for another version of the same course and branch. Usually used
when the head is updated (and thus the course x branch now points to this version)
"""
return CourseLocator(
org=self.org,
offering=self.offering,
branch=self.branch,
version_guid=version_guid
)
def _to_string(self):
"""
Return a string representing this location.
"""
parts = []
if self.package_id:
if self.offering:
parts.append(unicode(self.package_id))
if self.branch:
parts.append(u"{prefix}{branch}".format(prefix=BRANCH_PREFIX, branch=self.branch))
@@ -215,60 +285,150 @@ class CourseLocator(Locator):
parts.append(u"{prefix}{guid}".format(prefix=VERSION_PREFIX, guid=self.version_guid))
return u"/".join(parts)
def url(self):
"""
Return a string containing the URL for this location.
"""
return u'edx://' + unicode(self)
def _validate_args(self, url, version_guid, package_id):
class BlockUsageLocator(BlockLocatorBase, UsageKey): # TODO implement UsageKey methods
"""
Encodes a location.
Locations address modules (aka blocks) which are definitions situated in a
course instance. Thus, a Location must identify the course and the occurrence of
the defined element in the course. Courses can be a version of an offering, the
current draft head, or the current production version.
Locators can contain both a version and a org + offering w/ branch. The split mongo functions
may raise errors if these conflict w/ the current db state (i.e., the course's branch !=
the version_guid)
Locations can express as urls as well as dictionaries. They consist of
package_identifier: course_guid | version_guid
block : guid
branch : string
"""
CANONICAL_NAMESPACE = 'edx'
KEY_FIELDS = ('course_key', 'block_id')
# fake out class instrospection as this is an attr in this class's instances
course_key = None
def __init__(self, course_key, block_id):
"""
Validate provided arguments. Internal use only which is why it checks for each
arg and doesn't use keyword
Construct a BlockUsageLocator
"""
if not any((url, version_guid, package_id)):
raise InsufficientSpecificationError("Must provide one of url, version_guid, package_id")
block_id = self._parse_block_ref(block_id)
if block_id is None:
raise InvalidKeyError(self.__class__, "Missing block id")
super(BlockUsageLocator, self).__init__(course_key=course_key, block_id=block_id)
@classmethod
def _from_string(cls, serialized):
"""
Requests CourseLocator to deserialize its part and then adds the local deserialization of block
"""
course_key = CourseLocator._from_string(serialized)
parsed_parts = parse_url(serialized)
block_id = parsed_parts.get('block_id')
if block_id is None:
raise InvalidKeyError(cls, serialized)
return cls(course_key, block_id)
def version_agnostic(self):
"""
We don't care if the locator's version is not the current head; so, avoid version conflict
by reducing info.
Returns a copy of itself without any version info.
:raises: ValueError if the block locator has no org and offering
"""
return BlockUsageLocator(
course_key=self.course_key.version_agnostic(),
block_id=self.block_id
)
def course_agnostic(self):
"""
We only care about the locator's version not its course.
Returns a copy of itself without any course info.
:raises: ValueError if the block locator has no version_guid
"""
return BlockUsageLocator(
course_key=self.course_key.course_agnostic(),
block_id=self.block_id
)
def for_branch(self, branch):
"""
Return a UsageLocator for the same block in a different branch of the course.
"""
return BlockUsageLocator(
self.course_key.for_branch(branch),
block_id=self.block_id
)
@classmethod
def _parse_block_ref(cls, block_ref):
if isinstance(block_ref, LocalId):
return block_ref
else:
parse = parse_block_ref(block_ref)
if not parse:
raise InvalidKeyError(cls, block_ref)
return parse.get('block_id')
@property
def definition_key(self):
raise NotImplementedError()
@property
def org(self):
return self.course_key.org
@property
def offering(self):
return self.course_key.offering
@property
def package_id(self):
return self.course_key.package_id
@property
def branch(self):
return self.course_key.branch
@property
def version_guid(self):
return self.course_key.version_guid
@property
def name(self):
"""
The ambiguously named field from Location which code expects to find
"""
return self.block_id
def is_fully_specified(self):
"""
Returns True if either version_guid is specified, or package_id+branch
are specified.
This should always return True, since this should be validated in the constructor.
"""
return (self.version_guid is not None or
(self.package_id is not None and self.branch is not None))
return self.course_key.is_fully_specified()
def set_package_id(self, new):
@classmethod
def make_relative(cls, course_locator, block_id):
"""
Initialize package_id to new value.
If package_id has already been initialized to a different value, raise an exception.
Return a new instance which has the given block_id in the given course
:param course_locator: may be a BlockUsageLocator in the same snapshot
"""
self.set_property('package_id', new)
if hasattr(course_locator, 'course_key'):
course_locator = course_locator.course_key
return BlockUsageLocator(
course_key=course_locator,
block_id=block_id
)
def set_branch(self, new):
def map_into_course(self, course_key):
"""
Initialize branch to new value.
If branch has already been initialized to a different value, raise an exception.
Return a new instance which has the this block_id in the given course
:param course_key: a CourseKey object representing the new course to map into
"""
self.set_property('branch', new)
def set_version_guid(self, new):
"""
Initialize version_guid to new value.
If version_guid has already been initialized to a different value, raise an exception.
"""
self.set_property('version_guid', new)
def as_course_locator(self):
"""
Returns a copy of itself (downcasting) as a CourseLocator.
The copy has the same CourseLocator fields as the original.
The copy does not include subclass information, such as
a block_id (a property of BlockUsageLocator).
"""
return CourseLocator(package_id=self.package_id,
version_guid=self.version_guid,
branch=self.branch)
return BlockUsageLocator.make_relative(course_key, self.block_id)
def url_reverse(self, prefix, postfix=''):
"""
@@ -289,71 +449,15 @@ class CourseLocator(Locator):
postfix = ''
return prefix + unicode(self) + postfix
def init_from_url(self, url):
def _to_string(self):
"""
url must be a string beginning with 'edx://' and containing
either a valid version_guid or package_id (with optional branch), or both.
Return a string representing this location.
"""
if isinstance(url, Locator):
parse = url.__dict__
elif not isinstance(url, basestring):
raise TypeError('%s is not an instance of basestring' % url)
else:
parse = parse_url(url, tag_optional=True)
if not parse:
raise ValueError('Could not parse "%s" as a url' % url)
self._set_value(
parse, 'version_guid', lambda (new_guid): self.set_version_guid(self.as_object_id(new_guid))
return u"{course_key}/{BLOCK_PREFIX}{block_id}".format(
course_key=self.course_key._to_string(),
BLOCK_PREFIX=BLOCK_PREFIX,
block_id=self.block_id
)
self._set_value(parse, 'package_id', self.set_package_id)
self._set_value(parse, 'branch', self.set_branch)
def init_from_version_guid(self, version_guid):
"""
version_guid must be an instance of bson.objectid.ObjectId,
or able to be cast as one.
If it's a string, attempt to cast it as an ObjectId first.
"""
version_guid = self.as_object_id(version_guid)
if not isinstance(version_guid, ObjectId):
raise TypeError('%s is not an instance of ObjectId' % version_guid)
self.set_version_guid(version_guid)
def init_from_package_id(self, package_id, explicit_branch=None):
"""
package_id is a CourseLocator or a string like 'mit.eecs.6002x' or 'mit.eecs.6002x/branch/published'.
Revision (optional) is a string like 'published'.
It may be provided explicitly (explicit_branch) or embedded into package_id.
If branch is part of package_id (".../branch/published"), parse it out separately.
If branch is provided both ways, that's ok as long as they are the same value.
If a block ('/block/HW3') is a part of package_id, it is ignored.
"""
if package_id:
if isinstance(package_id, CourseLocator):
package_id = package_id.package_id
if not package_id:
raise ValueError("%s does not have a valid package_id" % package_id)
parse = parse_package_id(package_id)
if not parse or parse['package_id'] is None:
raise ValueError('Could not parse "%s" as a package_id' % package_id)
self.set_package_id(parse['package_id'])
rev = parse['branch']
if rev:
self.set_branch(rev)
if explicit_branch:
self.set_branch(explicit_branch)
def version(self):
"""
Returns the ObjectId referencing this specific location.
"""
return self.version_guid
def html_id(self):
"""
@@ -363,179 +467,31 @@ class CourseLocator(Locator):
place, but I have no way to override. We should clearly define the purpose and restrictions of this
(e.g., I'm assuming periods are fine).
"""
return self.package_id
def _set_value(self, parse, key, setter):
"""
Helper method that gets a value out of the dict returned by parse,
and then sets the corresponding bit of information in this locator
(via the supplied lambda 'setter'), unless the value is None.
"""
value = parse.get(key, None)
if value:
setter(value)
class BlockUsageLocator(CourseLocator):
"""
Encodes a location.
Locations address modules (aka blocks) which are definitions situated in a
course instance. Thus, a Location must identify the course and the occurrence of
the defined element in the course. Courses can be a version of an offering, the
current draft head, or the current production version.
Locators can contain both a version and a package_id w/ branch. The split mongo functions
may raise errors if these conflict w/ the current db state (i.e., the course's branch !=
the version_guid)
Locations can express as urls as well as dictionaries. They consist of
package_identifier: course_guid | version_guid
block : guid
branch : string
"""
# Default value
block_id = None
def __init__(self, url=None, version_guid=None, package_id=None,
branch=None, block_id=None):
"""
Construct a BlockUsageLocator
Caller may provide url, version_guid, or package_id, and optionally provide branch.
The block_id may be specified, either explictly or as part of
the url or package_id. If omitted, the locator is created but it
has not yet been initialized.
Resulting BlockUsageLocator will have a block_id property.
It will have either a version_guid property or a package_id (with optional branch) property, or both.
version_guid must be an instance of bson.objectid.ObjectId or None
url, package_id, branch, and block_id must be strings or None
"""
self._validate_args(url, version_guid, package_id)
if url:
self.init_block_ref_from_str(url)
if package_id:
self.init_block_ref_from_package_id(package_id)
if block_id:
self.init_block_ref(block_id)
super(BlockUsageLocator, self).__init__(
url=url,
version_guid=version_guid,
package_id=package_id,
branch=branch
)
def is_initialized(self):
"""
Returns True if block_id has been initialized, else returns False
"""
return self.block_id is not None
def version_agnostic(self):
"""
We don't care if the locator's version is not the current head; so, avoid version conflict
by reducing info.
Returns a copy of itself without any version info.
:raises: ValueError if the block locator has no package_id
"""
return BlockUsageLocator(package_id=self.package_id,
branch=self.branch,
block_id=self.block_id)
def course_agnostic(self):
"""
We only care about the locator's version not its course.
Returns a copy of itself without any course info.
:raises: ValueError if the block locator has no version_guid
"""
return BlockUsageLocator(version_guid=self.version_guid,
block_id=self.block_id)
def set_block_id(self, new):
"""
Initialize block_id to new value.
If block_id has already been initialized to a different value, raise an exception.
"""
self.set_property('block_id', new)
def init_block_ref(self, block_ref):
if isinstance(block_ref, LocalId):
self.set_block_id(block_ref)
else:
parse = parse_block_ref(block_ref)
if not parse:
raise ValueError('Could not parse "%s" as a block_ref' % block_ref)
self.set_block_id(parse['block'])
def init_block_ref_from_str(self, value):
"""
Create a block locator from the given string which may be a url or just the repr (no tag)
"""
if hasattr(value, 'block_id'):
self.init_block_ref(value.block_id)
return
if not isinstance(value, basestring):
return None
parse = parse_url(value, tag_optional=True)
if parse is None:
raise ValueError('Could not parse "%s" as a url' % value)
self._set_value(parse, 'block', self.set_block_id)
def init_block_ref_from_package_id(self, package_id):
if isinstance(package_id, CourseLocator):
package_id = package_id.package_id
assert package_id, "%s does not have a valid package_id"
parse = parse_package_id(package_id)
if parse is None:
raise ValueError('Could not parse "%s" as a package_id' % package_id)
self._set_value(parse, 'block', self.set_block_id)
@classmethod
def make_relative(cls, course_locator, block_id):
"""
Return a new instance which has the given block_id in the given course
:param course_locator: may be a BlockUsageLocator in the same snapshot
"""
return BlockUsageLocator(
package_id=course_locator.package_id,
version_guid=course_locator.version_guid,
branch=course_locator.branch,
block_id=block_id
)
def __unicode__(self):
"""
Return a string representing this location.
"""
rep = super(BlockUsageLocator, self).__unicode__()
return rep + '/' + BLOCK_PREFIX + unicode(self.block_id)
return re.sub('[^\w-]', '-', self._to_string())
class DefinitionLocator(Locator):
"""
Container for how to locate a description (the course-independent content).
"""
CANONICAL_NAMESPACE = 'defx'
KEY_FIELDS = ('definition_id',)
URL_RE = re.compile(r'^defx:' + VERSION_PREFIX + '([^/]+)$', re.IGNORECASE)
URL_RE = re.compile(r'^defx://' + VERSION_PREFIX + '([^/]+)$', re.IGNORECASE)
def __init__(self, definition_id):
if isinstance(definition_id, LocalId):
self.definition_id = definition_id
super(DefinitionLocator, self).__init__(definition_id)
elif isinstance(definition_id, basestring):
regex_match = self.URL_RE.match(definition_id)
if regex_match is not None:
self.definition_id = self.as_object_id(regex_match.group(1))
super(DefinitionLocator, self).__init__(self.as_object_id(regex_match.group(1)))
else:
self.definition_id = self.as_object_id(definition_id)
super(DefinitionLocator, self).__init__(self.as_object_id(definition_id))
else:
self.definition_id = self.as_object_id(definition_id)
super(DefinitionLocator, self).__init__(self.as_object_id(definition_id))
def __unicode__(self):
def _to_string(self):
'''
Return a string representing this location.
unicode(self) returns something like this: "version/519665f6223ebd6980884f2b"
@@ -545,9 +501,9 @@ class DefinitionLocator(Locator):
def url(self):
"""
Return a string containing the URL for this location.
url(self) returns something like this: 'defx://version/519665f6223ebd6980884f2b'
url(self) returns something like this: 'defx:version/519665f6223ebd6980884f2b'
"""
return u'defx://' + unicode(self)
return u'defx:' + self._to_string()
def version(self):
"""

299
common/lib/xmodule/xmodule/modulestore/mixed.py
View File

@@ -6,16 +6,18 @@ In this way, courses can be served up both - say - XMLModuleStore or MongoModule
"""
import logging
from uuid import uuid4
from opaque_keys import InvalidKeyError
from . import ModuleStoreWriteBase
from xmodule.modulestore.django import create_modulestore_instance, loc_mapper
from xmodule.modulestore import Location, SPLIT_MONGO_MODULESTORE_TYPE, XML_MODULESTORE_TYPE
from xmodule.modulestore.locator import CourseLocator, Locator
from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError
from uuid import uuid4
from xmodule.modulestore import Location, XML_MODULESTORE_TYPE
from xmodule.modulestore.locator import CourseLocator, Locator, BlockUsageLocator
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.keys import CourseKey, UsageKey
from xmodule.modulestore.mongo.base import MongoModuleStore
from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore
from xmodule.exceptions import UndefinedContext
from xmodule.modulestore.locations import SlashSeparatedCourseKey
log = logging.getLogger(__name__)
@@ -32,7 +34,13 @@ class MixedModuleStore(ModuleStoreWriteBase):
super(MixedModuleStore, self).__init__(**kwargs)
self.modulestores = {}
self.mappings = mappings
self.mappings = {}
for course_id, store_name in mappings.iteritems():
try:
self.mappings[CourseKey.from_string(course_id)] = store_name
except InvalidKeyError:
self.mappings[SlashSeparatedCourseKey.from_deprecated_string(course_id)] = store_name
if 'default' not in stores:
raise Exception('Missing a default modulestore in the MixedModuleStore __init__ method.')
@@ -43,7 +51,7 @@ class MixedModuleStore(ModuleStoreWriteBase):
# restrict xml to only load courses in mapping
store['OPTIONS']['course_ids'] = [
course_id
for course_id, store_key in self.mappings.iteritems()
for course_id, store_key in mappings.iteritems()
if store_key == key
]
self.modulestores[key] = create_modulestore_instance(
@@ -53,10 +61,6 @@ class MixedModuleStore(ModuleStoreWriteBase):
store['OPTIONS'],
i18n_service=i18n_service,
)
# If and when locations can identify their course, we won't need
# these loc maps. They're needed for figuring out which store owns these locations.
if is_xml:
self.ensure_loc_maps_exist(key)
def _get_modulestore_for_courseid(self, course_id):
"""
@@ -69,54 +73,50 @@ class MixedModuleStore(ModuleStoreWriteBase):
mapping = self.mappings.get(course_id, 'default')
return self.modulestores[mapping]
def has_item(self, course_id, reference):
def has_item(self, usage_key):
"""
Does the course include the xblock who's id is reference?
:param course_id: a course_id or package_id (slashed or dotted)
:param reference: a Location or BlockUsageLocator
"""
store = self._get_modulestore_for_courseid(course_id)
return store.has_item(course_id, reference)
store = self._get_modulestore_for_courseid(usage_key.course_key)
return store.has_item(usage_key)
def get_item(self, location, depth=0):
def get_item(self, usage_key, depth=0):
"""
This method is explicitly not implemented as we need a course_id to disambiguate
We should be able to fix this when the data-model rearchitecting is done
"""
# Although we shouldn't have both get_item and get_instance imho
raise NotImplementedError
store = self._get_modulestore_for_courseid(usage_key.course_key)
return store.get_item(usage_key, depth)
def get_instance(self, course_id, location, depth=0):
store = self._get_modulestore_for_courseid(course_id)
return store.get_instance(course_id, location, depth)
def get_items(self, location, course_id=None, depth=0, qualifiers=None):
def get_items(self, course_key, settings=None, content=None, **kwargs):
"""
Returns a list of XModuleDescriptor instances for the items
that match location. Any element of location that is None is treated
as a wildcard that matches any value. NOTE: don't use this to look for courses
as the course_id is required. Use get_courses.
Returns:
list of XModuleDescriptor instances for the matching items within the course with
the given course_key
location: either a Location possibly w/ None as wildcards for category or name or
a Locator with at least a package_id and branch but possibly no block_id.
NOTE: don't use this to look for courses
as the course_key is required. Use get_courses.
depth: An argument that some module stores may use to prefetch
descendants of the queried modules for more efficient results later
in the request. The depth is counted in the number of calls to
get_children() to cache. None indicates to cache all descendants
Args:
course_key (CourseKey): the course identifier
settings (dict): fields to look for which have settings scope. Follows same syntax
and rules as kwargs below
content (dict): fields to look for which have content scope. Follows same syntax and
rules as kwargs below.
kwargs (key=value): what to look for within the course.
Common qualifiers are ``category`` or any field name. if the target field is a list,
then it searches for the given value in the list not list equivalence.
Substring matching pass a regex object.
For some modulestores, ``name`` is another commonly provided key (Location based stores)
For some modulestores,
you can search by ``edited_by``, ``edited_on`` providing either a datetime for == (probably
useless) or a function accepting one arg to do inequality
"""
if not (course_id or hasattr(location, 'package_id')):
raise Exception("Must pass in a course_id when calling get_items()")
if not isinstance(course_key, CourseKey):
raise Exception("Must pass in a course_key when calling get_items()")
store = self._get_modulestore_for_courseid(course_id or getattr(location, 'package_id'))
return store.get_items(location, course_id, depth, qualifiers)
def _get_course_id_from_course_location(self, course_location):
"""
Get the proper course_id based on the type of course_location
"""
return getattr(course_location, 'course_id', None) or getattr(course_location, 'package_id', None)
store = self._get_modulestore_for_courseid(course_key)
return store.get_items(course_key, settings, content, **kwargs)
def get_courses(self):
'''
@@ -141,7 +141,8 @@ class MixedModuleStore(ModuleStoreWriteBase):
try:
# if there's no existing mapping, then the course can't have been in split
course_locator = loc_mapper().translate_location(
course.location.course_id, course.location, add_entry_if_missing=False
course.location,
add_entry_if_missing=False
)
if unicode(course_locator) not in courses:
courses[course_location] = course
@@ -152,27 +153,47 @@ class MixedModuleStore(ModuleStoreWriteBase):
return courses.values()
def get_course(self, course_id):
def get_course(self, course_key, depth=None):
"""
returns the course module associated with the course_id. If no such course exists,
it returns None
:param course_id: must be either a string course_id or a CourseLocator
:param course_key: must be a CourseKey
"""
store = self._get_modulestore_for_courseid(
course_id.package_id if hasattr(course_id, 'package_id') else course_id
)
assert(isinstance(course_key, CourseKey))
store = self._get_modulestore_for_courseid(course_key)
try:
return store.get_course(course_id)
return store.get_course(course_key, depth=depth)
except ItemNotFoundError:
return None
def get_parent_locations(self, location, course_id):
def has_course(self, course_id, ignore_case=False):
"""
returns the parent locations for a given location and course_id
returns whether the course exists
Args:
* course_id (CourseKey)
* ignore_case (bool): Tf True, do a case insensitive search. If
False, do a case sensitive search
"""
assert(isinstance(course_id, CourseKey))
store = self._get_modulestore_for_courseid(course_id)
return store.get_parent_locations(location, course_id)
return store.has_course(course_id, ignore_case)
def delete_course(self, course_key, user_id=None):
"""
Remove the given course from its modulestore.
"""
assert(isinstance(course_key, CourseKey))
store = self._get_modulestore_for_courseid(course_key)
return store.delete_course(course_key, user_id)
def get_parent_locations(self, location):
"""
returns the parent locations for a given location
"""
store = self._get_modulestore_for_courseid(location.course_key)
return store.get_parent_locations(location)
def get_modulestore_type(self, course_id):
"""
@@ -184,15 +205,14 @@ class MixedModuleStore(ModuleStoreWriteBase):
"""
return self._get_modulestore_for_courseid(course_id).get_modulestore_type(course_id)
def get_orphans(self, course_location, branch):
def get_orphans(self, course_key):
"""
Get all of the xblocks in the given course which have no parents and are not of types which are
usually orphaned. NOTE: may include xblocks which still have references via xblocks which don't
use children to point to their dependents.
"""
course_id = self._get_course_id_from_course_location(course_location)
store = self._get_modulestore_for_courseid(course_id)
return store.get_orphans(course_location, branch)
store = self._get_modulestore_for_courseid(course_key)
return store.get_orphans(course_key)
def get_errored_courses(self):
"""
@@ -204,106 +224,26 @@ class MixedModuleStore(ModuleStoreWriteBase):
errs.update(store.get_errored_courses())
return errs
def _get_course_id_from_block(self, block, store):
"""
Get the course_id from the block or from asking its store. Expensive.
"""
try:
return block.course_id
except UndefinedContext:
pass
try:
course = store._get_course_for_item(block.scope_ids.usage_id)
if course is not None:
return course.scope_ids.usage_id.course_id
except Exception: # sorry, that method just raises vanilla Exception if it doesn't find course
pass
def _infer_course_id_try(self, location):
"""
Create, Update, Delete operations don't require a fully-specified course_id, but
there's no complete & sound general way to compute the course_id except via the
proper modulestore. This method attempts several sound but not complete methods.
:param location: an old style Location
"""
if isinstance(location, CourseLocator):
return location.package_id
elif isinstance(location, basestring):
try:
location = Location(location)
except InvalidLocationError:
# try to parse as a course_id
try:
Location.parse_course_id(location)
# it's already a course_id
return location
except ValueError:
# cannot interpret the location
return None
# location is a Location at this point
if location.category == 'course': # easiest case
return location.course_id
# try finding in loc_mapper
try:
# see if the loc mapper knows the course id (requires double translation)
locator = loc_mapper().translate_location_to_course_locator(None, location)
location = loc_mapper().translate_locator_to_location(locator, get_course=True)
return location.course_id
except ItemNotFoundError:
pass
# expensive query against all location-based modulestores to look for location.
for store in self.modulestores.itervalues():
if isinstance(location, store.reference_type):
try:
xblock = store.get_item(location)
course_id = self._get_course_id_from_block(xblock, store)
if course_id is not None:
return course_id
except NotImplementedError:
blocks = store.get_items(location)
if len(blocks) == 1:
block = blocks[0]
try:
return block.course_id
except UndefinedContext:
pass
except ItemNotFoundError:
pass
# if we get here, it must be in a Locator based store, but we won't be able to find
# it.
return None
def create_course(self, course_id, user_id=None, store_name='default', **kwargs):
def create_course(self, org, offering, user_id=None, fields=None, store_name='default', **kwargs):
"""
Creates and returns the course.
:param org: the org
:param fields: a dict of xblock field name - value pairs for the course module.
:param metadata: the old way of setting fields by knowing which ones are scope.settings v scope.content
:param definition_data: the complement to metadata which is also a subset of fields
:returns: course xblock
Args:
org (str): the organization that owns the course
offering (str): the name of the course offering
user_id: id of the user creating the course
fields (dict): Fields to set on the course at initialization
store_name (str): the name of the modulestore that we will create this course within
kwargs: Any optional arguments understood by a subset of modulestores to customize instantiation
Returns: a CourseDescriptor
"""
store = self.modulestores[store_name]
if not hasattr(store, 'create_course'):
raise NotImplementedError(u"Cannot create a course on store %s" % store_name)
if store.get_modulestore_type(course_id) == SPLIT_MONGO_MODULESTORE_TYPE:
try:
course_dict = Location.parse_course_id(course_id)
org = course_dict['org']
course_id = "{org}.{course}.{name}".format(**course_dict)
except ValueError:
org = None
org = kwargs.pop('org', org)
fields = kwargs.pop('fields', {})
fields.update(kwargs.pop('metadata', {}))
fields.update(kwargs.pop('definition_data', {}))
course = store.create_course(course_id, org, user_id, fields=fields, **kwargs)
else: # assume mongo
course = store.create_course(course_id, **kwargs)
return course
return store.create_course(org, offering, user_id, fields, **kwargs)
def create_item(self, course_or_parent_loc, category, user_id=None, **kwargs):
"""
@@ -311,46 +251,30 @@ class MixedModuleStore(ModuleStoreWriteBase):
it installs the new item as a child of the parent (if the parent_loc is a specific
xblock reference).
:param course_or_parent_loc: Can be a course_id (org/course/run), CourseLocator,
Location, or BlockUsageLocator but must be what the persistence modulestore expects
:param course_or_parent_loc: Can be a CourseKey or UsageKey
:param category (str): The block_type of the item we are creating
"""
# find the store for the course
course_id = self._infer_course_id_try(course_or_parent_loc)
if course_id is None:
raise ItemNotFoundError(u"Cannot find modulestore for %s" % course_or_parent_loc)
course_id = getattr(course_or_parent_loc, 'course_key', course_or_parent_loc)
store = self._get_modulestore_for_courseid(course_id)
location = kwargs.pop('location', None)
# invoke its create_item
if isinstance(store, MongoModuleStore):
block_id = kwargs.pop('block_id', getattr(location, 'name', uuid4().hex))
# convert parent loc if it's legit
if isinstance(course_or_parent_loc, basestring):
parent_loc = None
if location is None:
loc_dict = Location.parse_course_id(course_id)
loc_dict['name'] = block_id
location = Location(category=category, **loc_dict)
else:
parent_loc = course_or_parent_loc
# must have a legitimate location, compute if appropriate
if location is None:
location = parent_loc.replace(category=category, name=block_id)
parent_loc = course_or_parent_loc if isinstance(course_or_parent_loc, UsageKey) else None
# must have a legitimate location, compute if appropriate
if location is None:
location = course_id.make_usage_key(category, block_id)
# do the actual creation
xblock = store.create_and_save_xmodule(location, **kwargs)
# don't forget to attach to parent
if parent_loc is not None and not 'detached' in xblock._class_tags:
parent = store.get_item(parent_loc)
parent.children.append(location.url())
parent.children.append(location)
store.update_item(parent)
elif isinstance(store, SplitMongoModuleStore):
if isinstance(course_or_parent_loc, basestring): # course_id
course_or_parent_loc = loc_mapper().translate_location_to_course_locator(
# hardcode draft version until we figure out how we're handling branches from app
course_or_parent_loc, None, published=False
)
elif not isinstance(course_or_parent_loc, CourseLocator):
if not isinstance(course_or_parent_loc, (CourseLocator, BlockUsageLocator)):
raise ValueError(u"Cannot create a child of {} in split. Wrong repr.".format(course_or_parent_loc))
# split handles all the fields in one dict not separated by scope
@@ -370,9 +294,7 @@ class MixedModuleStore(ModuleStoreWriteBase):
Update the xblock persisted to be the same as the given for all types of fields
(content, children, and metadata) attribute the change to the given user.
"""
course_id = self._infer_course_id_try(xblock.scope_ids.usage_id)
if course_id is None:
raise ItemNotFoundError(u"Cannot find modulestore for %s" % xblock.scope_ids.usage_id)
course_id = xblock.scope_ids.usage_id.course_key
store = self._get_modulestore_for_courseid(course_id)
return store.update_item(xblock, user_id)
@@ -380,9 +302,7 @@ class MixedModuleStore(ModuleStoreWriteBase):
"""
Delete the given item from persistence. kwargs allow modulestore specific parameters.
"""
course_id = self._infer_course_id_try(location)
if course_id is None:
raise ItemNotFoundError(u"Cannot find modulestore for %s" % location)
course_id = location.course_key
store = self._get_modulestore_for_courseid(course_id)
return store.delete_item(location, user_id=user_id, **kwargs)
@@ -396,21 +316,6 @@ class MixedModuleStore(ModuleStoreWriteBase):
elif hasattr(mstore, 'db'):
mstore.db.connection.close()
def ensure_loc_maps_exist(self, store_name):
"""
Ensure location maps exist for every course in the modulestore whose
name is the given name (mostly used for 'xml'). It creates maps for any
missing ones.
NOTE: will only work if the given store is Location based. If it's not,
it raises NotImplementedError
"""
store = self.modulestores[store_name]
if store.reference_type != Location:
raise ValueError(u"Cannot create maps from %s" % store.reference_type)
for course in store.get_courses():
loc_mapper().translate_location(course.location.course_id, course.location)
def get_courses_for_wiki(self, wiki_slug):
"""
Return the list of courses which use this wiki_slug

537
common/lib/xmodule/xmodule/modulestore/mongo/base.py
View File

@@ -8,7 +8,7 @@ structure:
'_id': <location.as_dict>,
'metadata': <dict containing all Scope.settings fields>
'definition': <dict containing all Scope.content fields>
'definition.children': <list of all child location.url()s>
'definition.children': <list of all child location.to_deprecated_string()s>
}
"""
@@ -16,26 +16,27 @@ import pymongo
import sys
import logging
import copy
import re
from bson.son import SON
from fs.osfs import OSFS
from itertools import repeat
from path import path
from importlib import import_module
from xmodule.errortracker import null_error_tracker, exc_info_to_str
from xmodule.mako_module import MakoDescriptorSystem
from xmodule.error_module import ErrorDescriptor
from xmodule.html_module import AboutDescriptor
from xblock.runtime import KvsFieldData
from xblock.exceptions import InvalidScopeError
from xblock.fields import Scope, ScopeIds
from xblock.fields import Scope, ScopeIds, Reference, ReferenceList, ReferenceValueDict
from xmodule.modulestore import ModuleStoreWriteBase, Location, MONGO_MODULESTORE_TYPE
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError
from xmodule.modulestore.inheritance import own_metadata, InheritanceMixin, inherit_metadata, InheritanceKeyValueStore
from xmodule.modulestore.xml import LocationReader
from xmodule.tabs import StaticTab, CourseTabList
from xblock.core import XBlock
from xmodule.modulestore.locations import SlashSeparatedCourseKey
log = logging.getLogger(__name__)
@@ -52,6 +53,7 @@ class InvalidWriteError(Exception):
Raised to indicate that writing to a particular key
in the KeyValueStore is disabled
"""
pass
class MongoKeyValueStore(InheritanceKeyValueStore):
@@ -120,7 +122,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
TODO (cdodge) when the 'split module store' work has been completed we can remove all
references to metadata_inheritance_tree
"""
def __init__(self, modulestore, module_data, default_class, cached_metadata, **kwargs):
def __init__(self, modulestore, course_key, module_data, default_class, cached_metadata, **kwargs):
"""
modulestore: the module store that can be used to retrieve additional modules
@@ -138,7 +140,6 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
MakoDescriptorSystem
"""
super(CachingDescriptorSystem, self).__init__(
id_reader=LocationReader(),
field_data=None,
load_item=self.load_item,
**kwargs
@@ -149,14 +150,14 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
self.default_class = default_class
# cdodge: other Systems have a course_id attribute defined. To keep things consistent, let's
# define an attribute here as well, even though it's None
self.course_id = None
self.course_id = course_key
self.cached_metadata = cached_metadata
def load_item(self, location):
"""
Return an XModule instance for the specified location
"""
location = Location(location)
assert isinstance(location, Location)
json_data = self.module_data.get(location)
if json_data is None:
module = self.modulestore.get_item(location)
@@ -170,6 +171,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
category = json_data['location']['category']
class_ = self.load_block_type(category)
definition = json_data.get('definition', {})
metadata = json_data.get('metadata', {})
for old_name, new_name in getattr(class_, 'metadata_translations', {}).items():
@@ -177,9 +179,19 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
metadata[new_name] = metadata[old_name]
del metadata[old_name]
children = [
location.course_key.make_usage_key_from_deprecated_string(childloc)
for childloc in definition.get('children', [])
]
data = definition.get('data', {})
if isinstance(data, basestring):
data = {'data': data}
mixed_class = self.mixologist.mix(class_)
data = self._convert_reference_fields_to_keys(mixed_class, location.course_key, data)
metadata = self._convert_reference_fields_to_keys(mixed_class, location.course_key, metadata)
kvs = MongoKeyValueStore(
definition.get('data', {}),
definition.get('children', []),
data,
children,
metadata,
)
@@ -193,7 +205,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
# Convert the serialized fields values in self.cached_metadata
# to python values
metadata_to_inherit = self.cached_metadata.get(non_draft_loc.url(), {})
metadata_to_inherit = self.cached_metadata.get(non_draft_loc.to_deprecated_string(), {})
inherit_metadata(module, metadata_to_inherit)
# decache any computed pending field settings
module.save()
@@ -203,31 +215,58 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
return ErrorDescriptor.from_json(
json_data,
self,
json_data['location'],
location,
error_msg=exc_info_to_str(sys.exc_info())
)
def _convert_reference_fields_to_keys(self, class_, course_key, jsonfields):
"""
Find all fields of type reference and convert the payload into UsageKeys
:param class_: the XBlock class
:param course_key: a CourseKey object for the given course
:param jsonfields: a dict of the jsonified version of the fields
"""
for field_name, value in jsonfields.iteritems():
if value:
field = class_.fields.get(field_name)
if field is None:
continue
elif isinstance(field, Reference):
jsonfields[field_name] = course_key.make_usage_key_from_deprecated_string(value)
elif isinstance(field, ReferenceList):
jsonfields[field_name] = [
course_key.make_usage_key_from_deprecated_string(ele) for ele in value
]
elif isinstance(field, ReferenceValueDict):
for key, subvalue in value.iteritems():
assert isinstance(subvalue, basestring)
value[key] = course_key.make_usage_key_from_deprecated_string(subvalue)
return jsonfields
def namedtuple_to_son(namedtuple, prefix=''):
def location_to_son(location, prefix='', tag='i4x'):
"""
Converts a namedtuple into a SON object with the same key order
Converts a location into a SON object with the same key order
"""
son = SON()
# pylint: disable=protected-access
for idx, field_name in enumerate(namedtuple._fields):
son[prefix + field_name] = namedtuple[idx]
son = SON({prefix + 'tag': tag})
for field_name in location.KEY_FIELDS:
# Filter the run, because the existing data doesn't have it stored
if field_name != 'run':
son[prefix + field_name] = getattr(location, field_name)
return son
def location_to_query(location, wildcard=True):
# The only thing using this w/ wildcards is contentstore.mongo for asset retrieval
def location_to_query(location, wildcard=True, tag='i4x'):
"""
Takes a Location and returns a SON object that will query for that location.
Takes a Location and returns a SON object that will query for that location by subfields
rather than subdoc. Note: location_to_son is faster in mongo as it does subdoc equivalence.
Fields in location that are None are ignored in the query
If `wildcard` is True, then a None in a location is treated as a wildcard
query. Otherwise, it is searched for literally
"""
query = namedtuple_to_son(Location(location), prefix='_id.')
query = location_to_son(location, prefix='_id.', tag=tag)
if wildcard:
for key, value in query.items():
@@ -239,11 +278,6 @@ def location_to_query(location, wildcard=True):
return query
def metadata_cache_key(location):
"""Turn a `Location` into a useful cache key."""
return u"{0.org}/{0.course}".format(location)
class MongoModuleStore(ModuleStoreWriteBase):
"""
A Mongodb backed ModuleStore
@@ -275,6 +309,8 @@ class MongoModuleStore(ModuleStoreWriteBase):
host=host,
port=port,
tz_aware=tz_aware,
# deserialize dicts as SONs
document_class=SON,
**kwargs
),
db
@@ -289,14 +325,6 @@ class MongoModuleStore(ModuleStoreWriteBase):
# Force mongo to report errors, at the expense of performance
self.collection.write_concern = {'w': 1}
# Force mongo to maintain an index over _id.* that is in the same order
# that is used when querying by a location
# pylint: disable=no-member, protected_access
self.collection.ensure_index(
zip(('_id.' + field for field in Location._fields), repeat(1)),
)
# pylint: enable=no-member, protected_access
if default_class is not None:
module_path, _, class_name = default_class.rpartition('.')
class_ = getattr(import_module(module_path), class_name)
@@ -308,20 +336,24 @@ class MongoModuleStore(ModuleStoreWriteBase):
self.render_template = render_template
self.i18n_service = i18n_service
self.ignore_write_events_on_courses = []
self.ignore_write_events_on_courses = set()
def compute_metadata_inheritance_tree(self, location):
def _compute_metadata_inheritance_tree(self, course_id):
'''
TODO (cdodge) This method can be deleted when the 'split module store' work has been completed
'''
# get all collections in the course, this query should not return any leaf nodes
# note this is a bit ugly as when we add new categories of containers, we have to add it here
block_types_with_children = set(name for name, class_ in XBlock.load_classes() if getattr(class_, 'has_children', False))
query = {'_id.org': location.org,
'_id.course': location.course,
'_id.category': {'$in': list(block_types_with_children)}
}
block_types_with_children = set(
name for name, class_ in XBlock.load_classes() if getattr(class_, 'has_children', False)
)
query = SON([
('_id.tag', 'i4x'),
('_id.org', course_id.org),
('_id.course', course_id.course),
('_id.category', {'$in': list(block_types_with_children)})
])
# we just want the Location, children, and inheritable metadata
record_filter = {'_id': 1, 'definition.children': 1}
@@ -333,24 +365,26 @@ class MongoModuleStore(ModuleStoreWriteBase):
# call out to the DB
resultset = self.collection.find(query, record_filter)
# it's ok to keep these as urls b/c the overall cache is indexed by course_key and this
# is a dictionary relative to that course
results_by_url = {}
root = None
# now go through the results and order them by the location url
for result in resultset:
location = Location(result['_id'])
# We need to collate between draft and non-draft
# i.e. draft verticals will have draft children but will have non-draft parents currently
location = location.replace(revision=None)
location_url = location.url()
# manually pick it apart b/c the db has tag and we want revision = None regardless
location = Location._from_deprecated_son(result['_id'], course_id.run).replace(revision=None)
location_url = location.to_deprecated_string()
if location_url in results_by_url:
# found either draft or live to complement the other revision
existing_children = results_by_url[location_url].get('definition', {}).get('children', [])
additional_children = result.get('definition', {}).get('children', [])
total_children = existing_children + additional_children
results_by_url[location_url].setdefault('definition', {})['children'] = total_children
results_by_url[location.url()] = result
results_by_url[location_url] = result
if location.category == 'course':
root = location.url()
root = location_url
# now traverse the tree and compute down the inherited metadata
metadata_to_inherit = {}
@@ -379,31 +413,30 @@ class MongoModuleStore(ModuleStoreWriteBase):
return metadata_to_inherit
def get_cached_metadata_inheritance_tree(self, location, force_refresh=False):
def _get_cached_metadata_inheritance_tree(self, course_id, force_refresh=False):
'''
TODO (cdodge) This method can be deleted when the 'split module store' work has been completed
'''
key = metadata_cache_key(location)
tree = {}
if not force_refresh:
# see if we are first in the request cache (if present)
if self.request_cache is not None and key in self.request_cache.data.get('metadata_inheritance', {}):
return self.request_cache.data['metadata_inheritance'][key]
if self.request_cache is not None and course_id in self.request_cache.data.get('metadata_inheritance', {}):
return self.request_cache.data['metadata_inheritance'][course_id]
# then look in any caching subsystem (e.g. memcached)
if self.metadata_inheritance_cache_subsystem is not None:
tree = self.metadata_inheritance_cache_subsystem.get(key, {})
tree = self.metadata_inheritance_cache_subsystem.get(course_id, {})
else:
logging.warning('Running MongoModuleStore without a metadata_inheritance_cache_subsystem. This is OK in localdev and testing environment. Not OK in production.')
if not tree:
# if not in subsystem, or we are on force refresh, then we have to compute
tree = self.compute_metadata_inheritance_tree(location)
tree = self._compute_metadata_inheritance_tree(course_id)
# now write out computed tree to caching subsystem (e.g. memcached), if available
if self.metadata_inheritance_cache_subsystem is not None:
self.metadata_inheritance_cache_subsystem.set(key, tree)
self.metadata_inheritance_cache_subsystem.set(course_id, tree)
# now populate a request_cache, if available. NOTE, we are outside of the
# scope of the above if: statement so that after a memcache hit, it'll get
@@ -413,18 +446,22 @@ class MongoModuleStore(ModuleStoreWriteBase):
# defined
if 'metadata_inheritance' not in self.request_cache.data:
self.request_cache.data['metadata_inheritance'] = {}
self.request_cache.data['metadata_inheritance'][key] = tree
self.request_cache.data['metadata_inheritance'][course_id] = tree
return tree
def refresh_cached_metadata_inheritance_tree(self, location):
def refresh_cached_metadata_inheritance_tree(self, course_id, runtime=None):
"""
Refresh the cached metadata inheritance tree for the org/course combination
for location
If given a runtime, it replaces the cached_metadata in that runtime. NOTE: failure to provide
a runtime may mean that some objects report old values for inherited data.
"""
pseudo_course_id = '/'.join([location.org, location.course])
if pseudo_course_id not in self.ignore_write_events_on_courses:
self.get_cached_metadata_inheritance_tree(location, force_refresh=True)
if course_id not in self.ignore_write_events_on_courses:
cached_metadata = self._get_cached_metadata_inheritance_tree(course_id, force_refresh=True)
if runtime:
runtime.cached_metadata = cached_metadata
def _clean_item_data(self, item):
"""
@@ -433,17 +470,19 @@ class MongoModuleStore(ModuleStoreWriteBase):
item['location'] = item['_id']
del item['_id']
def _query_children_for_cache_children(self, items):
def _query_children_for_cache_children(self, course_key, items):
"""
Generate a pymongo in query for finding the items and return the payloads
"""
# first get non-draft in a round-trip
query = {
'_id': {'$in': [namedtuple_to_son(Location(item)) for item in items]}
'_id': {'$in': [
location_to_son(course_key.make_usage_key_from_deprecated_string(item)) for item in items
]}
}
return list(self.collection.find(query))
def _cache_children(self, items, depth=0):
def _cache_children(self, course_key, items, depth=0):
"""
Returns a dictionary mapping Location -> item data, populated with json data
for all descendents of items up to the specified depth.
@@ -459,7 +498,7 @@ class MongoModuleStore(ModuleStoreWriteBase):
for item in to_process:
self._clean_item_data(item)
children.extend(item.get('definition', {}).get('children', []))
data[Location(item['location'])] = item
data[Location._from_deprecated_son(item['location'], course_key.run)] = item
if depth == 0:
break
@@ -469,7 +508,7 @@ class MongoModuleStore(ModuleStoreWriteBase):
# for or-query syntax
to_process = []
if children:
to_process = self._query_children_for_cache_children(children)
to_process = self._query_children_for_cache_children(course_key, children)
# If depth is None, then we just recurse until we hit all the descendents
if depth is not None:
@@ -477,11 +516,11 @@ class MongoModuleStore(ModuleStoreWriteBase):
return data
def _load_item(self, item, data_cache, apply_cached_metadata=True):
def _load_item(self, course_key, item, data_cache, apply_cached_metadata=True):
"""
Load an XModuleDescriptor from item, using the children stored in data_cache
"""
location = Location(item['location'])
location = Location._from_deprecated_son(item['location'], course_key.run)
data_dir = getattr(item, 'data_dir', location.course)
root = self.fs_root / data_dir
@@ -491,16 +530,15 @@ class MongoModuleStore(ModuleStoreWriteBase):
cached_metadata = {}
if apply_cached_metadata:
cached_metadata = self.get_cached_metadata_inheritance_tree(location)
cached_metadata = self._get_cached_metadata_inheritance_tree(course_key)
services = {}
if self.i18n_service:
services["i18n"] = self.i18n_service
# TODO (cdodge): When the 'split module store' work has been completed, we should remove
# the 'metadata_inheritance_tree' parameter
system = CachingDescriptorSystem(
modulestore=self,
course_key=course_key,
module_data=data_cache,
default_class=self.default_class,
resources_fs=resource_fs,
@@ -513,70 +551,95 @@ class MongoModuleStore(ModuleStoreWriteBase):
)
return system.load_item(location)
def _load_items(self, items, depth=0):
def _load_items(self, course_key, items, depth=0):
"""
Load a list of xmodules from the data in items, with children cached up
to specified depth
"""
data_cache = self._cache_children(items, depth)
data_cache = self._cache_children(course_key, items, depth)
# if we are loading a course object, if we're not prefetching children (depth != 0) then don't
# bother with the metadata inheritance
return [self._load_item(item, data_cache,
apply_cached_metadata=(item['location']['category'] != 'course' or depth != 0)) for item in items]
return [
self._load_item(
course_key, item, data_cache,
apply_cached_metadata=(item['location']['category'] != 'course' or depth != 0)
)
for item in items
]
def get_courses(self):
'''
Returns a list of course descriptors.
'''
course_filter = Location(category="course")
return [
course
for course
in self.get_items(course_filter)
if not (
course.location.org == 'edx' and
course.location.course == 'templates'
)
]
return sum(
[
self._load_items(
SlashSeparatedCourseKey(course['_id']['org'], course['_id']['course'], course['_id']['name']),
[course]
)
for course
# I tried to add '$and': [{'_id.org': {'$ne': 'edx'}}, {'_id.course': {'$ne': 'templates'}}]
# but it didn't do the right thing (it filtered all edx and all templates out)
in self.collection.find({'_id.category': 'course'})
if not ( # TODO kill this
course['_id']['org'] == 'edx' and
course['_id']['course'] == 'templates'
)
],
[]
)
def _find_one(self, location):
'''Look for a given location in the collection. If revision is not
specified, returns the latest. If the item is not present, raise
ItemNotFoundError.
'''
assert isinstance(location, Location)
item = self.collection.find_one(
location_to_query(location, wildcard=False),
{'_id': location_to_son(location)},
sort=[('revision', pymongo.ASCENDING)],
)
if item is None:
raise ItemNotFoundError(location)
return item
def get_course(self, course_id):
def get_course(self, course_key, depth=None):
"""
Get the course with the given courseid (org/course/run)
"""
id_components = Location.parse_course_id(course_id)
id_components['tag'] = 'i4x'
id_components['category'] = 'course'
assert(isinstance(course_key, SlashSeparatedCourseKey))
location = course_key.make_usage_key('course', course_key.run)
try:
return self.get_item(Location(id_components))
return self.get_item(location, depth=depth)
except ItemNotFoundError:
return None
def has_item(self, course_id, location):
def has_course(self, course_key, ignore_case=False):
"""
Is the given course in this modulestore
If ignore_case is True, do a case insensitive search,
otherwise, do a case sensitive search
"""
assert(isinstance(course_key, SlashSeparatedCourseKey))
course_query = self._course_key_to_son(course_key)
if ignore_case:
for key in course_query.iterkeys():
course_query[key] = re.compile(r"(?i)^{}$".format(course_query[key]))
return self.collection.find_one(course_query, fields={'_id': True}) is not None
def has_item(self, usage_key):
"""
Returns True if location exists in this ModuleStore.
"""
location = Location.ensure_fully_specified(location)
try:
self._find_one(location)
self._find_one(usage_key)
return True
except ItemNotFoundError:
return False
def get_item(self, location, depth=0):
def get_item(self, usage_key, depth=0):
"""
Returns an XModuleDescriptor instance for the item at location.
@@ -585,52 +648,135 @@ class MongoModuleStore(ModuleStoreWriteBase):
If no object is found at that location, raises
xmodule.modulestore.exceptions.ItemNotFoundError
location: a Location object
usage_key: a :class:`.UsageKey` instance
depth (int): An argument that some module stores may use to prefetch
descendents of the queried modules for more efficient results later
in the request. The depth is counted in the number of
calls to get_children() to cache. None indicates to cache all descendents.
"""
location = Location.ensure_fully_specified(location)
item = self._find_one(location)
module = self._load_items([item], depth)[0]
item = self._find_one(usage_key)
module = self._load_items(usage_key.course_key, [item], depth)[0]
return module
def get_instance(self, course_id, location, depth=0):
@staticmethod
def _course_key_to_son(course_id, tag='i4x'):
"""
TODO (vshnayder): implement policy tracking in mongo.
For now, just delegate to get_item and ignore policy.
depth (int): An argument that some module stores may use to prefetch
descendents of the queried modules for more efficient results later
in the request. The depth is counted in the number of
calls to get_children() to cache. None indicates to cache all descendents.
Generate the partial key to look up items relative to a given course
"""
return self.get_item(location, depth=depth)
return SON([
('_id.tag', tag),
('_id.org', course_id.org),
('_id.course', course_id.course),
])
def get_items(self, location, course_id=None, depth=0, qualifiers=None):
def get_items(self, course_id, settings=None, content=None, revision=None, **kwargs):
"""
Returns:
list of XModuleDescriptor instances for the matching items within the course with
the given course_id
NOTE: don't use this to look for courses
as the course_id is required. Use get_courses which is a lot faster anyway.
If you don't provide a value for revision, this limits the result to only ones in the
published course. Call this method on draft mongo store if you want to include drafts.
Args:
course_id (CourseKey): the course identifier
settings (dict): fields to look for which have settings scope. Follows same syntax
and rules as kwargs below
content (dict): fields to look for which have content scope. Follows same syntax and
rules as kwargs below.
revision (str): the revision of the items you're looking for.
kwargs (key=value): what to look for within the course.
Common qualifiers are ``category`` or any field name. if the target field is a list,
then it searches for the given value in the list not list equivalence.
Substring matching pass a regex object.
For this modulestore, ``name`` is a commonly provided key (Location based stores)
This modulestore does not allow searching dates by comparison or edited_by, previous_version,
update_version info.
"""
query = self._course_key_to_son(course_id)
query['_id.revision'] = revision
for field in ['category', 'name']:
if field in kwargs:
query['_id.' + field] = kwargs.pop(field)
for key, value in (settings or {}).iteritems():
query['metadata.' + key] = value
for key, value in (content or {}).iteritems():
query['definition.data.' + key] = value
if 'children' in kwargs:
query['definition.children'] = kwargs.pop('children')
query.update(kwargs)
items = self.collection.find(
location_to_query(location),
sort=[('revision', pymongo.ASCENDING)],
query,
sort=[('_id.revision', pymongo.ASCENDING)],
)
modules = self._load_items(list(items), depth)
modules = self._load_items(course_id, list(items))
return modules
def create_course(self, course_id, definition_data=None, metadata=None, runtime=None):
def create_course(self, org, offering, user_id=None, fields=None, **kwargs):
"""
Create a course with the given course_id.
Creates and returns the course.
Args:
org (str): the organization that owns the course
offering (str): the name of the course offering
user_id: id of the user creating the course
fields (dict): Fields to set on the course at initialization
kwargs: Any optional arguments understood by a subset of modulestores to customize instantiation
Returns: a CourseDescriptor
Raises:
InvalidLocationError: If a course with the same org and offering already exists
"""
if isinstance(course_id, Location):
location = course_id
if location.category != 'course':
raise ValueError(u"Course roots must be of category 'course': {}".format(unicode(location)))
else:
course_dict = Location.parse_course_id(course_id)
course_dict['category'] = 'course'
course_dict['tag'] = 'i4x'
location = Location(course_dict)
return self.create_and_save_xmodule(location, definition_data, metadata, runtime)
course, _, run = offering.partition('/')
course_id = SlashSeparatedCourseKey(org, course, run)
# Check if a course with this org/course has been defined before (case-insensitive)
course_search_location = SON([
('_id.tag', 'i4x'),
('_id.org', re.compile(u'^{}$'.format(course_id.org), re.IGNORECASE)),
('_id.course', re.compile(u'^{}$'.format(course_id.course), re.IGNORECASE)),
('_id.category', 'course'),
])
courses = self.collection.find(course_search_location, fields=('_id'))
if courses.count() > 0:
raise InvalidLocationError(
"There are already courses with the given org and course id: {}".format([
course['_id'] for course in courses
]))
location = course_id.make_usage_key('course', course_id.run)
course = self.create_and_save_xmodule(location, fields=fields, **kwargs)
# clone a default 'about' overview module as well
about_location = location.replace(
category='about',
name='overview'
)
overview_template = AboutDescriptor.get_template('overview.yaml')
self.create_and_save_xmodule(
about_location,
system=course.system,
definition_data=overview_template.get('data')
)
return course
def delete_course(self, course_key, user_id=None):
"""
The impl removes all of the db records for the course.
:param course_key:
:param user_id:
"""
course_query = self._course_key_to_son(course_key)
self.collection.remove(course_query, multi=True)
def create_xmodule(self, location, definition_data=None, metadata=None, system=None, fields={}):
"""
@@ -641,8 +787,6 @@ class MongoModuleStore(ModuleStoreWriteBase):
:param metadata: can be empty, the initial metadata for the kvs
:param system: if you already have an xblock from the course, the xblock.runtime value
"""
if not isinstance(location, Location):
location = Location(location)
# differs from split mongo in that I believe most of this logic should be above the persistence
# layer but added it here to enable quick conversion. I'll need to reconcile these.
if metadata is None:
@@ -659,6 +803,7 @@ class MongoModuleStore(ModuleStoreWriteBase):
system = CachingDescriptorSystem(
modulestore=self,
module_data={},
course_key=location.course_key,
default_class=self.default_class,
resources_fs=None,
error_tracker=self.error_tracker,
@@ -678,8 +823,9 @@ class MongoModuleStore(ModuleStoreWriteBase):
ScopeIds(None, location.category, location, location),
dbmodel,
)
for key, value in fields.iteritems():
setattr(xmodule, key, value)
if fields is not None:
for key, value in fields.iteritems():
setattr(xmodule, key, value)
# decache any pending field settings from init
xmodule.save()
return xmodule
@@ -700,7 +846,7 @@ class MongoModuleStore(ModuleStoreWriteBase):
# differs from split mongo in that I believe most of this logic should be above the persistence
# layer but added it here to enable quick conversion. I'll need to reconcile these.
new_object = self.create_xmodule(location, definition_data, metadata, system, fields)
location = new_object.location
location = new_object.scope_ids.usage_id
self.update_item(new_object, allow_not_found=True)
# VS[compat] cdodge: This is a hack because static_tabs also have references from the course module, so
@@ -712,47 +858,20 @@ class MongoModuleStore(ModuleStoreWriteBase):
course.tabs.append(
StaticTab(
name=new_object.display_name,
url_slug=new_object.location.name,
url_slug=new_object.scope_ids.usage_id.name,
)
)
self.update_item(course)
return new_object
def fire_updated_modulestore_signal(self, course_id, location):
"""
Send a signal using `self.modulestore_update_signal`, if that has been set
"""
if self.modulestore_update_signal is not None:
self.modulestore_update_signal.send(self, modulestore=self, course_id=course_id,
location=location)
def _get_course_for_item(self, location, depth=0):
'''
VS[compat]
cdodge: for a given Xmodule, return the course that it belongs to
NOTE: This makes a lot of assumptions about the format of the course location
for a given Xmodule, return the course that it belongs to
Also we have to assert that this module maps to only one course item - it'll throw an
assert if not
This is only used to support static_tabs as we need to be course module aware
'''
# @hack! We need to find the course location however, we don't
# know the 'name' parameter in this context, so we have
# to assume there's only one item in this query even though we are not specifying a name
course_search_location = Location('i4x', location.org, location.course, 'course', None)
courses = self.get_items(course_search_location, depth=depth)
# make sure we found exactly one match on this above course search
found_cnt = len(courses)
if found_cnt == 0:
raise Exception('Could not find course at {0}'.format(course_search_location))
if found_cnt > 1:
raise Exception('Found more than one course at {0}. There should only be one!!! '
'Dump = {1}'.format(course_search_location, courses))
return courses[0]
return self.get_course(location.course_key, depth)
def _update_single_item(self, location, update):
"""
@@ -763,7 +882,7 @@ class MongoModuleStore(ModuleStoreWriteBase):
# See http://www.mongodb.org/display/DOCS/Updating for
# atomic update syntax
result = self.collection.update(
{'_id': namedtuple_to_son(Location(location))},
{'_id': location_to_son(location)},
{'$set': update},
multi=False,
upsert=True,
@@ -774,50 +893,70 @@ class MongoModuleStore(ModuleStoreWriteBase):
if result['n'] == 0:
raise ItemNotFoundError(location)
def update_item(self, xblock, user=None, allow_not_found=False):
def update_item(self, xblock, user_id=None, allow_not_found=False, force=False):
"""
Update the persisted version of xblock to reflect its current values.
location: Something that can be passed to Location
data: A nested dictionary of problem data
xblock: which xblock to persist
user_id: who made the change (ignored for now by this modulestore)
allow_not_found: whether to create a new object if one didn't already exist or give an error
force: force is meaningless for this modulestore
"""
try:
definition_data = xblock.get_explicitly_set_fields_by_scope()
definition_data = self._convert_reference_fields_to_strings(xblock, xblock.get_explicitly_set_fields_by_scope())
payload = {
'definition.data': definition_data,
'metadata': own_metadata(xblock),
'metadata': self._convert_reference_fields_to_strings(xblock, own_metadata(xblock)),
}
if xblock.has_children:
# convert all to urls
xblock.children = [child.url() if isinstance(child, Location) else child
for child in xblock.children]
payload.update({'definition.children': xblock.children})
self._update_single_item(xblock.location, payload)
children = self._convert_reference_fields_to_strings(xblock, {'children': xblock.children})
payload.update({'definition.children': children['children']})
self._update_single_item(xblock.scope_ids.usage_id, payload)
# for static tabs, their containing course also records their display name
if xblock.category == 'static_tab':
course = self._get_course_for_item(xblock.location)
if xblock.scope_ids.block_type == 'static_tab':
course = self._get_course_for_item(xblock.scope_ids.usage_id)
# find the course's reference to this tab and update the name.
static_tab = CourseTabList.get_tab_by_slug(course.tabs, xblock.location.name)
static_tab = CourseTabList.get_tab_by_slug(course.tabs, xblock.scope_ids.usage_id.name)
# only update if changed
if static_tab and static_tab['name'] != xblock.display_name:
static_tab['name'] = xblock.display_name
self.update_item(course, user)
self.update_item(course, user_id)
# recompute (and update) the metadata inheritance tree which is cached
# was conditional on children or metadata having changed before dhm made one update to rule them all
self.refresh_cached_metadata_inheritance_tree(xblock.location)
self.refresh_cached_metadata_inheritance_tree(xblock.scope_ids.usage_id.course_key, xblock.runtime)
# fire signal that we've written to DB
self.fire_updated_modulestore_signal(get_course_id_no_run(xblock.location), xblock.location)
except ItemNotFoundError:
if not allow_not_found:
raise
def _convert_reference_fields_to_strings(self, xblock, jsonfields):
"""
Find all fields of type reference and convert the payload from UsageKeys to deprecated strings
:param xblock: the XBlock class
:param jsonfields: a dict of the jsonified version of the fields
"""
assert isinstance(jsonfields, dict)
for field_name, value in jsonfields.iteritems():
if value:
if isinstance(xblock.fields[field_name], Reference):
jsonfields[field_name] = value.to_deprecated_string()
elif isinstance(xblock.fields[field_name], ReferenceList):
jsonfields[field_name] = [
ele.to_deprecated_string() for ele in value
]
elif isinstance(xblock.fields[field_name], ReferenceValueDict):
for key, subvalue in value.iteritems():
assert isinstance(subvalue, Location)
value[key] = subvalue.to_deprecated_string()
return jsonfields
# pylint: disable=unused-argument
def delete_item(self, location, **kwargs):
"""
Delete an item from this modulestore
Delete an item from this modulestore.
location: Something that can be passed to Location
Args:
location (UsageKey)
"""
# pylint: enable=unused-argument
# VS[compat] cdodge: This is a hack because static_tabs also have references from the course module, so
@@ -825,26 +964,28 @@ class MongoModuleStore(ModuleStoreWriteBase):
# we should remove this once we can break this reference from the course to static tabs
if location.category == 'static_tab':
item = self.get_item(location)
course = self._get_course_for_item(item.location)
course = self._get_course_for_item(item.scope_ids.usage_id)
existing_tabs = course.tabs or []
course.tabs = [tab for tab in existing_tabs if tab.get('url_slug') != location.name]
self.update_item(course, '**replace_user**')
# Must include this to avoid the django debug toolbar (which defines the deprecated "safe=False")
# from overriding our default value set in the init method.
self.collection.remove({'_id': Location(location).dict()}, safe=self.collection.safe)
self.collection.remove({'_id': location_to_son(location)}, safe=self.collection.safe)
# recompute (and update) the metadata inheritance tree which is cached
self.refresh_cached_metadata_inheritance_tree(Location(location))
self.fire_updated_modulestore_signal(get_course_id_no_run(Location(location)), Location(location))
self.refresh_cached_metadata_inheritance_tree(location.course_key)
def get_parent_locations(self, location, course_id):
def get_parent_locations(self, location):
'''Find all locations that are the parents of this location in this
course. Needed for path_to_location().
'''
location = Location.ensure_fully_specified(location)
items = self.collection.find({'definition.children': location.url()},
{'_id': True})
return [Location(i['_id']) for i in items]
query = self._course_key_to_son(location.course_key)
query['definition.children'] = location.to_deprecated_string()
items = self.collection.find(query, {'_id': True})
return [
location.course_key.make_usage_key(i['_id']['category'], i['_id']['name'])
for i in items
]
def get_modulestore_type(self, course_id):
"""
@@ -856,21 +997,22 @@ class MongoModuleStore(ModuleStoreWriteBase):
"""
return MONGO_MODULESTORE_TYPE
def get_orphans(self, course_location, _branch):
def get_orphans(self, course_key):
"""
Return an array all of the locations for orphans in the course.
Return an array all of the locations (deprecated string format) for orphans in the course.
"""
detached_categories = [name for name, __ in XBlock.load_tagged_classes("detached")]
all_items = self.collection.find({
'_id.org': course_location.org,
'_id.course': course_location.course,
'_id.category': {'$nin': detached_categories}
})
query = self._course_key_to_son(course_key)
query['_id.category'] = {'$nin': detached_categories}
all_items = self.collection.find(query)
all_reachable = set()
item_locs = set()
for item in all_items:
if item['_id']['category'] != 'course':
item_locs.add(Location(item['_id']).replace(revision=None).url())
# It would be nice to change this method to return UsageKeys instead of the deprecated string.
item_locs.add(
Location._from_deprecated_son(item['_id'], course_key.run).replace(revision=None).to_deprecated_string()
)
all_reachable = all_reachable.union(item.get('definition', {}).get('children', []))
item_locs -= all_reachable
return list(item_locs)
@@ -882,7 +1024,8 @@ class MongoModuleStore(ModuleStoreWriteBase):
:return: list of course locations
"""
courses = self.collection.find({'definition.data.wiki_slug': wiki_slug})
return [Location(course['_id']) for course in courses]
# the course's run == its name. It's the only xblock for which that's necessarily true.
return [Location._from_deprecated_son(course['_id'], course['_id']['name']) for course in courses]
def _create_new_field_data(self, _category, _location, definition_data, metadata):
"""

138
common/lib/xmodule/xmodule/modulestore/mongo/draft.py
View File

@@ -7,14 +7,14 @@ and otherwise returns i4x://org/course/cat/name).
"""
from datetime import datetime
from xmodule.exceptions import InvalidVersionError
from xmodule.modulestore import Location
from xmodule.modulestore.exceptions import ItemNotFoundError, DuplicateItemError
from xmodule.modulestore.mongo.base import location_to_query, namedtuple_to_son, get_course_id_no_run, MongoModuleStore
import pymongo
from pytz import UTC
from xmodule.exceptions import InvalidVersionError
from xmodule.modulestore.exceptions import ItemNotFoundError, DuplicateItemError
from xmodule.modulestore.mongo.base import MongoModuleStore
from xmodule.modulestore.locations import Location
DRAFT = 'draft'
# Things w/ these categories should never be marked as version='draft'
DIRECT_ONLY_CATEGORIES = ['course', 'chapter', 'sequential', 'about', 'static_tab', 'course_info']
@@ -24,14 +24,14 @@ def as_draft(location):
"""
Returns the Location that is the draft for `location`
"""
return Location(location).replace(revision=DRAFT)
return location.replace(revision=DRAFT)
def as_published(location):
"""
Returns the Location that is the published version for `location`
"""
return Location(location).replace(revision=None)
return location.replace(revision=None)
def wrap_draft(item):
@@ -56,19 +56,19 @@ class DraftModuleStore(MongoModuleStore):
their children) to published modules.
"""
def get_item(self, location, depth=0):
def get_item(self, usage_key, depth=0):
"""
Returns an XModuleDescriptor instance for the item at location.
If location.revision is None, returns the item with the most
Returns an XModuleDescriptor instance for the item at usage_key.
If usage_key.revision is None, returns the item with the most
recent revision
If any segment of the location is None except revision, raises
If any segment of the usage_key is None except revision, raises
xmodule.modulestore.exceptions.InsufficientSpecificationError
If no object is found at that location, raises
If no object is found at that usage_key, raises
xmodule.modulestore.exceptions.ItemNotFoundError
location: Something that can be passed to Location
usage_key: A :class:`.UsageKey` instance
depth (int): An argument that some module stores may use to prefetch
descendents of the queried modules for more efficient results later
@@ -77,20 +77,9 @@ class DraftModuleStore(MongoModuleStore):
"""
try:
return wrap_draft(super(DraftModuleStore, self).get_item(as_draft(location), depth=depth))
return wrap_draft(super(DraftModuleStore, self).get_item(as_draft(usage_key), depth=depth))
except ItemNotFoundError:
return wrap_draft(super(DraftModuleStore, self).get_item(location, depth=depth))
def get_instance(self, course_id, location, depth=0):
"""
Get an instance of this location, with policy for course_id applied.
TODO (vshnayder): this may want to live outside the modulestore eventually
"""
try:
return wrap_draft(super(DraftModuleStore, self).get_instance(course_id, as_draft(location), depth=depth))
except ItemNotFoundError:
return wrap_draft(super(DraftModuleStore, self).get_instance(course_id, location, depth=depth))
return wrap_draft(super(DraftModuleStore, self).get_item(usage_key, depth=depth))
def create_xmodule(self, location, definition_data=None, metadata=None, system=None, fields={}):
"""
@@ -101,37 +90,42 @@ class DraftModuleStore(MongoModuleStore):
:param metadata: can be empty, the initial metadata for the kvs
:param system: if you already have an xmodule from the course, the xmodule.system value
"""
draft_loc = as_draft(location)
if draft_loc.category in DIRECT_ONLY_CATEGORIES:
if location.category in DIRECT_ONLY_CATEGORIES:
raise InvalidVersionError(location)
draft_loc = as_draft(location)
return super(DraftModuleStore, self).create_xmodule(draft_loc, definition_data, metadata, system, fields)
def get_items(self, location, course_id=None, depth=0, qualifiers=None):
def get_items(self, course_key, settings=None, content=None, **kwargs):
"""
Returns a list of XModuleDescriptor instances for the items
that match location. Any element of location that is None is treated
as a wildcard that matches any value
Returns:
list of XModuleDescriptor instances for the matching items within the course with
the given course_key
location: Something that can be passed to Location
NOTE: don't use this to look for courses
as the course_key is required. Use get_courses.
depth: An argument that some module stores may use to prefetch
descendents of the queried modules for more efficient results later
in the request. The depth is counted in the number of calls to
get_children() to cache. None indicates to cache all descendents
Args:
course_key (CourseKey): the course identifier
settings: not used
content: not used
kwargs (key=value): what to look for within the course.
Common qualifiers are ``category`` or any field name. if the target field is a list,
then it searches for the given value in the list not list equivalence.
Substring matching pass a regex object.
``name`` is another commonly provided key (Location based stores)
"""
draft_loc = as_draft(location)
draft_items = super(DraftModuleStore, self).get_items(draft_loc, course_id=course_id, depth=depth)
items = super(DraftModuleStore, self).get_items(location, course_id=course_id, depth=depth)
draft_locs_found = set(item.location.replace(revision=None) for item in draft_items)
non_draft_items = [
item
for item in items
if (item.location.revision != DRAFT
and item.location.replace(revision=None) not in draft_locs_found)
draft_items = [
wrap_draft(item) for item in
super(DraftModuleStore, self).get_items(course_key, revision='draft', **kwargs)
]
return [wrap_draft(item) for item in draft_items + non_draft_items]
draft_items_locations = {item.location for item in draft_items}
non_draft_items = [
item for item in
super(DraftModuleStore, self).get_items(course_key, revision=None, **kwargs)
# filter out items that are not already in draft
if item.location not in draft_items_locations
]
return draft_items + non_draft_items
def convert_to_draft(self, source_location):
"""
@@ -139,40 +133,38 @@ class DraftModuleStore(MongoModuleStore):
:param source: the location of the source (its revision must be None)
"""
original = self.collection.find_one(location_to_query(source_location))
original = self.collection.find_one({'_id': source_location.to_deprecated_son()})
draft_location = as_draft(source_location)
if draft_location.category in DIRECT_ONLY_CATEGORIES:
raise InvalidVersionError(source_location)
if not original:
raise ItemNotFoundError(source_location)
original['_id'] = namedtuple_to_son(draft_location)
original['_id'] = draft_location.to_deprecated_son()
try:
self.collection.insert(original)
except pymongo.errors.DuplicateKeyError:
raise DuplicateItemError(original['_id'])
self.refresh_cached_metadata_inheritance_tree(draft_location)
self.fire_updated_modulestore_signal(get_course_id_no_run(draft_location), draft_location)
self.refresh_cached_metadata_inheritance_tree(draft_location.course_key)
return self._load_items([original])[0]
return self._load_items(source_location.course_key, [original])[0]
def update_item(self, xblock, user=None, allow_not_found=False):
def update_item(self, xblock, user_id=None, allow_not_found=False, force=False):
"""
Save the current values to persisted version of the xblock
location: Something that can be passed to Location
data: A nested dictionary of problem data
See superclass doc.
In addition to the superclass's behavior, this method converts the unit to draft if it's not
already draft.
"""
draft_loc = as_draft(xblock.location)
try:
if not self.has_item(None, draft_loc):
if not self.has_item(draft_loc):
self.convert_to_draft(xblock.location)
except ItemNotFoundError:
if not allow_not_found:
raise
xblock.location = draft_loc
super(DraftModuleStore, self).update_item(xblock, user, allow_not_found)
super(DraftModuleStore, self).update_item(xblock, user_id, allow_not_found)
# don't allow locations to truly represent themselves as draft outside of this file
xblock.location = as_published(xblock.location)
@@ -188,14 +180,6 @@ class DraftModuleStore(MongoModuleStore):
return
def get_parent_locations(self, location, course_id):
'''Find all locations that are the parents of this location. Needed
for path_to_location().
returns an iterable of things that can be passed to Location.
'''
return super(DraftModuleStore, self).get_parent_locations(location, course_id)
def publish(self, location, published_by_id):
"""
Save a current draft to the underlying modulestore
@@ -216,8 +200,8 @@ class DraftModuleStore(MongoModuleStore):
# 2) child moved
for child in original_published.children:
if child not in draft.children:
rents = [Location(mom) for mom in self.get_parent_locations(child, None)]
if (len(rents) == 1 and rents[0] == Location(location)): # the 1 is this original_published
rents = self.get_parent_locations(child)
if (len(rents) == 1 and rents[0] == location): # the 1 is this original_published
self.delete_item(child, True)
super(DraftModuleStore, self).update_item(draft, '**replace_user**')
self.delete_item(location)
@@ -229,17 +213,19 @@ class DraftModuleStore(MongoModuleStore):
self.convert_to_draft(location)
super(DraftModuleStore, self).delete_item(location)
def _query_children_for_cache_children(self, items):
def _query_children_for_cache_children(self, course_key, items):
# first get non-draft in a round-trip
to_process_non_drafts = super(DraftModuleStore, self)._query_children_for_cache_children(items)
to_process_non_drafts = super(DraftModuleStore, self)._query_children_for_cache_children(course_key, items)
to_process_dict = {}
for non_draft in to_process_non_drafts:
to_process_dict[Location(non_draft["_id"])] = non_draft
to_process_dict[Location._from_deprecated_son(non_draft["_id"], course_key.run)] = non_draft
# now query all draft content in another round-trip
query = {
'_id': {'$in': [namedtuple_to_son(as_draft(Location(item))) for item in items]}
'_id': {'$in': [
as_draft(course_key.make_usage_key_from_deprecated_string(item)).to_deprecated_son() for item in items
]}
}
to_process_drafts = list(self.collection.find(query))
@@ -247,7 +233,7 @@ class DraftModuleStore(MongoModuleStore):
# with the draft. This is because the semantics of the DraftStore is to
# always return the draft - if available
for draft in to_process_drafts:
draft_loc = Location(draft["_id"])
draft_loc = Location._from_deprecated_son(draft["_id"], course_key.run)
draft_as_non_draft_loc = draft_loc.replace(revision=None)
# does non-draft exist in the collection

79
common/lib/xmodule/xmodule/modulestore/parsers.py
View File

@@ -7,15 +7,15 @@ BLOCK_PREFIX = r"block/"
# Prefix for the version portion of a locator URL, when it is preceded by a course ID
VERSION_PREFIX = r"version/"
ALLOWED_ID_CHARS = r'[\w\-~.:]'
ALLOWED_ID_CHARS = r'[\w\-~.:+]'
ALLOWED_ID_RE = re.compile(r'^{}+$'.format(ALLOWED_ID_CHARS), re.UNICODE)
# NOTE: if we need to support period in place of +, make it aggressive (take the first period in the string)
URL_RE_SOURCE = r"""
(?P<tag>edx://)?
((?P<package_id>{ALLOWED_ID_CHARS}+)/?)?
((?P<org>{ALLOWED_ID_CHARS}+)\+(?P<offering>{ALLOWED_ID_CHARS}+)/?)?
({BRANCH_PREFIX}(?P<branch>{ALLOWED_ID_CHARS}+)/?)?
({VERSION_PREFIX}(?P<version_guid>[A-F0-9]+)/?)?
({BLOCK_PREFIX}(?P<block>{ALLOWED_ID_CHARS}+))?
({BLOCK_PREFIX}(?P<block_id>{ALLOWED_ID_CHARS}+))?
""".format(
ALLOWED_ID_CHARS=ALLOWED_ID_CHARS, BRANCH_PREFIX=BRANCH_PREFIX,
VERSION_PREFIX=VERSION_PREFIX, BLOCK_PREFIX=BLOCK_PREFIX
@@ -24,40 +24,33 @@ URL_RE_SOURCE = r"""
URL_RE = re.compile('^' + URL_RE_SOURCE + '$', re.IGNORECASE | re.VERBOSE | re.UNICODE)
def parse_url(string, tag_optional=False):
def parse_url(string):
"""
A url usually begins with 'edx://' (case-insensitive match),
followed by either a version_guid or a package_id. If tag_optional, then
followed by either a version_guid or a org + offering pair. If tag_optional, then
the url does not have to start with the tag and edx will be assumed.
Examples:
'edx://version/0123FFFF'
'edx://mit.eecs.6002x'
'edx://mit.eecs.6002x/branch/published'
'edx://mit.eecs.6002x/branch/published/block/HW3'
'edx://mit.eecs.6002x/branch/published/version/000eee12345/block/HW3'
'edx:version/0123FFFF'
'edx:mit.eecs.6002x'
'edx:mit.eecs.6002x/branch/published'
'edx:mit.eecs.6002x/branch/published/block/HW3'
'edx:mit.eecs.6002x/branch/published/version/000eee12345/block/HW3'
This returns None if string cannot be parsed.
If it can be parsed as a version_guid with no preceding package_id, returns a dict
If it can be parsed as a version_guid with no preceding org + offering, returns a dict
with key 'version_guid' and the value,
If it can be parsed as a package_id, returns a dict
If it can be parsed as a org + offering, returns a dict
with key 'id' and optional keys 'branch' and 'version_guid'.
"""
match = URL_RE.match(string)
if not match:
return None
matched_dict = match.groupdict()
if matched_dict['tag'] is None and not tag_optional:
return None
return matched_dict
BLOCK_RE = re.compile(r'^' + ALLOWED_ID_CHARS + r'+$', re.IGNORECASE | re.UNICODE)
def parse_block_ref(string):
r"""
A block_ref is a string of url safe characters (see ALLOWED_ID_CHARS)
@@ -65,46 +58,6 @@ def parse_block_ref(string):
If string is a block_ref, returns a dict with key 'block_ref' and the value,
otherwise returns None.
"""
if len(string) > 0 and BLOCK_RE.match(string):
return {'block': string}
if ALLOWED_ID_RE.match(string):
return {'block_id': string}
return None
def parse_package_id(string):
r"""
A package_id has a main id component.
There may also be an optional branch (/branch/published or /branch/draft).
There may also be an optional version (/version/519665f6223ebd6980884f2b).
There may also be an optional block (/block/HW3 or /block/Quiz2).
Examples of valid package_ids:
'mit.eecs.6002x'
'mit.eecs.6002x/branch/published'
'mit.eecs.6002x/block/HW3'
'mit.eecs.6002x/branch/published/block/HW3'
'mit.eecs.6002x/branch/published/version/519665f6223ebd6980884f2b/block/HW3'
Syntax:
package_id = main_id [/branch/ branch] [/version/ version ] [/block/ block]
main_id = name [. name]*
branch = name
block = name
name = ALLOWED_ID_CHARS
If string is a package_id, returns a dict with keys 'id', 'branch', and 'block'.
Revision is optional: if missing returned_dict['branch'] is None.
Block is optional: if missing returned_dict['block'] is None.
Else returns None.
"""
match = URL_RE.match(string)
if not match:
return None
return match.groupdict()

65
common/lib/xmodule/xmodule/modulestore/search.py
View File

@@ -1,34 +1,28 @@
from itertools import repeat
from xmodule.course_module import CourseDescriptor
from .exceptions import (ItemNotFoundError, NoPathToItem)
from . import Location
def path_to_location(modulestore, course_id, location):
def path_to_location(modulestore, usage_key):
'''
Try to find a course_id/chapter/section[/position] path to location in
modulestore. The courseware insists that the first level in the course is
chapter, but any kind of module can be a "section".
location: something that can be passed to Location
course_id: Search for paths in this course.
Args:
modulestore: which store holds the relevant objects
usage_key: :class:`UsageKey` the id of the location to which to generate the path
raise ItemNotFoundError if the location doesn't exist.
Raises
ItemNotFoundError if the location doesn't exist.
NoPathToItem if the location exists, but isn't accessible via
a chapter/section path in the course(s) being searched.
raise NoPathToItem if the location exists, but isn't accessible via
a chapter/section path in the course(s) being searched.
Returns:
a tuple (course_id, chapter, section, position) suitable for the
courseware index view.
Return a tuple (course_id, chapter, section, position) suitable for the
courseware index view.
A location may be accessible via many paths. This method may
return any valid path.
If the section is a sequential or vertical, position will be the position
of this location in that sequence. Otherwise, position will
be None. TODO (vshnayder): Not true yet.
If the section is a sequential or vertical, position will be the children index
of this location under that sequence.
'''
def flatten(xs):
@@ -55,41 +49,38 @@ def path_to_location(modulestore, course_id, location):
# tuples (location, path-so-far). To avoid lots of
# copying, the path-so-far is stored as a lisp-style
# list--nested hd::tl tuples, and flattened at the end.
queue = [(location, ())]
queue = [(usage_key, ())]
while len(queue) > 0:
(loc, path) = queue.pop() # Takes from the end
loc = Location(loc)
(next_usage, path) = queue.pop() # Takes from the end
# get_parent_locations should raise ItemNotFoundError if location
# isn't found so we don't have to do it explicitly. Call this
# first to make sure the location is there (even if it's a course, and
# we would otherwise immediately exit).
parents = modulestore.get_parent_locations(loc, course_id)
parents = modulestore.get_parent_locations(next_usage)
# print 'Processing loc={0}, path={1}'.format(loc, path)
if loc.category == "course":
# confirm that this is the right course
if course_id == CourseDescriptor.location_to_id(loc):
# Found it!
path = (loc, path)
return flatten(path)
# print 'Processing loc={0}, path={1}'.format(next_usage, path)
if next_usage.definition_key.block_type == "course":
# Found it!
path = (next_usage, path)
return flatten(path)
# otherwise, add parent locations at the end
newpath = (loc, path)
newpath = (next_usage, path)
queue.extend(zip(parents, repeat(newpath)))
# If we're here, there is no path
return None
if not modulestore.has_item(course_id, location):
raise ItemNotFoundError
if not modulestore.has_item(usage_key):
raise ItemNotFoundError(usage_key)
path = find_path_to_course()
if path is None:
raise NoPathToItem(location)
raise NoPathToItem(usage_key)
n = len(path)
course_id = CourseDescriptor.location_to_id(path[0])
course_id = path[0].course_key
# pull out the location names
chapter = path[1].name if n > 1 else None
section = path[2].name if n > 2 else None
@@ -105,9 +96,9 @@ def path_to_location(modulestore, course_id, location):
if n > 3:
position_list = []
for path_index in range(2, n - 1):
category = path[path_index].category
category = path[path_index].definition_key.block_type
if category == 'sequential' or category == 'videosequence':
section_desc = modulestore.get_instance(course_id, path[path_index])
section_desc = modulestore.get_item(path[path_index])
child_locs = [c.location for c in section_desc.get_children()]
# positions are 1-indexed, and should be strings to be consistent with
# url parsing.

117
common/lib/xmodule/xmodule/modulestore/split_migrator.py
View File

@@ -6,9 +6,7 @@ Exists at the top level of modulestore b/c it needs to know about and access eac
In general, it's strategy is to treat the other modulestores as read-only and to never directly
manipulate storage but use existing api's.
'''
from xmodule.modulestore import Location
from xmodule.modulestore.locator import CourseLocator
from xmodule.modulestore.mongo import draft
from xblock.fields import Reference, ReferenceList, ReferenceValueDict
class SplitMigrator(object):
@@ -23,65 +21,64 @@ class SplitMigrator(object):
self.draft_modulestore = draft_modulestore
self.loc_mapper = loc_mapper
def migrate_mongo_course(self, course_location, user, new_package_id=None):
def migrate_mongo_course(self, course_key, user, new_org=None, new_offering=None):
"""
Create a new course in split_mongo representing the published and draft versions of the course from the
original mongo store. And return the new_package_id (which the caller can also get by calling
self.loc_mapper.translate_location(old_course_location)
original mongo store. And return the new CourseLocator
If the new course already exists, this raises DuplicateItemError
:param course_location: a Location whose category is 'course' and points to the course
:param user: the user whose action is causing this migration
:param new_package_id: (optional) the Locator.package_id for the new course. Defaults to
whatever translate_location_to_locator returns
:param new_org: (optional) the Locator.org for the new course. Defaults to
whatever translate_location_to_locator returns
:param new_offering: (optional) the Locator.offering for the new course. Defaults to
whatever translate_location_to_locator returns
"""
new_package_id = self.loc_mapper.create_map_entry(course_location, package_id=new_package_id)
old_course_id = course_location.course_id
new_course_locator = self.loc_mapper.create_map_entry(course_key, new_org, new_offering)
# the only difference in data between the old and split_mongo xblocks are the locations;
# so, any field which holds a location must change to a Locator; otherwise, the persistence
# layer and kvs's know how to store it.
# locations are in location, children, conditionals, course.tab
# create the course: set fields to explicitly_set for each scope, id_root = new_package_id, master_branch = 'production'
original_course = self.direct_modulestore.get_item(course_location)
new_course_root_locator = self.loc_mapper.translate_location(old_course_id, course_location)
# create the course: set fields to explicitly_set for each scope, id_root = new_course_locator, master_branch = 'production'
original_course = self.direct_modulestore.get_course(course_key)
new_course_root_locator = self.loc_mapper.translate_location(original_course.location)
new_course = self.split_modulestore.create_course(
new_package_id, course_location.org,
user.id,
fields=self._get_json_fields_translate_children(original_course, old_course_id, True),
new_course_root_locator.org, new_course_root_locator.offering, user.id,
fields=self._get_json_fields_translate_references(original_course, course_key, True),
root_block_id=new_course_root_locator.block_id,
master_branch=new_course_root_locator.branch
)
self._copy_published_modules_to_course(new_course, course_location, old_course_id, user)
self._add_draft_modules_to_course(new_package_id, old_course_id, course_location, user)
self._copy_published_modules_to_course(new_course, original_course.location, course_key, user)
self._add_draft_modules_to_course(new_course.id, course_key, user)
return new_package_id
return new_course_locator
def _copy_published_modules_to_course(self, new_course, old_course_loc, old_course_id, user):
def _copy_published_modules_to_course(self, new_course, old_course_loc, course_key, user):
"""
Copy all of the modules from the 'direct' version of the course to the new split course.
"""
course_version_locator = new_course.location.as_course_locator()
course_version_locator = new_course.id
# iterate over published course elements. Wildcarding rather than descending b/c some elements are orphaned (e.g.,
# course about pages, conditionals)
for module in self.direct_modulestore.get_items(
old_course_loc.replace(category=None, name=None, revision=None),
old_course_id
):
for module in self.direct_modulestore.get_items(course_key):
# don't copy the course again. No drafts should get here but check
if module.location != old_course_loc and not getattr(module, 'is_draft', False):
# create split_xblock using split.create_item
# where block_id is computed by translate_location_to_locator
new_locator = self.loc_mapper.translate_location(
old_course_id, module.location, True, add_entry_if_missing=True
module.location, True, add_entry_if_missing=True
)
# NOTE: the below auto populates the children when it migrates the parent; so,
# it doesn't need the parent as the first arg. That is, it translates and populates
# the 'children' field as it goes.
_new_module = self.split_modulestore.create_item(
course_version_locator, module.category, user.id,
block_id=new_locator.block_id,
fields=self._get_json_fields_translate_children(module, old_course_id, True),
fields=self._get_json_fields_translate_references(module, course_key, True),
continue_version=True
)
# after done w/ published items, add version for 'draft' pointing to the published structure
@@ -94,25 +91,22 @@ class SplitMigrator(object):
# children which meant some pointers were to non-existent locations in 'direct'
self.split_modulestore.internal_clean_children(course_version_locator)
def _add_draft_modules_to_course(self, new_package_id, old_course_id, old_course_loc, user):
def _add_draft_modules_to_course(self, published_course_key, course_key, user):
"""
update each draft. Create any which don't exist in published and attach to their parents.
"""
# each true update below will trigger a new version of the structure. We may want to just have one new version
# but that's for a later date.
new_draft_course_loc = CourseLocator(package_id=new_package_id, branch='draft')
new_draft_course_loc = published_course_key.for_branch('draft')
# to prevent race conditions of grandchilden being added before their parents and thus having no parent to
# add to
awaiting_adoption = {}
for module in self.draft_modulestore.get_items(
old_course_loc.replace(category=None, name=None, revision=draft.DRAFT),
old_course_id
):
for module in self.draft_modulestore.get_items(course_key):
if getattr(module, 'is_draft', False):
new_locator = self.loc_mapper.translate_location(
old_course_id, module.location, False, add_entry_if_missing=True
module.location, False, add_entry_if_missing=True
)
if self.split_modulestore.has_item(new_package_id, new_locator):
if self.split_modulestore.has_item(new_locator):
# was in 'direct' so draft is a new version
split_module = self.split_modulestore.get_item(new_locator)
# need to remove any no-longer-explicitly-set values and add/update any now set values.
@@ -131,25 +125,24 @@ class SplitMigrator(object):
_new_module = self.split_modulestore.create_item(
new_draft_course_loc, module.category, user.id,
block_id=new_locator.block_id,
fields=self._get_json_fields_translate_children(module, old_course_id, True)
fields=self._get_json_fields_translate_references(module, course_key, True)
)
awaiting_adoption[module.location] = new_locator.block_id
for draft_location, new_block_id in awaiting_adoption.iteritems():
for parent_loc in self.draft_modulestore.get_parent_locations(draft_location, old_course_id):
for parent_loc in self.draft_modulestore.get_parent_locations(draft_location):
old_parent = self.draft_modulestore.get_item(parent_loc)
new_parent = self.split_modulestore.get_item(
self.loc_mapper.translate_location(old_course_id, old_parent.location, False)
self.loc_mapper.translate_location(old_parent.location, False)
)
# this only occurs if the parent was also awaiting adoption
if new_block_id in new_parent.children:
break
# find index for module: new_parent may be missing quite a few of old_parent's children
new_parent_cursor = 0
draft_location = draft_location.url() # need as string
for old_child_loc in old_parent.children:
if old_child_loc == draft_location:
break
sibling_loc = self.loc_mapper.translate_location(old_course_id, Location(old_child_loc), False)
sibling_loc = self.loc_mapper.translate_location(old_child_loc, False)
# sibling may move cursor
for idx in range(new_parent_cursor, len(new_parent.children)):
if new_parent.children[idx] == sibling_loc.block_id:
@@ -158,24 +151,32 @@ class SplitMigrator(object):
new_parent.children.insert(new_parent_cursor, new_block_id)
new_parent = self.split_modulestore.update_item(new_parent, user.id)
def _get_json_fields_translate_children(self, xblock, old_course_id, published):
def _get_json_fields_translate_references(self, xblock, old_course_id, published):
"""
Return the json repr for explicitly set fields but convert all children to their block_id's
Return the json repr for explicitly set fields but convert all references to their block_id's
"""
fields = self.get_json_fields_explicitly_set(xblock)
# this will too generously copy the children even for ones that don't exist in the published b/c the old mongo
# had no way of not having parents point to draft only children :-(
if 'children' in fields:
fields['children'] = [
self.loc_mapper.translate_location(
old_course_id, Location(child), published, add_entry_if_missing=True
).block_id
for child in fields['children']]
return fields
# FIXME change split to take field values as pythonic values not json values
result = {}
for field_name, field in xblock.fields.iteritems():
if field.is_set_on(xblock):
if isinstance(field, Reference):
result[field_name] = unicode(self.loc_mapper.translate_location(
getattr(xblock, field_name), published, add_entry_if_missing=True
))
elif isinstance(field, ReferenceList):
result[field_name] = [
unicode(self.loc_mapper.translate_location(
ele, published, add_entry_if_missing=True
)) for ele in getattr(xblock, field_name)
]
elif isinstance(field, ReferenceValueDict):
result[field_name] = {
key: unicode(self.loc_mapper.translate_location(
subvalue, published, add_entry_if_missing=True
))
for key, subvalue in getattr(xblock, field_name).iteritems()
}
else:
result[field_name] = field.read_json(xblock)
def get_json_fields_explicitly_set(self, xblock):
"""
Get the json repr for fields set on this specific xblock
:param xblock:
"""
return {field.name: field.read_json(xblock) for field in xblock.fields.itervalues() if field.is_set_on(xblock)}
return result

55
common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py
View File

@@ -1,10 +1,10 @@
import sys
import logging
from xmodule.mako_module import MakoDescriptorSystem
from xmodule.modulestore.locator import BlockUsageLocator, LocalId
from xmodule.modulestore.locator import BlockUsageLocator, LocalId, CourseLocator
from xmodule.error_module import ErrorDescriptor
from xmodule.errortracker import exc_info_to_str
from xblock.runtime import KvsFieldData, IdReader
from xblock.runtime import KvsFieldData
from ..exceptions import ItemNotFoundError
from .split_mongo_kvs import SplitMongoKVS
from xblock.fields import ScopeIds
@@ -13,23 +13,6 @@ from xmodule.modulestore.loc_mapper_store import LocMapperStore
log = logging.getLogger(__name__)
class SplitMongoIdReader(IdReader):
"""
An :class:`~xblock.runtime.IdReader` associated with a particular
:class:`.CachingDescriptorSystem`.
"""
def __init__(self, system):
self.system = system
def get_definition_id(self, usage_id):
usage = self.system.load_item(usage_id)
return usage.definition_locator
def get_block_type(self, def_id):
definition = self.system.modulestore.db_connection.get_definition(def_id)
return definition['category']
class CachingDescriptorSystem(MakoDescriptorSystem):
"""
A system that has a cache of a course version's json that it will use to load modules
@@ -44,15 +27,14 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
modulestore: the module store that can be used to retrieve additional
modules
course_entry: the originally fetched enveloped course_structure w/ branch and package_id info.
course_entry: the originally fetched enveloped course_structure w/ branch and course id info.
Callers to _load_item provide an override but that function ignores the provided structure and
only looks at the branch and package_id
only looks at the branch and course id
module_data: a dict mapping Location -> json that was cached from the
underlying modulestore
"""
super(CachingDescriptorSystem, self).__init__(
id_reader=SplitMongoIdReader(self),
field_data=None,
load_item=self._load_item,
**kwargs
@@ -72,11 +54,14 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
self.local_modules = {}
def _load_item(self, block_id, course_entry_override=None):
if isinstance(block_id, BlockUsageLocator) and isinstance(block_id.block_id, LocalId):
try:
return self.local_modules[block_id]
except KeyError:
raise ItemNotFoundError
if isinstance(block_id, BlockUsageLocator):
if isinstance(block_id.block_id, LocalId):
try:
return self.local_modules[block_id]
except KeyError:
raise ItemNotFoundError
else:
block_id = block_id.block_id
json_data = self.module_data.get(block_id)
if json_data is None:
@@ -99,14 +84,15 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
# the thread is working with more than one named container pointing to the same specific structure is
# low; thus, the course_entry is most likely correct. If the thread is looking at > 1 named container
# pointing to the same structure, the access is likely to be chunky enough that the last known container
# is the intended one when not given a course_entry_override; thus, the caching of the last branch/package_id.
# is the intended one when not given a course_entry_override; thus, the caching of the last branch/course id.
def xblock_from_json(self, class_, block_id, json_data, course_entry_override=None):
if course_entry_override is None:
course_entry_override = self.course_entry
else:
# most recent retrieval is most likely the right one for next caller (see comment above fn)
self.course_entry['branch'] = course_entry_override['branch']
self.course_entry['package_id'] = course_entry_override['package_id']
self.course_entry['org'] = course_entry_override['org']
self.course_entry['offering'] = course_entry_override['offering']
# most likely a lazy loader or the id directly
definition = json_data.get('definition', {})
definition_id = self.modulestore.definition_locator(definition)
@@ -116,10 +102,13 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
block_id = LocalId()
block_locator = BlockUsageLocator(
version_guid=course_entry_override['structure']['_id'],
CourseLocator(
version_guid=course_entry_override['structure']['_id'],
org=course_entry_override.get('org'),
offering=course_entry_override.get('offering'),
branch=course_entry_override.get('branch'),
),
block_id=block_id,
package_id=course_entry_override.get('package_id'),
branch=course_entry_override.get('branch')
)
kvs = SplitMongoKVS(
@@ -141,7 +130,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
json_data,
self,
BlockUsageLocator(
version_guid=course_entry_override['structure']['_id'],
CourseLocator(version_guid=course_entry_override['structure']['_id']),
block_id=block_id
),
error_msg=exc_info_to_str(sys.exc_info())

24
common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py
View File

@@ -1,7 +1,9 @@
"""
Segregation of pymongo functions from the data modeling mechanisms for split modulestore.
"""
import re
import pymongo
from bson import son
class MongoConnection(object):
"""
@@ -18,6 +20,7 @@ class MongoConnection(object):
host=host,
port=port,
tz_aware=tz_aware,
document_class=son.SON,
**kwargs
),
db
@@ -63,11 +66,17 @@ class MongoConnection(object):
"""
self.structures.update({'_id': structure['_id']}, structure)
def get_course_index(self, key):
def get_course_index(self, key, ignore_case=False):
"""
Get the course_index from the persistence mechanism whose id is the given key
"""
return self.course_index.find_one({'_id': key})
case_regex = r"(?i)^{}$" if ignore_case else r"{}"
return self.course_index.find_one(
son.SON([
(key_attr, re.compile(case_regex.format(getattr(key, key_attr))))
for key_attr in ('org', 'offering')
])
)
def find_matching_course_indexes(self, query):
"""
@@ -86,13 +95,16 @@ class MongoConnection(object):
"""
Update the db record for course_index
"""
self.course_index.update({'_id': course_index['_id']}, course_index)
self.course_index.update(
son.SON([('org', course_index['org']), ('offering', course_index['offering'])]),
course_index
)
def delete_course_index(self, key):
def delete_course_index(self, course_index):
"""
Delete the course_index from the persistence mechanism whose id is the given key
Delete the course_index from the persistence mechanism whose id is the given course_index
"""
return self.course_index.remove({'_id': key})
return self.course_index.remove(son.SON([('org', course_index['org']), ('offering', course_index['offering'])]))
def get_definition(self, key):
"""

331
common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
View File

@@ -3,8 +3,9 @@ Provides full versioning CRUD and representation for collections of xblocks (e.g
Representation:
* course_index: a dictionary:
** '_id': package_id (e.g., myu.mydept.mycourse.myrun),
** '_id': a unique id which cannot change,
** 'org': the org's id. Only used for searching not identity,
** 'offering': the course's catalog number and run id or whatever user decides,
** 'edited_by': user_id of user who created the original entry,
** 'edited_on': the datetime of the original creation,
** 'versions': versions_dict: {branch_id: structure_id, ...}
@@ -47,7 +48,6 @@ Representation:
import threading
import datetime
import logging
import re
from importlib import import_module
from path import path
import copy
@@ -122,7 +122,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
self.thread_cache = threading.local()
if default_class is not None:
module_path, _, class_name = default_class.rpartition('.')
module_path, __, class_name = default_class.rpartition('.')
class_ = getattr(import_module(module_path), class_name)
self.default_class = class_
else:
@@ -235,7 +235,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
return the CourseDescriptor! It returns the actual db json from
structures.
Semantics: if package_id and branch given, then it will get that branch. If
Semantics: if course id and branch given, then it will get that branch. If
also give a version_guid, it will see if the current head of that branch == that guid. If not
it raises VersionConflictError (the version now differs from what it was when you got your
reference)
@@ -247,9 +247,9 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
if not course_locator.is_fully_specified():
raise InsufficientSpecificationError('Not fully specified: %s' % course_locator)
if course_locator.package_id is not None and course_locator.branch is not None:
# use the package_id
index = self.db_connection.get_course_index(course_locator.package_id)
if course_locator.org and course_locator.offering and course_locator.branch:
# use the course id
index = self.db_connection.get_course_index(course_locator)
if index is None:
raise ItemNotFoundError(course_locator)
if course_locator.branch not in index['versions']:
@@ -266,11 +266,12 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
version_guid = course_locator.as_object_id(version_guid)
entry = self.db_connection.get_structure(version_guid)
# b/c more than one course can use same structure, the 'package_id' and 'branch' are not intrinsic to structure
# b/c more than one course can use same structure, the 'org', 'offering', and 'branch' are not intrinsic to structure
# and the one assoc'd w/ it by another fetch may not be the one relevant to this fetch; so,
# add it in the envelope for the structure.
envelope = {
'package_id': course_locator.package_id,
'org': course_locator.org,
'offering': course_locator.offering,
'branch': course_locator.branch,
'structure': entry,
}
@@ -300,15 +301,17 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
for structure in matching:
version_guid = structure['versions'][branch]
version_guids.append(version_guid)
id_version_map[version_guid] = structure['_id']
id_version_map[version_guid] = structure
course_entries = self.db_connection.find_matching_structures({'_id': {'$in': version_guids}})
# get the block for the course element (s/b the root)
result = []
for entry in course_entries:
course_info = id_version_map[entry['_id']]
envelope = {
'package_id': id_version_map[entry['_id']],
'org': course_info['org'],
'offering': course_info['offering'],
'branch': branch,
'structure': entry,
}
@@ -316,42 +319,44 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
result.extend(self._load_items(envelope, [root], 0, lazy=True))
return result
def get_course(self, course_locator):
def get_course(self, course_id, depth=None):
'''
Gets the course descriptor for the course identified by the locator
which may or may not be a blockLocator.
raises InsufficientSpecificationError
'''
course_entry = self._lookup_course(course_locator)
assert(isinstance(course_id, CourseLocator))
course_entry = self._lookup_course(course_id)
root = course_entry['structure']['root']
result = self._load_items(course_entry, [root], 0, lazy=True)
return result[0]
def get_course_for_item(self, location):
def has_course(self, course_id, ignore_case=False):
'''
Provided for backward compatibility. Is equivalent to calling get_course
:param location:
Does this course exist in this modulestore.
'''
return self.get_course(location)
assert(isinstance(course_id, CourseLocator))
course_entry = self.db_connection.get_course_index(course_id, ignore_case)
return course_entry is not None
def has_item(self, package_id, block_location):
def has_item(self, usage_key):
"""
Returns True if location exists in its course. Returns false if
the course or the block w/in the course do not exist for the given version.
raises InsufficientSpecificationError if the locator does not id a block
"""
if block_location.block_id is None:
raise InsufficientSpecificationError(block_location)
if usage_key.block_id is None:
raise InsufficientSpecificationError(usage_key)
try:
course_structure = self._lookup_course(block_location)['structure']
course_structure = self._lookup_course(usage_key)['structure']
except ItemNotFoundError:
# this error only occurs if the course does not exist
return False
return self._get_block_from_structure(course_structure, block_location.block_id) is not None
return self._get_block_from_structure(course_structure, usage_key.block_id) is not None
def get_item(self, location, depth=0):
def get_item(self, usage_key, depth=0):
"""
depth (int): An argument that some module stores may use to prefetch
descendants of the queried modules for more efficient results later
@@ -361,52 +366,77 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
raises InsufficientSpecificationError or ItemNotFoundError
"""
# intended for temporary support of some pointers being old-style
if isinstance(location, Location):
if isinstance(usage_key, Location):
if self.loc_mapper is None:
raise InsufficientSpecificationError('No location mapper configured')
else:
location = self.loc_mapper.translate_location(
None, location, location.revision is None,
usage_key = self.loc_mapper.translate_location(
usage_key,
usage_key.revision is None,
add_entry_if_missing=False
)
assert isinstance(location, BlockUsageLocator)
if not location.is_initialized():
raise InsufficientSpecificationError("Not yet initialized: %s" % location)
course = self._lookup_course(location)
items = self._load_items(course, [location.block_id], depth, lazy=True)
assert isinstance(usage_key, BlockUsageLocator)
course = self._lookup_course(usage_key)
items = self._load_items(course, [usage_key.block_id], depth, lazy=True)
if len(items) == 0:
raise ItemNotFoundError(location)
raise ItemNotFoundError(usage_key)
return items[0]
def get_items(self, locator, course_id=None, depth=0, qualifiers=None):
def get_items(self, course_locator, settings=None, content=None, **kwargs):
"""
Get all of the modules in the given course matching the qualifiers. The
qualifiers should only be fields in the structures collection (sorry).
There will be a separate search method for searching through
definitions.
Returns:
list of XModuleDescriptor instances for the matching items within the course with
the given course_id
Common qualifiers are category, definition (provide definition id),
display_name, anyfieldname, children (return
block if its children includes the one given value). If you want
substring matching use {$regex: /acme.*corp/i} type syntax.
Although these
look like mongo queries, it is all done in memory; so, you cannot
try arbitrary queries.
:param locator: CourseLocator or BlockUsageLocator restricting search scope
:param course_id: ignored. Only included for API compatibility.
:param depth: ignored. Only included for API compatibility.
:param qualifiers: a dict restricting which elements should match
NOTE: don't use this to look for courses
as the course_id is required. Use get_courses.
Args:
course_locator (CourseLocator): the course identifier
settings (dict): fields to look for which have settings scope. Follows same syntax
and rules as kwargs below
content (dict): fields to look for which have content scope. Follows same syntax and
rules as kwargs below.
kwargs (key=value): what to look for within the course.
Common qualifiers are ``category`` or any field name. if the target field is a list,
then it searches for the given value in the list not list equivalence.
Substring matching pass a regex object.
For split,
you can search by ``edited_by``, ``edited_on`` providing a function testing limits.
"""
# TODO extend to only search a subdag of the course?
if qualifiers is None:
qualifiers = {}
course = self._lookup_course(locator)
course = self._lookup_course(course_locator)
items = []
def _block_matches_all(block_json):
"""
Check that the block matches all the criteria
"""
# do the checks which don't require loading any additional data
if (
self._block_matches(block_json, kwargs) and
self._block_matches(block_json.get('fields', {}), settings)
):
if content:
definition_block = self.db_connection.get_definition(block_json['definition'])
return self._block_matches(definition_block.get('fields', {}), content)
else:
return True
if settings is None:
settings = {}
if 'name' in kwargs:
# odd case where we don't search just confirm
block_id = kwargs.pop('name')
block = course['structure']['blocks'].get(block_id)
if _block_matches_all(block):
return self._load_items(course, [block_id], lazy=True)
else:
return []
# don't expect caller to know that children are in fields
if 'children' in kwargs:
settings['children'] = kwargs.pop('children')
for block_id, value in course['structure']['blocks'].iteritems():
if self._block_matches(value, qualifiers):
if _block_matches_all(value):
items.append(block_id)
if len(items) > 0:
@@ -414,20 +444,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
else:
return []
def get_instance(self, course_id, location, depth=0):
"""
Get an instance of this location.
For now, just delegate to get_item and ignore course policy.
depth (int): An argument that some module stores may use to prefetch
descendants of the queried modules for more efficient results later
in the request. The depth is counted in the number of
calls to get_children() to cache. None indicates to cache all descendants.
"""
return self.get_item(location, depth=depth)
def get_parent_locations(self, locator, course_id=None):
def get_parent_locations(self, locator):
'''
Return the locations (Locators w/ block_ids) for the parents of this location in this
course. Could use get_items(location, {'children': block_id}) but this is slightly faster.
@@ -438,20 +455,20 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
'''
course = self._lookup_course(locator)
items = self._get_parents_from_structure(locator.block_id, course['structure'])
return [BlockUsageLocator(
url=locator.as_course_locator(),
block_id=LocMapperStore.decode_key_from_mongo(parent_id),
)
for parent_id in items]
return [
BlockUsageLocator.make_relative(
locator,
block_id=LocMapperStore.decode_key_from_mongo(parent_id),
)
for parent_id in items
]
def get_orphans(self, package_id, branch):
def get_orphans(self, course_key):
"""
Return a dict of all of the orphans in the course.
:param package_id:
"""
detached_categories = [name for name, __ in XBlock.load_tagged_classes("detached")]
course = self._lookup_course(CourseLocator(package_id=package_id, branch=branch))
course = self._lookup_course(course_key)
items = {LocMapperStore.decode_key_from_mongo(block_id) for block_id in course['structure']['blocks'].keys()}
items.remove(course['structure']['root'])
for block_id, block_data in course['structure']['blocks'].iteritems():
@@ -459,7 +476,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
if block_data['category'] in detached_categories:
items.discard(LocMapperStore.decode_key_from_mongo(block_id))
return [
BlockUsageLocator(package_id=package_id, branch=branch, block_id=block_id)
BlockUsageLocator(course_key=course_key, block_id=block_id)
for block_id in items
]
@@ -468,7 +485,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
The index records the initial creation of the indexed course and tracks the current version
heads. This function is primarily for test verification but may serve some
more general purpose.
:param course_locator: must have a package_id set
:param course_locator: must have a org and offering set
:return {'org': string,
versions: {'draft': the head draft version id,
'published': the head published version id if any,
@@ -477,9 +494,9 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
'edited_on': when the course was originally created
}
"""
if course_locator.package_id is None:
if not (course_locator.offering and course_locator.org):
return None
index = self.db_connection.get_course_index(course_locator.package_id)
index = self.db_connection.get_course_index(course_locator)
return index
# TODO figure out a way to make this info accessible from the course descriptor
@@ -529,6 +546,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
if course_locator.version_guid is None:
course = self._lookup_course(course_locator)
version_guid = course['structure']['_id']
course_locator = course_locator.for_version(version_guid)
else:
version_guid = course_locator.version_guid
@@ -547,7 +565,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
for course_structure in next_versions:
result.setdefault(course_structure['previous_version'], []).append(
CourseLocator(version_guid=struct['_id']))
return VersionTree(CourseLocator(course_locator, version_guid=version_guid), result)
return VersionTree(course_locator, result)
def get_block_generations(self, block_locator):
@@ -562,8 +580,12 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
course_struct = self._lookup_course(block_locator.course_agnostic())['structure']
block_id = block_locator.block_id
update_version_field = 'blocks.{}.edit_info.update_version'.format(block_id)
all_versions_with_block = self.db_connection.find_matching_structures({'original_version': course_struct['original_version'],
update_version_field: {'$exists': True}})
all_versions_with_block = self.db_connection.find_matching_structures(
{
'original_version': course_struct['original_version'],
update_version_field: {'$exists': True}
}
)
# find (all) root versions and build map {previous: {successors}..}
possible_roots = []
result = {}
@@ -590,9 +612,14 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
return None
# convert the results value sets to locators
for k, versions in result.iteritems():
result[k] = [BlockUsageLocator(version_guid=version, block_id=block_id)
for version in versions]
return VersionTree(BlockUsageLocator(version_guid=possible_roots[0], block_id=block_id), result)
result[k] = [
BlockUsageLocator(CourseLocator(version_guid=version), block_id=block_id)
for version in versions
]
return VersionTree(
BlockUsageLocator(CourseLocator(version_guid=possible_roots[0]), block_id=block_id),
result
)
def get_definition_successors(self, definition_locator, version_history_depth=1):
'''
@@ -646,7 +673,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
# actual change b/c the descriptor and cache probably point to the same objects
old_definition = self.db_connection.get_definition(definition_locator.definition_id)
if old_definition is None:
raise ItemNotFoundError(definition_locator.url())
raise ItemNotFoundError(definition_locator.to_deprecated_string())
if needs_saved():
# new id to create new version
@@ -693,11 +720,12 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
:param course_or_parent_locator: If BlockUsageLocator, then it's assumed to be the parent.
If it's a CourseLocator, then it's
merely the containing course.
merely the containing course. If it has a version_guid and a course org + offering + branch, this
method ensures that the version is the head of the given course branch before making the change.
raises InsufficientSpecificationError if there is no course locator.
raises VersionConflictError if package_id and version_guid given and the current version head != version_guid
and force is not True.
raises VersionConflictError if the version_guid of the course_or_parent_locator is not the head
of the its course unless force is true.
:param force: fork the structure and don't update the course draftVersion if the above
:param continue_revision: for multistep transactions, continue revising the given version rather than creating
a new version. Setting force to True conflicts with setting this to True and will cause a VersionConflictError
@@ -722,11 +750,11 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
Rules for course locator:
* If the course locator specifies a package_id and either it doesn't
* If the course locator specifies a org and offering and either it doesn't
specify version_guid or the one it specifies == the current head of the branch,
it progresses the course to point
to the new head and sets the active version to point to the new head
* If the locator has a package_id but its version_guid != current head, it raises VersionConflictError.
* If the locator has a org and offering but its version_guid != current head, it raises VersionConflictError.
NOTE: using a version_guid will end up creating a new version of the course. Your new item won't be in
the course id'd by version_guid but instead in one w/ a new version_guid. Ensure in this case that you get
@@ -800,29 +828,36 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
if not continue_version:
self._update_head(index_entry, course_or_parent_locator.branch, new_id)
item_loc = BlockUsageLocator(
package_id=course_or_parent_locator.package_id,
branch=course_or_parent_locator.branch,
course_or_parent_locator.version_agnostic(),
block_id=new_block_id,
)
else:
item_loc = BlockUsageLocator(
CourseLocator(version_guid=new_id),
block_id=new_block_id,
version_guid=new_id,
)
# reconstruct the new_item from the cache
return self.get_item(item_loc)
def create_course(
self, course_id, org, user_id, fields=None,
self, org, offering, user_id, fields=None,
master_branch='draft', versions_dict=None, root_category='course',
root_block_id='course'
root_block_id='course', **kwargs
):
"""
Create a new entry in the active courses index which points to an existing or new structure. Returns
the course root of the resulting entry (the location has the course id)
course_id: If it's already taken, this method will raise DuplicateCourseError
Arguments:
org (str): the organization that owns the course
offering (str): the name of the course offering
user_id: id of the user creating the course
fields (dict): Fields to set on the course at initialization
kwargs: Any optional arguments understood by a subset of modulestores to customize instantiation
offering: If it's already taken, this method will raise DuplicateCourseError
fields: if scope.settings fields provided, will set the fields of the root course object in the
new course. If both
@@ -848,10 +883,11 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
provide any fields overrides, see above). if not provided, will create a mostly empty course
structure with just a category course root xblock.
"""
# check course_id's uniqueness
index = self.db_connection.get_course_index(course_id)
# check offering's uniqueness
locator = CourseLocator(org=org, offering=offering, branch=master_branch)
index = self.db_connection.get_course_index(locator)
if index is not None:
raise DuplicateCourseError(course_id, index)
raise DuplicateCourseError(locator, index)
partitioned_fields = self.partition_fields_by_scope(root_category, fields)
block_fields = partitioned_fields.setdefault(Scope.settings, {})
@@ -920,15 +956,16 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
versions_dict[master_branch] = new_id
index_entry = {
'_id': course_id,
'_id': ObjectId(),
'org': org,
'offering': offering,
'edited_by': user_id,
'edited_on': datetime.datetime.now(UTC),
'versions': versions_dict,
'schema_version': self.SCHEMA_VERSION,
}
self.db_connection.insert_course_index(index_entry)
return self.get_course(CourseLocator(package_id=course_id, branch=master_branch))
return self.get_course(locator)
def update_item(self, descriptor, user_id, allow_not_found=False, force=False):
"""
@@ -937,7 +974,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
raises ItemNotFoundError if the location does not exist.
Creates a new course version. If the descriptor's location has a package_id, it moves the course head
Creates a new course version. If the descriptor's location has a org and offering, it moves the course head
pointer. If the version_guid of the descriptor points to a non-head version and there's been an intervening
change to this item, it raises a VersionConflictError unless force is True. In the force case, it forks
the course but leaves the head pointer where it is (this change will not be in the course head).
@@ -983,10 +1020,16 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
# update the index entry if appropriate
if index_entry is not None:
self._update_head(index_entry, descriptor.location.branch, new_id)
course_key = CourseLocator(
org=index_entry['org'], offering=index_entry['offering'],
branch=descriptor.location.branch,
version_guid=new_id
)
else:
course_key = CourseLocator(version_guid=new_id)
# fetch and return the new item--fetching is unnecessary but a good qc step
new_locator = BlockUsageLocator(descriptor.location)
new_locator.version_guid = new_id
new_locator = BlockUsageLocator(course_key, descriptor.location.block_id)
return self.get_item(new_locator)
else:
# nothing changed, just return the one sent in
@@ -1060,10 +1103,8 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
# fetch and return the new item--fetching is unnecessary but a good qc step
return self.get_item(
BlockUsageLocator(
package_id=xblock.location.package_id,
xblock.location.course_key.for_version(new_id),
block_id=xblock.location.block_id,
branch=xblock.location.branch,
version_guid=new_id
)
)
else:
@@ -1088,7 +1129,8 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
if block_id is None:
block_id = self._generate_block_id(structure_blocks, xblock.category)
encoded_block_id = LocMapperStore.encode_key_for_mongo(block_id)
xblock.scope_ids.usage_id.block_id = block_id
new_usage_id = xblock.scope_ids.usage_id.replace(block_id=block_id)
xblock.scope_ids = xblock.scope_ids._replace(usage_id=new_usage_id) # pylint: disable=protected-access
else:
is_new = False
encoded_block_id = LocMapperStore.encode_key_for_mongo(xblock.location.block_id)
@@ -1179,7 +1221,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
"""
# get the destination's index, and source and destination structures.
source_structure = self._lookup_course(source_course)['structure']
index_entry = self.db_connection.get_course_index(destination_course.package_id)
index_entry = self.db_connection.get_course_index(destination_course)
if index_entry is None:
# brand new course
raise ItemNotFoundError(destination_course)
@@ -1244,13 +1286,13 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
raises ItemNotFoundError if the location does not exist.
raises ValueError if usage_locator points to the structure root
Creates a new course version. If the descriptor's location has a package_id, it moves the course head
Creates a new course version. If the descriptor's location has a org and offering, it moves the course head
pointer. If the version_guid of the descriptor points to a non-head version and there's been an intervening
change to this item, it raises a VersionConflictError unless force is True. In the force case, it forks
the course but leaves the head pointer where it is (this change will not be in the course head).
"""
assert isinstance(usage_locator, BlockUsageLocator) and usage_locator.is_initialized()
original_structure = self._lookup_course(usage_locator)['structure']
assert isinstance(usage_locator, BlockUsageLocator)
original_structure = self._lookup_course(usage_locator.course_key)['structure']
if original_structure['root'] == usage_locator.block_id:
raise ValueError("Cannot delete the root of a course")
index_entry = self._get_index_if_valid(usage_locator, force)
@@ -1283,32 +1325,29 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
# update index if appropriate and structures
self.db_connection.insert_structure(new_structure)
result = CourseLocator(version_guid=new_id)
# update the index entry if appropriate
if index_entry is not None:
# update the index entry if appropriate
self._update_head(index_entry, usage_locator.branch, new_id)
result.package_id = usage_locator.package_id
result.branch = usage_locator.branch
result = usage_locator.course_key.for_version(new_id)
else:
result = CourseLocator(version_guid=new_id)
return result
def delete_course(self, package_id):
def delete_course(self, course_key, user_id=None):
"""
Remove the given course from the course index.
Only removes the course from the index. The data remains. You can use create_course
with a versions hash to restore the course; however, the edited_on and
edited_by won't reflect the originals, of course.
:param package_id: uses package_id rather than locator to emphasize its global effect
"""
index = self.db_connection.get_course_index(package_id)
index = self.db_connection.get_course_index(course_key)
if index is None:
raise ItemNotFoundError(package_id)
raise ItemNotFoundError(course_key)
# this is the only real delete in the system. should it do something else?
log.info(u"deleting course from split-mongo: %s", package_id)
self.db_connection.delete_course_index(index['_id'])
log.info(u"deleting course from split-mongo: %s", course_key)
self.db_connection.delete_course_index(index)
def get_errored_courses(self):
"""
@@ -1413,38 +1452,6 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
# clear cache again b/c inheritance may be wrong over orphans
self._clear_cache(original_structure['_id'])
def _block_matches(self, value, qualifiers):
'''
Return True or False depending on whether the value (block contents)
matches the qualifiers as per get_items
:param value:
:param qualifiers:
'''
for key, criteria in qualifiers.iteritems():
if key in value:
target = value[key]
if not self._value_matches(target, criteria):
return False
elif criteria is not None:
return False
return True
def _value_matches(self, target, criteria):
''' helper for _block_matches '''
if isinstance(target, list):
return any(self._value_matches(ele, criteria)
for ele in target)
elif isinstance(criteria, dict):
if '$regex' in criteria:
return re.search(criteria['$regex'], target) is not None
elif not isinstance(target, dict):
return False
else:
return (isinstance(target, dict) and
self._block_matches(target, criteria))
else:
return criteria == target
def _get_index_if_valid(self, locator, force=False, continue_version=False):
"""
If the locator identifies a course and points to its draft (or plausibly its draft),
@@ -1458,7 +1465,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
:param continue_version: if True, assumes this operation requires a head version and will not create a new
version but instead continue an existing transaction on this version. This flag cannot be True if force is True.
"""
if locator.package_id is None or locator.branch is None:
if locator.org is None or locator.offering is None or locator.branch is None:
if continue_version:
raise InsufficientSpecificationError(
"To continue a version, the locator must point to one ({}).".format(locator)
@@ -1466,7 +1473,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
else:
return None
else:
index_entry = self.db_connection.get_course_index(locator.package_id)
index_entry = self.db_connection.get_course_index(locator)
is_head = (
locator.version_guid is None or
index_entry['versions'][locator.branch] == locator.version_guid

174
common/lib/xmodule/xmodule/modulestore/store_utilities.py
View File

@@ -2,7 +2,6 @@ import re
import logging
from xmodule.contentstore.content import StaticContent
from xmodule.modulestore import Location
def _prefix_only_url_replace_regex(prefix):
@@ -46,10 +45,6 @@ def rewrite_nonportable_content_links(source_course_id, dest_course_id, text):
"""
course_id_dict = Location.parse_course_id(source_course_id)
course_id_dict['tag'] = 'i4x'
course_id_dict['category'] = 'course'
def portable_asset_link_subtitution(match):
quote = match.group('quote')
rest = match.group('rest')
@@ -60,27 +55,21 @@ def rewrite_nonportable_content_links(source_course_id, dest_course_id, text):
rest = match.group('rest')
return quote + '/jump_to_id/' + rest + quote
def generic_courseware_link_substitution(match):
parts = Location.parse_course_id(dest_course_id)
parts['quote'] = match.group('quote')
parts['rest'] = match.group('rest')
return u'{quote}/courses/{org}/{course}/{name}/{rest}{quote}'.format(**parts)
course_location = Location(course_id_dict)
# NOTE: ultimately link updating is not a hard requirement, so if something blows up with
# the regex subsitution, log the error and continue
# the regex substitution, log the error and continue
c4x_link_base = StaticContent.get_base_url_path_for_course_assets(source_course_id)
try:
c4x_link_base = u'{0}/'.format(StaticContent.get_base_url_path_for_course_assets(course_location))
text = re.sub(_prefix_only_url_replace_regex(c4x_link_base), portable_asset_link_subtitution, text)
except Exception, e:
logging.warning("Error going regex subtituion %r on text = %r.\n\nError msg = %s", c4x_link_base, text, str(e))
except Exception as exc: # pylint: disable=broad-except
logging.warning("Error producing regex substitution %r for text = %r.\n\nError msg = %s", c4x_link_base, text, str(exc))
jump_to_link_base = u'/courses/{course_key_string}/jump_to/i4x://{course_key.org}/{course_key.course}/'.format(
course_key_string=source_course_id.to_deprecated_string(), course_key=source_course_id
)
try:
jump_to_link_base = u'/courses/{org}/{course}/{name}/jump_to/i4x://{org}/{course}/'.format(**course_id_dict)
text = re.sub(_prefix_and_category_url_replace_regex(jump_to_link_base), portable_jump_to_link_substitution, text)
except Exception, e:
logging.warning("Error going regex subtituion %r on text = %r.\n\nError msg = %s", jump_to_link_base, text, str(e))
except Exception as exc: # pylint: disable=broad-except
logging.warning("Error producing regex substitution %r for text = %r.\n\nError msg = %s", jump_to_link_base, text, str(exc))
# Also, there commonly is a set of link URL's used in the format:
# /courses/<org>/<course>/<name> which will be broken if migrated to a different course_id
@@ -90,65 +79,46 @@ def rewrite_nonportable_content_links(source_course_id, dest_course_id, text):
#
if source_course_id != dest_course_id:
try:
generic_courseware_link_base = u'/courses/{org}/{course}/{name}/'.format(**course_id_dict)
generic_courseware_link_base = u'/courses/{}/'.format(source_course_id.to_deprecated_string())
text = re.sub(_prefix_only_url_replace_regex(generic_courseware_link_base), portable_asset_link_subtitution, text)
except Exception, e:
logging.warning("Error going regex subtituion %r on text = %r.\n\nError msg = %s", generic_courseware_link_base, text, str(e))
except Exception as exc: # pylint: disable=broad-except
logging.warning("Error producing regex substitution %r for text = %r.\n\nError msg = %s", source_course_id, text, str(exc))
return text
def _clone_modules(modulestore, modules, source_location, dest_location):
def _clone_modules(modulestore, modules, source_course_id, dest_course_id):
for module in modules:
original_loc = Location(module.location)
if original_loc.category != 'course':
module.location = module.location._replace(
tag=dest_location.tag,
org=dest_location.org,
course=dest_location.course
)
else:
# on the course module we also have to update the module name
module.location = module.location._replace(
tag=dest_location.tag,
org=dest_location.org,
course=dest_location.course,
name=dest_location.name
)
original_loc = module.location
module.location = module.location.map_into_course(dest_course_id)
print "Cloning module {0} to {1}....".format(original_loc, module.location)
if 'data' in module.fields and module.fields['data'].is_set_on(module) and isinstance(module.data, basestring):
module.data = rewrite_nonportable_content_links(
source_location.course_id, dest_location.course_id, module.data
source_course_id, dest_course_id, module.data
)
# repoint children
if module.has_children:
new_children = []
for child_loc_url in module.children:
child_loc = Location(child_loc_url)
child_loc = child_loc._replace(
tag=dest_location.tag,
org=dest_location.org,
course=dest_location.course
)
new_children.append(child_loc.url())
for child_loc in module.children:
child_loc = child_loc.map_into_course(dest_course_id)
new_children.append(child_loc)
module.children = new_children
modulestore.update_item(module, '**replace_user**')
def clone_course(modulestore, contentstore, source_location, dest_location, delete_original=False):
def clone_course(modulestore, contentstore, source_course_id, dest_course_id):
# check to see if the dest_location exists as an empty course
# we need an empty course because the app layers manage the permissions and users
if not modulestore.has_item(dest_location.course_id, dest_location):
raise Exception("An empty course at {0} must have already been created. Aborting...".format(dest_location))
if not modulestore.has_course(dest_course_id):
raise Exception(u"An empty course at {0} must have already been created. Aborting...".format(dest_course_id))
# verify that the dest_location really is an empty course, which means only one with an optional 'overview'
dest_modules = modulestore.get_items([dest_location.tag, dest_location.org, dest_location.course, None, None, None])
dest_modules = modulestore.get_items(dest_course_id)
basically_empty = True
for module in dest_modules:
@@ -163,107 +133,63 @@ def clone_course(modulestore, contentstore, source_location, dest_location, dele
raise Exception("Course at destination {0} is not an empty course. You can only clone into an empty course. Aborting...".format(dest_location))
# check to see if the source course is actually there
if not modulestore.has_item(source_location.course_id, source_location):
raise Exception("Cannot find a course at {0}. Aborting".format(source_location))
if not modulestore.has_course(source_course_id):
raise Exception("Cannot find a course at {0}. Aborting".format(source_course_id))
# Get all modules under this namespace which is (tag, org, course) tuple
modules = modulestore.get_items([source_location.tag, source_location.org, source_location.course, None, None, None])
_clone_modules(modulestore, modules, source_location, dest_location)
modules = modulestore.get_items(source_course_id, revision=None)
_clone_modules(modulestore, modules, source_course_id, dest_course_id)
modules = modulestore.get_items([source_location.tag, source_location.org, source_location.course, None, None, 'draft'])
_clone_modules(modulestore, modules, source_location, dest_location)
modules = modulestore.get_items(source_course_id, revision='draft')
_clone_modules(modulestore, modules, source_course_id, dest_course_id)
# now iterate through all of the assets and clone them
# first the thumbnails
thumbs = contentstore.get_all_content_thumbnails_for_course(source_location)
for thumb in thumbs:
thumb_loc = Location(thumb["_id"])
content = contentstore.find(thumb_loc)
content.location = content.location._replace(org=dest_location.org,
course=dest_location.course)
thumb_keys = contentstore.get_all_content_thumbnails_for_course(source_course_id)
for thumb_key in thumb_keys:
content = contentstore.find(thumb_key)
content.location = content.location.map_into_course(dest_course_id)
print "Cloning thumbnail {0} to {1}".format(thumb_loc, content.location)
print "Cloning thumbnail {0} to {1}".format(thumb_key, content.location)
contentstore.save(content)
# now iterate through all of the assets, also updating the thumbnail pointer
assets, __ = contentstore.get_all_content_for_course(source_location)
for asset in assets:
asset_loc = Location(asset["_id"])
content = contentstore.find(asset_loc)
content.location = content.location._replace(org=dest_location.org,
course=dest_location.course)
asset_keys, __ = contentstore.get_all_content_for_course(source_course_id)
for asset_key in asset_keys:
content = contentstore.find(asset_key)
content.location = content.location.map_into_course(dest_course_id)
# be sure to update the pointer to the thumbnail
if content.thumbnail_location is not None:
content.thumbnail_location = content.thumbnail_location._replace(org=dest_location.org,
course=dest_location.course)
content.thumbnail_location = content.thumbnail_location.map_into_course(dest_course_id)
print "Cloning asset {0} to {1}".format(asset_loc, content.location)
print "Cloning asset {0} to {1}".format(asset_key, content.location)
contentstore.save(content)
return True
def _delete_modules_except_course(modulestore, modules, source_location, commit):
"""
This helper method will just enumerate through a list of modules and delete them, except for the
top-level course module
"""
for module in modules:
if module.category != 'course':
logging.warning("Deleting {0}...".format(module.location))
if commit:
# sanity check. Make sure we're not deleting a module in the incorrect course
if module.location.org != source_location.org or module.location.course != source_location.course:
raise Exception('Module {0} is not in same namespace as {1}. This should not happen! Aborting...'.format(module.location, source_location))
modulestore.delete_item(module.location)
def _delete_assets(contentstore, assets, commit):
"""
This helper method will enumerate through a list of assets and delete them
"""
for asset in assets:
asset_loc = Location(asset["_id"])
id = StaticContent.get_id_from_location(asset_loc)
logging.warning("Deleting {0}...".format(id))
if commit:
contentstore.delete(id)
def delete_course(modulestore, contentstore, source_location, commit=False):
def delete_course(modulestore, contentstore, course_key, commit=False):
"""
This method will actually do the work to delete all content in a course in a MongoDB backed
courseware store. BE VERY CAREFUL, this is not reversable.
"""
# check to see if the source course is actually there
if not modulestore.has_item(source_location.course_id, source_location):
raise Exception("Cannot find a course at {0}. Aborting".format(source_location))
if not modulestore.has_course(course_key):
raise Exception("Cannot find a course at {0}. Aborting".format(course_key))
# first delete all of the thumbnails
thumbs = contentstore.get_all_content_thumbnails_for_course(source_location)
_delete_assets(contentstore, thumbs, commit)
# then delete all of the assets
assets, __ = contentstore.get_all_content_for_course(source_location)
_delete_assets(contentstore, assets, commit)
# then delete all course modules
modules = modulestore.get_items([source_location.tag, source_location.org, source_location.course, None, None, None])
_delete_modules_except_course(modulestore, modules, source_location, commit)
# then delete all draft course modules
modules = modulestore.get_items([source_location.tag, source_location.org, source_location.course, None, None, 'draft'])
_delete_modules_except_course(modulestore, modules, source_location, commit)
# finally delete the top-level course module itself
print "Deleting {0}...".format(source_location)
if commit:
modulestore.delete_item(source_location)
print "Deleting assets and thumbnails {}".format(course_key)
contentstore.delete_all_course_assets(course_key)
# finally delete the course
print "Deleting {0}...".format(course_key)
if commit:
modulestore.delete_course(course_key, '**replace-user**')
return True

2
common/lib/xmodule/xmodule/modulestore/tests/django_utils.py
View File

@@ -205,7 +205,7 @@ class ModuleStoreTestCase(TestCase):
"""
store = editable_modulestore()
store.update_item(course, '**replace_user**')
updated_course = store.get_instance(course.id, course.location)
updated_course = store.get_course(course.id)
return updated_course
@staticmethod

22
common/lib/xmodule/xmodule/modulestore/tests/factories.py
View File

@@ -2,7 +2,8 @@ from factory import Factory, lazy_attribute_sequence, lazy_attribute
from factory.containers import CyclicDefinitionError
from uuid import uuid4
from xmodule.modulestore import Location, prefer_xmodules
from xmodule.modulestore import prefer_xmodules
from xmodule.modulestore.locations import Location
from xblock.core import XBlock
@@ -36,6 +37,7 @@ class CourseFactory(XModuleFactory):
number = '999'
display_name = 'Robot Super Course'
# pylint: disable=unused-argument
@classmethod
def _create(cls, target_class, **kwargs):
@@ -46,8 +48,10 @@ class CourseFactory(XModuleFactory):
# because the factory provides a default 'number' arg, prefer the non-defaulted 'course' arg if any
number = kwargs.pop('course', kwargs.pop('number', None))
store = kwargs.pop('modulestore')
name = kwargs.get('name', kwargs.get('run', Location.clean(kwargs.get('display_name'))))
run = kwargs.get('run', name)
location = Location('i4x', org, number, 'course', Location.clean(kwargs.get('display_name')))
location = Location(org, number, run, 'course', name)
# Write the data to the mongo datastore
new_course = store.create_xmodule(location, metadata=kwargs.get('metadata', None))
@@ -82,11 +86,15 @@ class ItemFactory(XModuleFactory):
else:
dest_name = self.display_name.replace(" ", "_")
return self.parent_location.replace(category=self.category, name=dest_name)
new_location = self.parent_location.course_key.make_usage_key(
self.category,
dest_name
)
return new_location
@lazy_attribute
def parent_location(self):
default_location = Location('i4x://MITx/999/course/Robot_Super_Course')
default_location = Location('MITx', '999', 'Robot_Super_Course', 'course', 'Robot_Super_Course', None)
try:
parent = self.parent
# This error is raised if the caller hasn't provided either parent or parent_location
@@ -127,12 +135,14 @@ class ItemFactory(XModuleFactory):
# catch any old style users before they get into trouble
assert 'template' not in kwargs
parent_location = Location(kwargs.pop('parent_location', None))
parent_location = kwargs.pop('parent_location', None)
data = kwargs.pop('data', None)
category = kwargs.pop('category', None)
display_name = kwargs.pop('display_name', None)
metadata = kwargs.pop('metadata', {})
location = kwargs.pop('location')
assert isinstance(location, Location)
assert location != parent_location
store = kwargs.pop('modulestore')
@@ -164,7 +174,7 @@ class ItemFactory(XModuleFactory):
store.update_item(module)
if 'detached' not in module._class_tags:
parent.children.append(location.url())
parent.children.append(location)
store.update_item(parent, '**replace_user**')
return store.get_item(location)

210
common/lib/xmodule/xmodule/modulestore/tests/test_location.py
View File

@@ -1,8 +1,11 @@
"""
Thorough tests of the Location class
"""
import ddt
from unittest import TestCase
from xmodule.modulestore import Location
from xmodule.modulestore.exceptions import InvalidLocationError
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import Location, AssetLocation, SlashSeparatedCourseKey
# Pairs for testing the clean* functions.
# The first item in the tuple is the input string.
@@ -23,117 +26,110 @@ class TestLocations(TestCase):
Tests of :class:`.Location`
"""
@ddt.data(
"tag://org/course/category/name",
"tag://org/course/category/name@revision"
"org+course+run+category+name",
"org+course+run+category+name@revision"
)
def test_string_roundtrip(self, url):
self.assertEquals(url, Location(url).url())
self.assertEquals(url, str(Location(url)))
self.assertEquals(url, Location._from_string(url)._to_string()) # pylint: disable=protected-access
@ddt.data(
{
'tag': 'tag',
"i4x://org/course/category/name",
"i4x://org/course/category/name@revision"
)
def test_deprecated_roundtrip(self, url):
course_id = SlashSeparatedCourseKey('org', 'course', 'run')
self.assertEquals(
url,
course_id.make_usage_key_from_deprecated_string(url).to_deprecated_string()
)
def test_invalid_chars_ssck(self):
"""
Test that the ssck constructor fails if given invalid chars
"""
valid_base = SlashSeparatedCourseKey(u'org.dept-1%2', u'course.sub-2%3', u'run.faster-4%5')
for key in SlashSeparatedCourseKey.KEY_FIELDS:
with self.assertRaises(InvalidKeyError):
# this ends up calling the constructor where the legality check should occur
valid_base.replace(**{key: u'funny thing'})
def test_invalid_chars_location(self):
"""
Test that the location constructor fails if given invalid chars
"""
course_key = SlashSeparatedCourseKey(u'org.dept-1%2', u'course.sub-2%3', u'run.faster-4%5')
valid_base = course_key.make_usage_key('tomato-again%9', 'block-head:sub-4%9')
for key in SlashSeparatedCourseKey.KEY_FIELDS:
with self.assertRaises(InvalidKeyError):
# this ends up calling the constructor where the legality check should occur
valid_base.replace(**{key: u'funny thing'})
@ddt.data(
((), {
'org': 'org',
'course': 'course',
'run': 'run',
'category': 'category',
'name': 'name',
'org': 'org'
},
{
'tag': 'tag',
}, 'org', 'course', 'run', 'category', 'name', None),
((), {
'org': 'org',
'course': 'course',
'run': 'run',
'category': 'category',
'name': 'name:more_name',
'org': 'org'
},
['tag', 'org', 'course', 'category', 'name'],
"tag://org/course/category/name",
"tag://org/course/category/name@revision",
u"tag://org/course/category/name",
u"tag://org/course/category/name@revision",
}, 'org', 'course', 'run', 'category', 'name:more_name', None),
(['org', 'course', 'run', 'category', 'name'], {}, 'org', 'course', 'run', 'category', 'name', None),
)
def test_is_valid(self, loc):
self.assertTrue(Location.is_valid(loc))
@ddt.unpack
def test_valid_locations(self, args, kwargs, org, course, run, category, name, revision):
location = Location(*args, **kwargs)
self.assertEquals(org, location.org)
self.assertEquals(course, location.course)
self.assertEquals(run, location.run)
self.assertEquals(category, location.category)
self.assertEquals(name, location.name)
self.assertEquals(revision, location.revision)
@ddt.data(
{
(("foo",), {}),
(["foo", "bar"], {}),
(["foo", "bar", "baz", "blat/blat", "foo"], {}),
(["foo", "bar", "baz", "blat", "foo/bar"], {}),
(["foo", "bar", "baz", "blat:blat", "foo:bar"], {}), # ':' ok in name, not in category
(('org', 'course', 'run', 'category', 'name with spaces', 'revision'), {}),
(('org', 'course', 'run', 'category', 'name/with/slashes', 'revision'), {}),
(('org', 'course', 'run', 'category', 'name', u'\xae'), {}),
(('org', 'course', 'run', 'category', u'\xae', 'revision'), {}),
((), {
'tag': 'tag',
'course': 'course',
'category': 'category',
'name': 'name@more_name',
'org': 'org'
},
{
}),
((), {
'tag': 'tag',
'course': 'course',
'category': 'category',
'name': 'name ', # extra space
'org': 'org'
},
"foo",
["foo"],
["foo", "bar"],
["foo", "bar", "baz", "blat:blat", "foo:bar"], # ':' ok in name, not in category
"tag://org/course/category/name with spaces@revision",
"tag://org/course/category/name/with/slashes@revision",
u"tag://org/course/category/name\xae", # No non-ascii characters for now
u"tag://org/course/category\xae/name", # No non-ascii characters for now
}),
)
def test_is_invalid(self, loc):
self.assertFalse(Location.is_valid(loc))
def test_dict(self):
input_dict = {
'tag': 'tag',
'course': 'course',
'category': 'category',
'name': 'name',
'org': 'org'
}
self.assertEquals("tag://org/course/category/name", Location(input_dict).url())
self.assertEquals(dict(revision=None, **input_dict), Location(input_dict).dict())
input_dict['revision'] = 'revision'
self.assertEquals("tag://org/course/category/name@revision", Location(input_dict).url())
self.assertEquals(input_dict, Location(input_dict).dict())
def test_list(self):
input_list = ['tag', 'org', 'course', 'category', 'name']
self.assertEquals("tag://org/course/category/name", Location(input_list).url())
self.assertEquals(input_list + [None], Location(input_list).list())
input_list.append('revision')
self.assertEquals("tag://org/course/category/name@revision", Location(input_list).url())
self.assertEquals(input_list, Location(input_list).list())
def test_location(self):
input_list = ['tag', 'org', 'course', 'category', 'name']
self.assertEquals("tag://org/course/category/name", Location(Location(input_list)).url())
def test_none(self):
self.assertEquals([None] * 6, Location(None).list())
@ddt.data(
"foo",
["foo", "bar"],
["foo", "bar", "baz", "blat/blat", "foo"],
["foo", "bar", "baz", "blat", "foo/bar"],
"tag://org/course/category/name with spaces@revision",
"tag://org/course/category/name/revision",
)
def test_invalid_locations(self, loc):
with self.assertRaises(InvalidLocationError):
Location(loc)
@ddt.unpack
def test_invalid_locations(self, *args, **kwargs):
with self.assertRaises(TypeError):
Location(*args, **kwargs)
def test_equality(self):
self.assertEquals(
Location('tag', 'org', 'course', 'category', 'name'),
Location('tag', 'org', 'course', 'category', 'name')
Location('tag', 'org', 'course', 'run', 'category', 'name'),
Location('tag', 'org', 'course', 'run', 'category', 'name')
)
self.assertNotEquals(
Location('tag', 'org', 'course', 'category', 'name1'),
Location('tag', 'org', 'course', 'category', 'name')
Location('tag', 'org', 'course', 'run', 'category', 'name1'),
Location('tag', 'org', 'course', 'run', 'category', 'name')
)
@ddt.data(
@@ -164,42 +160,38 @@ class TestLocations(TestCase):
self.assertEquals(Location.clean_for_html(pair[0]), pair[1])
def test_html_id(self):
loc = Location("tag://org/course/cat/name:more_name@rev")
self.assertEquals(loc.html_id(), "tag-org-course-cat-name_more_name-rev")
def test_course_id(self):
loc = Location('i4x', 'mitX', '103', 'course', 'test2')
self.assertEquals('mitX/103/test2', loc.course_id)
loc = Location('i4x', 'mitX', '103', '_not_a_course', 'test2')
with self.assertRaises(InvalidLocationError):
loc.course_id # pylint: disable=pointless-statement
loc = Location('org', 'course', 'run', 'cat', 'name:more_name', 'rev')
self.assertEquals(loc.html_id(), "i4x-org-course-cat-name_more_name-rev")
def test_replacement(self):
# pylint: disable=protected-access
self.assertEquals(
Location('t://o/c/c/n@r')._replace(name='new_name'),
Location('t://o/c/c/new_name@r'),
Location('o', 'c', 'r', 'c', 'n', 'r').replace(name='new_name'),
Location('o', 'c', 'r', 'c', 'new_name', 'r'),
)
with self.assertRaises(InvalidLocationError):
Location('t://o/c/c/n@r')._replace(name=u'name\xae')
with self.assertRaises(InvalidKeyError):
Location('o', 'c', 'r', 'c', 'n', 'r').replace(name=u'name\xae')
@ddt.data('org', 'course', 'category', 'name', 'revision')
def test_immutable(self, attr):
loc = Location('t://o/c/c/n@r')
loc = Location('o', 'c', 'r', 'c', 'n', 'r')
with self.assertRaises(AttributeError):
setattr(loc, attr, attr)
def test_parse_course_id(self):
"""
Test the parse_course_id class method
"""
source_string = "myorg/mycourse/myrun"
parsed = Location.parse_course_id(source_string)
self.assertEqual(parsed['org'], 'myorg')
self.assertEqual(parsed['course'], 'mycourse')
self.assertEqual(parsed['name'], 'myrun')
with self.assertRaises(ValueError):
Location.parse_course_id('notlegit.id/foo')
def test_map_into_course_location(self):
loc = Location('org', 'course', 'run', 'cat', 'name:more_name', 'rev')
course_key = SlashSeparatedCourseKey("edX", "toy", "2012_Fall")
self.assertEquals(
Location("edX", "toy", "2012_Fall", 'cat', 'name:more_name', 'rev'),
loc.map_into_course(course_key)
)
def test_map_into_course_asset_location(self):
loc = AssetLocation('org', 'course', 'run', 'asset', 'foo.bar')
course_key = SlashSeparatedCourseKey("edX", "toy", "2012_Fall")
self.assertEquals(
AssetLocation("edX", "toy", "2012_Fall", 'asset', 'foo.bar'),
loc.map_into_course(course_key)
)

289
common/lib/xmodule/xmodule/modulestore/tests/test_location_mapper.py
View File

@@ -1,15 +1,15 @@
'''
Created on Aug 5, 2013
@author: dmitchell
'''
"""
Test the loc mapper store
"""
import unittest
import uuid
from xmodule.modulestore import Location
from xmodule.modulestore.locator import BlockUsageLocator
from xmodule.modulestore.locator import BlockUsageLocator, CourseLocator
from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError
from xmodule.modulestore.loc_mapper_store import LocMapperStore
from mock import Mock
from xmodule.modulestore.locations import SlashSeparatedCourseKey
import bson.son
class LocMapperSetupSansDjango(unittest.TestCase):
@@ -41,41 +41,47 @@ class TestLocationMapper(LocMapperSetupSansDjango):
Test the location to locator mapper
"""
def test_create_map(self):
def _construct_course_son(org, course, run):
"""
Make a lookup son
"""
return bson.son.SON([
('org', org),
('course', course),
('name', run)
])
org = 'foo_org'
course = 'bar_course'
loc_mapper().create_map_entry(Location('i4x', org, course, 'course', 'baz_run'))
course1 = 'bar_course'
run = 'baz_run'
loc_mapper().create_map_entry(SlashSeparatedCourseKey(org, course1, run))
# pylint: disable=protected-access
entry = loc_mapper().location_map.find_one({
'_id': loc_mapper()._construct_location_son(org, course, 'baz_run')
'_id': _construct_course_son(org, course1, run)
})
self.assertIsNotNone(entry, "Didn't find entry")
self.assertEqual(entry['course_id'], '{}.{}.baz_run'.format(org, course))
self.assertEqual(entry['org'], org)
self.assertEqual(entry['offering'], '{}.{}'.format(course1, run))
self.assertEqual(entry['draft_branch'], 'draft')
self.assertEqual(entry['prod_branch'], 'published')
self.assertEqual(entry['block_map'], {})
# ensure create_entry does the right thing when not given a course (creates org/course
# rather than org/course/run course_id)
loc_mapper().create_map_entry(Location('i4x', org, course, 'vertical', 'baz_vert'))
# find the one which has no name
entry = loc_mapper().location_map.find_one({
'_id' : loc_mapper()._construct_location_son(org, course, None)
})
self.assertIsNotNone(entry, "Didn't find entry")
self.assertEqual(entry['course_id'], '{}.{}'.format(org, course))
course = 'quux_course'
course2 = 'quux_course'
# oldname: {category: newname}
block_map = {'abc123': {'problem': 'problem2'}}
loc_mapper().create_map_entry(
Location('i4x', org, course, 'problem', 'abc123', 'draft'),
'foo_org.geek_dept.quux_course.baz_run',
SlashSeparatedCourseKey(org, course2, run),
'foo_org.geek_dept',
'quux_course.baz_run',
'wip',
'live',
block_map)
entry = loc_mapper().location_map.find_one({'_id.org': org, '_id.course': course})
entry = loc_mapper().location_map.find_one({
'_id': _construct_course_son(org, course2, run)
})
self.assertIsNotNone(entry, "Didn't find entry")
self.assertEqual(entry['course_id'], 'foo_org.geek_dept.quux_course.baz_run')
self.assertEqual(entry['org'], 'foo_org.geek_dept')
self.assertEqual(entry['offering'], '{}.{}'.format(course2, run))
self.assertEqual(entry['draft_branch'], 'wip')
self.assertEqual(entry['prod_branch'], 'live')
self.assertEqual(entry['block_map'], block_map)
@@ -87,51 +93,50 @@ class TestLocationMapper(LocMapperSetupSansDjango):
org = u'foo_org'
course = u'bar_course'
run = u'baz_run'
course_location = Location('i4x', org, course, 'course', run)
course_locator = loc_mapper().translate_location(course_location.course_id, course_location)
course_location = SlashSeparatedCourseKey(org, course, run)
loc_mapper().create_map_entry(course_location)
# pylint: disable=protected-access
entry = loc_mapper().location_map.find_one({
'_id': loc_mapper()._construct_location_son(org, course, run)
'_id': loc_mapper()._construct_course_son(course_location)
})
self.assertIsNotNone(entry, 'Entry not found in loc_mapper')
self.assertEqual(entry['course_id'], u'{0}.{1}.{2}'.format(org, course, run))
self.assertEqual(entry['offering'], u'{1}.{2}'.format(org, course, run))
# now delete course location from loc_mapper and cache and test that course location no longer
# exists in loca_mapper and cache
loc_mapper().delete_course_mapping(course_location)
# pylint: disable=protected-access
entry = loc_mapper().location_map.find_one({
'_id': loc_mapper()._construct_location_son(org, course, run)
'_id': loc_mapper()._construct_course_son(course_location)
})
self.assertIsNone(entry, 'Entry found in loc_mapper')
# pylint: disable=protected-access
cached_value = loc_mapper()._get_location_from_cache(course_locator)
cached_value = loc_mapper()._get_location_from_cache(course_location.make_usage_key('course', run))
self.assertIsNone(cached_value, 'course_locator found in cache')
# pylint: disable=protected-access
cached_value = loc_mapper()._get_course_location_from_cache(course_locator.package_id)
cached_value = loc_mapper()._get_course_location_from_cache(course_location)
self.assertIsNone(cached_value, 'Entry found in cache')
def translate_n_check(self, location, old_style_course_id, new_style_package_id, block_id, branch, add_entry=False):
def translate_n_check(self, location, org, offering, block_id, branch, add_entry=False):
"""
Request translation, check package_id, block_id, and branch
Request translation, check org, offering, block_id, and branch
"""
prob_locator = loc_mapper().translate_location(
old_style_course_id,
location,
published= (branch=='published'),
published=(branch == 'published'),
add_entry_if_missing=add_entry
)
self.assertEqual(prob_locator.package_id, new_style_package_id)
self.assertEqual(prob_locator.org, org)
self.assertEqual(prob_locator.offering, offering)
self.assertEqual(prob_locator.block_id, block_id)
self.assertEqual(prob_locator.branch, branch)
course_locator = loc_mapper().translate_location_to_course_locator(
old_style_course_id,
location,
published=(branch == 'published'),
location.course_key,
published=(branch == 'published'),
)
self.assertEqual(course_locator.package_id, new_style_package_id)
self.assertEqual(course_locator.org, org)
self.assertEqual(course_locator.offering, offering)
self.assertEqual(course_locator.branch, branch)
def test_translate_location_read_only(self):
@@ -141,46 +146,45 @@ class TestLocationMapper(LocMapperSetupSansDjango):
# lookup before there are any maps
org = 'foo_org'
course = 'bar_course'
old_style_course_id = '{}/{}/{}'.format(org, course, 'baz_run')
run = 'baz_run'
slash_course_key = SlashSeparatedCourseKey(org, course, run)
with self.assertRaises(ItemNotFoundError):
_ = loc_mapper().translate_location(
old_style_course_id,
Location('i4x', org, course, 'problem', 'abc123'),
Location(org, course, run, 'problem', 'abc123'),
add_entry_if_missing=False
)
new_style_package_id = '{}.geek_dept.{}.baz_run'.format(org, course)
new_style_org = '{}.geek_dept'.format(org)
new_style_offering = '.{}.{}'.format(course, run)
block_map = {
'abc123': {'problem': 'problem2', 'vertical': 'vertical2'},
'def456': {'problem': 'problem4'},
'ghi789': {'problem': 'problem7'},
}
loc_mapper().create_map_entry(
Location('i4x', org, course, 'course', 'baz_run'),
new_style_package_id,
slash_course_key,
new_style_org, new_style_offering,
block_map=block_map
)
test_problem_locn = Location('i4x', org, course, 'problem', 'abc123')
# only one course matches
test_problem_locn = Location(org, course, run, 'problem', 'abc123')
# look for w/ only the Location (works b/c there's only one possible course match). Will force
# cache as default translation for this problemid
self.translate_n_check(test_problem_locn, None, new_style_package_id, 'problem2', 'published')
self.translate_n_check(test_problem_locn, new_style_org, new_style_offering, 'problem2', 'published')
# look for non-existent problem
with self.assertRaises(ItemNotFoundError):
loc_mapper().translate_location(
None,
Location('i4x', org, course, 'problem', '1def23'),
Location(org, course, run, 'problem', '1def23'),
add_entry_if_missing=False
)
test_no_cat_locn = test_problem_locn.replace(category=None)
with self.assertRaises(InvalidLocationError):
loc_mapper().translate_location(
old_style_course_id, test_no_cat_locn, False, False
slash_course_key.make_usage_key(None, 'abc123'), test_no_cat_locn, False, False
)
test_no_cat_locn = test_no_cat_locn.replace(name='def456')
# only one course matches
self.translate_n_check(test_no_cat_locn, old_style_course_id, new_style_package_id, 'problem4', 'published')
self.translate_n_check(
test_no_cat_locn, new_style_org, new_style_offering, 'problem4', 'published'
)
# add a distractor course (note that abc123 has a different translation in this one)
distractor_block_map = {
@@ -188,37 +192,23 @@ class TestLocationMapper(LocMapperSetupSansDjango):
'def456': {'problem': 'problem4'},
'ghi789': {'problem': 'problem7'},
}
test_delta_new_id = '{}.geek_dept.{}.{}'.format(org, course, 'delta_run')
test_delta_old_id = '{}/{}/{}'.format(org, course, 'delta_run')
run = 'delta_run'
test_delta_new_org = '{}.geek_dept'.format(org)
test_delta_new_offering = '{}.{}'.format(course, run)
loc_mapper().create_map_entry(
Location('i4x', org, course, 'course', 'delta_run'),
test_delta_new_id,
SlashSeparatedCourseKey(org, course, run),
test_delta_new_org, test_delta_new_offering,
block_map=distractor_block_map
)
# test that old translation still works
self.translate_n_check(test_problem_locn, old_style_course_id, new_style_package_id, 'problem2', 'published')
self.translate_n_check(
test_problem_locn, new_style_org, new_style_offering, 'problem2', 'published'
)
# and new returns new id
self.translate_n_check(test_problem_locn, test_delta_old_id, test_delta_new_id, 'problem3', 'published')
# look for default translation of uncached Location (not unique; so, just verify it returns something)
prob_locator = loc_mapper().translate_location(
None,
Location('i4x', org, course, 'problem', 'def456'),
add_entry_if_missing=False
self.translate_n_check(
test_problem_locn.replace(run=run), test_delta_new_org, test_delta_new_offering,
'problem3', 'published'
)
self.assertIsNotNone(prob_locator, "couldn't find ambiguous location")
# make delta_run default course: anything not cached using None as old_course_id will use this
loc_mapper().create_map_entry(
Location('i4x', org, course, 'problem', '789abc123efg456'),
test_delta_new_id,
block_map=block_map
)
# now an uncached ambiguous query should return delta
test_unused_locn = Location('i4x', org, course, 'problem', 'ghi789')
self.translate_n_check(test_unused_locn, None, test_delta_new_id, 'problem7', 'published')
# get the draft one (I'm sorry this is getting long)
self.translate_n_check(test_unused_locn, None, test_delta_new_id, 'problem7', 'draft')
def test_translate_location_dwim(self):
"""
@@ -227,27 +217,27 @@ class TestLocationMapper(LocMapperSetupSansDjango):
"""
org = 'foo_org'
course = 'bar_course'
old_style_course_id = '{}/{}/{}'.format(org, course, 'baz_run')
run = 'baz_run'
problem_name = 'abc123abc123abc123abc123abc123f9'
location = Location('i4x', org, course, 'problem', problem_name)
new_style_package_id = '{}.{}.{}'.format(org, course, 'baz_run')
self.translate_n_check(location, old_style_course_id, new_style_package_id, 'problemabc', 'published', True)
# look for w/ only the Location (works b/c there's only one possible course match): causes cache
self.translate_n_check(location, None, new_style_package_id, 'problemabc', 'published', True)
location = Location(org, course, run, 'problem', problem_name)
new_offering = '{}.{}'.format(course, run)
self.translate_n_check(location, org, new_offering, 'problemabc', 'published', True)
# create an entry w/o a guid name
other_location = Location('i4x', org, course, 'chapter', 'intro')
self.translate_n_check(other_location, old_style_course_id, new_style_package_id, 'intro', 'published', True)
other_location = Location(org, course, run, 'chapter', 'intro')
self.translate_n_check(other_location, org, new_offering, 'intro', 'published', True)
# add a distractor course
delta_new_package_id = '{}.geek_dept.{}.{}'.format(org, course, 'delta_run')
delta_course_locn = Location('i4x', org, course, 'course', 'delta_run')
delta_new_org = '{}.geek_dept'.format(org)
run = 'delta_run'
delta_new_offering = '{}.{}'.format(course, run)
delta_course_locn = SlashSeparatedCourseKey(org, course, run)
loc_mapper().create_map_entry(
delta_course_locn,
delta_new_package_id,
delta_new_org, delta_new_offering,
block_map={problem_name: {'problem': 'problem3'}}
)
self.translate_n_check(location, old_style_course_id, new_style_package_id, 'problemabc', 'published', True)
self.translate_n_check(location, org, new_offering, 'problemabc', 'published', True)
# add a new one to both courses (ensure name doesn't have same beginning)
new_prob_name = uuid.uuid4().hex
@@ -255,35 +245,11 @@ class TestLocationMapper(LocMapperSetupSansDjango):
new_prob_name = uuid.uuid4().hex
new_prob_locn = location.replace(name=new_prob_name)
new_usage_id = 'problem{}'.format(new_prob_name[:3])
self.translate_n_check(new_prob_locn, old_style_course_id, new_style_package_id, new_usage_id, 'published', True)
self.translate_n_check(new_prob_locn, org, new_offering, new_usage_id, 'published', True)
new_prob_locn = new_prob_locn.replace(run=run)
self.translate_n_check(
new_prob_locn, delta_course_locn.course_id, delta_new_package_id, new_usage_id, 'published', True
new_prob_locn, delta_new_org, delta_new_offering, new_usage_id, 'published', True
)
# look for w/ only the Location: causes caching and not unique; so, can't check which course
prob_locator = loc_mapper().translate_location(
None,
new_prob_locn,
add_entry_if_missing=True
)
self.assertIsNotNone(prob_locator, "couldn't find ambiguous location")
# add a default course pointing to the delta_run
loc_mapper().create_map_entry(
Location('i4x', org, course, 'problem', '789abc123efg456'),
delta_new_package_id,
block_map={problem_name: {'problem': 'problem3'}}
)
# now the ambiguous query should return delta
again_prob_name = uuid.uuid4().hex
while again_prob_name.startswith('abc') or again_prob_name.startswith(new_prob_name[:3]):
again_prob_name = uuid.uuid4().hex
again_prob_locn = location.replace(name=again_prob_name)
again_usage_id = 'problem{}'.format(again_prob_name[:3])
self.translate_n_check(again_prob_locn, old_style_course_id, new_style_package_id, again_usage_id, 'published', True)
self.translate_n_check(
again_prob_locn, delta_course_locn.course_id, delta_new_package_id, again_usage_id, 'published', True
)
self.translate_n_check(again_prob_locn, None, delta_new_package_id, again_usage_id, 'published', True)
def test_translate_locator(self):
"""
@@ -292,18 +258,23 @@ class TestLocationMapper(LocMapperSetupSansDjango):
# lookup for non-existent course
org = 'foo_org'
course = 'bar_course'
new_style_package_id = '{}.geek_dept.{}.baz_run'.format(org, course)
run = 'baz_run'
new_style_org = '{}.geek_dept'.format(org)
new_style_offering = '{}.{}'.format(course, run)
prob_course_key = CourseLocator(
org=new_style_org, offering=new_style_offering,
branch='published',
)
prob_locator = BlockUsageLocator(
package_id=new_style_package_id,
prob_course_key,
block_id='problem2',
branch='published'
)
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
self.assertIsNone(prob_location, 'found entry in empty map table')
loc_mapper().create_map_entry(
Location('i4x', org, course, 'course', 'baz_run'),
new_style_package_id,
SlashSeparatedCourseKey(org, course, run),
new_style_org, new_style_offering,
block_map={
'abc123': {'problem': 'problem2'},
'48f23a10395384929234': {'chapter': 'chapter48f'},
@@ -313,74 +284,56 @@ class TestLocationMapper(LocMapperSetupSansDjango):
# only one course matches
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
# default branch
self.assertEqual(prob_location, Location('i4x', org, course, 'problem', 'abc123', None))
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', None))
# test get_course keyword
prob_location = loc_mapper().translate_locator_to_location(prob_locator, get_course=True)
self.assertEqual(prob_location, Location('i4x', org, course, 'course', 'baz_run', None))
self.assertEqual(prob_location, SlashSeparatedCourseKey(org, course, run))
# explicit branch
prob_locator = BlockUsageLocator(
package_id=prob_locator.package_id, branch='draft', block_id=prob_locator.block_id
prob_course_key.for_branch('draft'), block_id=prob_locator.block_id
)
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
# Even though the problem was set as draft, we always return revision=None to work
# with old mongo/draft modulestores.
self.assertEqual(prob_location, Location('i4x', org, course, 'problem', 'abc123', None))
prob_locator = BlockUsageLocator(
package_id=new_style_package_id, block_id='problem2', branch='production'
)
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', None))
prob_locator = BlockUsageLocator(prob_course_key.for_branch('production'), block_id='problem2')
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
self.assertEqual(prob_location, Location('i4x', org, course, 'problem', 'abc123', None))
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', None))
# same for chapter except chapter cannot be draft in old system
chap_locator = BlockUsageLocator(
package_id=new_style_package_id,
prob_course_key.for_branch('production'),
block_id='chapter48f',
branch='production'
)
chap_location = loc_mapper().translate_locator_to_location(chap_locator)
self.assertEqual(chap_location, Location('i4x', org, course, 'chapter', '48f23a10395384929234'))
self.assertEqual(chap_location, Location(org, course, run, 'chapter', '48f23a10395384929234'))
# explicit branch
chap_locator.branch = 'draft'
chap_locator = chap_locator.for_branch('draft')
chap_location = loc_mapper().translate_locator_to_location(chap_locator)
self.assertEqual(chap_location, Location('i4x', org, course, 'chapter', '48f23a10395384929234'))
self.assertEqual(chap_location, Location(org, course, run, 'chapter', '48f23a10395384929234'))
chap_locator = BlockUsageLocator(
package_id=new_style_package_id, block_id='chapter48f', branch='production'
prob_course_key.for_branch('production'), block_id='chapter48f'
)
chap_location = loc_mapper().translate_locator_to_location(chap_locator)
self.assertEqual(chap_location, Location('i4x', org, course, 'chapter', '48f23a10395384929234'))
self.assertEqual(chap_location, Location(org, course, run, 'chapter', '48f23a10395384929234'))
# look for non-existent problem
prob_locator2 = BlockUsageLocator(
package_id=new_style_package_id,
branch='draft',
prob_course_key.for_branch('draft'),
block_id='problem3'
)
prob_location = loc_mapper().translate_locator_to_location(prob_locator2)
self.assertIsNone(prob_location, 'Found non-existent problem')
# add a distractor course
new_style_package_id = '{}.geek_dept.{}.{}'.format(org, course, 'delta_run')
delta_run = 'delta_run'
new_style_offering = '{}.{}'.format(course, delta_run)
loc_mapper().create_map_entry(
Location('i4x', org, course, 'course', 'delta_run'),
new_style_package_id,
SlashSeparatedCourseKey(org, course, delta_run),
new_style_org, new_style_offering,
block_map={'abc123': {'problem': 'problem3'}}
)
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
self.assertEqual(prob_location, Location('i4x', org, course, 'problem', 'abc123', None))
# add a default course pointing to the delta_run
loc_mapper().create_map_entry(
Location('i4x', org, course, 'problem', '789abc123efg456'),
new_style_package_id,
block_map={'abc123': {'problem': 'problem3'}}
)
# now query delta (2 entries point to it)
prob_locator = BlockUsageLocator(
package_id=new_style_package_id,
branch='production',
block_id='problem3'
)
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
self.assertEqual(prob_location, Location('i4x', org, course, 'problem', 'abc123'))
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', None))
def test_special_chars(self):
"""
@@ -390,10 +343,8 @@ class TestLocationMapper(LocMapperSetupSansDjango):
org = 'foo.org.edu'
course = 'bar.course-4'
name = 'baz.run_4-3'
old_style_course_id = '{}/{}/{}'.format(org, course, name)
location = Location('i4x', org, course, 'course', name)
location = Location(org, course, name, 'course', name)
prob_locator = loc_mapper().translate_location(
old_style_course_id,
location,
add_entry_if_missing=True
)
@@ -407,17 +358,17 @@ class TestLocationMapper(LocMapperSetupSansDjango):
org = "myorg"
course = "another_course"
name = "running_again"
course_location = Location('i4x', org, course, 'course', name)
course_xlate = loc_mapper().translate_location(None, course_location, add_entry_if_missing=True)
course_location = Location(org, course, name, 'course', name)
course_xlate = loc_mapper().translate_location(course_location, add_entry_if_missing=True)
self.assertEqual(course_location, loc_mapper().translate_locator_to_location(course_xlate))
eponymous_block = course_location.replace(category='chapter')
chapter_xlate = loc_mapper().translate_location(None, eponymous_block, add_entry_if_missing=True)
chapter_xlate = loc_mapper().translate_location(eponymous_block, add_entry_if_missing=True)
self.assertEqual(course_location, loc_mapper().translate_locator_to_location(course_xlate))
self.assertEqual(eponymous_block, loc_mapper().translate_locator_to_location(chapter_xlate))
# and a non-existent one w/o add
eponymous_block = course_location.replace(category='problem')
with self.assertRaises(ItemNotFoundError):
chapter_xlate = loc_mapper().translate_location(None, eponymous_block, add_entry_if_missing=False)
chapter_xlate = loc_mapper().translate_location(eponymous_block, add_entry_if_missing=False)
#==================================

351
common/lib/xmodule/xmodule/modulestore/tests/test_locators.py
View File

@@ -3,12 +3,12 @@ Tests for xmodule.modulestore.locator.
"""
from unittest import TestCase
import random
from bson.objectid import ObjectId
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locator import Locator, CourseLocator, BlockUsageLocator, DefinitionLocator
from xmodule.modulestore.parsers import BRANCH_PREFIX, BLOCK_PREFIX, VERSION_PREFIX
from xmodule.modulestore.exceptions import InsufficientSpecificationError, OverSpecificationError
from xmodule.modulestore import Location
import random
from ddt import ddt
class LocatorTest(TestCase):
@@ -19,232 +19,190 @@ class LocatorTest(TestCase):
def test_cant_instantiate_abstract_class(self):
self.assertRaises(TypeError, Locator)
def test_course_constructor_overspecified(self):
self.assertRaises(
OverSpecificationError,
CourseLocator,
url='edx://mit.eecs.6002x',
package_id='harvard.history',
branch='published',
version_guid=ObjectId())
self.assertRaises(
OverSpecificationError,
CourseLocator,
url='edx://mit.eecs.6002x',
package_id='harvard.history',
version_guid=ObjectId())
self.assertRaises(
OverSpecificationError,
CourseLocator,
url='edx://mit.eecs.6002x/' + BRANCH_PREFIX + 'published',
branch='draft')
self.assertRaises(
OverSpecificationError,
CourseLocator,
package_id='mit.eecs.6002x/' + BRANCH_PREFIX + 'published',
branch='draft')
def test_course_constructor_underspecified(self):
self.assertRaises(InsufficientSpecificationError, CourseLocator)
self.assertRaises(InsufficientSpecificationError, CourseLocator, branch='published')
with self.assertRaises(InvalidKeyError):
CourseLocator()
with self.assertRaises(InvalidKeyError):
CourseLocator(branch='published')
def test_course_constructor_bad_version_guid(self):
self.assertRaises(ValueError, CourseLocator, version_guid="012345")
self.assertRaises(InsufficientSpecificationError, CourseLocator, version_guid=None)
with self.assertRaises(ValueError):
CourseLocator(version_guid="012345")
with self.assertRaises(InvalidKeyError):
CourseLocator(version_guid=None)
def test_course_constructor_version_guid(self):
# generate a random location
test_id_1 = ObjectId()
test_id_1_loc = str(test_id_1)
testobj_1 = CourseLocator(version_guid=test_id_1)
self.check_course_locn_fields(testobj_1, 'version_guid', version_guid=test_id_1)
self.check_course_locn_fields(testobj_1, version_guid=test_id_1)
self.assertEqual(str(testobj_1.version_guid), test_id_1_loc)
self.assertEqual(str(testobj_1), VERSION_PREFIX + test_id_1_loc)
self.assertEqual(testobj_1.url(), 'edx://' + VERSION_PREFIX + test_id_1_loc)
self.assertEqual(testobj_1._to_string(), VERSION_PREFIX + test_id_1_loc)
# Test using a given string
test_id_2_loc = '519665f6223ebd6980884f2b'
test_id_2 = ObjectId(test_id_2_loc)
testobj_2 = CourseLocator(version_guid=test_id_2)
self.check_course_locn_fields(testobj_2, 'version_guid', version_guid=test_id_2)
self.check_course_locn_fields(testobj_2, version_guid=test_id_2)
self.assertEqual(str(testobj_2.version_guid), test_id_2_loc)
self.assertEqual(str(testobj_2), VERSION_PREFIX + test_id_2_loc)
self.assertEqual(testobj_2.url(), 'edx://' + VERSION_PREFIX + test_id_2_loc)
self.assertEqual(testobj_2._to_string(), VERSION_PREFIX + test_id_2_loc)
def test_course_constructor_bad_package_id(self):
@ddt.data(
' mit.eecs',
'mit.eecs ',
VERSION_PREFIX + 'mit.eecs',
BLOCK_PREFIX + 'black/mit.eecs',
'mit.ee cs',
'mit.ee,cs',
'mit.ee/cs',
'mit.ee&cs',
'mit.ee()cs',
BRANCH_PREFIX + 'this',
'mit.eecs/' + BRANCH_PREFIX,
'mit.eecs/' + BRANCH_PREFIX + 'this/' + BRANCH_PREFIX + 'that',
'mit.eecs/' + BRANCH_PREFIX + 'this/' + BRANCH_PREFIX,
'mit.eecs/' + BRANCH_PREFIX + 'this ',
'mit.eecs/' + BRANCH_PREFIX + 'th%is ',
)
def test_course_constructor_bad_package_id(self, bad_id):
"""
Test all sorts of badly-formed package_ids (and urls with those package_ids)
"""
for bad_id in (' mit.eecs',
'mit.eecs ',
VERSION_PREFIX + 'mit.eecs',
BLOCK_PREFIX + 'black/mit.eecs',
'mit.ee cs',
'mit.ee,cs',
'mit.ee/cs',
'mit.ee&cs',
'mit.ee()cs',
BRANCH_PREFIX + 'this',
'mit.eecs/' + BRANCH_PREFIX,
'mit.eecs/' + BRANCH_PREFIX + 'this/' + BRANCH_PREFIX + 'that',
'mit.eecs/' + BRANCH_PREFIX + 'this/' + BRANCH_PREFIX,
'mit.eecs/' + BRANCH_PREFIX + 'this ',
'mit.eecs/' + BRANCH_PREFIX + 'th%is ',
):
self.assertRaises(ValueError, CourseLocator, package_id=bad_id)
self.assertRaises(ValueError, CourseLocator, url='edx://' + bad_id)
with self.assertRaises(InvalidKeyError):
CourseLocator(org=bad_id, offering='test')
def test_course_constructor_bad_url(self):
for bad_url in ('edx://',
'edx:/mit.eecs',
'http://mit.eecs',
'edx//mit.eecs'):
self.assertRaises(ValueError, CourseLocator, url=bad_url)
with self.assertRaises(InvalidKeyError):
CourseLocator(org='test', offering=bad_id)
def test_course_constructor_redundant_001(self):
testurn = 'mit.eecs.6002x'
testobj = CourseLocator(package_id=testurn, url='edx://' + testurn)
self.check_course_locn_fields(testobj, 'package_id', package_id=testurn)
with self.assertRaises(InvalidKeyError):
CourseLocator.from_string('course-locator:' + bad_id)
def test_course_constructor_redundant_002(self):
testurn = 'mit.eecs.6002x/' + BRANCH_PREFIX + 'published'
expected_urn = 'mit.eecs.6002x'
expected_rev = 'published'
testobj = CourseLocator(package_id=testurn, url='edx://' + testurn)
self.check_course_locn_fields(testobj, 'package_id',
package_id=expected_urn,
branch=expected_rev)
@ddt.data('course-locator:', 'course-locator:/mit.eecs', 'http:mit.eecs', 'course-locator//mit.eecs')
def test_course_constructor_bad_url(self, bad_url):
with self.assertRaises(InvalidKeyError):
CourseLocator.from_string(bad_url)
def test_course_constructor_url(self):
# Test parsing a url when it starts with a version ID and there is also a block ID.
# This hits the parsers parse_guid method.
test_id_loc = '519665f6223ebd6980884f2b'
testobj = CourseLocator(url="edx://{}{}/{}hw3".format(VERSION_PREFIX, test_id_loc, BLOCK_PREFIX))
testobj = CourseLocator.from_string("course-locator:{}{}/{}hw3".format(VERSION_PREFIX, test_id_loc, BLOCK_PREFIX))
self.check_course_locn_fields(
testobj,
'test_block constructor',
version_guid=ObjectId(test_id_loc)
)
def test_course_constructor_url_package_id_and_version_guid(self):
test_id_loc = '519665f6223ebd6980884f2b'
testobj = CourseLocator(url='edx://mit.eecs-honors.6002x/' + VERSION_PREFIX + test_id_loc)
self.check_course_locn_fields(testobj, 'error parsing url with both course ID and version GUID',
package_id='mit.eecs-honors.6002x',
version_guid=ObjectId(test_id_loc))
testobj = CourseLocator.from_string('course-locator:mit.eecs+honors.6002x/' + VERSION_PREFIX + test_id_loc)
self.check_course_locn_fields(
testobj,
org='mit.eecs',
offering='honors.6002x',
version_guid=ObjectId(test_id_loc)
)
def test_course_constructor_url_package_id_branch_and_version_guid(self):
test_id_loc = '519665f6223ebd6980884f2b'
testobj = CourseLocator(url='edx://mit.eecs.~6002x/' + BRANCH_PREFIX + 'draft-1/' + VERSION_PREFIX + test_id_loc)
self.check_course_locn_fields(testobj, 'error parsing url with both course ID branch, and version GUID',
package_id='mit.eecs.~6002x',
branch='draft-1',
version_guid=ObjectId(test_id_loc))
org = 'mit.eecs'
offering = '~6002x'
testobj = CourseLocator.from_string('course-locator:{}+{}/{}draft-1/{}{}'.format(
org, offering, BRANCH_PREFIX, VERSION_PREFIX, test_id_loc
))
self.check_course_locn_fields(
testobj,
org=org,
offering=offering,
branch='draft-1',
version_guid=ObjectId(test_id_loc)
)
def test_course_constructor_package_id_no_branch(self):
testurn = 'mit.eecs.6002x'
testobj = CourseLocator(package_id=testurn)
self.check_course_locn_fields(testobj, 'package_id', package_id=testurn)
org = 'mit.eecs'
offering = '6002x'
testurn = '{}+{}'.format(org, offering)
testobj = CourseLocator(org=org, offering=offering)
self.check_course_locn_fields(testobj, org=org, offering=offering)
self.assertEqual(testobj.package_id, testurn)
self.assertEqual(str(testobj), testurn)
self.assertEqual(testobj.url(), 'edx://' + testurn)
def test_course_constructor_package_id_with_branch(self):
testurn = 'mit.eecs.6002x/' + BRANCH_PREFIX + 'published'
expected_id = 'mit.eecs.6002x'
expected_branch = 'published'
testobj = CourseLocator(package_id=testurn)
self.check_course_locn_fields(testobj, 'package_id with branch',
package_id=expected_id,
branch=expected_branch,
)
self.assertEqual(testobj.package_id, expected_id)
self.assertEqual(testobj.branch, expected_branch)
self.assertEqual(str(testobj), testurn)
self.assertEqual(testobj.url(), 'edx://' + testurn)
self.assertEqual(testobj._to_string(), testurn)
def test_course_constructor_package_id_separate_branch(self):
test_id = 'mit.eecs.6002x'
org = 'mit.eecs'
offering = '6002x'
testurn = '{}+{}'.format(org, offering)
test_branch = 'published'
expected_urn = 'mit.eecs.6002x/' + BRANCH_PREFIX + 'published'
testobj = CourseLocator(package_id=test_id, branch=test_branch)
self.check_course_locn_fields(testobj, 'package_id with separate branch',
package_id=test_id,
branch=test_branch,
)
self.assertEqual(testobj.package_id, test_id)
expected_urn = '{}+{}/{}{}'.format(org, offering, BRANCH_PREFIX, test_branch)
testobj = CourseLocator(org=org, offering=offering, branch=test_branch)
self.check_course_locn_fields(
testobj,
org=org,
offering=offering,
branch=test_branch,
)
self.assertEqual(testobj.package_id, testurn)
self.assertEqual(testobj.branch, test_branch)
self.assertEqual(str(testobj), expected_urn)
self.assertEqual(testobj.url(), 'edx://' + expected_urn)
def test_course_constructor_package_id_repeated_branch(self):
"""
The same branch appears in the package_id and the branch field.
"""
test_id = 'mit.eecs.6002x/' + BRANCH_PREFIX + 'published'
test_branch = 'published'
expected_id = 'mit.eecs.6002x'
expected_urn = test_id
testobj = CourseLocator(package_id=test_id, branch=test_branch)
self.check_course_locn_fields(testobj, 'package_id with repeated branch',
package_id=expected_id,
branch=test_branch,
)
self.assertEqual(testobj.package_id, expected_id)
self.assertEqual(testobj.branch, test_branch)
self.assertEqual(str(testobj), expected_urn)
self.assertEqual(testobj.url(), 'edx://' + expected_urn)
self.assertEqual(testobj._to_string(), expected_urn)
def test_block_constructor(self):
testurn = 'mit.eecs.6002x/' + BRANCH_PREFIX + 'published/' + BLOCK_PREFIX + 'HW3'
expected_id = 'mit.eecs.6002x'
expected_org = 'mit.eecs'
expected_offering = '6002x'
expected_branch = 'published'
expected_block_ref = 'HW3'
testobj = BlockUsageLocator(url=testurn)
self.check_block_locn_fields(testobj, 'test_block constructor',
package_id=expected_id,
testurn = 'edx:{}+{}/{}{}/{}{}'.format(
expected_org, expected_offering, BRANCH_PREFIX, expected_branch, BLOCK_PREFIX, 'HW3'
)
testobj = BlockUsageLocator.from_string(testurn)
self.check_block_locn_fields(testobj,
org=expected_org,
offering=expected_offering,
branch=expected_branch,
block=expected_block_ref)
self.assertEqual(str(testobj), testurn)
self.assertEqual(testobj.url(), 'edx://' + testurn)
testobj = BlockUsageLocator(url=testurn, version_guid=ObjectId())
self.assertEqual(unicode(testobj), testurn)
testobj = BlockUsageLocator(testobj.course_key.for_version(ObjectId()), testobj.block_id)
agnostic = testobj.version_agnostic()
self.assertIsNone(agnostic.version_guid)
self.check_block_locn_fields(agnostic, 'test_block constructor',
package_id=expected_id,
self.check_block_locn_fields(agnostic,
org=expected_org,
offering=expected_offering,
branch=expected_branch,
block=expected_block_ref)
def test_block_constructor_url_version_prefix(self):
test_id_loc = '519665f6223ebd6980884f2b'
testobj = BlockUsageLocator(
url='edx://mit.eecs.6002x/{}{}/{}lab2'.format(VERSION_PREFIX, test_id_loc, BLOCK_PREFIX)
testobj = BlockUsageLocator.from_string(
'edx:mit.eecs+6002x/{}{}/{}lab2'.format(VERSION_PREFIX, test_id_loc, BLOCK_PREFIX)
)
self.check_block_locn_fields(
testobj, 'error parsing URL with version and block',
package_id='mit.eecs.6002x',
testobj,
org='mit.eecs',
offering='6002x',
block='lab2',
version_guid=ObjectId(test_id_loc)
)
agnostic = testobj.course_agnostic()
self.check_block_locn_fields(
agnostic, 'error parsing URL with version and block',
agnostic,
block='lab2',
package_id=None,
org=None,
offering=None,
version_guid=ObjectId(test_id_loc)
)
self.assertIsNone(agnostic.package_id)
self.assertIsNone(agnostic.offering)
self.assertIsNone(agnostic.org)
def test_block_constructor_url_kitchen_sink(self):
test_id_loc = '519665f6223ebd6980884f2b'
testobj = BlockUsageLocator(
url='edx://mit.eecs.6002x/{}draft/{}{}/{}lab2'.format(
testobj = BlockUsageLocator.from_string(
'edx:mit.eecs+6002x/{}draft/{}{}/{}lab2'.format(
BRANCH_PREFIX, VERSION_PREFIX, test_id_loc, BLOCK_PREFIX
)
)
self.check_block_locn_fields(
testobj, 'error parsing URL with branch, version, and block',
package_id='mit.eecs.6002x',
testobj,
org='mit.eecs',
offering='6002x',
branch='draft',
block='lab2',
version_guid=ObjectId(test_id_loc)
@@ -254,71 +212,50 @@ class LocatorTest(TestCase):
"""
It seems we used to use colons in names; so, ensure they're acceptable.
"""
package_id = 'mit.eecs-1'
org = 'mit.eecs'
offering = '1'
branch = 'foo'
block_id = 'problem:with-colon~2'
testobj = BlockUsageLocator(package_id=package_id, branch=branch, block_id=block_id)
self.check_block_locn_fields(testobj, 'Cannot handle colon', package_id=package_id, branch=branch, block=block_id)
testobj = BlockUsageLocator(
CourseLocator(org=org, offering=offering, branch=branch),
block_id=block_id
)
self.check_block_locn_fields(
testobj, org=org, offering=offering, branch=branch, block=block_id
)
def test_relative(self):
"""
Test making a relative usage locator.
"""
package_id = 'mit.eecs-1'
org = 'mit.eecs'
offering = '1'
branch = 'foo'
baseobj = CourseLocator(package_id=package_id, branch=branch)
baseobj = CourseLocator(org=org, offering=offering, branch=branch)
block_id = 'problem:with-colon~2'
testobj = BlockUsageLocator.make_relative(baseobj, block_id)
self.check_block_locn_fields(
testobj, 'Cannot make relative to course', package_id=package_id, branch=branch, block=block_id
testobj, org=org, offering=offering, branch=branch, block=block_id
)
block_id = 'completely_different'
testobj = BlockUsageLocator.make_relative(testobj, block_id)
self.check_block_locn_fields(
testobj, 'Cannot make relative to block usage', package_id=package_id, branch=branch, block=block_id
testobj, org=org, offering=offering, branch=branch, block=block_id
)
def test_repr(self):
testurn = 'mit.eecs.6002x/' + BRANCH_PREFIX + 'published/' + BLOCK_PREFIX + 'HW3'
testobj = BlockUsageLocator(package_id=testurn)
self.assertEqual('BlockUsageLocator("mit.eecs.6002x/branch/published/block/HW3")', repr(testobj))
def test_old_location_helpers(self):
"""
Test the functions intended to help with the conversion from old locations to locators
"""
location_tuple = ('i4x', 'mit', 'eecs.6002x', 'course', 't3_2013')
location = Location(location_tuple)
self.assertEqual(location, Locator.to_locator_or_location(location))
self.assertEqual(location, Locator.to_locator_or_location(location_tuple))
self.assertEqual(location, Locator.to_locator_or_location(list(location_tuple)))
self.assertEqual(location, Locator.to_locator_or_location(location.dict()))
locator = BlockUsageLocator(package_id='foo.bar', branch='alpha', block_id='deep')
self.assertEqual(locator, Locator.to_locator_or_location(locator))
self.assertEqual(locator.as_course_locator(), Locator.to_locator_or_location(locator.as_course_locator()))
self.assertEqual(location, Locator.to_locator_or_location(location.url()))
self.assertEqual(locator, Locator.to_locator_or_location(locator.url()))
self.assertEqual(locator, Locator.to_locator_or_location(locator.__dict__))
asset_location = Location(['c4x', 'mit', 'eecs.6002x', 'asset', 'selfie.jpeg'])
self.assertEqual(asset_location, Locator.to_locator_or_location(asset_location))
self.assertEqual(asset_location, Locator.to_locator_or_location(asset_location.url()))
def_location_url = "defx://version/" + '{:024x}'.format(random.randrange(16 ** 24))
self.assertEqual(DefinitionLocator(def_location_url), Locator.to_locator_or_location(def_location_url))
with self.assertRaises(ValueError):
Locator.to_locator_or_location(22)
with self.assertRaises(ValueError):
Locator.to_locator_or_location("hello.world.not.a.url")
self.assertIsNone(Locator.parse_url("unknown://foo.bar/baz"))
testurn = 'edx:mit.eecs+6002x/' + BRANCH_PREFIX + 'published/' + BLOCK_PREFIX + 'HW3'
testobj = BlockUsageLocator.from_string(testurn)
self.assertEqual("BlockUsageLocator(CourseLocator(u'mit.eecs', u'6002x', u'published', None), u'HW3')", repr(testobj))
def test_url_reverse(self):
"""
Test the url_reverse method
"""
locator = CourseLocator(package_id="a.fancy_course-id", branch="branch_1.2-3")
locator = BlockUsageLocator(
CourseLocator(org="a", offering="fancy_course-id", branch="branch_1.2-3"),
block_id='element'
)
self.assertEqual(
'/expression/{}/format'.format(unicode(locator)),
locator.url_reverse('expression', 'format')
@@ -339,8 +276,7 @@ class LocatorTest(TestCase):
def test_description_locator_url(self):
object_id = '{:024x}'.format(random.randrange(16 ** 24))
definition_locator = DefinitionLocator(object_id)
self.assertEqual('defx://' + VERSION_PREFIX + object_id, definition_locator.url())
self.assertEqual(definition_locator, DefinitionLocator(definition_locator.url()))
self.assertEqual('defx:' + VERSION_PREFIX + object_id, unicode(definition_locator))
def test_description_locator_version(self):
object_id = '{:024x}'.format(random.randrange(16 ** 24))
@@ -350,20 +286,21 @@ class LocatorTest(TestCase):
# ------------------------------------------------------------------
# Utilities
def check_course_locn_fields(self, testobj, msg, version_guid=None,
package_id=None, branch=None):
def check_course_locn_fields(self, testobj, version_guid=None,
org=None, offering=None, branch=None):
"""
Checks the version, package_id, and branch in testobj
Checks the version, org, offering, and branch in testobj
"""
self.assertEqual(testobj.version_guid, version_guid, msg)
self.assertEqual(testobj.package_id, package_id, msg)
self.assertEqual(testobj.branch, branch, msg)
self.assertEqual(testobj.version_guid, version_guid)
self.assertEqual(testobj.org, org)
self.assertEqual(testobj.offering, offering)
self.assertEqual(testobj.branch, branch)
def check_block_locn_fields(self, testobj, msg, version_guid=None,
package_id=None, branch=None, block=None):
def check_block_locn_fields(self, testobj, version_guid=None,
org=None, offering=None, branch=None, block=None):
"""
Does adds a block id check over and above the check_course_locn_fields tests
"""
self.check_course_locn_fields(testobj, msg, version_guid, package_id,
self.check_course_locn_fields(testobj, version_guid, org, offering,
branch)
self.assertEqual(testobj.block_id, block)

175
common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py
View File

@@ -14,6 +14,8 @@ from xmodule.modulestore.tests.test_location_mapper import LocMapperSetupSansDja
# Mixed modulestore depends on django, so we'll manually configure some django settings
# before importing the module
from django.conf import settings
from xmodule.modulestore.locations import SlashSeparatedCourseKey
import bson.son
if not settings.configured:
settings.configure()
from xmodule.modulestore.mixed import MixedModuleStore
@@ -83,7 +85,7 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
"""
AssertEqual replacement for CourseLocator
"""
if not (loc1.package_id == loc2.package_id and loc1.branch == loc2.branch and loc1.block_id == loc2.block_id):
if loc1.version_agnostic() != loc2.version_agnostic():
self.fail(self._formatMessage(msg, u"{} != {}".format(unicode(loc1), unicode(loc2))))
def setUp(self):
@@ -95,6 +97,7 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
host=self.HOST,
port=self.PORT,
tz_aware=True,
document_class=bson.son.SON,
)
self.connection.drop_database(self.DB)
self.addCleanup(self.connection.drop_database, self.DB)
@@ -109,29 +112,40 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
patcher.start()
self.addCleanup(patcher.stop)
self.addTypeEqualityFunc(BlockUsageLocator, '_compareIgnoreVersion')
self.addTypeEqualityFunc(CourseLocator, '_compareIgnoreVersion')
# define attrs which get set in initdb to quell pylint
self.import_chapter_location = self.store = self.fake_location = self.xml_chapter_location = None
self.course_locations = []
# pylint: disable=invalid-name
def _create_course(self, default, course_id):
def _create_course(self, default, course_key):
"""
Create a course w/ one item in the persistence store using the given course & item location.
"""
course = self.store.create_course(course_id, store_name=default)
if default == 'split':
offering = course_key.offering.replace('/', '.')
else:
offering = course_key.offering
course = self.store.create_course(course_key.org, offering, store_name=default)
category = self.import_chapter_location.category
block_id = self.import_chapter_location.name
chapter = self.store.create_item(
# don't use course_location as it may not be the repr
course.location, category, location=self.import_chapter_location, block_id=block_id
)
if isinstance(course.location, CourseLocator):
if isinstance(course.id, CourseLocator):
self.course_locations[self.MONGO_COURSEID] = course.location.version_agnostic()
self.import_chapter_location = chapter.location.version_agnostic()
else:
self.assertEqual(course.location.course_id, course_id)
self.assertEqual(course.id, course_key)
self.assertEqual(chapter.location, self.import_chapter_location)
def _course_key_from_string(self, string):
"""
Get the course key for the given course string
"""
return self.course_locations[string].course_key
def initdb(self, default):
"""
Initialize the database and create one test course in it
@@ -141,11 +155,17 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
self.store = MixedModuleStore(**self.options)
self.addCleanup(self.store.close_all_connections)
# convert to CourseKeys
self.course_locations = {
course_id: generate_location(course_id)
course_id: SlashSeparatedCourseKey.from_deprecated_string(course_id)
for course_id in [self.MONGO_COURSEID, self.XML_COURSEID1, self.XML_COURSEID2]
}
self.fake_location = Location('i4x', 'foo', 'bar', 'vertical', 'baz')
# and then to the root UsageKey
self.course_locations = {
course_id: course_key.make_usage_key('course', course_key.run)
for course_id, course_key in self.course_locations.iteritems() # pylint: disable=maybe-no-member
}
self.fake_location = Location('foo', 'bar', 'slowly', 'vertical', 'baz')
self.import_chapter_location = self.course_locations[self.MONGO_COURSEID].replace(
category='chapter', name='Overview'
)
@@ -154,9 +174,9 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
)
# get Locators and set up the loc mapper if app is Locator based
if default == 'split':
self.fake_location = loc_mapper().translate_location('foo/bar/2012_Fall', self.fake_location)
self.fake_location = loc_mapper().translate_location(self.fake_location)
self._create_course(default, self.MONGO_COURSEID)
self._create_course(default, self.course_locations[self.MONGO_COURSEID].course_key)
@ddt.data('direct', 'split')
def test_get_modulestore_type(self, default_ms):
@@ -164,57 +184,54 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
Make sure we get back the store type we expect for given mappings
"""
self.initdb(default_ms)
self.assertEqual(self.store.get_modulestore_type(self.XML_COURSEID1), XML_MODULESTORE_TYPE)
self.assertEqual(self.store.get_modulestore_type(self.XML_COURSEID2), XML_MODULESTORE_TYPE)
self.assertEqual(self.store.get_modulestore_type(
self._course_key_from_string(self.XML_COURSEID1)), XML_MODULESTORE_TYPE
)
self.assertEqual(self.store.get_modulestore_type(
self._course_key_from_string(self.XML_COURSEID2)), XML_MODULESTORE_TYPE
)
mongo_ms_type = MONGO_MODULESTORE_TYPE if default_ms == 'direct' else SPLIT_MONGO_MODULESTORE_TYPE
self.assertEqual(self.store.get_modulestore_type(self.MONGO_COURSEID), mongo_ms_type)
self.assertEqual(self.store.get_modulestore_type(
self._course_key_from_string(self.MONGO_COURSEID)), mongo_ms_type
)
# try an unknown mapping, it should be the 'default' store
self.assertEqual(self.store.get_modulestore_type('foo/bar/2012_Fall'), mongo_ms_type)
self.assertEqual(self.store.get_modulestore_type(
SlashSeparatedCourseKey('foo', 'bar', '2012_Fall')), mongo_ms_type
)
@ddt.data('direct', 'split')
def test_has_item(self, default_ms):
self.initdb(default_ms)
for course_id, course_locn in self.course_locations.iteritems():
self.assertTrue(self.store.has_item(course_id, course_locn))
for course_locn in self.course_locations.itervalues(): # pylint: disable=maybe-no-member
self.assertTrue(self.store.has_item(course_locn))
# try negative cases
self.assertFalse(self.store.has_item(
self.XML_COURSEID1,
self.course_locations[self.XML_COURSEID1].replace(name='not_findable', category='problem')
))
self.assertFalse(self.store.has_item(self.MONGO_COURSEID, self.fake_location))
self.assertFalse(self.store.has_item(self.fake_location))
@ddt.data('direct', 'split')
def test_get_item(self, default_ms):
self.initdb(default_ms)
with self.assertRaises(NotImplementedError):
self.store.get_item(self.fake_location)
@ddt.data('direct', 'split')
def test_get_instance(self, default_ms):
self.initdb(default_ms)
for course_id, course_locn in self.course_locations.iteritems():
self.assertIsNotNone(self.store.get_instance(course_id, course_locn))
for course_locn in self.course_locations.itervalues(): # pylint: disable=maybe-no-member
self.assertIsNotNone(self.store.get_item(course_locn))
# try negative cases
with self.assertRaises(ItemNotFoundError):
self.store.get_instance(
self.XML_COURSEID1,
self.store.get_item(
self.course_locations[self.XML_COURSEID1].replace(name='not_findable', category='problem')
)
with self.assertRaises(ItemNotFoundError):
self.store.get_instance(self.MONGO_COURSEID, self.fake_location)
self.store.get_item(self.fake_location)
@ddt.data('direct', 'split')
def test_get_items(self, default_ms):
self.initdb(default_ms)
for course_id, course_locn in self.course_locations.iteritems():
if hasattr(course_locn, 'as_course_locator'):
locn = course_locn.as_course_locator()
else:
locn = course_locn.replace(org=None, course=None, name=None)
for course_locn in self.course_locations.itervalues(): # pylint: disable=maybe-no-member
locn = course_locn.course_key
# NOTE: use get_course if you just want the course. get_items is expensive
modules = self.store.get_items(locn, course_id, qualifiers={'category': 'course'})
modules = self.store.get_items(locn, category='course')
self.assertEqual(len(modules), 1)
self.assertEqual(modules[0].location, course_locn)
@@ -224,25 +241,19 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
Update should fail for r/o dbs and succeed for r/w ones
"""
self.initdb(default_ms)
course_id = self.XML_COURSEID1
course = self.store.get_course(course_id)
course = self.store.get_course(self.course_locations[self.XML_COURSEID1].course_key)
# if following raised, then the test is really a noop, change it
self.assertFalse(course.show_calculator, "Default changed making test meaningless")
course.show_calculator = True
with self.assertRaises(NotImplementedError):
with self.assertRaises(AttributeError): # ensure it doesn't allow writing
self.store.update_item(course, None)
# now do it for a r/w db
# get_course api's are inconsistent: one takes Locators the other an old style course id
if hasattr(self.course_locations[self.MONGO_COURSEID], 'as_course_locator'):
locn = self.course_locations[self.MONGO_COURSEID]
else:
locn = self.MONGO_COURSEID
course = self.store.get_course(locn)
course = self.store.get_course(self.course_locations[self.MONGO_COURSEID].course_key)
# if following raised, then the test is really a noop, change it
self.assertFalse(course.show_calculator, "Default changed making test meaningless")
course.show_calculator = True
self.store.update_item(course, None)
course = self.store.get_course(locn)
course = self.store.get_course(self.course_locations[self.MONGO_COURSEID].course_key)
self.assertTrue(course.show_calculator)
@ddt.data('direct', 'split')
@@ -251,13 +262,13 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
Delete should reject on r/o db and work on r/w one
"""
self.initdb(default_ms)
# r/o try deleting the course
with self.assertRaises(NotImplementedError):
# r/o try deleting the course (is here to ensure it can't be deleted)
with self.assertRaises(AttributeError):
self.store.delete_item(self.xml_chapter_location)
self.store.delete_item(self.import_chapter_location, '**replace_user**')
# verify it's gone
with self.assertRaises(ItemNotFoundError):
self.store.get_instance(self.MONGO_COURSEID, self.import_chapter_location)
self.store.get_item(self.import_chapter_location)
@ddt.data('direct', 'split')
def test_get_courses(self, default_ms):
@@ -281,9 +292,9 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
self.initdb('direct')
courses = self.store.modulestores['xml'].get_courses()
self.assertEqual(len(courses), 2)
course_ids = [course.location.course_id for course in courses]
self.assertIn(self.XML_COURSEID1, course_ids)
self.assertIn(self.XML_COURSEID2, course_ids)
course_ids = [course.id for course in courses]
self.assertIn(self.course_locations[self.XML_COURSEID1].course_key, course_ids)
self.assertIn(self.course_locations[self.XML_COURSEID2].course_key, course_ids)
# this course is in the directory from which we loaded courses but not in the map
self.assertNotIn("edX/toy/TT_2012_Fall", course_ids)
@@ -293,35 +304,25 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
"""
self.initdb('direct')
with self.assertRaises(NotImplementedError):
self.store.create_course("org/course/run", store_name='xml')
self.store.create_course("org", "course/run", store_name='xml')
@ddt.data('direct', 'split')
def test_get_course(self, default_ms):
self.initdb(default_ms)
for course_locn in self.course_locations.itervalues():
if hasattr(course_locn, 'as_course_locator'):
locn = course_locn.as_course_locator()
else:
locn = course_locn.course_id
for course_location in self.course_locations.itervalues(): # pylint: disable=maybe-no-member
# NOTE: use get_course if you just want the course. get_items is expensive
course = self.store.get_course(locn)
course = self.store.get_course(course_location.course_key)
self.assertIsNotNone(course)
self.assertEqual(course.location, course_locn)
self.assertEqual(course.id, course_location.course_key)
@ddt.data('direct', 'split')
def test_get_parent_locations(self, default_ms):
self.initdb(default_ms)
parents = self.store.get_parent_locations(
self.import_chapter_location,
self.MONGO_COURSEID
)
parents = self.store.get_parent_locations(self.import_chapter_location)
self.assertEqual(len(parents), 1)
self.assertEqual(parents[0], self.course_locations[self.MONGO_COURSEID])
parents = self.store.get_parent_locations(
self.xml_chapter_location,
self.XML_COURSEID1
)
parents = self.store.get_parent_locations(self.xml_chapter_location)
self.assertEqual(len(parents), 1)
self.assertEqual(parents[0], self.course_locations[self.XML_COURSEID1])
@@ -329,33 +330,13 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
def test_get_orphans(self, default_ms):
self.initdb(default_ms)
# create an orphan
if default_ms == 'split':
course_id = self.course_locations[self.MONGO_COURSEID].as_course_locator()
branch = course_id.branch
else:
course_id = self.MONGO_COURSEID
branch = None
course_id = self.course_locations[self.MONGO_COURSEID].course_key
orphan = self.store.create_item(course_id, 'problem', block_id='orphan')
found_orphans = self.store.get_orphans(self.course_locations[self.MONGO_COURSEID], branch)
found_orphans = self.store.get_orphans(self.course_locations[self.MONGO_COURSEID].course_key)
if default_ms == 'split':
self.assertEqual(found_orphans, [orphan.location.version_agnostic()])
else:
self.assertEqual(found_orphans, [unicode(orphan.location)])
@ddt.data('split')
def test_create_item_from_course_id(self, default_ms):
"""
Test code paths missed by the above:
* passing an old-style course_id which has a loc map to split's create_item
"""
self.initdb(default_ms)
# create loc_map entry
loc_mapper().translate_location(self.MONGO_COURSEID, generate_location(self.MONGO_COURSEID))
orphan = self.store.create_item(self.MONGO_COURSEID, 'problem', block_id='orphan')
self.assertEqual(
orphan.location.version_agnostic().as_course_locator(),
self.course_locations[self.MONGO_COURSEID].as_course_locator()
)
self.assertEqual(found_orphans, [orphan.location.to_deprecated_string()])
@ddt.data('direct')
def test_create_item_from_parent_location(self, default_ms):
@@ -365,7 +346,7 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
"""
self.initdb(default_ms)
self.store.create_item(self.course_locations[self.MONGO_COURSEID], 'problem', block_id='orphan')
orphans = self.store.get_orphans(self.course_locations[self.MONGO_COURSEID], None)
orphans = self.store.get_orphans(self.course_locations[self.MONGO_COURSEID].course_key)
self.assertEqual(len(orphans), 0, "unexpected orphans: {}".format(orphans))
@ddt.data('direct')
@@ -376,11 +357,11 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
self.initdb(default_ms)
course_locations = self.store.get_courses_for_wiki('toy')
self.assertEqual(len(course_locations), 1)
self.assertIn(Location('i4x', 'edX', 'toy', 'course', '2012_Fall'), course_locations)
self.assertIn(self.course_locations[self.XML_COURSEID1], course_locations)
course_locations = self.store.get_courses_for_wiki('simple')
self.assertEqual(len(course_locations), 1)
self.assertIn(Location('i4x', 'edX', 'simple', 'course', '2012_Fall'), course_locations)
self.assertIn(self.course_locations[self.XML_COURSEID2], course_locations)
self.assertEqual(len(self.store.get_courses_for_wiki('edX.simple.2012_Fall')), 0)
self.assertEqual(len(self.store.get_courses_for_wiki('no_such_wiki')), 0)
@@ -413,13 +394,3 @@ def create_modulestore_instance(engine, doc_store_config, options, i18n_service=
doc_store_config=doc_store_config,
**options
)
def generate_location(course_id):
"""
Generate the locations for the given ids
"""
course_dict = Location.parse_course_id(course_id)
course_dict['tag'] = 'i4x'
course_dict['category'] = 'course'
return Location(course_dict)

62
common/lib/xmodule/xmodule/modulestore/tests/test_modulestore.py
View File

@@ -1,26 +1,70 @@
from nose.tools import assert_equals, assert_raises # pylint: disable=E0611
from nose.tools import assert_equals, assert_raises, assert_true, assert_false # pylint: disable=E0611
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.search import path_to_location
from xmodule.modulestore.locations import SlashSeparatedCourseKey
def check_path_to_location(modulestore):
"""
Make sure that path_to_location works: should be passed a modulestore
with the toy and simple courses loaded.
"""
course_id = SlashSeparatedCourseKey("edX", "toy", "2012_Fall")
should_work = (
("i4x://edX/toy/video/Welcome",
("edX/toy/2012_Fall", "Overview", "Welcome", None)),
("i4x://edX/toy/chapter/Overview",
("edX/toy/2012_Fall", "Overview", None, None)),
(course_id.make_usage_key('video', 'Welcome'),
(course_id, "Overview", "Welcome", None)),
(course_id.make_usage_key('chapter', 'Overview'),
(course_id, "Overview", None, None)),
)
course_id = "edX/toy/2012_Fall"
for location, expected in should_work:
assert_equals(path_to_location(modulestore, course_id, location), expected)
assert_equals(path_to_location(modulestore, location), expected)
not_found = (
"i4x://edX/toy/video/WelcomeX", "i4x://edX/toy/course/NotHome"
course_id.make_usage_key('video', 'WelcomeX'),
course_id.make_usage_key('course', 'NotHome'),
)
for location in not_found:
assert_raises(ItemNotFoundError, path_to_location, modulestore, course_id, location)
with assert_raises(ItemNotFoundError):
path_to_location(modulestore, location)
def check_has_course_method(modulestore, locator, locator_key_fields):
error_message = "Called has_course with query {0} and ignore_case is {1}."
for ignore_case in [True, False]:
# should find the course with exact locator
assert_true(modulestore.has_course(locator, ignore_case))
for key_field in locator_key_fields:
locator_changes_that_should_not_be_found = [ # pylint: disable=invalid-name
# replace value for one of the keys
{key_field: 'fake'},
# add a character at the end
{key_field: getattr(locator, key_field) + 'X'},
# add a character in the beginning
{key_field: 'X' + getattr(locator, key_field)},
]
for changes in locator_changes_that_should_not_be_found:
search_locator = locator.replace(**changes)
assert_false(
modulestore.has_course(search_locator),
error_message.format(search_locator, ignore_case)
)
# test case [in]sensitivity
locator_case_changes = [
{key_field: getattr(locator, key_field).upper()},
{key_field: getattr(locator, key_field).capitalize()},
{key_field: getattr(locator, key_field).capitalize().swapcase()},
]
for changes in locator_case_changes:
search_locator = locator.replace(**changes)
assert_equals(
modulestore.has_course(search_locator, ignore_case),
ignore_case,
error_message.format(search_locator, ignore_case)
)

240
common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py
View File

@@ -1,28 +1,33 @@
from pprint import pprint
# pylint: disable=E0611
from nose.tools import assert_equals, assert_raises, \
assert_not_equals, assert_false
from itertools import ifilter
assert_not_equals, assert_false, assert_true, assert_greater, assert_is_instance
# pylint: enable=E0611
import pymongo
import logging
from uuid import uuid4
import unittest
import bson.son
from xblock.core import XBlock
from xblock.fields import Scope
from xblock.fields import Scope, Reference, ReferenceList, ReferenceValueDict
from xblock.runtime import KeyValueStore
from xblock.exceptions import InvalidScopeError
from xblock.plugin import Plugin
from xmodule.tests import DATA_DIR
from xmodule.modulestore import Location, MONGO_MODULESTORE_TYPE
from xmodule.modulestore.mongo import MongoModuleStore, MongoKeyValueStore
from xmodule.modulestore.draft import DraftModuleStore
from xmodule.modulestore.locations import SlashSeparatedCourseKey, AssetLocation
from xmodule.modulestore.xml_importer import import_from_xml, perform_xlint
from xmodule.contentstore.mongo import MongoContentStore
from xmodule.modulestore.tests.test_modulestore import check_path_to_location
from nose.tools import assert_in
from xmodule.exceptions import NotFoundError
from xmodule.modulestore.exceptions import InsufficientSpecificationError
from git.test.lib.asserts import assert_not_none
from xmodule.x_module import XModuleMixin
log = logging.getLogger(__name__)
@@ -35,7 +40,17 @@ DEFAULT_CLASS = 'xmodule.raw_module.RawDescriptor'
RENDER_TEMPLATE = lambda t_n, d, ctx = None, nsp = 'main': ''
class TestMongoModuleStore(object):
class ReferenceTestXBlock(XBlock):
"""
Test xblock type to test the reference field types
"""
has_children = True
reference_link = Reference(default=None, scope=Scope.content)
reference_list = ReferenceList(scope=Scope.content)
reference_dict = ReferenceValueDict(scope=Scope.settings)
class TestMongoModuleStore(unittest.TestCase):
'''Tests!'''
# Explicitly list the courses to load (don't want the big one)
courses = ['toy', 'simple', 'simple_with_draft', 'test_unicode']
@@ -46,6 +61,7 @@ class TestMongoModuleStore(object):
host=HOST,
port=PORT,
tz_aware=True,
document_class=bson.son.SON,
)
cls.connection.drop_database(DB)
@@ -57,6 +73,7 @@ class TestMongoModuleStore(object):
@classmethod
def teardownClass(cls):
# cls.patcher.stop()
if cls.connection:
cls.connection.drop_database(DB)
cls.connection.close()
@@ -69,7 +86,10 @@ class TestMongoModuleStore(object):
'db': DB,
'collection': COLLECTION,
}
store = MongoModuleStore(doc_store_config, FS_ROOT, RENDER_TEMPLATE, default_class=DEFAULT_CLASS)
store = MongoModuleStore(
doc_store_config, FS_ROOT, RENDER_TEMPLATE, default_class=DEFAULT_CLASS,
xblock_mixins=(XModuleMixin,)
)
# since MongoModuleStore and MongoContentStore are basically assumed to be together, create this class
# as well
content_store = MongoContentStore(HOST, DB)
@@ -103,25 +123,10 @@ class TestMongoModuleStore(object):
def tearDown(self):
pass
def get_course_by_id(self, name):
"""
Returns the first course with `id` of `name`, or `None` if there are none.
"""
courses = self.store.get_courses()
return next(ifilter(lambda x: x.id == name, courses), None)
def course_with_id_exists(self, name):
"""
Returns true iff there exists some course with `id` of `name`.
"""
return (self.get_course_by_id(name) is not None)
def test_init(self):
'''Make sure the db loads, and print all the locations in the db.
Call this directly from failing tests to see what is loaded'''
'''Make sure the db loads'''
ids = list(self.connection[DB][COLLECTION].find({}, {'_id': True}))
pprint([Location(i['_id']).url() for i in ids])
assert_greater(len(ids), 12)
def test_mongo_modulestore_type(self):
store = MongoModuleStore(
@@ -134,53 +139,64 @@ class TestMongoModuleStore(object):
'''Make sure the course objects loaded properly'''
courses = self.store.get_courses()
assert_equals(len(courses), 5)
assert self.course_with_id_exists('edX/simple/2012_Fall')
assert self.course_with_id_exists('edX/simple_with_draft/2012_Fall')
assert self.course_with_id_exists('edX/test_import_course/2012_Fall')
assert self.course_with_id_exists('edX/test_unicode/2012_Fall')
assert self.course_with_id_exists('edX/toy/2012_Fall')
course_ids = [course.id for course in courses]
for course_key in [
SlashSeparatedCourseKey(*fields)
for fields in [
['edX', 'simple', '2012_Fall'], ['edX', 'simple_with_draft', '2012_Fall'],
['edX', 'test_import_course', '2012_Fall'], ['edX', 'test_unicode', '2012_Fall'],
['edX', 'toy', '2012_Fall']
]
]:
assert_in(course_key, course_ids)
course = self.store.get_course(course_key)
assert_not_none(course)
def test_loads(self):
assert_not_equals(
self.store.get_item("i4x://edX/toy/course/2012_Fall"),
None)
assert_not_none(
self.store.get_item(Location('edX', 'toy', '2012_Fall', 'course', '2012_Fall'))
)
assert_not_equals(
self.store.get_item("i4x://edX/simple/course/2012_Fall"),
None)
assert_not_none(
self.store.get_item(Location('edX', 'simple', '2012_Fall', 'course', '2012_Fall')),
)
assert_not_equals(
self.store.get_item("i4x://edX/toy/video/Welcome"),
None)
assert_not_none(
self.store.get_item(Location('edX', 'toy', '2012_Fall', 'video', 'Welcome')),
)
def test_unicode_loads(self):
assert_not_equals(
self.store.get_item("i4x://edX/test_unicode/course/2012_Fall"),
None)
"""
Test that getting items from the test_unicode course works
"""
assert_not_none(
self.store.get_item(Location('edX', 'test_unicode', '2012_Fall', 'course', '2012_Fall')),
)
# All items with ascii-only filenames should load properly.
assert_not_equals(
self.store.get_item("i4x://edX/test_unicode/video/Welcome"),
None)
assert_not_equals(
self.store.get_item("i4x://edX/test_unicode/video/Welcome"),
None)
assert_not_equals(
self.store.get_item("i4x://edX/test_unicode/chapter/Overview"),
None)
assert_not_none(
self.store.get_item(Location('edX', 'test_unicode', '2012_Fall', 'video', 'Welcome')),
)
assert_not_none(
self.store.get_item(Location('edX', 'test_unicode', '2012_Fall', 'video', 'Welcome')),
)
assert_not_none(
self.store.get_item(Location('edX', 'test_unicode', '2012_Fall', 'chapter', 'Overview')),
)
def test_find_one(self):
assert_not_equals(
self.store._find_one(Location("i4x://edX/toy/course/2012_Fall")),
None)
assert_not_none(
self.store._find_one(Location('edX', 'toy', '2012_Fall', 'course', '2012_Fall')),
)
assert_not_equals(
self.store._find_one(Location("i4x://edX/simple/course/2012_Fall")),
None)
assert_not_none(
self.store._find_one(Location('edX', 'simple', '2012_Fall', 'course', '2012_Fall')),
)
assert_not_equals(
self.store._find_one(Location("i4x://edX/toy/video/Welcome")),
None)
assert_not_none(
self.store._find_one(Location('edX', 'toy', '2012_Fall', 'video', 'Welcome')),
)
def test_path_to_location(self):
'''Make sure that path_to_location works'''
@@ -209,7 +225,7 @@ class TestMongoModuleStore(object):
Assumes the information is desired for courses[4] ('toy' course).
"""
course = self.get_course_by_id('edX/toy/2012_Fall')
course = self.store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
return course.tabs[index]['name']
# There was a bug where model.save was not getting called after the static tab name
@@ -224,29 +240,32 @@ class TestMongoModuleStore(object):
"""
Test getting, setting, and defaulting the locked attr and arbitrary attrs.
"""
location = Location('i4x', 'edX', 'toy', 'course', '2012_Fall')
course_content, __ = TestMongoModuleStore.content_store.get_all_content_for_course(location)
assert len(course_content) > 0
location = Location('edX', 'toy', '2012_Fall', 'course', '2012_Fall')
course_content, __ = TestMongoModuleStore.content_store.get_all_content_for_course(location.course_key)
assert_true(len(course_content) > 0)
# a bit overkill, could just do for content[0]
for content in course_content:
assert not content.get('locked', False)
assert not TestMongoModuleStore.content_store.get_attr(content['_id'], 'locked', False)
attrs = TestMongoModuleStore.content_store.get_attrs(content['_id'])
asset_key = AssetLocation._from_deprecated_son(content['_id'], location.run)
assert not TestMongoModuleStore.content_store.get_attr(asset_key, 'locked', False)
attrs = TestMongoModuleStore.content_store.get_attrs(asset_key)
assert_in('uploadDate', attrs)
assert not attrs.get('locked', False)
TestMongoModuleStore.content_store.set_attr(content['_id'], 'locked', True)
assert TestMongoModuleStore.content_store.get_attr(content['_id'], 'locked', False)
attrs = TestMongoModuleStore.content_store.get_attrs(content['_id'])
TestMongoModuleStore.content_store.set_attr(asset_key, 'locked', True)
assert TestMongoModuleStore.content_store.get_attr(asset_key, 'locked', False)
attrs = TestMongoModuleStore.content_store.get_attrs(asset_key)
assert_in('locked', attrs)
assert attrs['locked'] is True
TestMongoModuleStore.content_store.set_attrs(content['_id'], {'miscel': 99})
assert_equals(TestMongoModuleStore.content_store.get_attr(content['_id'], 'miscel'), 99)
TestMongoModuleStore.content_store.set_attrs(asset_key, {'miscel': 99})
assert_equals(TestMongoModuleStore.content_store.get_attr(asset_key, 'miscel'), 99)
asset_key = AssetLocation._from_deprecated_son(course_content[0]['_id'], location.run)
assert_raises(
AttributeError, TestMongoModuleStore.content_store.set_attr, course_content[0]['_id'],
AttributeError, TestMongoModuleStore.content_store.set_attr, asset_key,
'md5', 'ff1532598830e3feac91c2449eaa60d6'
)
assert_raises(
AttributeError, TestMongoModuleStore.content_store.set_attrs, course_content[0]['_id'],
AttributeError, TestMongoModuleStore.content_store.set_attrs, asset_key,
{'foo': 9, 'md5': 'ff1532598830e3feac91c2449eaa60d6'}
)
assert_raises(
@@ -269,7 +288,7 @@ class TestMongoModuleStore(object):
{'displayname': 'hello'}
)
assert_raises(
InsufficientSpecificationError, TestMongoModuleStore.content_store.set_attrs,
NotFoundError, TestMongoModuleStore.content_store.set_attrs,
Location('bogus', 'bogus', 'bogus', 'asset', None),
{'displayname': 'hello'}
)
@@ -281,13 +300,13 @@ class TestMongoModuleStore(object):
for course_number in self.courses:
course_locations = self.store.get_courses_for_wiki(course_number)
assert_equals(len(course_locations), 1)
assert_equals(Location('i4x', 'edX', course_number, 'course', '2012_Fall'), course_locations[0])
assert_equals(Location('edX', course_number, '2012_Fall', 'course', '2012_Fall'), course_locations[0])
course_locations = self.store.get_courses_for_wiki('no_such_wiki')
assert_equals(len(course_locations), 0)
# set toy course to share the wiki with simple course
toy_course = self.store.get_course('edX/toy/2012_Fall')
toy_course = self.store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
toy_course.wiki_slug = 'simple'
self.store.update_item(toy_course)
@@ -299,17 +318,78 @@ class TestMongoModuleStore(object):
course_locations = self.store.get_courses_for_wiki('simple')
assert_equals(len(course_locations), 2)
for course_number in ['toy', 'simple']:
assert_in(Location('i4x', 'edX', course_number, 'course', '2012_Fall'), course_locations)
assert_in(Location('edX', course_number, '2012_Fall', 'course', '2012_Fall'), course_locations)
# configure simple course to use unique wiki_slug.
simple_course = self.store.get_course('edX/simple/2012_Fall')
simple_course = self.store.get_course(SlashSeparatedCourseKey('edX', 'simple', '2012_Fall'))
simple_course.wiki_slug = 'edX.simple.2012_Fall'
self.store.update_item(simple_course)
# it should be retrievable with its new wiki_slug
course_locations = self.store.get_courses_for_wiki('edX.simple.2012_Fall')
assert_equals(len(course_locations), 1)
assert_in(Location('i4x', 'edX', 'simple', 'course', '2012_Fall'), course_locations)
assert_in(Location('edX', 'simple', '2012_Fall', 'course', '2012_Fall'), course_locations)
@Plugin.register_temp_plugin(ReferenceTestXBlock, 'ref_test')
def test_reference_converters(self):
"""
Test that references types get deserialized correctly
"""
course_key = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
def setup_test():
course = self.store.get_course(course_key)
# can't use item factory as it depends on django settings
p1ele = self.store.create_and_save_xmodule(course.id.make_usage_key('problem', 'p1'))
p2ele = self.store.create_and_save_xmodule(course.id.make_usage_key('problem', 'p2'))
self.refloc = course.id.make_usage_key('ref_test', 'ref_test')
self.store.create_and_save_xmodule(
self.refloc, fields={
'reference_link': p1ele.location,
'reference_list': [p1ele.location, p2ele.location],
'reference_dict': {'p1': p1ele.location, 'p2': p2ele.location},
'children': [p1ele.location, p2ele.location],
}
)
def check_xblock_fields():
def check_children(xblock):
for child in xblock.children:
assert_is_instance(child, Location)
course = self.store.get_course(course_key)
check_children(course)
refele = self.store.get_item(self.refloc)
check_children(refele)
assert_is_instance(refele.reference_link, Location)
assert_greater(len(refele.reference_list), 0)
for ref in refele.reference_list:
assert_is_instance(ref, Location)
assert_greater(len(refele.reference_dict), 0)
for ref in refele.reference_dict.itervalues():
assert_is_instance(ref, Location)
def check_mongo_fields():
def get_item(location):
return self.store._find_one(location)
def check_children(payload):
for child in payload['definition']['children']:
assert_is_instance(child, basestring)
refele = get_item(self.refloc)
check_children(refele)
assert_is_instance(refele['definition']['data']['reference_link'], basestring)
assert_greater(len(refele['definition']['data']['reference_list']), 0)
for ref in refele['definition']['data']['reference_list']:
assert_is_instance(ref, basestring)
assert_greater(len(refele['metadata']['reference_dict']), 0)
for ref in refele['metadata']['reference_dict'].itervalues():
assert_is_instance(ref, basestring)
setup_test()
check_xblock_fields()
check_mongo_fields()
class TestMongoKeyValueStore(object):
"""
@@ -318,8 +398,8 @@ class TestMongoKeyValueStore(object):
def setUp(self):
self.data = {'foo': 'foo_value'}
self.location = Location('i4x://org/course/category/name@version')
self.children = ['i4x://org/course/child/a', 'i4x://org/course/child/b']
self.course_id = SlashSeparatedCourseKey('org', 'course', 'run')
self.children = [self.course_id.make_usage_key('child', 'a'), self.course_id.make_usage_key('child', 'b')]
self.metadata = {'meta': 'meta_val'}
self.kvs = MongoKeyValueStore(self.data, self.children, self.metadata)

156
common/lib/xmodule/xmodule/modulestore/tests/test_orphan.py
View File

@@ -1,158 +1,52 @@
import uuid
import mock
import unittest
import random
import datetime
from xmodule.modulestore.inheritance import InheritanceMixin
from xmodule.modulestore.mongo import MongoModuleStore
from xmodule.modulestore.split_mongo import SplitMongoModuleStore
from xmodule.modulestore import Location
from xmodule.fields import Date
from xmodule.modulestore.locator import BlockUsageLocator, CourseLocator
from xmodule.modulestore.tests.test_split_w_old_mongo import SplitWMongoCourseBoostrapper
class TestOrphan(unittest.TestCase):
class TestOrphan(SplitWMongoCourseBoostrapper):
"""
Test the orphan finding code
"""
# Snippet of what would be in the django settings envs file
db_config = {
'host': 'localhost',
'db': 'test_xmodule',
}
modulestore_options = {
'default_class': 'xmodule.raw_module.RawDescriptor',
'fs_root': '',
'render_template': mock.Mock(return_value=""),
'xblock_mixins': (InheritanceMixin,)
}
split_package_id = 'test_org.test_course.runid'
def setUp(self):
self.db_config['collection'] = 'modulestore{0}'.format(uuid.uuid4().hex[:5])
self.userid = random.getrandbits(32)
super(TestOrphan, self).setUp()
self.split_mongo = SplitMongoModuleStore(
self.db_config,
**self.modulestore_options
)
self.addCleanup(self.tear_down_split)
self.old_mongo = MongoModuleStore(self.db_config, **self.modulestore_options)
self.addCleanup(self.tear_down_mongo)
self.course_location = None
self._create_course()
def tear_down_split(self):
"""
Remove the test collections, close the db connection
"""
split_db = self.split_mongo.db
split_db.drop_collection(split_db.course_index)
split_db.drop_collection(split_db.structures)
split_db.drop_collection(split_db.definitions)
split_db.connection.close()
def tear_down_mongo(self):
"""
Remove the test collections, close the db connection
"""
split_db = self.split_mongo.db
# old_mongo doesn't give a db attr, but all of the dbs are the same
split_db.drop_collection(self.old_mongo.collection)
def _create_item(self, category, name, data, metadata, parent_category, parent_name, runtime):
"""
Create the item of the given category and block id in split and old mongo, add it to the optional
parent. The parent category is only needed because old mongo requires it for the id.
"""
location = Location('i4x', 'test_org', 'test_course', category, name)
self.old_mongo.create_and_save_xmodule(location, data, metadata, runtime)
if isinstance(data, basestring):
fields = {'data': data}
else:
fields = data.copy()
fields.update(metadata)
if parent_name:
# add child to parent in mongo
parent_location = Location('i4x', 'test_org', 'test_course', parent_category, parent_name)
parent = self.old_mongo.get_item(parent_location)
parent.children.append(location.url())
self.old_mongo.update_item(parent, self.userid)
# create pointer for split
course_or_parent_locator = BlockUsageLocator(
package_id=self.split_package_id,
branch='draft',
block_id=parent_name
)
else:
course_or_parent_locator = CourseLocator(
package_id='test_org.test_course.runid',
branch='draft',
)
self.split_mongo.create_item(course_or_parent_locator, category, self.userid, block_id=name, fields=fields)
def _create_course(self):
"""
* some detached items
* some attached children
* some orphans
"""
date_proxy = Date()
metadata = {
'start': date_proxy.to_json(datetime.datetime(2000, 3, 13, 4)),
'display_name': 'Migration test course',
}
data = {
'wiki_slug': 'test_course_slug'
}
fields = metadata.copy()
fields.update(data)
# split requires the course to be created separately from creating items
self.split_mongo.create_course(
self.split_package_id, 'test_org', self.userid, fields=fields, root_block_id='runid'
)
self.course_location = Location('i4x', 'test_org', 'test_course', 'course', 'runid')
self.old_mongo.create_and_save_xmodule(self.course_location, data, metadata)
runtime = self.old_mongo.get_item(self.course_location).runtime
super(TestOrphan, self)._create_course()
self._create_item('chapter', 'Chapter1', {}, {'display_name': 'Chapter 1'}, 'course', 'runid', runtime)
self._create_item('chapter', 'Chapter2', {}, {'display_name': 'Chapter 2'}, 'course', 'runid', runtime)
self._create_item('chapter', 'OrphanChapter', {}, {'display_name': 'Orphan Chapter'}, None, None, runtime)
self._create_item('vertical', 'Vert1', {}, {'display_name': 'Vertical 1'}, 'chapter', 'Chapter1', runtime)
self._create_item('vertical', 'OrphanVert', {}, {'display_name': 'Orphan Vertical'}, None, None, runtime)
self._create_item('html', 'Html1', "<p>Goodbye</p>", {'display_name': 'Parented Html'}, 'vertical', 'Vert1', runtime)
self._create_item('html', 'OrphanHtml', "<p>Hello</p>", {'display_name': 'Orphan html'}, None, None, runtime)
self._create_item('static_tab', 'staticuno', "<p>tab</p>", {'display_name': 'Tab uno'}, None, None, runtime)
self._create_item('about', 'overview', "<p>overview</p>", {}, None, None, runtime)
self._create_item('course_info', 'updates', "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>", {}, None, None, runtime)
self._create_item('chapter', 'Chapter1', {}, {'display_name': 'Chapter 1'}, 'course', 'runid')
self._create_item('chapter', 'Chapter2', {}, {'display_name': 'Chapter 2'}, 'course', 'runid')
self._create_item('chapter', 'OrphanChapter', {}, {'display_name': 'Orphan Chapter'}, None, None)
self._create_item('vertical', 'Vert1', {}, {'display_name': 'Vertical 1'}, 'chapter', 'Chapter1')
self._create_item('vertical', 'OrphanVert', {}, {'display_name': 'Orphan Vertical'}, None, None)
self._create_item('html', 'Html1', "<p>Goodbye</p>", {'display_name': 'Parented Html'}, 'vertical', 'Vert1')
self._create_item('html', 'OrphanHtml', "<p>Hello</p>", {'display_name': 'Orphan html'}, None, None)
self._create_item('static_tab', 'staticuno', "<p>tab</p>", {'display_name': 'Tab uno'}, None, None)
self._create_item('about', 'overview', "<p>overview</p>", {}, None, None)
self._create_item('course_info', 'updates', "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>", {}, None, None)
def test_mongo_orphan(self):
"""
Test that old mongo finds the orphans
"""
orphans = self.old_mongo.get_orphans(self.course_location, None)
orphans = self.old_mongo.get_orphans(self.old_course_key)
self.assertEqual(len(orphans), 3, "Wrong # {}".format(orphans))
location = self.course_location.replace(category='chapter', name='OrphanChapter')
self.assertIn(location.url(), orphans)
location = self.course_location.replace(category='vertical', name='OrphanVert')
self.assertIn(location.url(), orphans)
location = self.course_location.replace(category='html', name='OrphanHtml')
self.assertIn(location.url(), orphans)
location = self.old_course_key.make_usage_key('chapter', name='OrphanChapter')
self.assertIn(location.to_deprecated_string(), orphans)
location = self.old_course_key.make_usage_key('vertical', name='OrphanVert')
self.assertIn(location.to_deprecated_string(), orphans)
location = self.old_course_key.make_usage_key('html', 'OrphanHtml')
self.assertIn(location.to_deprecated_string(), orphans)
def test_split_orphan(self):
"""
Test that old mongo finds the orphans
Test that split mongo finds the orphans
"""
orphans = self.split_mongo.get_orphans(self.split_package_id, 'draft')
orphans = self.split_mongo.get_orphans(self.split_course_key)
self.assertEqual(len(orphans), 3, "Wrong # {}".format(orphans))
location = BlockUsageLocator(package_id=self.split_package_id, branch='draft', block_id='OrphanChapter')
location = self.split_course_key.make_usage_key('chapter', 'OrphanChapter')
self.assertIn(location, orphans)
location = BlockUsageLocator(package_id=self.split_package_id, branch='draft', block_id='OrphanVert')
location = self.split_course_key.make_usage_key('vertical', 'OrphanVert')
self.assertIn(location, orphans)
location = BlockUsageLocator(package_id=self.split_package_id, branch='draft', block_id='OrphanHtml')
location = self.split_course_key.make_usage_key('html', 'OrphanHtml')
self.assertIn(location, orphans)

141
common/lib/xmodule/xmodule/modulestore/tests/test_publish.py
View File

@@ -1,103 +1,26 @@
"""
Test the publish code (primary causing orphans)
Test the publish code (mostly testing that publishing doesn't result in orphans)
"""
import uuid
import mock
import unittest
import datetime
import random
from xmodule.modulestore.inheritance import InheritanceMixin
from xmodule.modulestore.mongo import MongoModuleStore, DraftMongoModuleStore
from xmodule.modulestore import Location
from xmodule.fields import Date
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.mongo.draft import DIRECT_ONLY_CATEGORIES
from xmodule.modulestore.tests.test_split_w_old_mongo import SplitWMongoCourseBoostrapper
class TestPublish(unittest.TestCase):
class TestPublish(SplitWMongoCourseBoostrapper):
"""
Test the publish code (primary causing orphans)
"""
# Snippet of what would be in the django settings envs file
db_config = {
'host': 'localhost',
'db': 'test_xmodule',
}
modulestore_options = {
'default_class': 'xmodule.raw_module.RawDescriptor',
'fs_root': '',
'render_template': mock.Mock(return_value=""),
'xblock_mixins': (InheritanceMixin,)
}
def setUp(self):
self.db_config['collection'] = 'modulestore{0}'.format(uuid.uuid4().hex[:5])
self.old_mongo = MongoModuleStore(self.db_config, **self.modulestore_options)
self.draft_mongo = DraftMongoModuleStore(self.db_config, **self.modulestore_options)
self.addCleanup(self.tear_down_mongo)
self.course_location = None
def tear_down_mongo(self):
# old_mongo doesn't give a db attr, but all of the dbs are the same and draft and pub use same collection
dbref = self.old_mongo.collection.database
dbref.drop_collection(self.old_mongo.collection)
dbref.connection.close()
def _create_item(self, category, name, data, metadata, parent_category, parent_name, runtime):
"""
Create the item in either draft or direct based on category and attach to its parent.
"""
location = self.course_location.replace(category=category, name=name)
if category in DIRECT_ONLY_CATEGORIES:
mongo = self.old_mongo
else:
mongo = self.draft_mongo
mongo.create_and_save_xmodule(location, data, metadata, runtime)
if isinstance(data, basestring):
fields = {'data': data}
else:
fields = data.copy()
fields.update(metadata)
if parent_name:
# add child to parent in mongo
parent_location = self.course_location.replace(category=parent_category, name=parent_name)
parent = self.draft_mongo.get_item(parent_location)
parent.children.append(location.url())
if parent_category in DIRECT_ONLY_CATEGORIES:
mongo = self.old_mongo
else:
mongo = self.draft_mongo
mongo.update_item(parent, '**replace_user**')
def _create_course(self):
"""
Create the course, publish all verticals
* some detached items
"""
date_proxy = Date()
metadata = {
'start': date_proxy.to_json(datetime.datetime(2000, 3, 13, 4)),
'display_name': 'Migration test course',
}
data = {
'wiki_slug': 'test_course_slug'
}
fields = metadata.copy()
fields.update(data)
super(TestPublish, self)._create_course(split=False)
self.course_location = Location('i4x', 'test_org', 'test_course', 'course', 'runid')
self.old_mongo.create_and_save_xmodule(self.course_location, data, metadata)
runtime = self.draft_mongo.get_item(self.course_location).runtime
self._create_item('chapter', 'Chapter1', {}, {'display_name': 'Chapter 1'}, 'course', 'runid', runtime)
self._create_item('chapter', 'Chapter2', {}, {'display_name': 'Chapter 2'}, 'course', 'runid', runtime)
self._create_item('vertical', 'Vert1', {}, {'display_name': 'Vertical 1'}, 'chapter', 'Chapter1', runtime)
self._create_item('vertical', 'Vert2', {}, {'display_name': 'Vertical 2'}, 'chapter', 'Chapter1', runtime)
self._create_item('html', 'Html1', "<p>Goodbye</p>", {'display_name': 'Parented Html'}, 'vertical', 'Vert1', runtime)
self._create_item('chapter', 'Chapter1', {}, {'display_name': 'Chapter 1'}, 'course', 'runid', split=False)
self._create_item('chapter', 'Chapter2', {}, {'display_name': 'Chapter 2'}, 'course', 'runid', split=False)
self._create_item('vertical', 'Vert1', {}, {'display_name': 'Vertical 1'}, 'chapter', 'Chapter1', split=False)
self._create_item('vertical', 'Vert2', {}, {'display_name': 'Vertical 2'}, 'chapter', 'Chapter1', split=False)
self._create_item('html', 'Html1', "<p>Goodbye</p>", {'display_name': 'Parented Html'}, 'vertical', 'Vert1', split=False)
self._create_item(
'discussion', 'Discussion1',
"discussion discussion_category=\"Lecture 1\" discussion_id=\"a08bfd89b2aa40fa81f2c650a9332846\" discussion_target=\"Lecture 1\"/>\n",
@@ -107,9 +30,10 @@ class TestPublish(unittest.TestCase):
"display_name": "Lecture 1 Discussion",
"discussion_id": "a08bfd89b2aa40fa81f2c650a9332846"
},
'vertical', 'Vert1', runtime
'vertical', 'Vert1',
split=False
)
self._create_item('html', 'Html2', "<p>Hellow</p>", {'display_name': 'Hollow Html'}, 'vertical', 'Vert1', runtime)
self._create_item('html', 'Html2', "<p>Hellow</p>", {'display_name': 'Hollow Html'}, 'vertical', 'Vert1', split=False)
self._create_item(
'discussion', 'Discussion2',
"discussion discussion_category=\"Lecture 2\" discussion_id=\"b08bfd89b2aa40fa81f2c650a9332846\" discussion_target=\"Lecture 2\"/>\n",
@@ -119,11 +43,12 @@ class TestPublish(unittest.TestCase):
"display_name": "Lecture 2 Discussion",
"discussion_id": "b08bfd89b2aa40fa81f2c650a9332846"
},
'vertical', 'Vert2', runtime
'vertical', 'Vert2',
split=False
)
self._create_item('static_tab', 'staticuno', "<p>tab</p>", {'display_name': 'Tab uno'}, None, None, runtime)
self._create_item('about', 'overview', "<p>overview</p>", {}, None, None, runtime)
self._create_item('course_info', 'updates', "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>", {}, None, None, runtime)
self._create_item('static_tab', 'staticuno', "<p>tab</p>", {'display_name': 'Tab uno'}, None, None, split=False)
self._create_item('about', 'overview', "<p>overview</p>", {}, None, None, split=False)
self._create_item('course_info', 'updates', "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>", {}, None, None, split=False)
def _xmodule_recurse(self, item, action):
"""
@@ -142,13 +67,11 @@ class TestPublish(unittest.TestCase):
To reproduce a bug (STUD-811) publish a vertical, convert to draft, delete a child, move a child, publish.
See if deleted and moved children still is connected or exists in db (bug was disconnected but existed)
"""
self._create_course()
userid = random.getrandbits(32)
location = self.course_location.replace(category='vertical', name='Vert1')
location = self.old_course_key.make_usage_key('vertical', name='Vert1')
item = self.draft_mongo.get_item(location, 2)
self._xmodule_recurse(
item,
lambda i: self.draft_mongo.publish(i.location, userid)
lambda i: self.draft_mongo.publish(i.location, self.userid)
)
# verify status
item = self.draft_mongo.get_item(location, 0)
@@ -164,26 +87,26 @@ class TestPublish(unittest.TestCase):
self.assertFalse(getattr(item, 'is_draft', False), "Published item doesn't say so")
# delete the discussion (which oddly is not in draft mode)
location = self.course_location.replace(category='discussion', name='Discussion1')
location = self.old_course_key.make_usage_key('discussion', name='Discussion1')
self.draft_mongo.delete_item(location)
# remove pointer from draft vertical (verify presence first to ensure process is valid)
self.assertIn(location.url(), draft_vert.children)
draft_vert.children.remove(location.url())
self.assertIn(location, draft_vert.children)
draft_vert.children.remove(location)
# move the other child
other_child_loc = self.course_location.replace(category='html', name='Html2')
draft_vert.children.remove(other_child_loc.url())
other_vert = self.draft_mongo.get_item(self.course_location.replace(category='vertical', name='Vert2'), 0)
other_vert.children.append(other_child_loc.url())
self.draft_mongo.update_item(draft_vert, '**replace_user**')
self.draft_mongo.update_item(other_vert, '**replace_user**')
other_child_loc = self.old_course_key.make_usage_key('html', name='Html2')
draft_vert.children.remove(other_child_loc)
other_vert = self.draft_mongo.get_item(self.old_course_key.make_usage_key('vertical', name='Vert2'), 0)
other_vert.children.append(other_child_loc)
self.draft_mongo.update_item(draft_vert, self.userid)
self.draft_mongo.update_item(other_vert, self.userid)
# publish
self._xmodule_recurse(
draft_vert,
lambda i: self.draft_mongo.publish(i.location, userid)
lambda i: self.draft_mongo.publish(i.location, self.userid)
)
item = self.old_mongo.get_item(draft_vert.location, 0)
self.assertNotIn(location.url(), item.children)
self.assertNotIn(location, item.children)
with self.assertRaises(ItemNotFoundError):
self.draft_mongo.get_item(location)
self.assertNotIn(other_child_loc.url(), item.children)
self.assertTrue(self.draft_mongo.has_item(None, other_child_loc), "Oops, lost moved item")
self.assertNotIn(other_child_loc, item.children)
self.assertTrue(self.draft_mongo.has_item(other_child_loc), "Oops, lost moved item")

271
common/lib/xmodule/xmodule/modulestore/tests/test_split_migrator.py
View File

@@ -1,81 +1,37 @@
"""
Created on Sep 10, 2013
@author: dmitchell
Tests for split_migrator
"""
import unittest
import uuid
import random
import mock
import datetime
from xmodule.fields import Date
from xmodule.modulestore import Location
from xmodule.modulestore.inheritance import InheritanceMixin
from xmodule.modulestore.loc_mapper_store import LocMapperStore
from xmodule.modulestore.mongo.draft import DraftModuleStore
from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore
from xmodule.modulestore.mongo.base import MongoModuleStore
from xmodule.modulestore.split_migrator import SplitMigrator
from xmodule.modulestore.mongo import draft
from xmodule.modulestore.tests import test_location_mapper
from xmodule.modulestore.tests.test_split_w_old_mongo import SplitWMongoCourseBoostrapper
from nose.tools import nottest
class TestMigration(unittest.TestCase):
@nottest
class TestMigration(SplitWMongoCourseBoostrapper):
"""
Test the split migrator
"""
# Snippet of what would be in the django settings envs file
db_config = {
'host': 'localhost',
'db': 'test_xmodule',
'collection': 'modulestore{0}'.format(uuid.uuid4().hex[:5]),
}
modulestore_options = {
'default_class': 'xmodule.raw_module.RawDescriptor',
'fs_root': '',
'render_template': mock.Mock(return_value=""),
'xblock_mixins': (InheritanceMixin,)
}
def setUp(self):
super(TestMigration, self).setUp()
# pylint: disable=W0142
self.loc_mapper = LocMapperStore(test_location_mapper.TrivialCache(), **self.db_config)
self.old_mongo = MongoModuleStore(self.db_config, **self.modulestore_options)
self.draft_mongo = DraftModuleStore(self.db_config, **self.modulestore_options)
self.split_mongo = SplitMongoModuleStore(
doc_store_config=self.db_config,
loc_mapper=self.loc_mapper,
**self.modulestore_options
)
self.split_mongo.loc_mapper = self.loc_mapper
self.migrator = SplitMigrator(self.split_mongo, self.old_mongo, self.draft_mongo, self.loc_mapper)
self.course_location = None
self.create_source_course()
def tearDown(self):
dbref = self.loc_mapper.db
dbref.drop_collection(self.loc_mapper.location_map)
split_db = self.split_mongo.db
split_db.drop_collection(self.split_mongo.db_connection.course_index)
split_db.drop_collection(self.split_mongo.db_connection.structures)
split_db.drop_collection(self.split_mongo.db_connection.definitions)
# old_mongo doesn't give a db attr, but all of the dbs are the same
dbref.drop_collection(self.old_mongo.collection)
dbref.connection.close()
super(TestMigration, self).tearDown()
def _create_and_get_item(self, store, location, data, metadata, runtime=None):
store.create_and_save_xmodule(location, data, metadata, runtime)
return store.get_item(location)
def create_source_course(self):
def _create_course(self):
"""
A course testing all of the conversion mechanisms:
* some inheritable settings
@@ -83,150 +39,138 @@ class TestMigration(unittest.TestCase):
only the live ones get to published. Some are only draft, some are both, some are only live.
* about, static_tab, and conditional documents
"""
location = Location('i4x', 'test_org', 'test_course', 'course', 'runid')
self.course_location = location
date_proxy = Date()
metadata = {
'start': date_proxy.to_json(datetime.datetime(2000, 3, 13, 4)),
'display_name': 'Migration test course',
}
data = {
'wiki_slug': 'test_course_slug'
}
course_root = self._create_and_get_item(self.old_mongo, location, data, metadata)
runtime = course_root.runtime
super(TestMigration, self)._create_course(split=False)
# chapters
location = location.replace(category='chapter', name=uuid.uuid4().hex)
chapter1 = self._create_and_get_item(self.old_mongo, location, {}, {'display_name': 'Chapter 1'}, runtime)
course_root.children.append(chapter1.location.url())
location = location.replace(category='chapter', name=uuid.uuid4().hex)
chapter2 = self._create_and_get_item(self.old_mongo, location, {}, {'display_name': 'Chapter 2'}, runtime)
course_root.children.append(chapter2.location.url())
self.old_mongo.update_item(course_root, '**replace_user**')
chapter1_name = uuid.uuid4().hex
self._create_item('chapter', chapter1_name, {}, {'display_name': 'Chapter 1'}, 'course', 'runid', split=False)
chap2_loc = self.old_course_key.make_usage_key('chapter', uuid.uuid4().hex)
self._create_item(
chap2_loc.category, chap2_loc.name, {}, {'display_name': 'Chapter 2'}, 'course', 'runid', split=False
)
# vertical in live only
location = location.replace(category='vertical', name=uuid.uuid4().hex)
live_vert = self._create_and_get_item(self.old_mongo, location, {}, {'display_name': 'Live vertical'}, runtime)
chapter1.children.append(live_vert.location.url())
self.create_random_units(self.old_mongo, live_vert)
live_vert_name = uuid.uuid4().hex
self._create_item(
'vertical', live_vert_name, {}, {'display_name': 'Live vertical'}, 'chapter', chapter1_name,
draft=False, split=False
)
self.create_random_units(False, self.old_course_key.make_usage_key('vertical', live_vert_name))
# vertical in both live and draft
location = location.replace(category='vertical', name=uuid.uuid4().hex)
both_vert = self._create_and_get_item(
self.old_mongo, location, {}, {'display_name': 'Both vertical'}, runtime
both_vert_loc = self.old_course_key.make_usage_key('vertical', uuid.uuid4().hex)
self._create_item(
both_vert_loc.category, both_vert_loc.name, {}, {'display_name': 'Both vertical'}, 'chapter', chapter1_name,
draft=False, split=False
)
draft_both = self._create_and_get_item(
self.draft_mongo, location, {}, {'display_name': 'Both vertical renamed'}, runtime
)
chapter1.children.append(both_vert.location.url())
self.create_random_units(self.old_mongo, both_vert, self.draft_mongo, draft_both)
self.create_random_units(False, both_vert_loc)
draft_both = self.draft_mongo.get_item(both_vert_loc)
draft_both.display_name = 'Both vertical renamed'
self.draft_mongo.update_item(draft_both)
self.create_random_units(True, both_vert_loc)
# vertical in draft only (x2)
location = location.replace(category='vertical', name=uuid.uuid4().hex)
draft_vert = self._create_and_get_item(
self.draft_mongo,
location, {}, {'display_name': 'Draft vertical'}, runtime)
chapter1.children.append(draft_vert.location.url())
self.create_random_units(self.draft_mongo, draft_vert)
location = location.replace(category='vertical', name=uuid.uuid4().hex)
draft_vert = self._create_and_get_item(
self.draft_mongo,
location, {}, {'display_name': 'Draft vertical2'}, runtime)
chapter1.children.append(draft_vert.location.url())
self.create_random_units(self.draft_mongo, draft_vert)
# and finally one in live only (so published has to skip 2)
location = location.replace(category='vertical', name=uuid.uuid4().hex)
live_vert = self._create_and_get_item(
self.old_mongo,
location, {}, {'display_name': 'Live vertical end'}, runtime)
chapter1.children.append(live_vert.location.url())
self.create_random_units(self.old_mongo, live_vert)
# update the chapter
self.old_mongo.update_item(chapter1, '**replace_user**')
# now the other one w/ the conditional
# first create some show children
indirect1 = self._create_and_get_item(
self.old_mongo,
location.replace(category='discussion', name=uuid.uuid4().hex),
"", {'display_name': 'conditional show 1'}, runtime
draft_vert_loc = self.old_course_key.make_usage_key('vertical', uuid.uuid4().hex)
self._create_item(
draft_vert_loc.category, draft_vert_loc.name, {}, {'display_name': 'Draft vertical'}, 'chapter', chapter1_name,
draft=True, split=False
)
indirect2 = self._create_and_get_item(
self.old_mongo,
location.replace(category='html', name=uuid.uuid4().hex),
"", {'display_name': 'conditional show 2'}, runtime
self.create_random_units(True, draft_vert_loc)
draft_vert_loc = self.old_course_key.make_usage_key('vertical', uuid.uuid4().hex)
self._create_item(
draft_vert_loc.category, draft_vert_loc.name, {}, {'display_name': 'Draft vertical2'}, 'chapter', chapter1_name,
draft=True, split=False
)
location = location.replace(category='conditional', name=uuid.uuid4().hex)
metadata = {
'xml_attributes': {
'sources': [live_vert.location.url(), ],
'completed': True,
self.create_random_units(True, draft_vert_loc)
# and finally one in live only (so published has to skip 2 preceding sibs)
live_vert_loc = self.old_course_key.make_usage_key('vertical', uuid.uuid4().hex)
self._create_item(
live_vert_loc.category, live_vert_loc.name, {}, {'display_name': 'Live vertical end'}, 'chapter', chapter1_name,
draft=False, split=False
)
self.create_random_units(True, draft_vert_loc)
# now the other chapter w/ the conditional
# create pointers to children (before the children exist)
indirect1_loc = self.old_course_key.make_usage_key('discussion', uuid.uuid4().hex)
indirect2_loc = self.old_course_key.make_usage_key('html', uuid.uuid4().hex)
conditional_loc = self.old_course_key.make_usage_key('conditional', uuid.uuid4().hex)
self._create_item(
conditional_loc.category, conditional_loc.name,
{
'show_tag_list': [indirect1_loc, indirect2_loc],
'sources_list': [live_vert_loc, ],
},
}
data = {
'show_tag_list': [indirect1.location.url(), indirect2.location.url()]
}
conditional = self._create_and_get_item(self.old_mongo, location, data, metadata, runtime)
conditional.children = [indirect1.location.url(), indirect2.location.url()]
{
'xml_attributes': {
'completed': True,
},
},
chap2_loc.category, chap2_loc.name,
draft=False, split=False
)
# create the children
self._create_item(
indirect1_loc.category, indirect1_loc.name, {'data': ""}, {'display_name': 'conditional show 1'},
conditional_loc.category, conditional_loc.name,
draft=False, split=False
)
self._create_item(
indirect2_loc.category, indirect2_loc.name, {'data': ""}, {'display_name': 'conditional show 2'},
conditional_loc.category, conditional_loc.name,
draft=False, split=False
)
# add direct children
self.create_random_units(self.old_mongo, conditional)
chapter2.children.append(conditional.location.url())
self.old_mongo.update_item(chapter2, '**replace_user**')
self.create_random_units(False, conditional_loc)
# and the ancillary docs (not children)
location = location.replace(category='static_tab', name=uuid.uuid4().hex)
# the below automatically adds the tab to the course
_tab = self._create_and_get_item(self.old_mongo, location, "", {'display_name': 'Tab uno'}, runtime)
location = location.replace(category='about', name='overview')
_overview = self._create_and_get_item(self.old_mongo, location, "<p>test</p>", {}, runtime)
location = location.replace(category='course_info', name='updates')
_overview = self._create_and_get_item(
self.old_mongo,
location, "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>", {}, runtime
self._create_item(
'static_tab', uuid.uuid4().hex, {'data': ""}, {'display_name': 'Tab uno'},
None, None, draft=False, split=False
)
self._create_item(
'about', 'overview', {'data': "<p>test</p>"}, {},
None, None, draft=False, split=False
)
self._create_item(
'course_info', 'updates', {'data': "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>"}, {},
None, None, draft=False, split=False
)
def create_random_units(self, store, parent, cc_store=None, cc_parent=None):
def create_random_units(self, draft, parent_loc):
"""
Create a random selection of units under the given parent w/ random names & attrs
:param store: which store (e.g., direct/draft) to create them in
:param parent: the parent to have point to them
:param cc_store: (optional) if given, make a small change and save also to this store but w/ same location
(only makes sense if store is 'direct' and this is 'draft' or vice versa)
"""
for _ in range(random.randrange(6)):
location = parent.location.replace(
location = parent_loc.replace(
category=random.choice(['html', 'video', 'problem', 'discussion']),
name=uuid.uuid4().hex
)
metadata = {'display_name': str(uuid.uuid4()), 'graded': True}
data = {}
element = self._create_and_get_item(store, location, data, metadata, parent.runtime)
parent.children.append(element.location.url())
if cc_store is not None:
# change display_name and remove graded to test the delta
element = self._create_and_get_item(
cc_store, location, data, {'display_name': str(uuid.uuid4())}, parent.runtime
)
cc_parent.children.append(element.location.url())
store.update_item(parent, '**replace_user**')
if cc_store is not None:
cc_store.update_item(cc_parent, '**replace_user**')
self._create_item(
location.category, location.name, data, metadata, parent_loc.category, parent_loc.name,
draft=draft, split=False
)
def compare_courses(self, presplit, published):
# descend via children to do comparison
old_root = presplit.get_item(self.course_location, depth=None)
new_root_locator = self.loc_mapper.translate_location(
self.course_location.course_id, self.course_location, published, add_entry_if_missing=False
old_root = presplit.get_course(self.old_course_key)
new_root_locator = self.loc_mapper.translate_location_to_course_locator(
old_root.id, published
)
new_root = self.split_mongo.get_course(new_root_locator)
self.compare_dags(presplit, old_root, new_root, published)
# grab the detached items to compare they should be in both published and draft
for category in ['conditional', 'about', 'course_info', 'static_tab']:
location = self.course_location.replace(name=None, category=category)
for conditional in presplit.get_items(location):
for conditional in presplit.get_items(self.old_course_key, category=category):
locator = self.loc_mapper.translate_location(
self.course_location.course_id,
conditional.location, published, add_entry_if_missing=False
conditional.location,
published,
add_entry_if_missing=False
)
self.compare_dags(presplit, conditional, self.split_mongo.get_item(locator), published)
@@ -262,9 +206,10 @@ class TestMigration(unittest.TestCase):
# compare children
if presplit_dag_root.has_children:
self.assertEqual(
len(presplit_dag_root.get_children()), len(split_dag_root.get_children()),
"{0.category} '{0.display_name}': children count {1} != {2}".format(
presplit_dag_root, len(presplit_dag_root.get_children()), split_dag_root.children
# need get_children to filter out drafts
len(presplit_dag_root.get_children()), len(split_dag_root.children),
"{0.category} '{0.display_name}': children {1} != {2}".format(
presplit_dag_root, presplit_dag_root.children, split_dag_root.children
)
)
for pre_child, split_child in zip(presplit_dag_root.get_children(), split_dag_root.get_children()):
@@ -272,7 +217,7 @@ class TestMigration(unittest.TestCase):
def test_migrator(self):
user = mock.Mock(id=1)
self.migrator.migrate_mongo_course(self.course_location, user)
self.migrator.migrate_mongo_course(self.old_course_key, user)
# now compare the migrated to the original course
self.compare_courses(self.old_mongo, True)
self.compare_courses(self.draft_mongo, False)

445
common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py
View File

@@ -17,8 +17,8 @@ from xmodule.modulestore.locator import CourseLocator, BlockUsageLocator, Versio
from xmodule.modulestore.inheritance import InheritanceMixin
from xmodule.x_module import XModuleMixin
from xmodule.fields import Date, Timedelta
from bson.objectid import ObjectId
from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore
from xmodule.modulestore.tests.test_modulestore import check_has_course_method
class SplitModuleTest(unittest.TestCase):
@@ -56,6 +56,7 @@ class SplitModuleTest(unittest.TestCase):
COURSE_CONTENT = {
"testx.GreekHero": {
"org": "testx",
"offering": "GreekHero",
"root_block_id": "head12345",
"user_id": "test@edx.org",
"fields": {
@@ -185,7 +186,7 @@ class SplitModuleTest(unittest.TestCase):
}}
},
{"user_id": "testassist@edx.org",
"update":
"update":
{"head12345": {
"end": _date_field.from_json("2013-06-13T04:30"),
"grading_policy": {
@@ -272,9 +273,10 @@ class SplitModuleTest(unittest.TestCase):
]
},
]
},
},
"testx.wonderful": {
"org": "testx",
"offering": "wonderful",
"root_block_id": "head23456",
"user_id": "test@edx.org",
"fields": {
@@ -377,9 +379,10 @@ class SplitModuleTest(unittest.TestCase):
}
}
]
},
},
"guestx.contender": {
"org": "guestx",
"offering": "contender",
"root_block_id": "head345679",
"user_id": "test@guestx.edu",
"fields": {
@@ -441,7 +444,7 @@ class SplitModuleTest(unittest.TestCase):
split_store = modulestore()
for course_id, course_spec in SplitModuleTest.COURSE_CONTENT.iteritems():
course = split_store.create_course(
course_id, course_spec['org'], course_spec['user_id'],
course_spec['org'], course_spec['offering'], course_spec['user_id'],
fields=course_spec['fields'],
root_block_id=course_spec['root_block_id']
)
@@ -452,7 +455,7 @@ class SplitModuleTest(unittest.TestCase):
block = course
else:
block_usage = BlockUsageLocator.make_relative(course.location, block_id)
block = split_store.get_instance(course.location.package_id, block_usage)
block = split_store.get_item(block_usage)
for key, value in fields.iteritems():
setattr(block, key, value)
# create new blocks into dag: parent must already exist; thus, order is important
@@ -464,7 +467,7 @@ class SplitModuleTest(unittest.TestCase):
parent = course
else:
block_usage = BlockUsageLocator.make_relative(course.location, spec['parent'])
parent = split_store.get_instance(course.location.package_id, block_usage)
parent = split_store.get_item(block_usage)
block_id = LocalId(spec['id'])
child = split_store.create_xblock(
course.runtime, spec['category'], spec['fields'], block_id, parent_xblock=parent
@@ -472,8 +475,11 @@ class SplitModuleTest(unittest.TestCase):
new_ele_dict[spec['id']] = child
course = split_store.persist_xblock_dag(course, revision['user_id'])
# publish "testx.wonderful"
to_publish = BlockUsageLocator(package_id="testx.wonderful", branch="draft", block_id="head23456")
destination = CourseLocator(package_id="testx.wonderful", branch="published")
to_publish = BlockUsageLocator(
CourseLocator(org="testx", offering="wonderful", branch="draft"),
block_id="head23456"
)
destination = CourseLocator(org="testx", offering="wonderful", branch="published")
split_store.xblock_publish("test@edx.org", to_publish, destination, [to_publish.block_id], None)
def tearDown(self):
@@ -509,7 +515,7 @@ class SplitModuleCourseTests(SplitModuleTest):
self.assertEqual(len(courses), 3, "Wrong number of courses")
# check metadata -- NOTE no promised order
course = self.findByIdInResult(courses, "head12345")
self.assertEqual(course.location.package_id, "testx.GreekHero")
self.assertEqual(course.location.org, "testx")
self.assertEqual(course.category, 'course', 'wrong category')
self.assertEqual(len(course.tabs), 6, "wrong number of tabs")
self.assertEqual(
@@ -532,7 +538,8 @@ class SplitModuleCourseTests(SplitModuleTest):
self.assertEqual(len(courses_published), 1, len(courses_published))
course = self.findByIdInResult(courses_published, "head23456")
self.assertIsNotNone(course, "published courses")
self.assertEqual(course.location.package_id, "testx.wonderful")
self.assertEqual(course.location.course_key.org, "testx")
self.assertEqual(course.location.course_key.offering, "wonderful")
self.assertEqual(course.category, 'course', 'wrong category')
self.assertEqual(len(course.tabs), 4, "wrong number of tabs")
self.assertEqual(course.display_name, "The most wonderful course",
@@ -550,16 +557,27 @@ class SplitModuleCourseTests(SplitModuleTest):
self.assertIsNotNone(self.findByIdInResult(courses, "head12345"))
self.assertIsNotNone(self.findByIdInResult(courses, "head23456"))
def test_has_course(self):
'''
Test the various calling forms for has_course
'''
check_has_course_method(
modulestore(),
CourseLocator(org='testx', offering='wonderful', branch="draft"),
locator_key_fields=['org', 'offering']
)
def test_get_course(self):
'''
Test the various calling forms for get_course
'''
locator = CourseLocator(package_id="testx.GreekHero", branch="draft")
locator = CourseLocator(org='testx', offering='GreekHero', branch="draft")
head_course = modulestore().get_course(locator)
self.assertNotEqual(head_course.location.version_guid, head_course.previous_version)
locator = CourseLocator(version_guid=head_course.previous_version)
course = modulestore().get_course(locator)
self.assertIsNone(course.location.package_id)
self.assertIsNone(course.location.course_key.org)
self.assertEqual(course.location.version_guid, head_course.previous_version)
self.assertEqual(course.category, 'course')
self.assertEqual(len(course.tabs), 6)
@@ -572,9 +590,10 @@ class SplitModuleCourseTests(SplitModuleTest):
self.assertEqual(course.edited_by, "testassist@edx.org")
self.assertDictEqual(course.grade_cutoffs, {"Pass": 0.55})
locator = CourseLocator(package_id='testx.GreekHero', branch='draft')
locator = CourseLocator(org='testx', offering='GreekHero', branch='draft')
course = modulestore().get_course(locator)
self.assertEqual(course.location.package_id, "testx.GreekHero")
self.assertEqual(course.location.course_key.org, "testx")
self.assertEqual(course.location.course_key.offering, "GreekHero")
self.assertEqual(course.category, 'course')
self.assertEqual(len(course.tabs), 6)
self.assertEqual(course.display_name, "The Ancient Greek Hero")
@@ -584,29 +603,28 @@ class SplitModuleCourseTests(SplitModuleTest):
self.assertEqual(course.edited_by, "testassist@edx.org")
self.assertDictEqual(course.grade_cutoffs, {"Pass": 0.45})
locator = CourseLocator(package_id='testx.wonderful', branch='published')
locator = CourseLocator(org='testx', offering='wonderful', branch='published')
course = modulestore().get_course(locator)
published_version = course.location.version_guid
locator = CourseLocator(package_id='testx.wonderful', branch='draft')
locator = CourseLocator(org='testx', offering='wonderful', branch='draft')
course = modulestore().get_course(locator)
self.assertNotEqual(course.location.version_guid, published_version)
def test_get_course_negative(self):
# Now negative testing
self.assertRaises(InsufficientSpecificationError,
modulestore().get_course, CourseLocator(package_id='edu.meh.blah'))
self.assertRaises(ItemNotFoundError,
modulestore().get_course, CourseLocator(package_id='nosuchthing', branch='draft'))
self.assertRaises(ItemNotFoundError,
modulestore().get_course,
CourseLocator(package_id='testx.GreekHero', branch='published'))
with self.assertRaises(InsufficientSpecificationError):
modulestore().get_course(CourseLocator(org='edu', offering='meh.blah'))
with self.assertRaises(ItemNotFoundError):
modulestore().get_course(CourseLocator(org='edu', offering='nosuchthing', branch='draft'))
with self.assertRaises(ItemNotFoundError):
modulestore().get_course(CourseLocator(org='testx', offering='GreekHero', branch='published'))
def test_cache(self):
"""
Test that the mechanics of caching work.
"""
locator = CourseLocator(package_id='testx.GreekHero', branch='draft')
locator = CourseLocator(org='testx', offering='GreekHero', branch='draft')
course = modulestore().get_course(locator)
block_map = modulestore().cache_items(course.system, course.children, depth=3)
self.assertIn('chapter1', block_map)
@@ -616,7 +634,7 @@ class SplitModuleCourseTests(SplitModuleTest):
"""
get_course_successors(course_locator, version_history_depth=1)
"""
locator = CourseLocator(package_id='testx.GreekHero', branch='draft')
locator = CourseLocator(org='testx', offering='GreekHero', branch='draft')
course = modulestore().get_course(locator)
versions = [course.location.version_guid, course.previous_version]
locator = CourseLocator(version_guid=course.previous_version)
@@ -626,7 +644,7 @@ class SplitModuleCourseTests(SplitModuleTest):
locator = CourseLocator(version_guid=course.previous_version)
result = modulestore().get_course_successors(locator)
self.assertIsInstance(result, VersionTree)
self.assertIsNone(result.locator.package_id)
self.assertIsNone(result.locator.org)
self.assertEqual(result.locator.version_guid, versions[-1])
self.assertEqual(len(result.children), 1)
self.assertEqual(result.children[0].locator.version_guid, versions[-2])
@@ -652,87 +670,85 @@ class SplitModuleItemTests(SplitModuleTest):
'''
has_item(BlockUsageLocator)
'''
package_id = 'testx.GreekHero'
locator = CourseLocator(package_id=package_id, branch='draft')
course = modulestore().get_course(locator)
org = 'testx'
offering = 'GreekHero'
course_locator = CourseLocator(org=org, offering=offering, branch='draft')
course = modulestore().get_course(course_locator)
previous_version = course.previous_version
# positive tests of various forms
locator = BlockUsageLocator(version_guid=previous_version, block_id='head12345')
locator = BlockUsageLocator(CourseLocator(version_guid=previous_version), block_id='head12345')
self.assertTrue(
modulestore().has_item(package_id, locator), "couldn't find in %s" % previous_version
modulestore().has_item(locator), "couldn't find in %s" % previous_version
)
locator = BlockUsageLocator(package_id='testx.GreekHero', block_id='head12345', branch='draft')
locator = BlockUsageLocator(course_locator, block_id='head12345')
self.assertTrue(
modulestore().has_item(locator.package_id, locator),
modulestore().has_item(locator),
)
self.assertFalse(
modulestore().has_item(locator.package_id, BlockUsageLocator(
package_id=locator.package_id,
branch='published',
block_id=locator.block_id)),
modulestore().has_item(
BlockUsageLocator(
locator.course_key.for_branch('published'),
block_id=locator.block_id
)
),
"found in published head"
)
# not a course obj
locator = BlockUsageLocator(package_id='testx.GreekHero', block_id='chapter1', branch='draft')
locator = BlockUsageLocator(course_locator, block_id='chapter1')
self.assertTrue(
modulestore().has_item(locator.package_id, locator),
modulestore().has_item(locator),
"couldn't find chapter1"
)
# in published course
locator = BlockUsageLocator(package_id="testx.wonderful", block_id="head23456", branch='draft')
locator = BlockUsageLocator(
CourseLocator(org="testx", offering="wonderful", branch='draft'),
block_id="head23456"
)
self.assertTrue(
modulestore().has_item(
locator.package_id,
BlockUsageLocator(package_id=locator.package_id, block_id=locator.block_id, branch='published')
BlockUsageLocator(locator.course_key.for_branch("published"), block_id=locator.block_id)
)
)
locator.branch = 'published'
self.assertTrue(modulestore().has_item(package_id, locator), "couldn't find in published")
locator = locator.for_branch('published')
self.assertTrue(modulestore().has_item(locator), "couldn't find in published")
def test_negative_has_item(self):
# negative tests--not found
# no such course or block
package_id = 'testx.GreekHero'
locator = BlockUsageLocator(package_id="doesnotexist", block_id="head23456", branch='draft')
self.assertFalse(modulestore().has_item(package_id, locator))
locator = BlockUsageLocator(package_id="testx.wonderful", block_id="doesnotexist", branch='draft')
self.assertFalse(modulestore().has_item(package_id, locator))
# negative tests--insufficient specification
self.assertRaises(InsufficientSpecificationError, BlockUsageLocator)
locator = CourseLocator(package_id=package_id, branch='draft')
course = modulestore().get_course(locator)
previous_version = course.previous_version
with self.assertRaises(InsufficientSpecificationError):
modulestore().has_item(None, BlockUsageLocator(version_guid=previous_version))
with self.assertRaises(InsufficientSpecificationError):
modulestore().has_item(None, BlockUsageLocator(package_id='testx.GreekHero'))
locator = BlockUsageLocator(
CourseLocator(org="foo", offering="doesnotexist", branch='draft'),
block_id="head23456"
)
self.assertFalse(modulestore().has_item(locator))
locator = BlockUsageLocator(
CourseLocator(org="testx", offering="wonderful", branch='draft'),
block_id="doesnotexist"
)
self.assertFalse(modulestore().has_item(locator))
def test_get_item(self):
'''
get_item(blocklocator)
'''
locator = CourseLocator(package_id="testx.GreekHero", branch='draft')
course = modulestore().get_course(locator)
hero_locator = CourseLocator(org="testx", offering="GreekHero", branch='draft')
course = modulestore().get_course(hero_locator)
previous_version = course.previous_version
# positive tests of various forms
locator = BlockUsageLocator(version_guid=previous_version, block_id='head12345')
locator = BlockUsageLocator(CourseLocator(version_guid=previous_version), block_id='head12345')
block = modulestore().get_item(locator)
self.assertIsInstance(block, CourseDescriptor)
# get_instance just redirects to get_item, ignores package_id
self.assertIsInstance(modulestore().get_instance("package_id", locator), CourseDescriptor)
self.assertIsInstance(modulestore().get_item(locator), CourseDescriptor)
def verify_greek_hero(block):
"""
Check contents of block
"""
self.assertEqual(block.location.package_id, "testx.GreekHero")
self.assertEqual(block.location.org, "testx")
self.assertEqual(block.location.offering, "GreekHero")
self.assertEqual(len(block.tabs), 6, "wrong number of tabs")
self.assertEqual(block.display_name, "The Ancient Greek Hero")
self.assertEqual(block.advertised_start, "Fall 2013")
@@ -743,18 +759,17 @@ class SplitModuleItemTests(SplitModuleTest):
block.grade_cutoffs, {"Pass": 0.45},
)
locator = BlockUsageLocator(package_id='testx.GreekHero', block_id='head12345', branch='draft')
locator = BlockUsageLocator(hero_locator, block_id='head12345')
verify_greek_hero(modulestore().get_item(locator))
# get_instance just redirects to get_item, ignores package_id
verify_greek_hero(modulestore().get_instance("package_id", locator))
# try to look up other branches
self.assertRaises(ItemNotFoundError,
modulestore().get_item,
BlockUsageLocator(package_id=locator.as_course_locator(),
block_id=locator.block_id,
branch='published'))
locator.branch = 'draft'
with self.assertRaises(ItemNotFoundError):
modulestore().get_item(
BlockUsageLocator(
hero_locator.for_branch("published"),
block_id=locator.block_id,
)
)
self.assertIsInstance(
modulestore().get_item(locator),
CourseDescriptor
@@ -762,15 +777,19 @@ class SplitModuleItemTests(SplitModuleTest):
def test_get_non_root(self):
# not a course obj
locator = BlockUsageLocator(package_id='testx.GreekHero', block_id='chapter1', branch='draft')
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='GreekHero', branch='draft'), 'chapter1'
)
block = modulestore().get_item(locator)
self.assertEqual(block.location.package_id, "testx.GreekHero")
self.assertEqual(block.location.package_id, "testx+GreekHero")
self.assertEqual(block.category, 'chapter')
self.assertEqual(block.display_name, "Hercules")
self.assertEqual(block.edited_by, "testassist@edx.org")
# in published course
locator = BlockUsageLocator(package_id="testx.wonderful", block_id="head23456", branch='published')
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='wonderful', branch='published'), 'head23456'
)
self.assertIsInstance(
modulestore().get_item(locator),
CourseDescriptor
@@ -778,19 +797,17 @@ class SplitModuleItemTests(SplitModuleTest):
# negative tests--not found
# no such course or block
locator = BlockUsageLocator(package_id="doesnotexist", block_id="head23456", branch='draft')
locator = BlockUsageLocator(
CourseLocator(org='doesnotexist', offering='doesnotexist', branch='draft'), 'head23456'
)
with self.assertRaises(ItemNotFoundError):
modulestore().get_item(locator)
locator = BlockUsageLocator(package_id="testx.wonderful", block_id="doesnotexist", branch='draft')
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='wonderful', branch='draft'), 'doesnotexist'
)
with self.assertRaises(ItemNotFoundError):
modulestore().get_item(locator)
# negative tests--insufficient specification
with self.assertRaises(InsufficientSpecificationError):
modulestore().get_item(BlockUsageLocator(version_guid=ObjectId()))
with self.assertRaises(InsufficientSpecificationError):
modulestore().get_item(BlockUsageLocator(package_id='testx.GreekHero', branch='draft'))
# pylint: disable=W0212
def test_matching(self):
'''
@@ -800,66 +817,65 @@ class SplitModuleItemTests(SplitModuleTest):
self.assertFalse(modulestore()._value_matches('help', 'Help'))
self.assertTrue(modulestore()._value_matches(['distract', 'help', 'notme'], 'help'))
self.assertFalse(modulestore()._value_matches(['distract', 'Help', 'notme'], 'help'))
self.assertFalse(modulestore()._value_matches({'field': ['distract', 'Help', 'notme']}, {'field': 'help'}))
self.assertFalse(modulestore()._value_matches(['distract', 'Help', 'notme'], {'field': 'help'}))
self.assertTrue(modulestore()._value_matches(
self.assertFalse(modulestore()._block_matches({'field': ['distract', 'Help', 'notme']}, {'field': 'help'}))
self.assertTrue(modulestore()._block_matches(
{'field': ['distract', 'help', 'notme'],
'irrelevant': 2},
{'field': 'help'}))
self.assertTrue(modulestore()._value_matches('I need some help', {'$regex': 'help'}))
self.assertTrue(modulestore()._value_matches(['I need some help', 'today'], {'$regex': 'help'}))
self.assertFalse(modulestore()._value_matches('I need some help', {'$regex': 'Help'}))
self.assertFalse(modulestore()._value_matches(['I need some help', 'today'], {'$regex': 'Help'}))
self.assertTrue(modulestore()._value_matches('I need some help', re.compile(r'help')))
self.assertTrue(modulestore()._value_matches(['I need some help', 'today'], re.compile(r'help')))
self.assertFalse(modulestore()._value_matches('I need some help', re.compile(r'Help')))
self.assertTrue(modulestore()._value_matches(['I need some help', 'today'], re.compile(r'Help', re.IGNORECASE)))
self.assertTrue(modulestore()._block_matches({'a': 1, 'b': 2}, {'a': 1}))
self.assertTrue(modulestore()._block_matches({'a': 1, 'b': 2}, {'c': None}))
self.assertTrue(modulestore()._block_matches({'a': 1, 'b': 2}, {'a': 1, 'c': None}))
self.assertFalse(modulestore()._block_matches({'a': 1, 'b': 2}, {'a': 2}))
self.assertFalse(modulestore()._block_matches({'a': 1, 'b': 2}, {'c': 1}))
self.assertFalse(modulestore()._block_matches({'a': 1, 'b': 2}, {'a': 1, 'c': 1}))
self.assertTrue(modulestore()._block_matches({'a': 1, 'b': 2}, {'a': lambda i: 0 < i < 2}))
def test_get_items(self):
'''
get_items(locator, qualifiers, [branch])
'''
locator = CourseLocator(package_id="testx.GreekHero", branch='draft')
locator = CourseLocator(org='testx', offering='GreekHero', branch='draft')
# get all modules
matches = modulestore().get_items(locator)
self.assertEqual(len(matches), 6)
matches = modulestore().get_items(locator, qualifiers={})
matches = modulestore().get_items(locator)
self.assertEqual(len(matches), 6)
matches = modulestore().get_items(locator, qualifiers={'category': 'chapter'})
matches = modulestore().get_items(locator, category='chapter')
self.assertEqual(len(matches), 3)
matches = modulestore().get_items(locator, qualifiers={'category': 'garbage'})
matches = modulestore().get_items(locator, category='garbage')
self.assertEqual(len(matches), 0)
matches = modulestore().get_items(
locator,
qualifiers=
{
'category': 'chapter',
'fields': {'display_name': {'$regex': 'Hera'}}
}
category='chapter',
settings={'display_name': re.compile(r'Hera')},
)
self.assertEqual(len(matches), 2)
matches = modulestore().get_items(locator, qualifiers={'fields': {'children': 'chapter2'}})
matches = modulestore().get_items(locator, children='chapter2')
self.assertEqual(len(matches), 1)
self.assertEqual(matches[0].location.block_id, 'head12345')
def test_get_parents(self):
'''
get_parent_locations(locator, [block_id], [branch]): [BlockUsageLocator]
get_parent_locations(locator): [BlockUsageLocator]
'''
locator = BlockUsageLocator(package_id="testx.GreekHero", branch='draft', block_id='chapter1')
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='GreekHero', branch='draft'),
block_id='chapter1'
)
parents = modulestore().get_parent_locations(locator)
self.assertEqual(len(parents), 1)
self.assertEqual(parents[0].block_id, 'head12345')
self.assertEqual(parents[0].package_id, "testx.GreekHero")
locator.block_id = 'chapter2'
self.assertEqual(parents[0].org, "testx")
self.assertEqual(parents[0].offering, "GreekHero")
locator = locator.course_key.make_usage_key('Chapter', 'chapter2')
parents = modulestore().get_parent_locations(locator)
self.assertEqual(len(parents), 1)
self.assertEqual(parents[0].block_id, 'head12345')
locator.block_id = 'nosuchblock'
locator = locator.course_key.make_usage_key('garbage', 'nosuchblock')
parents = modulestore().get_parent_locations(locator)
self.assertEqual(len(parents), 0)
@@ -867,7 +883,9 @@ class SplitModuleItemTests(SplitModuleTest):
"""
Test the existing get_children method on xdescriptors
"""
locator = BlockUsageLocator(package_id="testx.GreekHero", block_id="head12345", branch='draft')
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='GreekHero', branch='draft'), 'head12345'
)
block = modulestore().get_item(locator)
children = block.get_children()
expected_ids = [
@@ -909,7 +927,7 @@ class TestItemCrud(SplitModuleTest):
create_item(course_or_parent_locator, category, user, definition_locator=None, fields): new_desciptor
"""
# grab link to course to ensure new versioning works
locator = CourseLocator(package_id="testx.GreekHero", branch='draft')
locator = CourseLocator(org='testx', offering='GreekHero', branch='draft')
premod_course = modulestore().get_course(locator)
premod_history = modulestore().get_course_history_info(premod_course.location)
# add minimal one w/o a parent
@@ -919,7 +937,7 @@ class TestItemCrud(SplitModuleTest):
fields={'display_name': 'new sequential'}
)
# check that course version changed and course's previous is the other one
self.assertEqual(new_module.location.package_id, "testx.GreekHero")
self.assertEqual(new_module.location.offering, "GreekHero")
self.assertNotEqual(new_module.location.version_guid, premod_course.location.version_guid)
self.assertIsNone(locator.version_guid, "Version inadvertently filled in")
current_course = modulestore().get_course(locator)
@@ -935,7 +953,7 @@ class TestItemCrud(SplitModuleTest):
self.assertEqual(new_module.display_name, 'new sequential')
# check that block does not exist in previous version
locator = BlockUsageLocator(
version_guid=premod_course.location.version_guid,
CourseLocator(version_guid=premod_course.location.version_guid),
block_id=new_module.location.block_id
)
self.assertRaises(ItemNotFoundError, modulestore().get_item, locator)
@@ -944,11 +962,16 @@ class TestItemCrud(SplitModuleTest):
"""
Test create_item w/ specifying the parent of the new item
"""
locator = BlockUsageLocator(package_id="testx.GreekHero", branch='draft', block_id='chapter2')
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='GreekHero', branch='draft'),
block_id='chapter2'
)
original = modulestore().get_item(locator)
locator = BlockUsageLocator(package_id="testx.wonderful", block_id="head23456", branch='draft')
premod_course = modulestore().get_course(locator)
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='wonderful', branch='draft'), 'head23456'
)
premod_course = modulestore().get_course(locator.course_key)
category = 'chapter'
new_module = modulestore().create_item(
locator, category, 'user123',
@@ -967,10 +990,15 @@ class TestItemCrud(SplitModuleTest):
a definition id and new def data that it branches the definition in the db.
Actually, this tries to test all create_item features not tested above.
"""
locator = BlockUsageLocator(package_id="testx.GreekHero", branch='draft', block_id='problem1')
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='GreekHero', branch='draft'),
block_id='problem1'
)
original = modulestore().get_item(locator)
locator = BlockUsageLocator(package_id="guestx.contender", block_id="head345679", branch='draft')
locator = BlockUsageLocator(
CourseLocator(org='guestx', offering='contender', branch='draft'), 'head345679'
)
category = 'problem'
new_payload = "<problem>empty</problem>"
new_module = modulestore().create_item(
@@ -1002,8 +1030,9 @@ class TestItemCrud(SplitModuleTest):
"""
Check that using odd characters in block id don't break ability to add and retrieve block.
"""
parent_locator = BlockUsageLocator(package_id="guestx.contender", block_id="head345679", branch='draft')
chapter_locator = BlockUsageLocator(package_id="guestx.contender", block_id="foo.bar_-~:0", branch='draft')
course_key = CourseLocator(org='guestx', offering='contender', branch='draft')
parent_locator = BlockUsageLocator(course_key, block_id="head345679")
chapter_locator = BlockUsageLocator(course_key, block_id="foo.bar_-~:0")
modulestore().create_item(
parent_locator, 'chapter', 'anotheruser',
block_id=chapter_locator.block_id,
@@ -1014,7 +1043,7 @@ class TestItemCrud(SplitModuleTest):
self.assertEqual(new_module.location.block_id, "foo.bar_-~:0") # hardcode to ensure BUL init didn't change
# now try making that a parent of something
new_payload = "<problem>empty</problem>"
problem_locator = BlockUsageLocator(package_id="guestx.contender", block_id="prob.bar_-~:99a", branch='draft')
problem_locator = BlockUsageLocator(course_key, block_id="prob.bar_-~:99a")
modulestore().create_item(
chapter_locator, 'problem', 'anotheruser',
block_id=problem_locator.block_id,
@@ -1032,15 +1061,13 @@ class TestItemCrud(SplitModuleTest):
"""
# start transaction w/ simple creation
user = random.getrandbits(32)
new_course = modulestore().create_course('test_org.test_transaction', 'test_org', user)
new_course_locator = new_course.location.as_course_locator()
new_course = modulestore().create_course('test_org', 'test_transaction', user)
new_course_locator = new_course.id
index_history_info = modulestore().get_course_history_info(new_course.location)
course_block_prev_version = new_course.previous_version
course_block_update_version = new_course.update_version
self.assertIsNotNone(new_course_locator.version_guid, "Want to test a definite version")
versionless_course_locator = CourseLocator(
package_id=new_course_locator.package_id, branch=new_course_locator.branch
)
versionless_course_locator = new_course_locator.version_agnostic()
# positive simple case: no force, add chapter
new_ele = modulestore().create_item(
@@ -1093,9 +1120,8 @@ class TestItemCrud(SplitModuleTest):
# add new child to old parent in continued (leave off version_guid)
course_module_locator = BlockUsageLocator(
package_id=new_course.location.package_id,
new_course.location.course_key.version_agnostic(),
block_id=new_course.location.block_id,
branch=new_course.location.branch
)
new_ele = modulestore().create_item(
course_module_locator, 'chapter', user,
@@ -1115,7 +1141,10 @@ class TestItemCrud(SplitModuleTest):
"""
test updating an items metadata ensuring the definition doesn't version but the course does if it should
"""
locator = BlockUsageLocator(package_id="testx.GreekHero", block_id="problem3_2", branch='draft')
locator = BlockUsageLocator(
CourseLocator(org="testx", offering="GreekHero", branch='draft'),
block_id="problem3_2"
)
problem = modulestore().get_item(locator)
pre_def_id = problem.definition_locator.definition_id
pre_version_guid = problem.location.version_guid
@@ -1132,13 +1161,13 @@ class TestItemCrud(SplitModuleTest):
self.assertEqual(updated_problem.max_attempts, 4)
# refetch to ensure original didn't change
original_location = BlockUsageLocator(
version_guid=pre_version_guid,
CourseLocator(version_guid=pre_version_guid),
block_id=problem.location.block_id
)
problem = modulestore().get_item(original_location)
self.assertNotEqual(problem.max_attempts, 4, "original changed")
current_course = modulestore().get_course(locator)
current_course = modulestore().get_course(locator.course_key)
self.assertEqual(updated_problem.location.version_guid, current_course.location.version_guid)
history_info = modulestore().get_course_history_info(current_course.location)
@@ -1149,7 +1178,9 @@ class TestItemCrud(SplitModuleTest):
"""
test updating an item's children ensuring the definition doesn't version but the course does if it should
"""
locator = BlockUsageLocator(package_id="testx.GreekHero", block_id="chapter3", branch='draft')
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='GreekHero', branch='draft'), 'chapter3'
)
block = modulestore().get_item(locator)
pre_def_id = block.definition_locator.definition_id
pre_version_guid = block.location.version_guid
@@ -1164,10 +1195,9 @@ class TestItemCrud(SplitModuleTest):
self.assertNotEqual(updated_problem.location.version_guid, pre_version_guid)
self.assertEqual(updated_problem.children, block.children)
self.assertNotIn(moved_child, updated_problem.children)
locator.block_id = "chapter1"
locator = locator.course_key.make_usage_key('Chapter', "chapter1")
other_block = modulestore().get_item(locator)
other_block.children.append(moved_child)
other_block.save() # decache model changes
other_updated = modulestore().update_item(other_block, '**replace_user**')
self.assertIn(moved_child, other_updated.children)
@@ -1175,7 +1205,9 @@ class TestItemCrud(SplitModuleTest):
"""
test updating an item's definition: ensure it gets versioned as well as the course getting versioned
"""
locator = BlockUsageLocator(package_id="testx.GreekHero", block_id="head12345", branch='draft')
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='GreekHero', branch='draft'), 'head12345'
)
block = modulestore().get_item(locator)
pre_def_id = block.definition_locator.definition_id
pre_version_guid = block.location.version_guid
@@ -1192,10 +1224,16 @@ class TestItemCrud(SplitModuleTest):
"""
Test updating metadata, children, and definition in a single call ensuring all the versioning occurs
"""
locator = BlockUsageLocator(package_id="testx.GreekHero", branch='draft', block_id='problem1')
locator = BlockUsageLocator(
CourseLocator('testx', 'GreekHero', branch='draft'),
block_id='problem1'
)
original = modulestore().get_item(locator)
# first add 2 children to the course for the update to manipulate
locator = BlockUsageLocator(package_id="guestx.contender", block_id="head345679", branch='draft')
locator = BlockUsageLocator(
CourseLocator('guestx', 'contender', branch='draft'),
block_id="head345679"
)
category = 'problem'
new_payload = "<problem>empty</problem>"
modulestore().create_item(
@@ -1231,33 +1269,28 @@ class TestItemCrud(SplitModuleTest):
def test_delete_item(self):
course = self.create_course_for_deletion()
self.assertRaises(ValueError,
modulestore().delete_item,
course.location,
'deleting_user')
reusable_location = BlockUsageLocator(
package_id=course.location.package_id,
block_id=course.location.block_id,
branch='draft')
with self.assertRaises(ValueError):
modulestore().delete_item(course.location, 'deleting_user')
reusable_location = course.id.version_agnostic().for_branch('draft')
# delete a leaf
problems = modulestore().get_items(reusable_location, {'category': 'problem'})
problems = modulestore().get_items(reusable_location, category='problem')
locn_to_del = problems[0].location
new_course_loc = modulestore().delete_item(locn_to_del, 'deleting_user', delete_children=False)
deleted = BlockUsageLocator(package_id=reusable_location.package_id,
branch=reusable_location.branch,
block_id=locn_to_del.block_id)
self.assertFalse(modulestore().has_item(reusable_location.package_id, deleted))
self.assertRaises(VersionConflictError, modulestore().has_item, reusable_location.package_id, locn_to_del)
deleted = locn_to_del.version_agnostic()
self.assertFalse(modulestore().has_item(deleted))
with self.assertRaises(VersionConflictError):
modulestore().has_item(locn_to_del)
locator = BlockUsageLocator(
version_guid=locn_to_del.version_guid,
CourseLocator(version_guid=locn_to_del.version_guid),
block_id=locn_to_del.block_id
)
self.assertTrue(modulestore().has_item(reusable_location.package_id, locator))
self.assertTrue(modulestore().has_item(locator))
self.assertNotEqual(new_course_loc.version_guid, course.location.version_guid)
# delete a subtree
nodes = modulestore().get_items(reusable_location, {'category': 'chapter'})
nodes = modulestore().get_items(reusable_location, category='chapter')
new_course_loc = modulestore().delete_item(nodes[0].location, 'deleting_user', delete_children=True)
# check subtree
@@ -1267,15 +1300,23 @@ class TestItemCrud(SplitModuleTest):
"""
if node:
node_loc = node.location
self.assertFalse(modulestore().has_item(reusable_location.package_id,
BlockUsageLocator(
package_id=node_loc.package_id,
branch=node_loc.branch,
block_id=node.location.block_id)))
self.assertFalse(
modulestore().has_item(
BlockUsageLocator(
CourseLocator(
org=node_loc.org,
offering=node_loc.offering,
branch=node_loc.branch,
),
block_id=node_loc.block_id
)
)
)
locator = BlockUsageLocator(
version_guid=node.location.version_guid,
block_id=node.location.block_id)
self.assertTrue(modulestore().has_item(reusable_location.package_id, locator))
CourseLocator(version_guid=node.location.version_guid),
block_id=node.location.block_id
)
self.assertTrue(modulestore().has_item(locator))
if node.has_children:
for sub in node.get_children():
check_subtree(sub)
@@ -1285,11 +1326,11 @@ class TestItemCrud(SplitModuleTest):
"""
Create a course we can delete
"""
course = modulestore().create_course('nihilx.deletion', 'nihilx', 'deleting_user')
course = modulestore().create_course('nihilx', 'deletion', 'deleting_user')
root = BlockUsageLocator(
package_id=course.location.package_id,
course.id.version_agnostic().for_branch('draft'),
block_id=course.location.block_id,
branch='draft')
)
for _ in range(4):
self.create_subtree_for_deletion(root, ['chapter', 'vertical', 'problem'])
return modulestore().get_item(root)
@@ -1300,8 +1341,8 @@ class TestItemCrud(SplitModuleTest):
"""
if not category_queue:
return
node = modulestore().create_item(parent, category_queue[0], 'deleting_user')
node_loc = BlockUsageLocator(parent.as_course_locator(), block_id=node.location.block_id)
node = modulestore().create_item(parent.version_agnostic(), category_queue[0], 'deleting_user')
node_loc = BlockUsageLocator(parent.course_key, block_id=node.location.block_id)
for _ in range(4):
self.create_subtree_for_deletion(node_loc, category_queue[1:])
@@ -1315,7 +1356,7 @@ class TestCourseCreation(SplitModuleTest):
The simplest case but probing all expected results from it.
"""
# Oddly getting differences of 200nsec
new_course = modulestore().create_course('test_org.test_course', 'test_org', 'create_user')
new_course = modulestore().create_course('test_org', 'test_course', 'create_user')
new_locator = new_course.location
# check index entry
index_info = modulestore().get_course_index_info(new_locator)
@@ -1340,13 +1381,13 @@ class TestCourseCreation(SplitModuleTest):
"""
Test making a course which points to an existing draft and published but not making any changes to either.
"""
original_locator = CourseLocator(package_id="testx.wonderful", branch='draft')
original_locator = CourseLocator(org='testx', offering='wonderful', branch='draft')
original_index = modulestore().get_course_index_info(original_locator)
new_draft = modulestore().create_course(
'best', 'leech', 'leech_master',
versions_dict=original_index['versions'])
new_draft_locator = new_draft.location
self.assertRegexpMatches(new_draft_locator.package_id, 'best')
self.assertRegexpMatches(new_draft_locator.org, 'best')
# the edited_by and other meta fields on the new course will be the original author not this one
self.assertEqual(new_draft.edited_by, 'test@edx.org')
self.assertEqual(new_draft_locator.version_guid, original_index['versions']['draft'])
@@ -1354,7 +1395,7 @@ class TestCourseCreation(SplitModuleTest):
new_index = modulestore().get_course_index_info(new_draft_locator)
self.assertEqual(new_index['edited_by'], 'leech_master')
new_published_locator = CourseLocator(package_id=new_draft_locator.package_id, branch='published')
new_published_locator = new_draft_locator.course_key.for_branch("published")
new_published = modulestore().get_course(new_published_locator)
self.assertEqual(new_published.edited_by, 'test@edx.org')
self.assertEqual(new_published.location.version_guid, original_index['versions']['published'])
@@ -1365,7 +1406,7 @@ class TestCourseCreation(SplitModuleTest):
new_draft.location, 'chapter', 'leech_master',
fields={'display_name': 'new chapter'}
)
new_draft_locator.version_guid = None
new_draft_locator = new_draft_locator.course_key.version_agnostic()
new_index = modulestore().get_course_index_info(new_draft_locator)
self.assertNotEqual(new_index['versions']['draft'], original_index['versions']['draft'])
new_draft = modulestore().get_course(new_draft_locator)
@@ -1377,18 +1418,12 @@ class TestCourseCreation(SplitModuleTest):
original_course = modulestore().get_course(original_locator)
self.assertEqual(original_course.location.version_guid, original_index['versions']['draft'])
self.assertFalse(
modulestore().has_item(new_draft_locator.package_id, BlockUsageLocator(
original_locator,
block_id=new_item.location.block_id
))
)
def test_derived_course(self):
"""
Create a new course which overrides metadata and course_data
"""
original_locator = CourseLocator(package_id="guestx.contender", branch='draft')
original_locator = CourseLocator(org='guestx', offering='contender', branch='draft')
original = modulestore().get_course(original_locator)
original_index = modulestore().get_course_index_info(original_locator)
fields = {}
@@ -1410,7 +1445,7 @@ class TestCourseCreation(SplitModuleTest):
fields=fields
)
new_draft_locator = new_draft.location
self.assertRegexpMatches(new_draft_locator.package_id, 'counter')
self.assertRegexpMatches(new_draft_locator.org, 'counter')
# the edited_by and other meta fields on the new course will be the original author not this one
self.assertEqual(new_draft.edited_by, 'leech_master')
self.assertNotEqual(new_draft_locator.version_guid, original_index['versions']['draft'])
@@ -1425,19 +1460,12 @@ class TestCourseCreation(SplitModuleTest):
def test_update_course_index(self):
"""
Test changing the org, pretty id, etc of a course. Test that it doesn't allow changing the id, etc.
Test the versions pointers. NOTE: you can change the org, offering, or other things, but
it's not clear how you'd find them again or associate them w/ existing student history since
we use course_key so many places as immutable.
"""
locator = CourseLocator(package_id="testx.GreekHero", branch='draft')
locator = CourseLocator(org='testx', offering='GreekHero', branch='draft')
course_info = modulestore().get_course_index_info(locator)
course_info['org'] = 'funkyU'
modulestore().update_course_index(course_info)
course_info = modulestore().get_course_index_info(locator)
self.assertEqual(course_info['org'], 'funkyU')
course_info['org'] = 'moreFunky'
modulestore().update_course_index(course_info)
course_info = modulestore().get_course_index_info(locator)
self.assertEqual(course_info['org'], 'moreFunky')
# an allowed but not necessarily recommended way to revert the draft version
head_course = modulestore().get_course(locator)
@@ -1450,7 +1478,7 @@ class TestCourseCreation(SplitModuleTest):
# an allowed but not recommended way to publish a course
versions['published'] = versions['draft']
modulestore().update_course_index(course_info)
course = modulestore().get_course(CourseLocator(package_id=locator.package_id, branch="published"))
course = modulestore().get_course(locator.for_branch("published"))
self.assertEqual(course.location.version_guid, versions['draft'])
def test_create_with_root(self):
@@ -1459,7 +1487,7 @@ class TestCourseCreation(SplitModuleTest):
"""
user = random.getrandbits(32)
new_course = modulestore().create_course(
'test_org.test_transaction', 'test_org', user,
'test_org', 'test_transaction', user,
root_block_id='top', root_category='chapter'
)
self.assertEqual(new_course.location.block_id, 'top')
@@ -1480,7 +1508,8 @@ class TestCourseCreation(SplitModuleTest):
user = random.getrandbits(32)
courses = modulestore().get_courses()
with self.assertRaises(DuplicateCourseError):
modulestore().create_course(courses[0].location.package_id, 'org', 'pretty', user)
dupe_course_key = courses[0].location.course_key
modulestore().create_course(dupe_course_key.org, dupe_course_key.offering, user)
class TestInheritance(SplitModuleTest):
@@ -1493,11 +1522,15 @@ class TestInheritance(SplitModuleTest):
"""
# Note, not testing value where defined (course) b/c there's no
# defined accessor for it on CourseDescriptor.
locator = BlockUsageLocator(package_id="testx.GreekHero", block_id="problem3_2", branch='draft')
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='GreekHero', branch='draft'), 'problem3_2'
)
node = modulestore().get_item(locator)
# inherited
self.assertEqual(node.graceperiod, datetime.timedelta(hours=2))
locator = BlockUsageLocator(package_id="testx.GreekHero", block_id="problem1", branch='draft')
locator = BlockUsageLocator(
CourseLocator(org='testx', offering='GreekHero', branch='draft'), 'problem1'
)
node = modulestore().get_item(locator)
# overridden
self.assertEqual(node.graceperiod, datetime.timedelta(hours=4))
@@ -1518,8 +1551,8 @@ class TestPublish(SplitModuleTest):
"""
Test the standard patterns: publish to new branch, revise and publish
"""
source_course = CourseLocator(package_id="testx.GreekHero", branch='draft')
dest_course = CourseLocator(package_id="testx.GreekHero", branch="published")
source_course = CourseLocator(org='testx', offering='GreekHero', branch='draft')
dest_course = CourseLocator(org='testx', offering='GreekHero', branch="published")
modulestore().xblock_publish(self.user, source_course, dest_course, ["head12345"], ["chapter2", "chapter3"])
expected = ["head12345", "chapter1"]
self._check_course(
@@ -1560,13 +1593,13 @@ class TestPublish(SplitModuleTest):
"""
Test the exceptions which preclude successful publication
"""
source_course = CourseLocator(package_id="testx.GreekHero", branch='draft')
source_course = CourseLocator(org='testx', offering='GreekHero', branch='draft')
# destination does not exist
destination_course = CourseLocator(package_id="Unknown", branch="published")
destination_course = CourseLocator(org='fake', offering='Unknown', branch="published")
with self.assertRaises(ItemNotFoundError):
modulestore().xblock_publish(self.user, source_course, destination_course, ["chapter3"], None)
# publishing into a new branch w/o publishing the root
destination_course = CourseLocator(package_id="testx.GreekHero", branch="published")
destination_course = CourseLocator(org='testx', offering='GreekHero', branch="published")
with self.assertRaises(ItemNotFoundError):
modulestore().xblock_publish(self.user, source_course, destination_course, ["chapter3"], None)
# publishing a subdag w/o the parent already in course
@@ -1578,8 +1611,8 @@ class TestPublish(SplitModuleTest):
"""
Test publishing moves and deletes.
"""
source_course = CourseLocator(package_id="testx.GreekHero", branch='draft')
dest_course = CourseLocator(package_id="testx.GreekHero", branch="published")
source_course = CourseLocator(org='testx', offering='GreekHero', branch='draft')
dest_course = CourseLocator(org='testx', offering='GreekHero', branch="published")
modulestore().xblock_publish(self.user, source_course, dest_course, ["head12345"], ["chapter2"])
expected = ["head12345", "chapter1", "chapter3", "problem1", "problem3_2"]
self._check_course(source_course, dest_course, expected, ["chapter2"])

137
common/lib/xmodule/xmodule/modulestore/tests/test_split_w_old_mongo.py Normal file
View File

@@ -0,0 +1,137 @@
import unittest
import mock
import datetime
import uuid
import random
from xmodule.modulestore.inheritance import InheritanceMixin
from xmodule.modulestore.locator import CourseLocator, BlockUsageLocator
from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore
from xmodule.modulestore.mongo import MongoModuleStore, DraftMongoModuleStore
from xmodule.modulestore.mongo.draft import DIRECT_ONLY_CATEGORIES
class SplitWMongoCourseBoostrapper(unittest.TestCase):
"""
Helper for tests which need to construct split mongo & old mongo based courses to get interesting internal structure.
Override _create_course and after invoking the super() _create_course, have it call _create_item for
each xblock you want in the course.
This class ensures the db gets created, opened, and cleaned up in addition to creating the course
Defines the following attrs on self:
* userid: a random non-registered mock user id
* split_mongo: a pointer to the split mongo instance
* old_mongo: a pointer to the old_mongo instance
* draft_mongo: a pointer to the old draft instance
* split_course_key (CourseLocator): of the new course
* old_course_key: the SlashSpecifiedCourseKey for the course
"""
# Snippet of what would be in the django settings envs file
db_config = {
'host': 'localhost',
'db': 'test_xmodule',
}
modulestore_options = {
'default_class': 'xmodule.raw_module.RawDescriptor',
'fs_root': '',
'render_template': mock.Mock(return_value=""),
'xblock_mixins': (InheritanceMixin,)
}
split_course_key = CourseLocator('test_org', 'test_course.runid', branch='draft')
def setUp(self):
self.db_config['collection'] = 'modulestore{0}'.format(uuid.uuid4().hex[:5])
self.userid = random.getrandbits(32)
super(SplitWMongoCourseBoostrapper, self).setUp()
self.split_mongo = SplitMongoModuleStore(
self.db_config,
**self.modulestore_options
)
self.addCleanup(self.split_mongo.db.connection.close)
self.addCleanup(self.tear_down_split)
self.old_mongo = MongoModuleStore(self.db_config, **self.modulestore_options)
self.draft_mongo = DraftMongoModuleStore(self.db_config, **self.modulestore_options)
self.addCleanup(self.tear_down_mongo)
self.old_course_key = None
self.runtime = None
self._create_course()
def tear_down_split(self):
"""
Remove the test collections, close the db connection
"""
split_db = self.split_mongo.db
split_db.drop_collection(split_db.course_index)
split_db.drop_collection(split_db.structures)
split_db.drop_collection(split_db.definitions)
def tear_down_mongo(self):
"""
Remove the test collections, close the db connection
"""
split_db = self.split_mongo.db
# old_mongo doesn't give a db attr, but all of the dbs are the same
split_db.drop_collection(self.old_mongo.collection)
def _create_item(self, category, name, data, metadata, parent_category, parent_name, draft=True, split=True):
"""
Create the item of the given category and block id in split and old mongo, add it to the optional
parent. The parent category is only needed because old mongo requires it for the id.
"""
location = self.old_course_key.make_usage_key(category, name)
if not draft or category in DIRECT_ONLY_CATEGORIES:
mongo = self.old_mongo
else:
mongo = self.draft_mongo
mongo.create_and_save_xmodule(location, data, metadata, self.runtime)
if isinstance(data, basestring):
fields = {'data': data}
else:
fields = data.copy()
fields.update(metadata)
if parent_name:
# add child to parent in mongo
parent_location = self.old_course_key.make_usage_key(parent_category, parent_name)
if not draft or parent_category in DIRECT_ONLY_CATEGORIES:
mongo = self.old_mongo
else:
mongo = self.draft_mongo
parent = mongo.get_item(parent_location)
parent.children.append(location)
mongo.update_item(parent, self.userid)
# create pointer for split
course_or_parent_locator = BlockUsageLocator(
course_key=self.split_course_key,
block_id=parent_name
)
else:
course_or_parent_locator = self.split_course_key
if split:
self.split_mongo.create_item(course_or_parent_locator, category, self.userid, block_id=name, fields=fields)
def _create_course(self, split=True):
"""
* some detached items
* some attached children
* some orphans
"""
metadata = {
'start': datetime.datetime(2000, 3, 13, 4),
'display_name': 'Migration test course',
}
data = {
'wiki_slug': 'test_course_slug'
}
fields = metadata.copy()
fields.update(data)
if split:
# split requires the course to be created separately from creating items
self.split_mongo.create_course(
self.split_course_key.org, self.split_course_key.offering, self.userid, fields=fields, root_block_id='runid'
)
old_course = self.old_mongo.create_course(self.split_course_key.org, 'test_course/runid', fields=fields)
self.old_course_key = old_course.id
self.runtime = old_course.runtime

35
common/lib/xmodule/xmodule/modulestore/tests/test_xml.py
View File

@@ -7,12 +7,13 @@ import unittest
from glob import glob
from mock import patch
from xmodule.course_module import CourseDescriptor
from xmodule.modulestore.xml import XMLModuleStore
from xmodule.modulestore import Location, XML_MODULESTORE_TYPE
from .test_modulestore import check_path_to_location
from xmodule.tests import DATA_DIR
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from xmodule.modulestore.tests.test_modulestore import check_has_course_method
def glob_tildes_at_end(path):
@@ -58,22 +59,16 @@ class TestXMLModuleStore(unittest.TestCase):
modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy'], load_error_modules=False)
# Look up the errors during load. There should be none.
location = CourseDescriptor.id_to_location("edX/toy/2012_Fall")
errors = modulestore.get_item_errors(location)
errors = modulestore.get_course_errors(SlashSeparatedCourseKey("edX", "toy", "2012_Fall"))
assert errors == []
@patch("xmodule.modulestore.xml.glob.glob", side_effect=glob_tildes_at_end)
def test_tilde_files_ignored(self, _fake_glob):
modulestore = XMLModuleStore(DATA_DIR, course_dirs=['tilde'], load_error_modules=False)
course_module = modulestore.modules['edX/tilde/2012_Fall']
about_location = Location({
'tag': 'i4x',
'org': 'edX',
'course': 'tilde',
'category': 'about',
'name': 'index',
})
about_module = course_module[about_location]
about_location = SlashSeparatedCourseKey('edX', 'tilde', '2012_Fall').make_usage_key(
'about', 'index',
)
about_module = modulestore.get_item(about_location)
self.assertIn("GREEN", about_module.data)
self.assertNotIn("RED", about_module.data)
@@ -85,13 +80,13 @@ class TestXMLModuleStore(unittest.TestCase):
for course in store.get_courses():
course_locations = store.get_courses_for_wiki(course.wiki_slug)
self.assertEqual(len(course_locations), 1)
self.assertIn(Location('i4x', 'edX', course.location.course, 'course', '2012_Fall'), course_locations)
self.assertIn(course.location, course_locations)
course_locations = store.get_courses_for_wiki('no_such_wiki')
self.assertEqual(len(course_locations), 0)
# now set toy course to share the wiki with simple course
toy_course = store.get_course('edX/toy/2012_Fall')
toy_course = store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
toy_course.wiki_slug = 'simple'
course_locations = store.get_courses_for_wiki('toy')
@@ -100,4 +95,14 @@ class TestXMLModuleStore(unittest.TestCase):
course_locations = store.get_courses_for_wiki('simple')
self.assertEqual(len(course_locations), 2)
for course_number in ['toy', 'simple']:
self.assertIn(Location('i4x', 'edX', course_number, 'course', '2012_Fall'), course_locations)
self.assertIn(Location('edX', course_number, '2012_Fall', 'course', '2012_Fall'), course_locations)
def test_has_course(self):
"""
Test the has_course method
"""
check_has_course_method(
XMLModuleStore(DATA_DIR, course_dirs=['toy', 'simple']),
SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'),
locator_key_fields=SlashSeparatedCourseKey.KEY_FIELDS
)

153
common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py
View File

@@ -1,7 +1,6 @@
"""
Tests for XML importer.
"""
from unittest import TestCase
import mock
from xblock.core import XBlock
from xblock.fields import String, Scope, ScopeIds
@@ -9,7 +8,93 @@ from xblock.runtime import Runtime, KvsFieldData, DictKeyValueStore
from xmodule.x_module import XModuleMixin
from xmodule.modulestore import Location
from xmodule.modulestore.inheritance import InheritanceMixin
from xmodule.modulestore.xml_importer import remap_namespace
from xmodule.modulestore.xml_importer import import_module
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from xmodule.tests import DATA_DIR
from uuid import uuid4
import unittest
import importlib
class ModuleStoreNoSettings(unittest.TestCase):
"""
A mixin to create a mongo modulestore that avoids settings
"""
HOST = 'localhost'
PORT = 27017
DB = 'test_mongo_%s' % uuid4().hex[:5]
COLLECTION = 'modulestore'
FS_ROOT = DATA_DIR
DEFAULT_CLASS = 'xmodule.modulestore.tests.test_xml_importer.StubXBlock'
RENDER_TEMPLATE = lambda t_n, d, ctx = None, nsp = 'main': ''
modulestore_options = {
'default_class': DEFAULT_CLASS,
'fs_root': DATA_DIR,
'render_template': RENDER_TEMPLATE,
}
DOC_STORE_CONFIG = {
'host': HOST,
'db': DB,
'collection': COLLECTION,
}
MODULESTORE = {
'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': modulestore_options
}
modulestore = None
def cleanup_modulestore(self):
"""
cleanup
"""
if modulestore:
connection = self.modulestore.database.connection
connection.drop_database(self.modulestore.database)
connection.close()
def setUp(self):
"""
Add cleanups
"""
self.addCleanup(self.cleanup_modulestore)
super(ModuleStoreNoSettings, self).setUp()
#===========================================
def modulestore():
"""
Mock the django dependent global modulestore function to disentangle tests from django
"""
def load_function(engine_path):
"""
Load the given engine
"""
module_path, _, name = engine_path.rpartition('.')
return getattr(importlib.import_module(module_path), name)
if ModuleStoreNoSettings.modulestore is None:
class_ = load_function(ModuleStoreNoSettings.MODULESTORE['ENGINE'])
options = {}
options.update(ModuleStoreNoSettings.MODULESTORE['OPTIONS'])
options['render_template'] = render_to_template_mock
# pylint: disable=W0142
ModuleStoreNoSettings.modulestore = class_(
ModuleStoreNoSettings.MODULESTORE['DOC_STORE_CONFIG'],
**options
)
return ModuleStoreNoSettings.modulestore
# pylint: disable=W0613
def render_to_template_mock(*args):
pass
class StubXBlock(XBlock, XModuleMixin, InheritanceMixin):
@@ -29,7 +114,7 @@ class StubXBlock(XBlock, XModuleMixin, InheritanceMixin):
)
class RemapNamespaceTest(TestCase):
class RemapNamespaceTest(ModuleStoreNoSettings):
"""
Test that remapping the namespace from import to the actual course location.
"""
@@ -42,81 +127,99 @@ class RemapNamespaceTest(TestCase):
self.field_data = KvsFieldData(kvs=DictKeyValueStore())
self.scope_ids = ScopeIds('Bob', 'stubxblock', '123', 'import')
self.xblock = StubXBlock(self.runtime, self.field_data, self.scope_ids)
super(RemapNamespaceTest, self).setUp()
def test_remap_namespace_native_xblock(self):
# Set the XBlock's location
self.xblock.location = Location("i4x://import/org/run/stubxblock")
self.xblock.location = Location("org", "import", "run", "category", "stubxblock")
# Explicitly set the content and settings fields
self.xblock.test_content_field = "Explicitly set"
self.xblock.test_settings_field = "Explicitly set"
self.xblock.save()
# Remap the namespace
target_location_namespace = Location("i4x://course/org/run/stubxblock")
remap_namespace(self.xblock, target_location_namespace)
# Move to different runtime w/ different course id
target_location_namespace = SlashSeparatedCourseKey("org", "course", "run")
new_version = import_module(
self.xblock,
modulestore(),
self.xblock.location.course_key,
target_location_namespace,
do_import_static=False
)
# Check the XBlock's location
self.assertEqual(self.xblock.location, target_location_namespace)
self.assertEqual(new_version.location.course_key, target_location_namespace)
# Check the values of the fields.
# The content and settings fields should be preserved
self.assertEqual(self.xblock.test_content_field, 'Explicitly set')
self.assertEqual(self.xblock.test_settings_field, 'Explicitly set')
self.assertEqual(new_version.test_content_field, 'Explicitly set')
self.assertEqual(new_version.test_settings_field, 'Explicitly set')
# Expect that these fields are marked explicitly set
self.assertIn(
'test_content_field',
self.xblock.get_explicitly_set_fields_by_scope(scope=Scope.content)
new_version.get_explicitly_set_fields_by_scope(scope=Scope.content)
)
self.assertIn(
'test_settings_field',
self.xblock.get_explicitly_set_fields_by_scope(scope=Scope.settings)
new_version.get_explicitly_set_fields_by_scope(scope=Scope.settings)
)
def test_remap_namespace_native_xblock_default_values(self):
# Set the XBlock's location
self.xblock.location = Location("i4x://import/org/run/stubxblock")
self.xblock.location = Location("org", "import", "run", "category", "stubxblock")
# Do NOT set any values, so the fields should use the defaults
self.xblock.save()
# Remap the namespace
target_location_namespace = Location("i4x://course/org/run/stubxblock")
remap_namespace(self.xblock, target_location_namespace)
target_location_namespace = Location("org", "course", "run", "category", "stubxblock")
new_version = import_module(
self.xblock,
modulestore(),
self.xblock.location.course_key,
target_location_namespace.course_key,
do_import_static=False
)
# Check the values of the fields.
# The content and settings fields should be the default values
self.assertEqual(self.xblock.test_content_field, 'default value')
self.assertEqual(self.xblock.test_settings_field, 'default value')
self.assertEqual(new_version.test_content_field, 'default value')
self.assertEqual(new_version.test_settings_field, 'default value')
# The fields should NOT appear in the explicitly set fields
self.assertNotIn(
'test_content_field',
self.xblock.get_explicitly_set_fields_by_scope(scope=Scope.content)
new_version.get_explicitly_set_fields_by_scope(scope=Scope.content)
)
self.assertNotIn(
'test_settings_field',
self.xblock.get_explicitly_set_fields_by_scope(scope=Scope.settings)
new_version.get_explicitly_set_fields_by_scope(scope=Scope.settings)
)
def test_remap_namespace_native_xblock_inherited_values(self):
# Set the XBlock's location
self.xblock.location = Location("i4x://import/org/run/stubxblock")
self.xblock.location = Location("org", "import", "run", "category", "stubxblock")
self.xblock.save()
# Remap the namespace
target_location_namespace = Location("i4x://course/org/run/stubxblock")
remap_namespace(self.xblock, target_location_namespace)
target_location_namespace = Location("org", "course", "run", "category", "stubxblock")
new_version = import_module(
self.xblock,
modulestore(),
self.xblock.location.course_key,
target_location_namespace.course_key,
do_import_static=False
)
# Inherited fields should NOT be explicitly set
self.assertNotIn(
'start', self.xblock.get_explicitly_set_fields_by_scope(scope=Scope.settings)
'start', new_version.get_explicitly_set_fields_by_scope(scope=Scope.settings)
)
self.assertNotIn(
'graded', self.xblock.get_explicitly_set_fields_by_scope(scope=Scope.settings)
'graded', new_version.get_explicitly_set_fields_by_scope(scope=Scope.settings)
)

223
common/lib/xmodule/xmodule/modulestore/xml.py
View File

@@ -16,21 +16,23 @@ from path import path
from xmodule.error_module import ErrorDescriptor
from xmodule.errortracker import make_error_tracker, exc_info_to_str
from xmodule.course_module import CourseDescriptor
from xmodule.mako_module import MakoDescriptorSystem
from xmodule.x_module import XMLParsingSystem, policy_key
from xmodule.modulestore.xml_exporter import DEFAULT_CONTENT_FIELDS
from xmodule.tabs import CourseTabList
from xmodule.modulestore.keys import UsageKey
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from xblock.fields import ScopeIds
from xblock.field_data import DictFieldData
from xblock.runtime import DictKeyValueStore, IdReader, IdGenerator
from xblock.runtime import DictKeyValueStore, IdGenerator
from . import ModuleStoreReadBase, Location, XML_MODULESTORE_TYPE
from .exceptions import ItemNotFoundError
from .inheritance import compute_inherited_metadata, inheriting_field_data
from xblock.fields import ScopeIds, Reference, ReferenceList, ReferenceValueDict
edx_xml_parser = etree.XMLParser(dtd_validation=False, load_dtd=False,
remove_comments=True, remove_blank_text=True)
@@ -51,7 +53,7 @@ def clean_out_mako_templating(xml_string):
class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
def __init__(self, xmlstore, course_id, course_dir,
error_tracker, parent_tracker,
load_error_modules=True, id_reader=None, **kwargs):
load_error_modules=True, **kwargs):
"""
A class that handles loading from xml. Does some munging to ensure that
all elements have unique slugs.
@@ -60,13 +62,7 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
"""
self.unnamed = defaultdict(int) # category -> num of new url_names for that category
self.used_names = defaultdict(set) # category -> set of used url_names
course_id_dict = Location.parse_course_id(course_id)
self.org = course_id_dict['org']
self.course = course_id_dict['course']
self.url_name = course_id_dict['name']
if id_reader is None:
id_reader = LocationReader()
id_generator = CourseLocationGenerator(self.org, self.course)
id_generator = CourseLocationGenerator(course_id)
# cdodge: adding the course_id as passed in for later reference rather than having to recomine the org/course/url_name
self.course_id = course_id
@@ -178,7 +174,7 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
self,
id_generator,
)
except Exception as err:
except Exception as err: # pylint: disable=broad-except
if not self.load_error_modules:
raise
@@ -224,9 +220,9 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
# TODO (vshnayder): we are somewhat architecturally confused in the loading code:
# load_item should actually be get_instance, because it expects the course-specific
# policy to be loaded. For now, just add the course_id here...
def load_item(location):
def load_item(usage_key):
"""Return the XBlock for the specified location"""
return xmlstore.get_instance(course_id, Location(location))
return xmlstore.get_item(usage_key)
resources_fs = OSFS(xmlstore.data_dir / course_dir)
@@ -236,7 +232,6 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
render_template=render_template,
error_tracker=error_tracker,
process_xml=process_xml,
id_reader=id_reader,
**kwargs
)
@@ -247,37 +242,53 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
block.children.append(child_block.scope_ids.usage_id)
class LocationReader(IdReader):
"""
IdReader for definition and usage ids that are Locations
"""
def get_definition_id(self, usage_id):
return usage_id
def get_block_type(self, def_id):
location = def_id
return location.category
class CourseLocationGenerator(IdGenerator):
"""
IdGenerator for Location-based definition ids and usage ids
based within a course
"""
def __init__(self, org, course):
self.org = org
self.course = course
def __init__(self, course_id):
self.course_id = course_id
self.autogen_ids = itertools.count(0)
def create_usage(self, def_id):
return Location(def_id)
return def_id
def create_definition(self, block_type, slug=None):
assert block_type is not None
if slug is None:
slug = 'autogen_{}_{}'.format(block_type, self.autogen_ids.next())
location = Location('i4x', self.org, self.course, block_type, slug)
return location
return self.course_id.make_usage_key(block_type, slug)
def _make_usage_key(course_key, value):
"""
Makes value into a UsageKey inside the specified course.
If value is already a UsageKey, returns that.
"""
if isinstance(value, UsageKey):
return value
return course_key.make_usage_key_from_deprecated_string(value)
def _convert_reference_fields_to_keys(xblock): # pylint: disable=invalid-name
"""
Find all fields of type reference and convert the payload into UsageKeys
"""
course_key = xblock.scope_ids.usage_id.course_key
for field in xblock.fields.itervalues():
if field.is_set_on(xblock):
field_value = getattr(xblock, field.name)
if isinstance(field, Reference):
setattr(xblock, field.name, _make_usage_key(course_key, field_value))
elif isinstance(field, ReferenceList):
setattr(xblock, field.name, [_make_usage_key(course_key, ele) for ele in field_value])
elif isinstance(field, ReferenceValueDict):
for key, subvalue in field_value.iteritems():
assert isinstance(subvalue, basestring)
field_value[key] = _make_usage_key(course_key, subvalue)
setattr(xblock, field.name, field_value)
def create_block_from_xml(xml_data, system, id_generator):
@@ -309,6 +320,9 @@ def create_block_from_xml(xml_data, system, id_generator):
scope_ids = ScopeIds(None, block_type, def_id, usage_id)
xblock = xblock_class.parse_xml(node, system, scope_ids, id_generator)
_convert_reference_fields_to_keys(xblock)
return xblock
@@ -327,8 +341,8 @@ class ParentTracker(object):
child and parent must be :class:`.Location` instances.
"""
s = self._parents.setdefault(child, set())
s.add(parent)
setp = self._parents.setdefault(child, set())
setp.add(parent)
def is_known(self, child):
"""
@@ -359,13 +373,14 @@ class XMLModuleStore(ModuleStoreReadBase):
"""
Initialize an XMLModuleStore from data_dir
data_dir: path to data directory containing the course directories
Args:
data_dir (str): path to data directory containing the course directories
default_class: dot-separated string defining the default descriptor
class to use if none is specified in entry_points
default_class (str): dot-separated string defining the default descriptor
class to use if none is specified in entry_points
course_dirs or course_ids: If specified, the list of course_dirs or course_ids to load. Otherwise,
load all courses. Note, providing both
course_dirs or course_ids (list of str): If specified, the list of course_dirs or course_ids to load. Otherwise,
load all courses. Note, providing both
"""
super(XMLModuleStore, self).__init__(**kwargs)
@@ -374,6 +389,9 @@ class XMLModuleStore(ModuleStoreReadBase):
self.courses = {} # course_dir -> XBlock for the course
self.errored_courses = {} # course_dir -> errorlog, for dirs that failed to load
if course_ids is not None:
course_ids = [SlashSeparatedCourseKey.from_deprecated_string(course_id) for course_id in course_ids]
self.load_error_modules = load_error_modules
if default_class is None:
@@ -415,9 +433,9 @@ class XMLModuleStore(ModuleStoreReadBase):
course_descriptor = None
try:
course_descriptor = self.load_course(course_dir, course_ids, errorlog.tracker)
except Exception as e:
except Exception as exc: # pylint: disable=broad-except
msg = "ERROR: Failed to load course '{0}': {1}".format(
course_dir.encode("utf-8"), unicode(e)
course_dir.encode("utf-8"), unicode(exc)
)
log.exception(msg)
errorlog.tracker(msg)
@@ -430,7 +448,7 @@ class XMLModuleStore(ModuleStoreReadBase):
self.errored_courses[course_dir] = errorlog
else:
self.courses[course_dir] = course_descriptor
self._location_errors[course_descriptor.scope_ids.usage_id] = errorlog
self._course_errors[course_descriptor.id] = errorlog
self.parent_trackers[course_descriptor.id].make_known(course_descriptor.scope_ids.usage_id)
def __unicode__(self):
@@ -521,7 +539,7 @@ class XMLModuleStore(ModuleStoreReadBase):
raise ValueError("Can't load a course without a 'url_name' "
"(or 'name') set. Set url_name.")
course_id = CourseDescriptor.make_id(org, course, url_name)
course_id = SlashSeparatedCourseKey(org, course, url_name)
if course_ids is not None and course_id not in course_ids:
return None
@@ -670,42 +688,20 @@ class XMLModuleStore(ModuleStoreReadBase):
module.save()
self.modules[course_descriptor.id][module.scope_ids.usage_id] = module
except Exception, e:
except Exception as exc: # pylint: disable=broad-except
logging.exception("Failed to load %s. Skipping... \
Exception: %s", filepath, unicode(e))
system.error_tracker("ERROR: " + unicode(e))
Exception: %s", filepath, unicode(exc))
system.error_tracker("ERROR: " + unicode(exc))
def get_instance(self, course_id, location, depth=0):
"""
Returns an XBlock instance for the item at
location, with the policy for course_id. (In case two xml
dirs have different content at the same location, return the
one for this course_id.)
If any segment of the location is None except revision, raises
xmodule.modulestore.exceptions.InsufficientSpecificationError
If no object is found at that location, raises
xmodule.modulestore.exceptions.ItemNotFoundError
location: Something that can be passed to Location
"""
location = Location(location)
try:
return self.modules[course_id][location]
except KeyError:
raise ItemNotFoundError(location)
def has_item(self, course_id, location):
def has_item(self, usage_key):
"""
Returns True if location exists in this ModuleStore.
"""
location = Location(location)
return location in self.modules[course_id]
return usage_key in self.modules[usage_key.course_key]
def get_item(self, location, depth=0):
def get_item(self, usage_key, depth=0):
"""
Returns an XBlock instance for the item at location.
Returns an XBlock instance for the item for this UsageKey.
If any segment of the location is None except revision, raises
xmodule.modulestore.exceptions.InsufficientSpecificationError
@@ -713,26 +709,56 @@ class XMLModuleStore(ModuleStoreReadBase):
If no object is found at that location, raises
xmodule.modulestore.exceptions.ItemNotFoundError
location: Something that can be passed to Location
usage_key: a UsageKey that matches the module we are looking for.
"""
raise NotImplementedError("XMLModuleStores can't guarantee that definitions"
" are unique. Use get_instance.")
try:
return self.modules[usage_key.course_key][usage_key]
except KeyError:
raise ItemNotFoundError(usage_key)
def get_items(self, location, course_id=None, depth=0, qualifiers=None):
def get_items(self, course_id, settings=None, content=None, **kwargs):
"""
Returns:
list of XModuleDescriptor instances for the matching items within the course with
the given course_id
NOTE: don't use this to look for courses
as the course_id is required. Use get_courses.
Args:
course_id (CourseKey): the course identifier
settings (dict): fields to look for which have settings scope. Follows same syntax
and rules as kwargs below
content (dict): fields to look for which have content scope. Follows same syntax and
rules as kwargs below.
kwargs (key=value): what to look for within the course.
Common qualifiers are ``category`` or any field name. if the target field is a list,
then it searches for the given value in the list not list equivalence.
Substring matching pass a regex object.
For this modulestore, ``name`` is another commonly provided key (Location based stores)
(but not revision!)
For this modulestore,
you can search dates by providing either a datetime for == (probably
useless) or a tuple (">"|"<" datetime) for after or before, etc.
"""
items = []
def _add_get_items(self, location, modules):
for mod_loc, module in modules.iteritems():
# Locations match if each value in `location` is None or if the value from `location`
# matches the value from `mod_loc`
if all(goal is None or goal == value for goal, value in zip(location, mod_loc)):
items.append(module)
category = kwargs.pop('category', None)
name = kwargs.pop('name', None)
if course_id is None:
for _, modules in self.modules.iteritems():
_add_get_items(self, location, modules)
else:
_add_get_items(self, location, self.modules[course_id])
def _block_matches_all(mod_loc, module):
if category and mod_loc.category != category:
return False
if name and mod_loc.name != name:
return False
return all(
self._block_matches(module, fields or {})
for fields in [settings, content, kwargs]
)
for mod_loc, module in self.modules[course_id].iteritems():
if _block_matches_all(mod_loc, module):
items.append(module)
return items
@@ -750,7 +776,7 @@ class XMLModuleStore(ModuleStoreReadBase):
"""
return dict((k, self.errored_courses[k].errors) for k in self.errored_courses)
def get_orphans(self, course_location, _branch):
def get_orphans(self, course_key):
"""
Get all of the xblocks in the given course which have no parents and are not of types which are
usually orphaned. NOTE: may include xblocks which still have references via xblocks which don't
@@ -759,28 +785,17 @@ class XMLModuleStore(ModuleStoreReadBase):
# here just to quell the abstractmethod. someone could write the impl if needed
raise NotImplementedError
def update_item(self, xblock, user, **kwargs):
"""
Set the data in the item specified by the location to
data
location: Something that can be passed to Location
data: A nested dictionary of problem data
"""
raise NotImplementedError("XMLModuleStores are read-only")
def get_parent_locations(self, location, course_id):
def get_parent_locations(self, location):
'''Find all locations that are the parents of this location in this
course. Needed for path_to_location().
returns an iterable of things that can be passed to Location. This may
be empty if there are no parents.
'''
location = Location.ensure_fully_specified(location)
if not self.parent_trackers[course_id].is_known(location):
raise ItemNotFoundError("{0} not in {1}".format(location, course_id))
if not self.parent_trackers[location.course_key].is_known(location):
raise ItemNotFoundError("{0} not in {1}".format(location, location.course_key))
return self.parent_trackers[course_id].parents(location)
return self.parent_trackers[location.course_key].parents(location)
def get_modulestore_type(self, course_id):
"""

39
common/lib/xmodule/xmodule/modulestore/xml_exporter.py
View File

@@ -32,7 +32,7 @@ class EdxJSONEncoder(json.JSONEncoder):
"""
def default(self, obj):
if isinstance(obj, Location):
return obj.url()
return obj.to_deprecated_string()
elif isinstance(obj, datetime.datetime):
if obj.tzinfo is not None:
if obj.utcoffset() is None:
@@ -45,24 +45,23 @@ class EdxJSONEncoder(json.JSONEncoder):
return super(EdxJSONEncoder, self).default(obj)
def export_to_xml(modulestore, contentstore, course_location, root_dir, course_dir, draft_modulestore=None):
def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir, draft_modulestore=None):
"""
Export all modules from `modulestore` and content from `contentstore` as xml to `root_dir`.
`modulestore`: A `ModuleStore` object that is the source of the modules to export
`contentstore`: A `ContentStore` object that is the source of the content to export, can be None
`course_location`: The `Location` of the `CourseModuleDescriptor` to export
`course_key`: The `CourseKey` of the `CourseModuleDescriptor` to export
`root_dir`: The directory to write the exported xml to
`course_dir`: The name of the directory inside `root_dir` to write the course content to
`draft_modulestore`: An optional `DraftModuleStore` that contains draft content, which will be exported
alongside the public content in the course.
"""
course_id = course_location.course_id
course = modulestore.get_course(course_id)
course = modulestore.get_course(course_key)
fs = OSFS(root_dir)
export_fs = course.runtime.export_fs = fs.makeopendir(course_dir)
fsm = OSFS(root_dir)
export_fs = course.runtime.export_fs = fsm.makeopendir(course_dir)
root = lxml.etree.Element('unknown')
course.add_xml_to_node(root)
@@ -74,22 +73,22 @@ def export_to_xml(modulestore, contentstore, course_location, root_dir, course_d
policies_dir = export_fs.makeopendir('policies')
if contentstore:
contentstore.export_all_for_course(
course_location,
course_key,
root_dir + '/' + course_dir + '/static/',
root_dir + '/' + course_dir + '/policies/assets.json',
)
# export the static tabs
export_extra_content(export_fs, modulestore, course_id, course_location, 'static_tab', 'tabs', '.html')
export_extra_content(export_fs, modulestore, course_key, 'static_tab', 'tabs', '.html')
# export the custom tags
export_extra_content(export_fs, modulestore, course_id, course_location, 'custom_tag_template', 'custom_tags')
export_extra_content(export_fs, modulestore, course_key, 'custom_tag_template', 'custom_tags')
# export the course updates
export_extra_content(export_fs, modulestore, course_id, course_location, 'course_info', 'info', '.html')
export_extra_content(export_fs, modulestore, course_key, 'course_info', 'info', '.html')
# export the 'about' data (e.g. overview, etc.)
export_extra_content(export_fs, modulestore, course_id, course_location, 'about', 'about', '.html')
export_extra_content(export_fs, modulestore, course_key, 'about', 'about', '.html')
# export the grading policy
course_run_policy_dir = policies_dir.makeopendir(course.location.name)
@@ -106,18 +105,17 @@ def export_to_xml(modulestore, contentstore, course_location, root_dir, course_d
# should we change the application, then this assumption will no longer
# be valid
if draft_modulestore is not None:
draft_verticals = draft_modulestore.get_items([None, course_location.org, course_location.course,
'vertical', None, 'draft'])
draft_verticals = draft_modulestore.get_items(course_key, category='vertical')
if len(draft_verticals) > 0:
draft_course_dir = export_fs.makeopendir(DRAFT_DIR)
for draft_vertical in draft_verticals:
parent_locs = draft_modulestore.get_parent_locations(draft_vertical.location, course.location.course_id)
parent_locs = draft_modulestore.get_parent_locations(draft_vertical.location)
# Don't try to export orphaned items.
if len(parent_locs) > 0:
logging.debug('parent_locs = {0}'.format(parent_locs))
draft_vertical.xml_attributes['parent_sequential_url'] = Location(parent_locs[0]).url()
sequential = modulestore.get_item(Location(parent_locs[0]))
index = sequential.children.index(draft_vertical.location.url())
draft_vertical.xml_attributes['parent_sequential_url'] = parent_locs[0].to_deprecated_string()
sequential = modulestore.get_item(parent_locs[0])
index = sequential.children.index(draft_vertical.location)
draft_vertical.xml_attributes['index_in_children_list'] = str(index)
draft_vertical.runtime.export_fs = draft_course_dir
node = lxml.etree.Element('unknown')
@@ -138,9 +136,8 @@ def _export_field_content(xblock_item, item_dir):
field_content_file.write(dumps(module_data.get(field_name, {}), cls=EdxJSONEncoder))
def export_extra_content(export_fs, modulestore, course_id, course_location, category_type, dirname, file_suffix=''):
query_loc = Location('i4x', course_location.org, course_location.course, category_type, None)
items = modulestore.get_items(query_loc, course_id)
def export_extra_content(export_fs, modulestore, course_key, category_type, dirname, file_suffix=''):
items = modulestore.get_items(course_key, category=category_type)
if len(items) > 0:
item_dir = export_fs.makeopendir(dirname)

405
common/lib/xmodule/xmodule/modulestore/xml_importer.py
View File

@@ -5,22 +5,23 @@ from path import path
import json
from .xml import XMLModuleStore, ImportSystem, ParentTracker
from xmodule.modulestore import Location
from xblock.runtime import KvsFieldData, DictKeyValueStore
from xmodule.x_module import XModuleDescriptor
from xmodule.modulestore.keys import UsageKey
from xblock.fields import Scope, Reference, ReferenceList, ReferenceValueDict
from xmodule.contentstore.content import StaticContent
from .inheritance import own_metadata
from xmodule.errortracker import make_error_tracker
from .store_utilities import rewrite_nonportable_content_links
import xblock
from xmodule.tabs import CourseTabList
log = logging.getLogger(__name__)
def import_static_content(
modules, course_loc, course_data_path, static_content_store,
target_location_namespace, subpath='static', verbose=False):
course_data_path, static_content_store,
target_course_id, subpath='static', verbose=False):
remap_dict = {}
@@ -65,12 +66,9 @@ def import_static_content(
fullname_with_subpath = content_path.replace(static_dir, '')
if fullname_with_subpath.startswith('/'):
fullname_with_subpath = fullname_with_subpath[1:]
content_loc = StaticContent.compute_location(
target_location_namespace.org, target_location_namespace.course,
fullname_with_subpath
)
asset_key = StaticContent.compute_location(target_course_id, fullname_with_subpath)
policy_ele = policy.get(content_loc.name, {})
policy_ele = policy.get(asset_key.path, {})
displayname = policy_ele.get('displayname', filename)
locked = policy_ele.get('locked', False)
mime_type = policy_ele.get('contentType')
@@ -79,7 +77,7 @@ def import_static_content(
if not mime_type or mime_type not in mimetypes_list:
mime_type = mimetypes.guess_type(filename)[0] # Assign guessed mimetype
content = StaticContent(
content_loc, displayname, mime_type, data,
asset_key, displayname, mime_type, data,
import_path=fullname_with_subpath, locked=locked
)
@@ -99,7 +97,7 @@ def import_static_content(
# store the remapping information which will be needed
# to subsitute in the module data
remap_dict[fullname_with_subpath] = content_loc.name
remap_dict[fullname_with_subpath] = asset_key
return remap_dict
@@ -108,8 +106,8 @@ def import_from_xml(
store, data_dir, course_dirs=None,
default_class='xmodule.raw_module.RawDescriptor',
load_error_modules=True, static_content_store=None,
target_location_namespace=None, verbose=False, draft_store=None,
do_import_static=True):
target_course_id=None, verbose=False, draft_store=None,
do_import_static=True, create_new_course=False):
"""
Import the specified xml data_dir into the "store" modulestore,
using org and course as the location org and course.
@@ -117,8 +115,7 @@ def import_from_xml(
course_dirs: If specified, the list of course_dirs to load. Otherwise, load
all course dirs
target_location_namespace is the namespace [passed as Location]
(i.e. {tag},{org},{course}) that all modules in the should be remapped to
target_course_id is the CourseKey that all modules should be remapped to
after import off disk. We do this remapping as a post-processing step
because there's logic in the importing which expects a 'url_name' as an
identifier to where things are on disk
@@ -132,6 +129,9 @@ def import_from_xml(
time the course is loaded. Static content for some courses may also be
served directly by nginx, instead of going through django.
: create_new_course:
If True, then courses whose ids already exist in the store are not imported.
The check for existing courses is case-insensitive.
"""
xml_module_store = XMLModuleStore(
@@ -143,6 +143,12 @@ def import_from_xml(
xblock_select=store.xblock_select,
)
# If we're going to remap the course_id, then we can only do that with
# a single course
if target_course_id:
assert(len(xml_module_store.modules) == 1)
# NOTE: the XmlModuleStore does not implement get_items()
# which would be a preferable means to enumerate the entire collection
# of course modules. It will be left as a TBD to implement that
@@ -150,21 +156,28 @@ def import_from_xml(
course_items = []
for course_id in xml_module_store.modules.keys():
if target_location_namespace is not None:
pseudo_course_id = u'{0.org}/{0.course}'.format(target_location_namespace)
if target_course_id is not None:
dest_course_id = target_course_id
else:
course_id_components = Location.parse_course_id(course_id)
pseudo_course_id = u'{org}/{course}'.format(**course_id_components)
dest_course_id = course_id
if create_new_course:
if store.has_course(dest_course_id, ignore_case=True):
log.debug(
"Skipping import of course with id, {0},"
"since it collides with an existing one".format(dest_course_id)
)
continue
else:
store.create_course(dest_course_id.org, dest_course_id.offering)
try:
# turn off all write signalling while importing as this
# is a high volume operation on stores that need it
if (hasattr(store, 'ignore_write_events_on_courses') and
pseudo_course_id not in store.ignore_write_events_on_courses):
store.ignore_write_events_on_courses.append(pseudo_course_id)
if hasattr(store, 'ignore_write_events_on_courses'):
store.ignore_write_events_on_courses.add(dest_course_id)
course_data_path = None
course_location = None
if verbose:
log.debug("Scanning {0} for course module...".format(course_id))
@@ -175,40 +188,11 @@ def import_from_xml(
for module in xml_module_store.modules[course_id].itervalues():
if module.scope_ids.block_type == 'course':
course_data_path = path(data_dir) / module.data_dir
course_location = module.location
course_prefix = u'{0.org}/{0.course}'.format(course_location)
# Check to see if a course with the same
# pseudo_course_id, but different run exists in
# the passed store to avoid broken courses
courses = store.get_courses()
bad_run = False
for course in courses:
if course.location.course_id.startswith(course_prefix):
log.debug('Import is overwriting existing course')
# Importing over existing course, check
# that runs match or fail
if course.location.name != module.location.name:
log.error(
'A course with ID %s exists, and this '
'course has the same organization and '
'course number, but a different term that '
'is fully identified as %s.',
course.location.course_id,
module.location.course_id
)
bad_run = True
break
if bad_run:
# Skip this course, but keep trying to import courses
continue
log.debug('======> IMPORTING course to location {loc}'.format(
loc=course_location
log.debug(u'======> IMPORTING course {course_id}'.format(
course_id=course_id,
))
module = remap_namespace(module, target_location_namespace)
if not do_import_static:
# for old-style xblock where this was actually linked to kvs
module.static_asset_path = module.data_dir
@@ -219,6 +203,35 @@ def import_from_xml(
log.debug('course data_dir={0}'.format(module.data_dir))
course = import_module(
module, store,
course_id,
dest_course_id,
do_import_static=do_import_static
)
for entry in course.pdf_textbooks:
for chapter in entry.get('chapters', []):
if StaticContent.is_c4x_path(chapter.get('url', '')):
asset_key = StaticContent.get_location_from_path(chapter['url'])
chapter['url'] = StaticContent.get_static_path_from_location(asset_key)
# Original wiki_slugs had value location.course. To make them unique this was changed to 'org.course.name'.
# If we are importing into a course with a different course_id and wiki_slug is equal to either of these default
# values then remap it so that the wiki does not point to the old wiki.
if course_id != course.id:
original_unique_wiki_slug = u'{0}.{1}.{2}'.format(
course_id.org,
course_id.course,
course_id.run
)
if course.wiki_slug == original_unique_wiki_slug or course.wiki_slug == course_id.course:
course.wiki_slug = u'{0}.{1}.{2}'.format(
course.id.org,
course.id.course,
course.id.run,
)
# cdodge: more hacks (what else). Seems like we have a
# problem when importing a course (like 6.002) which
# does not have any tabs defined in the policy file.
@@ -227,36 +240,19 @@ def import_from_xml(
# the LMS barfs because it expects that -- if there are
# *any* tabs -- then there at least needs to be
# some predefined ones
if module.tabs is None or len(module.tabs) == 0:
module.tabs = [
{"type": "courseware"},
{"type": "course_info", "name": "Course Info"},
{"type": "discussion", "name": "Discussion"},
{"type": "wiki", "name": "Wiki"},
# note, add 'progress' when we can support it on Edge
]
if course.tabs is None or len(course.tabs) == 0:
CourseTabList.initialize_default(course)
import_module(
module, store, course_data_path, static_content_store,
course_location,
target_location_namespace or course_location,
do_import_static=do_import_static
)
store.update_item(course)
course_items.append(module)
course_items.append(course)
# then import all the static content
if static_content_store is not None and do_import_static:
if target_location_namespace is not None:
_namespace_rename = target_location_namespace
else:
_namespace_rename = course_location
# first pass to find everything in /static/
import_static_content(
xml_module_store.modules[course_id], course_location,
course_data_path, static_content_store,
_namespace_rename, subpath='static', verbose=verbose
dest_course_id, subpath='static', verbose=verbose
)
elif verbose and not do_import_static:
@@ -277,15 +273,9 @@ def import_from_xml(
simport = 'static_import'
if os.path.exists(course_data_path / simport):
if target_location_namespace is not None:
_namespace_rename = target_location_namespace
else:
_namespace_rename = course_location
import_static_content(
xml_module_store.modules[course_id], course_location,
course_data_path, static_content_store,
_namespace_rename, subpath=simport, verbose=verbose
dest_course_id, subpath=simport, verbose=verbose
)
# finally loop through all the modules
@@ -295,20 +285,17 @@ def import_from_xml(
# of the loop so just skip over it in the inner loop
continue
# remap module to the new namespace
if target_location_namespace is not None:
module = remap_namespace(module, target_location_namespace)
if verbose:
log.debug('importing module location {loc}'.format(
loc=module.location
))
import_module(
module, store, course_data_path, static_content_store,
course_location,
target_location_namespace if target_location_namespace else course_location,
do_import_static=do_import_static
module, store,
course_id,
dest_course_id,
do_import_static=do_import_static,
system=course.runtime
)
# now import any 'draft' items
@@ -319,51 +306,93 @@ def import_from_xml(
draft_store,
course_data_path,
static_content_store,
course_location,
target_location_namespace if target_location_namespace else course_location
course_id,
dest_course_id,
course.runtime
)
finally:
# turn back on all write signalling on stores that need it
if (hasattr(store, 'ignore_write_events_on_courses') and
pseudo_course_id in store.ignore_write_events_on_courses):
store.ignore_write_events_on_courses.remove(pseudo_course_id)
store.refresh_cached_metadata_inheritance_tree(
target_location_namespace if target_location_namespace is not None else course_location
)
dest_course_id in store.ignore_write_events_on_courses):
store.ignore_write_events_on_courses.remove(dest_course_id)
store.refresh_cached_metadata_inheritance_tree(dest_course_id)
return xml_module_store, course_items
def import_module(
module, store, course_data_path, static_content_store,
source_course_location, dest_course_location, allow_not_found=False,
do_import_static=True):
module, store,
source_course_id, dest_course_id,
do_import_static=True, system=None):
logging.debug(u'processing import of module {}...'.format(module.location.url()))
logging.debug(u'processing import of module {}...'.format(module.location.to_deprecated_string()))
if do_import_static and 'data' in module.fields and isinstance(module.fields['data'], xblock.fields.String):
# we want to convert all 'non-portable' links in the module_data
# (if it is a string) to portable strings (e.g. /static/)
module.data = rewrite_nonportable_content_links(
source_course_location.course_id,
dest_course_location.course_id, module.data
source_course_id,
dest_course_id,
module.data
)
# remove any export/import only xml_attributes
# which are used to wire together draft imports
if 'parent_sequential_url' in getattr(module, 'xml_attributes', []):
del module.xml_attributes['parent_sequential_url']
if 'index_in_children_list' in getattr(module, 'xml_attributes', []):
del module.xml_attributes['index_in_children_list']
# Move the module to a new course
new_usage_key = module.scope_ids.usage_id.map_into_course(dest_course_id)
if new_usage_key.category == 'course':
new_usage_key = new_usage_key.replace(name=dest_course_id.run)
new_module = store.create_xmodule(new_usage_key, system=system)
store.update_item(module, '**replace_user**', allow_not_found=allow_not_found)
def _convert_reference_fields_to_new_namespace(reference):
"""
Convert a reference to the new namespace, but only
if the original namespace matched the original course.
Otherwise, returns the input value.
"""
assert isinstance(reference, UsageKey)
if source_course_id == reference.course_key:
return reference.map_into_course(dest_course_id)
else:
return reference
for field_name, field in module.fields.iteritems():
if field.is_set_on(module):
if isinstance(field, Reference):
new_ref = _convert_reference_fields_to_new_namespace(getattr(module, field_name))
setattr(new_module, field_name, new_ref)
elif isinstance(field, ReferenceList):
references = getattr(module, field_name)
new_references = [_convert_reference_fields_to_new_namespace(reference) for reference in references]
setattr(new_module, field_name, new_references)
elif isinstance(field, ReferenceValueDict):
reference_dict = getattr(module, field_name)
new_reference_dict = {
key: _convert_reference_fields_to_new_namespace(reference)
for key, reference
in reference_dict.items()
}
setattr(new_module, field_name, new_reference_dict)
elif field_name == 'xml_attributes':
value = getattr(module, field_name)
# remove any export/import only xml_attributes
# which are used to wire together draft imports
if 'parent_sequential_url' in value:
del value['parent_sequential_url']
if 'index_in_children_list' in value:
del value['index_in_children_list']
setattr(new_module, field_name, value)
else:
setattr(new_module, field_name, getattr(module, field_name))
store.update_item(new_module, '**replace_user**', allow_not_found=True)
return new_module
def import_course_draft(
xml_module_store, store, draft_store, course_data_path,
static_content_store, source_location_namespace,
target_location_namespace):
static_content_store, source_course_id,
target_course_id, mongo_runtime):
'''
This will import all the content inside of the 'drafts' folder, if it exists
NOTE: This is not a full course import, basically in our current
@@ -388,7 +417,7 @@ def import_course_draft(
draft_course_dir = draft_dir.replace(data_dir, '', 1)
system = ImportSystem(
xmlstore=xml_module_store,
course_id=target_location_namespace.course_id,
course_id=target_course_id,
course_dir=draft_course_dir,
error_tracker=errorlog.tracker,
parent_tracker=ParentTracker(),
@@ -458,14 +487,13 @@ def import_course_draft(
else:
drafts[index] = [descriptor]
except Exception, e:
logging.exception('There was an error. {err}'.format(
err=unicode(e)
))
except Exception:
logging.exception('Error while parsing course xml.')
# For each index_in_children_list key, there is a list of vertical descriptors.
for key in sorted(drafts.iterkeys()):
for descriptor in drafts[key]:
course_key = descriptor.location.course_key
try:
def _import_module(module):
# Update the module's location to "draft" revision
@@ -482,141 +510,29 @@ def import_course_draft(
sequential_url = module.xml_attributes['parent_sequential_url']
index = int(module.xml_attributes['index_in_children_list'])
seq_location = Location(sequential_url)
seq_location = course_key.make_usage_key_from_deprecated_string(sequential_url)
# IMPORTANT: Be sure to update the sequential
# in the NEW namespace
seq_location = seq_location.replace(
org=target_location_namespace.org,
course=target_location_namespace.course
)
seq_location = seq_location.map_into_course(target_course_id)
sequential = store.get_item(seq_location, depth=0)
if non_draft_location.url() not in sequential.children:
sequential.children.insert(index, non_draft_location.url())
if non_draft_location not in sequential.children:
sequential.children.insert(index, non_draft_location)
store.update_item(sequential, '**replace_user**')
import_module(
module, draft_store, course_data_path,
static_content_store, source_location_namespace,
target_location_namespace, allow_not_found=True
module, draft_store,
source_course_id,
target_course_id, system=mongo_runtime
)
for child in module.get_children():
_import_module(child)
_import_module(descriptor)
except Exception, e:
logging.exception('There was an error. {err}'.format(
err=unicode(e)
))
def remap_namespace(module, target_location_namespace):
if target_location_namespace is None:
return module
original_location = module.location
# This looks a bit wonky as we need to also change the 'name' of the
# imported course to be what the caller passed in
if module.location.category != 'course':
_update_module_location(
module,
module.location.replace(
tag=target_location_namespace.tag,
org=target_location_namespace.org,
course=target_location_namespace.course
)
)
else:
#
# module is a course module
#
module.location = module.location.replace(
tag=target_location_namespace.tag,
org=target_location_namespace.org,
course=target_location_namespace.course,
name=target_location_namespace.name
)
# There is more re-namespacing work we have to do when
# importing course modules
# remap pdf_textbook urls to portable static URLs
for entry in module.pdf_textbooks:
for chapter in entry.get('chapters', []):
if StaticContent.is_c4x_path(chapter.get('url', '')):
chapter_loc = StaticContent.get_location_from_path(chapter['url'])
chapter['url'] = StaticContent.get_static_path_from_location(
chapter_loc
)
# Original wiki_slugs had value location.course. To make them unique this was changed to 'org.course.name'.
# If we are importing into a course with a different course_id and wiki_slug is equal to either of these default
# values then remap it so that the wiki does not point to the old wiki.
if original_location.course_id != target_location_namespace.course_id:
original_unique_wiki_slug = u'{0}.{1}.{2}'.format(
original_location.org,
original_location.course,
original_location.name
)
if module.wiki_slug == original_unique_wiki_slug or module.wiki_slug == original_location.course:
module.wiki_slug = u'{0}.{1}.{2}'.format(
target_location_namespace.org,
target_location_namespace.course,
target_location_namespace.name,
)
module.save()
all_fields = module.get_explicitly_set_fields_by_scope(Scope.content)
all_fields.update(module.get_explicitly_set_fields_by_scope(Scope.settings))
if hasattr(module, 'children'):
all_fields['children'] = module.children
def convert_ref(reference):
"""
Convert a reference to the new namespace, but only
if the original namespace matched the original course.
Otherwise, returns the input value.
"""
new_ref = reference
ref = Location(reference)
in_original_namespace = (original_location.tag == ref.tag and
original_location.org == ref.org and
original_location.course == ref.course)
if in_original_namespace:
new_ref = ref.replace(
tag=target_location_namespace.tag,
org=target_location_namespace.org,
course=target_location_namespace.course
).url()
return new_ref
for field_name in all_fields:
field_object = module.fields.get(field_name)
if isinstance(field_object, Reference):
new_ref = convert_ref(getattr(module, field_name))
setattr(module, field_name, new_ref)
module.save()
elif isinstance(field_object, ReferenceList):
references = getattr(module, field_name)
new_references = [convert_ref(reference) for reference in references]
setattr(module, field_name, new_references)
module.save()
elif isinstance(field_object, ReferenceValueDict):
reference_dict = getattr(module, field_name)
new_reference_dict = {
key: convert_ref(reference)
for key, reference
in reference_dict.items()
}
setattr(module, field_name, new_reference_dict)
module.save()
return module
except Exception:
logging.exception('There while importing draft descriptor %s', descriptor)
def allowed_metadata_by_category(category):
@@ -648,7 +564,7 @@ def check_module_metadata_editability(module):
print(
": found non-editable metadata on {url}. "
"These metadata keys are not supported = {keys}".format(
url=module.location.url(), keys=illegal_keys
url=module.location.to_deprecated_string(), keys=illegal_keys
)
)
@@ -676,7 +592,7 @@ def validate_category_hierarchy(
parents.append(module)
for parent in parents:
for child_loc in [Location(child) for child in parent.children]:
for child_loc in parent.children:
if child_loc.category != expected_child_category:
err_cnt += 1
print(
@@ -767,7 +683,7 @@ def perform_xlint(
warn_cnt += _warn_cnt
# first count all errors and warnings as part of the XMLModuleStore import
for err_log in module_store._location_errors.itervalues():
for err_log in module_store._course_errors.itervalues():
for err_log_entry in err_log.errors:
msg = err_log_entry[0]
if msg.startswith('ERROR:'):
@@ -815,12 +731,7 @@ def perform_xlint(
)
# check for a presence of a course marketing video
location_elements = Location.parse_course_id(course_id)
location_elements['tag'] = 'i4x'
location_elements['category'] = 'about'
location_elements['name'] = 'video'
loc = Location(location_elements)
if loc not in module_store.modules[course_id]:
if not module_store.has_item(course_id.make_usage_key('about', 'video')):
print(
"WARN: Missing course marketing video. It is recommended "
"that every course have a marketing video."

4
common/lib/xmodule/xmodule/open_ended_grading_classes/combined_open_ended_modulev1.py
View File

@@ -412,7 +412,7 @@ class CombinedOpenEndedV1Module():
:param message: A message to put in the log.
:return: None
"""
info_message = "Combined open ended user state for user {0} in location {1} was invalid. It has been reset, and you now have a new attempt. {2}".format(self.system.anonymous_student_id, self.location.url(), message)
info_message = "Combined open ended user state for user {0} in location {1} was invalid. It has been reset, and you now have a new attempt. {2}".format(self.system.anonymous_student_id, self.location.to_deprecated_string(), message)
self.current_task_number = 0
self.student_attempts = 0
self.old_task_states.append(self.task_states)
@@ -800,7 +800,7 @@ class CombinedOpenEndedV1Module():
success = False
allowed_to_submit = True
try:
response = self.peer_gs.get_data_for_location(self.location.url(), student_id)
response = self.peer_gs.get_data_for_location(self.location.to_deprecated_string(), student_id)
count_graded = response['count_graded']
count_required = response['count_required']
student_sub_count = response['student_sub_count']

2
common/lib/xmodule/xmodule/open_ended_grading_classes/combined_open_ended_rubric.py
View File

@@ -96,7 +96,7 @@ class CombinedOpenEndedRubric(object):
if not success:
#This is a staff_facing_error
error_message = "Could not parse rubric : {0} for location {1}. Contact the learning sciences group for assistance.".format(
rubric_string, location.url())
rubric_string, location.to_deprecated_string())
log.error(error_message)
raise RubricParsingError(error_message)

4
common/lib/xmodule/xmodule/open_ended_grading_classes/open_ended_module.py
View File

@@ -105,7 +105,7 @@ class OpenEndedModule(openendedchild.OpenEndedChild):
# NOTE: self.system.location is valid because the capa_module
# __init__ adds it (easiest way to get problem location into
# response types)
except TypeError, ValueError:
except (TypeError, ValueError):
# This is a dev_facing_error
log.exception(
"Grader payload from external open ended grading server is not a json object! Object: {0}".format(
@@ -116,7 +116,7 @@ class OpenEndedModule(openendedchild.OpenEndedChild):
parsed_grader_payload.update({
'location': self.location_string,
'course_id': system.course_id,
'course_id': system.course_id.to_deprecated_string(),
'prompt': prompt_string,
'rubric': rubric_string,
'initial_display': self.initial_display,

2
common/lib/xmodule/xmodule/open_ended_grading_classes/openendedchild.py
View File

@@ -157,7 +157,7 @@ class OpenEndedChild(object):
self.location_string = location
try:
self.location_string = self.location_string.url()
self.location_string = self.location_string.to_deprecated_string()
except:
pass

5
common/lib/xmodule/xmodule/open_ended_grading_classes/peer_grading_service.py
View File

@@ -87,6 +87,11 @@ class PeerGradingService(GradingService):
def get_problem_list(self, course_id, grader_id):
params = {'course_id': course_id, 'student_id': grader_id}
result = self.get(self.get_problem_list_url, params)
if 'problem_list' in result:
for problem in result['problem_list']:
problem['location'] = course_id.make_usage_key_from_deprecated_string(problem['location'])
self._record_result('get_problem_list', result)
dog_stats_api.histogram(
self._metric_name('get_problem_list.result.length'),

10
common/lib/xmodule/xmodule/peer_grading_module.py
View File

@@ -8,7 +8,6 @@ from xblock.fields import Dict, String, Scope, Boolean, Float, Reference
from xmodule.capa_module import ComplexEncoder
from xmodule.fields import Date, Timedelta
from xmodule.modulestore import Location
from xmodule.modulestore.exceptions import ItemNotFoundError, NoPathToItem
from xmodule.raw_module import RawDescriptor
from xmodule.timeinfo import TimeInfo
@@ -261,7 +260,7 @@ class PeerGradingModule(PeerGradingFields, XModule):
if not success:
log.exception(
"No instance data found and could not get data from controller for loc {0} student {1}".format(
self.system.location.url(), self.system.anonymous_student_id
self.system.location.to_deprecated_string(), self.system.anonymous_student_id
))
return None
count_graded = response['count_graded']
@@ -563,7 +562,7 @@ class PeerGradingModule(PeerGradingFields, XModule):
good_problem_list = []
for problem in problem_list:
problem_location = Location(problem['location'])
problem_location = problem['location']
try:
descriptor = self._find_corresponding_module_for_location(problem_location)
except (NoPathToItem, ItemNotFoundError):
@@ -588,7 +587,6 @@ class PeerGradingModule(PeerGradingFields, XModule):
ajax_url = self.ajax_url
html = self.system.render_template('peer_grading/peer_grading.html', {
'course_id': self.course_id,
'ajax_url': ajax_url,
'success': success,
'problem_list': good_problem_list,
@@ -611,10 +609,10 @@ class PeerGradingModule(PeerGradingFields, XModule):
log.error(
"Peer grading problem in peer_grading_module called with no get parameters, but use_for_single_location is False.")
return {'html': "", 'success': False}
problem_location = Location(self.link_to_location)
problem_location = self.link_to_location
elif data.get('location') is not None:
problem_location = Location(data.get('location'))
problem_location = self.course_id.make_usage_key_from_deprecated_string(data.get('location'))
module = self._find_corresponding_module_for_location(problem_location)

4
common/lib/xmodule/xmodule/seq_module.py
View File

@@ -96,7 +96,7 @@ class SequenceModule(SequenceFields, XModule):
'progress_status': Progress.to_js_status_str(progress),
'progress_detail': Progress.to_js_detail_str(progress),
'type': child.get_icon_class(),
'id': child.id,
'id': child.scope_ids.usage_id.to_deprecated_string(),
}
if childinfo['title'] == '':
childinfo['title'] = child.display_name_with_default
@@ -104,7 +104,7 @@ class SequenceModule(SequenceFields, XModule):
params = {'items': contents,
'element_id': self.location.html_id(),
'item_id': self.id,
'item_id': self.location.to_deprecated_string(),
'position': self.position,
'tag': self.location.category,
'ajax_url': self.system.ajax_url,

10
common/lib/xmodule/xmodule/split_test_module.py
View File

@@ -82,7 +82,7 @@ class SplitTestModule(SplitTestFields, XModule):
# we've picked a choice. Use self.descriptor.get_children() instead.
for child in self.descriptor.get_children():
if child.location.url() == location:
if child.location == location:
return child
return None
@@ -182,7 +182,7 @@ class SplitTestModule(SplitTestFields, XModule):
fragment.add_frag_resources(rendered_child)
contents.append({
'id': child.id,
'id': child.location.to_deprecated_string(),
'content': rendered_child.content
})
@@ -252,7 +252,11 @@ class SplitTestDescriptor(SplitTestFields, SequenceDescriptor):
def definition_to_xml(self, resource_fs):
xml_object = etree.Element('split_test')
xml_object.set('group_id_to_child', json.dumps(self.group_id_to_child))
renderable_groups = {}
# json.dumps doesn't know how to handle Location objects
for group in self.group_id_to_child:
renderable_groups[group] = self.group_id_to_child[group].to_deprecated_string()
xml_object.set('group_id_to_child', json.dumps(renderable_groups))
xml_object.set('user_partition_id', str(self.user_partition_id))
for child in self.get_children():
self.runtime.add_block_as_child_node(child, xml_object)

10
common/lib/xmodule/xmodule/tabs.py
View File

@@ -456,7 +456,7 @@ class StaticTab(CourseTab):
super(StaticTab, self).__init__(
name=tab_dict['name'] if tab_dict else name,
tab_id='static_tab_{0}'.format(self.url_slug),
link_func=lambda course, reverse_func: reverse_func(self.type, args=[course.id, self.url_slug]),
link_func=lambda course, reverse_func: reverse_func(self.type, args=[course.id.to_deprecated_string(), self.url_slug]),
)
def __getitem__(self, key):
@@ -537,7 +537,7 @@ class TextbookTabs(TextbookTabsBase):
yield SingleTextbookTab(
name=textbook.title,
tab_id='textbook/{0}'.format(index),
link_func=lambda course, reverse_func: reverse_func('book', args=[course.id, index]),
link_func=lambda course, reverse_func: reverse_func('book', args=[course.id.to_deprecated_string(), index]),
)
@@ -557,7 +557,7 @@ class PDFTextbookTabs(TextbookTabsBase):
yield SingleTextbookTab(
name=textbook['tab_title'],
tab_id='pdftextbook/{0}'.format(index),
link_func=lambda course, reverse_func: reverse_func('pdf_book', args=[course.id, index]),
link_func=lambda course, reverse_func: reverse_func('pdf_book', args=[course.id.to_deprecated_string(), index]),
)
@@ -577,7 +577,7 @@ class HtmlTextbookTabs(TextbookTabsBase):
yield SingleTextbookTab(
name=textbook['tab_title'],
tab_id='htmltextbook/{0}'.format(index),
link_func=lambda course, reverse_func: reverse_func('html_book', args=[course.id, index]),
link_func=lambda course, reverse_func: reverse_func('html_book', args=[course.id.to_deprecated_string(), index]),
)
@@ -884,7 +884,7 @@ def link_reverse_func(reverse_name):
Returns a function that takes in a course and reverse_url_func,
and calls the reverse_url_func with the given reverse_name and course' ID.
"""
return lambda course, reverse_url_func: reverse_url_func(reverse_name, args=[course.id])
return lambda course, reverse_url_func: reverse_url_func(reverse_name, args=[course.id.to_deprecated_string()])
def link_value_func(value):

25
common/lib/xmodule/xmodule/tests/__init__.py
View File

@@ -16,12 +16,13 @@ from mock import Mock
from path import path
from xblock.field_data import DictFieldData
from xblock.fields import ScopeIds
from xmodule.x_module import ModuleSystem, XModuleDescriptor, XModuleMixin
from xmodule.modulestore.inheritance import InheritanceMixin
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from xmodule.mako_module import MakoDescriptorSystem
from xmodule.error_module import ErrorDescriptor
from xmodule.modulestore.xml import LocationReader
MODULE_DIR = path(__file__).dirname()
@@ -45,13 +46,21 @@ class TestModuleSystem(ModuleSystem): # pylint: disable=abstract-method
ModuleSystem for testing
"""
def handler_url(self, block, handler, suffix='', query='', thirdparty=False):
return str(block.scope_ids.usage_id) + '/' + handler + '/' + suffix + '?' + query
return '{usage_id}/{handler}{suffix}?{query}'.format(
usage_id=block.scope_ids.usage_id.to_deprecated_string(),
handler=handler,
suffix=suffix,
query=query,
)
def local_resource_url(self, block, uri):
return 'resource/' + str(block.scope_ids.block_type) + '/' + uri
return 'resource/{usage_id}/{uri}'.format(
usage_id=block.scope_ids.usage_id.to_deprecated_string(),
uri=uri,
)
def get_test_system(course_id=''):
def get_test_system(course_id=SlashSeparatedCourseKey('org', 'course', 'run')):
"""
Construct a test ModuleSystem instance.
@@ -96,7 +105,6 @@ def get_test_descriptor_system():
render_template=mock_render_template,
mixins=(InheritanceMixin, XModuleMixin),
field_data=DictFieldData({}),
id_reader=LocationReader(),
)
@@ -131,12 +139,15 @@ class LogicTest(unittest.TestCase):
url_name = ''
category = 'test'
self.system = get_test_system(course_id='test/course/id')
self.system = get_test_system()
self.descriptor = EmptyClass()
self.xmodule_class = self.descriptor_class.module_class
usage_key = self.system.course_id.make_usage_key(self.descriptor.category, 'test_loc')
# ScopeIds has 4 fields: user_id, block_type, def_id, usage_id
scope_ids = ScopeIds(1, self.descriptor.category, usage_key, usage_key)
self.xmodule = self.xmodule_class(
self.descriptor, self.system, DictFieldData(self.raw_field_data), Mock()
self.descriptor, self.system, DictFieldData(self.raw_field_data), scope_ids
)
def ajax_request(self, dispatch, data):

2
common/lib/xmodule/xmodule/tests/test_annotatable_module.py
View File

@@ -35,7 +35,7 @@ class AnnotatableModuleTestCase(unittest.TestCase):
Mock(),
get_test_system(),
DictFieldData({'data': self.sample_xml}),
ScopeIds(None, None, None, None)
ScopeIds(None, None, None, Location('org', 'course', 'run', 'category', 'name', None))
)
def test_annotation_data_attr(self):

10
common/lib/xmodule/xmodule/tests/test_capa_module.py
View File

@@ -101,8 +101,14 @@ class CapaFactory(object):
attempts: also added to instance state. Will be converted to an int.
"""
location = Location(["i4x", "edX", "capa_test", "problem",
"SampleProblem{0}".format(cls.next_num())])
location = Location(
"edX",
"capa_test",
"2012_Fall",
"problem",
"SampleProblem{0}".format(cls.next_num()),
None
)
if xml is None:
xml = cls.sample_problem_xml
field_data = {'data': xml}

52
common/lib/xmodule/xmodule/tests/test_combined_open_ended.py
View File

@@ -12,7 +12,7 @@ import unittest
from datetime import datetime
from lxml import etree
from mock import Mock, MagicMock, ANY, patch
from mock import Mock, MagicMock, patch
from pytz import UTC
from webob.multidict import MultiDict
@@ -20,7 +20,6 @@ from xmodule.open_ended_grading_classes.openendedchild import OpenEndedChild
from xmodule.open_ended_grading_classes.open_ended_module import OpenEndedModule
from xmodule.open_ended_grading_classes.self_assessment_module import SelfAssessmentModule
from xmodule.open_ended_grading_classes.combined_open_ended_modulev1 import CombinedOpenEndedV1Module
from xmodule.open_ended_grading_classes.grading_service_module import GradingServiceError
from xmodule.combined_open_ended_module import CombinedOpenEndedModule
from xmodule.modulestore import Location
from xmodule.tests import get_test_system, test_util_open_ended
@@ -48,8 +47,7 @@ class OpenEndedChildTest(unittest.TestCase):
"""
Test the open ended child class
"""
location = Location(["i4x", "edX", "sa_test", "selfassessment",
"SampleQuestion"])
location = Location("edX", "sa_test", "2012_Fall", "selfassessment", "SampleQuestion")
metadata = json.dumps({'attempts': '10'})
prompt = etree.XML("<prompt>This is a question prompt</prompt>")
@@ -173,8 +171,7 @@ class OpenEndedModuleTest(unittest.TestCase):
"""
Test the open ended module class
"""
location = Location(["i4x", "edX", "sa_test", "selfassessment",
"SampleQuestion"])
location = Location("edX", "sa_test", "2012_Fall", "selfassessment", "SampleQuestion")
metadata = json.dumps({'attempts': '10'})
prompt = etree.XML("<prompt>This is a question prompt</prompt>")
@@ -446,8 +443,7 @@ class CombinedOpenEndedModuleTest(unittest.TestCase):
"""
Unit tests for the combined open ended xmodule
"""
location = Location(["i4x", "edX", "open_ended", "combinedopenended",
"SampleQuestion"])
location = Location("edX", "open_ended", "2012_Fall", "combinedopenended", "SampleQuestion")
definition_template = """
<combinedopenended attempts="10000">
{rubric}
@@ -517,6 +513,9 @@ class CombinedOpenEndedModuleTest(unittest.TestCase):
descriptor = Mock(data=full_definition)
test_system = get_test_system()
test_system.open_ended_grading_interface = None
usage_key = test_system.course_id.make_usage_key('combinedopenended', 'test_loc')
# ScopeIds has 4 fields: user_id, block_type, def_id, usage_id
scope_ids = ScopeIds(1, 'combinedopenended', usage_key, usage_key)
combinedoe_container = CombinedOpenEndedModule(
descriptor=descriptor,
runtime=test_system,
@@ -524,7 +523,7 @@ class CombinedOpenEndedModuleTest(unittest.TestCase):
'data': full_definition,
'weight': '1',
}),
scope_ids=ScopeIds(None, None, None, None),
scope_ids=scope_ids,
)
def setUp(self):
@@ -799,8 +798,7 @@ class CombinedOpenEndedModuleConsistencyTest(unittest.TestCase):
# location, definition_template, prompt, rubric, max_score, metadata, oeparam, task_xml1, task_xml2
# All these variables are used to construct the xmodule descriptor.
location = Location(["i4x", "edX", "open_ended", "combinedopenended",
"SampleQuestion"])
location = Location("edX", "open_ended", "2012_Fall", "combinedopenended", "SampleQuestion")
definition_template = """
<combinedopenended attempts="10000">
{rubric}
@@ -871,6 +869,9 @@ class CombinedOpenEndedModuleConsistencyTest(unittest.TestCase):
descriptor = Mock(data=full_definition)
test_system = get_test_system()
test_system.open_ended_grading_interface = None
usage_key = test_system.course_id.make_usage_key('combinedopenended', 'test_loc')
# ScopeIds has 4 fields: user_id, block_type, def_id, usage_id
scope_ids = ScopeIds(1, 'combinedopenended', usage_key, usage_key)
combinedoe_container = CombinedOpenEndedModule(
descriptor=descriptor,
runtime=test_system,
@@ -878,7 +879,7 @@ class CombinedOpenEndedModuleConsistencyTest(unittest.TestCase):
'data': full_definition,
'weight': '1',
}),
scope_ids=ScopeIds(None, None, None, None),
scope_ids=scope_ids,
)
def setUp(self):
@@ -964,7 +965,7 @@ class OpenEndedModuleXmlTest(unittest.TestCase, DummyModulestore):
"""
Test the student flow in the combined open ended xmodule
"""
problem_location = Location(["i4x", "edX", "open_ended", "combinedopenended", "SampleQuestion"])
problem_location = Location("edX", "open_ended", "2012_Fall", "combinedopenended", "SampleQuestion")
answer = "blah blah"
assessment = [0, 1]
hint = "blah"
@@ -999,7 +1000,7 @@ class OpenEndedModuleXmlTest(unittest.TestCase, DummyModulestore):
return result
def _module(self):
return self.get_module_from_location(self.problem_location, COURSE)
return self.get_module_from_location(self.problem_location)
def test_open_ended_load_and_save(self):
"""
@@ -1212,7 +1213,7 @@ class OpenEndedModuleXmlAttemptTest(unittest.TestCase, DummyModulestore):
"""
Test if student is able to reset the problem
"""
problem_location = Location(["i4x", "edX", "open_ended", "combinedopenended", "SampleQuestion1Attempt"])
problem_location = Location("edX", "open_ended", "2012_Fall", "combinedopenended", "SampleQuestion1Attempt")
answer = "blah blah"
assessment = [0, 1]
hint = "blah"
@@ -1241,7 +1242,7 @@ class OpenEndedModuleXmlAttemptTest(unittest.TestCase, DummyModulestore):
return result
def _module(self):
return self.get_module_from_location(self.problem_location, COURSE)
return self.get_module_from_location(self.problem_location)
def test_reset_fail(self):
"""
@@ -1283,12 +1284,13 @@ class OpenEndedModuleXmlImageUploadTest(unittest.TestCase, DummyModulestore):
"""
Test if student is able to upload images properly.
"""
problem_location = Location(["i4x", "edX", "open_ended", "combinedopenended", "SampleQuestionImageUpload"])
problem_location = Location("edX", "open_ended", "2012_Fall", "combinedopenended", "SampleQuestionImageUpload")
answer_text = "Hello, this is my amazing answer."
file_text = "Hello, this is my amazing file."
file_name = "Student file 1"
answer_link = "http://www.edx.org"
autolink_tag = '<a target="_blank" href='
autolink_tag_swapped = '<a href='
def get_module_system(self, descriptor):
test_system = get_test_system()
@@ -1306,7 +1308,7 @@ class OpenEndedModuleXmlImageUploadTest(unittest.TestCase, DummyModulestore):
"""
Test to see if a student submission without a file attached fails.
"""
module = self.get_module_from_location(self.problem_location, COURSE)
module = self.get_module_from_location(self.problem_location)
# Simulate a student saving an answer
response = module.handle_ajax("save_answer", {"student_answer": self.answer_text})
@@ -1326,7 +1328,7 @@ class OpenEndedModuleXmlImageUploadTest(unittest.TestCase, DummyModulestore):
"""
Test to see if a student submission with a file is handled properly.
"""
module = self.get_module_from_location(self.problem_location, COURSE)
module = self.get_module_from_location(self.problem_location)
# Simulate a student saving an answer with a file
response = module.handle_ajax("save_answer", {
@@ -1338,13 +1340,14 @@ class OpenEndedModuleXmlImageUploadTest(unittest.TestCase, DummyModulestore):
response = json.loads(response)
self.assertTrue(response['success'])
self.assertIn(self.file_name, response['student_response'])
self.assertIn(self.autolink_tag, response['student_response'])
self.assertTrue(self.autolink_tag in response['student_response'] or
self.autolink_tag_swapped in response['student_response'])
def test_link_submission_success(self):
"""
Students can submit links instead of files. Check that the link is properly handled.
"""
module = self.get_module_from_location(self.problem_location, COURSE)
module = self.get_module_from_location(self.problem_location)
# Simulate a student saving an answer with a link.
response = module.handle_ajax("save_answer", {
@@ -1355,7 +1358,8 @@ class OpenEndedModuleXmlImageUploadTest(unittest.TestCase, DummyModulestore):
self.assertTrue(response['success'])
self.assertIn(self.answer_link, response['student_response'])
self.assertIn(self.autolink_tag, response['student_response'])
self.assertTrue(self.autolink_tag in response['student_response'] or
self.autolink_tag_swapped in response['student_response'])
class OpenEndedModuleUtilTest(unittest.TestCase):
@@ -1369,7 +1373,7 @@ class OpenEndedModuleUtilTest(unittest.TestCase):
embed_dirty = u'<embed height="200" id="cats" onhover="eval()" src="http://example.com/lolcats.swf" width="200"/>'
embed_clean = u'<embed width="200" height="200" id="cats" src="http://example.com/lolcats.swf">'
iframe_dirty = u'<iframe class="cats" height="200" onerror="eval()" src="http://example.com/lolcats" width="200"/>'
iframe_clean = u'<iframe height="200" class="cats" width="200" src="http://example.com/lolcats"></iframe>'
iframe_clean = ur'<iframe (height="200" ?|class="cats" ?|width="200" ?|src="http://example.com/lolcats" ?)+></iframe>'
text = u'I am a \u201c\xfcber student\u201d'
text_lessthan_noencd = u'This used to be broken < by the other parser. 3>5'
@@ -1402,7 +1406,7 @@ class OpenEndedModuleUtilTest(unittest.TestCase):
"""
Basic test for passing through iframe, but stripping bad attr
"""
self.assertEqual(OpenEndedChild.sanitize_html(self.iframe_dirty), self.iframe_clean)
self.assertRegexpMatches(OpenEndedChild.sanitize_html(self.iframe_dirty), self.iframe_clean)
def test_text(self):
"""

27
common/lib/xmodule/xmodule/tests/test_conditional.py
View File

@@ -1,4 +1,3 @@
from ast import literal_eval
import json
import unittest
@@ -8,7 +7,7 @@ from mock import Mock, patch
from xblock.field_data import DictFieldData
from xblock.fields import ScopeIds
from xmodule.error_module import NonStaffErrorDescriptor
from xmodule.modulestore import Location
from xmodule.modulestore.locations import SlashSeparatedCourseKey, Location
from xmodule.modulestore.xml import ImportSystem, XMLModuleStore, CourseLocationGenerator
from xmodule.conditional_module import ConditionalDescriptor
from xmodule.tests import DATA_DIR, get_test_system, get_test_descriptor_system
@@ -54,13 +53,13 @@ class ConditionalFactory(object):
descriptor_system = get_test_descriptor_system()
# construct source descriptor and module:
source_location = Location(["i4x", "edX", "conditional_test", "problem", "SampleProblem"])
source_location = Location("edX", "conditional_test", "test_run", "problem", "SampleProblem", None)
if source_is_error_module:
# Make an error descriptor and module
source_descriptor = NonStaffErrorDescriptor.from_xml(
'some random xml data',
system,
id_generator=CourseLocationGenerator(source_location.org, source_location.course),
id_generator=CourseLocationGenerator(SlashSeparatedCourseKey('edX', 'conditional_test', 'test_run')),
error_msg='random error message'
)
else:
@@ -78,15 +77,19 @@ class ConditionalFactory(object):
child_descriptor.runtime = descriptor_system
child_descriptor.xmodule_runtime = get_test_system()
child_descriptor.render = lambda view, context=None: descriptor_system.render(child_descriptor, view, context)
child_descriptor.location = source_location.replace(category='html', name='child')
descriptor_system.load_item = {'child': child_descriptor, 'source': source_descriptor}.get
descriptor_system.load_item = {
child_descriptor.location: child_descriptor,
source_location: source_descriptor
}.get
# construct conditional module:
cond_location = Location(["i4x", "edX", "conditional_test", "conditional", "SampleConditional"])
cond_location = Location("edX", "conditional_test", "test_run", "conditional", "SampleConditional", None)
field_data = DictFieldData({
'data': '<conditional/>',
'xml_attributes': {'attempted': 'true'},
'children': ['child'],
'children': [child_descriptor.location],
})
cond_descriptor = ConditionalDescriptor(
@@ -130,7 +133,6 @@ class ConditionalModuleBasicTest(unittest.TestCase):
expected = modules['cond_module'].xmodule_runtime.render_template('conditional_ajax.html', {
'ajax_url': modules['cond_module'].xmodule_runtime.ajax_url,
'element_id': u'i4x-edX-conditional_test-conditional-SampleConditional',
'id': u'i4x://edX/conditional_test/conditional/SampleConditional',
'depends': u'i4x-edX-conditional_test-problem-SampleProblem',
})
self.assertEquals(expected, html)
@@ -198,14 +200,14 @@ class ConditionalModuleXmlTest(unittest.TestCase):
def inner_get_module(descriptor):
if isinstance(descriptor, Location):
location = descriptor
descriptor = self.modulestore.get_instance(course.id, location, depth=None)
descriptor = self.modulestore.get_item(location, depth=None)
descriptor.xmodule_runtime = get_test_system()
descriptor.xmodule_runtime.get_module = inner_get_module
return descriptor
# edx - HarvardX
# cond_test - ER22x
location = Location(["i4x", "HarvardX", "ER22x", "conditional", "condone"])
location = Location("HarvardX", "ER22x", "2013_Spring", "conditional", "condone")
def replace_urls(text, staticfiles_prefix=None, replace_prefix='/static/', course_namespace=None):
return text
@@ -224,9 +226,8 @@ class ConditionalModuleXmlTest(unittest.TestCase):
'conditional_ajax.html',
{
# Test ajax url is just usage-id / handler_name
'ajax_url': 'i4x://HarvardX/ER22x/conditional/condone/xmodule_handler',
'ajax_url': '{}/xmodule_handler'.format(location.to_deprecated_string()),
'element_id': u'i4x-HarvardX-ER22x-conditional-condone',
'id': u'i4x://HarvardX/ER22x/conditional/condone',
'depends': u'i4x-HarvardX-ER22x-problem-choiceprob'
}
)
@@ -242,7 +243,7 @@ class ConditionalModuleXmlTest(unittest.TestCase):
self.assertFalse(any(['This is a secret' in item for item in html]))
# Now change state of the capa problem to make it completed
inner_module = inner_get_module(Location('i4x://HarvardX/ER22x/problem/choiceprob'))
inner_module = inner_get_module(location.replace(category="problem", name='choiceprob'))
inner_module.attempts = 1
# Save our modifications to the underlying KeyValueStore so they can be persisted
inner_module.save()

22
common/lib/xmodule/xmodule/tests/test_content.py
View File

@@ -1,7 +1,7 @@
import unittest
from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.content import ContentStore
from xmodule.modulestore import Location
from xmodule.modulestore.locations import SlashSeparatedCourseKey, AssetLocation
class Content:
@@ -21,18 +21,28 @@ class ContentTest(unittest.TestCase):
self.assertIsNone(content.thumbnail_location)
def test_static_url_generation_from_courseid(self):
url = StaticContent.convert_legacy_static_url_with_course_id('images_course_image.jpg', 'foo/bar/bz')
course_key = SlashSeparatedCourseKey('foo', 'bar', 'bz')
url = StaticContent.convert_legacy_static_url_with_course_id('images_course_image.jpg', course_key)
self.assertEqual(url, '/c4x/foo/bar/asset/images_course_image.jpg')
def test_generate_thumbnail_image(self):
contentStore = ContentStore()
content = Content(Location(u'c4x', u'mitX', u'800', u'asset', u'monsters__.jpg'), None)
content = Content(AssetLocation(u'mitX', u'800', u'ignore_run', u'asset', u'monsters__.jpg'), None)
(thumbnail_content, thumbnail_file_location) = contentStore.generate_thumbnail(content)
self.assertIsNone(thumbnail_content)
self.assertEqual(Location(u'c4x', u'mitX', u'800', u'thumbnail', u'monsters__.jpg'), thumbnail_file_location)
self.assertEqual(AssetLocation(u'mitX', u'800', u'ignore_run', u'thumbnail', u'monsters__.jpg'), thumbnail_file_location)
def test_compute_location(self):
# We had a bug that __ got converted into a single _. Make sure that substitution of INVALID_CHARS (like space)
# still happen.
asset_location = StaticContent.compute_location('mitX', '400', 'subs__1eo_jXvZnE .srt.sjson')
self.assertEqual(Location(u'c4x', u'mitX', u'400', u'asset', u'subs__1eo_jXvZnE_.srt.sjson', None), asset_location)
asset_location = StaticContent.compute_location(
SlashSeparatedCourseKey('mitX', '400', 'ignore'), 'subs__1eo_jXvZnE .srt.sjson'
)
self.assertEqual(AssetLocation(u'mitX', u'400', u'ignore', u'asset', u'subs__1eo_jXvZnE_.srt.sjson', None), asset_location)
def test_get_location_from_path(self):
asset_location = StaticContent.get_location_from_path(u'/c4x/foo/bar/asset/images_course_image.jpg')
self.assertEqual(
AssetLocation(u'foo', u'bar', None, u'asset', u'images_course_image.jpg', None),
asset_location
)

6
common/lib/xmodule/xmodule/tests/test_course_module.py
View File

@@ -8,7 +8,8 @@ from mock import Mock, patch
from xblock.runtime import KvsFieldData, DictKeyValueStore
import xmodule.course_module
from xmodule.modulestore.xml import ImportSystem, XMLModuleStore, LocationReader
from xmodule.modulestore.xml import ImportSystem, XMLModuleStore
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from django.utils.timezone import UTC
@@ -32,7 +33,7 @@ class DummySystem(ImportSystem):
xmlstore = XMLModuleStore("data_dir", course_dirs=[],
load_error_modules=load_error_modules)
course_id = "/".join([ORG, COURSE, 'test_run'])
course_id = SlashSeparatedCourseKey(ORG, COURSE, 'test_run')
course_dir = "test_dir"
error_tracker = Mock()
parent_tracker = Mock()
@@ -45,7 +46,6 @@ class DummySystem(ImportSystem):
parent_tracker=parent_tracker,
load_error_modules=load_error_modules,
field_data=KvsFieldData(DictKeyValueStore()),
id_reader=LocationReader(),
)

3
common/lib/xmodule/xmodule/tests/test_delay_between_attempts.py
View File

@@ -84,8 +84,7 @@ class CapaFactoryWithDelay(object):
"""
Optional parameters here are cut down to what we actually use vs. the regular CapaFactory.
"""
location = Location(["i4x", "edX", "capa_test", "problem",
"SampleProblem{0}".format(cls.next_num())])
location = Location("edX", "capa_test", "run", "problem", "SampleProblem{0}".format(cls.next_num()))
field_data = {'data': cls.sample_problem_xml}
if max_attempts is not None:

3
common/lib/xmodule/xmodule/tests/test_editing_module.py
View File

@@ -5,6 +5,7 @@ import logging
from mock import Mock
from pkg_resources import resource_string
from xmodule.modulestore.locations import Location
from xmodule.editing_module import TabsEditingDescriptor
from xblock.field_data import DictFieldData
from xblock.fields import ScopeIds
@@ -46,7 +47,7 @@ class TabsEditingDescriptorTestCase(unittest.TestCase):
TabsEditingDescriptor.tabs = self.tabs
self.descriptor = system.construct_xblock_from_class(
TabsEditingDescriptor,
scope_ids=ScopeIds(None, None, None, None),
scope_ids=ScopeIds(None, None, None, Location('org', 'course', 'run', 'category', 'name', 'revision')),
field_data=DictFieldData({}),
)

15
common/lib/xmodule/xmodule/tests/test_error_module.py
View File

@@ -4,8 +4,8 @@ Tests for ErrorModule and NonStaffErrorModule
import unittest
from xmodule.tests import get_test_system
from xmodule.error_module import ErrorDescriptor, ErrorModule, NonStaffErrorDescriptor
from xmodule.modulestore import Location
from xmodule.modulestore.xml import CourseLocationGenerator
from xmodule.modulestore.locations import SlashSeparatedCourseKey, Location
from xmodule.x_module import XModuleDescriptor, XModule
from mock import MagicMock, Mock, patch
from xblock.runtime import Runtime, IdReader
@@ -17,9 +17,8 @@ from xblock.test.tools import unabc
class SetupTestErrorModules():
def setUp(self):
self.system = get_test_system()
self.org = "org"
self.course = "course"
self.location = Location(['i4x', self.org, self.course, None, None])
self.course_id = SlashSeparatedCourseKey('org', 'course', 'run')
self.location = self.course_id.make_usage_key('foo', 'bar')
self.valid_xml = u"<problem>ABC \N{SNOWMAN}</problem>"
self.error_msg = "Error"
@@ -35,7 +34,7 @@ class TestErrorModule(unittest.TestCase, SetupTestErrorModules):
descriptor = ErrorDescriptor.from_xml(
self.valid_xml,
self.system,
CourseLocationGenerator(self.org, self.course),
CourseLocationGenerator(self.course_id),
self.error_msg
)
self.assertIsInstance(descriptor, ErrorDescriptor)
@@ -70,7 +69,7 @@ class TestNonStaffErrorModule(unittest.TestCase, SetupTestErrorModules):
descriptor = NonStaffErrorDescriptor.from_xml(
self.valid_xml,
self.system,
CourseLocationGenerator(self.org, self.course)
CourseLocationGenerator(self.course_id)
)
self.assertIsInstance(descriptor, NonStaffErrorDescriptor)
@@ -78,7 +77,7 @@ class TestNonStaffErrorModule(unittest.TestCase, SetupTestErrorModules):
descriptor = NonStaffErrorDescriptor.from_xml(
self.valid_xml,
self.system,
CourseLocationGenerator(self.org, self.course)
CourseLocationGenerator(self.course_id)
)
descriptor.xmodule_runtime = self.system
context_repr = self.system.render(descriptor, 'student_view').content
@@ -130,7 +129,7 @@ class TestErrorModuleConstruction(unittest.TestCase):
self.descriptor = BrokenDescriptor(
TestRuntime(Mock(spec=IdReader), field_data),
field_data,
ScopeIds(None, None, None, 'i4x://org/course/broken/name')
ScopeIds(None, None, None, Location('org', 'course', 'run', 'broken', 'name', None))
)
self.descriptor.xmodule_runtime = TestRuntime(Mock(spec=IdReader), field_data)
self.descriptor.xmodule_runtime.error_descriptor_class = ErrorDescriptor

10
common/lib/xmodule/xmodule/tests/test_export.py
View File

@@ -36,7 +36,7 @@ def strip_filenames(descriptor):
"""
Recursively strips 'filename' from all children's definitions.
"""
print("strip filename from {desc}".format(desc=descriptor.location.url()))
print("strip filename from {desc}".format(desc=descriptor.location.to_deprecated_string()))
if descriptor._field_data.has(descriptor, 'filename'):
descriptor._field_data.delete(descriptor, 'filename')
@@ -173,11 +173,11 @@ class TestEdxJsonEncoder(unittest.TestCase):
self.null_utc_tz = NullTZ()
def test_encode_location(self):
loc = Location('i4x', 'org', 'course', 'category', 'name')
self.assertEqual(loc.url(), self.encoder.default(loc))
loc = Location('org', 'course', 'run', 'category', 'name', None)
self.assertEqual(loc.to_deprecated_string(), self.encoder.default(loc))
loc = Location('i4x', 'org', 'course', 'category', 'name', 'version')
self.assertEqual(loc.url(), self.encoder.default(loc))
loc = Location('org', 'course', 'run', 'category', 'name', 'version')
self.assertEqual(loc.to_deprecated_string(), self.encoder.default(loc))
def test_encode_naive_datetime(self):
self.assertEqual(

76
common/lib/xmodule/xmodule/tests/test_import.py
View File

@@ -12,12 +12,13 @@ from django.utils.timezone import UTC
from xmodule.xml_module import is_pointer_tag
from xmodule.modulestore import Location, only_xmodules
from xmodule.modulestore.xml import ImportSystem, XMLModuleStore, LocationReader
from xmodule.modulestore.xml import ImportSystem, XMLModuleStore
from xmodule.modulestore.inheritance import compute_inherited_metadata
from xmodule.x_module import XModuleMixin
from xmodule.fields import Date
from xmodule.tests import DATA_DIR
from xmodule.modulestore.inheritance import InheritanceMixin
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from xblock.core import XBlock
from xblock.fields import Scope, String, Integer
@@ -34,7 +35,7 @@ class DummySystem(ImportSystem):
def __init__(self, load_error_modules):
xmlstore = XMLModuleStore("data_dir", course_dirs=[], load_error_modules=load_error_modules)
course_id = "/".join([ORG, COURSE, 'test_run'])
course_id = SlashSeparatedCourseKey(ORG, COURSE, 'test_run')
course_dir = "test_dir"
error_tracker = Mock()
parent_tracker = Mock()
@@ -48,7 +49,6 @@ class DummySystem(ImportSystem):
load_error_modules=load_error_modules,
mixins=(InheritanceMixin, XModuleMixin),
field_data=KvsFieldData(DictKeyValueStore()),
id_reader=LocationReader(),
)
def render_template(self, _template, _context):
@@ -343,7 +343,7 @@ class ImportTestCase(BaseCourseTestCase):
def check_for_key(key, node, value):
"recursive check for presence of key"
print("Checking {0}".format(node.location.url()))
print("Checking {0}".format(node.location.to_deprecated_string()))
self.assertEqual(getattr(node, key), value)
for c in node.get_children():
check_for_key(key, c, value)
@@ -383,12 +383,10 @@ class ImportTestCase(BaseCourseTestCase):
modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy', 'two_toys'])
toy_id = "edX/toy/2012_Fall"
two_toy_id = "edX/toy/TT_2012_Fall"
location = Location(["i4x", "edX", "toy", "video", "Welcome"])
toy_video = modulestore.get_instance(toy_id, location)
two_toy_video = modulestore.get_instance(two_toy_id, location)
location = Location("edX", "toy", "2012_Fall", "video", "Welcome", None)
toy_video = modulestore.get_item(location)
location_two = Location("edX", "toy", "TT_2012_Fall", "video", "Welcome", None)
two_toy_video = modulestore.get_item(location_two)
self.assertEqual(toy_video.youtube_id_1_0, "p2Q6BrNhdh8")
self.assertEqual(two_toy_video.youtube_id_1_0, "p2Q6BrNhdh9")
@@ -401,10 +399,9 @@ class ImportTestCase(BaseCourseTestCase):
courses = modulestore.get_courses()
self.assertEquals(len(courses), 1)
course = courses[0]
course_id = course.id
print("course errors:")
for (msg, err) in modulestore.get_item_errors(course.location):
for (msg, err) in modulestore.get_course_errors(course.id):
print(msg)
print(err)
@@ -416,13 +413,12 @@ class ImportTestCase(BaseCourseTestCase):
print("Ch2 location: ", ch2.location)
also_ch2 = modulestore.get_instance(course_id, ch2.location)
also_ch2 = modulestore.get_item(ch2.location)
self.assertEquals(ch2, also_ch2)
print("making sure html loaded")
cloc = course.location
loc = Location(cloc.tag, cloc.org, cloc.course, 'html', 'secret:toylab')
html = modulestore.get_instance(course_id, loc)
loc = course.id.make_usage_key('html', 'secret:toylab')
html = modulestore.get_item(loc)
self.assertEquals(html.display_name, "Toy lab")
def test_unicode(self):
@@ -442,12 +438,16 @@ class ImportTestCase(BaseCourseTestCase):
# Expect to find an error/exception about characters in "®esources"
expect = "Invalid characters"
errors = [(msg.encode("utf-8"), err.encode("utf-8"))
for msg, err in
modulestore.get_item_errors(course.location)]
errors = [
(msg.encode("utf-8"), err.encode("utf-8"))
for msg, err
in modulestore.get_course_errors(course.id)
]
self.assertTrue(any(expect in msg or expect in err
for msg, err in errors))
self.assertTrue(any(
expect in msg or expect in err
for msg, err in errors
))
chapters = course.get_children()
self.assertEqual(len(chapters), 4)
@@ -458,7 +458,7 @@ class ImportTestCase(BaseCourseTestCase):
modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy'])
toy_id = "edX/toy/2012_Fall"
toy_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
course = modulestore.get_course(toy_id)
chapters = course.get_children()
@@ -484,20 +484,12 @@ class ImportTestCase(BaseCourseTestCase):
self.assertEqual(len(sections), 1)
location = course.location
conditional_location = Location(
location.tag, location.org, location.course,
'conditional', 'condone'
)
module = modulestore.get_instance(course.id, conditional_location)
conditional_location = course.id.make_usage_key('conditional', 'condone')
module = modulestore.get_item(conditional_location)
self.assertEqual(len(module.children), 1)
poll_location = Location(
location.tag, location.org, location.course,
'poll_question', 'first_poll'
)
module = modulestore.get_instance(course.id, poll_location)
poll_location = course.id.make_usage_key('poll_question', 'first_poll')
module = modulestore.get_item(poll_location)
self.assertEqual(len(module.get_children()), 0)
self.assertEqual(module.voted, False)
self.assertEqual(module.poll_answer, '')
@@ -527,9 +519,9 @@ class ImportTestCase(BaseCourseTestCase):
'''
modulestore = XMLModuleStore(DATA_DIR, course_dirs=['graphic_slider_tool'])
sa_id = "edX/gst_test/2012_Fall"
location = Location(["i4x", "edX", "gst_test", "graphical_slider_tool", "sample_gst"])
gst_sample = modulestore.get_instance(sa_id, location)
sa_id = SlashSeparatedCourseKey("edX", "gst_test", "2012_Fall")
location = sa_id.make_usage_key("graphical_slider_tool", "sample_gst")
gst_sample = modulestore.get_item(location)
render_string_from_sample_gst_xml = """
<slider var="a" style="width:400px;float:left;"/>\
<plot style="margin-top:15px;margin-bottom:15px;"/>""".strip()
@@ -545,12 +537,8 @@ class ImportTestCase(BaseCourseTestCase):
self.assertEqual(len(sections), 1)
location = course.location
location = Location(
location.tag, location.org, location.course,
'word_cloud', 'cloud1'
)
module = modulestore.get_instance(course.id, location)
location = course.id.make_usage_key('word_cloud', 'cloud1')
module = modulestore.get_item(location)
self.assertEqual(len(module.get_children()), 0)
self.assertEqual(module.num_inputs, 5)
self.assertEqual(module.num_top_words, 250)
@@ -561,7 +549,7 @@ class ImportTestCase(BaseCourseTestCase):
"""
modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy'])
toy_id = "edX/toy/2012_Fall"
toy_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
course = modulestore.get_course(toy_id)

6
common/lib/xmodule/xmodule/tests/test_import_static.py
View File

@@ -3,8 +3,8 @@ Tests that check that we ignore the appropriate files when importing courses.
"""
import unittest
from mock import Mock
from xmodule.modulestore import Location
from xmodule.modulestore.xml_importer import import_static_content
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from xmodule.tests import DATA_DIR
@@ -12,10 +12,10 @@ class IgnoredFilesTestCase(unittest.TestCase):
"Tests for ignored files"
def test_ignore_tilde_static_files(self):
course_dir = DATA_DIR / "tilde"
loc = Location("edX", "tilde", "Fall_2012")
course_id = SlashSeparatedCourseKey("edX", "tilde", "Fall_2012")
content_store = Mock()
content_store.generate_thumbnail.return_value = ("content", "location")
import_static_content(Mock(), Mock(), course_dir, content_store, loc)
import_static_content(course_dir, content_store, course_id)
saved_static_content = [call[0][0] for call in content_store.save.call_args_list]
name_val = {sc.name: sc.data for sc in saved_static_content}
self.assertIn("example.txt", name_val)

46
common/lib/xmodule/xmodule/tests/test_lti_unit.py
View File

@@ -262,26 +262,22 @@ class LTIModuleTest(LogicTest):
self.assertEqual(real_resource_link_id, expected_resource_link_id)
def test_lis_result_sourcedid(self):
with patch('xmodule.lti_module.LTIModule.location', new_callable=PropertyMock) as mock_location:
self.xmodule.location.html_id = lambda: 'i4x-2-3-lti-31de800015cf4afb973356dbe81496df'
expected_sourcedId = u':'.join(urllib.quote(i) for i in (
self.system.course_id,
urllib.quote(self.unquoted_resource_link_id),
self.user_id
))
real_lis_result_sourcedid = self.xmodule.get_lis_result_sourcedid()
self.assertEqual(real_lis_result_sourcedid, expected_sourcedId)
expected_sourcedId = u':'.join(urllib.quote(i) for i in (
self.system.course_id.to_deprecated_string(),
self.xmodule.get_resource_link_id(),
self.user_id
))
real_lis_result_sourcedid = self.xmodule.get_lis_result_sourcedid()
self.assertEqual(real_lis_result_sourcedid, expected_sourcedId)
@patch('xmodule.course_module.CourseDescriptor.id_to_location')
def test_client_key_secret(self, test):
def test_client_key_secret(self):
"""
LTI module gets client key and secret provided.
"""
#this adds lti passports to system
mocked_course = Mock(lti_passports = ['lti_id:test_client:test_secret'])
modulestore = Mock()
modulestore.get_item.return_value = mocked_course
modulestore.get_course.return_value = mocked_course
runtime = Mock(modulestore=modulestore)
self.xmodule.descriptor.runtime = runtime
self.xmodule.lti_id = "lti_id"
@@ -289,8 +285,7 @@ class LTIModuleTest(LogicTest):
expected = ('test_client', 'test_secret')
self.assertEqual(expected, (key, secret))
@patch('xmodule.course_module.CourseDescriptor.id_to_location')
def test_client_key_secret_not_provided(self, test):
def test_client_key_secret_not_provided(self):
"""
LTI module attempts to get client key and secret provided in cms.
@@ -300,7 +295,7 @@ class LTIModuleTest(LogicTest):
#this adds lti passports to system
mocked_course = Mock(lti_passports = ['test_id:test_client:test_secret'])
modulestore = Mock()
modulestore.get_item.return_value = mocked_course
modulestore.get_course.return_value = mocked_course
runtime = Mock(modulestore=modulestore)
self.xmodule.descriptor.runtime = runtime
#set another lti_id
@@ -309,8 +304,7 @@ class LTIModuleTest(LogicTest):
expected = ('','')
self.assertEqual(expected, key_secret)
@patch('xmodule.course_module.CourseDescriptor.id_to_location')
def test_bad_client_key_secret(self, test):
def test_bad_client_key_secret(self):
"""
LTI module attempts to get client key and secret provided in cms.
@@ -319,16 +313,16 @@ class LTIModuleTest(LogicTest):
#this adds lti passports to system
mocked_course = Mock(lti_passports = ['test_id_test_client_test_secret'])
modulestore = Mock()
modulestore.get_item.return_value = mocked_course
modulestore.get_course.return_value = mocked_course
runtime = Mock(modulestore=modulestore)
self.xmodule.descriptor.runtime = runtime
self.xmodule.lti_id = 'lti_id'
with self.assertRaises(LTIError):
self.xmodule.get_client_key_secret()
@patch('xmodule.lti_module.signature.verify_hmac_sha1', return_value=True)
@patch('xmodule.lti_module.LTIModule.get_client_key_secret', return_value=('test_client_key', u'test_client_secret'))
def test_successful_verify_oauth_body_sign(self, get_key_secret, mocked_verify):
@patch('xmodule.lti_module.signature.verify_hmac_sha1', Mock(return_value=True))
@patch('xmodule.lti_module.LTIModule.get_client_key_secret', Mock(return_value=('test_client_key', u'test_client_secret')))
def test_successful_verify_oauth_body_sign(self):
"""
Test if OAuth signing was successful.
"""
@@ -337,9 +331,9 @@ class LTIModuleTest(LogicTest):
except LTIError:
self.fail("verify_oauth_body_sign() raised LTIError unexpectedly!")
@patch('xmodule.lti_module.signature.verify_hmac_sha1', return_value=False)
@patch('xmodule.lti_module.LTIModule.get_client_key_secret', return_value=('test_client_key', u'test_client_secret'))
def test_failed_verify_oauth_body_sign(self, get_key_secret, mocked_verify):
@patch('xmodule.lti_module.signature.verify_hmac_sha1', Mock(return_value=False))
@patch('xmodule.lti_module.LTIModule.get_client_key_secret', Mock(return_value=('test_client_key', u'test_client_secret')))
def test_failed_verify_oauth_body_sign(self):
"""
Oauth signing verify fail.
"""
@@ -411,4 +405,4 @@ class LTIModuleTest(LogicTest):
"""
Tests that LTI parameter context_id is equal to course_id.
"""
self.assertEqual(self.system.course_id, self.xmodule.context_id)
self.assertEqual(self.system.course_id.to_deprecated_string(), self.xmodule.context_id)

65
common/lib/xmodule/xmodule/tests/test_peer_grading.py
View File

@@ -7,7 +7,7 @@ from webob.multidict import MultiDict
from xblock.field_data import DictFieldData
from xblock.fields import ScopeIds
from xmodule.modulestore import Location
from xmodule.modulestore.locations import Location, SlashSeparatedCourseKey
from xmodule.tests import get_test_system, get_test_descriptor_system
from xmodule.tests.test_util_open_ended import DummyModulestore
from xmodule.open_ended_grading_classes.peer_grading_service import MockPeerGradingService
@@ -16,20 +16,17 @@ from xmodule.modulestore.exceptions import ItemNotFoundError, NoPathToItem
log = logging.getLogger(__name__)
ORG = "edX"
COURSE = "open_ended"
class PeerGradingModuleTest(unittest.TestCase, DummyModulestore):
"""
Test peer grading xmodule at the unit level. More detailed tests are difficult, as the module relies on an
external grading service.
"""
problem_location = Location(["i4x", "edX", "open_ended", "peergrading",
"PeerGradingSample"])
coe_location = Location(["i4x", "edX", "open_ended", "combinedopenended", "SampleQuestion"])
course_id = SlashSeparatedCourseKey('edX', 'open_ended', '2012_Fall')
problem_location = course_id.make_usage_key("peergrading", "PeerGradingSample")
coe_location = course_id.make_usage_key("combinedopenended", "SampleQuestion")
calibrated_dict = {'location': "blah"}
coe_dict = {'location': coe_location.url()}
coe_dict = {'location': coe_location.to_deprecated_string()}
save_dict = MultiDict({
'location': "blah",
'submission_id': 1,
@@ -42,7 +39,7 @@ class PeerGradingModuleTest(unittest.TestCase, DummyModulestore):
save_dict.extend(('rubric_scores[]', val) for val in (0, 1))
def get_module_system(self, descriptor):
test_system = get_test_system()
test_system = get_test_system(self.course_id)
test_system.open_ended_grading_interface = None
return test_system
@@ -51,9 +48,9 @@ class PeerGradingModuleTest(unittest.TestCase, DummyModulestore):
Create a peer grading module from a test system
@return:
"""
self.setup_modulestore(COURSE)
self.peer_grading = self.get_module_from_location(self.problem_location, COURSE)
self.coe = self.get_module_from_location(self.coe_location, COURSE)
self.setup_modulestore(self.course_id.course)
self.peer_grading = self.get_module_from_location(self.problem_location)
self.coe = self.get_module_from_location(self.coe_location)
def test_module_closed(self):
"""
@@ -75,7 +72,7 @@ class PeerGradingModuleTest(unittest.TestCase, DummyModulestore):
Try getting data from the external grading service
@return:
"""
success, _data = self.peer_grading.query_data_for_location(self.problem_location.url())
success, _data = self.peer_grading.query_data_for_location(self.problem_location.to_deprecated_string())
self.assertTrue(success)
def test_get_score_none(self):
@@ -149,8 +146,11 @@ class PeerGradingModuleTest(unittest.TestCase, DummyModulestore):
Mainly for diff coverage
@return:
"""
# pylint: disable=protected-access
with self.assertRaises(ItemNotFoundError):
self.peer_grading._find_corresponding_module_for_location(Location('i4x', 'a', 'b', 'c', 'd'))
self.peer_grading._find_corresponding_module_for_location(
Location('org', 'course', 'run', 'category', 'name', 'revision')
)
def test_get_instance_state(self):
"""
@@ -235,7 +235,13 @@ class MockPeerGradingServiceProblemList(MockPeerGradingService):
def get_problem_list(self, course_id, grader_id):
return {'success': True,
'problem_list': [
{"num_graded": 3, "num_pending": 681, "num_required": 3, "location": "i4x://edX/open_ended/combinedopenended/SampleQuestion", "problem_name": "Peer-Graded Essay"},
{
"num_graded": 3,
"num_pending": 681,
"num_required": 3,
"location": course_id.make_usage_key('combinedopenended', 'SampleQuestion'),
"problem_name": "Peer-Graded Essay"
},
]}
@@ -244,12 +250,12 @@ class PeerGradingModuleScoredTest(unittest.TestCase, DummyModulestore):
Test peer grading xmodule at the unit level. More detailed tests are difficult, as the module relies on an
external grading service.
"""
problem_location = Location(
["i4x", "edX", "open_ended", "peergrading", "PeerGradingScored"]
)
course_id = SlashSeparatedCourseKey('edX', 'open_ended', '2012_Fall')
problem_location = course_id.make_usage_key("peergrading", "PeerGradingScored")
def get_module_system(self, descriptor):
test_system = get_test_system()
test_system = get_test_system(self.course_id)
test_system.open_ended_grading_interface = None
return test_system
@@ -258,10 +264,10 @@ class PeerGradingModuleScoredTest(unittest.TestCase, DummyModulestore):
Create a peer grading module from a test system
@return:
"""
self.setup_modulestore(COURSE)
self.setup_modulestore(self.course_id.course)
def test_metadata_load(self):
peer_grading = self.get_module_from_location(self.problem_location, COURSE)
peer_grading = self.get_module_from_location(self.problem_location)
self.assertFalse(peer_grading.closed())
def test_problem_list(self):
@@ -270,7 +276,7 @@ class PeerGradingModuleScoredTest(unittest.TestCase, DummyModulestore):
"""
# Initialize peer grading module.
peer_grading = self.get_module_from_location(self.problem_location, COURSE)
peer_grading = self.get_module_from_location(self.problem_location)
# Ensure that it cannot find any peer grading.
html = peer_grading.peer_grading()
@@ -286,13 +292,12 @@ class PeerGradingModuleLinkedTest(unittest.TestCase, DummyModulestore):
"""
Test peer grading that is linked to an open ended module.
"""
problem_location = Location(["i4x", "edX", "open_ended", "peergrading",
"PeerGradingLinked"])
coe_location = Location(["i4x", "edX", "open_ended", "combinedopenended",
"SampleQuestion"])
course_id = SlashSeparatedCourseKey('edX', 'open_ended', '2012_Fall')
problem_location = course_id.make_usage_key("peergrading", "PeerGradingLinked")
coe_location = course_id.make_usage_key("combinedopenended", "SampleQuestion")
def get_module_system(self, descriptor):
test_system = get_test_system()
test_system = get_test_system(self.course_id)
test_system.open_ended_grading_interface = None
return test_system
@@ -300,7 +305,7 @@ class PeerGradingModuleLinkedTest(unittest.TestCase, DummyModulestore):
"""
Create a peer grading module from a test system.
"""
self.setup_modulestore(COURSE)
self.setup_modulestore(self.course_id.course)
@property
def field_data(self):
@@ -312,7 +317,7 @@ class PeerGradingModuleLinkedTest(unittest.TestCase, DummyModulestore):
'data': '<peergrading/>',
'location': self.problem_location,
'use_for_single_location': True,
'link_to_location': self.coe_location.url(),
'link_to_location': self.coe_location.to_deprecated_string(),
'graded': True,
})
@@ -424,7 +429,7 @@ class PeerGradingModuleLinkedTest(unittest.TestCase, DummyModulestore):
peer_grading = self._create_peer_grading_with_linked_problem(self.coe_location)
# If we specify a location, it will render the problem for that location.
data = peer_grading.handle_ajax('problem', {'location': self.coe_location})
data = peer_grading.handle_ajax('problem', {'location': self.coe_location.to_deprecated_string()})
self.assertTrue(json.loads(data)['success'])
# If we don't specify a location, it should use the linked location.

9
common/lib/xmodule/xmodule/tests/test_self_assessment.py
View File

@@ -4,6 +4,7 @@ import unittest
from mock import Mock, MagicMock
from webob.multidict import MultiDict
from pytz import UTC
from xblock.fields import ScopeIds
from xmodule.open_ended_grading_classes.self_assessment_module import SelfAssessmentModule
from xmodule.modulestore import Location
from lxml import etree
@@ -29,8 +30,7 @@ class SelfAssessmentTest(unittest.TestCase):
'hintprompt': 'Consider this...',
}
location = Location(["i4x", "edX", "sa_test", "selfassessment",
"SampleQuestion"])
location = Location("edX", "sa_test", "run", "selfassessment", "SampleQuestion", None)
descriptor = Mock()
@@ -56,7 +56,10 @@ class SelfAssessmentTest(unittest.TestCase):
}
system = get_test_system()
system.xmodule_instance = Mock(scope_ids=Mock(usage_id='dummy-usage-id'))
usage_key = system.course_id.make_usage_key('combinedopenended', 'test_loc')
scope_ids = ScopeIds(1, 'combinedopenended', usage_key, usage_key)
system.xmodule_instance = Mock(scope_ids=scope_ids)
self.module = SelfAssessmentModule(
system,
self.location,

24
common/lib/xmodule/xmodule/tests/test_tabs.py
View File

@@ -2,6 +2,7 @@
from mock import MagicMock
import xmodule.tabs as tabs
import unittest
from xmodule.modulestore.locations import SlashSeparatedCourseKey
class TabTestCase(unittest.TestCase):
@@ -9,7 +10,7 @@ class TabTestCase(unittest.TestCase):
def setUp(self):
self.course = MagicMock()
self.course.id = 'edX/toy/2012_Fall'
self.course.id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
self.fake_dict_tab = {'fake_key': 'fake_value'}
self.settings = MagicMock()
self.settings.FEATURES = {}
@@ -137,7 +138,7 @@ class ProgressTestCase(TabTestCase):
return self.check_tab(
tab_class=tabs.ProgressTab,
dict_tab={'type': tabs.ProgressTab.type, 'name': 'same'},
expected_link=self.reverse('progress', args=[self.course.id]),
expected_link=self.reverse('progress', args=[self.course.id.to_deprecated_string()]),
expected_tab_id=tabs.ProgressTab.type,
invalid_dict_tab=None,
)
@@ -161,7 +162,7 @@ class WikiTestCase(TabTestCase):
return self.check_tab(
tab_class=tabs.WikiTab,
dict_tab={'type': tabs.WikiTab.type, 'name': 'same'},
expected_link=self.reverse('course_wiki', args=[self.course.id]),
expected_link=self.reverse('course_wiki', args=[self.course.id.to_deprecated_string()]),
expected_tab_id=tabs.WikiTab.type,
invalid_dict_tab=self.fake_dict_tab,
)
@@ -220,7 +221,7 @@ class StaticTabTestCase(TabTestCase):
tab = self.check_tab(
tab_class=tabs.StaticTab,
dict_tab={'type': tabs.StaticTab.type, 'name': 'same', 'url_slug': url_slug},
expected_link=self.reverse('static_tab', args=[self.course.id, url_slug]),
expected_link=self.reverse('static_tab', args=[self.course.id.to_deprecated_string(), url_slug]),
expected_tab_id='static_tab_schmug',
invalid_dict_tab=self.fake_dict_tab,
)
@@ -257,7 +258,10 @@ class TextbooksTestCase(TabTestCase):
# verify all textbook type tabs
if isinstance(tab, tabs.SingleTextbookTab):
book_type, book_index = tab.tab_id.split("/", 1)
expected_link = self.reverse(type_to_reverse_name[book_type], args=[self.course.id, book_index])
expected_link = self.reverse(
type_to_reverse_name[book_type],
args=[self.course.id.to_deprecated_string(), book_index]
)
self.assertEqual(tab.link_func(self.course, self.reverse), expected_link)
self.assertTrue(tab.name.startswith('Book{0}'.format(book_index)))
num_textbooks_found = num_textbooks_found + 1
@@ -279,7 +283,7 @@ class GradingTestCase(TabTestCase):
tab_class=tab_class,
dict_tab={'type': tab_class.type, 'name': name},
expected_name=name,
expected_link=self.reverse(link_value, args=[self.course.id]),
expected_link=self.reverse(link_value, args=[self.course.id.to_deprecated_string()]),
expected_tab_id=tab_class.type,
invalid_dict_tab=None,
)
@@ -314,7 +318,7 @@ class NotesTestCase(TabTestCase):
return self.check_tab(
tab_class=tabs.NotesTab,
dict_tab={'type': tabs.NotesTab.type, 'name': 'same'},
expected_link=self.reverse('notes', args=[self.course.id]),
expected_link=self.reverse('notes', args=[self.course.id.to_deprecated_string()]),
expected_tab_id=tabs.NotesTab.type,
invalid_dict_tab=self.fake_dict_tab,
)
@@ -341,7 +345,7 @@ class SyllabusTestCase(TabTestCase):
tab_class=tabs.SyllabusTab,
dict_tab={'type': tabs.SyllabusTab.type, 'name': name},
expected_name=name,
expected_link=self.reverse('syllabus', args=[self.course.id]),
expected_link=self.reverse('syllabus', args=[self.course.id.to_deprecated_string()]),
expected_tab_id=tabs.SyllabusTab.type,
invalid_dict_tab=None,
)
@@ -365,7 +369,7 @@ class InstructorTestCase(TabTestCase):
tab_class=tabs.InstructorTab,
dict_tab={'type': tabs.InstructorTab.type, 'name': name},
expected_name=name,
expected_link=self.reverse('instructor_dashboard', args=[self.course.id]),
expected_link=self.reverse('instructor_dashboard', args=[self.course.id.to_deprecated_string()]),
expected_tab_id=tabs.InstructorTab.type,
invalid_dict_tab=None,
)
@@ -603,7 +607,7 @@ class DiscussionLinkTestCase(TabTestCase):
"""Custom reverse function"""
def reverse_discussion_link(viewname, args):
"""reverse lookup for discussion link"""
if viewname == "django_comment_client.forum.views.forum_form_discussion" and args == [course.id]:
if viewname == "django_comment_client.forum.views.forum_form_discussion" and args == [course.id.to_deprecated_string()]:
return "default_discussion_link"
return reverse_discussion_link

7
common/lib/xmodule/xmodule/tests/test_util_open_ended.py
View File

@@ -92,11 +92,8 @@ class DummyModulestore(object):
courses = self.modulestore.get_courses()
return courses[0]
def get_module_from_location(self, location, course):
course = self.get_course(course)
if not isinstance(location, Location):
location = Location(location)
descriptor = self.modulestore.get_instance(course.id, location, depth=None)
def get_module_from_location(self, usage_key):
descriptor = self.modulestore.get_item(usage_key, depth=None)
descriptor.xmodule_runtime = self.get_module_system(descriptor)
return descriptor

21
common/lib/xmodule/xmodule/tests/test_video.py
View File

@@ -125,7 +125,7 @@ class VideoDescriptorTest(unittest.TestCase):
def setUp(self):
system = get_test_descriptor_system()
location = Location('i4x://org/course/video/name')
location = Location('org', 'course', 'run', 'video', 'name', None)
self.descriptor = system.construct_xblock_from_class(
VideoDescriptor,
scope_ids=ScopeIds(None, None, location, location),
@@ -138,7 +138,7 @@ class VideoDescriptorTest(unittest.TestCase):
back out to XML.
"""
system = DummySystem(load_error_modules=True)
location = Location(["i4x", "edX", "video", "default", "SampleProblem1"])
location = Location("edX", 'course', 'run', "video", 'SampleProblem1', None)
field_data = DictFieldData({'location': location})
descriptor = VideoDescriptor(system, field_data, Mock())
descriptor.youtube_id_0_75 = 'izygArpw-Qo'
@@ -154,7 +154,7 @@ class VideoDescriptorTest(unittest.TestCase):
in the output string.
"""
system = DummySystem(load_error_modules=True)
location = Location(["i4x", "edX", "video", "default", "SampleProblem1"])
location = Location("edX", 'course', 'run', "video", "SampleProblem1", None)
field_data = DictFieldData({'location': location})
descriptor = VideoDescriptor(system, field_data, Mock())
descriptor.youtube_id_0_75 = 'izygArpw-Qo'
@@ -194,8 +194,7 @@ class VideoDescriptorImportTestCase(unittest.TestCase):
<transcript language="ge" src="german_translation.srt" />
</video>
'''
location = Location(["i4x", "edX", "video", "default",
"SampleProblem1"])
location = Location("edX", 'course', 'run', "video", "SampleProblem1", None)
field_data = DictFieldData({
'data': sample_xml,
'location': location
@@ -498,6 +497,9 @@ class VideoExportTestCase(unittest.TestCase):
Make sure that VideoDescriptor can export itself to XML
correctly.
"""
def setUp(self):
self.location = Location("edX", 'course', 'run', "video", "SampleProblem1", None)
def assertXmlEqual(self, expected, xml):
for attr in ['tag', 'attrib', 'text', 'tail']:
self.assertEqual(getattr(expected, attr), getattr(xml, attr))
@@ -507,8 +509,7 @@ class VideoExportTestCase(unittest.TestCase):
def test_export_to_xml(self):
"""Test that we write the correct XML on export."""
module_system = DummySystem(load_error_modules=True)
location = Location(["i4x", "edX", "video", "default", "SampleProblem1"])
desc = VideoDescriptor(module_system, DictFieldData({}), ScopeIds(None, None, location, location))
desc = VideoDescriptor(module_system, DictFieldData({}), ScopeIds(None, None, self.location, self.location))
desc.youtube_id_0_75 = 'izygArpw-Qo'
desc.youtube_id_1_0 = 'p2Q6BrNhdh8'
@@ -540,8 +541,7 @@ class VideoExportTestCase(unittest.TestCase):
def test_export_to_xml_empty_end_time(self):
"""Test that we write the correct XML on export."""
module_system = DummySystem(load_error_modules=True)
location = Location(["i4x", "edX", "video", "default", "SampleProblem1"])
desc = VideoDescriptor(module_system, DictFieldData({}), ScopeIds(None, None, location, location))
desc = VideoDescriptor(module_system, DictFieldData({}), ScopeIds(None, None, self.location, self.location))
desc.youtube_id_0_75 = 'izygArpw-Qo'
desc.youtube_id_1_0 = 'p2Q6BrNhdh8'
@@ -569,8 +569,7 @@ class VideoExportTestCase(unittest.TestCase):
def test_export_to_xml_empty_parameters(self):
"""Test XML export with defaults."""
module_system = DummySystem(load_error_modules=True)
location = Location(["i4x", "edX", "video", "default", "SampleProblem1"])
desc = VideoDescriptor(module_system, DictFieldData({}), ScopeIds(None, None, location, location))
desc = VideoDescriptor(module_system, DictFieldData({}), ScopeIds(None, None, self.location, self.location))
xml = desc.definition_to_xml(None)
expected = '<video url_name="SampleProblem1"/>\n'

2
common/lib/xmodule/xmodule/tests/test_xblock_wrappers.py
View File

@@ -190,7 +190,7 @@ class LeafDescriptorFactory(Factory):
@lazy_attribute
def location(self):
return Location('i4x://org/course/category/{}'.format(self.url_name))
return Location('org', 'course', 'run', 'category', self.url_name, None)
@lazy_attribute
def block_type(self):

14
common/lib/xmodule/xmodule/tests/xml/__init__.py
View File

@@ -7,8 +7,8 @@ from unittest import TestCase
from xmodule.x_module import XMLParsingSystem, policy_key
from xmodule.mako_module import MakoDescriptorSystem
from xmodule.modulestore.xml import create_block_from_xml, LocationReader, CourseLocationGenerator
from xmodule.modulestore import Location
from xmodule.modulestore.xml import create_block_from_xml, CourseLocationGenerator
from xmodule.modulestore.locations import SlashSeparatedCourseKey, Location
from xblock.runtime import KvsFieldData, DictKeyValueStore
@@ -18,8 +18,7 @@ class InMemorySystem(XMLParsingSystem, MakoDescriptorSystem): # pylint: disable
The simplest possible XMLParsingSystem
"""
def __init__(self, xml_import_data):
self.org = xml_import_data.org
self.course = xml_import_data.course
self.course_id = SlashSeparatedCourseKey.from_deprecated_string(xml_import_data.course_id)
self.default_class = xml_import_data.default_class
self._descriptors = {}
@@ -37,7 +36,6 @@ class InMemorySystem(XMLParsingSystem, MakoDescriptorSystem): # pylint: disable
select=xml_import_data.xblock_select,
render_template=lambda template, context: pprint.pformat((template, context)),
field_data=KvsFieldData(DictKeyValueStore()),
id_reader=LocationReader(),
)
def process_xml(self, xml): # pylint: disable=method-hidden
@@ -45,14 +43,14 @@ class InMemorySystem(XMLParsingSystem, MakoDescriptorSystem): # pylint: disable
descriptor = create_block_from_xml(
xml,
self,
CourseLocationGenerator(self.org, self.course),
CourseLocationGenerator(self.course_id),
)
self._descriptors[descriptor.location.url()] = descriptor
self._descriptors[descriptor.location.to_deprecated_string()] = descriptor
return descriptor
def load_item(self, location): # pylint: disable=method-hidden
"""Return the descriptor loaded for `location`"""
return self._descriptors[Location(location).url()]
return self._descriptors[location.to_deprecated_string()]
class XModuleXmlImportTest(TestCase):

12
common/lib/xmodule/xmodule/tests/xml/factories.py
View File

@@ -17,15 +17,14 @@ class XmlImportData(object):
Class to capture all of the data needed to actually run an XML import,
so that the Factories have something to generate
"""
def __init__(self, xml_node, xml=None, org=None, course=None,
def __init__(self, xml_node, xml=None, course_id=None,
default_class=None, policy=None,
filesystem=None, parent=None,
xblock_mixins=(), xblock_select=None):
self._xml_node = xml_node
self._xml_string = xml
self.org = org
self.course = course
self.course_id = course_id
self.default_class = default_class
self.filesystem = filesystem
self.xblock_mixins = xblock_mixins
@@ -47,8 +46,8 @@ class XmlImportData(object):
def __repr__(self):
return u"XmlImportData{!r}".format((
self._xml_node, self._xml_string, self.org,
self.course, self.default_class, self.policy,
self._xml_node, self._xml_string, self.course_id,
self.default_class, self.policy,
self.filesystem, self.parent, self.xblock_mixins,
self.xblock_select,
))
@@ -74,6 +73,7 @@ class XmlImportFactory(Factory):
policy = {}
inline_xml = True
tag = 'unknown'
course_id = 'edX/xml_test_course/101'
@classmethod
def _adjust_kwargs(cls, **kwargs):
@@ -136,8 +136,6 @@ class XmlImportFactory(Factory):
class CourseFactory(XmlImportFactory):
"""Factory for <course> nodes"""
tag = 'course'
org = 'edX'
course = 'xml_test_course'
name = '101'
static_asset_path = 'xml_test_course'

2
common/lib/xmodule/xmodule/vertical_module.py
View File

@@ -25,7 +25,7 @@ class VerticalModule(VerticalFields, XModule):
fragment.add_frag_resources(rendered_child)
contents.append({
'id': child.id,
'id': child.location.to_deprecated_string(),
'content': rendered_child.content
})

7
common/lib/xmodule/xmodule/video_module/transcripts_utils.py
View File

@@ -289,9 +289,7 @@ def copy_or_rename_transcript(new_name, old_name, item, delete_old=False, user=N
If `delete_old` is True, removes `old_name` files from storage.
"""
filename = 'subs_{0}.srt.sjson'.format(old_name)
content_location = StaticContent.compute_location(
item.location.org, item.location.course, filename
)
content_location = StaticContent.compute_location(item.location.course_key, filename)
transcripts = contentstore().find(content_location).data
save_subs_to_store(json.loads(transcripts), new_name, item)
item.sub = new_name
@@ -532,7 +530,7 @@ class Transcript(object):
"""
Return asset location. `location` is module location.
"""
return StaticContent.compute_location(location.org, location.course, filename)
return StaticContent.compute_location(location.course_key, filename)
@staticmethod
def delete_asset(location, filename):
@@ -545,4 +543,5 @@ class Transcript(object):
log.info("Transcript asset %s was removed from store.", filename)
except NotFoundError:
pass
return StaticContent.compute_location(location.course_key, filename)

53
common/lib/xmodule/xmodule/x_module.py
View File

@@ -18,14 +18,12 @@ from webob.multidict import MultiDict
from xblock.core import XBlock
from xblock.fields import Scope, Integer, Float, List, XBlockMixin, String, Dict
from xblock.fragment import Fragment
from xblock.plugin import default_select
from xblock.runtime import Runtime
from xmodule.fields import RelativeTime
from xmodule.errortracker import exc_info_to_str
from xmodule.modulestore import Location
from xmodule.modulestore.exceptions import ItemNotFoundError, InsufficientSpecificationError, InvalidLocationError
from xmodule.modulestore.locator import BlockUsageLocator
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.keys import OpaqueKeyReader, UsageKey
from xmodule.exceptions import UndefinedContext
from dogapi import dog_stats_api
@@ -156,11 +154,7 @@ class XModuleMixin(XBlockMixin):
@property
def course_id(self):
return self.runtime.course_id
@property
def id(self):
return self.location.url()
return self.location.course_key
@property
def category(self):
@@ -168,16 +162,11 @@ class XModuleMixin(XBlockMixin):
@property
def location(self):
try:
return Location(self.scope_ids.usage_id)
except InvalidLocationError:
if isinstance(self.scope_ids.usage_id, BlockUsageLocator):
return self.scope_ids.usage_id
else:
return BlockUsageLocator(self.scope_ids.usage_id)
return self.scope_ids.usage_id
@location.setter
def location(self, value):
assert isinstance(value, UsageKey)
self.scope_ids = self.scope_ids._replace(
def_id=value,
usage_id=value,
@@ -185,12 +174,7 @@ class XModuleMixin(XBlockMixin):
@property
def url_name(self):
if isinstance(self.location, Location):
return self.location.name
elif isinstance(self.location, BlockUsageLocator):
return self.location.block_id
else:
raise InsufficientSpecificationError()
return self.location.name
@property
def display_name_with_default(self):
@@ -203,6 +187,17 @@ class XModuleMixin(XBlockMixin):
name = self.url_name.replace('_', ' ')
return name
@property
def xblock_kvs(self):
"""
Retrieves the internal KeyValueStore for this XModule.
Should only be used by the persistence layer. Use with caution.
"""
# if caller wants kvs, caller's assuming it's up to date; so, decache it
self.save()
return self._field_data._kvs # pylint: disable=protected-access
def get_explicitly_set_fields_by_scope(self, scope=Scope.content):
"""
Get a dictionary of the fields for the given scope which are set explicitly on this xblock. (Including
@@ -214,15 +209,6 @@ class XModuleMixin(XBlockMixin):
result[field.name] = field.read_json(self)
return result
@property
def xblock_kvs(self):
"""
Use w/ caution. Really intended for use by the persistence layer.
"""
# if caller wants kvs, caller's assuming it's up to date; so, decache it
self.save()
return self._field_data._kvs # pylint: disable=protected-access
def get_content_titles(self):
"""
Returns list of content titles for all of self's children.
@@ -684,7 +670,6 @@ class XModuleDescriptor(XModuleMixin, HTMLSnippet, ResourceTemplates, XBlock):
Interpret the parsed XML in `node`, creating an XModuleDescriptor.
"""
xml = etree.tostring(node)
# TODO: change from_xml to not take org and course, it can use self.system.
block = cls.from_xml(xml, runtime, id_generator)
return block
@@ -1023,7 +1008,7 @@ class DescriptorSystem(MetricsMixin, ConfigurableFragmentWrapper, Runtime): # p
local_resource_url: an implementation of :meth:`xblock.runtime.Runtime.local_resource_url`
"""
super(DescriptorSystem, self).__init__(**kwargs)
super(DescriptorSystem, self).__init__(id_reader=OpaqueKeyReader(), **kwargs)
# This is used by XModules to write out separate files during xml export
self.export_fs = None
@@ -1217,7 +1202,7 @@ class ModuleSystem(MetricsMixin,ConfigurableFragmentWrapper, Runtime): # pylint
# Usage_store is unused, and field_data is often supplanted with an
# explicit field_data during construct_xblock.
super(ModuleSystem, self).__init__(id_reader=None, field_data=field_data, **kwargs)
super(ModuleSystem, self).__init__(id_reader=OpaqueKeyReader(), field_data=field_data, **kwargs)
self.STATIC_URL = static_url
self.xqueue = xqueue