Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #16511 from edx/noraiz/EDUCATOR-1696

Browse Source 
optimize libraries iter for studio dashboard
This commit is contained in:
Noraiz Anwar
2017-11-14 15:41:39 +05:00
committed by GitHub
parent 08e6a069a9 73f9c02d6d
commit 439e367dea
6 changed files with 122 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
cms/djangoapps/contentstore/views/course.py
View File

@@ -484,7 +484,7 @@ def _accessible_libraries_iter(user, org=None):
if org is not None:
libraries = [] if org == '' else modulestore().get_libraries(org=org)
else:
libraries = modulestore().get_libraries()
libraries = modulestore().get_library_summaries()
# No need to worry about ErrorDescriptors - split's get_libraries() never returns them.
return (lib for lib in libraries if has_studio_read_access(user, lib.location.library_key))
@@ -493,7 +493,7 @@ def _accessible_libraries_iter(user, org=None):
@ensure_csrf_cookie
def course_listing(request):
"""
List all courses available to the logged in user
List all courses and libraries available to the logged in user
"""
optimization_enabled = GlobalStaff().has_user(request.user) and \
@@ -502,7 +502,10 @@ def course_listing(request):
org = request.GET.get('org', '') if optimization_enabled else None
courses_iter, in_process_course_actions = get_courses_accessible_to_user(request, org)
user = request.user
# remove this temporary log after getting results
start_time = time.time()
libraries = _accessible_libraries_iter(request.user, org) if LIBRARIES_ENABLED else []
log.info("_accessible_libraries_iter completed in [%f]", (time.time() - start_time))
def format_in_process_course_view(uca):
"""
@@ -529,6 +532,7 @@ def course_listing(request):
"""
Return a dict of the data which the view requires for each library
"""
return {
u'display_name': library.display_name,
u'library_key': unicode(library.location.library_key),

36
common/lib/xmodule/xmodule/library_content_module.py
View File

@@ -640,3 +640,39 @@ class LibraryContentDescriptor(LibraryContentFields, MakoModuleDescriptor, XmlDe
if field.is_set_on(self):
xml_object.set(field_name, unicode(field.read_from(self)))
return xml_object
class LibrarySummary(object):
"""
A library summary object which contains the fields required for library listing on studio.
"""
def __init__(self, library_locator, display_name):
"""
Initialize LibrarySummary
Arguments:
library_locator (LibraryLocator): LibraryLocator object of the library.
display_name (unicode): display name of the library.
"""
self.display_name = display_name if display_name else _(u"Empty")
self.id = library_locator # pylint: disable=invalid-name
self.location = library_locator.make_usage_key('library', 'library')
@property
def display_org_with_default(self):
"""
Org display names are not implemented. This just provides API compatibility with CourseDescriptor.
Always returns the raw 'org' field from the key.
"""
return self.location.library_key.org
@property
def display_number_with_default(self):
"""
Display numbers are not implemented. This just provides API compatibility with CourseDescriptor.
Always returns the raw 'library' field from the key.
"""
return self.location.library_key.library

7
common/lib/xmodule/xmodule/modulestore/exceptions.py
View File

@@ -18,6 +18,13 @@ class MultipleCourseBlocksFound(Exception):
pass
class MultipleLibraryBlocksFound(Exception):
"""
Raise this exception when Iterating over the library blocks return multiple library blocks.
"""
pass
class InsufficientSpecificationError(Exception):
pass

17
common/lib/xmodule/xmodule/modulestore/mixed.py
View File

@@ -321,6 +321,23 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
courses[course_id] = course
return courses.values()
@strip_key
def get_library_summaries(self, **kwargs):
"""
Returns a list of LibrarySummary objects.
Information contains `location`, `display_name`, `locator` of the libraries in this modulestore.
"""
library_summaries = {}
for store in self.modulestores:
if not hasattr(store, 'get_libraries'):
continue
# fetch library summaries and filter out any duplicated entry across/within stores
for library_summary in store.get_library_summaries(**kwargs):
library_id = self._clean_locator_for_mapping(library_summary.location)
if library_id not in library_summaries:
library_summaries[library_id] = library_summary
return library_summaries.values()
@strip_key
def get_libraries(self, **kwargs):
"""

9
common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py
View File

@@ -362,20 +362,21 @@ class MongoConnection(object):
return docs
@autoretry_read()
def find_course_blocks_by_id(self, ids, course_context=None):
def find_courselike_blocks_by_id(self, ids, block_type, course_context=None):
"""
Find all structures that specified in `ids`. Among the blocks only return block whose type is `course`.
Find all structures that specified in `ids`. Among the blocks only return block whose type is `block_type`.
Arguments:
ids (list): A list of structure ids
block_type: type of block to return
"""
with TIMER.timer("find_course_blocks_by_id", course_context) as tagger:
with TIMER.timer("find_courselike_blocks_by_id", course_context) as tagger:
tagger.measure("requested_ids", len(ids))
docs = [
structure_from_mongo(structure, course_context)
for structure in self.structures.find(
{'_id': {'$in': ids}},
{'blocks': {'$elemMatch': {'block_type': 'course'}}, 'root': 1}
{'blocks': {'$elemMatch': {'block_type': block_type}}, 'root': 1}
)
]
tagger.measure("structures", len(docs))

60
common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
View File

@@ -70,6 +70,7 @@ from bson.objectid import ObjectId
from xblock.core import XBlock
from xblock.fields import Scope, Reference, ReferenceList, ReferenceValueDict
from xmodule.course_module import CourseSummary
from xmodule.library_content_module import LibrarySummary
from xmodule.errortracker import null_error_tracker
from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.locator import (
@@ -582,16 +583,15 @@ class SplitBulkWriteMixin(BulkOperationsMixin):
return course_indexes
def find_course_blocks_by_id(self, ids):
def find_courselike_blocks_by_id(self, ids, block_type):
"""
Find all structures that specified in `ids`. Filter the course blocks to only return whose
`block_type` is `course`
Find all structures that specified in `ids`. Return blocks matching with block_type.
Arguments:
ids (list): A list of structure ids
block_type: type of block to return
"""
ids = set(ids)
return self.db_connection.find_course_blocks_by_id(list(ids))
return self.db_connection.find_courselike_blocks_by_id(list(ids), block_type)
def find_structures_by_id(self, ids):
"""
@@ -691,6 +691,9 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
# version) but those functions will have an optional arg for setting these.
SEARCH_TARGET_DICT = ['wiki_slug']
DEFAULT_ROOT_LIBRARY_BLOCK_TYPE = 'library'
DEFAULT_ROOT_COURSE_BLOCK_TYPE = 'course'
def __init__(self, contentstore, doc_store_config, fs_root, render_template,
default_class=None,
error_tracker=null_error_tracker,
@@ -913,16 +916,19 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
# add it in the envelope for the structure.
return CourseEnvelope(course_key.replace(version_guid=version_guid), entry)
def _get_course_blocks_for_branch(self, branch, **kwargs):
def _get_courselike_blocks_for_branch(self, branch, **kwargs):
"""
Internal generator for fetching lists of courses without loading them.
Internal generator for fetching lists of courselike without loading them.
"""
version_guids, id_version_map = self.collect_ids_from_matching_indexes(branch, **kwargs)
if not version_guids:
return
for entry in self.find_course_blocks_by_id(version_guids):
block_type = SplitMongoModuleStore.DEFAULT_ROOT_LIBRARY_BLOCK_TYPE \
if branch == 'library' else SplitMongoModuleStore.DEFAULT_ROOT_COURSE_BLOCK_TYPE
for entry in self.find_courselike_blocks_by_id(version_guids, block_type):
for course_index in id_version_map[entry['_id']]:
yield entry, course_index
@@ -1039,7 +1045,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
}
courses_summaries = []
for entry, structure_info in self._get_course_blocks_for_branch(branch, **kwargs):
for entry, structure_info in self._get_courselike_blocks_for_branch(branch, **kwargs):
course_locator = self._create_course_locator(structure_info, branch=None)
course_block = [
block_data
@@ -1059,6 +1065,42 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
)
return courses_summaries
@autoretry_read()
def get_library_summaries(self, **kwargs):
"""
Returns a list of `LibrarySummary` objects.
kwargs can be valid db fields to match against active_versions
collection e.g org='example_org'.
"""
branch = 'library'
libraries_summaries = []
for entry, structure_info in self._get_courselike_blocks_for_branch(branch, **kwargs):
library_locator = self._create_library_locator(structure_info, branch=None)
library_block = [
block_data
for block_key, block_data in entry['blocks'].items()
if block_key.type == "library"
]
if not library_block:
raise ItemNotFoundError
if len(library_block) > 1:
raise MultipleLibraryBlocksFound(
"Expected 1 library block, but found {0}".format(len(library_block))
)
library_block_fields = library_block[0].fields
display_name = ''
if 'display_name' in library_block_fields:
display_name = library_block_fields['display_name']
libraries_summaries.append(
LibrarySummary(library_locator, display_name)
)
return libraries_summaries
def get_libraries(self, branch="library", **kwargs):
"""
Returns a list of "library" root blocks matching any given qualifiers.