From e59b650fab1b66dcff9360233eb008f6e0e108e8 Mon Sep 17 00:00:00 2001 From: Chris Dodge Date: Mon, 18 Mar 2013 15:02:49 -0400 Subject: [PATCH] add an array of courses to not refresh the metadata cache on writes. This is needed for imports where we are doing bulk updates otherwise, the DB gets thrashed and the import takes too long, timing out the browser --- common/lib/xmodule/xmodule/modulestore/mongo.py | 14 ++++++++++---- .../xmodule/xmodule/modulestore/xml_importer.py | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/common/lib/xmodule/xmodule/modulestore/mongo.py b/common/lib/xmodule/xmodule/modulestore/mongo.py index c5e5bbfdf8..4ec7a699e0 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo.py @@ -246,6 +246,7 @@ class MongoModuleStore(ModuleStoreBase): self.fs_root = path(fs_root) self.error_tracker = error_tracker self.render_template = render_template + self.ignore_write_events_on_courses = [] def get_metadata_inheritance_tree(self, location): ''' @@ -329,6 +330,11 @@ class MongoModuleStore(ModuleStoreBase): return tree + def refresh_cached_metadata_inheritance_tree(self, location): + pseudo_course_id = '/'.join([location.org, location.course]) + if pseudo_course_id not in self.ignore_write_events_on_courses: + self.get_cached_metadata_inheritance_tree(location, force_refresh = True) + def clear_cached_metadata_inheritance_tree(self, location): key_name = '{0}/{1}'.format(location.org, location.course) if self.metadata_inheritance_cache is not None: @@ -519,7 +525,7 @@ class MongoModuleStore(ModuleStoreBase): raise DuplicateItemError(location) # recompute (and update) the metadata inheritance tree which is cached - self.get_cached_metadata_inheritance_tree(Location(location), force_refresh = True) + self.refresh_cached_metadata_inheritance_tree(Location(location)) def get_course_for_item(self, location, depth=0): ''' @@ -586,7 +592,7 @@ class MongoModuleStore(ModuleStoreBase): self._update_single_item(location, {'definition.children': children}) # recompute (and update) the metadata inheritance tree which is cached - self.get_cached_metadata_inheritance_tree(Location(location), force_refresh = True) + self.refresh_cached_metadata_inheritance_tree(Location(location)) def update_metadata(self, location, metadata): """ @@ -612,7 +618,7 @@ class MongoModuleStore(ModuleStoreBase): self._update_single_item(location, {'metadata': metadata}) # recompute (and update) the metadata inheritance tree which is cached - self.get_cached_metadata_inheritance_tree(loc, force_refresh = True) + self.refresh_cached_metadata_inheritance_tree(loc) def delete_item(self, location): """ @@ -632,7 +638,7 @@ class MongoModuleStore(ModuleStoreBase): self.collection.remove({'_id': Location(location).dict()}) # recompute (and update) the metadata inheritance tree which is cached - self.get_cached_metadata_inheritance_tree(Location(location), force_refresh = True) + self.refresh_cached_metadata_inheritance_tree(Location(location)) def get_parent_locations(self, location, course_id): diff --git a/common/lib/xmodule/xmodule/modulestore/xml_importer.py b/common/lib/xmodule/xmodule/modulestore/xml_importer.py index fa232596f2..5955b0ebe7 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml_importer.py +++ b/common/lib/xmodule/xmodule/modulestore/xml_importer.py @@ -201,6 +201,16 @@ def import_from_xml(store, data_dir, course_dirs=None, course_items = [] for course_id in module_store.modules.keys(): + if target_location_namespace is not None: + pseudo_course_id = '/'.join([target_location_namespace.org, target_location_namespace.course]) + else: + course_id_components = course_id.split('/') + pseudo_course_id = '/'.join([course_id_components[0], course_id_components[1]]) + + # turn off all write signalling while importing as this is a high volume operation + if pseudo_course_id not in store.ignore_write_events_on_courses: + store.ignore_write_events_on_courses.append(pseudo_course_id) + course_data_path = None course_location = None @@ -296,6 +306,12 @@ def import_from_xml(store, data_dir, course_dirs=None, # inherited metadata everywhere. store.update_metadata(module.location, dict(own_metadata(module))) + # turn back on all write signalling + if pseudo_course_id in store.ignore_write_events_on_courses: + store.ignore_write_events_on_courses.remove(pseudo_course_id) + store.refresh_cached_metadata_inheritance_tree(target_location_namespace if + target_location_namespace is not None else course_location) + return module_store, course_items def remap_namespace(module, target_location_namespace):