From ba993c59375b0d136ec31242cd8b6f62f621eca9 Mon Sep 17 00:00:00 2001 From: Kyle Mulka Date: Tue, 7 May 2019 10:36:59 -0400 Subject: [PATCH] =?UTF-8?q?INCR-226=20run=20modernize=20on=20common/lib/xm?= =?UTF-8?q?odule/xmodule/modulestore/mong=E2=80=A6=20(#20431)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * INCR-226 run modernize on common/lib/xmodule/xmodule/modulestore/mongo, split_mongo, and perf_tests * fix pep8 error --- .../xmodule/modulestore/mongo/__init__.py | 3 +- .../xmodule/xmodule/modulestore/mongo/base.py | 107 +++++++------- .../xmodule/modulestore/mongo/draft.py | 31 ++-- .../perf_tests/generate_asset_xml.py | 19 ++- .../modulestore/perf_tests/generate_report.py | 6 +- .../perf_tests/test_asset_import_export.py | 25 ++-- .../modulestore/split_mongo/__init__.py | 5 +- .../split_mongo/caching_descriptor_system.py | 25 ++-- .../split_mongo/definition_lazy_loader.py | 5 +- .../modulestore/split_mongo/id_manager.py | 7 +- .../split_mongo/mongo_connection.py | 39 ++--- .../xmodule/modulestore/split_mongo/split.py | 138 ++++++++++-------- .../modulestore/split_mongo/split_draft.py | 15 +- .../split_mongo/split_mongo_kvs.py | 18 ++- 14 files changed, 250 insertions(+), 193 deletions(-) diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/__init__.py b/common/lib/xmodule/xmodule/modulestore/mongo/__init__.py index c20ca43e84..584ed10283 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo/__init__.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo/__init__.py @@ -2,8 +2,9 @@ Provide names as exported by older mongo.py module """ -from xmodule.modulestore.mongo.base import MongoModuleStore, MongoKeyValueStore +from __future__ import absolute_import +from xmodule.modulestore.mongo.base import MongoKeyValueStore, MongoModuleStore # Backwards compatibility for prod systems that refererence # xmodule.modulestore.mongo.DraftMongoModuleStore from xmodule.modulestore.mongo.draft import DraftModuleStore as DraftMongoModuleStore diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/base.py b/common/lib/xmodule/xmodule/modulestore/mongo/base.py index b2c25ba948..26c6181b24 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo/base.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo/base.py @@ -11,47 +11,48 @@ structure: 'definition.children': } """ -import six +from __future__ import absolute_import + import copy -from datetime import datetime -from importlib import import_module import logging -import pymongo import re import sys +from datetime import datetime +from importlib import import_module from uuid import uuid4 +import pymongo +import six from bson.son import SON from contracts import contract, new_contract from fs.osfs import OSFS from mongodb_proxy import autoretry_read -from opaque_keys.edx.keys import UsageKey, CourseKey, AssetKey +from opaque_keys.edx.keys import AssetKey, CourseKey, UsageKey from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator, LibraryLocator from path import Path as path from pytz import UTC from xblock.core import XBlock from xblock.exceptions import InvalidScopeError -from xblock.fields import Scope, ScopeIds, Reference, ReferenceList, ReferenceValueDict +from xblock.fields import Reference, ReferenceList, ReferenceValueDict, Scope, ScopeIds from xblock.runtime import KvsFieldData from xmodule.assetstore import AssetMetadata, CourseAssetsFromStorage from xmodule.course_module import CourseSummary from xmodule.error_module import ErrorDescriptor -from xmodule.errortracker import null_error_tracker, exc_info_to_str +from xmodule.errortracker import exc_info_to_str, null_error_tracker from xmodule.exceptions import HeartbeatFailure from xmodule.mako_module import MakoDescriptorSystem -from xmodule.mongo_utils import connect_to_mongodb, create_collection_index -from xmodule.modulestore import ModuleStoreWriteBase, ModuleStoreEnum, BulkOperationsMixin, BulkOpsRecord -from xmodule.modulestore.draft_and_published import ModuleStoreDraftAndPublished, DIRECT_ONLY_CATEGORIES +from xmodule.modulestore import BulkOperationsMixin, BulkOpsRecord, ModuleStoreEnum, ModuleStoreWriteBase +from xmodule.modulestore.draft_and_published import DIRECT_ONLY_CATEGORIES, ModuleStoreDraftAndPublished from xmodule.modulestore.edit_info import EditInfoRuntimeMixin -from xmodule.modulestore.exceptions import ItemNotFoundError, DuplicateCourseError, ReferentialIntegrityError -from xmodule.modulestore.inheritance import InheritanceMixin, inherit_metadata, InheritanceKeyValueStore -from xmodule.partitions.partitions_service import PartitionService -from xmodule.modulestore.xml import CourseLocationManager +from xmodule.modulestore.exceptions import DuplicateCourseError, ItemNotFoundError, ReferentialIntegrityError +from xmodule.modulestore.inheritance import InheritanceKeyValueStore, InheritanceMixin, inherit_metadata from xmodule.modulestore.store_utilities import DETACHED_XBLOCK_TYPES +from xmodule.modulestore.xml import CourseLocationManager +from xmodule.mongo_utils import connect_to_mongodb, create_collection_index +from xmodule.partitions.partitions_service import PartitionService from xmodule.services import SettingsService - log = logging.getLogger(__name__) new_contract('CourseKey', CourseKey) @@ -183,10 +184,10 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin): def __repr__(self): return "CachingDescriptorSystem{!r}".format(( self.modulestore, - unicode(self.course_id), - [unicode(key) for key in self.module_data.keys()], + six.text_type(self.course_id), + [six.text_type(key) for key in self.module_data.keys()], self.default_class, - [unicode(key) for key in self.cached_metadata.keys()], + [six.text_type(key) for key in self.cached_metadata.keys()], )) def __init__(self, modulestore, course_key, module_data, default_class, cached_metadata, **kwargs): @@ -265,7 +266,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin): parent = None if self.cached_metadata is not None: # fish the parent out of here if it's available - parent_url = self.cached_metadata.get(unicode(location), {}).get('parent', {}).get( + parent_url = self.cached_metadata.get(six.text_type(location), {}).get('parent', {}).get( ModuleStoreEnum.Branch.published_only if location.branch is None else ModuleStoreEnum.Branch.draft_preferred ) @@ -281,7 +282,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin): ) data = definition.get('data', {}) - if isinstance(data, basestring): + if isinstance(data, six.string_types): data = {'data': data} mixed_class = self.mixologist.mix(class_) @@ -305,7 +306,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin): # Convert the serialized fields values in self.cached_metadata # to python values - metadata_to_inherit = self.cached_metadata.get(unicode(non_draft_loc), {}) + metadata_to_inherit = self.cached_metadata.get(six.text_type(non_draft_loc), {}) inherit_metadata(module, metadata_to_inherit) module._edit_info = json_data.get('edit_info') @@ -351,7 +352,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin): :param jsonfields: a dict of the jsonified version of the fields """ result = {} - for field_name, value in jsonfields.iteritems(): + for field_name, value in six.iteritems(jsonfields): field = class_.fields.get(field_name) if field is None: continue @@ -365,7 +366,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin): ] elif isinstance(field, ReferenceValueDict): result[field_name] = { - key: self._convert_reference_to_key(subvalue) for key, subvalue in value.iteritems() + key: self._convert_reference_to_key(subvalue) for key, subvalue in six.iteritems(value) } else: result[field_name] = value @@ -516,17 +517,17 @@ class ParentLocationCache(dict): """ # pylint: disable=missing-docstring - @contract(key=unicode) + @contract(key=six.text_type) def has(self, key): return key in self - @contract(key=unicode, value="BlockUsageLocator | None") + @contract(key=six.text_type, value="BlockUsageLocator | None") def set(self, key, value): self[key] = value @contract(value="BlockUsageLocator") def delete_by_value(self, value): - keys_to_delete = [k for k, v in self.iteritems() if v == value] + keys_to_delete = [k for k, v in six.iteritems(self) if v == value] for key in keys_to_delete: del self[key] @@ -720,7 +721,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo # manually pick it apart b/c the db has tag and we want as_published revision regardless location = as_published(BlockUsageLocator._from_deprecated_son(result['_id'], course_id.run)) - location_url = unicode(location) + location_url = six.text_type(location) if location_url in results_by_url: # found either draft or live to complement the other revision # FIXME this is wrong. If the child was moved in draft from one parent to the other, it will @@ -777,12 +778,12 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo course_id = self.fill_in_run(course_id) if not force_refresh: # see if we are first in the request cache (if present) - if self.request_cache is not None and unicode(course_id) in self.request_cache.data.get('metadata_inheritance', {}): - return self.request_cache.data['metadata_inheritance'][unicode(course_id)] + if self.request_cache is not None and six.text_type(course_id) in self.request_cache.data.get('metadata_inheritance', {}): + return self.request_cache.data['metadata_inheritance'][six.text_type(course_id)] # then look in any caching subsystem (e.g. memcached) if self.metadata_inheritance_cache_subsystem is not None: - tree = self.metadata_inheritance_cache_subsystem.get(unicode(course_id), {}) + tree = self.metadata_inheritance_cache_subsystem.get(six.text_type(course_id), {}) else: logging.warning( 'Running MongoModuleStore without a metadata_inheritance_cache_subsystem. This is \ @@ -795,7 +796,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo # now write out computed tree to caching subsystem (e.g. memcached), if available if self.metadata_inheritance_cache_subsystem is not None: - self.metadata_inheritance_cache_subsystem.set(unicode(course_id), tree) + self.metadata_inheritance_cache_subsystem.set(six.text_type(course_id), tree) # now populate a request_cache, if available. NOTE, we are outside of the # scope of the above if: statement so that after a memcache hit, it'll get @@ -805,7 +806,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo # defined if 'metadata_inheritance' not in self.request_cache.data: self.request_cache.data['metadata_inheritance'] = {} - self.request_cache.data['metadata_inheritance'][unicode(course_id)] = tree + self.request_cache.data['metadata_inheritance'][six.text_type(course_id)] = tree return tree @@ -1022,7 +1023,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo for course_key in course_keys: course_query = { '_id.{}'.format(value_attr): getattr(course_key, key_attr) - for key_attr, value_attr in {'org': 'org', 'course': 'course', 'run': 'name'}.iteritems() + for key_attr, value_attr in six.iteritems({'org': 'org', 'course': 'course', 'run': 'name'}) } course_query.update(query) course_queries.append(course_query) @@ -1148,8 +1149,8 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo location = course_key.make_usage_key('course', course_key.run) if ignore_case: course_query = location.to_deprecated_son('_id.') - for key in course_query.iterkeys(): - if isinstance(course_query[key], basestring): + for key in six.iterkeys(course_query): + if isinstance(course_query[key], six.string_types): course_query[key] = re.compile(r"(?i)^{}$".format(course_query[key])) else: course_query = {'_id': location.to_deprecated_son()} @@ -1273,9 +1274,9 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo qualifier_value = {'$in': qualifier_value} query['_id.' + field] = qualifier_value - for key, value in (settings or {}).iteritems(): + for key, value in six.iteritems((settings or {})): query['metadata.' + key] = value - for key, value in (content or {}).iteritems(): + for key, value in six.iteritems((content or {})): query['definition.data.' + key] = value if 'children' in qualifiers: query['definition.children'] = qualifiers.pop('children') @@ -1395,7 +1396,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo for_parent=kwargs.get('for_parent'), ) if fields is not None: - for key, value in fields.iteritems(): + for key, value in six.iteritems(fields): setattr(xmodule, key, value) # decache any pending field settings from init xmodule.save() @@ -1551,7 +1552,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo parent_cache = self._get_parent_cache(self.get_branch_setting()) parent_cache.delete_by_value(xblock.location) for child in xblock.children: - parent_cache.set(unicode(child), xblock.location) + parent_cache.set(six.text_type(child), xblock.location) self._update_single_item(xblock.scope_ids.usage_id, payload, allow_not_found=allow_not_found) @@ -1585,19 +1586,19 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo :param jsonfields: a dict of the jsonified version of the fields """ jsonfields = {} - for field_name, field in xblock.fields.iteritems(): + for field_name, field in six.iteritems(xblock.fields): if field.scope == scope and field.is_set_on(xblock): if field.scope == Scope.parent: continue elif isinstance(field, Reference): - jsonfields[field_name] = unicode(field.read_from(xblock)) + jsonfields[field_name] = six.text_type(field.read_from(xblock)) elif isinstance(field, ReferenceList): jsonfields[field_name] = [ - unicode(ele) for ele in field.read_from(xblock) + six.text_type(ele) for ele in field.read_from(xblock) ] elif isinstance(field, ReferenceValueDict): jsonfields[field_name] = { - key: unicode(subvalue) for key, subvalue in field.read_from(xblock).iteritems() + key: six.text_type(subvalue) for key, subvalue in six.iteritems(field.read_from(xblock)) } else: jsonfields[field_name] = field.read_json(xblock) @@ -1648,19 +1649,19 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo or revision == ModuleStoreEnum.RevisionOption.draft_preferred parent_cache = self._get_parent_cache(self.get_branch_setting()) - if parent_cache.has(unicode(location)): - return parent_cache.get(unicode(location)) + if parent_cache.has(six.text_type(location)): + return parent_cache.get(six.text_type(location)) # create a query with tag, org, course, and the children field set to the given location query = self._course_key_to_son(location.course_key) - query['definition.children'] = unicode(location) + query['definition.children'] = six.text_type(location) # if only looking for the PUBLISHED parent, set the revision in the query to None if revision == ModuleStoreEnum.RevisionOption.published_only: query['_id.revision'] = MongoRevisionKey.published def cache_and_return(parent_loc): # pylint:disable=missing-docstring - parent_cache.set(unicode(location), parent_loc) + parent_cache.set(six.text_type(location), parent_loc) return parent_loc # query the collection, sorting by DRAFT first @@ -1753,7 +1754,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo if item['_id']['category'] != 'course': # It would be nice to change this method to return UsageKeys instead of the deprecated string. item_locs.add( - unicode(as_published(BlockUsageLocator._from_deprecated_son(item['_id'], course_key.run))) + six.text_type(as_published(BlockUsageLocator._from_deprecated_son(item['_id'], course_key.run))) ) all_reachable = all_reachable.union(item.get('definition', {}).get('children', [])) item_locs -= all_reachable @@ -1811,7 +1812,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo else: # Complete course key, so query for asset metadata. course_assets = self.asset_collection.find_one( - {'course_id': unicode(course_key)}, + {'course_id': six.text_type(course_key)}, ) doc_id = None if course_assets is None else course_assets['_id'] @@ -1821,7 +1822,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo raise ItemNotFoundError(course_key) else: # Course exists, so create matching assets document. - course_assets = {'course_id': unicode(course_key), 'assets': {}} + course_assets = {'course_id': six.text_type(course_key), 'assets': {}} doc_id = self.asset_collection.insert(course_assets) elif isinstance(course_assets['assets'], list): # This record is in the old course assets format. @@ -1858,7 +1859,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo # Build an update set with potentially multiple embedded fields. updates_by_type = {} - for asset_type, assets in assets_by_type.iteritems(): + for asset_type, assets in six.iteritems(assets_by_type): updates_by_type[self._make_mongo_asset_key(asset_type)] = list(assets) # Update the document. @@ -1911,8 +1912,8 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo dest_course_key (CourseKey): identifier of course to copy to """ source_assets = self._find_course_assets(source_course_key) - dest_assets = {'assets': source_assets.asset_md.copy(), 'course_id': unicode(dest_course_key)} - self.asset_collection.remove({'course_id': unicode(dest_course_key)}) + dest_assets = {'assets': source_assets.asset_md.copy(), 'course_id': six.text_type(dest_course_key)} + self.asset_collection.remove({'course_id': six.text_type(dest_course_key)}) # Update the document. self.asset_collection.insert(dest_assets) diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/draft.py b/common/lib/xmodule/xmodule/modulestore/mongo/draft.py index deec0fb500..9a264fae1d 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo/draft.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo/draft.py @@ -6,24 +6,35 @@ returns the i4x://org/course/cat/name@draft object if that exists, and otherwise returns i4x://org/course/cat/name). """ -import pymongo +from __future__ import absolute_import + import logging +import pymongo +import six from opaque_keys.edx.keys import UsageKey from opaque_keys.edx.locator import BlockUsageLocator from six import text_type -from openedx.core.lib.cache_utils import request_cached from xblock.core import XBlock + +from openedx.core.lib.cache_utils import request_cached from xmodule.exceptions import InvalidVersionError from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.draft_and_published import DIRECT_ONLY_CATEGORIES, UnsupportedRevisionError from xmodule.modulestore.exceptions import ( - ItemNotFoundError, DuplicateItemError, DuplicateCourseError, InvalidBranchSetting + DuplicateCourseError, + DuplicateItemError, + InvalidBranchSetting, + ItemNotFoundError ) from xmodule.modulestore.mongo.base import ( - MongoModuleStore, MongoRevisionKey, as_draft, as_published, SORT_REVISION_FAVOR_DRAFT + SORT_REVISION_FAVOR_DRAFT, + MongoModuleStore, + MongoRevisionKey, + as_draft, + as_published ) from xmodule.modulestore.store_utilities import rewrite_nonportable_content_links -from xmodule.modulestore.draft_and_published import UnsupportedRevisionError, DIRECT_ONLY_CATEGORIES log = logging.getLogger(__name__) @@ -207,7 +218,7 @@ class DraftModuleStore(MongoModuleStore): ) else: # update fields on existing course - for key, value in fields.iteritems(): + for key, value in six.iteritems(fields): setattr(new_course, key, value) self.update_item(new_course, user_id) @@ -232,7 +243,7 @@ class DraftModuleStore(MongoModuleStore): log.info("Cloning module %s to %s....", original_loc, module.location) - if 'data' in module.fields and module.fields['data'].is_set_on(module) and isinstance(module.data, basestring): + if 'data' in module.fields and module.fields['data'].is_set_on(module) and isinstance(module.data, six.string_types): module.data = rewrite_nonportable_content_links( original_loc.course_key, dest_course_id, module.data ) @@ -653,7 +664,7 @@ class DraftModuleStore(MongoModuleStore): @request_cached( # use the XBlock's location value in the cache key - arg_map_function=lambda arg: unicode(arg.location if isinstance(arg, XBlock) else arg), + arg_map_function=lambda arg: six.text_type(arg.location if isinstance(arg, XBlock) else arg), # use this store's request_cache request_cache_getter=lambda args, kwargs: args[1], ) @@ -847,7 +858,7 @@ class DraftModuleStore(MongoModuleStore): try: source_item = self.get_item(item_location) except ItemNotFoundError: - log.error('Unable to find the item %s', unicode(item_location)) + log.error('Unable to find the item %s', six.text_type(item_location)) return if source_item.parent and source_item.parent.block_id != original_parent_location.block_id: @@ -886,7 +897,7 @@ class DraftModuleStore(MongoModuleStore): to_process_dict[draft_as_non_draft_loc] = draft # convert the dict - which is used for look ups - back into a list - queried_children = to_process_dict.values() + queried_children = list(to_process_dict.values()) return queried_children diff --git a/common/lib/xmodule/xmodule/modulestore/perf_tests/generate_asset_xml.py b/common/lib/xmodule/xmodule/modulestore/perf_tests/generate_asset_xml.py index 21000f90b1..f06c423290 100644 --- a/common/lib/xmodule/xmodule/modulestore/perf_tests/generate_asset_xml.py +++ b/common/lib/xmodule/xmodule/modulestore/perf_tests/generate_asset_xml.py @@ -4,13 +4,16 @@ """ Generates fake XML for asset metadata. """ -from __future__ import print_function +from __future__ import absolute_import, print_function import random -from lxml import etree from datetime import datetime, timedelta -from xmodule.assetstore import AssetMetadata + +from lxml import etree from opaque_keys.edx.keys import CourseKey +from six.moves import range + +from xmodule.assetstore import AssetMetadata try: import click @@ -52,7 +55,7 @@ def filename(): Fake a filename. """ fname = u'' - for __ in xrange(random.randint(10, 30)): + for __ in range(random.randint(10, 30)): fname += random.choice(NAME_CHARS_W_UNICODE) fname += random.choice(('.jpg', '.pdf', '.png', '.txt')) return fname @@ -63,8 +66,8 @@ def pathname(): Fake a pathname. """ pname = u'' - for __ in xrange(random.randint(2, 3)): - for __ in xrange(random.randint(5, 10)): + for __ in range(random.randint(2, 3)): + for __ in range(random.randint(5, 10)): pname += random.choice(NAME_CHARS) pname += '/' return pname @@ -149,7 +152,7 @@ def generate_random_asset_md(): return AssetMetadata( asset_key, pathname=pathname(), - internal_name=str([filename() for __ in xrange(10)]), + internal_name=str([filename() for __ in range(10)]), locked=locked(), contenttype=contenttype(), thumbnail=filename(), @@ -170,7 +173,7 @@ def make_asset_md(amount): Make a number of fake AssetMetadata objects. """ all_asset_md = [] - for __ in xrange(amount): + for __ in range(amount): all_asset_md.append(generate_random_asset_md()) return all_asset_md diff --git a/common/lib/xmodule/xmodule/modulestore/perf_tests/generate_report.py b/common/lib/xmodule/xmodule/modulestore/perf_tests/generate_report.py index 5b01e11e14..77595ffa02 100644 --- a/common/lib/xmodule/xmodule/modulestore/perf_tests/generate_report.py +++ b/common/lib/xmodule/xmodule/modulestore/perf_tests/generate_report.py @@ -4,11 +4,13 @@ Reads the data generated by performance tests and generates a savable report which can be viewed over time to examine the performance effects of code changes on various parts of the system. """ -from __future__ import print_function +from __future__ import absolute_import, print_function import sqlite3 -from lxml.builder import E + import lxml.html +from lxml.builder import E + try: import click except ImportError: diff --git a/common/lib/xmodule/xmodule/modulestore/perf_tests/test_asset_import_export.py b/common/lib/xmodule/xmodule/modulestore/perf_tests/test_asset_import_export.py index 47b89d6c33..a3fbf3cfe1 100644 --- a/common/lib/xmodule/xmodule/modulestore/perf_tests/test_asset_import_export.py +++ b/common/lib/xmodule/xmodule/modulestore/perf_tests/test_asset_import_export.py @@ -1,26 +1,25 @@ """ Performance test for asset metadata in the modulestore. """ -from path import Path as path -import unittest -from tempfile import mkdtemp -import itertools -from shutil import rmtree -from bson.code import Code +from __future__ import absolute_import + import datetime +import itertools +import unittest +from shutil import rmtree +from tempfile import mkdtemp + import ddt import pytest +from bson.code import Code +from path import Path as path from xmodule.assetstore import AssetMetadata from xmodule.modulestore import ModuleStoreEnum -from xmodule.modulestore.xml_importer import import_course_from_xml +from xmodule.modulestore.perf_tests.generate_asset_xml import ASSET_XSD_FILE, make_asset_xml, validate_xml +from xmodule.modulestore.tests.utils import MODULESTORE_SETUPS, SHORT_NAME_MAP, TEST_DATA_DIR from xmodule.modulestore.xml_exporter import export_course_to_xml -from xmodule.modulestore.tests.utils import ( - MODULESTORE_SETUPS, - SHORT_NAME_MAP, - TEST_DATA_DIR, -) -from xmodule.modulestore.perf_tests.generate_asset_xml import make_asset_xml, validate_xml, ASSET_XSD_FILE +from xmodule.modulestore.xml_importer import import_course_from_xml # Number of assets saved in the modulestore per test run. ASSET_AMOUNT_PER_TEST = (0, 1, 10, 100, 1000, 10000) diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/__init__.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/__init__.py index 06ae542506..f8c8a1b485 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/__init__.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/__init__.py @@ -2,8 +2,11 @@ General utilities """ +from __future__ import absolute_import + from collections import namedtuple -from contracts import contract, check + +from contracts import check, contract from opaque_keys.edx.locator import BlockUsageLocator diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py index 2e5d3f7cfe..f6a02302c2 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py @@ -1,25 +1,28 @@ -import sys -import logging +from __future__ import absolute_import +import logging +import sys + +import six from contracts import contract, new_contract from fs.osfs import OSFS from lazy import lazy -from xblock.runtime import KvsFieldData, KeyValueStore -from xblock.fields import ScopeIds +from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator, DefinitionLocator, LibraryLocator, LocalId from xblock.core import XBlock -from opaque_keys.edx.locator import BlockUsageLocator, LocalId, CourseLocator, LibraryLocator, DefinitionLocator +from xblock.fields import ScopeIds +from xblock.runtime import KeyValueStore, KvsFieldData -from xmodule.library_tools import LibraryToolsService -from xmodule.mako_module import MakoDescriptorSystem from xmodule.error_module import ErrorDescriptor from xmodule.errortracker import exc_info_to_str +from xmodule.library_tools import LibraryToolsService +from xmodule.mako_module import MakoDescriptorSystem from xmodule.modulestore import BlockData from xmodule.modulestore.edit_info import EditInfoRuntimeMixin from xmodule.modulestore.exceptions import ItemNotFoundError -from xmodule.modulestore.inheritance import inheriting_field_data, InheritanceMixin +from xmodule.modulestore.inheritance import InheritanceMixin, inheriting_field_data from xmodule.modulestore.split_mongo import BlockKey, CourseEnvelope -from xmodule.modulestore.split_mongo.id_manager import SplitMongoIdManager from xmodule.modulestore.split_mongo.definition_lazy_loader import DefinitionLazyLoader +from xmodule.modulestore.split_mongo.id_manager import SplitMongoIdManager from xmodule.modulestore.split_mongo.split_mongo_kvs import SplitMongoKVS from xmodule.x_module import XModuleMixin @@ -88,7 +91,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin): @contract(returns="dict(BlockKey: BlockKey)") def _parent_map(self): parent_map = {} - for block_key, block in self.course_entry.structure['blocks'].iteritems(): + for block_key, block in six.iteritems(self.course_entry.structure['blocks']): for child in block.fields.get('children', []): parent_map[child] = block_key return parent_map @@ -382,7 +385,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin): new_aside = super(CachingDescriptorSystem, self).get_aside_of_type(block, aside_type) new_aside._field_data = block._field_data # pylint: disable=protected-access - for key, _ in new_aside.fields.iteritems(): + for key, _ in six.iteritems(new_aside.fields): if isinstance(key, KeyValueStore.Key) and block._field_data.has(new_aside, key): # pylint: disable=protected-access try: value = block._field_data.get(new_aside, key) # pylint: disable=protected-access diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/definition_lazy_loader.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/definition_lazy_loader.py index 8bbc9e0ab5..6cddedfc97 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/definition_lazy_loader.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/definition_lazy_loader.py @@ -1,6 +1,9 @@ -from opaque_keys.edx.locator import DefinitionLocator +from __future__ import absolute_import + import copy +from opaque_keys.edx.locator import DefinitionLocator + class DefinitionLazyLoader(object): """ diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/id_manager.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/id_manager.py index 8de9a72277..cf337abc1d 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/id_manager.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/id_manager.py @@ -3,9 +3,12 @@ An implementation of IdReader and IdGenerator that manages ids for the SplitMong mechanism. """ -from opaque_keys.edx.locator import LocalId, DefinitionLocator -from xmodule.x_module import OpaqueKeyReader, AsideKeyGenerator +from __future__ import absolute_import + +from opaque_keys.edx.locator import DefinitionLocator, LocalId + from xmodule.modulestore.split_mongo import BlockKey +from xmodule.x_module import AsideKeyGenerator, OpaqueKeyReader # TODO: Migrate split_mongo to use this class for all key mapping/creation. diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py index 42062d5469..470ff55f80 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py @@ -1,35 +1,36 @@ """ Segregation of pymongo functions from the data modeling mechanisms for split modulestore. """ +from __future__ import absolute_import + import datetime -import cPickle as pickle +import logging import math -import zlib -import pymongo -import pytz import re +import zlib from contextlib import contextmanager from time import time +import pymongo +import pytz +import six +import six.moves.cPickle as pickle +from contracts import check, new_contract +from mongodb_proxy import autoretry_read # Import this just to export it from pymongo.errors import DuplicateKeyError # pylint: disable=unused-import +from xmodule.exceptions import HeartbeatFailure +from xmodule.modulestore import BlockData +from xmodule.modulestore.split_mongo import BlockKey +from xmodule.mongo_utils import connect_to_mongodb, create_collection_index + try: from django.core.cache import caches, InvalidCacheBackendError DJANGO_AVAILABLE = True except ImportError: DJANGO_AVAILABLE = False -import logging - -from contracts import check, new_contract -from mongodb_proxy import autoretry_read -from xmodule.exceptions import HeartbeatFailure -from xmodule.modulestore import BlockData -from xmodule.modulestore.split_mongo import BlockKey -from xmodule.mongo_utils import connect_to_mongodb, create_collection_index - - new_contract('BlockData', BlockData) log = logging.getLogger(__name__) @@ -82,7 +83,7 @@ class Tagger(object): **kwargs: Each keyword is treated as a tag name, and the value of the argument is the tag value. """ - self.added_tags.extend(kwargs.items()) + self.added_tags.extend(list(kwargs.items())) @property def tags(self): @@ -187,14 +188,14 @@ def structure_to_mongo(structure, course_context=None): check('BlockKey', structure['root']) check('dict(BlockKey: BlockData)', structure['blocks']) - for block in structure['blocks'].itervalues(): + for block in six.itervalues(structure['blocks']): if 'children' in block.fields: check('list(BlockKey)', block.fields['children']) new_structure = dict(structure) new_structure['blocks'] = [] - for block_key, block in structure['blocks'].iteritems(): + for block_key, block in six.iteritems(structure['blocks']): new_block = dict(block.to_storable()) new_block.setdefault('block_type', block_key.type) new_block['block_id'] = block_key.id @@ -311,7 +312,7 @@ class MongoConnection(object): if doc is None: log.warning( "doc was None when attempting to retrieve structure for item with key %s", - unicode(key) + six.text_type(key) ) return None tagger_find_one.measure("blocks", len(doc['blocks'])) @@ -459,7 +460,7 @@ class MongoConnection(object): query['versions.{}'.format(branch)] = {'$exists': True} if search_targets: - for key, value in search_targets.iteritems(): + for key, value in six.iteritems(search_targets): query['search_targets.{}'.format(key)] = value if org_target: diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py index 93e2ff1510..4531ebd67c 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py @@ -53,47 +53,63 @@ Representation: *** 'original_version': definition_id of the root of the previous version relation on this definition. Acts as a pseudo-object identifier. """ +from __future__ import absolute_import + import copy import datetime import hashlib import logging -import six - -from contracts import contract, new_contract +from collections import defaultdict from importlib import import_module -from mongodb_proxy import autoretry_read -from path import Path as path -from pytz import UTC -from bson.objectid import ObjectId +from types import NoneType -from xblock.core import XBlock -from xblock.fields import Scope, Reference, ReferenceList, ReferenceValueDict -from xmodule.course_module import CourseSummary -from xmodule.library_content_module import LibrarySummary -from xmodule.errortracker import null_error_tracker +import six +from bson.objectid import ObjectId +from ccx_keys.locator import CCXBlockUsageLocator, CCXLocator +from contracts import contract, new_contract +from mongodb_proxy import autoretry_read from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.locator import ( - BlockUsageLocator, DefinitionLocator, CourseLocator, LibraryLocator, VersionTree, LocalId, + BlockUsageLocator, + CourseLocator, + DefinitionLocator, + LibraryLocator, + LocalId, + VersionTree ) -from ccx_keys.locator import CCXLocator, CCXBlockUsageLocator -from xmodule.modulestore.exceptions import InsufficientSpecificationError, VersionConflictError, DuplicateItemError, \ - DuplicateCourseError, MultipleCourseBlocksFound +from path import Path as path +from pytz import UTC +from xblock.core import XBlock +from xblock.fields import Reference, ReferenceList, ReferenceValueDict, Scope + +from xmodule.assetstore import AssetMetadata +from xmodule.course_module import CourseSummary +from xmodule.error_module import ErrorDescriptor +from xmodule.errortracker import null_error_tracker +from xmodule.library_content_module import LibrarySummary from xmodule.modulestore import ( - inheritance, ModuleStoreWriteBase, ModuleStoreEnum, - BulkOpsRecord, BulkOperationsMixin, SortedAssetList, BlockData + BlockData, + BulkOperationsMixin, + BulkOpsRecord, + ModuleStoreEnum, + ModuleStoreWriteBase, + SortedAssetList, + inheritance ) +from xmodule.modulestore.exceptions import ( + DuplicateCourseError, + DuplicateItemError, + InsufficientSpecificationError, + MultipleCourseBlocksFound, + VersionConflictError +) +from xmodule.modulestore.split_mongo import BlockKey, CourseEnvelope +from xmodule.modulestore.split_mongo.mongo_connection import DuplicateKeyError, MongoConnection +from xmodule.modulestore.store_utilities import DETACHED_XBLOCK_TYPES +from xmodule.partitions.partitions_service import PartitionService from ..exceptions import ItemNotFoundError from .caching_descriptor_system import CachingDescriptorSystem -from xmodule.partitions.partitions_service import PartitionService -from xmodule.modulestore.split_mongo.mongo_connection import MongoConnection, DuplicateKeyError -from xmodule.modulestore.split_mongo import BlockKey, CourseEnvelope -from xmodule.modulestore.store_utilities import DETACHED_XBLOCK_TYPES -from xmodule.error_module import ErrorDescriptor -from collections import defaultdict -from types import NoneType -from xmodule.assetstore import AssetMetadata - log = logging.getLogger(__name__) @@ -152,7 +168,7 @@ class SplitBulkWriteRecord(BulkOpsRecord): # If there was no index in the database to start with, then all branches # are dirty by definition if self.initial_index is None: - return self.index.get('versions', {}).keys() + return list(self.index.get('versions', {}).keys()) # Return branches whose ids differ between self.index and self.initial_index return [ @@ -247,7 +263,7 @@ class SplitBulkWriteMixin(BulkOperationsMixin): dirty = False # If the content is dirty, then update the database - for _id in bulk_write_record.structures.viewkeys() - bulk_write_record.structures_in_db: + for _id in six.viewkeys(bulk_write_record.structures) - bulk_write_record.structures_in_db: dirty = True try: @@ -258,7 +274,7 @@ class SplitBulkWriteMixin(BulkOperationsMixin): # append only, so if it's already been written, we can just keep going. log.debug("Attempted to insert duplicate structure %s", _id) - for _id in bulk_write_record.definitions.viewkeys() - bulk_write_record.definitions_in_db: + for _id in six.viewkeys(bulk_write_record.definitions) - bulk_write_record.definitions_in_db: dirty = True try: @@ -439,7 +455,7 @@ class SplitBulkWriteMixin(BulkOperationsMixin): defs_from_db = list(self.db_connection.get_definitions(list(ids), course_key)) defs_dict = {d.get('_id'): d for d in defs_from_db} # Add the retrieved definitions to the cache. - bulk_write_record.definitions_in_db.update(defs_dict.iterkeys()) + bulk_write_record.definitions_in_db.update(six.iterkeys(defs_dict)) bulk_write_record.definitions.update(defs_dict) definitions.extend(defs_from_db) return definitions @@ -555,7 +571,7 @@ class SplitBulkWriteMixin(BulkOperationsMixin): 'search_targets' not in record.index or field not in record.index['search_targets'] or record.index['search_targets'][field] != value - for field, value in search_targets.iteritems() + for field, value in six.iteritems(search_targets) ): continue # if we've specified a filter by org, @@ -789,14 +805,14 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): course_key, [ block.definition - for block in new_module_data.itervalues() + for block in six.itervalues(new_module_data) ] ) # Turn definitions into a map. definitions = {definition['_id']: definition for definition in descendent_definitions} - for block in new_module_data.itervalues(): + for block in six.itervalues(new_module_data): if block.definition in definitions: definition = definitions[block.definition] # convert_fields gets done later in the runtime's xblock_from_json @@ -1281,7 +1297,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): # odd case where we don't search just confirm block_name = qualifiers.pop('name') block_ids = [] - for block_id, block in course.structure['blocks'].iteritems(): + for block_id, block in six.iteritems(course.structure['blocks']): # Don't do an in comparison blindly; first check to make sure # that the name qualifier we're looking at isn't a plain string; # if it is a string, then it should match exactly. If it's other @@ -1312,7 +1328,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): path_cache = {} parents_cache = self.build_block_key_to_parents_mapping(course.structure) - for block_id, value in course.structure['blocks'].iteritems(): + for block_id, value in six.iteritems(course.structure['blocks']): if _block_matches_all(value): if not include_orphans: if ( # pylint: disable=bad-continuation @@ -1338,7 +1354,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): :return dict: a dictionary containing mapping of block_keys against their parents. """ children_to_parents = defaultdict(list) - for parent_key, value in structure['blocks'].iteritems(): + for parent_key, value in six.iteritems(structure['blocks']): for child_key in value.fields.get('children', []): children_to_parents[child_key].append(parent_key) @@ -1430,7 +1446,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): items = set(course.structure['blocks'].keys()) items.remove(course.structure['root']) blocks = course.structure['blocks'] - for block_id, block_data in blocks.iteritems(): + for block_id, block_data in six.iteritems(blocks): items.difference_update(BlockKey(*child) for child in block_data.fields.get('children', [])) if block_data.block_type in detached_categories: items.discard(block_id) @@ -1583,7 +1599,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): elif len(possible_roots) == 0: return None # convert the results value sets to locators - for k, versions in result.iteritems(): + for k, versions in six.iteritems(result): result[k] = [ block_locator.for_version(version) for version in versions @@ -1648,10 +1664,10 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): :param user_id: request.user """ def needs_saved(): - for key, value in new_def_data.iteritems(): + for key, value in six.iteritems(new_def_data): if key not in old_definition['fields'] or value != old_definition['fields'][key]: return True - for key, value in old_definition.get('fields', {}).iteritems(): + for key, value in six.iteritems(old_definition.get('fields', {})): if key not in new_def_data: return True @@ -2094,7 +2110,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): with self.bulk_operations(course_key): if allow_not_found and isinstance(block_key.id, (LocalId, NoneType)): fields = {} - for subfields in partitioned_fields.itervalues(): + for subfields in six.itervalues(partitioned_fields): fields.update(subfields) return self.create_item( user_id, course_key, block_key.type, fields=fields, asides=asides, force=force @@ -2107,7 +2123,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): if original_entry is None: if allow_not_found: fields = {} - for subfields in partitioned_fields.itervalues(): + for subfields in six.itervalues(partitioned_fields): fields.update(subfields) return self.create_item(user_id, course_key, block_key.type, block_id=block_key.id, fields=fields, asides=asides, force=force) @@ -2238,7 +2254,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): BlockData(**json_data), **kwargs ) - for field_name, value in (fields or {}).iteritems(): + for field_name, value in six.iteritems((fields or {})): setattr(new_block, field_name, value) if parent_xblock is not None: @@ -2359,13 +2375,13 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): :param settings: :param original_fields: """ - original_keys = original_fields.keys() + original_keys = list(original_fields.keys()) if 'children' in original_keys: original_keys.remove('children') if len(settings) != len(original_keys): return True else: - new_keys = settings.keys() + new_keys = list(settings.keys()) for key in original_keys: if key not in new_keys or original_fields[key] != settings[key]: return True @@ -2578,7 +2594,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): # Compute a new block ID. This new block ID must be consistent when this # method is called with the same (source_key, dest_structure) pair unique_data = "{}:{}:{}".format( - unicode(hashable_source_id).encode("utf-8"), + six.text_type(hashable_source_id).encode("utf-8"), block_key.id, new_parent_block_key.id, ) @@ -2618,7 +2634,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): # Setting it to the source_block_info structure version here breaks split_draft's has_changes() method. new_block_info.edit_info.edited_by = user_id new_block_info.edit_info.edited_on = datetime.datetime.now(UTC) - new_block_info.edit_info.original_usage = unicode(usage_key.replace(branch=None, version_guid=None)) + new_block_info.edit_info.original_usage = six.text_type(usage_key.replace(branch=None, version_guid=None)) new_block_info.edit_info.original_usage_version = source_block_info.edit_info.update_version dest_structure['blocks'][new_block_key] = new_block_info @@ -2711,7 +2727,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): """ # create mapping from each child's key to its parents' keys child_parent_map = defaultdict(set) - for block_key, block_data in blocks.iteritems(): + for block_key, block_data in six.iteritems(blocks): for child in block_data.fields.get('children', []): child_parent_map[BlockKey(*child)].add(block_key) @@ -2890,7 +2906,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): course_key, asset_metadata_list, course_assets, user_id, import_only ) - for asset_type, assets in assets_by_type.iteritems(): + for asset_type, assets in six.iteritems(assets_by_type): new_structure['assets'][asset_type] = list(assets) # update index if appropriate and structures @@ -2999,7 +3015,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): original_structure = self._lookup_course(course_locator).structure index_entry = self._get_index_if_valid(course_locator) new_structure = self.version_structure(course_locator, original_structure, user_id) - for block in new_structure['blocks'].itervalues(): + for block in six.itervalues(new_structure['blocks']): if 'children' in block.fields: block.fields['children'] = [ block_id for block_id in block.fields['children'] @@ -3042,7 +3058,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): xblock_class = self.mixologist.mix(xblock_class) # Make a shallow copy, so that we aren't manipulating a cached field dictionary output_fields = dict(jsonfields) - for field_name, value in output_fields.iteritems(): + for field_name, value in six.iteritems(output_fields): if value: try: field = xblock_class.fields.get(field_name) @@ -3053,7 +3069,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): elif isinstance(field, ReferenceList): output_fields[field_name] = [robust_usage_key(ele) for ele in value] elif isinstance(field, ReferenceValueDict): - for key, subvalue in value.iteritems(): + for key, subvalue in six.iteritems(value): value[key] = robust_usage_key(subvalue) return output_fields @@ -3102,7 +3118,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): :param fields: a dictionary of fields and values usually only those explicitly set and already ready for persisting (e.g., references converted to block_ids) """ - for field_name, field_value in fields.iteritems(): + for field_name, field_value in six.iteritems(fields): if field_name in self.SEARCH_TARGET_DICT: index_entry.setdefault('search_targets', {})[field_name] = field_value @@ -3126,7 +3142,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): # explicitly_set_fields_by_scope converts to json; so, avoiding it # the existing partition_fields_by_scope works on a dict not an xblock result = defaultdict(dict) - for field in xblock.fields.itervalues(): + for field in six.itervalues(xblock.fields): if field.is_set_on(xblock): result[field.scope][field.name] = field.read_from(xblock) return result @@ -3149,13 +3165,13 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): Handle client possibly setting field to strings rather than keys to get the block_id """ # perhaps replace by fixing the views or Field Reference*.from_json to return a Key - if isinstance(reference, basestring): + if isinstance(reference, six.string_types): reference = BlockUsageLocator.from_string(reference) elif isinstance(reference, BlockKey): return reference return BlockKey.from_usage_key(reference) - for field_name, value in fields.iteritems(): + for field_name, value in six.iteritems(fields): if value is not None: if isinstance(xblock_class.fields[field_name], Reference): fields[field_name] = reference_block_id(value) @@ -3164,7 +3180,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): reference_block_id(ele) for ele in value ] elif isinstance(xblock_class.fields[field_name], ReferenceValueDict): - for key, subvalue in value.iteritems(): + for key, subvalue in six.iteritems(value): value[key] = reference_block_id(subvalue) # should this recurse down dicts and lists just in case they contain datetime? elif not isinstance(value, datetime.datetime): # don't convert datetimes! @@ -3207,7 +3223,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): """ return [ parent_block_key - for parent_block_key, value in structure['blocks'].iteritems() + for parent_block_key, value in six.iteritems(structure['blocks']) if block_key in value.fields.get('children', []) ] @@ -3279,7 +3295,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): block_defaults=new_block.defaults ) # Extend the block's new edit_info with any extra edit_info fields from the source (e.g. original_usage): - for key, val in new_block.edit_info.to_storable().iteritems(): + for key, val in six.iteritems(new_block.edit_info.to_storable()): if getattr(destination_block.edit_info, key) is None: setattr(destination_block.edit_info, key, val) @@ -3386,7 +3402,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): result_list.append(aside) if tmp_new_asides_data: - for _, asd in tmp_new_asides_data.iteritems(): + for _, asd in six.iteritems(tmp_new_asides_data): result_list.append(asd) updated = True diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py index dd192ee76a..7ddc7c170f 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py @@ -2,16 +2,21 @@ Module for the dual-branch fall-back Draft->Published Versioning ModuleStore """ -from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore, EXCLUDE_ALL +from __future__ import absolute_import + +from contracts import contract +from opaque_keys.edx.locator import CourseLocator, LibraryLocator, LibraryUsageLocator + from xmodule.exceptions import InvalidVersionError from xmodule.modulestore import ModuleStoreEnum -from xmodule.modulestore.exceptions import InsufficientSpecificationError, ItemNotFoundError from xmodule.modulestore.draft_and_published import ( - ModuleStoreDraftAndPublished, DIRECT_ONLY_CATEGORIES, UnsupportedRevisionError + DIRECT_ONLY_CATEGORIES, + ModuleStoreDraftAndPublished, + UnsupportedRevisionError ) -from opaque_keys.edx.locator import CourseLocator, LibraryLocator, LibraryUsageLocator +from xmodule.modulestore.exceptions import InsufficientSpecificationError, ItemNotFoundError from xmodule.modulestore.split_mongo import BlockKey -from contracts import contract +from xmodule.modulestore.split_mongo.split import EXCLUDE_ALL, SplitMongoModuleStore class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPublished): diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/split_mongo_kvs.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/split_mongo_kvs.py index 417f383d24..5e3db544b3 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/split_mongo_kvs.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/split_mongo_kvs.py @@ -1,12 +1,18 @@ +from __future__ import absolute_import + import copy -from contracts import contract, new_contract -from xblock.fields import Scope from collections import namedtuple -from xblock.exceptions import InvalidScopeError -from .definition_lazy_loader import DefinitionLazyLoader -from xmodule.modulestore.inheritance import InheritanceKeyValueStore + +import six +from contracts import contract, new_contract from opaque_keys.edx.locator import BlockUsageLocator from xblock.core import XBlockAside +from xblock.exceptions import InvalidScopeError +from xblock.fields import Scope + +from xmodule.modulestore.inheritance import InheritanceKeyValueStore + +from .definition_lazy_loader import DefinitionLazyLoader # id is a BlockUsageLocator, def_id is the definition's guid SplitMongoKVSid = namedtuple('SplitMongoKVSid', 'id, def_id') @@ -186,7 +192,7 @@ class SplitMongoKVS(InheritanceKeyValueStore): aside_fields_p = persisted_definition.get('aside_fields') if aside_fields_p: aside_fields = self._definition.field_converter(aside_fields_p) - for aside_type, fields in aside_fields.iteritems(): + for aside_type, fields in six.iteritems(aside_fields): self.aside_fields.setdefault(aside_type, {}).update(fields) # do we want to cache any of the edit_info? self._definition = None # already loaded