Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

INCR-412 python3 compatibility

Browse Source
This commit is contained in:
Ayub khan
2019-07-11 15:32:08 +05:00
parent 6db7a08268
commit 0698c10d29
8 changed files with 116 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
common/lib/xmodule/xmodule/modulestore/__init__.py
View File

@@ -3,32 +3,35 @@ This module provides an abstraction for working with XModuleDescriptors
that are stored in a database an accessible using their Location as an identifier
"""
from __future__ import absolute_import
import datetime
import logging
import re
import datetime
from pytz import UTC
import threading
from abc import ABCMeta, abstractmethod
from collections import defaultdict
from contextlib import contextmanager
import threading
from operator import itemgetter
from sortedcontainers import SortedKeyList
from abc import ABCMeta, abstractmethod
import six
from contracts import contract, new_contract
from xblock.plugin import default_select
from .exceptions import InvalidLocationError, InsufficientSpecificationError
from xmodule.errortracker import make_error_tracker
from xmodule.assetstore import AssetMetadata
from opaque_keys.edx.keys import CourseKey, AssetKey
from opaque_keys.edx.keys import AssetKey, CourseKey
from opaque_keys.edx.locations import Location # For import backwards compatibility
from xblock.runtime import Mixologist
from pytz import UTC
from six.moves import range
from sortedcontainers import SortedKeyList
from xblock.core import XBlock
from xblock.plugin import default_select
from xblock.runtime import Mixologist
# The below import is not used within this module, but ir is still needed becuase
# other modules are imorting EdxJSONEncoder from here
from openedx.core.lib.json_utils import EdxJSONEncoder # pylint: disable=unused-import
from xmodule.assetstore import AssetMetadata
from xmodule.errortracker import make_error_tracker
from .exceptions import InsufficientSpecificationError, InvalidLocationError
log = logging.getLogger('edx.modulestore')
@@ -207,7 +210,7 @@ class BulkOperationsMixin(object):
# Retrieve the bulk record based on matching org/course/run (possibly ignoring case)
if ignore_case:
for key, record in self._active_bulk_ops.records.iteritems():
for key, record in six.iteritems(self._active_bulk_ops.records):
# Shortcut: check basic equivalence for cases where org/course/run might be None.
if (key == course_key) or (
(key.org and key.org.lower() == course_key.org.lower()) and
@@ -223,7 +226,7 @@ class BulkOperationsMixin(object):
"""
Yield all active (CourseLocator, BulkOpsRecord) tuples.
"""
for course_key, record in self._active_bulk_ops.records.iteritems():
for course_key, record in six.iteritems(self._active_bulk_ops.records):
if record.active:
yield (course_key, record)
@@ -634,7 +637,7 @@ class ModuleStoreAssetBase(object):
if asset_type is None:
# Add assets of all types to the sorted list.
all_assets = SortedAssetList(iterable=[], key=key_func)
for asset_type, val in course_assets.iteritems():
for asset_type, val in six.iteritems(course_assets):
all_assets.update(val)
else:
# Add assets of a single type to the sorted list.
@@ -655,7 +658,7 @@ class ModuleStoreAssetBase(object):
end_idx = (num_assets - 1) - end_idx
ret_assets = []
for idx in xrange(start_idx, end_idx, step_incr):
for idx in range(start_idx, end_idx, step_incr):
raw_asset = all_assets[idx]
asset_key = course_key.make_asset_key(raw_asset['asset_type'], raw_asset['filename'])
new_asset = AssetMetadata(asset_key)
@@ -776,14 +779,12 @@ class ModuleStoreAssetWriteInterface(ModuleStoreAssetBase):
pass
class ModuleStoreRead(ModuleStoreAssetBase):
class ModuleStoreRead(six.with_metaclass(ABCMeta, ModuleStoreAssetBase)):
"""
An abstract interface for a database backend that stores XModuleDescriptor
instances and extends read-only functionality
"""
__metaclass__ = ABCMeta
@abstractmethod
def has_item(self, usage_key):
"""
@@ -877,7 +878,7 @@ class ModuleStoreRead(ModuleStoreAssetBase):
else:
return True, field
for key, criteria in qualifiers.iteritems():
for key, criteria in six.iteritems(qualifiers):
is_set, value = _is_set_on(key)
if isinstance(criteria, dict) and '$exists' in criteria and criteria['$exists'] == is_set:
continue
@@ -1033,14 +1034,12 @@ class ModuleStoreRead(ModuleStoreAssetBase):
pass
class ModuleStoreWrite(ModuleStoreRead, ModuleStoreAssetWriteInterface):
class ModuleStoreWrite(six.with_metaclass(ABCMeta, ModuleStoreRead, ModuleStoreAssetWriteInterface)):
"""
An abstract interface for a database backend that stores XModuleDescriptor
instances and extends both read and write functionality
"""
__metaclass__ = ABCMeta
@abstractmethod
def update_item(self, xblock, user_id, allow_not_found=False, force=False, **kwargs):
"""
@@ -1300,7 +1299,7 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
if fields is None:
return result
cls = self.mixologist.mix(XBlock.load_class(category, select=prefer_xmodules))
for field_name, value in fields.iteritems():
for field_name, value in six.iteritems(fields):
field = getattr(cls, field_name)
result[field.scope][field_name] = value
return result

2
common/lib/xmodule/xmodule/modulestore/draft.py
View File

@@ -5,4 +5,6 @@ This modulestore has been moved to xmodule.modulestore.mongo.draft
"""
# pylint: disable=unused-import
from __future__ import absolute_import
from xmodule.modulestore.mongo.draft import DraftModuleStore

18
common/lib/xmodule/xmodule/modulestore/mongoengine_fields.py
View File

@@ -2,12 +2,16 @@
Custom field types for mongoengine
"""
from __future__ import absolute_import
from types import NoneType
import mongoengine
from opaque_keys.edx.locations import Location
from opaque_keys.edx.keys import CourseKey, UsageKey
from six import text_type
import six
from opaque_keys.edx.keys import CourseKey, UsageKey
from opaque_keys.edx.locations import Location
from six import text_type
class CourseKeyField(mongoengine.StringField):
"""
@@ -34,7 +38,7 @@ class CourseKeyField(mongoengine.StringField):
"""
# calling super b/c it decodes utf (and doesn't have circularity of from_python)
course_key = super(CourseKeyField, self).to_python(course_key)
assert isinstance(course_key, (type(None), six.string_types, CourseKey))
assert isinstance(course_key, (NoneType, six.string_types, CourseKey))
if course_key == '':
return None
if isinstance(course_key, six.string_types):
@@ -43,7 +47,7 @@ class CourseKeyField(mongoengine.StringField):
return course_key
def validate(self, value):
assert isinstance(value, (type(None), six.string_types, CourseKey))
assert isinstance(value, (NoneType, six.string_types, CourseKey))
if isinstance(value, CourseKey):
return super(CourseKeyField, self).validate(text_type(value))
else:
@@ -70,7 +74,7 @@ class UsageKeyField(mongoengine.StringField):
"""
Deserialize to a UsageKey instance: for now it's a location missing the run
"""
assert isinstance(location, (type(None), six.string_types, UsageKey))
assert isinstance(location, (NoneType, six.string_types, UsageKey))
if location == '':
return None
if isinstance(location, six.string_types):
@@ -80,7 +84,7 @@ class UsageKeyField(mongoengine.StringField):
return location
def validate(self, value):
assert isinstance(value, (type(None), six.string_types, UsageKey))
assert isinstance(value, (NoneType, six.string_types, UsageKey))
if isinstance(value, UsageKey):
return super(UsageKeyField, self).validate(text_type(value))
else:

6
common/lib/xmodule/xmodule/modulestore/search.py
View File

@@ -1,7 +1,11 @@
''' useful functions for finding content and its position '''
from __future__ import absolute_import
from logging import getLogger
from .exceptions import (ItemNotFoundError, NoPathToItem)
from six.moves import range
from .exceptions import ItemNotFoundError, NoPathToItem
LOGGER = getLogger(__name__)

15
common/lib/xmodule/xmodule/modulestore/store_utilities.py
View File

@@ -1,8 +1,11 @@
import re
from __future__ import absolute_import
import logging
import re
import uuid
from collections import namedtuple
import uuid
import six
from xblock.core import XBlock
DETACHED_XBLOCK_TYPES = set(name for name, __ in XBlock.load_tagged_classes("detached"))
@@ -42,7 +45,7 @@ def rewrite_nonportable_content_links(source_course_id, dest_course_id, text):
# create a serialized template for what the id will look like in the source_course but with
# the block_id as a regex pattern
placeholder_id = uuid.uuid4().hex
asset_block_pattern = unicode(source_course_id.make_asset_key('asset', placeholder_id))
asset_block_pattern = six.text_type(source_course_id.make_asset_key('asset', placeholder_id))
asset_block_pattern = asset_block_pattern.replace(placeholder_id, r'(?P<block_id>.*?)')
try:
text = _prefix_only_url_replace_regex(asset_block_pattern).sub(portable_asset_link_subtitution, text)
@@ -50,11 +53,11 @@ def rewrite_nonportable_content_links(source_course_id, dest_course_id, text):
logging.warning("Error producing regex substitution %r for text = %r.\n\nError msg = %s", asset_block_pattern, text, str(exc))
placeholder_category = 'cat_{}'.format(uuid.uuid4().hex)
usage_block_pattern = unicode(source_course_id.make_usage_key(placeholder_category, placeholder_id))
usage_block_pattern = six.text_type(source_course_id.make_usage_key(placeholder_category, placeholder_id))
usage_block_pattern = usage_block_pattern.replace(placeholder_category, r'(?P<category>[^/+@]+)')
usage_block_pattern = usage_block_pattern.replace(placeholder_id, r'(?P<block_id>.*?)')
jump_to_link_base = u'/courses/{course_key_string}/jump_to/{usage_key_string}'.format(
course_key_string=unicode(source_course_id), usage_key_string=usage_block_pattern
course_key_string=six.text_type(source_course_id), usage_key_string=usage_block_pattern
)
try:
text = _prefix_only_url_replace_regex(jump_to_link_base).sub(portable_jump_to_link_substitution, text)
@@ -69,7 +72,7 @@ def rewrite_nonportable_content_links(source_course_id, dest_course_id, text):
#
if source_course_id != dest_course_id:
try:
generic_courseware_link_base = u'/courses/{}/'.format(unicode(source_course_id))
generic_courseware_link_base = u'/courses/{}/'.format(six.text_type(source_course_id))
text = re.sub(_prefix_only_url_replace_regex(generic_courseware_link_base), portable_asset_link_subtitution, text)
except Exception as exc: # pylint: disable=broad-except
logging.warning("Error producing regex substitution %r for text = %r.\n\nError msg = %s", source_course_id, text, str(exc))

60
common/lib/xmodule/xmodule/modulestore/xml.py
View File

@@ -1,3 +1,6 @@
from __future__ import absolute_import
import glob
import hashlib
import itertools
import json
@@ -5,37 +8,38 @@ import logging
import os
import re
import sys
import glob
from collections import defaultdict
from contextlib import contextmanager
from importlib import import_module
from io import BytesIO
import six
from fs.osfs import OSFS
from importlib import import_module
from lxml import etree
from path import Path as path
from contextlib import contextmanager
from lazy import lazy
from lxml import etree
from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator, LibraryLocator
from path import Path as path
from xblock.field_data import DictFieldData
from xblock.fields import ScopeIds
from xblock.runtime import DictKeyValueStore
from xmodule.error_module import ErrorDescriptor
from xmodule.errortracker import make_error_tracker, exc_info_to_str
from xmodule.errortracker import exc_info_to_str, make_error_tracker
from xmodule.mako_module import MakoDescriptorSystem
from xmodule.x_module import (
XMLParsingSystem, policy_key,
OpaqueKeyReader, AsideKeyGenerator, DEPRECATION_VSCOMPAT_EVENT
)
from xmodule.modulestore import COURSE_ROOT, LIBRARY_ROOT, ModuleStoreEnum, ModuleStoreReadBase
from xmodule.modulestore.xml_exporter import DEFAULT_CONTENT_FIELDS
from xmodule.modulestore import ModuleStoreEnum, ModuleStoreReadBase, LIBRARY_ROOT, COURSE_ROOT
from xmodule.tabs import CourseTabList
from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.locator import CourseLocator, LibraryLocator, BlockUsageLocator
from xblock.field_data import DictFieldData
from xblock.runtime import DictKeyValueStore
from xblock.fields import ScopeIds
from xmodule.x_module import (
DEPRECATION_VSCOMPAT_EVENT,
AsideKeyGenerator,
OpaqueKeyReader,
XMLParsingSystem,
policy_key
)
from .exceptions import ItemNotFoundError
from .inheritance import compute_inherited_metadata, inheriting_field_data, InheritanceKeyValueStore
from .inheritance import InheritanceKeyValueStore, compute_inherited_metadata, inheriting_field_data
edx_xml_parser = etree.XMLParser(dtd_validation=False, load_dtd=False,
remove_comments=True, remove_blank_text=True)
@@ -190,14 +194,14 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
msg = "Error loading from xml. %s"
log.warning(
msg,
unicode(err)[:200],
six.text_type(err)[:200],
# Normally, we don't want lots of exception traces in our logs from common
# content problems. But if you're debugging the xml loading code itself,
# uncomment the next line.
# exc_info=True
)
msg = msg % (unicode(err)[:200])
msg = msg % (six.text_type(err)[:200])
self.error_tracker(msg)
err_msg = msg + "\n" + exc_info_to_str(sys.exc_info())
@@ -277,7 +281,7 @@ class CourseLocationManager(OpaqueKeyReader, AsideKeyGenerator):
def create_definition(self, block_type, slug=None):
assert block_type is not None
if slug is None:
slug = 'autogen_{}_{}'.format(block_type, self.autogen_ids.next())
slug = 'autogen_{}_{}'.format(block_type, next(self.autogen_ids))
return self.course_id.make_usage_key(block_type, slug)
def get_definition_id(self, usage_id):
@@ -387,7 +391,7 @@ class XMLModuleStore(ModuleStoreReadBase):
course_descriptor = self.load_course(course_dir, course_ids, errorlog.tracker, target_course_id)
except Exception as exc: # pylint: disable=broad-except
msg = "ERROR: Failed to load courselike '{0}': {1}".format(
course_dir.encode("utf-8"), unicode(exc)
course_dir.encode("utf-8"), six.text_type(exc)
)
log.exception(msg)
errorlog.tracker(msg)
@@ -651,7 +655,7 @@ class XMLModuleStore(ModuleStoreReadBase):
try:
# get and update data field in xblock runtime
module = system.load_item(loc)
for key, value in data_content.iteritems():
for key, value in six.iteritems(data_content):
setattr(module, key, value)
module.save()
except ItemNotFoundError:
@@ -694,8 +698,8 @@ class XMLModuleStore(ModuleStoreReadBase):
self.modules[course_descriptor.id][module.scope_ids.usage_id] = module
except Exception as exc: # pylint: disable=broad-except
logging.exception("Failed to load %s. Skipping... \
Exception: %s", filepath, unicode(exc))
system.error_tracker("ERROR: " + unicode(exc))
Exception: %s", filepath, six.text_type(exc))
system.error_tracker("ERROR: " + six.text_type(exc))
def has_item(self, usage_key):
"""
@@ -769,7 +773,7 @@ class XMLModuleStore(ModuleStoreReadBase):
for fields in [settings, content, qualifiers]
)
for mod_loc, module in self.modules[course_id].iteritems():
for mod_loc, module in six.iteritems(self.modules[course_id]):
if _block_matches_all(mod_loc, module):
items.append(module)
@@ -796,7 +800,7 @@ class XMLModuleStore(ModuleStoreReadBase):
Returns a list of course descriptors. If there were errors on loading,
some of these may be ErrorDescriptors instead.
"""
return self.courses.values()
return list(self.courses.values())
def get_course_summaries(self, **kwargs):
"""

29
common/lib/xmodule/xmodule/modulestore/xml_exporter.py
View File

@@ -2,24 +2,27 @@
Methods for exporting course data to XML
"""
from __future__ import absolute_import
import logging
import os
from abc import abstractmethod
from six import text_type
from json import dumps
import lxml.etree
from xblock.fields import Scope, Reference, ReferenceList, ReferenceValueDict
import six
from fs.osfs import OSFS
from opaque_keys.edx.locator import CourseLocator, LibraryLocator
from six import text_type
from xblock.fields import Reference, ReferenceList, ReferenceValueDict, Scope
from xmodule.assetstore import AssetMetadata
from xmodule.contentstore.content import StaticContent
from xmodule.exceptions import NotFoundError
from xmodule.assetstore import AssetMetadata
from xmodule.modulestore import EdxJSONEncoder, ModuleStoreEnum
from xmodule.modulestore import LIBRARY_ROOT, EdxJSONEncoder, ModuleStoreEnum
from xmodule.modulestore.draft_and_published import DIRECT_ONLY_CATEGORIES
from xmodule.modulestore.inheritance import own_metadata
from xmodule.modulestore.store_utilities import draft_node_constructor, get_draft_subtree_roots
from xmodule.modulestore import LIBRARY_ROOT
from fs.osfs import OSFS
from json import dumps
import os
from xmodule.modulestore.draft_and_published import DIRECT_ONLY_CATEGORIES
from opaque_keys.edx.locator import CourseLocator, LibraryLocator
DRAFT_DIR = "drafts"
PUBLISHED_DIR = "published"
@@ -355,7 +358,7 @@ def adapt_references(subtree, destination_course_key, export_fs):
Map every reference in the subtree into destination_course_key and set it back into the xblock fields
"""
subtree.runtime.export_fs = export_fs # ensure everything knows where it's going!
for field_name, field in subtree.fields.iteritems():
for field_name, field in six.iteritems(subtree.fields):
if field.is_set_on(subtree):
if isinstance(field, Reference):
value = field.read_from(subtree)
@@ -372,7 +375,7 @@ def adapt_references(subtree, destination_course_key, export_fs):
elif isinstance(field, ReferenceValueDict):
field.write_to(
subtree, {
key: ele.map_into_course(destination_course_key) for key, ele in field.read_from(subtree).iteritems()
key: ele.map_into_course(destination_course_key) for key, ele in six.iteritems(field.read_from(subtree))
}
)

32
common/lib/xmodule/xmodule/modulestore/xml_importer.py
View File

@@ -20,7 +20,8 @@ Modulestore virtual | XML physical (draft, published)
(a, a) | (a, a) | (x, a) | (x, x) | (x, y) | (a, x)
(a, b) | (a, b) | (x, b) | (x, x) | (x, y) | (a, x)
"""
from __future__ import print_function
from __future__ import absolute_import, print_function
import json
import logging
import mimetypes
@@ -28,6 +29,7 @@ import os
import re
from abc import abstractmethod
import six
import xblock
from lxml import etree
from opaque_keys.edx.keys import UsageKey
@@ -95,7 +97,7 @@ class StaticContentImporter:
mimetypes.add_type('application/octet-stream', '.sjson')
mimetypes.add_type('application/octet-stream', '.srt')
self.mimetypes_list = mimetypes.types_map.values()
self.mimetypes_list = list(mimetypes.types_map.values())
def import_static_content_directory(self, content_subdir=DEFAULT_STATIC_CONTENT_SUBDIR, verbose=False):
remap_dict = {}
@@ -736,7 +738,7 @@ def _update_and_import_module(
Update all the module reference fields to the destination course id,
then import the module into the destination course.
"""
logging.debug(u'processing import of module %s...', unicode(module.location))
logging.debug(u'processing import of module %s...', six.text_type(module.location))
def _update_module_references(module, source_course_id, dest_course_id):
"""
@@ -756,7 +758,7 @@ def _update_and_import_module(
return reference
fields = {}
for field_name, field in module.fields.iteritems():
for field_name, field in six.iteritems(module.fields):
if field.scope != Scope.parent and field.is_set_on(module):
if isinstance(field, Reference):
value = field.read_from(module)
@@ -772,7 +774,7 @@ def _update_and_import_module(
fields[field_name] = {
key: _convert_ref_fields_to_new_namespace(reference)
for key, reference
in reference_dict.iteritems()
in six.iteritems(reference_dict)
}
elif field_name == 'xml_attributes':
value = field.read_from(module)
@@ -946,7 +948,7 @@ def _import_course_draft(
index = index_in_children_list(descriptor)
parent_url = get_parent_url(descriptor, xml)
draft_url = unicode(descriptor.location)
draft_url = six.text_type(descriptor.location)
draft = draft_node_constructor(
module=descriptor, url=draft_url, parent_url=parent_url, index=index
@@ -995,7 +997,7 @@ def check_module_metadata_editability(module):
print(
": found non-editable metadata on {url}. "
"These metadata keys are not supported = {keys}".format(
url=unicode(module.location), keys=illegal_keys
url=six.text_type(module.location), keys=illegal_keys
)
)
@@ -1041,7 +1043,7 @@ def create_xml_attributes(module, xml):
Make up for modules which don't define xml_attributes by creating them here and populating
"""
xml_attrs = {}
for attr, val in xml.attrib.iteritems():
for attr, val in six.iteritems(xml.attrib):
if attr not in module.fields:
# translate obsolete attr
if attr == 'parent_sequential_url':
@@ -1068,7 +1070,7 @@ def validate_category_hierarchy(
parents = []
# get all modules of parent_category
for module in module_store.modules[course_id].itervalues():
for module in six.itervalues(module_store.modules[course_id]):
if module.location.block_type == parent_category:
parents.append(module)
@@ -1124,7 +1126,7 @@ def validate_course_policy(module_store, course_id):
"""
# is there a reliable way to get the module location just given the course_id?
warn_cnt = 0
for module in module_store.modules[course_id].itervalues():
for module in six.itervalues(module_store.modules[course_id]):
if module.location.block_type == 'course':
if not module._field_data.has(module, 'rerandomize'):
warn_cnt += 1
@@ -1166,7 +1168,7 @@ def perform_xlint(
warn_cnt += _warn_cnt
# first count all errors and warnings as part of the XMLModuleStore import
for err_log in module_store._course_errors.itervalues(): # pylint: disable=protected-access
for err_log in six.itervalues(module_store._course_errors): # pylint: disable=protected-access
for err_log_entry in err_log.errors:
msg = err_log_entry[0]
if msg.startswith('ERROR:'):
@@ -1175,7 +1177,7 @@ def perform_xlint(
warn_cnt += 1
# then count outright all courses that failed to load at all
for err_log in module_store.errored_courses.itervalues():
for err_log in six.itervalues(module_store.errored_courses):
for err_log_entry in err_log.errors:
msg = err_log_entry[0]
print(msg)
@@ -1266,9 +1268,9 @@ def _update_module_location(module, new_location):
rekey_fields = []
else:
rekey_fields = (
module.get_explicitly_set_fields_by_scope(Scope.content).keys() +
module.get_explicitly_set_fields_by_scope(Scope.settings).keys() +
module.get_explicitly_set_fields_by_scope(Scope.children).keys()
list(module.get_explicitly_set_fields_by_scope(Scope.content).keys()) +
list(module.get_explicitly_set_fields_by_scope(Scope.settings).keys()) +
list(module.get_explicitly_set_fields_by_scope(Scope.children).keys())
)
module.location = new_location