Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

BOM-2378 (C) : pyupgrade in common/lib/xmodule (#26736)

Browse Source 
* refactor: pyupgrade in common/lib/xmodule

Co-authored-by: Usama Sadiq <usama.sadiq@arbisoft.com>
This commit is contained in:
M. Zulqarnain
2021-03-10 17:56:39 +05:00
committed by GitHub
parent 8e8420dd46
commit b4614d75f0
50 changed files with 632 additions and 694 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
common/lib/xmodule/xmodule/annotatable_module.py
View File

@@ -46,7 +46,7 @@ class AnnotatableBlock(
data = String(
help=_("XML data for the annotation"),
scope=Scope.content,
default=textwrap.dedent(HTML(u"""
default=textwrap.dedent(HTML("""
<annotatable>
<instructions>
<p>Enter your (optional) instructions for the exercise in HTML format.</p>

5
common/lib/xmodule/xmodule/annotator_mixin.py
View File

@@ -3,12 +3,11 @@ Annotations Tool Mixin
This file contains global variables and functions used in the various Annotation Tools.
"""
from html.parser import HTMLParser
from os.path import basename, splitext
from urllib.parse import urlparse
from lxml import etree
from six.moves.html_parser import HTMLParser
from six.moves.urllib.parse import urlparse
def get_instructions(xmltree):

22
common/lib/xmodule/xmodule/assetstore/__init__.py
View File

@@ -8,7 +8,6 @@ from datetime import datetime
import dateutil.parser
import pytz
import six
from contracts import contract, new_contract
from lxml import etree
from opaque_keys.edx.keys import AssetKey, CourseKey
@@ -16,16 +15,13 @@ from opaque_keys.edx.keys import AssetKey, CourseKey
new_contract('AssetKey', AssetKey)
new_contract('CourseKey', CourseKey)
new_contract('datetime', datetime)
new_contract('basestring', six.string_types[0])
if six.PY2:
new_contract('long', long) # lint-amnesty, pylint: disable=undefined-variable
else:
new_contract('long', int)
new_contract('basestring', (str,)[0])
new_contract('long', int)
new_contract('AssetElement', lambda x: isinstance(x, etree._Element) and x.tag == "asset") # pylint: disable=protected-access
new_contract('AssetsElement', lambda x: isinstance(x, etree._Element) and x.tag == "assets") # pylint: disable=protected-access
class AssetMetadata(object):
class AssetMetadata:
"""
Stores the metadata associated with a particular course asset. The asset metadata gets stored
in the modulestore.
@@ -50,10 +46,10 @@ class AssetMetadata(object):
ASSET_XML_TAG = 'asset'
# Top-level directory name in exported course XML which holds asset metadata.
EXPORTED_ASSET_DIR = u'assets'
EXPORTED_ASSET_DIR = 'assets'
# Filename of all asset metadata exported as XML.
EXPORTED_ASSET_FILENAME = u'assets.xml'
EXPORTED_ASSET_FILENAME = 'assets.xml'
@contract(asset_id='AssetKey',
pathname='str|None', internal_name='str|None',
@@ -128,7 +124,7 @@ class AssetMetadata(object):
Arguments:
attr_dict: Prop, val dictionary of all attributes to set.
"""
for attr, val in six.iteritems(attr_dict):
for attr, val in attr_dict.items():
if attr in self.ATTRS_ALLOWED_TO_UPDATE:
setattr(self, attr, val)
else:
@@ -238,7 +234,7 @@ class AssetMetadata(object):
elif isinstance(value, dict):
value = json.dumps(value)
else:
value = six.text_type(value)
value = str(value)
child.text = value
@staticmethod
@@ -253,7 +249,7 @@ class AssetMetadata(object):
asset.to_xml(asset_node)
class CourseAssetsFromStorage(object):
class CourseAssetsFromStorage:
"""
Wrapper class for asset metadata lists returned from modulestore storage.
"""
@@ -304,7 +300,7 @@ class CourseAssetsFromStorage(object):
"""
Iterates over the items of the asset dict.
"""
return six.iteritems(self.asset_md)
return self.asset_md.items()
def items(self):
"""

2
common/lib/xmodule/xmodule/assetstore/assetmgr.py
View File

@@ -43,7 +43,7 @@ class AssetMetadataFoundTemporary(AssetException):
pass # lint-amnesty, pylint: disable=unnecessary-pass
class AssetManager(object):
class AssetManager:
"""
Manager for saving/loading course assets.
"""

3
common/lib/xmodule/xmodule/assetstore/tests/test_asset_xml.py
View File

@@ -10,7 +10,6 @@ from contracts import ContractNotRespected
from lxml import etree
from opaque_keys.edx.locator import CourseLocator
from path import Path as path
from six.moves import zip
from xmodule.assetstore import AssetMetadata
from xmodule.modulestore.tests.test_assetstore import AssetStoreTestData
@@ -22,7 +21,7 @@ class TestAssetXml(unittest.TestCase):
"""
def setUp(self):
super(TestAssetXml, self).setUp() # lint-amnesty, pylint: disable=super-with-arguments
super().setUp()
xsd_filename = "assets.xsd"

100
common/lib/xmodule/xmodule/capa_base.py
View File

@@ -12,13 +12,11 @@ import struct
import sys
import traceback
import six
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.utils.encoding import smart_text
from django.utils.functional import cached_property
from pytz import utc
from six import text_type
from xblock.fields import Boolean, Dict, Float, Integer, Scope, String, XMLString
from xblock.scorable import ScorableXBlockMixin, Score
@@ -51,7 +49,7 @@ except ImproperlyConfigured:
FEATURES = {}
class SHOWANSWER(object):
class SHOWANSWER:
"""
Constants for when to show answer
"""
@@ -69,7 +67,7 @@ class SHOWANSWER(object):
ATTEMPTED_NO_PAST_DUE = "attempted_no_past_due"
class RANDOMIZATION(object):
class RANDOMIZATION:
"""
Constants for problem randomization
"""
@@ -88,8 +86,8 @@ def randomization_bin(seed, problem_id):
we'll combine the system's per-student seed with the problem id in picking the bin.
"""
r_hash = hashlib.sha1()
r_hash.update(six.b(str(seed)))
r_hash.update(six.b(str(problem_id)))
r_hash.update(str(seed).encode())
r_hash.update(str(problem_id).encode())
# get the first few digits of the hash, convert to an int, then mod.
return int(r_hash.hexdigest()[:7], 16) % NUM_RANDOMIZATION_BINS
@@ -117,11 +115,11 @@ class ComplexEncoder(json.JSONEncoder):
Print a nicely formatted complex number, or default to the JSON encoder
"""
if isinstance(obj, complex):
return u"{real:.7g}{imag:+.7g}*j".format(real=obj.real, imag=obj.imag)
return f"{obj.real:.7g}{obj.imag:+.7g}*j"
return json.JSONEncoder.default(self, obj)
class CapaFields(object):
class CapaFields:
"""
Define the possible fields for a Capa problem
"""
@@ -298,9 +296,9 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
try:
lcp = self.new_lcp(self.get_state_for_lcp())
except Exception as err: # pylint: disable=broad-except
msg = u'cannot create LoncapaProblem {loc}: {err}'.format(
loc=text_type(self.location), err=err)
six.reraise(Exception, Exception(msg), sys.exc_info()[2])
msg = 'cannot create LoncapaProblem {loc}: {err}'.format(
loc=str(self.location), err=err)
raise Exception(msg).with_traceback(sys.exc_info()[2])
if self.score is None:
self.set_score(self.score_from_lcp(lcp))
@@ -316,7 +314,7 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
self.seed = 1
elif self.rerandomize == RANDOMIZATION.PER_STUDENT and hasattr(self.runtime, 'seed'):
# see comment on randomization_bin
self.seed = randomization_bin(self.runtime.seed, six.text_type(self.location).encode('utf-8'))
self.seed = randomization_bin(self.runtime.seed, str(self.location).encode('utf-8'))
else:
self.seed = struct.unpack('i', os.urandom(4))[0]
@@ -437,7 +435,7 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
return self.runtime.render_template('problem_ajax.html', {
'element_id': self.location.html_id(),
'id': text_type(self.location),
'id': str(self.location),
'ajax_url': self.ajax_url,
'current_score': curr_score,
'total_possible': total_possible,
@@ -447,16 +445,16 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
})
def handle_fatal_lcp_error(self, error): # lint-amnesty, pylint: disable=missing-function-docstring
log.exception(u"LcpFatalError Encountered for {block}".format(block=str(self.location)))
log.exception("LcpFatalError Encountered for {block}".format(block=str(self.location)))
if error:
return(
HTML(u'<p>Error formatting HTML for problem:</p><p><pre style="color:red">{msg}</pre></p>').format(
msg=text_type(error))
HTML('<p>Error formatting HTML for problem:</p><p><pre style="color:red">{msg}</pre></p>').format(
msg=str(error))
)
else:
return HTML(
u'<p>Could not format HTML for problem. '
u'Contact course staff in the discussion forum for assistance.</p>'
'<p>Could not format HTML for problem. '
'Contact course staff in the discussion forum for assistance.</p>'
)
def submit_button_name(self):
@@ -565,24 +563,24 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
`err` is the Exception encountered while rendering the problem HTML.
"""
problem_display_name = self.display_name_with_default
problem_location = text_type(self.location)
problem_location = str(self.location)
log.exception(
u"ProblemGetHtmlError: %r, %r, %s",
"ProblemGetHtmlError: %r, %r, %s",
problem_display_name,
problem_location,
text_type(err)
str(err)
)
# TODO (vshnayder): another switch on DEBUG.
if self.runtime.DEBUG:
msg = HTML(
u'[courseware.capa.capa_module] '
u'Failed to generate HTML for problem {url}'
'[courseware.capa.capa_module] '
'Failed to generate HTML for problem {url}'
).format(
url=text_type(self.location)
url=str(self.location)
)
msg += HTML(u'<p>Error:</p><p><pre>{msg}</pre></p>').format(msg=text_type(err))
msg += HTML(u'<p><pre>{tb}</pre></p>').format(tb=traceback.format_exc())
msg += HTML('<p>Error:</p><p><pre>{msg}</pre></p>').format(msg=str(err))
msg += HTML('<p><pre>{tb}</pre></p>').format(tb=traceback.format_exc())
html = msg
else:
@@ -630,10 +628,10 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
except Exception as error:
# Couldn't do it. Give up.
log.exception(
u"ProblemGetHtmlError: Unable to generate html from LoncapaProblem: %r, %r, %s",
"ProblemGetHtmlError: Unable to generate html from LoncapaProblem: %r, %r, %s",
problem_display_name,
problem_location,
text_type(error)
str(error)
)
raise
@@ -696,7 +694,7 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
# Log this demand-hint request. Note that this only logs the last hint requested (although now
# all previously shown hints are still displayed).
event_info = dict()
event_info['module_id'] = text_type(self.location)
event_info['module_id'] = str(self.location)
event_info['hint_index'] = hint_index
event_info['hint_len'] = len(demand_hints)
event_info['hint_text'] = get_inner_html_from_xpath(demand_hints[hint_index])
@@ -759,12 +757,12 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
save_message = None
if self.has_saved_answers:
save_message = _(
u"Your answers were previously saved. Click '{button_name}' to grade them."
"Your answers were previously saved. Click '{button_name}' to grade them."
).format(button_name=self.submit_button_name())
context = {
'problem': content,
'id': text_type(self.location),
'id': str(self.location),
'short_id': self.location.html_id(),
'submit_button': submit_button,
'submit_button_submitting': submit_button_submitting,
@@ -786,7 +784,7 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
html = self.runtime.render_template('problem.html', context)
if encapsulate:
html = HTML(u'<div id="problem_{id}" class="problem" data-url="{ajax_url}">{html}</div>').format(
html = HTML('<div id="problem_{id}" class="problem" data-url="{ajax_url}">{html}</div>').format(
id=self.location.html_id(), ajax_url=self.ajax_url, html=HTML(html)
)
@@ -876,7 +874,7 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
'correcthint', 'regexphint', 'additional_answer', 'stringequalhint', 'compoundhint',
'stringequalhint']
for tag in tags:
html = re.sub(r'<%s.*?>.*?</%s>' % (tag, tag), '', html, flags=re.DOTALL) # xss-lint: disable=python-interpolate-html # lint-amnesty, pylint: disable=line-too-long
html = re.sub(fr'<{tag}.*?>.*?</{tag}>', '', html, flags=re.DOTALL) # xss-lint: disable=python-interpolate-html # lint-amnesty, pylint: disable=line-too-long
# Some of these tags span multiple lines
# Note: could probably speed this up by calling sub() once with a big regex
# vs. simply calling sub() many times as we have here.
@@ -1064,7 +1062,7 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
(and also screen reader text).
"""
event_info = dict()
event_info['problem_id'] = text_type(self.location)
event_info['problem_id'] = str(self.location)
self.track_function_unmask('showanswer', event_info)
if not self.answer_available(): # lint-amnesty, pylint: disable=no-else-raise
raise NotFoundError('Answer is not available')
@@ -1084,7 +1082,7 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
answer_content = self.runtime.replace_jump_to_id_urls(answer_content)
new_answer = {answer_id: answer_content}
except TypeError:
log.debug(u'Unable to perform URL substitution on answers[%s]: %s',
log.debug('Unable to perform URL substitution on answers[%s]: %s',
answer_id, answers[answer_id])
new_answer = {answer_id: answers[answer_id]}
new_answers.update(new_answer)
@@ -1156,7 +1154,7 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
# will return (key, '', '')
# We detect this and raise an error
if not name: # lint-amnesty, pylint: disable=no-else-raise
raise ValueError(u"{key} must contain at least one underscore".format(key=key))
raise ValueError(f"{key} must contain at least one underscore")
else:
# This allows for answers which require more than one value for
@@ -1177,7 +1175,7 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
# If the submission wasn't deserializable, raise an error.
except(KeyError, ValueError):
raise ValueError( # lint-amnesty, pylint: disable=raise-missing-from
u"Invalid submission: {val} for {key}".format(val=data[key], key=key)
"Invalid submission: {val} for {key}".format(val=data[key], key=key)
)
else:
val = data[key]
@@ -1185,7 +1183,7 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
# If the name already exists, then we don't want
# to override it. Raise an error instead
if name in answers: # lint-amnesty, pylint: disable=no-else-raise
raise ValueError(u"Key {name} already exists in answers dict".format(name=name))
raise ValueError(f"Key {name} already exists in answers dict")
else:
answers[name] = val
@@ -1220,14 +1218,14 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
"""
event_info = dict()
event_info['state'] = self.lcp.get_state()
event_info['problem_id'] = text_type(self.location)
event_info['problem_id'] = str(self.location)
self.lcp.has_saved_answers = False
answers = self.make_dict_of_responses(data)
answers_without_files = convert_files_to_filenames(answers)
event_info['answers'] = answers_without_files
metric_name = u'capa.check_problem.{}'.format # lint-amnesty, pylint: disable=unused-variable
metric_name = 'capa.check_problem.{}'.format # lint-amnesty, pylint: disable=unused-variable
# Can override current time
current_time = datetime.datetime.now(utc)
if override_time is not False:
@@ -1239,7 +1237,7 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
if self.closed():
log.error(
'ProblemClosedError: Problem %s, close date: %s, due:%s, is_past_due: %s, attempts: %s/%s,',
text_type(self.location),
str(self.location),
self.close_date,
self.due,
self.is_past_due(),
@@ -1263,7 +1261,7 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
waittime_between_requests = self.runtime.xqueue['waittime']
if (current_time - prev_submit_time).total_seconds() < waittime_between_requests:
msg = _(u"You must wait at least {wait} seconds between submissions.").format(
msg = _("You must wait at least {wait} seconds between submissions.").format(
wait=waittime_between_requests)
return {'success': msg, 'html': ''}
@@ -1272,7 +1270,7 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
seconds_since_submission = (current_time - self.last_submission_time).total_seconds()
if seconds_since_submission < self.submission_wait_seconds:
remaining_secs = int(self.submission_wait_seconds - seconds_since_submission)
msg = _(u'You must wait at least {wait_secs} between submissions. {remaining_secs} remaining.').format(
msg = _('You must wait at least {wait_secs} between submissions. {remaining_secs} remaining.').format(
wait_secs=self.pretty_print_seconds(self.submission_wait_seconds),
remaining_secs=self.pretty_print_seconds(remaining_secs))
return {
@@ -1308,7 +1306,7 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
# the full exception, including traceback,
# in the response
if self.runtime.user_is_staff:
msg = u"Staff debug info: {tb}".format(tb=traceback.format_exc())
msg = f"Staff debug info: {traceback.format_exc()}"
# Otherwise, display just an error message,
# without a stack trace
@@ -1328,8 +1326,8 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
self.set_score(self.score_from_lcp(self.lcp))
if self.runtime.DEBUG:
msg = u"Error checking problem: {}".format(text_type(err))
msg += u'\nTraceback:\n{}'.format(traceback.format_exc())
msg = "Error checking problem: {}".format(str(err))
msg += f'\nTraceback:\n{traceback.format_exc()}'
return {'success': msg}
raise
published_grade = self.publish_grade()
@@ -1481,14 +1479,14 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
strings ''.
"""
input_metadata = {}
for input_id, internal_answer in six.iteritems(answers):
for input_id, internal_answer in answers.items():
answer_input = self.lcp.inputs.get(input_id)
if answer_input is None:
log.warning('Input id %s is not mapped to an input type.', input_id)
answer_response = None
for responder in six.itervalues(self.lcp.responders):
for responder in self.lcp.responders.values():
if input_id in responder.answer_ids:
answer_response = responder
@@ -1533,7 +1531,7 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
"""
event_info = dict()
event_info['state'] = self.lcp.get_state()
event_info['problem_id'] = text_type(self.location)
event_info['problem_id'] = str(self.location)
answers = self.make_dict_of_responses(data)
event_info['answers'] = answers
@@ -1593,7 +1591,7 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
"""
event_info = dict()
event_info['old_state'] = self.lcp.get_state()
event_info['problem_id'] = text_type(self.location)
event_info['problem_id'] = str(self.location)
_ = self.runtime.service(self, "i18n").ugettext
if self.closed():
@@ -1658,7 +1656,7 @@ class CapaMixin(ScorableXBlockMixin, CapaFields):
Returns the error messages for exceptions occurring while performing
the rescoring, rather than throwing them.
"""
event_info = {'state': self.lcp.get_state(), 'problem_id': text_type(self.location)}
event_info = {'state': self.lcp.get_state(), 'problem_id': str(self.location)}
_ = self.runtime.service(self, "i18n").ugettext

23
common/lib/xmodule/xmodule/capa_module.py
View File

@@ -6,7 +6,6 @@ import logging
import re
import sys
import six
from bleach.sanitizer import Cleaner
from lxml import etree
from pkg_resources import resource_string
@@ -96,7 +95,7 @@ class ProblemBlock(
}
def bind_for_student(self, *args, **kwargs): # lint-amnesty, pylint: disable=signature-differs
super(ProblemBlock, self).bind_for_student(*args, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
super().bind_for_student(*args, **kwargs)
# Capa was an XModule. When bind_for_student() was called on it with a new runtime, a new CapaModule object
# was initialized when XModuleDescriptor._xmodule() was called next. self.lcp was constructed in CapaModule
@@ -132,7 +131,7 @@ class ProblemBlock(
return self.student_view(context)
else:
# Show a message that this content requires users to login/enroll.
return super(ProblemBlock, self).public_view(context) # lint-amnesty, pylint: disable=super-with-arguments
return super().public_view(context)
def author_view(self, context):
"""
@@ -189,7 +188,7 @@ class ProblemBlock(
)
if dispatch not in handlers:
return 'Error: {} is not a known capa action'.format(dispatch)
return f'Error: {dispatch} is not a known capa action'
before = self.get_progress()
before_attempts = self.attempts
@@ -197,7 +196,7 @@ class ProblemBlock(
try:
result = handlers[dispatch](data)
except NotFoundError:
except NotFoundError as ex:
log.info(
"Unable to find data when dispatching %s to %s for user %s",
dispatch,
@@ -205,9 +204,9 @@ class ProblemBlock(
self.scope_ids.user_id
)
_, _, traceback_obj = sys.exc_info()
six.reraise(ProcessingError, ProcessingError(not_found_error_message), traceback_obj)
raise ProcessingError(not_found_error_message).with_traceback(traceback_obj) from ex
except Exception: # lint-amnesty, pylint: disable=broad-except
except Exception as ex: # lint-amnesty, pylint: disable=broad-except
log.exception(
"Unknown error when dispatching %s to %s for user %s",
dispatch,
@@ -215,7 +214,7 @@ class ProblemBlock(
self.scope_ids.user_id
)
_, _, traceback_obj = sys.exc_info()
six.reraise(ProcessingError, ProcessingError(generic_error_message), traceback_obj)
raise ProcessingError(generic_error_message).with_traceback(traceback_obj) from ex
after = self.get_progress()
after_attempts = self.attempts
@@ -275,7 +274,7 @@ class ProblemBlock(
@property
def non_editable_metadata_fields(self):
non_editable_fields = super(ProblemBlock, self).non_editable_metadata_fields # lint-amnesty, pylint: disable=super-with-arguments
non_editable_fields = super().non_editable_metadata_fields
non_editable_fields.extend([
ProblemBlock.due,
ProblemBlock.graceperiod,
@@ -292,7 +291,7 @@ class ProblemBlock(
try:
tree = etree.XML(self.data)
except etree.XMLSyntaxError:
log.error('Error parsing problem types from xml for capa module {}'.format(self.display_name))
log.error(f'Error parsing problem types from xml for capa module {self.display_name}')
return None # short-term fix to prevent errors (TNL-5057). Will be more properly addressed in TNL-4525.
registered_tags = responsetypes.registry.registered_tags()
return {node.tag for node in tree.iter() if node.tag in registered_tags}
@@ -301,7 +300,7 @@ class ProblemBlock(
"""
Return dictionary prepared with module content and type for indexing.
"""
xblock_body = super(ProblemBlock, self).index_dictionary() # lint-amnesty, pylint: disable=super-with-arguments
xblock_body = super().index_dictionary()
# Make optioninput's options index friendly by replacing the actual tag with the values
capa_content = re.sub(r'<optioninput options="\(([^"]+)\)".*?>\s*|\S*<\/optioninput>', r'\1', self.data)
@@ -384,7 +383,7 @@ class ProblemBlock(
minimal_init=True,
)
except responsetypes.LoncapaProblemError:
log.exception(u"LcpFatalError for block {} while getting max score".format(str(self.location)))
log.exception("LcpFatalError for block {} while getting max score".format(str(self.location)))
maximum_score = 0
else:
maximum_score = lcp.get_max_score()

22
common/lib/xmodule/xmodule/conditional_module.py
View File

@@ -6,12 +6,10 @@ ConditionalBlock is an XBlock which you can use for disabling some XBlocks by co
import json
import logging
import six
from lazy import lazy
from lxml import etree
from opaque_keys.edx.locator import BlockUsageLocator
from pkg_resources import resource_string
from six import text_type
from web_fragments.fragment import Fragment
from xblock.fields import ReferenceList, Scope, String
@@ -194,11 +192,11 @@ class ConditionalBlock(
"""
Create an instance of the Conditional XBlock.
"""
super(ConditionalBlock, self).__init__(*args, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(*args, **kwargs)
# Convert sources xml_attribute to a ReferenceList field type so Location/Locator
# substitution can be done.
if not self.sources_list:
if 'sources' in self.xml_attributes and isinstance(self.xml_attributes['sources'], six.string_types):
if 'sources' in self.xml_attributes and isinstance(self.xml_attributes['sources'], str):
self.sources_list = [
# TODO: it is not clear why we are replacing the run here (which actually is a no-op
# for old-style course locators. However, this is the implementation of
@@ -343,7 +341,7 @@ class ConditionalBlock(
children = []
show_tag_list = []
definition = {}
for conditional_attr in six.iterkeys(cls.conditions_map):
for conditional_attr in cls.conditions_map:
conditional_value = xml_object.get(conditional_attr)
if conditional_value is not None:
definition.update({
@@ -377,28 +375,28 @@ class ConditionalBlock(
self.runtime.add_block_as_child_node(child, xml_object)
if self.show_tag_list:
show_str = HTML(u'<show sources="{sources}" />').format(
sources=Text(';'.join(text_type(location) for location in self.show_tag_list)))
show_str = HTML('<show sources="{sources}" />').format(
sources=Text(';'.join(str(location) for location in self.show_tag_list)))
xml_object.append(etree.fromstring(show_str))
# Overwrite the original sources attribute with the value from sources_list, as
# Locations may have been changed to Locators.
stringified_sources_list = [text_type(loc) for loc in self.sources_list]
stringified_sources_list = [str(loc) for loc in self.sources_list]
self.xml_attributes['sources'] = ';'.join(stringified_sources_list)
self.xml_attributes[self.conditional_attr] = self.conditional_value
self.xml_attributes['message'] = self.conditional_message
return xml_object
def validate(self):
validation = super(ConditionalBlock, self).validate() # lint-amnesty, pylint: disable=super-with-arguments
validation = super().validate()
if not self.sources_list:
conditional_validation = StudioValidation(self.location)
conditional_validation.add(
StudioValidationMessage(
StudioValidationMessage.NOT_CONFIGURED,
_(u"This component has no source components configured yet."),
_("This component has no source components configured yet."),
action_class='edit-button',
action_label=_(u"Configure list of sources")
action_label=_("Configure list of sources")
)
)
validation = StudioValidation.copy(validation)
@@ -407,7 +405,7 @@ class ConditionalBlock(
@property
def non_editable_metadata_fields(self):
non_editable_fields = super(ConditionalBlock, self).non_editable_metadata_fields # lint-amnesty, pylint: disable=super-with-arguments
non_editable_fields = super().non_editable_metadata_fields
non_editable_fields.extend([
ConditionalBlock.due,
ConditionalBlock.show_tag_list,

29
common/lib/xmodule/xmodule/contentstore/content.py
View File

@@ -5,13 +5,12 @@ import os
import re
import uuid
from io import BytesIO
from urllib.parse import parse_qsl, quote_plus, urlencode, urlparse, urlunparse
import six
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import AssetKey, CourseKey
from opaque_keys.edx.locator import AssetLocator
from PIL import Image
from six.moves.urllib.parse import parse_qsl, quote_plus, urlencode, urlparse, urlunparse
from xmodule.assetstore.assetmgr import AssetManager
from xmodule.exceptions import NotFoundError
@@ -26,7 +25,7 @@ VERSIONED_ASSETS_PREFIX = '/assets/courseware'
VERSIONED_ASSETS_PATTERN = r'/assets/courseware/(v[\d]/)?([a-f0-9]{32})'
class StaticContent(object): # lint-amnesty, pylint: disable=missing-class-docstring
class StaticContent: # lint-amnesty, pylint: disable=missing-class-docstring
def __init__(self, loc, name, content_type, data, last_modified_at=None, thumbnail_location=None, import_path=None,
length=None, locked=False, content_digest=None):
self.location = loc
@@ -58,13 +57,13 @@ class StaticContent(object): # lint-amnesty, pylint: disable=missing-class-docs
name_root, ext = os.path.splitext(original_name)
if not ext == extension:
name_root = name_root + ext.replace(u'.', u'-')
name_root = name_root + ext.replace('.', '-')
if dimensions:
width, height = dimensions
name_root += "-{}x{}".format(width, height)
name_root += f"-{width}x{height}"
return u"{name_root}{extension}".format(
return "{name_root}{extension}".format(
name_root=name_root,
extension=extension,
)
@@ -118,7 +117,7 @@ class StaticContent(object): # lint-amnesty, pylint: disable=missing-class-docs
the actual /c4x/... path which the client needs to reference static content
"""
if location is not None:
return u"/static/{name}".format(name=location.block_id)
return f"/static/{location.block_id}"
else:
return None
@@ -180,9 +179,9 @@ class StaticContent(object): # lint-amnesty, pylint: disable=missing-class-docs
if StaticContent.is_versioned_asset_path(path):
return path
structure_version = 'v{}'.format(STATIC_CONTENT_VERSION)
structure_version = f'v{STATIC_CONTENT_VERSION}'
return u'{}/{}/{}{}'.format(VERSIONED_ASSETS_PREFIX, structure_version, version, path)
return f'{VERSIONED_ASSETS_PREFIX}/{structure_version}/{version}{path}'
@staticmethod
def get_asset_key_from_path(course_key, path):
@@ -301,7 +300,7 @@ class StaticContent(object): # lint-amnesty, pylint: disable=missing-class-docs
Legacy code expects the serialized asset key to start w/ a slash; so, do that in one place
:param asset_key:
"""
url = six.text_type(asset_key)
url = str(asset_key)
if not url.startswith('/'):
url = '/' + url # TODO - re-address this once LMS-11198 is tackled.
return url
@@ -310,9 +309,9 @@ class StaticContent(object): # lint-amnesty, pylint: disable=missing-class-docs
class StaticContentStream(StaticContent): # lint-amnesty, pylint: disable=missing-class-docstring
def __init__(self, loc, name, content_type, stream, last_modified_at=None, thumbnail_location=None, import_path=None, # lint-amnesty, pylint: disable=line-too-long
length=None, locked=False, content_digest=None):
super(StaticContentStream, self).__init__(loc, name, content_type, None, last_modified_at=last_modified_at, # lint-amnesty, pylint: disable=super-with-arguments
thumbnail_location=thumbnail_location, import_path=import_path,
length=length, locked=locked, content_digest=content_digest)
super().__init__(loc, name, content_type, None, last_modified_at=last_modified_at,
thumbnail_location=thumbnail_location, import_path=import_path,
length=length, locked=locked, content_digest=content_digest)
self._stream = stream
def stream_data(self):
@@ -349,7 +348,7 @@ class StaticContentStream(StaticContent): # lint-amnesty, pylint: disable=missi
return content
class ContentStore(object):
class ContentStore:
'''
Abstraction for all ContentStore providers (e.g. MongoDB)
'''
@@ -459,7 +458,7 @@ class ContentStore(object):
except Exception as exc: # pylint: disable=broad-except
# log and continue as thumbnails are generally considered as optional
logging.exception(
u"Failed to generate thumbnail for {0}. Exception: {1}".format(content.location, str(exc))
"Failed to generate thumbnail for {}. Exception: {}".format(content.location, str(exc))
)
return thumbnail_content, thumbnail_file_location

39
common/lib/xmodule/xmodule/contentstore/mongo.py
View File

@@ -8,7 +8,6 @@ import os
import gridfs
import pymongo
import six
from bson.son import SON
from fs.osfs import OSFS
from gridfs.errors import NoFile, FileExists
@@ -93,7 +92,7 @@ class MongoContentStore(ContentStore):
self.delete(content_id) # delete is a noop if the entry doesn't exist; so, don't waste time checking
thumbnail_location = content.thumbnail_location.to_deprecated_list_repr() if content.thumbnail_location else None # lint-amnesty, pylint: disable=line-too-long
with self.fs.new_file(_id=content_id, filename=six.text_type(content.location), content_type=content.content_type, # lint-amnesty, pylint: disable=line-too-long
with self.fs.new_file(_id=content_id, filename=str(content.location), content_type=content.content_type, # lint-amnesty, pylint: disable=line-too-long
displayname=content.name, content_son=content_son,
thumbnail_location=thumbnail_location,
import_path=content.import_path,
@@ -103,14 +102,14 @@ class MongoContentStore(ContentStore):
# It seems that this code thought that only some specific object would have the `__iter__` attribute
# but many more objects have this in python3 and shouldn't be using the chunking logic. For string and
# byte streams we write them directly to gridfs and convert them to byetarrys if necessary.
if hasattr(content.data, '__iter__') and not isinstance(content.data, (six.binary_type, six.string_types)):
if hasattr(content.data, '__iter__') and not isinstance(content.data, (bytes, (str,))):
for chunk in content.data:
fp.write(chunk)
else:
# Ideally we could just ensure that we don't get strings in here and only byte streams
# but being confident of that wolud be a lot more work than we have time for so we just
# handle both cases here.
if isinstance(content.data, six.text_type):
if isinstance(content.data, str):
fp.write(content.data.encode('utf-8'))
else:
fp.write(content.data)
@@ -217,7 +216,7 @@ class MongoContentStore(ContentStore):
# When debugging course exports, this might be a good place
# to look. -- pmitros
self.export(asset['asset_key'], output_directory)
for attr, value in six.iteritems(asset):
for attr, value in asset.items():
if attr not in ['_id', 'md5', 'uploadDate', 'length', 'chunkSize', 'asset_key']:
policy.setdefault(asset['asset_key'].block_id, {})[attr] = value
@@ -240,9 +239,9 @@ class MongoContentStore(ContentStore):
assets_to_delete = 0
for prefix in ['_id', 'content_son']:
query = SON([
('{}.tag'.format(prefix), XASSET_LOCATION_TAG),
('{}.category'.format(prefix), 'asset'),
('{}.name'.format(prefix), {'$regex': ASSET_IGNORE_REGEX}),
(f'{prefix}.tag', XASSET_LOCATION_TAG),
(f'{prefix}.category', 'asset'),
(f'{prefix}.name', {'$regex': ASSET_IGNORE_REGEX}),
])
items = self.fs_files.find(query)
for asset in items:
@@ -375,9 +374,9 @@ class MongoContentStore(ContentStore):
:param location: a c4x asset location
"""
for attr in six.iterkeys(attr_dict):
for attr in attr_dict.keys():
if attr in ['_id', 'md5', 'uploadDate', 'length']:
raise AttributeError("{} is a protected attribute.".format(attr))
raise AttributeError(f"{attr} is a protected attribute.")
asset_db_key, __ = self.asset_db_key(location)
# catch upsert error and raise NotFoundError if asset doesn't exist
result = self.fs_files.update_one({'_id': asset_db_key}, {"$set": attr_dict}, upsert=False)
@@ -413,7 +412,7 @@ class MongoContentStore(ContentStore):
asset_key = self.make_id_son(asset)
# don't convert from string until fs access
source_content = self.fs.get(asset_key)
if isinstance(asset_key, six.string_types):
if isinstance(asset_key, str):
asset_key = AssetKey.from_string(asset_key)
__, asset_key = self.asset_db_key(asset_key)
# Need to replace dict IDs with SON for chunk lookup to work under Python 3
@@ -428,7 +427,7 @@ class MongoContentStore(ContentStore):
asset_id = asset_key
else: # add the run, since it's the last field, we're golden
asset_key['run'] = dest_course_key.run
asset_id = six.text_type(
asset_id = str(
dest_course_key.make_asset_key(asset_key['category'], asset_key['name']).for_branch(None)
)
try:
@@ -496,7 +495,7 @@ class MongoContentStore(ContentStore):
# NOTE, there's no need to state that run doesn't exist in the negative case b/c access via
# SON requires equivalence (same keys and values in exact same order)
dbkey['run'] = location.run
content_id = six.text_type(location.for_branch(None))
content_id = str(location.for_branch(None))
return content_id, dbkey
def make_id_son(self, fs_entry):
@@ -506,7 +505,7 @@ class MongoContentStore(ContentStore):
fs_entry: the element returned by self.fs_files.find
"""
_id_field = fs_entry.get('_id', fs_entry)
if isinstance(_id_field, six.string_types):
if isinstance(_id_field, str):
return _id_field
dbkey = SON((field_name, _id_field.get(field_name)) for field_name in self.ordered_key_fields)
if 'run' in _id_field:
@@ -613,14 +612,14 @@ def query_for_course(course_key, category=None):
else:
prefix = 'content_son'
dbkey = SON([
('{}.tag'.format(prefix), XASSET_LOCATION_TAG),
('{}.org'.format(prefix), course_key.org),
('{}.course'.format(prefix), course_key.course),
(f'{prefix}.tag', XASSET_LOCATION_TAG),
(f'{prefix}.org', course_key.org),
(f'{prefix}.course', course_key.course),
])
if category:
dbkey['{}.category'.format(prefix)] = category
dbkey[f'{prefix}.category'] = category
if getattr(course_key, 'deprecated', False):
dbkey['{}.run'.format(prefix)] = {'$exists': False}
dbkey[f'{prefix}.run'] = {'$exists': False}
else:
dbkey['{}.run'.format(prefix)] = course_key.run
dbkey[f'{prefix}.run'] = course_key.run
return dbkey

4
common/lib/xmodule/xmodule/contentstore/utils.py
View File

@@ -15,13 +15,13 @@ def empty_asset_trashcan(course_locs):
# first delete all of the thumbnails
thumbs = store.get_all_content_thumbnails_for_course(course_loc)
for thumb in thumbs:
print("Deleting {0}...".format(thumb))
print(f"Deleting {thumb}...")
store.delete(thumb['asset_key'])
# then delete all of the assets
assets, __ = store.get_all_content_for_course(course_loc)
for asset in assets:
print("Deleting {0}...".format(asset))
print(f"Deleting {asset}...")
store.delete(asset['asset_key'])

3
common/lib/xmodule/xmodule/course_metadata_utils.py
View File

@@ -12,7 +12,6 @@ from datetime import datetime, timedelta
from math import exp
import dateutil.parser
import six
from pytz import utc
DEFAULT_START_DATE = datetime(2030, 1, 1, tzinfo=utc)
@@ -66,7 +65,7 @@ def clean_course_key(course_key, padding_char):
padding_char (str): Character used for padding at end of the encoded
string. The standard value for this is '='.
"""
encoded = b32encode(six.text_type(course_key).encode('utf8')).decode('utf8')
encoded = b32encode(str(course_key).encode('utf8')).decode('utf8')
return "course_{}".format(
encoded.replace('=', padding_char)
)

47
common/lib/xmodule/xmodule/course_module.py
View File

@@ -10,14 +10,12 @@ from io import BytesIO
import dateutil.parser
import requests
import six
from django.conf import settings
from django.core.validators import validate_email
from lazy import lazy
from lxml import etree
from path import Path as path
from pytz import utc
from six import text_type
from xblock.fields import Boolean, Dict, Float, Integer, List, Scope, String
from openedx.core.djangoapps.video_pipeline.models import VideoUploadsEnabledByDefault
from openedx.core.lib.license import LicenseMixin
@@ -63,7 +61,7 @@ class StringOrDate(Date): # lint-amnesty, pylint: disable=missing-class-docstri
if present, assume it's a string if it doesn't parse.
"""
try:
result = super(StringOrDate, self).from_json(value) # lint-amnesty, pylint: disable=super-with-arguments
result = super().from_json(value)
except ValueError:
return value
if result is None:
@@ -76,7 +74,7 @@ class StringOrDate(Date): # lint-amnesty, pylint: disable=missing-class-docstri
Convert a time struct or string to a string.
"""
try:
result = super(StringOrDate, self).to_json(value) # lint-amnesty, pylint: disable=super-with-arguments
result = super().to_json(value)
except: # lint-amnesty, pylint: disable=bare-except
return value
if result is None:
@@ -103,7 +101,7 @@ edx_xml_parser = etree.XMLParser(dtd_validation=False, load_dtd=False,
_cached_toc = {}
class Textbook(object): # lint-amnesty, pylint: disable=missing-class-docstring
class Textbook: # lint-amnesty, pylint: disable=missing-class-docstring,eq-without-hash
def __init__(self, title, book_url):
self.title = title
self.book_url = book_url
@@ -154,7 +152,7 @@ class Textbook(object): # lint-amnesty, pylint: disable=missing-class-docstring
try:
r = requests.get(toc_url)
except Exception as err:
msg = 'Error %s: Unable to retrieve textbook table of contents at %s' % (err, toc_url)
msg = f'Error {err}: Unable to retrieve textbook table of contents at {toc_url}'
log.error(msg)
raise Exception(msg) # lint-amnesty, pylint: disable=raise-missing-from
@@ -162,7 +160,7 @@ class Textbook(object): # lint-amnesty, pylint: disable=missing-class-docstring
try:
table_of_contents = etree.fromstring(r.text)
except Exception as err:
msg = 'Error %s: Unable to parse XML for textbook table of contents at %s' % (err, toc_url)
msg = f'Error {err}: Unable to parse XML for textbook table of contents at {toc_url}'
log.error(msg)
raise Exception(msg) # lint-amnesty, pylint: disable=raise-missing-from
@@ -185,7 +183,7 @@ class TextbookList(List): # lint-amnesty, pylint: disable=missing-class-docstri
except: # lint-amnesty, pylint: disable=bare-except
# If we can't get to S3 (e.g. on a train with no internet), don't break
# the rest of the courseware.
log.exception("Couldn't load textbook ({0}, {1})".format(title, book_url))
log.exception(f"Couldn't load textbook ({title}, {book_url})")
continue
return textbooks
@@ -213,7 +211,7 @@ class ProctoringProvider(String):
and include any inherited values from the platform default.
"""
errors = []
value = super(ProctoringProvider, self).from_json(value) # lint-amnesty, pylint: disable=super-with-arguments
value = super().from_json(value)
provider_errors = self._validate_proctoring_provider(value)
errors.extend(provider_errors)
@@ -263,7 +261,7 @@ class ProctoringProvider(String):
"""
Return default value for ProctoringProvider.
"""
default = super(ProctoringProvider, self).default # lint-amnesty, pylint: disable=super-with-arguments
default = super().default
proctoring_backend_settings = getattr(settings, 'PROCTORING_BACKENDS', None)
@@ -312,7 +310,7 @@ class TeamsConfigField(Dict):
return value.cleaned_data
class CourseFields(object): # lint-amnesty, pylint: disable=missing-class-docstring
class CourseFields: # lint-amnesty, pylint: disable=missing-class-docstring
lti_passports = List(
display_name=_("LTI Passports"),
help=_('Enter the passports for course LTI tools in the following format: "id:client_key:client_secret".'),
@@ -1040,6 +1038,9 @@ class CourseBlock(
resources_dir = None
def __init__(self, *args, **kwargs):
"""
Expects the same arguments as XModuleDescriptor.__init__
"""
super().__init__(*args, **kwargs)
_ = self.runtime.service(self, "i18n").ugettext
@@ -1076,7 +1077,7 @@ class CourseBlock(
if not getattr(self, "tabs", []):
CourseTabList.initialize_default(self)
except InvalidTabsException as err:
raise type(err)('{msg} For course: {course_id}'.format(msg=text_type(err), course_id=six.text_type(self.id))) # lint-amnesty, pylint: disable=line-too-long
raise type(err)('{msg} For course: {course_id}'.format(msg=str(err), course_id=str(self.id))) # lint-amnesty, pylint: disable=line-too-long
self.set_default_certificate_available_date()
@@ -1118,14 +1119,14 @@ class CourseBlock(
for policy_path in paths:
if not system.resources_fs.exists(policy_path):
continue
log.debug("Loading grading policy from {0}".format(policy_path))
log.debug(f"Loading grading policy from {policy_path}")
try:
with system.resources_fs.open(policy_path) as grading_policy_file:
policy_str = grading_policy_file.read()
# if we successfully read the file, stop looking at backups
break
except IOError:
msg = "Unable to load course settings file from '{0}'".format(policy_path)
except OSError:
msg = f"Unable to load course settings file from '{policy_path}'"
log.warning(msg)
return policy_str
@@ -1136,7 +1137,7 @@ class CourseBlock(
# bleh, have to parse the XML here to just pull out the url_name attribute
# I don't think it's stored anywhere in the instance.
if isinstance(xml_data, six.text_type):
if isinstance(xml_data, str):
xml_data = xml_data.encode('ascii', 'ignore')
course_file = BytesIO(xml_data)
xml_obj = etree.parse(course_file, parser=edx_xml_parser).getroot()
@@ -1144,12 +1145,12 @@ class CourseBlock(
policy_dir = None
url_name = xml_obj.get('url_name', xml_obj.get('slug'))
if url_name:
policy_dir = u'policies/' + url_name
policy_dir = 'policies/' + url_name
# Try to load grading policy
paths = [u'grading_policy.json']
paths = ['grading_policy.json']
if policy_dir:
paths = [policy_dir + u'/grading_policy.json'] + paths
paths = [policy_dir + '/grading_policy.json'] + paths
try:
policy = json.loads(cls.read_grading_policy(paths, system))
@@ -1366,7 +1367,7 @@ class CourseBlock(
return True
else:
return False
elif isinstance(flag, six.string_types):
elif isinstance(flag, str):
return flag.lower() in ['true', 'yes', 'y']
else:
return bool(flag)
@@ -1557,14 +1558,14 @@ class CourseBlock(
return datetime.now(utc) <= self.start
class CourseSummary(object):
class CourseSummary:
"""
A lightweight course summary class, which constructs split/mongo course summary without loading
the course. It is used at cms for listing courses to global staff user.
"""
course_info_fields = ['display_name', 'display_coursenumber', 'display_organization', 'end']
def __init__(self, course_locator, display_name=u"Empty", display_coursenumber=None, display_organization=None,
def __init__(self, course_locator, display_name="Empty", display_coursenumber=None, display_organization=None,
end=None):
"""
Initialize and construct course summary
@@ -1624,7 +1625,7 @@ class CourseSummary(object):
except TypeError as e:
log.warning(
"Course '{course_id}' has an improperly formatted end date '{end_date}'. Error: '{err}'.".format(
course_id=six.text_type(self.id), end_date=self.end, err=e
course_id=str(self.id), end_date=self.end, err=e
)
)
modified_end = self.end.replace(tzinfo=utc)

4
common/lib/xmodule/xmodule/editing_module.py
View File

@@ -11,7 +11,7 @@ from xmodule.mako_module import MakoModuleDescriptor, MakoTemplateBlockBase
log = logging.getLogger(__name__)
class EditingFields(object):
class EditingFields:
"""Contains specific template information (the raw data body)"""
data = String(scope=Scope.content, default='')
@@ -32,7 +32,7 @@ class EditingMixin(EditingFields, MakoTemplateBlockBase):
"""
`data` should not be editable in the Studio settings editor.
"""
non_editable_fields = super(EditingMixin, self).non_editable_metadata_fields # lint-amnesty, pylint: disable=super-with-arguments
non_editable_fields = super().non_editable_metadata_fields
non_editable_fields.append(self.fields['data'])
return non_editable_fields

5
common/lib/xmodule/xmodule/error_module.py
View File

@@ -9,7 +9,6 @@ import json
import logging
import sys
import six
from lxml import etree
from web_fragments.fragment import Fragment
from xblock.field_data import DictFieldData
@@ -35,7 +34,7 @@ log = logging.getLogger(__name__)
# decides whether to create a staff or not-staff module.
class ErrorFields(object):
class ErrorFields:
"""
XBlock fields used by the ErrorBlocks
"""
@@ -107,7 +106,7 @@ class ErrorBlock(
# real metadata stays in the content, but add a display name
field_data = DictFieldData({
'error_msg': six.text_type(error_msg),
'error_msg': str(error_msg),
'contents': contents,
'location': location,
'category': 'error'

6
common/lib/xmodule/xmodule/exceptions.py
View File

@@ -21,7 +21,7 @@ class InvalidVersionError(Exception):
for a non-leaf node)
"""
def __init__(self, location):
super(InvalidVersionError, self).__init__() # lint-amnesty, pylint: disable=super-with-arguments
super().__init__()
self.location = location
@@ -30,7 +30,7 @@ class SerializationError(Exception):
Thrown when a module cannot be exported to XML
"""
def __init__(self, location, msg):
super(SerializationError, self).__init__(msg) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(msg)
self.location = location
@@ -51,4 +51,4 @@ class HeartbeatFailure(Exception):
In addition to a msg, provide the name of the service.
"""
self.service = service
super(HeartbeatFailure, self).__init__(msg) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(msg)

26
common/lib/xmodule/xmodule/fields.py
View File

@@ -6,9 +6,7 @@ import re
import time
import dateutil.parser
import six
from pytz import UTC
from six import text_type
from xblock.fields import JSONField
from xblock.scorable import Score
@@ -37,7 +35,7 @@ class Date(JSONField):
result = dateutil.parser.parse(field, default=self.PREVENT_DEFAULT_DAY_MON_SEED1)
result_other = dateutil.parser.parse(field, default=self.PREVENT_DEFAULT_DAY_MON_SEED2)
if result != result_other:
log.warning("Field {0} is missing month or day".format(self.name))
log.warning(f"Field {self.name} is missing month or day")
return None
if result.tzinfo is None:
result = result.replace(tzinfo=UTC)
@@ -53,16 +51,16 @@ class Date(JSONField):
return field
elif field == "":
return None
elif isinstance(field, six.string_types):
elif isinstance(field, str):
return self._parse_date_wo_default_month_day(field)
elif isinstance(field, six.integer_types) or isinstance(field, float): # lint-amnesty, pylint: disable=consider-merging-isinstance
elif isinstance(field, int) or isinstance(field, float): # lint-amnesty, pylint: disable=consider-merging-isinstance
return datetime.datetime.fromtimestamp(field / 1000, UTC)
elif isinstance(field, time.struct_time):
return datetime.datetime.fromtimestamp(time.mktime(field), UTC)
elif isinstance(field, datetime.datetime):
return field
else:
msg = "Field {0} has bad value '{1}'".format(
msg = "Field {} has bad value '{}'".format(
self.name, field)
raise TypeError(msg)
@@ -86,7 +84,7 @@ class Date(JSONField):
else:
return value.isoformat()
else:
raise TypeError("Cannot convert {!r} to json".format(value))
raise TypeError(f"Cannot convert {value!r} to json")
enforce_type = from_json
@@ -118,7 +116,7 @@ class Timedelta(JSONField): # lint-amnesty, pylint: disable=missing-class-docst
return
parts = parts.groupdict()
time_params = {}
for (name, param) in six.iteritems(parts):
for (name, param) in parts.items():
if param:
time_params[name] = int(param)
return datetime.timedelta(**time_params)
@@ -179,7 +177,7 @@ class RelativeTime(JSONField):
except ValueError as e:
raise ValueError( # lint-amnesty, pylint: disable=raise-missing-from
"Incorrect RelativeTime value {!r} was set in XML or serialized. "
"Original parse message is {}".format(value, text_type(e))
"Original parse message is {}".format(value, str(e))
)
return datetime.timedelta(
hours=obj_time.tm_hour,
@@ -204,10 +202,10 @@ class RelativeTime(JSONField):
if isinstance(value, float):
return datetime.timedelta(seconds=value)
if isinstance(value, six.string_types):
if isinstance(value, str):
return self.isotime_to_timedelta(value)
msg = "RelativeTime Field {0} has bad value '{1!r}'".format(self.name, value)
msg = f"RelativeTime Field {self.name} has bad value '{value!r}'"
raise TypeError(msg)
def to_json(self, value):
@@ -235,7 +233,7 @@ class RelativeTime(JSONField):
)
return self.timedelta_to_string(value)
raise TypeError("RelativeTime: cannot convert {!r} to json".format(value))
raise TypeError(f"RelativeTime: cannot convert {value!r} to json")
def timedelta_to_string(self, value):
"""
@@ -284,7 +282,7 @@ class ScoreField(JSONField):
if raw_possible < 0:
raise ValueError(
'Error deserializing field of type {0}: Expected a positive number for raw_possible, got {1}.'.format(
'Error deserializing field of type {}: Expected a positive number for raw_possible, got {}.'.format(
self.display_name,
raw_possible,
)
@@ -292,7 +290,7 @@ class ScoreField(JSONField):
if not (0 <= raw_earned <= raw_possible): # lint-amnesty, pylint: disable=superfluous-parens
raise ValueError(
'Error deserializing field of type {0}: Expected raw_earned between 0 and {1}, got {2}.'.format(
'Error deserializing field of type {}: Expected raw_earned between 0 and {}, got {}.'.format(
self.display_name,
raw_possible,
raw_earned

34
common/lib/xmodule/xmodule/graders.py
View File

@@ -11,11 +11,9 @@ import sys
from collections import OrderedDict
from datetime import datetime
import six
from contracts import contract
from pytz import UTC
from django.utils.translation import ugettext_lazy as _
from six.moves import range
from xmodule.util.misc import get_short_labeler
@@ -23,7 +21,7 @@ from xmodule.util.misc import get_short_labeler
log = logging.getLogger("edx.courseware")
class ScoreBase(six.with_metaclass(abc.ABCMeta, object)):
class ScoreBase(metaclass=abc.ABCMeta): # pylint: disable=eq-without-hash
"""
Abstract base class for encapsulating fields of values scores.
"""
@@ -51,7 +49,7 @@ class ScoreBase(six.with_metaclass(abc.ABCMeta, object)):
return not self.__eq__(other)
def __repr__(self):
return u"{class_name}({fields})".format(class_name=self.__class__.__name__, fields=self.__dict__)
return f"{self.__class__.__name__}({self.__dict__})"
class ProblemScore(ScoreBase):
@@ -78,7 +76,7 @@ class ProblemScore(ScoreBase):
:param weight: Weight of this problem
:type weight: int|float|None
"""
super(ProblemScore, self).__init__(*args, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(*args, **kwargs)
self.raw_earned = float(raw_earned) if raw_earned is not None else None
self.raw_possible = float(raw_possible) if raw_possible is not None else None
self.earned = float(weighted_earned) if weighted_earned is not None else None
@@ -101,7 +99,7 @@ class AggregatedScore(ScoreBase):
:param tw_possible: Total aggregated sum of all weighted possible values
:type tw_possible: int|float|None
"""
super(AggregatedScore, self).__init__(*args, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(*args, **kwargs)
self.earned = float(tw_earned) if tw_earned is not None else None
self.possible = float(tw_possible) if tw_possible is not None else None
@@ -180,7 +178,7 @@ def grader_from_conf(conf):
bad_args = invalid_args(subgrader_class.__init__, subgraderconf)
if bad_args:
log.warning(u"Invalid arguments for a subgrader: %s", bad_args)
log.warning("Invalid arguments for a subgrader: %s", bad_args)
for key in bad_args:
del subgraderconf[key]
@@ -192,12 +190,12 @@ def grader_from_conf(conf):
msg = ("Unable to parse grader configuration:\n " +
str(subgraderconf) +
"\n Error was:\n " + str(error))
six.reraise(ValueError, ValueError(msg), sys.exc_info()[2])
raise ValueError(msg).with_traceback(sys.exc_info()[2])
return WeightedSubsectionsGrader(subgraders)
class CourseGrader(six.with_metaclass(abc.ABCMeta, object)):
class CourseGrader(metaclass=abc.ABCMeta):
"""
A course grader takes the totaled scores for each graded section (that a student has
started) in the course. From these scores, the grader calculates an overall percentage
@@ -279,7 +277,7 @@ class WeightedSubsectionsGrader(CourseGrader):
subgrade_result = subgrader.grade(grade_sheet, generate_random_scores)
weighted_percent = subgrade_result['percent'] * weight
section_detail = _(u"{assignment_type} = {weighted_percent:.2%} of a possible {weight:.2%}").format(
section_detail = _("{assignment_type} = {weighted_percent:.2%} of a possible {weight:.2%}").format(
assignment_type=assignment_type,
weighted_percent=weighted_percent,
weight=weight)
@@ -393,7 +391,7 @@ class AssignmentFormatGrader(CourseGrader):
section_name = scores[i].display_name
percentage = scores[i].percent_graded
summary_format = u"{section_type} {index} - {name} - {percent:.0%} ({earned:.3n}/{possible:.3n})"
summary_format = "{section_type} {index} - {name} - {percent:.0%} ({earned:.3n}/{possible:.3n})"
summary = summary_format.format(
index=i + self.starting_index,
section_type=self.section_type,
@@ -405,7 +403,7 @@ class AssignmentFormatGrader(CourseGrader):
else:
percentage = 0.0
# Translators: "Homework 1 - Unreleased - 0% (?/?)" The section has not been released for viewing.
summary = _(u"{section_type} {index} Unreleased - 0% (?/?)").format(
summary = _("{section_type} {index} Unreleased - 0% (?/?)").format(
index=i + self.starting_index,
section_type=self.section_type
)
@@ -418,7 +416,7 @@ class AssignmentFormatGrader(CourseGrader):
for dropped_index in dropped_indices:
breakdown[dropped_index]['mark'] = {
'detail': _(u"The lowest {drop_count} {section_type} scores are dropped.").format(
'detail': _("The lowest {drop_count} {section_type} scores are dropped.").format(
drop_count=self.drop_count,
section_type=self.section_type
)
@@ -427,21 +425,21 @@ class AssignmentFormatGrader(CourseGrader):
if len(breakdown) == 1:
# if there is only one entry in a section, suppress the existing individual entry and the average,
# and just display a single entry for the section.
total_detail = u"{section_type} = {percent:.0%}".format(
total_detail = "{section_type} = {percent:.0%}".format(
percent=total_percent,
section_type=self.section_type,
)
total_label = u"{short_label}".format(short_label=self.short_label)
total_label = f"{self.short_label}"
breakdown = [{'percent': total_percent, 'label': total_label,
'detail': total_detail, 'category': self.category, 'prominent': True}, ]
else:
# Translators: "Homework Average = 0%"
total_detail = _(u"{section_type} Average = {percent:.0%}").format(
total_detail = _("{section_type} Average = {percent:.0%}").format(
percent=total_percent,
section_type=self.section_type
)
# Translators: Avg is short for Average
total_label = _(u"{short_label} Avg").format(short_label=self.short_label)
total_label = _("{short_label} Avg").format(short_label=self.short_label)
if self.show_only_average:
breakdown = []
@@ -476,7 +474,7 @@ def _min_or_none(itr):
return None
class ShowCorrectness(object):
class ShowCorrectness:
"""
Helper class for determining whether correctness is currently hidden for a block.

33
common/lib/xmodule/xmodule/html_module.py
View File

@@ -10,7 +10,6 @@ from datetime import datetime
from pkg_resources import resource_string
import six
from django.conf import settings
from fs.errors import ResourceNotFound
from lxml import etree
@@ -59,7 +58,7 @@ class HtmlBlockMixin( # lint-amnesty, pylint: disable=abstract-method
# use display_name_with_default for those
default=_("Text")
)
data = String(help=_("Html contents to display for this module"), default=u"", scope=Scope.content)
data = String(help=_("Html contents to display for this module"), default="", scope=Scope.content)
source_code = String(
help=_("Source code for LaTeX documents. This feature is not well-supported."),
scope=Scope.settings
@@ -111,7 +110,7 @@ class HtmlBlockMixin( # lint-amnesty, pylint: disable=abstract-method
else:
return {
'enabled': False,
'message': 'To enable, set FEATURES["{}"]'.format(self.ENABLE_HTML_XBLOCK_STUDENT_VIEW_DATA)
'message': f'To enable, set FEATURES["{self.ENABLE_HTML_XBLOCK_STUDENT_VIEW_DATA}"]'
}
def get_html(self):
@@ -256,7 +255,7 @@ class HtmlBlockMixin( # lint-amnesty, pylint: disable=abstract-method
)
base = path(pointer_path).dirname()
# log.debug("base = {0}, base.dirname={1}, filename={2}".format(base, base.dirname(), filename))
filepath = u"{base}/{name}.html".format(base=base, name=filename)
filepath = f"{base}/{filename}.html"
# log.debug("looking for html file for {0} at {1}".format(location, filepath))
# VS[compat]
@@ -278,7 +277,7 @@ class HtmlBlockMixin( # lint-amnesty, pylint: disable=abstract-method
html = infile.read()
# Log a warning if we can't parse the file, but don't error
if not check_html(html) and len(html) > 0:
msg = "Couldn't parse html in {0}, content = {1}".format(filepath, html)
msg = f"Couldn't parse html in {filepath}, content = {html}"
log.warning(msg)
system.error_tracker("Warning: " + msg)
@@ -291,10 +290,10 @@ class HtmlBlockMixin( # lint-amnesty, pylint: disable=abstract-method
return definition, []
except ResourceNotFound as err:
msg = 'Unable to load file contents at path {0}: {1} '.format(
msg = 'Unable to load file contents at path {}: {} '.format(
filepath, err)
# add more info and re-raise
six.reraise(Exception, Exception(msg), sys.exc_info()[2])
raise Exception(msg).with_traceback(sys.exc_info()[2])
@classmethod
def parse_xml_new_runtime(cls, node, runtime, keys):
@@ -319,7 +318,7 @@ class HtmlBlockMixin( # lint-amnesty, pylint: disable=abstract-method
# Write html to file, return an empty tag
pathname = name_to_pathname(self.url_name)
filepath = u'{category}/{pathname}.html'.format(
filepath = '{category}/{pathname}.html'.format(
category=self.category,
pathname=pathname
)
@@ -341,12 +340,12 @@ class HtmlBlockMixin( # lint-amnesty, pylint: disable=abstract-method
"""
`use_latex_compiler` should not be editable in the Studio settings editor.
"""
non_editable_fields = super(HtmlBlockMixin, self).non_editable_metadata_fields # lint-amnesty, pylint: disable=super-with-arguments
non_editable_fields = super().non_editable_metadata_fields
non_editable_fields.append(HtmlBlockMixin.use_latex_compiler)
return non_editable_fields
def index_dictionary(self):
xblock_body = super(HtmlBlockMixin, self).index_dictionary() # lint-amnesty, pylint: disable=super-with-arguments
xblock_body = super().index_dictionary()
# Removing script and style
html_content = re.sub(
re.compile(
@@ -380,7 +379,7 @@ class HtmlBlock(HtmlBlockMixin): # lint-amnesty, pylint: disable=abstract-metho
"""
class AboutFields(object): # lint-amnesty, pylint: disable=missing-class-docstring
class AboutFields: # lint-amnesty, pylint: disable=missing-class-docstring
display_name = String(
help=_("The display name for this component."),
scope=Scope.settings,
@@ -388,7 +387,7 @@ class AboutFields(object): # lint-amnesty, pylint: disable=missing-class-docstr
)
data = String(
help=_("Html contents to display for this module"),
default=u"",
default="",
scope=Scope.content
)
@@ -402,7 +401,7 @@ class AboutBlock(AboutFields, HtmlBlockMixin): # lint-amnesty, pylint: disable=
template_dir_name = "about"
class StaticTabFields(object):
class StaticTabFields:
"""
The overrides for Static Tabs
"""
@@ -420,7 +419,7 @@ class StaticTabFields(object):
scope=Scope.settings
)
data = String(
default=textwrap.dedent(u"""\
default=textwrap.dedent("""\
<p>Add the content you want students to see on this page.</p>
"""),
scope=Scope.content,
@@ -437,7 +436,7 @@ class StaticTabBlock(StaticTabFields, HtmlBlockMixin): # lint-amnesty, pylint:
template_dir_name = None
class CourseInfoFields(object):
class CourseInfoFields:
"""
Field overrides
"""
@@ -448,7 +447,7 @@ class CourseInfoFields(object):
)
data = String(
help=_("Html contents to display for this module"),
default=u"<ol></ol>",
default="<ol></ol>",
scope=Scope.content
)
@@ -482,7 +481,7 @@ class CourseInfoBlock(CourseInfoFields, HtmlBlockMixin): # lint-amnesty, pylint
'visible_updates': course_updates[:3],
'hidden_updates': course_updates[3:],
}
return self.system.render_template("{0}/course_updates.html".format(self.TEMPLATE_DIR), context)
return self.system.render_template(f"{self.TEMPLATE_DIR}/course_updates.html", context)
@classmethod
def order_updates(self, updates): # lint-amnesty, pylint: disable=bad-classmethod-argument

64
common/lib/xmodule/xmodule/library_content_module.py
View File

@@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
"""
LibraryContent: The XBlock used to include blocks from a library in a course.
"""
@@ -10,14 +9,11 @@ import random
from copy import copy
from gettext import ngettext
import six
import bleach
from lazy import lazy
from lxml import etree
from opaque_keys.edx.locator import LibraryLocator
from pkg_resources import resource_string
from six import text_type
from six.moves import zip
from web_fragments.fragment import Fragment
from webob import Response
from xblock.completable import XBlockCompletionMode
@@ -196,10 +192,10 @@ class LibraryContentBlock(
"""
rand = random.Random()
selected_keys = set(tuple(k) for k in selected) # set of (block_type, block_id) tuples assigned to this student
selected_keys = {tuple(k) for k in selected} # set of (block_type, block_id) tuples assigned to this student
# Determine which of our children we will show:
valid_block_keys = set((c.block_type, c.block_id) for c in children)
valid_block_keys = {(c.block_type, c.block_id) for c in children}
# Remove any selected blocks that are no longer valid:
invalid_block_keys = (selected_keys - valid_block_keys)
@@ -244,13 +240,13 @@ class LibraryContentBlock(
Helper method to publish an event for analytics purposes
"""
event_data = {
"location": six.text_type(self.location),
"location": str(self.location),
"result": result,
"previous_count": getattr(self, "_last_event_result_count", len(self.selected)),
"max_count": self.max_count,
}
event_data.update(kwargs)
self.runtime.publish(self, "edx.librarycontentblock.content.{}".format(event_name), event_data)
self.runtime.publish(self, f"edx.librarycontentblock.content.{event_name}", event_data)
self._last_event_result_count = len(result) # pylint: disable=attribute-defined-outside-init
@classmethod
@@ -364,7 +360,7 @@ class LibraryContentBlock(
rendered_child = displayable.render(STUDENT_VIEW, child_context)
fragment.add_fragment_resources(rendered_child)
contents.append({
'id': text_type(displayable.location),
'id': str(displayable.location),
'content': rendered_child.content,
})
@@ -478,7 +474,7 @@ class LibraryContentBlock(
"""
Copy any overrides the user has made on blocks in this library.
"""
for field in six.itervalues(source.fields):
for field in source.fields.values():
if field.scope == Scope.settings and field.is_set_on(source):
setattr(dest, field.name, field.read_from(source))
if source.has_children:
@@ -515,16 +511,16 @@ class LibraryContentBlock(
"""
latest_version = lib_tools.get_library_version(library_key)
if latest_version is not None:
if version is None or version != six.text_type(latest_version):
if version is None or version != str(latest_version):
validation.set_summary(
StudioValidationMessage(
StudioValidationMessage.WARNING,
_(u'This component is out of date. The library has new content.'),
_('This component is out of date. The library has new content.'),
# TODO: change this to action_runtime_event='...' once the unit page supports that feature.
# See https://openedx.atlassian.net/browse/TNL-993
action_class='library-update-btn',
# Translators: {refresh_icon} placeholder is substituted to "↻" (without double quotes)
action_label=_(u"{refresh_icon} Update now.").format(refresh_icon=u"↻")
action_label=_("{refresh_icon} Update now.").format(refresh_icon="↻")
)
)
return False
@@ -532,9 +528,9 @@ class LibraryContentBlock(
validation.set_summary(
StudioValidationMessage(
StudioValidationMessage.ERROR,
_(u'Library is invalid, corrupt, or has been deleted.'),
_('Library is invalid, corrupt, or has been deleted.'),
action_class='edit-button',
action_label=_(u"Edit Library List.")
action_label=_("Edit Library List.")
)
)
return False
@@ -560,8 +556,8 @@ class LibraryContentBlock(
StudioValidationMessage(
StudioValidationMessage.ERROR,
_(
u"This course does not support content libraries. "
u"Contact your system administrator for more information."
"This course does not support content libraries. "
"Contact your system administrator for more information."
)
)
)
@@ -570,9 +566,9 @@ class LibraryContentBlock(
validation.set_summary(
StudioValidationMessage(
StudioValidationMessage.NOT_CONFIGURED,
_(u"A library has not yet been selected."),
_("A library has not yet been selected."),
action_class='edit-button',
action_label=_(u"Select a Library.")
action_label=_("Select a Library.")
)
)
return validation
@@ -587,9 +583,9 @@ class LibraryContentBlock(
validation,
StudioValidationMessage(
StudioValidationMessage.WARNING,
_(u'There are no matching problem types in the specified libraries.'),
_('There are no matching problem types in the specified libraries.'),
action_class='edit-button',
action_label=_(u"Select another problem type.")
action_label=_("Select another problem type.")
)
)
@@ -600,18 +596,18 @@ class LibraryContentBlock(
StudioValidationMessage.WARNING,
(
ngettext(
u'The specified library is configured to fetch {count} problem, ',
u'The specified library is configured to fetch {count} problems, ',
'The specified library is configured to fetch {count} problem, ',
'The specified library is configured to fetch {count} problems, ',
self.max_count
) +
ngettext(
u'but there is only {actual} matching problem.',
u'but there are only {actual} matching problems.',
'but there is only {actual} matching problem.',
'but there are only {actual} matching problems.',
matching_children_count
)
).format(count=self.max_count, actual=matching_children_count),
action_class='edit-button',
action_label=_(u"Edit the library configuration.")
action_label=_("Edit the library configuration.")
)
)
@@ -625,13 +621,13 @@ class LibraryContentBlock(
user_perms = self.runtime.service(self, 'studio_user_permissions')
all_libraries = [
(key, bleach.clean(name)) for key, name in lib_tools.list_available_libraries()
if user_perms.can_read(key) or self.source_library_id == six.text_type(key)
if user_perms.can_read(key) or self.source_library_id == str(key)
]
all_libraries.sort(key=lambda entry: entry[1]) # Sort by name
if self.source_library_id and self.source_library_key not in [entry[0] for entry in all_libraries]:
all_libraries.append((self.source_library_id, _(u"Invalid Library")))
all_libraries = [(u"", _("No Library Selected"))] + all_libraries
values = [{"display_name": name, "value": six.text_type(key)} for key, name in all_libraries]
all_libraries.append((self.source_library_id, _("Invalid Library")))
all_libraries = [("", _("No Library Selected"))] + all_libraries
values = [{"display_name": name, "value": str(key)} for key, name in all_libraries]
return values
def editor_saved(self, user, old_metadata, old_content): # lint-amnesty, pylint: disable=unused-argument
@@ -684,15 +680,15 @@ class LibraryContentBlock(
for child in self.get_children():
self.runtime.add_block_as_child_node(child, xml_object)
# Set node attributes based on our fields.
for field_name, field in six.iteritems(self.fields):
for field_name, field in self.fields.items(): # pylint: disable=no-member
if field_name in ('children', 'parent', 'content'):
continue
if field.is_set_on(self):
xml_object.set(field_name, six.text_type(field.read_from(self)))
xml_object.set(field_name, str(field.read_from(self)))
return xml_object
class LibrarySummary(object):
class LibrarySummary:
"""
A library summary object which contains the fields required for library listing on studio.
"""
@@ -706,7 +702,7 @@ class LibrarySummary(object):
display_name (unicode): display name of the library.
"""
self.display_name = display_name if display_name else _(u"Empty")
self.display_name = display_name if display_name else _("Empty")
self.id = library_locator # pylint: disable=invalid-name
self.location = library_locator.make_usage_key('library', 'library')

7
common/lib/xmodule/xmodule/library_root_xblock.py
View File

@@ -4,7 +4,6 @@
import logging
import six
from django.utils.encoding import python_2_unicode_compatible
from web_fragments.fragment import Fragment
@@ -50,7 +49,7 @@ class LibraryRoot(XBlock):
has_author_view = True
def __str__(self):
return u"Library: {}".format(self.display_name)
return f"Library: {self.display_name}"
def author_view(self, context):
"""
@@ -91,7 +90,7 @@ class LibraryRoot(XBlock):
child = self.runtime.get_block(child_key)
child_view_name = StudioEditableModule.get_preview_view_name(child)
if six.text_type(child.location) == force_render:
if str(child.location) == force_render:
child_context['show_preview'] = True
if child_context['show_preview']:
@@ -101,7 +100,7 @@ class LibraryRoot(XBlock):
fragment.add_fragment_resources(rendered_child)
contents.append({
'id': six.text_type(child.location),
'id': str(child.location),
'content': rendered_child.content,
})

10
common/lib/xmodule/xmodule/library_sourced_block.py
View File

@@ -55,7 +55,7 @@ class LibrarySourcedBlock(StudioEditableXBlockMixin, EditableChildrenMixin, XBlo
MAX_BLOCKS_ALLOWED = 10
def __str__(self):
return "LibrarySourcedBlock: {}".format(self.display_name)
return f"LibrarySourcedBlock: {self.display_name}"
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -121,7 +121,7 @@ class LibrarySourcedBlock(StudioEditableXBlockMixin, EditableChildrenMixin, XBlo
validation.add(
ValidationMessage(
ValidationMessage.ERROR,
_(u"A maximum of {0} components may be added.").format(self.MAX_BLOCKS_ALLOWED)
_("A maximum of {0} components may be added.").format(self.MAX_BLOCKS_ALLOWED)
)
)
@@ -137,9 +137,9 @@ class LibrarySourcedBlock(StudioEditableXBlockMixin, EditableChildrenMixin, XBlo
validation.set_summary(
StudioValidationMessage(
StudioValidationMessage.NOT_CONFIGURED,
_(u"No XBlock has been configured for this component. Use the editor to select the target blocks."),
_("No XBlock has been configured for this component. Use the editor to select the target blocks."),
action_class='edit-button',
action_label=_(u"Open Editor")
action_label=_("Open Editor")
)
)
return validation
@@ -156,5 +156,5 @@ class LibrarySourcedBlock(StudioEditableXBlockMixin, EditableChildrenMixin, XBlo
lib_tools.import_from_blockstore(self, self.source_block_ids)
except Exception as err: # pylint: disable=broad-except
log.exception(err)
return Response(_(u"Importing Library Block failed - are the IDs valid and readable?"), status=400)
return Response(_("Importing Library Block failed - are the IDs valid and readable?"), status=400)
return response

17
common/lib/xmodule/xmodule/library_tools.py
View File

@@ -3,7 +3,6 @@ XBlock runtime services for LibraryContentBlock
"""
import hashlib
import six
from django.contrib.auth.models import User # lint-amnesty, pylint: disable=imported-auth-user
from django.core.exceptions import PermissionDenied
from opaque_keys.edx.keys import UsageKey
@@ -26,7 +25,7 @@ def normalize_key_for_search(library_key):
return library_key.replace(version_guid=None, branch=None)
class LibraryToolsService(object):
class LibraryToolsService:
"""
Service that allows LibraryContentBlock to interact with libraries in the
modulestore.
@@ -81,9 +80,9 @@ class LibraryToolsService(object):
""" Basic information about the given block """
orig_key, orig_version = self.store.get_block_original_usage(usage_key)
return {
"usage_key": six.text_type(usage_key),
"original_usage_key": six.text_type(orig_key) if orig_key else None,
"original_usage_version": six.text_type(orig_version) if orig_version else None,
"usage_key": str(usage_key),
"original_usage_key": str(orig_key) if orig_key else None,
"original_usage_version": str(orig_version) if orig_version else None,
}
result_json = []
@@ -109,7 +108,7 @@ class LibraryToolsService(object):
search_engine = SearchEngine.get_search_engine(index="library_index")
if search_engine:
filter_clause = {
"library": six.text_type(normalize_key_for_search(library.location.library_key)),
"library": str(normalize_key_for_search(library.location.library_key)),
"content_type": ProblemBlock.INDEX_CONTENT_TYPE,
"problem_types": capa_type
}
@@ -161,7 +160,7 @@ class LibraryToolsService(object):
library_key = library_key.replace(branch=ModuleStoreEnum.BranchName.library, version_guid=version)
library = self._get_library(library_key)
if library is None:
raise ValueError("Requested library {0} not found.".format(library_key))
raise ValueError(f"Requested library {library_key} not found.")
if user_perms and not user_perms.can_read(library_key):
raise PermissionDenied()
filter_children = (dest_block.capa_type != ANY_CAPA_TYPE_VALUE)
@@ -202,7 +201,7 @@ class LibraryToolsService(object):
"""
dest_key = dest_block.scope_ids.usage_id
if not isinstance(dest_key, BlockUsageLocator):
raise TypeError("Destination {} should be a modulestore course.".format(dest_key))
raise TypeError(f"Destination {dest_key} should be a modulestore course.")
if self.user_id is None:
raise ValueError("Cannot check user permissions - LibraryTools user_id is None")
@@ -286,7 +285,7 @@ class LibraryToolsService(object):
if isinstance(field_value, str):
# If string field (which may also be JSON/XML data), rewrite /static/... URLs to point to blockstore
for asset in all_assets:
field_value = field_value.replace('/static/{}'.format(asset.path), asset.url)
field_value = field_value.replace(f'/static/{asset.path}', asset.url)
# Make sure the URL is one that will work from the user's browser when using the docker devstack
field_value = blockstore_api.force_browser_url(field_value)
setattr(new_block, field_name, field_value)

55
common/lib/xmodule/xmodule/lti_2_util.py
View File

@@ -9,11 +9,10 @@ import hashlib
import json
import logging
import re
from unittest import mock
from urllib import parse
import mock
import six
from oauthlib.oauth1 import Client
from six import text_type
from webob import Response
from xblock.core import XBlock
@@ -29,7 +28,7 @@ class LTIError(Exception):
"""Error class for LTIBlock and LTI20BlockMixin"""
class LTI20BlockMixin(object):
class LTI20BlockMixin:
"""
This class MUST be mixed into LTIBlock. It does not do anything on its own. It's just factored
out for modularity.
@@ -81,7 +80,7 @@ class LTI20BlockMixin(object):
real_user = self.system.get_real_user(anon_id)
if not real_user: # that means we can't save to database, as we do not have real user id.
msg = "[LTI]: Real user not found against anon_id: {}".format(anon_id)
msg = f"[LTI]: Real user not found against anon_id: {anon_id}"
log.info(msg)
return Response(status=404) # have to do 404 due to spec, but 400 is better, with error msg in body
if request.method == "PUT":
@@ -108,22 +107,22 @@ class LTI20BlockMixin(object):
"""
sha1 = hashlib.sha1()
sha1.update(request.body)
oauth_body_hash = six.text_type(base64.b64encode(sha1.digest()))
log.debug("[LTI] oauth_body_hash = {}".format(oauth_body_hash))
oauth_body_hash = str(base64.b64encode(sha1.digest()))
log.debug(f"[LTI] oauth_body_hash = {oauth_body_hash}")
client_key, client_secret = self.get_client_key_secret()
client = Client(client_key, client_secret)
mock_request = mock.Mock(
uri=six.text_type(six.moves.urllib.parse.unquote(request.url)),
uri=str(parse.unquote(request.url)),
headers=request.headers,
body=u"",
decoded_body=u"",
http_method=six.text_type(request.method),
body="",
decoded_body="",
http_method=str(request.method),
)
params = client.get_oauth_params(mock_request)
mock_request.oauth_params = params
mock_request.oauth_params.append((u'oauth_body_hash', oauth_body_hash))
mock_request.oauth_params.append(('oauth_body_hash', oauth_body_hash))
sig = client.get_oauth_signature(mock_request)
mock_request.oauth_params.append((u'oauth_signature', sig))
mock_request.oauth_params.append(('oauth_signature', sig))
_, headers, _ = client._render(mock_request) # pylint: disable=protected-access
log.debug("\n\n#### COPY AND PASTE AUTHORIZATION HEADER ####\n{}\n####################################\n\n"
@@ -151,7 +150,7 @@ class LTI20BlockMixin(object):
return match_obj.group('anon_id')
# fall-through handles all error cases
msg = "No valid user id found in endpoint URL"
log.info("[LTI]: {}".format(msg))
log.info(f"[LTI]: {msg}")
raise LTIError(msg)
def _lti_2_0_result_get_handler(self, request, real_user):
@@ -236,7 +235,7 @@ class LTI20BlockMixin(object):
"""
self.set_user_module_score(user, None, None, score_deleted=True)
def set_user_module_score(self, user, score, max_score, comment=u"", score_deleted=False):
def set_user_module_score(self, user, score, max_score, comment="", score_deleted=False):
"""
Sets the module user state, including grades and comments, and also scoring in db's courseware_studentmodule
@@ -286,15 +285,15 @@ class LTI20BlockMixin(object):
"""
content_type = request.headers.get('Content-Type')
if verify_content_type and content_type != LTI_2_0_JSON_CONTENT_TYPE:
log.info("[LTI]: v2.0 result service -- bad Content-Type: {}".format(content_type))
log.info(f"[LTI]: v2.0 result service -- bad Content-Type: {content_type}")
raise LTIError(
"For LTI 2.0 result service, Content-Type must be {}. Got {}".format(LTI_2_0_JSON_CONTENT_TYPE,
content_type))
try:
self.verify_oauth_body_sign(request, content_type=LTI_2_0_JSON_CONTENT_TYPE)
except (ValueError, LTIError) as err:
log.info("[LTI]: v2.0 result service -- OAuth body verification failed: {}".format(text_type(err)))
raise LTIError(text_type(err)) # lint-amnesty, pylint: disable=raise-missing-from
log.info("[LTI]: v2.0 result service -- OAuth body verification failed: {}".format(str(err)))
raise LTIError(str(err)) # lint-amnesty, pylint: disable=raise-missing-from
def parse_lti_2_0_result_json(self, json_str):
"""
@@ -319,8 +318,8 @@ class LTI20BlockMixin(object):
try:
json_obj = json.loads(json_str)
except (ValueError, TypeError):
msg = "Supplied JSON string in request body could not be decoded: {}".format(json_str)
log.info("[LTI] {}".format(msg))
msg = f"Supplied JSON string in request body could not be decoded: {json_str}"
log.info(f"[LTI] {msg}")
raise LTIError(msg) # lint-amnesty, pylint: disable=raise-missing-from
# the standard supports a list of objects, who knows why. It must contain at least 1 element, and the
@@ -331,22 +330,22 @@ class LTI20BlockMixin(object):
else:
msg = ("Supplied JSON string is a list that does not contain an object as the first element. {}"
.format(json_str))
log.info("[LTI] {}".format(msg))
log.info(f"[LTI] {msg}")
raise LTIError(msg)
# '@type' must be "Result"
result_type = json_obj.get("@type")
if result_type != "Result":
msg = "JSON object does not contain correct @type attribute (should be 'Result', is {})".format(result_type)
log.info("[LTI] {}".format(msg))
msg = f"JSON object does not contain correct @type attribute (should be 'Result', is {result_type})"
log.info(f"[LTI] {msg}")
raise LTIError(msg)
# '@context' must be present as a key
REQUIRED_KEYS = ["@context"] # pylint: disable=invalid-name
for key in REQUIRED_KEYS:
if key not in json_obj:
msg = "JSON object does not contain required key {}".format(key)
log.info("[LTI] {}".format(msg))
msg = f"JSON object does not contain required key {key}"
log.info(f"[LTI] {msg}")
raise LTIError(msg)
# 'resultScore' is not present. If this was a PUT this means it's actually a DELETE according
@@ -360,11 +359,11 @@ class LTI20BlockMixin(object):
score = float(json_obj.get('resultScore', "unconvertable")) # Check if float is present and the right type
if not 0 <= score <= 1:
msg = 'score value outside the permitted range of 0-1.'
log.info("[LTI] {}".format(msg))
log.info(f"[LTI] {msg}")
raise LTIError(msg)
except (TypeError, ValueError) as err:
msg = "Could not convert resultScore to float: {}".format(text_type(err))
log.info("[LTI] {}".format(msg))
msg = "Could not convert resultScore to float: {}".format(str(err))
log.info(f"[LTI] {msg}")
raise LTIError(msg) # lint-amnesty, pylint: disable=raise-missing-from
return score, json_obj.get('comment', "")

89
common/lib/xmodule/xmodule/lti_module.py
View File

@@ -60,16 +60,15 @@ import hashlib
import logging
import textwrap
from xml.sax.saxutils import escape
from unittest import mock
from urllib import parse
import bleach
import mock
import oauthlib.oauth1
import six
from lxml import etree
from oauthlib.oauth1.rfc5849 import signature
from pkg_resources import resource_string
from pytz import UTC
from six import text_type
from webob import Response
from web_fragments.fragment import Fragment
from xblock.core import List, Scope, String, XBlock
@@ -106,7 +105,7 @@ BREAK_TAG = '<br />'
_ = lambda text: text
class LTIFields(object):
class LTIFields:
"""
Fields to define and obtain LTI tool from provider are set here,
except credentials, which should be set in course settings::
@@ -457,7 +456,7 @@ class LTIBlock(
except ValueError:
_ = self.runtime.service(self, "i18n").ugettext
msg = _('Could not parse custom parameter: {custom_parameter}. Should be "x=y" string.').format(
custom_parameter="{0!r}".format(custom_parameter)
custom_parameter=f"{custom_parameter!r}"
)
raise LTIError(msg) # lint-amnesty, pylint: disable=raise-missing-from
@@ -465,7 +464,7 @@ class LTIBlock(
if param_name not in PARAMETERS:
param_name = 'custom_' + param_name
custom_parameters[six.text_type(param_name)] = six.text_type(param_value)
custom_parameters[str(param_name)] = str(param_value)
return self.oauth_params(
custom_parameters,
@@ -532,7 +531,7 @@ class LTIBlock(
def get_user_id(self):
user_id = self.runtime.anonymous_student_id
assert user_id is not None
return six.text_type(six.moves.urllib.parse.quote(user_id))
return str(parse.quote(user_id))
def get_outcome_service_url(self, service_name="grade_handler"):
"""
@@ -578,7 +577,7 @@ class LTIBlock(
i4x-2-3-lti-31de800015cf4afb973356dbe81496df this part of resource_link_id:
makes resource_link_id to be unique among courses inside same system.
"""
return six.text_type(six.moves.urllib.parse.quote("{}-{}".format(self.system.hostname, self.location.html_id()))) # lint-amnesty, pylint: disable=line-too-long
return str(parse.quote(f"{self.system.hostname}-{self.location.html_id()}")) # lint-amnesty, pylint: disable=line-too-long
def get_lis_result_sourcedid(self):
"""
@@ -590,7 +589,7 @@ class LTIBlock(
This field is generally optional, but is required for grading.
"""
return "{context}:{resource_link}:{user_id}".format(
context=six.moves.urllib.parse.quote(self.context_id),
context=parse.quote(self.context_id),
resource_link=self.get_resource_link_id(),
user_id=self.get_user_id()
)
@@ -609,7 +608,7 @@ class LTIBlock(
context_id is an opaque identifier that uniquely identifies the context (e.g., a course)
that contains the link being launched.
"""
return text_type(self.course_id)
return str(self.course_id)
@property
def role(self):
@@ -617,11 +616,11 @@ class LTIBlock(
Get system user role and convert it to LTI role.
"""
roles = {
'student': u'Student',
'staff': u'Administrator',
'instructor': u'Instructor',
'student': 'Student',
'staff': 'Administrator',
'instructor': 'Instructor',
}
return roles.get(self.system.get_user_role(), u'Student')
return roles.get(self.system.get_user_role(), 'Student')
def get_icon_class(self):
""" Returns the icon class """
@@ -640,29 +639,29 @@ class LTIBlock(
"""
client = oauthlib.oauth1.Client(
client_key=text_type(client_key),
client_secret=text_type(client_secret)
client_key=str(client_key),
client_secret=str(client_secret)
)
# Must have parameters for correct signing from LTI:
body = {
u'user_id': self.get_user_id(),
u'oauth_callback': u'about:blank',
u'launch_presentation_return_url': '',
u'lti_message_type': u'basic-lti-launch-request',
u'lti_version': 'LTI-1p0',
u'roles': self.role,
'user_id': self.get_user_id(),
'oauth_callback': 'about:blank',
'launch_presentation_return_url': '',
'lti_message_type': 'basic-lti-launch-request',
'lti_version': 'LTI-1p0',
'roles': self.role,
# Parameters required for grading:
u'resource_link_id': self.get_resource_link_id(),
u'lis_result_sourcedid': self.get_lis_result_sourcedid(),
'resource_link_id': self.get_resource_link_id(),
'lis_result_sourcedid': self.get_lis_result_sourcedid(),
u'context_id': self.context_id,
'context_id': self.context_id,
}
if self.has_score:
body.update({
u'lis_outcome_service_url': self.get_outcome_service_url()
'lis_outcome_service_url': self.get_outcome_service_url()
})
self.user_email = "" # lint-amnesty, pylint: disable=attribute-defined-outside-init
@@ -697,21 +696,21 @@ class LTIBlock(
try:
__, headers, __ = client.sign(
six.text_type(self.launch_url.strip()),
http_method=u'POST',
str(self.launch_url.strip()),
http_method='POST',
body=body,
headers=headers)
except ValueError: # Scheme not in url.
# https://github.com/idan/oauthlib/blob/master/oauthlib/oauth1/rfc5849/signature.py#L136
# Stubbing headers for now:
log.info(
u"LTI module %s in course %s does not have oauth parameters correctly configured.",
"LTI module %s in course %s does not have oauth parameters correctly configured.",
self.location,
self.location.course_key,
)
headers = {
u'Content-Type': u'application/x-www-form-urlencoded',
u'Authorization': u'OAuth oauth_nonce="80966668944732164491378916897", \
'Content-Type': 'application/x-www-form-urlencoded',
'Authorization': 'OAuth oauth_nonce="80966668944732164491378916897", \
oauth_timestamp="1378916897", oauth_version="1.0", oauth_signature_method="HMAC-SHA1", \
oauth_consumer_key="", oauth_signature="frVp4JuvT1mVXlxktiAUjQ7%2F1cw%3D"'}
@@ -719,15 +718,15 @@ oauth_consumer_key="", oauth_signature="frVp4JuvT1mVXlxktiAUjQ7%2F1cw%3D"'}
# Parse headers to pass to template as part of context:
params = dict([param.strip().replace('"', '').split('=') for param in params.split(',')])
params[u'oauth_nonce'] = params[u'OAuth oauth_nonce']
del params[u'OAuth oauth_nonce']
params['oauth_nonce'] = params['OAuth oauth_nonce']
del params['OAuth oauth_nonce']
# oauthlib encodes signature with
# 'Content-Type': 'application/x-www-form-urlencoded'
# so '='' becomes '%3D'.
# We send form via browser, so browser will encode it again,
# So we need to decode signature back:
params[u'oauth_signature'] = six.moves.urllib.parse.unquote(params[u'oauth_signature']).encode('utf-8').decode('utf8') # lint-amnesty, pylint: disable=line-too-long
params['oauth_signature'] = parse.unquote(params['oauth_signature']).encode('utf-8').decode('utf8') # lint-amnesty, pylint: disable=line-too-long
# Add LTI parameters to OAuth parameters for sending in form.
params.update(body)
@@ -816,7 +815,7 @@ oauth_consumer_key="", oauth_signature="frVp4JuvT1mVXlxktiAUjQ7%2F1cw%3D"'}
try:
imsx_messageIdentifier, sourcedId, score, action = self.parse_grade_xml_body(request.body)
except Exception as e: # lint-amnesty, pylint: disable=broad-except
error_message = "Request body XML parsing error: " + escape(text_type(e))
error_message = "Request body XML parsing error: " + escape(str(e))
log.debug("[LTI]: " + error_message) # lint-amnesty, pylint: disable=logging-not-lazy
failure_values['imsx_description'] = error_message
return Response(response_xml_template.format(**failure_values), content_type="application/xml")
@@ -826,12 +825,12 @@ oauth_consumer_key="", oauth_signature="frVp4JuvT1mVXlxktiAUjQ7%2F1cw%3D"'}
self.verify_oauth_body_sign(request)
except (ValueError, LTIError) as e:
failure_values['imsx_messageIdentifier'] = escape(imsx_messageIdentifier)
error_message = "OAuth verification error: " + escape(text_type(e))
error_message = "OAuth verification error: " + escape(str(e))
failure_values['imsx_description'] = error_message
log.debug("[LTI]: " + error_message) # lint-amnesty, pylint: disable=logging-not-lazy
return Response(response_xml_template.format(**failure_values), content_type="application/xml")
real_user = self.system.get_real_user(six.moves.urllib.parse.unquote(sourcedId.split(':')[-1]))
real_user = self.system.get_real_user(parse.unquote(sourcedId.split(':')[-1]))
if not real_user: # that means we can't save to database, as we do not have real user id.
failure_values['imsx_messageIdentifier'] = escape(imsx_messageIdentifier)
failure_values['imsx_description'] = "User not found."
@@ -842,7 +841,7 @@ oauth_consumer_key="", oauth_signature="frVp4JuvT1mVXlxktiAUjQ7%2F1cw%3D"'}
values = {
'imsx_codeMajor': 'success',
'imsx_description': 'Score for {sourced_id} is now {score}'.format(sourced_id=sourcedId, score=score),
'imsx_description': f'Score for {sourcedId} is now {score}',
'imsx_messageIdentifier': escape(imsx_messageIdentifier),
'response': '<replaceResultResponse/>'
}
@@ -900,7 +899,7 @@ oauth_consumer_key="", oauth_signature="frVp4JuvT1mVXlxktiAUjQ7%2F1cw%3D"'}
client_key, client_secret = self.get_client_key_secret() # lint-amnesty, pylint: disable=unused-variable
headers = {
'Authorization': six.text_type(request.headers.get('Authorization')),
'Authorization': str(request.headers.get('Authorization')),
'Content-Type': content_type,
}
@@ -911,14 +910,14 @@ oauth_consumer_key="", oauth_signature="frVp4JuvT1mVXlxktiAUjQ7%2F1cw%3D"'}
oauth_headers = dict(oauth_params)
oauth_signature = oauth_headers.pop('oauth_signature')
mock_request_lti_1 = mock.Mock(
uri=six.text_type(six.moves.urllib.parse.unquote(self.get_outcome_service_url())),
http_method=six.text_type(request.method),
uri=str(parse.unquote(self.get_outcome_service_url())),
http_method=str(request.method),
params=list(oauth_headers.items()),
signature=oauth_signature
)
mock_request_lti_2 = mock.Mock(
uri=six.text_type(six.moves.urllib.parse.unquote(request.url)),
http_method=six.text_type(request.method),
uri=str(parse.unquote(request.url)),
http_method=str(request.method),
params=list(oauth_headers.items()),
signature=oauth_signature
)
@@ -940,7 +939,7 @@ oauth_consumer_key="", oauth_signature="frVp4JuvT1mVXlxktiAUjQ7%2F1cw%3D"'}
"headers:{} url:{} method:{}".format(
oauth_headers,
self.get_outcome_service_url(),
six.text_type(request.method)
str(request.method)
))
raise LTIError("OAuth signature verification has failed.")
@@ -955,7 +954,7 @@ oauth_consumer_key="", oauth_signature="frVp4JuvT1mVXlxktiAUjQ7%2F1cw%3D"'}
except ValueError:
_ = self.runtime.service(self, "i18n").ugettext
msg = _('Could not parse LTI passport: {lti_passport}. Should be "id:key:secret" string.').format(
lti_passport='{0!r}'.format(lti_passport)
lti_passport=f'{lti_passport!r}'
)
raise LTIError(msg) # lint-amnesty, pylint: disable=raise-missing-from

6
common/lib/xmodule/xmodule/mako_module.py
View File

@@ -10,12 +10,12 @@ from .x_module import DescriptorSystem, XModuleDescriptor, shim_xmodule_js
class MakoDescriptorSystem(DescriptorSystem): # lint-amnesty, pylint: disable=abstract-method
def __init__(self, render_template, **kwargs):
super(MakoDescriptorSystem, self).__init__(**kwargs) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(**kwargs)
self.render_template = render_template
class MakoTemplateBlockBase(object):
class MakoTemplateBlockBase:
"""
XBlock intended as a mixin that uses a mako template
to specify the module html.
@@ -27,7 +27,7 @@ class MakoTemplateBlockBase(object):
# pylint: disable=no-member
def __init__(self, *args, **kwargs):
super(MakoTemplateBlockBase, self).__init__(*args, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(*args, **kwargs)
if getattr(self.runtime, 'render_template', None) is None:
raise TypeError(
'{runtime} must have a render_template function'

62
common/lib/xmodule/xmodule/modulestore/__init__.py
View File

@@ -13,12 +13,10 @@ from collections import defaultdict
from contextlib import contextmanager
from operator import itemgetter
import six
from contracts import contract, new_contract
from opaque_keys.edx.keys import AssetKey, CourseKey
from opaque_keys.edx.locations import Location # For import backwards compatibility
from pytz import UTC
from six.moves import range
from sortedcontainers import SortedKeyList
from xblock.core import XBlock
from xblock.plugin import default_select
@@ -39,8 +37,8 @@ new_contract('AssetKey', AssetKey)
new_contract('AssetMetadata', AssetMetadata)
new_contract('XBlock', XBlock)
LIBRARY_ROOT = u'library.xml'
COURSE_ROOT = u'course.xml'
LIBRARY_ROOT = 'library.xml'
COURSE_ROOT = 'course.xml'
# List of names of computed fields on xmodules that are of type usage keys.
# This list can be used to determine which fields need to be stripped of
@@ -48,19 +46,19 @@ COURSE_ROOT = u'course.xml'
XMODULE_FIELDS_WITH_USAGE_KEYS = ['location', 'parent']
class ModuleStoreEnum(object):
class ModuleStoreEnum:
"""
A class to encapsulate common constants that are used with the various modulestores.
"""
class Type(object):
class Type:
"""
The various types of modulestores provided
"""
split = 'split'
mongo = 'mongo'
class RevisionOption(object):
class RevisionOption:
"""
Revision constants to use for Module Store operations
Note: These values are passed into store APIs and only used at run time
@@ -77,7 +75,7 @@ class ModuleStoreEnum(object):
# all revisions are queried
all = 'rev-opt-all'
class Branch(object):
class Branch:
"""
Branch constants to use for stores, such as Mongo, that have only 2 branches: DRAFT and PUBLISHED
Note: These values are taken from server configuration settings, so should not be changed without alerting DevOps # lint-amnesty, pylint: disable=line-too-long
@@ -85,7 +83,7 @@ class ModuleStoreEnum(object):
draft_preferred = 'draft-preferred'
published_only = 'published-only'
class BranchName(object):
class BranchName:
"""
Branch constants to use for stores, such as Split, that have named branches
"""
@@ -93,7 +91,7 @@ class ModuleStoreEnum(object):
published = 'published-branch'
library = 'library'
class UserID(object):
class UserID:
"""
Values for user ID defaults
"""
@@ -112,7 +110,7 @@ class ModuleStoreEnum(object):
# user ID for automatic update by the system
system = -4
class SortOrder(object):
class SortOrder:
"""
Values for sorting asset metadata.
"""
@@ -120,7 +118,7 @@ class ModuleStoreEnum(object):
descending = 2
class BulkOpsRecord(object):
class BulkOpsRecord:
"""
For handling nesting of bulk operations
"""
@@ -161,11 +159,11 @@ class ActiveBulkThread(threading.local):
Add the expected vars to the thread.
"""
def __init__(self, bulk_ops_record_type, **kwargs):
super(ActiveBulkThread, self).__init__(**kwargs) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(**kwargs)
self.records = defaultdict(bulk_ops_record_type)
class BulkOperationsMixin(object):
class BulkOperationsMixin:
"""
This implements the :meth:`bulk_operations` modulestore semantics which handles nested invocations
@@ -178,7 +176,7 @@ class BulkOperationsMixin(object):
mongo_connection.
"""
def __init__(self, *args, **kwargs):
super(BulkOperationsMixin, self).__init__(*args, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(*args, **kwargs)
self._active_bulk_ops = ActiveBulkThread(self._bulk_ops_record_type)
self.signal_handler = None
@@ -209,7 +207,7 @@ class BulkOperationsMixin(object):
# Retrieve the bulk record based on matching org/course/run (possibly ignoring case)
if ignore_case:
for key, record in six.iteritems(self._active_bulk_ops.records):
for key, record in self._active_bulk_ops.records.items():
# Shortcut: check basic equivalence for cases where org/course/run might be None.
if (key == course_key) or ( # lint-amnesty, pylint: disable=too-many-boolean-expressions
(key.org and key.org.lower() == course_key.org.lower()) and
@@ -225,7 +223,7 @@ class BulkOperationsMixin(object):
"""
Yield all active (CourseLocator, BulkOpsRecord) tuples.
"""
for course_key, record in six.iteritems(self._active_bulk_ops.records):
for course_key, record in self._active_bulk_ops.records.items():
if record.active:
yield (course_key, record)
@@ -336,7 +334,7 @@ class BulkOperationsMixin(object):
bulk_ops_record.has_library_updated_item = False
class EditInfo(object):
class EditInfo: # pylint: disable=eq-without-hash
"""
Encapsulates the editing info of a block.
"""
@@ -415,7 +413,7 @@ class EditInfo(object):
return not self == edit_info
class BlockData(object):
class BlockData: # pylint: disable=eq-without-hash
"""
Wrap the block data in an object instead of using a straight Python dictionary.
Allows the storing of meta-information about a structure that doesn't persist along with
@@ -519,7 +517,7 @@ class SortedAssetList(SortedKeyList): # lint-amnesty, pylint: disable=abstract-
if key_func is None:
kwargs['key'] = itemgetter('filename')
self.filename_sort = True
super(SortedAssetList, self).__init__(**kwargs) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(**kwargs)
@contract(asset_id=AssetKey)
def find(self, asset_id):
@@ -556,7 +554,7 @@ class SortedAssetList(SortedKeyList): # lint-amnesty, pylint: disable=abstract-
self.add(metadata_to_insert)
class ModuleStoreAssetBase(object):
class ModuleStoreAssetBase:
"""
The methods for accessing assets and their metadata
"""
@@ -634,7 +632,7 @@ class ModuleStoreAssetBase(object):
if asset_type is None:
# Add assets of all types to the sorted list.
all_assets = SortedAssetList(iterable=[], key=key_func)
for asset_type, val in six.iteritems(course_assets): # lint-amnesty, pylint: disable=redefined-argument-from-local
for asset_type, val in course_assets.items(): # lint-amnesty, pylint: disable=redefined-argument-from-local
all_assets.update(val)
else:
# Add assets of a single type to the sorted list.
@@ -776,7 +774,7 @@ class ModuleStoreAssetWriteInterface(ModuleStoreAssetBase):
pass # lint-amnesty, pylint: disable=unnecessary-pass
class ModuleStoreRead(six.with_metaclass(ABCMeta, ModuleStoreAssetBase)):
class ModuleStoreRead(ModuleStoreAssetBase, metaclass=ABCMeta):
"""
An abstract interface for a database backend that stores XModuleDescriptor
instances and extends read-only functionality
@@ -875,7 +873,7 @@ class ModuleStoreRead(six.with_metaclass(ABCMeta, ModuleStoreAssetBase)):
else:
return True, field
for key, criteria in six.iteritems(qualifiers):
for key, criteria in qualifiers.items():
is_set, value = _is_set_on(key)
if isinstance(criteria, dict) and '$exists' in criteria and criteria['$exists'] == is_set:
continue
@@ -1031,7 +1029,7 @@ class ModuleStoreRead(six.with_metaclass(ABCMeta, ModuleStoreAssetBase)):
pass # lint-amnesty, pylint: disable=unnecessary-pass
class ModuleStoreWrite(six.with_metaclass(ABCMeta, ModuleStoreRead, ModuleStoreAssetWriteInterface)):
class ModuleStoreWrite(ModuleStoreRead, ModuleStoreAssetWriteInterface, metaclass=ABCMeta):
"""
An abstract interface for a database backend that stores XModuleDescriptor
instances and extends both read and write functionality
@@ -1172,7 +1170,7 @@ class ModuleStoreReadBase(BulkOperationsMixin, ModuleStoreRead):
'''
Set up the error-tracking logic.
'''
super(ModuleStoreReadBase, self).__init__(**kwargs) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(**kwargs)
self._course_errors = defaultdict(make_error_tracker) # location -> ErrorLog
# TODO move the inheritance_cache_subsystem to classes which use it
self.metadata_inheritance_cache_subsystem = metadata_inheritance_cache_subsystem
@@ -1260,7 +1258,7 @@ class ModuleStoreReadBase(BulkOperationsMixin, ModuleStoreRead):
"""
if self.contentstore:
self.contentstore.close_connections()
super(ModuleStoreReadBase, self).close_connections() # lint-amnesty, pylint: disable=super-with-arguments
super().close_connections()
@contextmanager
def default_store(self, store_type):
@@ -1268,7 +1266,7 @@ class ModuleStoreReadBase(BulkOperationsMixin, ModuleStoreRead):
A context manager for temporarily changing the default store
"""
if self.get_modulestore_type(None) != store_type:
raise ValueError(u"Cannot set default store to type {}".format(store_type))
raise ValueError(f"Cannot set default store to type {store_type}")
yield
@@ -1278,7 +1276,7 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
Implement interface functionality that can be shared.
'''
def __init__(self, contentstore, **kwargs):
super(ModuleStoreWriteBase, self).__init__(contentstore=contentstore, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(contentstore=contentstore, **kwargs)
self.mixologist = Mixologist(self.xblock_mixins)
def partition_fields_by_scope(self, category, fields):
@@ -1293,7 +1291,7 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
return result
classes = XBlock.load_class(category, select=prefer_xmodules)
cls = self.mixologist.mix(classes)
for field_name, value in six.iteritems(fields):
for field_name, value in fields.items():
field = getattr(cls, field_name)
result[field.scope][field_name] = value
return result
@@ -1338,7 +1336,7 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
# delete the assets
if self.contentstore:
self.contentstore.delete_all_course_assets(course_key)
super(ModuleStoreWriteBase, self).delete_course(course_key, user_id) # lint-amnesty, pylint: disable=super-with-arguments
super().delete_course(course_key, user_id)
def _drop_database(self, database=True, collections=True, connections=True):
"""
@@ -1354,7 +1352,7 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
"""
if self.contentstore:
self.contentstore._drop_database(database, collections, connections) # pylint: disable=protected-access
super(ModuleStoreWriteBase, self)._drop_database(database, collections, connections) # lint-amnesty, pylint: disable=super-with-arguments
super()._drop_database(database, collections, connections)
def create_child(self, user_id, parent_usage_key, block_type, block_id=None, fields=None, **kwargs):
"""

19
common/lib/xmodule/xmodule/modulestore/django.py
View File

@@ -8,7 +8,6 @@ from importlib import import_module
import gettext
import logging
import six
from pkg_resources import resource_filename
import re # lint-amnesty, pylint: disable=wrong-import-order
@@ -68,7 +67,7 @@ class SwitchedSignal(django.dispatch.Signal):
All other args are passed to the constructor for django.dispatch.Signal.
"""
super(SwitchedSignal, self).__init__(*args, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(*args, **kwargs)
self.name = name
self._allow_signals = True
@@ -103,7 +102,7 @@ class SwitchedSignal(django.dispatch.Signal):
"ALLOW" if self._allow_signals else "BLOCK"
)
if self._allow_signals:
return super(SwitchedSignal, self).send(*args, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
return super().send(*args, **kwargs)
return []
def send_robust(self, *args, **kwargs):
@@ -121,14 +120,14 @@ class SwitchedSignal(django.dispatch.Signal):
"ALLOW" if self._allow_signals else "BLOCK"
)
if self._allow_signals:
return super(SwitchedSignal, self).send_robust(*args, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
return super().send_robust(*args, **kwargs)
return []
def __repr__(self):
return u"SwitchedSignal('{}')".format(self.name)
return f"SwitchedSignal('{self.name}')"
class SignalHandler(object):
class SignalHandler:
"""
This class is to allow the modulestores to emit signals that can be caught
by other parts of the Django application. If your app needs to do something
@@ -252,7 +251,7 @@ def create_modulestore_instance(
FUNCTION_KEYS = ['render_template']
for key in FUNCTION_KEYS:
if key in _options and isinstance(_options[key], six.string_types):
if key in _options and isinstance(_options[key], str):
_options[key] = load_function(_options[key])
request_cache = DEFAULT_REQUEST_CACHE
@@ -345,7 +344,7 @@ def clear_existing_modulestores():
_MIXED_MODULESTORE = None
class ModuleI18nService(object):
class ModuleI18nService:
"""
Implement the XBlock runtime "i18n" service.
@@ -379,12 +378,12 @@ class ModuleI18nService(object):
xblock_locale_path,
[to_locale(selected_language if selected_language else settings.LANGUAGE_CODE)]
)
except IOError:
except OSError:
# Fall back to the default Django translator if the XBlock translator is not found.
pass
def __getattr__(self, name):
name = 'gettext' if six.PY3 and name == 'ugettext' else name
name = 'gettext' if name == 'ugettext' else name
return getattr(self.translator, name)
def strftime(self, *args, **kwargs):

25
common/lib/xmodule/xmodule/modulestore/draft_and_published.py
View File

@@ -8,9 +8,6 @@ import threading
from abc import ABCMeta, abstractmethod
from contextlib import contextmanager
import six
from six import text_type
from . import BulkOperationsMixin, ModuleStoreEnum
from .exceptions import ItemNotFoundError
@@ -20,7 +17,7 @@ DIRECT_ONLY_CATEGORIES = ['course', 'chapter', 'sequential', 'about', 'static_ta
log = logging.getLogger(__name__)
class BranchSettingMixin(object):
class BranchSettingMixin:
"""
A mixin to manage a module store's branch setting.
The order of override is (from higher precedence to lower):
@@ -38,7 +35,7 @@ class BranchSettingMixin(object):
'branch_setting_func',
lambda: ModuleStoreEnum.Branch.published_only
)
super(BranchSettingMixin, self).__init__(*args, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(*args, **kwargs)
# cache the branch setting on a local thread to support a multi-threaded environment
self.thread_cache = threading.local()
@@ -71,7 +68,7 @@ class BranchSettingMixin(object):
return self.default_branch_setting_func()
class ModuleStoreDraftAndPublished(six.with_metaclass(ABCMeta, BranchSettingMixin, BulkOperationsMixin)):
class ModuleStoreDraftAndPublished(BranchSettingMixin, BulkOperationsMixin, metaclass=ABCMeta):
"""
A mixin for a read-write database backend that supports two branches, Draft and Published, with
options to prefer Draft and fallback to Published.
@@ -161,7 +158,7 @@ class ModuleStoreDraftAndPublished(six.with_metaclass(ABCMeta, BranchSettingMixi
old_parent_item = self.get_item(old_parent_location) # pylint: disable=no-member
new_parent_item = self.get_item(new_parent_location) # pylint: disable=no-member
except ItemNotFoundError as exception:
log.error('Unable to find the item : %s', text_type(exception))
log.error('Unable to find the item : %s', str(exception))
return
# Remove item from the list of children of old parent.
@@ -170,8 +167,8 @@ class ModuleStoreDraftAndPublished(six.with_metaclass(ABCMeta, BranchSettingMixi
self.update_item(old_parent_item, user_id) # pylint: disable=no-member
log.info(
'%s removed from %s children',
text_type(source_item.location),
text_type(old_parent_item.location)
str(source_item.location),
str(old_parent_item.location)
)
# Add item to new parent at particular location.
@@ -183,8 +180,8 @@ class ModuleStoreDraftAndPublished(six.with_metaclass(ABCMeta, BranchSettingMixi
self.update_item(new_parent_item, user_id) # pylint: disable=no-member
log.info(
'%s added to %s children',
text_type(source_item.location),
text_type(new_parent_item.location)
str(source_item.location),
str(new_parent_item.location)
)
# Update parent attribute of the item block
@@ -192,8 +189,8 @@ class ModuleStoreDraftAndPublished(six.with_metaclass(ABCMeta, BranchSettingMixi
self.update_item(source_item, user_id) # pylint: disable=no-member
log.info(
'%s parent updated to %s',
text_type(source_item.location),
text_type(new_parent_item.location)
str(source_item.location),
str(new_parent_item.location)
)
return source_item.location
@@ -209,4 +206,4 @@ class UnsupportedRevisionError(ValueError):
ModuleStoreEnum.RevisionOption.published_only,
ModuleStoreEnum.RevisionOption.draft_only
]
super(UnsupportedRevisionError, self).__init__('revision not one of {}'.format(allowed_revisions)) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(f'revision not one of {allowed_revisions}')

3
common/lib/xmodule/xmodule/modulestore/edit_info.py
View File

@@ -5,7 +5,6 @@ Access methods to get EditInfo for xblocks
from abc import ABCMeta, abstractmethod
import six
from xblock.core import XBlockMixin
@@ -56,7 +55,7 @@ class EditInfoMixin(XBlockMixin):
return self.runtime.get_published_on(self)
class EditInfoRuntimeMixin(six.with_metaclass(ABCMeta, object)):
class EditInfoRuntimeMixin(metaclass=ABCMeta):
"""
An abstract mixin class for the functions which the :class: `EditInfoMixin` methods call on the runtime
"""

10
common/lib/xmodule/xmodule/modulestore/exceptions.py
View File

@@ -55,7 +55,7 @@ class DuplicateItemError(Exception):
Attempted to create an item which already exists.
"""
def __init__(self, element_id, store=None, collection=None):
super(DuplicateItemError, self).__init__() # lint-amnesty, pylint: disable=super-with-arguments
super().__init__()
self.element_id = element_id
self.store = store
self.collection = collection
@@ -77,7 +77,7 @@ class VersionConflictError(Exception):
The caller asked for either draft or published head and gave a version which conflicted with it.
"""
def __init__(self, requestedLocation, currentHeadVersionGuid):
super(VersionConflictError, self).__init__(u'Requested {}, but current head is {}'.format( # lint-amnesty, pylint: disable=super-with-arguments
super().__init__('Requested {}, but current head is {}'.format(
requestedLocation,
currentHeadVersionGuid
))
@@ -91,8 +91,8 @@ class DuplicateCourseError(Exception):
"""
existing_entry will have the who, when, and other properties of the existing entry
"""
super(DuplicateCourseError, self).__init__( # lint-amnesty, pylint: disable=super-with-arguments
u'Cannot create course {}, which duplicates {}'.format(course_id, existing_entry)
super().__init__(
f'Cannot create course {course_id}, which duplicates {existing_entry}'
)
self.course_id = course_id
self.existing_entry = existing_entry
@@ -103,6 +103,6 @@ class InvalidBranchSetting(Exception):
Raised when the process' branch setting did not match the required setting for the attempted operation on a store.
"""
def __init__(self, expected_setting, actual_setting):
super(InvalidBranchSetting, self).__init__(u"Invalid branch: expected {} but got {}".format(expected_setting, actual_setting)) # lint-amnesty, pylint: disable=line-too-long, super-with-arguments
super().__init__(f"Invalid branch: expected {expected_setting} but got {actual_setting}") # lint-amnesty, pylint: disable=line-too-long, super-with-arguments
self.expected_setting = expected_setting
self.actual_setting = actual_setting

8
common/lib/xmodule/xmodule/modulestore/inheritance.py
View File

@@ -318,7 +318,7 @@ class InheritingFieldData(KvsFieldData):
parents.
"""
super(InheritingFieldData, self).__init__(**kwargs) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(**kwargs)
self.inheritable_names = set(inheritable_names)
def has_default_value(self, name):
@@ -349,14 +349,14 @@ class InheritingFieldData(KvsFieldData):
if ancestor and \
ancestor.location.block_type == 'library_content' and \
self.has_default_value(name):
return super(InheritingFieldData, self).default(block, name) # lint-amnesty, pylint: disable=super-with-arguments
return super().default(block, name)
while ancestor is not None:
if field.is_set_on(ancestor):
return field.read_json(ancestor)
else:
ancestor = ancestor.get_parent()
return super(InheritingFieldData, self).default(block, name) # lint-amnesty, pylint: disable=super-with-arguments
return super().default(block, name)
def inheriting_field_data(kvs):
@@ -375,7 +375,7 @@ class InheritanceKeyValueStore(KeyValueStore):
Note: inherited_settings is a dict of key to json values (internal xblock field repr)
"""
def __init__(self, initial_values=None, inherited_settings=None):
super(InheritanceKeyValueStore, self).__init__() # lint-amnesty, pylint: disable=super-with-arguments
super().__init__()
self.inherited_settings = inherited_settings or {}
self._fields = initial_values or {}

32
common/lib/xmodule/xmodule/modulestore/mixed.py
View File

@@ -11,7 +11,6 @@ import itertools
import logging
from contextlib import contextmanager
import six
from contracts import contract, new_contract
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import AssetKey, CourseKey
@@ -28,10 +27,7 @@ new_contract('CourseKey', CourseKey)
new_contract('AssetKey', AssetKey)
new_contract('AssetMetadata', AssetMetadata)
new_contract('LibraryLocator', LibraryLocator)
if six.PY2:
new_contract('long', long) # lint-amnesty, pylint: disable=undefined-variable
else:
new_contract('long', int)
new_contract('long', int)
log = logging.getLogger(__name__)
@@ -85,7 +81,7 @@ def strip_key(func):
if isinstance(field_value, list):
field_value = [strip_key_func(fv) for fv in field_value]
elif isinstance(field_value, dict):
for key, val in six.iteritems(field_value):
for key, val in field_value.items():
field_value[key] = strip_key_func(val)
else:
field_value = strip_key_func(field_value)
@@ -125,7 +121,7 @@ def prepare_asides_to_store(asides_source):
asides = []
for asd in asides_source:
aside_fields = {}
for asd_field_key, asd_field_val in six.iteritems(asd.fields):
for asd_field_key, asd_field_val in asd.fields.items():
aside_fields[asd_field_key] = asd_field_val.read_from(asd)
asides.append({
'aside_type': asd.scope_ids.block_type,
@@ -154,7 +150,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
Initialize a MixedModuleStore. Here we look into our passed in kwargs which should be a
collection of other modulestore configuration information
"""
super(MixedModuleStore, self).__init__(contentstore, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(contentstore, **kwargs)
if create_modulestore_instance is None:
raise ValueError('MixedModuleStore constructor must be passed a create_modulestore_instance function')
@@ -162,7 +158,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
self.modulestores = []
self.mappings = {}
for course_id, store_name in six.iteritems(mappings):
for course_id, store_name in mappings.items():
try:
self.mappings[CourseKey.from_string(course_id)] = store_name
except InvalidKeyError:
@@ -182,7 +178,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
signal_handler=signal_handler,
)
# replace all named pointers to the store into actual pointers
for course_key, store_name in six.iteritems(self.mappings):
for course_key, store_name in self.mappings.items():
if store_name == key:
self.mappings[course_key] = store
self.modulestores.append(store)
@@ -306,7 +302,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
# Check if course is indeed unique. Save it in result if unique
if course_id in course_summaries:
log.warning(
u"Modulestore %s have duplicate courses %s; skipping from result.", store, course_id
"Modulestore %s have duplicate courses %s; skipping from result.", store, course_id
)
else:
course_summaries[course_id] = course_summary
@@ -338,10 +334,10 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
for store in self.modulestores:
if not hasattr(store, 'get_library_keys'):
continue
all_library_keys |= set(
all_library_keys |= {
self._clean_locator_for_mapping(library_key)
for library_key in store.get_library_keys()
)
}
return list(all_library_keys)
@strip_key
@@ -386,7 +382,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
This key may represent a course that doesn't exist in this modulestore.
"""
# If there is a mapping that match this org/course/run, use that
for course_id, store in six.iteritems(self.mappings):
for course_id, store in self.mappings.items():
candidate_key = store.make_course_key(org, course, run)
if candidate_key == course_id:
return candidate_key
@@ -745,7 +741,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
dest_course_id.course, dest_course_id.run, fields, **kwargs)
# the super handles assets and any other necessities
super(MixedModuleStore, self).clone_course(source_course_id, dest_course_id, user_id, fields, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
super().clone_course(source_course_id, dest_course_id, user_id, fields, **kwargs)
else:
raise NotImplementedError("No code for cloning from {} to {}".format(
source_modulestore, dest_modulestore
@@ -916,7 +912,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
# could be done in parallel threads if needed
return dict(
itertools.chain.from_iterable(
six.iteritems(store.heartbeat())
store.heartbeat().items()
for store in self.modulestores
)
)
@@ -994,7 +990,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
if hasattr(store, method):
return store
else:
raise NotImplementedError(u"Cannot call {} on store {}".format(method, store))
raise NotImplementedError(f"Cannot call {method} on store {store}")
@property
def default_modulestore(self):
@@ -1017,7 +1013,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
# find the store corresponding to the given type
store = next((store for store in self.modulestores if store.get_modulestore_type() == store_type), None)
if not store:
raise Exception(u"Cannot find store of type {}".format(store_type))
raise Exception(f"Cannot find store of type {store_type}")
prev_thread_local_store = getattr(self.thread_cache, 'default_store', None)
try:

6
common/lib/xmodule/xmodule/modulestore/modulestore_settings.py
View File

@@ -6,8 +6,6 @@ This file contains helper functions for configuring module_store_setting setting
import copy
import warnings
import six
def convert_module_store_setting_if_needed(module_store_setting):
"""
@@ -19,7 +17,7 @@ def convert_module_store_setting_if_needed(module_store_setting):
Converts and returns the given stores in old (unordered) dict-style format to the new (ordered) list format
"""
new_store_list = []
for store_name, store_settings in six.iteritems(old_stores):
for store_name, store_settings in old_stores.items():
store_settings['NAME'] = store_name
if store_name == 'default':
@@ -125,7 +123,7 @@ def update_module_store_settings(
mixed_stores.remove(store)
mixed_stores.insert(0, store)
return
raise Exception("Could not find setting for requested default store: {}".format(default_store))
raise Exception(f"Could not find setting for requested default store: {default_store}")
if mappings and 'mappings' in module_store_setting['default']['OPTIONS']:
module_store_setting['default']['OPTIONS']['mappings'] = mappings

124
common/lib/xmodule/xmodule/modulestore/mongo/base.py
View File

@@ -22,7 +22,6 @@ from importlib import import_module
from uuid import uuid4
import pymongo
import six
from bson.son import SON
from contracts import contract, new_contract
from fs.osfs import OSFS
@@ -58,10 +57,7 @@ log = logging.getLogger(__name__)
new_contract('CourseKey', CourseKey)
new_contract('AssetKey', AssetKey)
new_contract('AssetMetadata', AssetMetadata)
if six.PY2:
new_contract('long', long) # lint-amnesty, pylint: disable=undefined-variable
else:
new_contract('long', int)
new_contract('long', int)
new_contract('BlockUsageLocator', BlockUsageLocator)
# sort order that returns DRAFT items first
@@ -70,9 +66,9 @@ SORT_REVISION_FAVOR_DRAFT = ('_id.revision', pymongo.DESCENDING)
# sort order that returns PUBLISHED items first
SORT_REVISION_FAVOR_PUBLISHED = ('_id.revision', pymongo.ASCENDING)
BLOCK_TYPES_WITH_CHILDREN = list(set(
BLOCK_TYPES_WITH_CHILDREN = list({
name for name, class_ in XBlock.load_classes() if getattr(class_, 'has_children', False)
))
})
# Allow us to call _from_deprecated_(son|string) throughout the file
# pylint: disable=protected-access
@@ -81,7 +77,7 @@ BLOCK_TYPES_WITH_CHILDREN = list(set(
_OSFS_INSTANCE = {}
class MongoRevisionKey(object):
class MongoRevisionKey:
"""
Key Revision constants to use for Location and Usage Keys in the Mongo modulestore
Note: These values are persisted in the database, so should not be changed without migrations
@@ -104,7 +100,7 @@ class MongoKeyValueStore(InheritanceKeyValueStore):
known to the MongoModuleStore (data, children, and metadata)
"""
def __init__(self, data, parent, children, metadata):
super(MongoKeyValueStore, self).__init__() # lint-amnesty, pylint: disable=super-with-arguments
super().__init__()
if not isinstance(data, dict):
self._data = {'data': data}
else:
@@ -184,10 +180,10 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin): # li
def __repr__(self):
return "CachingDescriptorSystem{!r}".format((
self.modulestore,
six.text_type(self.course_id),
[six.text_type(key) for key in self.module_data.keys()],
str(self.course_id),
[str(key) for key in self.module_data.keys()],
self.default_class,
[six.text_type(key) for key in self.cached_metadata.keys()],
[str(key) for key in self.cached_metadata.keys()],
))
def __init__(self, modulestore, course_key, module_data, default_class, cached_metadata, **kwargs):
@@ -214,7 +210,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin): # li
id_manager = CourseLocationManager(course_key)
kwargs.setdefault('id_reader', id_manager)
kwargs.setdefault('id_generator', id_manager)
super(CachingDescriptorSystem, self).__init__( # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(
field_data=None,
load_item=self.load_item,
**kwargs
@@ -266,7 +262,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin): # li
parent = None
if self.cached_metadata is not None:
# fish the parent out of here if it's available
parent_url = self.cached_metadata.get(six.text_type(location), {}).get('parent', {}).get(
parent_url = self.cached_metadata.get(str(location), {}).get('parent', {}).get(
ModuleStoreEnum.Branch.published_only if location.branch is None
else ModuleStoreEnum.Branch.draft_preferred
)
@@ -282,7 +278,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin): # li
)
data = definition.get('data', {})
if isinstance(data, six.string_types):
if isinstance(data, str):
data = {'data': data}
mixed_class = self.mixologist.mix(class_)
@@ -306,7 +302,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin): # li
# Convert the serialized fields values in self.cached_metadata
# to python values
metadata_to_inherit = self.cached_metadata.get(six.text_type(non_draft_loc), {})
metadata_to_inherit = self.cached_metadata.get(str(non_draft_loc), {})
inherit_metadata(module, metadata_to_inherit)
module._edit_info = json_data.get('edit_info')
@@ -352,7 +348,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin): # li
:param jsonfields: a dict of the jsonified version of the fields
"""
result = {}
for field_name, value in six.iteritems(jsonfields):
for field_name, value in jsonfields.items():
field = class_.fields.get(field_name)
if field is None:
continue
@@ -366,7 +362,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin): # li
]
elif isinstance(field, ReferenceValueDict):
result[field_name] = {
key: self._convert_reference_to_key(subvalue) for key, subvalue in six.iteritems(value)
key: self._convert_reference_to_key(subvalue) for key, subvalue in value.items()
}
else:
result[field_name] = value
@@ -473,7 +469,7 @@ class MongoBulkOpsRecord(BulkOpsRecord):
Tracks whether there've been any writes per course and disables inheritance generation
"""
def __init__(self):
super(MongoBulkOpsRecord, self).__init__() # lint-amnesty, pylint: disable=super-with-arguments
super().__init__()
self.dirty = False
@@ -506,7 +502,7 @@ class MongoBulkOpsMixin(BulkOperationsMixin):
"""
Returns whether a bulk operation is in progress for the given course.
"""
return super(MongoBulkOpsMixin, self)._is_in_bulk_operation( # lint-amnesty, pylint: disable=super-with-arguments
return super()._is_in_bulk_operation(
course_id.for_branch(None), ignore_case
)
@@ -516,17 +512,17 @@ class ParentLocationCache(dict):
Dict-based object augmented with a more cache-like interface, for internal use.
"""
@contract(key=six.text_type)
@contract(key=str)
def has(self, key):
return key in self
@contract(key=six.text_type, value="BlockUsageLocator | None")
@contract(key=str, value="BlockUsageLocator | None")
def set(self, key, value):
self[key] = value
@contract(value="BlockUsageLocator")
def delete_by_value(self, value):
keys_to_delete = [k for k, v in six.iteritems(self) if v == value]
keys_to_delete = [k for k, v in self.items() if v == value]
for key in keys_to_delete:
del self[key]
@@ -555,7 +551,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
:param doc_store_config: must have a host, db, and collection entries. Other common entries: port, tz_aware.
"""
super(MongoModuleStore, self).__init__(contentstore=contentstore, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(contentstore=contentstore, **kwargs)
def do_connection(
db, collection, host, port=27017, tz_aware=True, user=None, password=None, asset_collection=None, **kwargs
@@ -617,7 +613,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
If connections is True, then close the connection to the database as well.
"""
# drop the assets
super(MongoModuleStore, self)._drop_database(database, collections, connections) # lint-amnesty, pylint: disable=super-with-arguments
super()._drop_database(database, collections, connections)
connection = self.collection.database.client
@@ -673,7 +669,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
ParentLocationCaches associated with the current request (if any).
"""
if self.request_cache is not None:
return self.request_cache.data.setdefault('parent-location-{}'.format(branch), ParentLocationCache())
return self.request_cache.data.setdefault(f'parent-location-{branch}', ParentLocationCache())
else:
return ParentLocationCache()
@@ -698,7 +694,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
# just get the inheritable metadata since that is all we need for the computation
# this minimizes both data pushed over the wire
for field_name in InheritanceMixin.fields:
record_filter['metadata.{0}'.format(field_name)] = 1
record_filter[f'metadata.{field_name}'] = 1
# call out to the DB
resultset = self.collection.find(query, record_filter)
@@ -713,7 +709,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
# manually pick it apart b/c the db has tag and we want as_published revision regardless
location = as_published(BlockUsageLocator._from_deprecated_son(result['_id'], course_id.run))
location_url = six.text_type(location)
location_url = str(location)
if location_url in results_by_url:
# found either draft or live to complement the other revision
# FIXME this is wrong. If the child was moved in draft from one parent to the other, it will
@@ -770,12 +766,12 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
course_id = self.fill_in_run(course_id)
if not force_refresh:
# see if we are first in the request cache (if present)
if self.request_cache is not None and six.text_type(course_id) in self.request_cache.data.get('metadata_inheritance', {}): # lint-amnesty, pylint: disable=line-too-long
return self.request_cache.data['metadata_inheritance'][six.text_type(course_id)]
if self.request_cache is not None and str(course_id) in self.request_cache.data.get('metadata_inheritance', {}): # lint-amnesty, pylint: disable=line-too-long
return self.request_cache.data['metadata_inheritance'][str(course_id)]
# then look in any caching subsystem (e.g. memcached)
if self.metadata_inheritance_cache_subsystem is not None:
tree = self.metadata_inheritance_cache_subsystem.get(six.text_type(course_id), {})
tree = self.metadata_inheritance_cache_subsystem.get(str(course_id), {})
else:
logging.warning(
'Running MongoModuleStore without a metadata_inheritance_cache_subsystem. This is \
@@ -788,7 +784,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
# now write out computed tree to caching subsystem (e.g. memcached), if available
if self.metadata_inheritance_cache_subsystem is not None:
self.metadata_inheritance_cache_subsystem.set(six.text_type(course_id), tree)
self.metadata_inheritance_cache_subsystem.set(str(course_id), tree)
# now populate a request_cache, if available. NOTE, we are outside of the
# scope of the above if: statement so that after a memcache hit, it'll get
@@ -798,7 +794,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
# defined
if 'metadata_inheritance' not in self.request_cache.data:
self.request_cache.data['metadata_inheritance'] = {}
self.request_cache.data['metadata_inheritance'][six.text_type(course_id)] = tree
self.request_cache.data['metadata_inheritance'][str(course_id)] = tree
return tree
@@ -1014,8 +1010,8 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
course_queries = []
for course_key in course_keys:
course_query = {
'_id.{}'.format(value_attr): getattr(course_key, key_attr)
for key_attr, value_attr in six.iteritems({'org': 'org', 'course': 'course', 'run': 'name'})
f'_id.{value_attr}': getattr(course_key, key_attr)
for key_attr, value_attr in {'org': 'org', 'course': 'course', 'run': 'name'}.items()
}
course_query.update(query)
course_queries.append(course_query)
@@ -1141,8 +1137,8 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
location = course_key.make_usage_key('course', course_key.run)
if ignore_case:
course_query = location.to_deprecated_son('_id.')
for key in six.iterkeys(course_query):
if isinstance(course_query[key], six.string_types):
for key in course_query.keys():
if isinstance(course_query[key], str):
course_query[key] = re.compile(r"(?i)^{}$".format(course_query[key]))
else:
course_query = {'_id': location.to_deprecated_son()}
@@ -1266,9 +1262,9 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
qualifier_value = {'$in': qualifier_value}
query['_id.' + field] = qualifier_value
for key, value in six.iteritems((settings or {})):
for key, value in (settings or {}).items():
query['metadata.' + key] = value
for key, value in six.iteritems((content or {})):
for key, value in (content or {}).items():
query['definition.data.' + key] = value
if 'children' in qualifiers:
query['definition.children'] = qualifiers.pop('children')
@@ -1308,8 +1304,8 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
# Check if a course with this org/course has been defined before (case-insensitive)
course_search_location = SON([
('_id.tag', 'i4x'),
('_id.org', re.compile(u'^{}$'.format(course_id.org), re.IGNORECASE)),
('_id.course', re.compile(u'^{}$'.format(course_id.course), re.IGNORECASE)),
('_id.org', re.compile(f'^{course_id.org}$', re.IGNORECASE)),
('_id.course', re.compile(f'^{course_id.course}$', re.IGNORECASE)),
('_id.category', 'course'),
])
courses = self.collection.find(course_search_location, projection={'_id': True})
@@ -1323,7 +1319,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
xblock = self.create_item(user_id, course_id, 'course', course_id.run, fields=fields, **kwargs)
# create any other necessary things as a side effect
super(MongoModuleStore, self).create_course( # lint-amnesty, pylint: disable=super-with-arguments
super().create_course(
org, course, run, user_id, runtime=xblock.runtime, **kwargs
)
@@ -1350,7 +1346,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
if block_type == 'course':
block_id = course_key.run
else:
block_id = u'{}_{}'.format(block_type, uuid4().hex[:5])
block_id = '{}_{}'.format(block_type, uuid4().hex[:5])
if runtime is None:
services = {}
@@ -1391,7 +1387,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
for_parent=kwargs.get('for_parent'),
)
if fields is not None:
for key, value in six.iteritems(fields):
for key, value in fields.items():
setattr(xmodule, key, value)
# decache any pending field settings from init
xmodule.save()
@@ -1415,7 +1411,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
if block_type == 'course':
block_id = course_key.run
else:
block_id = u'{}_{}'.format(block_type, uuid4().hex[:5])
block_id = '{}_{}'.format(block_type, uuid4().hex[:5])
runtime = kwargs.pop('runtime', None)
xblock = self.create_xblock(runtime, course_key, block_type, block_id, **kwargs)
@@ -1545,7 +1541,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
parent_cache = self._get_parent_cache(self.get_branch_setting())
parent_cache.delete_by_value(xblock.location)
for child in xblock.children:
parent_cache.set(six.text_type(child), xblock.location)
parent_cache.set(str(child), xblock.location)
self._update_single_item(xblock.scope_ids.usage_id, payload, allow_not_found=allow_not_found)
@@ -1579,19 +1575,19 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
:param jsonfields: a dict of the jsonified version of the fields
"""
jsonfields = {}
for field_name, field in six.iteritems(xblock.fields):
for field_name, field in xblock.fields.items():
if field.scope == scope and field.is_set_on(xblock):
if field.scope == Scope.parent:
continue
elif isinstance(field, Reference):
jsonfields[field_name] = six.text_type(field.read_from(xblock))
jsonfields[field_name] = str(field.read_from(xblock))
elif isinstance(field, ReferenceList):
jsonfields[field_name] = [
six.text_type(ele) for ele in field.read_from(xblock)
str(ele) for ele in field.read_from(xblock)
]
elif isinstance(field, ReferenceValueDict):
jsonfields[field_name] = {
key: six.text_type(subvalue) for key, subvalue in six.iteritems(field.read_from(xblock))
key: str(subvalue) for key, subvalue in field.read_from(xblock).items()
}
else:
jsonfields[field_name] = field.read_json(xblock)
@@ -1641,19 +1637,19 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
or revision == ModuleStoreEnum.RevisionOption.draft_preferred # lint-amnesty, pylint: disable=consider-using-in
parent_cache = self._get_parent_cache(self.get_branch_setting())
if parent_cache.has(six.text_type(location)):
return parent_cache.get(six.text_type(location))
if parent_cache.has(str(location)):
return parent_cache.get(str(location))
# create a query with tag, org, course, and the children field set to the given location
query = self._course_key_to_son(location.course_key)
query['definition.children'] = six.text_type(location)
query['definition.children'] = str(location)
# if only looking for the PUBLISHED parent, set the revision in the query to None
if revision == ModuleStoreEnum.RevisionOption.published_only:
query['_id.revision'] = MongoRevisionKey.published
def cache_and_return(parent_loc):
parent_cache.set(six.text_type(location), parent_loc)
parent_cache.set(str(location), parent_loc)
return parent_loc
# query the collection, sorting by DRAFT first
@@ -1674,7 +1670,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
if len(non_orphan_parents) > 1: # lint-amnesty, pylint: disable=no-else-raise
# should never have multiple PUBLISHED parents
raise ReferentialIntegrityError(
u"{} parents claim {}".format(len(parents), location)
"{} parents claim {}".format(len(parents), location)
)
else:
return cache_and_return(non_orphan_parents[0].replace(run=location.course_key.run))
@@ -1746,7 +1742,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
if item['_id']['category'] != 'course':
# It would be nice to change this method to return UsageKeys instead of the deprecated string.
item_locs.add(
six.text_type(as_published(BlockUsageLocator._from_deprecated_son(item['_id'], course_key.run)))
str(as_published(BlockUsageLocator._from_deprecated_son(item['_id'], course_key.run)))
)
all_reachable = all_reachable.union(item.get('definition', {}).get('children', []))
item_locs -= all_reachable
@@ -1797,14 +1793,14 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
# A single document exists per course to store the course asset metadata.
course_key = self.fill_in_run(course_key)
if course_key.run is None:
log.warning(u'No run found for combo org "{}" course "{}" on asset request.'.format(
log.warning('No run found for combo org "{}" course "{}" on asset request.'.format(
course_key.org, course_key.course
))
course_assets = None
else:
# Complete course key, so query for asset metadata.
course_assets = self.asset_collection.find_one(
{'course_id': six.text_type(course_key)},
{'course_id': str(course_key)},
)
doc_id = None if course_assets is None else course_assets['_id']
@@ -1814,7 +1810,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
raise ItemNotFoundError(course_key)
else:
# Course exists, so create matching assets document.
course_assets = {'course_id': six.text_type(course_key), 'assets': {}}
course_assets = {'course_id': str(course_key), 'assets': {}}
doc_id = self.asset_collection.insert_one(course_assets).inserted_id
elif isinstance(course_assets['assets'], list):
# This record is in the old course assets format.
@@ -1833,7 +1829,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
"""
Given a asset type, form a key needed to update the proper embedded field in the Mongo doc.
"""
return 'assets.{}'.format(asset_type)
return f'assets.{asset_type}'
@contract(asset_metadata_list='list(AssetMetadata)', user_id='int|long')
def _save_asset_metadata_list(self, asset_metadata_list, user_id, import_only):
@@ -1851,7 +1847,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
# Build an update set with potentially multiple embedded fields.
updates_by_type = {}
for asset_type, assets in six.iteritems(assets_by_type):
for asset_type, assets in assets_by_type.items():
updates_by_type[self._make_mongo_asset_key(asset_type)] = list(assets)
# Update the document.
@@ -1904,8 +1900,8 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
dest_course_key (CourseKey): identifier of course to copy to
"""
source_assets = self._find_course_assets(source_course_key)
dest_assets = {'assets': source_assets.asset_md.copy(), 'course_id': six.text_type(dest_course_key)}
self.asset_collection.delete_many({'course_id': six.text_type(dest_course_key)})
dest_assets = {'assets': source_assets.asset_md.copy(), 'course_id': str(dest_course_key)}
self.asset_collection.delete_many({'course_id': str(dest_course_key)})
# Update the document.
self.asset_collection.insert_one(dest_assets)
@@ -1991,7 +1987,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
self.database.client.admin.command('ismaster')
return {ModuleStoreEnum.Type.mongo: True}
except pymongo.errors.ConnectionFailure:
raise HeartbeatFailure("Can't connect to {}".format(self.database.name), 'mongo') # lint-amnesty, pylint: disable=raise-missing-from
raise HeartbeatFailure(f"Can't connect to {self.database.name}", 'mongo') # lint-amnesty, pylint: disable=raise-missing-from
def ensure_indexes(self):
"""

34
common/lib/xmodule/xmodule/modulestore/mongo/draft.py
View File

@@ -10,10 +10,8 @@ and otherwise returns i4x://org/course/cat/name).
import logging
import pymongo
import six
from opaque_keys.edx.keys import UsageKey
from opaque_keys.edx.locator import BlockUsageLocator
from six import text_type
from xblock.core import XBlock
from openedx.core.lib.cache_utils import request_cached
@@ -173,7 +171,7 @@ class DraftModuleStore(MongoModuleStore):
# Note: does not need to inform the bulk mechanism since after the course is deleted,
# it can't calculate inheritance anyway. Nothing is there to be dirty.
# delete the assets
super(DraftModuleStore, self).delete_course(course_key, user_id) # lint-amnesty, pylint: disable=super-with-arguments
super().delete_course(course_key, user_id) # lint-amnesty, pylint: disable=super-with-arguments
# delete all of the db records for the course
course_query = self._course_key_to_son(course_key)
@@ -189,7 +187,7 @@ class DraftModuleStore(MongoModuleStore):
"""
# check to see if the source course is actually there
if not self.has_course(source_course_id):
raise ItemNotFoundError("Cannot find a course at {0}. Aborting".format(source_course_id))
raise ItemNotFoundError(f"Cannot find a course at {source_course_id}. Aborting")
with self.bulk_operations(dest_course_id):
# verify that the dest_location really is an empty course
@@ -199,14 +197,14 @@ class DraftModuleStore(MongoModuleStore):
if self.collection.count_documents(query, limit=1) > 0:
raise DuplicateCourseError(
dest_course_id,
"Course at destination {0} is not an empty course. "
"Course at destination {} is not an empty course. "
"You can only clone into an empty course. Aborting...".format(
dest_course_id
)
)
# clone the assets
super(DraftModuleStore, self).clone_course(source_course_id, dest_course_id, user_id, fields) # lint-amnesty, pylint: disable=super-with-arguments
super().clone_course(source_course_id, dest_course_id, user_id, fields) # lint-amnesty, pylint: disable=super-with-arguments
# get the whole old course
new_course = self.get_course(dest_course_id)
@@ -217,7 +215,7 @@ class DraftModuleStore(MongoModuleStore):
)
else:
# update fields on existing course
for key, value in six.iteritems(fields):
for key, value in fields.items():
setattr(new_course, key, value)
self.update_item(new_course, user_id)
@@ -242,7 +240,7 @@ class DraftModuleStore(MongoModuleStore):
log.info("Cloning module %s to %s....", original_loc, module.location)
if 'data' in module.fields and module.fields['data'].is_set_on(module) and isinstance(module.data, six.string_types): # lint-amnesty, pylint: disable=line-too-long
if 'data' in module.fields and module.fields['data'].is_set_on(module) and isinstance(module.data, str): # lint-amnesty, pylint: disable=line-too-long
module.data = rewrite_nonportable_content_links(
original_loc.course_key, dest_course_id, module.data
)
@@ -275,7 +273,7 @@ class DraftModuleStore(MongoModuleStore):
# create a query to find all items in the course that have the given location listed as a child
query = self._course_key_to_son(location.course_key)
query['definition.children'] = text_type(location)
query['definition.children'] = str(location)
# find all the items that satisfy the query
parents = self.collection.find(query, {'_id': True}, sort=[SORT_REVISION_FAVOR_DRAFT])
@@ -319,7 +317,7 @@ class DraftModuleStore(MongoModuleStore):
revision = ModuleStoreEnum.RevisionOption.published_only \
if self.get_branch_setting() == ModuleStoreEnum.Branch.published_only \
else ModuleStoreEnum.RevisionOption.draft_preferred
return super(DraftModuleStore, self).get_parent_location(location, revision, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
return super().get_parent_location(location, revision, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
def create_xblock(self, runtime, course_key, block_type, block_id=None, fields=None, **kwargs): # lint-amnesty, pylint: disable=arguments-differ
"""
@@ -332,7 +330,7 @@ class DraftModuleStore(MongoModuleStore):
:param runtime: if you already have an xmodule from the course, the xmodule.runtime value
:param fields: a dictionary of field names and values for the new xmodule
"""
new_block = super(DraftModuleStore, self).create_xblock( # lint-amnesty, pylint: disable=super-with-arguments
new_block = super().create_xblock( # lint-amnesty, pylint: disable=super-with-arguments
runtime, course_key, block_type, block_id, fields, **kwargs
)
new_block.location = self.for_branch_setting(new_block.location)
@@ -481,13 +479,13 @@ class DraftModuleStore(MongoModuleStore):
# if the revision is published, defer to base
if draft_loc.branch == MongoRevisionKey.published:
item = super(DraftModuleStore, self).update_item(xblock, user_id, allow_not_found) # lint-amnesty, pylint: disable=super-with-arguments
item = super().update_item(xblock, user_id, allow_not_found) # lint-amnesty, pylint: disable=super-with-arguments
course_key = xblock.location.course_key
if isPublish or (item.category in DIRECT_ONLY_CATEGORIES and not child_update):
self._flag_publish_event(course_key)
return item
if not super(DraftModuleStore, self).has_item(draft_loc): # lint-amnesty, pylint: disable=super-with-arguments
if not super().has_item(draft_loc): # lint-amnesty, pylint: disable=super-with-arguments
try:
# ignore any descendants which are already draft
self._convert_to_draft(xblock.location, user_id, ignore_if_draft=True)
@@ -499,7 +497,7 @@ class DraftModuleStore(MongoModuleStore):
raise
xblock.location = draft_loc
super(DraftModuleStore, self).update_item(xblock, user_id, allow_not_found, isPublish=isPublish) # lint-amnesty, pylint: disable=super-with-arguments
super().update_item(xblock, user_id, allow_not_found, isPublish=isPublish) # lint-amnesty, pylint: disable=super-with-arguments
return wrap_draft(xblock)
def delete_item(self, location, user_id, revision=None, **kwargs): # lint-amnesty, pylint: disable=arguments-differ
@@ -560,7 +558,7 @@ class DraftModuleStore(MongoModuleStore):
if self.collection.count_documents(query) > 1:
continue
parent_block = super(DraftModuleStore, self).get_item(parent_location) # lint-amnesty, pylint: disable=super-with-arguments
parent_block = super().get_item(parent_location) # lint-amnesty, pylint: disable=super-with-arguments
parent_block.children.remove(location)
parent_block.location = parent_location # ensure the location is with the correct revision
self.update_item(parent_block, user_id, child_update=True)
@@ -663,7 +661,7 @@ class DraftModuleStore(MongoModuleStore):
@request_cached(
# use the XBlock's location value in the cache key
arg_map_function=lambda arg: six.text_type(arg.location if isinstance(arg, XBlock) else arg),
arg_map_function=lambda arg: str(arg.location if isinstance(arg, XBlock) else arg),
# use this store's request_cache
request_cache_getter=lambda args, kwargs: args[1],
)
@@ -858,7 +856,7 @@ class DraftModuleStore(MongoModuleStore):
try:
source_item = self.get_item(item_location)
except ItemNotFoundError:
log.error('Unable to find the item %s', six.text_type(item_location))
log.error('Unable to find the item %s', str(item_location))
return
if source_item.parent and source_item.parent.block_id != original_parent_location.block_id:
@@ -867,7 +865,7 @@ class DraftModuleStore(MongoModuleStore):
def _query_children_for_cache_children(self, course_key, items):
# first get non-draft in a round-trip
to_process_non_drafts = super(DraftModuleStore, self)._query_children_for_cache_children(course_key, items) # lint-amnesty, pylint: disable=super-with-arguments
to_process_non_drafts = super()._query_children_for_cache_children(course_key, items) # lint-amnesty, pylint: disable=super-with-arguments
to_process_dict = {}
for non_draft in to_process_non_drafts:

32
common/lib/xmodule/xmodule/modulestore/mongoengine_fields.py
View File

@@ -4,11 +4,9 @@ Custom field types for mongoengine
import mongoengine
import six
from opaque_keys.edx.keys import CourseKey, UsageKey
from opaque_keys.edx.locations import Location
from six import text_type
class CourseKeyField(mongoengine.StringField):
@@ -17,7 +15,7 @@ class CourseKeyField(mongoengine.StringField):
"""
def __init__(self, **kwargs):
# it'd be useful to add init args such as support_deprecated, force_deprecated
super(CourseKeyField, self).__init__(**kwargs) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(**kwargs)
def to_mongo(self, course_key): # lint-amnesty, pylint: disable=arguments-differ
"""
@@ -26,7 +24,7 @@ class CourseKeyField(mongoengine.StringField):
assert isinstance(course_key, (type(None), CourseKey))
if course_key:
# don't call super as base.BaseField.to_mongo calls to_python() for some odd reason
return text_type(course_key)
return str(course_key)
else:
return None
@@ -35,21 +33,21 @@ class CourseKeyField(mongoengine.StringField):
Deserialize to a CourseKey instance
"""
# calling super b/c it decodes utf (and doesn't have circularity of from_python)
course_key = super(CourseKeyField, self).to_python(course_key) # lint-amnesty, pylint: disable=super-with-arguments
assert isinstance(course_key, (type(None), six.string_types, CourseKey))
course_key = super().to_python(course_key)
assert isinstance(course_key, (type(None), (str,), CourseKey))
if course_key == '':
return None
if isinstance(course_key, six.string_types):
if isinstance(course_key, str):
return CourseKey.from_string(course_key)
else:
return course_key
def validate(self, value):
assert isinstance(value, (type(None), six.string_types, CourseKey))
assert isinstance(value, (type(None), str, CourseKey))
if isinstance(value, CourseKey):
return super(CourseKeyField, self).validate(text_type(value)) # lint-amnesty, pylint: disable=super-with-arguments
return super().validate(str(value))
else:
return super(CourseKeyField, self).validate(value) # lint-amnesty, pylint: disable=super-with-arguments
return super().validate(value)
def prepare_query_value(self, _opt, value):
return self.to_mongo(value)
@@ -66,27 +64,27 @@ class UsageKeyField(mongoengine.StringField):
assert isinstance(location, (type(None), UsageKey))
if location is None:
return None
return super(UsageKeyField, self).to_mongo(text_type(location)) # lint-amnesty, pylint: disable=super-with-arguments
return super().to_mongo(str(location))
def to_python(self, location): # lint-amnesty, pylint: disable=arguments-differ
"""
Deserialize to a UsageKey instance: for now it's a location missing the run
"""
assert isinstance(location, (type(None), six.string_types, UsageKey))
assert isinstance(location, (type(None), str, UsageKey))
if location == '':
return None
if isinstance(location, six.string_types):
location = super(UsageKeyField, self).to_python(location) # lint-amnesty, pylint: disable=super-with-arguments
if isinstance(location, str):
location = super().to_python(location)
return Location.from_string(location)
else:
return location
def validate(self, value):
assert isinstance(value, (type(None), six.string_types, UsageKey))
assert isinstance(value, (type(None), str, UsageKey))
if isinstance(value, UsageKey):
return super(UsageKeyField, self).validate(text_type(value)) # lint-amnesty, pylint: disable=super-with-arguments
return super().validate(str(value))
else:
return super(UsageKeyField, self).validate(value) # lint-amnesty, pylint: disable=super-with-arguments
return super().validate(value)
def prepare_query_value(self, _opt, value):
return self.to_mongo(value)

16
common/lib/xmodule/xmodule/modulestore/perf_tests/generate_asset_xml.py
View File

@@ -1,5 +1,4 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Generates fake XML for asset metadata.
@@ -11,7 +10,6 @@ from datetime import datetime, timedelta
from lxml import etree
from opaque_keys.edx.keys import CourseKey
from six.moves import range
from xmodule.assetstore import AssetMetadata
@@ -24,8 +22,8 @@ except ImportError:
ASSET_XSD_FILE = 'assets.xsd'
# Characters used in name generation below.
NAME_CHARS = u'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-'
NAME_CHARS_W_UNICODE = NAME_CHARS + u'àĚŘDžΦШΩΣӔ'
NAME_CHARS = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-'
NAME_CHARS_W_UNICODE = NAME_CHARS + 'àĚŘDžΦШΩΣӔ'
def coin_flip():
@@ -54,7 +52,7 @@ def filename():
"""
Fake a filename.
"""
fname = u''
fname = ''
for __ in range(random.randint(10, 30)):
fname += random.choice(NAME_CHARS_W_UNICODE)
fname += random.choice(('.jpg', '.pdf', '.png', '.txt'))
@@ -65,7 +63,7 @@ def pathname():
"""
Fake a pathname.
"""
pname = u''
pname = ''
for __ in range(random.randint(2, 3)):
for __ in range(random.randint(5, 10)):
pname += random.choice(NAME_CHARS)
@@ -113,7 +111,7 @@ def versions():
"""
Version string.
"""
return 'v{}.0'.format(ver)
return f'v{ver}.0'
return (ver_str(curr_ver), ver_str(prev_ver))
@@ -195,13 +193,13 @@ def validate_xml(xsd_filename, xml_filename):
"""
Validate a generated XML file against the XSD.
"""
with open(xsd_filename, 'r') as f:
with open(xsd_filename) as f:
schema_root = etree.XML(f.read())
schema = etree.XMLSchema(schema_root)
xmlparser = etree.XMLParser(schema=schema)
with open(xml_filename, 'r') as f:
with open(xml_filename) as f:
etree.fromstring(f.read(), xmlparser)
if click is not None:

23
common/lib/xmodule/xmodule/modulestore/perf_tests/generate_report.py
View File

@@ -1,4 +1,3 @@
"""
Reads the data generated by performance tests and generates a savable
report which can be viewed over time to examine the performance effects of code changes on
@@ -20,7 +19,7 @@ except ImportError:
DB_NAME = 'block_times.db'
class HTMLTable(object):
class HTMLTable:
"""
Simple wrapper for an HTML table.
"""
@@ -53,7 +52,7 @@ class HTMLTable(object):
)
class HTMLDocument(object):
class HTMLDocument:
"""
Simple wrapper for an entire HTML document.
"""
@@ -64,7 +63,7 @@ class HTMLDocument(object):
def add_header(self, level, text):
"""Add a header to the document."""
func_name = "H{}".format(level)
func_name = f"H{level}"
self.body.append(getattr(E, func_name)(text))
def add_to_body(self, elem):
@@ -76,7 +75,7 @@ class HTMLDocument(object):
return lxml.html.tostring(self.html, pretty_print=pretty_print)
class ReportGenerator(object):
class ReportGenerator:
"""
Base class for report generation.
"""
@@ -95,7 +94,7 @@ class ImportExportReportGen(ReportGenerator):
Class which generates report for course import/export performance test data.
"""
def __init__(self, db_name):
super(ImportExportReportGen, self).__init__(db_name) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(db_name)
self._read_timing_data()
def _read_timing_data(self):
@@ -147,7 +146,7 @@ class ImportExportReportGen(ReportGenerator):
columns = ["Asset Metadata Amount", ]
ms_keys = sorted(self.all_modulestore_combos)
for k in ms_keys:
columns.append("{} ({})".format(k, table_type))
columns.append(f"{k} ({table_type})")
phase_table = HTMLTable(columns)
# Make a row for each amount of asset metadata.
@@ -168,7 +167,7 @@ class ImportExportReportGen(ReportGenerator):
value = 0
else:
value = (per_amount[modulestore] - per_phase['0'][modulestore]) / float(amount)
row.append("{}".format(value))
row.append(f"{value}")
phase_table.add_row(row)
# Add the table title and the table.
@@ -183,7 +182,7 @@ class FindReportGen(ReportGenerator):
Class which generates report for asset access performance test data.
"""
def __init__(self, db_name):
super(FindReportGen, self).__init__(db_name) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(db_name)
self._read_timing_data()
def _read_timing_data(self):
@@ -236,7 +235,7 @@ class FindReportGen(ReportGenerator):
columns = ["Asset Metadata Amount", ]
ms_keys = sorted(self.all_modulestores)
for k in ms_keys:
columns.append("Time Taken (ms) ({})".format(k))
columns.append(f"Time Taken (ms) ({k})")
phase_table = HTMLTable(columns)
if phase != 'get_asset_list':
for amount in sorted(per_phase.keys()):
@@ -244,7 +243,7 @@ class FindReportGen(ReportGenerator):
row = [amount, ]
for modulestore in ms_keys:
time_taken = per_amount[modulestore]
row.append("{}".format(time_taken))
row.append(f"{time_taken}")
phase_table.add_row(row)
html.add_header(2, phase)
html.add_to_body(phase_table.table)
@@ -260,7 +259,7 @@ class FindReportGen(ReportGenerator):
for modulestore in ms_keys:
# Each sort has two different ranges retrieved.
time_taken = per_amount[modulestore] / 2.0
row.append("{}".format(time_taken))
row.append(f"{time_taken}")
sort_table.add_row(row)
html.add_header(3, sort)
html.add_to_body(sort_table.table)

2
common/lib/xmodule/xmodule/modulestore/perf_tests/test_asset_import_export.py
View File

@@ -61,7 +61,7 @@ class CrossStoreXMLRoundtrip(unittest.TestCase):
perf_test = True
def setUp(self):
super(CrossStoreXMLRoundtrip, self).setUp() # lint-amnesty, pylint: disable=super-with-arguments
super().setUp()
self.export_dir = mkdtemp()
self.addCleanup(rmtree, self.export_dir, ignore_errors=True)

3
common/lib/xmodule/xmodule/modulestore/search.py
View File

@@ -3,7 +3,6 @@
from logging import getLogger
from six.moves import range
from lms.djangoapps.courseware.masquerade import MASQUERADE_SETTINGS_KEY
from common.djangoapps.student.roles import GlobalStaff
@@ -182,7 +181,7 @@ def navigation_index(position):
try:
navigation_position = int(position.split('_', 1)[0])
except (ValueError, TypeError):
LOGGER.exception(u'Bad position %r passed to navigation_index, will assume first position', position)
LOGGER.exception('Bad position %r passed to navigation_index, will assume first position', position)
navigation_position = 1
return navigation_position

22
common/lib/xmodule/xmodule/modulestore/split_migrator.py
View File

@@ -10,9 +10,7 @@ manipulate storage but use existing api's.
import logging
import six
from opaque_keys.edx.locator import CourseLocator
from six.moves import range
from xblock.fields import Reference, ReferenceList, ReferenceValueDict
from xmodule.modulestore import ModuleStoreEnum
@@ -21,13 +19,13 @@ from xmodule.modulestore.exceptions import ItemNotFoundError
log = logging.getLogger(__name__)
class SplitMigrator(object):
class SplitMigrator:
"""
Copies courses from old mongo to split mongo and sets up location mapping so any references to the old
name will be able to find the new elements.
"""
def __init__(self, split_modulestore, source_modulestore):
super(SplitMigrator, self).__init__() # lint-amnesty, pylint: disable=super-with-arguments
super().__init__()
self.split_modulestore = split_modulestore
self.source_modulestore = source_modulestore
@@ -53,7 +51,7 @@ class SplitMigrator(object):
# create the course: set fields to explicitly_set for each scope, id_root = new_course_locator, master_branch = 'production' # lint-amnesty, pylint: disable=line-too-long
original_course = self.source_modulestore.get_course(source_course_key, **kwargs)
if original_course is None:
raise ItemNotFoundError(six.text_type(source_course_key))
raise ItemNotFoundError(str(source_course_key))
if new_org is None:
new_org = source_course_key.org
@@ -144,12 +142,12 @@ class SplitMigrator(object):
# was in 'direct' so draft is a new version
split_module = self.split_modulestore.get_item(new_locator, **kwargs)
# need to remove any no-longer-explicitly-set values and add/update any now set values.
for name, field in six.iteritems(split_module.fields):
for name, field in split_module.fields.items():
if field.is_set_on(split_module) and not module.fields[name].is_set_on(module):
field.delete_from(split_module)
for field, value in six.iteritems(self._get_fields_translate_references(
for field, value in self._get_fields_translate_references(
module, new_draft_course_loc, published_course_usage_key.block_id, field_names=False
)):
).items():
field.write_to(split_module, value)
_new_module = self.split_modulestore.update_item(split_module, user_id, **kwargs)
@@ -165,12 +163,12 @@ class SplitMigrator(object):
**kwargs
)
awaiting_adoption[module.location] = new_locator
for draft_location, new_locator in six.iteritems(awaiting_adoption):
for draft_location, new_locator in awaiting_adoption.items():
parent_loc = self.source_modulestore.get_parent_location(
draft_location, revision=ModuleStoreEnum.RevisionOption.draft_preferred, **kwargs
)
if parent_loc is None:
log.warning(u'No parent found in source course for %s', draft_location)
log.warning('No parent found in source course for %s', draft_location)
continue
old_parent = self.source_modulestore.get_item(parent_loc, **kwargs)
split_parent_loc = new_draft_course_loc.make_usage_key(
@@ -212,7 +210,7 @@ class SplitMigrator(object):
)
result = {}
for field_name, field in six.iteritems(xblock.fields):
for field_name, field in xblock.fields.items():
if field.is_set_on(xblock):
field_value = field.read_from(xblock)
field_key = field_name if field_names else field
@@ -225,7 +223,7 @@ class SplitMigrator(object):
elif isinstance(field, ReferenceValueDict):
result[field_key] = {
key: get_translation(subvalue)
for key, subvalue in six.iteritems(field_value)
for key, subvalue in field_value.items()
}
else:
result[field_key] = field_value

2
common/lib/xmodule/xmodule/modulestore/split_mongo/__init__.py
View File

@@ -14,7 +14,7 @@ class BlockKey(namedtuple('BlockKey', 'type id')): # lint-amnesty, pylint: disa
@contract(type="string[>0]")
def __new__(cls, type, id): # lint-amnesty, pylint: disable=redefined-builtin
return super(BlockKey, cls).__new__(cls, type, id)
return super().__new__(cls, type, id)
@classmethod
@contract(usage_key=BlockUsageLocator)

9
common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py
View File

@@ -3,7 +3,6 @@
import logging
import sys
import six
from contracts import contract, new_contract
from fs.osfs import OSFS
from lazy import lazy
@@ -70,7 +69,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin): # li
kwargs.setdefault('id_reader', id_manager)
kwargs.setdefault('id_generator', id_manager)
super(CachingDescriptorSystem, self).__init__( # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(
field_data=None,
load_item=self._load_item,
resources_fs=OSFS(root),
@@ -91,7 +90,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin): # li
@contract(returns="dict(BlockKey: BlockKey)")
def _parent_map(self): # lint-amnesty, pylint: disable=missing-function-docstring
parent_map = {}
for block_key, block in six.iteritems(self.course_entry.structure['blocks']):
for block_key, block in self.course_entry.structure['blocks'].items():
for child in block.fields.get('children', []):
parent_map[child] = block_key
return parent_map
@@ -382,10 +381,10 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin): # li
if aside.scope_ids.block_type == aside_type:
return aside
new_aside = super(CachingDescriptorSystem, self).get_aside_of_type(block, aside_type) # lint-amnesty, pylint: disable=super-with-arguments
new_aside = super().get_aside_of_type(block, aside_type)
new_aside._field_data = block._field_data # pylint: disable=protected-access
for key, _ in six.iteritems(new_aside.fields):
for key, _ in new_aside.fields.items():
if isinstance(key, KeyValueStore.Key) and block._field_data.has(new_aside, key): # pylint: disable=protected-access
try:
value = block._field_data.get(new_aside, key) # pylint: disable=protected-access

2
common/lib/xmodule/xmodule/modulestore/split_mongo/definition_lazy_loader.py
View File

@@ -5,7 +5,7 @@ import copy
from opaque_keys.edx.locator import DefinitionLocator
class DefinitionLazyLoader(object):
class DefinitionLazyLoader:
"""
A placeholder to put into an xblock in place of its definition which
when accessed knows how to get its content. Only useful if the containing

40
common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py
View File

@@ -6,6 +6,7 @@ Segregation of pymongo functions from the data modeling mechanisms for split mod
import datetime
import logging
import math
import pickle
import re
import zlib
from contextlib import contextmanager
@@ -13,8 +14,6 @@ from time import time
import pymongo
import pytz
import six
from six.moves import cPickle as pickle
from contracts import check, new_contract
from mongodb_proxy import autoretry_read
# Import this just to export it
@@ -53,7 +52,7 @@ def round_power_2(value):
return math.pow(2, math.ceil(math.log(value, 2)))
class Tagger(object):
class Tagger:
"""
An object used by :class:`QueryTimer` to allow timed code blocks
to add measurements and tags to the timer.
@@ -94,12 +93,12 @@ class Tagger(object):
'{}:{}'.format(name, round_power_2(size))
for name, size in self.measures
] + [
'{}:{}'.format(name, value)
f'{name}:{value}'
for name, value in self.added_tags
]
class QueryTimer(object):
class QueryTimer:
"""
An object that allows timing a block of code while also recording measurements
about that code.
@@ -127,7 +126,7 @@ class QueryTimer(object):
course_context: The course which the query is being made for.
"""
tagger = Tagger(self._sample_rate)
metric_name = "{}.{}".format(self._metric_base, metric_name)
metric_name = f"{self._metric_base}.{metric_name}"
start = time() # lint-amnesty, pylint: disable=unused-variable
try:
@@ -135,7 +134,7 @@ class QueryTimer(object):
finally:
end = time() # lint-amnesty, pylint: disable=unused-variable
tags = tagger.tags
tags.append('course:{}'.format(course_context))
tags.append(f'course:{course_context}')
TIMER = QueryTimer(__name__, 0.01)
@@ -187,14 +186,14 @@ def structure_to_mongo(structure, course_context=None):
check('BlockKey', structure['root'])
check('dict(BlockKey: BlockData)', structure['blocks'])
for block in six.itervalues(structure['blocks']):
for block in structure['blocks'].values():
if 'children' in block.fields:
check('list(BlockKey)', block.fields['children'])
new_structure = dict(structure)
new_structure['blocks'] = []
for block_key, block in six.iteritems(structure['blocks']):
for block_key, block in structure['blocks'].items():
new_block = dict(block.to_storable())
new_block.setdefault('block_type', block_key.type)
new_block['block_id'] = block_key.id
@@ -203,7 +202,7 @@ def structure_to_mongo(structure, course_context=None):
return new_structure
class CourseStructureCache(object):
class CourseStructureCache:
"""
Wrapper around django cache object to cache course structure objects.
The course structures are pickled and compressed when cached.
@@ -239,10 +238,7 @@ class CourseStructureCache(object):
pickled_data = zlib.decompress(compressed_pickled_data)
tagger.measure('uncompressed_size', len(pickled_data))
if six.PY2:
return pickle.loads(pickled_data)
else:
return pickle.loads(pickled_data, encoding='latin-1')
return pickle.loads(pickled_data, encoding='latin-1')
except Exception: # lint-amnesty, pylint: disable=broad-except
# The cached data is corrupt in some way, get rid of it.
log.warning("CourseStructureCache: Bad data in cache for %s", course_context)
@@ -266,7 +262,7 @@ class CourseStructureCache(object):
self.cache.set(key, compressed_pickled_data, None)
class MongoConnection(object):
class MongoConnection:
"""
Segregation of pymongo functions from the data modeling mechanisms for split modulestore.
"""
@@ -300,7 +296,7 @@ class MongoConnection(object):
self.database.client.admin.command('ismaster')
return True
except pymongo.errors.ConnectionFailure:
raise HeartbeatFailure("Can't connect to {}".format(self.database.name), 'mongo') # lint-amnesty, pylint: disable=raise-missing-from
raise HeartbeatFailure(f"Can't connect to {self.database.name}", 'mongo') # lint-amnesty, pylint: disable=raise-missing-from
def get_structure(self, key, course_context=None):
"""
@@ -322,7 +318,7 @@ class MongoConnection(object):
if doc is None:
log.warning(
"doc was None when attempting to retrieve structure for item with key %s",
six.text_type(key)
str(key)
)
return None
tagger_find_one.measure("blocks", len(doc['blocks']))
@@ -431,7 +427,7 @@ class MongoConnection(object):
with TIMER.timer("get_course_index", key):
if ignore_case:
query = {
key_attr: re.compile(u'^{}$'.format(re.escape(getattr(key, key_attr))), re.IGNORECASE)
key_attr: re.compile('^{}$'.format(re.escape(getattr(key, key_attr))), re.IGNORECASE)
for key_attr in ('org', 'course', 'run')
}
else:
@@ -467,11 +463,11 @@ class MongoConnection(object):
query['$or'] = courses_queries
else:
if branch is not None:
query['versions.{}'.format(branch)] = {'$exists': True}
query[f'versions.{branch}'] = {'$exists': True}
if search_targets:
for key, value in six.iteritems(search_targets):
query['search_targets.{}'.format(key)] = value
for key, value in search_targets.items():
query[f'search_targets.{key}'] = value
if org_target:
query['org'] = org_target
@@ -485,7 +481,7 @@ class MongoConnection(object):
courses_queries = []
query = {}
if branch:
query = {'versions.{}'.format(branch): {'$exists': True}}
query = {f'versions.{branch}': {'$exists': True}}
for course_key in course_keys:
course_query = {

105
common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
View File

@@ -62,7 +62,6 @@ import logging
from collections import defaultdict
from importlib import import_module
import six
from bson.objectid import ObjectId
from ccx_keys.locator import CCXBlockUsageLocator, CCXLocator
from contracts import contract, new_contract
@@ -142,7 +141,7 @@ new_contract('XBlock', XBlock)
class SplitBulkWriteRecord(BulkOpsRecord): # lint-amnesty, pylint: disable=missing-class-docstring
def __init__(self):
super(SplitBulkWriteRecord, self).__init__() # lint-amnesty, pylint: disable=super-with-arguments
super().__init__()
self.initial_index = None
self.index = None
self.structures = {}
@@ -187,7 +186,7 @@ class SplitBulkWriteRecord(BulkOpsRecord): # lint-amnesty, pylint: disable=miss
self.structures[structure['_id']] = structure
def __repr__(self):
return u"SplitBulkWriteRecord<{!r}, {!r}, {!r}, {!r}, {!r}>".format(
return "SplitBulkWriteRecord<{!r}, {!r}, {!r}, {!r}, {!r}>".format(
self._active_count,
self.initial_index,
self.index,
@@ -220,7 +219,7 @@ class SplitBulkWriteMixin(BulkOperationsMixin):
return self._bulk_ops_record_type()
if not isinstance(course_key, (CourseLocator, LibraryLocator)):
raise TypeError(u'{!r} is not a CourseLocator or LibraryLocator'.format(course_key))
raise TypeError(f'{course_key!r} is not a CourseLocator or LibraryLocator')
# handle version_guid based retrieval locally
if course_key.org is None or course_key.course is None or course_key.run is None:
return self._active_bulk_ops.records[
@@ -228,7 +227,7 @@ class SplitBulkWriteMixin(BulkOperationsMixin):
]
# handle ignore case and general use
return super(SplitBulkWriteMixin, self)._get_bulk_ops_record( # lint-amnesty, pylint: disable=super-with-arguments
return super()._get_bulk_ops_record(
course_key.replace(branch=None, version_guid=None), ignore_case
)
@@ -237,7 +236,7 @@ class SplitBulkWriteMixin(BulkOperationsMixin):
Clear the record for this course
"""
if not isinstance(course_key, (CourseLocator, LibraryLocator)):
raise TypeError('{!r} is not a CourseLocator or LibraryLocator'.format(course_key))
raise TypeError(f'{course_key!r} is not a CourseLocator or LibraryLocator')
if course_key.org and course_key.course and course_key.run:
del self._active_bulk_ops.records[course_key.replace(branch=None, version_guid=None)]
@@ -263,7 +262,7 @@ class SplitBulkWriteMixin(BulkOperationsMixin):
dirty = False
# If the content is dirty, then update the database
for _id in six.viewkeys(bulk_write_record.structures) - bulk_write_record.structures_in_db:
for _id in bulk_write_record.structures.keys() - bulk_write_record.structures_in_db:
dirty = True
try:
@@ -274,7 +273,7 @@ class SplitBulkWriteMixin(BulkOperationsMixin):
# append only, so if it's already been written, we can just keep going.
log.debug("Attempted to insert duplicate structure %s", _id)
for _id in six.viewkeys(bulk_write_record.definitions) - bulk_write_record.definitions_in_db:
for _id in bulk_write_record.definitions.keys() - bulk_write_record.definitions_in_db:
dirty = True
try:
@@ -455,7 +454,7 @@ class SplitBulkWriteMixin(BulkOperationsMixin):
defs_from_db = list(self.db_connection.get_definitions(list(ids), course_key))
defs_dict = {d.get('_id'): d for d in defs_from_db}
# Add the retrieved definitions to the cache.
bulk_write_record.definitions_in_db.update(six.iterkeys(defs_dict))
bulk_write_record.definitions_in_db.update(defs_dict.keys())
bulk_write_record.definitions.update(defs_dict)
definitions.extend(defs_from_db)
return definitions
@@ -571,7 +570,7 @@ class SplitBulkWriteMixin(BulkOperationsMixin):
'search_targets' not in record.index or
field not in record.index['search_targets'] or
record.index['search_targets'][field] != value
for field, value in six.iteritems(search_targets)
for field, value in search_targets.items()
):
continue
# if we've specified a filter by org,
@@ -718,7 +717,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
:param doc_store_config: must have a host, db, and collection entries. Other common entries: port, tz_aware.
"""
super(SplitMongoModuleStore, self).__init__(contentstore, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(contentstore, **kwargs)
self.db_connection = MongoConnection(**doc_store_config)
@@ -765,7 +764,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
If connections is True, then close the connection to the database as well.
"""
# drop the assets
super(SplitMongoModuleStore, self)._drop_database(database, collections, connections) # lint-amnesty, pylint: disable=super-with-arguments
super()._drop_database(database, collections, connections)
self.db_connection._drop_database(database, collections, connections) # pylint: disable=protected-access
@@ -799,14 +798,14 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
course_key,
[
block.definition
for block in six.itervalues(new_module_data)
for block in new_module_data.values()
]
)
# Turn definitions into a map.
definitions = {definition['_id']: definition
for definition in descendent_definitions}
for block in six.itervalues(new_module_data):
for block in new_module_data.values():
if block.definition in definitions:
definition = definitions[block.definition]
# convert_fields gets done later in the runtime's xblock_from_json
@@ -917,7 +916,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
entry = self.get_structure(course_key, version_guid)
if entry is None:
raise ItemNotFoundError('Structure: {}'.format(version_guid))
raise ItemNotFoundError(f'Structure: {version_guid}')
# b/c more than one course can use same structure, the 'org', 'course',
# 'run', and 'branch' are not intrinsic to structure
@@ -1065,7 +1064,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
if len(course_block) > 1:
raise MultipleCourseBlocksFound(
"Expected 1 course block to be found in the course, but found {0}".format(len(course_block))
"Expected 1 course block to be found in the course, but found {}".format(len(course_block))
)
course_summary = extract_course_summary(course_block[0])
courses_summaries.append(
@@ -1107,7 +1106,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
if len(library_block) > 1:
raise MultipleLibraryBlocksFound(
"Expected 1 library block, but found {0}".format(len(library_block))
"Expected 1 library block, but found {}".format(len(library_block))
)
library_block_fields = library_block[0].fields
@@ -1245,7 +1244,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
if len(items) == 0: # lint-amnesty, pylint: disable=no-else-raise
raise ItemNotFoundError(usage_key)
elif len(items) > 1:
log.debug("Found more than one item for '{}'".format(usage_key))
log.debug(f"Found more than one item for '{usage_key}'")
return items[0]
def get_items(self, course_locator, settings=None, content=None, qualifiers=None, include_orphans=True, **kwargs): # lint-amnesty, pylint: disable=arguments-differ
@@ -1303,14 +1302,14 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
# odd case where we don't search just confirm
block_name = qualifiers.pop('name')
block_ids = []
for block_id, block in six.iteritems(course.structure['blocks']):
for block_id, block in course.structure['blocks'].items():
# Don't do an in comparison blindly; first check to make sure
# that the name qualifier we're looking at isn't a plain string;
# if it is a string, then it should match exactly. If it's other
# than a string, we check whether it contains the block ID; this
# is so a list or other iterable can be passed with multiple
# valid qualifiers.
if isinstance(block_name, six.string_types):
if isinstance(block_name, str):
name_matches = block_id.id == block_name
else:
name_matches = block_id.id in block_name
@@ -1334,7 +1333,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
path_cache = {}
parents_cache = self.build_block_key_to_parents_mapping(course.structure)
for block_id, value in six.iteritems(course.structure['blocks']):
for block_id, value in course.structure['blocks'].items():
if _block_matches_all(value):
if not include_orphans:
if (
@@ -1360,7 +1359,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
:return dict: a dictionary containing mapping of block_keys against their parents.
"""
children_to_parents = defaultdict(list)
for parent_key, value in six.iteritems(structure['blocks']):
for parent_key, value in structure['blocks'].items():
for child_key in value.fields.get('children', []):
children_to_parents[child_key].append(parent_key)
@@ -1452,7 +1451,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
items = set(course.structure['blocks'].keys())
items.remove(course.structure['root'])
blocks = course.structure['blocks']
for block_id, block_data in six.iteritems(blocks):
for block_id, block_data in blocks.items():
items.difference_update(BlockKey(*child) for child in block_data.fields.get('children', []))
if block_data.block_type in detached_categories:
items.discard(block_id)
@@ -1605,7 +1604,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
elif len(possible_roots) == 0:
return None
# convert the results value sets to locators
for k, versions in six.iteritems(result):
for k, versions in result.items():
result[k] = [
block_locator.for_version(version)
for version in versions
@@ -1670,10 +1669,10 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
:param user_id: request.user
"""
def needs_saved():
for key, value in six.iteritems(new_def_data):
for key, value in new_def_data.items():
if key not in old_definition['fields'] or value != old_definition['fields'][key]:
return True
for key, value in six.iteritems(old_definition.get('fields', {})):
for key, value in old_definition.get('fields', {}).items():
if key not in new_def_data:
return True
@@ -1720,7 +1719,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
# moves, its id won't change and will be confusing
serial = 1
while True:
potential_key = BlockKey(category, "{}{}".format(category, serial))
potential_key = BlockKey(category, f"{category}{serial}")
if potential_key not in course_blocks:
return potential_key
serial += 1
@@ -1910,7 +1909,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
"""
source_index = self.get_course_index_info(source_course_id)
if source_index is None:
raise ItemNotFoundError("Cannot find a course at {0}. Aborting".format(source_course_id))
raise ItemNotFoundError(f"Cannot find a course at {source_course_id}. Aborting")
with self.bulk_operations(dest_course_id):
new_course = self.create_course(
@@ -1923,7 +1922,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
**kwargs
)
# don't copy assets until we create the course in case something's awry
super(SplitMongoModuleStore, self).clone_course(source_course_id, dest_course_id, user_id, fields, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
super().clone_course(source_course_id, dest_course_id, user_id, fields, **kwargs)
return new_course
DEFAULT_ROOT_COURSE_BLOCK_ID = 'course'
@@ -2116,7 +2115,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
with self.bulk_operations(course_key):
if allow_not_found and isinstance(block_key.id, (LocalId, type(None))):
fields = {}
for subfields in six.itervalues(partitioned_fields):
for subfields in partitioned_fields.values():
fields.update(subfields)
return self.create_item(
user_id, course_key, block_key.type, fields=fields, asides=asides, force=force
@@ -2129,7 +2128,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
if original_entry is None:
if allow_not_found:
fields = {}
for subfields in six.itervalues(partitioned_fields):
for subfields in partitioned_fields.values():
fields.update(subfields)
return self.create_item(user_id, course_key, block_key.type, block_id=block_key.id, fields=fields,
asides=asides, force=force)
@@ -2260,7 +2259,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
BlockData(**json_data),
**kwargs
)
for field_name, value in six.iteritems((fields or {})):
for field_name, value in (fields or {}).items():
setattr(new_block, field_name, value)
if parent_xblock is not None:
@@ -2437,7 +2436,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
# must be copying the dag root if there's no current dag
root_block_key = source_structure['root']
if not any(root_block_key == BlockKey.from_usage_key(subtree) for subtree in subtree_list):
raise ItemNotFoundError(u'Must publish course root {}'.format(root_block_key))
raise ItemNotFoundError(f'Must publish course root {root_block_key}')
root_source = source_structure['blocks'][root_block_key]
# create branch
destination_structure = self._new_structure(
@@ -2600,7 +2599,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
# Compute a new block ID. This new block ID must be consistent when this
# method is called with the same (source_key, dest_structure) pair
unique_data = "{}:{}:{}".format(
six.text_type(hashable_source_id).encode("utf-8"),
str(hashable_source_id).encode("utf-8"),
block_key.id,
new_parent_block_key.id,
)
@@ -2640,7 +2639,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
# Setting it to the source_block_info structure version here breaks split_draft's has_changes() method.
new_block_info.edit_info.edited_by = user_id
new_block_info.edit_info.edited_on = datetime.datetime.now(UTC)
new_block_info.edit_info.original_usage = six.text_type(usage_key.replace(branch=None, version_guid=None))
new_block_info.edit_info.original_usage = str(usage_key.replace(branch=None, version_guid=None))
new_block_info.edit_info.original_usage_version = source_block_info.edit_info.update_version
dest_structure['blocks'][new_block_key] = new_block_info
@@ -2733,7 +2732,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
"""
# create mapping from each child's key to its parents' keys
child_parent_map = defaultdict(set)
for block_key, block_data in six.iteritems(blocks):
for block_key, block_data in blocks.items():
for child in block_data.fields.get('children', []):
child_parent_map[BlockKey(*child)].add(block_key)
@@ -2764,7 +2763,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
edited_by won't reflect the originals, of course.
"""
# this is the only real delete in the system. should it do something else?
log.info(u"deleting course from split-mongo: %s", course_key)
log.info("deleting course from split-mongo: %s", course_key)
self.delete_course_index(course_key)
# We do NOT call the super class here since we need to keep the assets
@@ -2805,7 +2804,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
for child in block_fields.get('children', []):
try:
if child in inherited_from:
raise Exception(u'Infinite loop detected when inheriting to {}, having already inherited from {}'.format(child, inherited_from)) # lint-amnesty, pylint: disable=line-too-long
raise Exception(f'Infinite loop detected when inheriting to {child}, having already inherited from {inherited_from}') # lint-amnesty, pylint: disable=line-too-long
self.inherit_settings(
block_map,
BlockKey(*child),
@@ -2855,8 +2854,8 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
try:
course_assets = self._lookup_course(course_key).structure.get('assets', {})
except (InsufficientSpecificationError, VersionConflictError) as err: # lint-amnesty, pylint: disable=unused-variable
log.warning(u'Error finding assets for org "%s" course "%s" on asset '
u'request. Either version of course_key is None or invalid.',
log.warning('Error finding assets for org "%s" course "%s" on asset '
'request. Either version of course_key is None or invalid.',
course_key.org, course_key.course)
return {}
@@ -2912,7 +2911,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
course_key, asset_metadata_list, course_assets, user_id, import_only
)
for asset_type, assets in six.iteritems(assets_by_type):
for asset_type, assets in assets_by_type.items():
new_structure['assets'][asset_type] = list(assets)
# update index if appropriate and structures
@@ -3021,7 +3020,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
original_structure = self._lookup_course(course_locator).structure
index_entry = self._get_index_if_valid(course_locator)
new_structure = self.version_structure(course_locator, original_structure, user_id)
for block in six.itervalues(new_structure['blocks']):
for block in new_structure['blocks'].values():
if 'children' in block.fields:
block.fields['children'] = [
block_id for block_id in block.fields['children']
@@ -3064,7 +3063,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
xblock_class = self.mixologist.mix(xblock_class)
# Make a shallow copy, so that we aren't manipulating a cached field dictionary
output_fields = dict(jsonfields)
for field_name, value in six.iteritems(output_fields):
for field_name, value in output_fields.items():
if value:
try:
field = xblock_class.fields.get(field_name)
@@ -3075,7 +3074,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
elif isinstance(field, ReferenceList):
output_fields[field_name] = [robust_usage_key(ele) for ele in value]
elif isinstance(field, ReferenceValueDict):
for key, subvalue in six.iteritems(value):
for key, subvalue in value.items():
value[key] = robust_usage_key(subvalue)
return output_fields
@@ -3124,7 +3123,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
:param fields: a dictionary of fields and values usually only those explicitly set and already
ready for persisting (e.g., references converted to block_ids)
"""
for field_name, field_value in six.iteritems(fields):
for field_name, field_value in fields.items():
if field_name in self.SEARCH_TARGET_DICT:
index_entry.setdefault('search_targets', {})[field_name] = field_value
@@ -3137,7 +3136,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
:param new_id:
"""
if not isinstance(new_id, ObjectId):
raise TypeError('new_id must be an ObjectId, but is {!r}'.format(new_id))
raise TypeError(f'new_id must be an ObjectId, but is {new_id!r}')
index_entry['versions'][branch] = new_id
self.update_course_index(course_key, index_entry)
@@ -3148,7 +3147,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
# explicitly_set_fields_by_scope converts to json; so, avoiding it
# the existing partition_fields_by_scope works on a dict not an xblock
result = defaultdict(dict)
for field in six.itervalues(xblock.fields):
for field in xblock.fields.values():
if field.is_set_on(xblock):
result[field.scope][field.name] = field.read_from(xblock)
return result
@@ -3171,13 +3170,13 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
Handle client possibly setting field to strings rather than keys to get the block_id
"""
# perhaps replace by fixing the views or Field Reference*.from_json to return a Key
if isinstance(reference, six.string_types):
if isinstance(reference, str):
reference = BlockUsageLocator.from_string(reference)
elif isinstance(reference, BlockKey):
return reference
return BlockKey.from_usage_key(reference)
for field_name, value in six.iteritems(fields):
for field_name, value in fields.items():
if value is not None:
if isinstance(xblock_class.fields[field_name], Reference):
fields[field_name] = reference_block_id(value)
@@ -3186,7 +3185,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
reference_block_id(ele) for ele in value
]
elif isinstance(xblock_class.fields[field_name], ReferenceValueDict):
for key, subvalue in six.iteritems(value):
for key, subvalue in value.items():
value[key] = reference_block_id(subvalue)
# should this recurse down dicts and lists just in case they contain datetime?
elif not isinstance(value, datetime.datetime): # don't convert datetimes!
@@ -3229,7 +3228,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
"""
return [
parent_block_key
for parent_block_key, value in six.iteritems(structure['blocks'])
for parent_block_key, value in structure['blocks'].items()
if block_key in value.fields.get('children', [])
]
@@ -3301,7 +3300,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
block_defaults=new_block.defaults
)
# Extend the block's new edit_info with any extra edit_info fields from the source (e.g. original_usage):
for key, val in six.iteritems(new_block.edit_info.to_storable()):
for key, val in new_block.edit_info.to_storable().items():
if getattr(destination_block.edit_info, key) is None:
setattr(destination_block.edit_info, key, val)
@@ -3408,7 +3407,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
result_list.append(aside)
if tmp_new_asides_data:
for _, asd in six.iteritems(tmp_new_asides_data):
for _, asd in tmp_new_asides_data.items():
result_list.append(asd)
updated = True

66
common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py
View File

@@ -38,7 +38,7 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
"""
master_branch = kwargs.pop('master_branch', ModuleStoreEnum.BranchName.draft)
with self.bulk_operations(CourseLocator(org, course, run), ignore_case=True):
item = super(DraftVersioningModuleStore, self).create_course( # lint-amnesty, pylint: disable=super-with-arguments
item = super().create_course(
org, course, run, user_id, master_branch=master_branch, **kwargs
)
if master_branch == ModuleStoreEnum.BranchName.draft and not skip_auto_publish:
@@ -59,7 +59,7 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
def get_course(self, course_id, depth=0, **kwargs):
course_id = self._map_revision_to_branch(course_id)
return super(DraftVersioningModuleStore, self).get_course(course_id, depth=depth, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
return super().get_course(course_id, depth=depth, **kwargs)
def get_library(self, library_id, depth=0, head_validation=True, **kwargs):
if not head_validation and library_id.version_guid:
@@ -67,14 +67,14 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
self, library_id, depth=depth, head_validation=head_validation, **kwargs
)
library_id = self._map_revision_to_branch(library_id)
return super(DraftVersioningModuleStore, self).get_library(library_id, depth=depth, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
return super().get_library(library_id, depth=depth, **kwargs)
def clone_course(self, source_course_id, dest_course_id, user_id, fields=None, revision=None, **kwargs): # lint-amnesty, pylint: disable=arguments-differ
"""
See :py:meth: xmodule.modulestore.split_mongo.split.SplitMongoModuleStore.clone_course
"""
dest_course_id = self._map_revision_to_branch(dest_course_id, revision=revision)
return super(DraftVersioningModuleStore, self).clone_course( # lint-amnesty, pylint: disable=super-with-arguments
return super().clone_course(
source_course_id, dest_course_id, user_id, fields=fields, **kwargs
)
@@ -84,11 +84,11 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
"""
branch_setting = self.get_branch_setting()
if branch_setting == ModuleStoreEnum.Branch.draft_preferred:
return super(DraftVersioningModuleStore, self).get_course_summaries( # lint-amnesty, pylint: disable=super-with-arguments
return super().get_course_summaries(
ModuleStoreEnum.BranchName.draft, **kwargs
)
elif branch_setting == ModuleStoreEnum.Branch.published_only:
return super(DraftVersioningModuleStore, self).get_course_summaries( # lint-amnesty, pylint: disable=super-with-arguments
return super().get_course_summaries(
ModuleStoreEnum.BranchName.published, **kwargs
)
else:
@@ -100,9 +100,9 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
"""
branch_setting = self.get_branch_setting()
if branch_setting == ModuleStoreEnum.Branch.draft_preferred:
return super(DraftVersioningModuleStore, self).get_courses(ModuleStoreEnum.BranchName.draft, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
return super().get_courses(ModuleStoreEnum.BranchName.draft, **kwargs)
elif branch_setting == ModuleStoreEnum.Branch.published_only:
return super(DraftVersioningModuleStore, self).get_courses(ModuleStoreEnum.BranchName.published, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
return super().get_courses(ModuleStoreEnum.BranchName.published, **kwargs)
else:
raise InsufficientSpecificationError()
@@ -124,7 +124,7 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
source_keys = [self._map_revision_to_branch(key) for key in source_keys]
dest_key = self._map_revision_to_branch(dest_key)
head_validation = kwargs.get('head_validation')
new_keys = super(DraftVersioningModuleStore, self).copy_from_template( # lint-amnesty, pylint: disable=super-with-arguments
new_keys = super().copy_from_template(
source_keys, dest_key, user_id, head_validation
)
if dest_key.branch == ModuleStoreEnum.BranchName.draft:
@@ -148,7 +148,7 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
or descriptor.location.block_type in DIRECT_ONLY_CATEGORIES
with self.bulk_operations(descriptor.location.course_key, emit_signals=emit_signals):
item = super(DraftVersioningModuleStore, self).update_item( # lint-amnesty, pylint: disable=super-with-arguments
item = super().update_item(
descriptor,
user_id,
allow_not_found=allow_not_found,
@@ -169,7 +169,7 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
emit_signals = course_key.branch == ModuleStoreEnum.BranchName.published \
or block_type in DIRECT_ONLY_CATEGORIES
with self.bulk_operations(course_key, emit_signals=emit_signals):
item = super(DraftVersioningModuleStore, self).create_item( # lint-amnesty, pylint: disable=super-with-arguments
item = super().create_item(
user_id, course_key, block_type, block_id=block_id,
definition_locator=definition_locator, fields=fields, asides=asides,
force=force, **kwargs
@@ -184,7 +184,7 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
):
parent_usage_key = self._map_revision_to_branch(parent_usage_key)
with self.bulk_operations(parent_usage_key.course_key):
item = super(DraftVersioningModuleStore, self).create_child( # lint-amnesty, pylint: disable=super-with-arguments
item = super().create_child(
user_id, parent_usage_key, block_type, block_id=block_id,
fields=fields, asides=asides, **kwargs
)
@@ -238,7 +238,7 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
self._flag_publish_event(location.course_key)
for branch in branches_to_delete:
branched_location = location.for_branch(branch)
super(DraftVersioningModuleStore, self).delete_item(branched_location, user_id) # lint-amnesty, pylint: disable=super-with-arguments
super().delete_item(branched_location, user_id)
if autopublish_parent:
self.publish(parent_loc.version_agnostic(), user_id, blacklist=EXCLUDE_ALL, **kwargs)
@@ -274,14 +274,14 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
Returns True if location exists in this ModuleStore.
"""
usage_key = self._map_revision_to_branch(usage_key, revision=revision)
return super(DraftVersioningModuleStore, self).has_item(usage_key) # lint-amnesty, pylint: disable=super-with-arguments
return super().has_item(usage_key)
def get_item(self, usage_key, depth=0, revision=None, **kwargs): # lint-amnesty, pylint: disable=arguments-differ
"""
Returns the item identified by usage_key and revision.
"""
usage_key = self._map_revision_to_branch(usage_key, revision=revision)
return super(DraftVersioningModuleStore, self).get_item(usage_key, depth=depth, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
return super().get_item(usage_key, depth=depth, **kwargs)
def get_items(self, course_locator, revision=None, **kwargs): # lint-amnesty, pylint: disable=arguments-differ
"""
@@ -289,7 +289,7 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
the given course_locator.
"""
course_locator = self._map_revision_to_branch(course_locator, revision=revision)
return super(DraftVersioningModuleStore, self).get_items(course_locator, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
return super().get_items(course_locator, **kwargs)
def get_parent_location(self, location, revision=None, **kwargs): # lint-amnesty, pylint: disable=arguments-differ
'''
@@ -306,7 +306,7 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
if revision == ModuleStoreEnum.RevisionOption.draft_preferred:
revision = ModuleStoreEnum.RevisionOption.draft_only
location = self._map_revision_to_branch(location, revision=revision)
return super(DraftVersioningModuleStore, self).get_parent_location(location, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
return super().get_parent_location(location, **kwargs)
def get_block_original_usage(self, usage_key):
"""
@@ -315,18 +315,18 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
copy was inherited.
"""
usage_key = self._map_revision_to_branch(usage_key)
return super(DraftVersioningModuleStore, self).get_block_original_usage(usage_key) # lint-amnesty, pylint: disable=super-with-arguments
return super().get_block_original_usage(usage_key)
def get_orphans(self, course_key, **kwargs):
course_key = self._map_revision_to_branch(course_key)
return super(DraftVersioningModuleStore, self).get_orphans(course_key, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
return super().get_orphans(course_key, **kwargs)
def fix_not_found(self, course_key, user_id): # lint-amnesty, pylint: disable=arguments-differ
"""
Fix any children which point to non-existent blocks in the course's published and draft branches
"""
for branch in [ModuleStoreEnum.RevisionOption.published_only, ModuleStoreEnum.RevisionOption.draft_only]:
super(DraftVersioningModuleStore, self).fix_not_found( # lint-amnesty, pylint: disable=super-with-arguments
super().fix_not_found(
self._map_revision_to_branch(course_key, branch),
user_id
)
@@ -373,7 +373,7 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
Publishes the subtree under location from the draft branch to the published branch
Returns the newly published item.
"""
super(DraftVersioningModuleStore, self).copy( # lint-amnesty, pylint: disable=super-with-arguments
super().copy(
user_id,
# Directly using the replace function rather than the for_branch function
# because for_branch obliterates the version_guid and will lead to missed version conflicts.
@@ -519,14 +519,14 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
See :py:meth `xmodule.modulestore.split_mongo.split.SplitMongoModuleStore.get_course_history_info`
"""
course_locator = self._map_revision_to_branch(course_locator)
return super(DraftVersioningModuleStore, self).get_course_history_info(course_locator) # lint-amnesty, pylint: disable=super-with-arguments
return super().get_course_history_info(course_locator)
def get_course_successors(self, course_locator, version_history_depth=1):
"""
See :py:meth `xmodule.modulestore.split_mongo.split.SplitMongoModuleStore.get_course_successors`
"""
course_locator = self._map_revision_to_branch(course_locator)
return super(DraftVersioningModuleStore, self).get_course_successors( # lint-amnesty, pylint: disable=super-with-arguments
return super().get_course_successors(
course_locator, version_history_depth=version_history_depth
)
@@ -535,7 +535,7 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
See :py:meth `xmodule.modulestore.split_mongo.split.SplitMongoModuleStore.get_block_generations`
"""
block_locator = self._map_revision_to_branch(block_locator)
return super(DraftVersioningModuleStore, self).get_block_generations(block_locator) # lint-amnesty, pylint: disable=super-with-arguments
return super().get_block_generations(block_locator)
def has_published_version(self, xblock):
"""
@@ -614,12 +614,12 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
@contract(asset_key='AssetKey')
def find_asset_metadata(self, asset_key, **kwargs):
return super(DraftVersioningModuleStore, self).find_asset_metadata( # lint-amnesty, pylint: disable=super-with-arguments
return super().find_asset_metadata(
self._map_revision_to_branch(asset_key), **kwargs
)
def get_all_asset_metadata(self, course_key, asset_type, start=0, maxresults=-1, sort=None, **kwargs):
return super(DraftVersioningModuleStore, self).get_all_asset_metadata( # lint-amnesty, pylint: disable=super-with-arguments
return super().get_all_asset_metadata(
self._map_revision_to_branch(course_key), asset_type, start, maxresults, sort, **kwargs
)
@@ -628,11 +628,11 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
Updates both the published and draft branches
"""
# if one call gets an exception, don't do the other call but pass on the exception
super(DraftVersioningModuleStore, self)._update_course_assets( # lint-amnesty, pylint: disable=super-with-arguments
super()._update_course_assets(
user_id, self._map_revision_to_branch(asset_key, ModuleStoreEnum.RevisionOption.published_only),
update_function
)
super(DraftVersioningModuleStore, self)._update_course_assets( # lint-amnesty, pylint: disable=super-with-arguments
super()._update_course_assets(
user_id, self._map_revision_to_branch(asset_key, ModuleStoreEnum.RevisionOption.draft_only),
update_function
)
@@ -646,17 +646,17 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
for asset_md in asset_metadata_list:
asset_key = asset_md.asset_id
asset_md.asset_id = self._map_revision_to_branch(asset_key, ModuleStoreEnum.RevisionOption.published_only)
super(DraftVersioningModuleStore, self).save_asset_metadata_list(asset_metadata_list, user_id, import_only) # lint-amnesty, pylint: disable=super-with-arguments
super().save_asset_metadata_list(asset_metadata_list, user_id, import_only)
for asset_md in asset_metadata_list:
asset_key = asset_md.asset_id
asset_md.asset_id = self._map_revision_to_branch(asset_key, ModuleStoreEnum.RevisionOption.draft_only)
super(DraftVersioningModuleStore, self).save_asset_metadata_list(asset_metadata_list, user_id, import_only) # lint-amnesty, pylint: disable=super-with-arguments
super().save_asset_metadata_list(asset_metadata_list, user_id, import_only)
# Change each asset key back to its original state.
for k in asset_keys:
asset_md.asset_id = k
def _find_course_asset(self, asset_key):
return super(DraftVersioningModuleStore, self)._find_course_asset( # lint-amnesty, pylint: disable=super-with-arguments
return super()._find_course_asset(
self._map_revision_to_branch(asset_key)
)
@@ -664,7 +664,7 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
"""
Split specific lookup
"""
return super(DraftVersioningModuleStore, self)._find_course_assets( # lint-amnesty, pylint: disable=super-with-arguments
return super()._find_course_assets(
self._map_revision_to_branch(course_key)
)
@@ -673,7 +673,7 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
Copies to and from both branches
"""
for revision in [ModuleStoreEnum.RevisionOption.published_only, ModuleStoreEnum.RevisionOption.draft_only]:
super(DraftVersioningModuleStore, self).copy_all_asset_metadata( # lint-amnesty, pylint: disable=super-with-arguments
super().copy_all_asset_metadata(
self._map_revision_to_branch(source_course_key, revision),
self._map_revision_to_branch(dest_course_key, revision),
user_id

7
common/lib/xmodule/xmodule/modulestore/split_mongo/split_mongo_kvs.py
View File

@@ -3,7 +3,6 @@
import copy
from collections import namedtuple
import six
from contracts import contract, new_contract
from opaque_keys.edx.locator import BlockUsageLocator
from xblock.core import XBlockAside
@@ -37,7 +36,7 @@ class SplitMongoKVS(InheritanceKeyValueStore):
(copied from a template block with copy_from_template)
"""
# deepcopy so that manipulations of fields does not pollute the source
super(SplitMongoKVS, self).__init__(copy.deepcopy(initial_values)) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(copy.deepcopy(initial_values))
self._definition = definition # either a DefinitionLazyLoader or the db id of the definition.
# if the db id, then the definition is presumed to be loaded into _fields
@@ -178,7 +177,7 @@ class SplitMongoKVS(InheritanceKeyValueStore):
if self._defaults and key.field_name in self._defaults:
return self._defaults[key.field_name]
# If not, try inheriting from a parent, then use the XBlock type's normal default value:
return super(SplitMongoKVS, self).default(key) # lint-amnesty, pylint: disable=super-with-arguments
return super().default(key)
def _load_definition(self):
"""
@@ -192,7 +191,7 @@ class SplitMongoKVS(InheritanceKeyValueStore):
aside_fields_p = persisted_definition.get('aside_fields')
if aside_fields_p:
aside_fields = self._definition.field_converter(aside_fields_p)
for aside_type, fields in six.iteritems(aside_fields):
for aside_type, fields in aside_fields.items():
self.aside_fields.setdefault(aside_type, {}).update(fields)
# do we want to cache any of the edit_info?
self._definition = None # already loaded

17
common/lib/xmodule/xmodule/modulestore/store_utilities.py
View File

@@ -5,17 +5,16 @@ import re
import uuid
from collections import namedtuple
import six
from xblock.core import XBlock
DETACHED_XBLOCK_TYPES = set(name for name, __ in XBlock.load_tagged_classes("detached"))
DETACHED_XBLOCK_TYPES = {name for name, __ in XBlock.load_tagged_classes("detached")}
def _prefix_only_url_replace_regex(pattern):
"""
Match urls in quotes pulling out the fields from pattern
"""
return re.compile(u"""
return re.compile("""
(?x) # flags=re.VERBOSE
(?P<quote>\\\\?['"]) # the opening quotes
{}
@@ -45,19 +44,19 @@ def rewrite_nonportable_content_links(source_course_id, dest_course_id, text):
# create a serialized template for what the id will look like in the source_course but with
# the block_id as a regex pattern
placeholder_id = uuid.uuid4().hex
asset_block_pattern = six.text_type(source_course_id.make_asset_key('asset', placeholder_id))
asset_block_pattern = str(source_course_id.make_asset_key('asset', placeholder_id))
asset_block_pattern = asset_block_pattern.replace(placeholder_id, r'(?P<block_id>.*?)')
try:
text = _prefix_only_url_replace_regex(asset_block_pattern).sub(portable_asset_link_subtitution, text)
except Exception as exc: # pylint: disable=broad-except
logging.warning("Error producing regex substitution %r for text = %r.\n\nError msg = %s", asset_block_pattern, text, str(exc)) # lint-amnesty, pylint: disable=line-too-long
placeholder_category = 'cat_{}'.format(uuid.uuid4().hex)
usage_block_pattern = six.text_type(source_course_id.make_usage_key(placeholder_category, placeholder_id))
placeholder_category = f'cat_{uuid.uuid4().hex}'
usage_block_pattern = str(source_course_id.make_usage_key(placeholder_category, placeholder_id))
usage_block_pattern = usage_block_pattern.replace(placeholder_category, r'(?P<category>[^/+@]+)')
usage_block_pattern = usage_block_pattern.replace(placeholder_id, r'(?P<block_id>.*?)')
jump_to_link_base = u'/courses/{course_key_string}/jump_to/{usage_key_string}'.format(
course_key_string=six.text_type(source_course_id), usage_key_string=usage_block_pattern
jump_to_link_base = '/courses/{course_key_string}/jump_to/{usage_key_string}'.format(
course_key_string=str(source_course_id), usage_key_string=usage_block_pattern
)
try:
text = _prefix_only_url_replace_regex(jump_to_link_base).sub(portable_jump_to_link_substitution, text)
@@ -72,7 +71,7 @@ def rewrite_nonportable_content_links(source_course_id, dest_course_id, text):
#
if source_course_id != dest_course_id:
try:
generic_courseware_link_base = u'/courses/{}/'.format(six.text_type(source_course_id))
generic_courseware_link_base = '/courses/{}/'.format(str(source_course_id))
text = re.sub(_prefix_only_url_replace_regex(generic_courseware_link_base), portable_asset_link_subtitution, text) # lint-amnesty, pylint: disable=line-too-long
except Exception as exc: # pylint: disable=broad-except
logging.warning("Error producing regex substitution %r for text = %r.\n\nError msg = %s", source_course_id, text, str(exc)) # lint-amnesty, pylint: disable=line-too-long