Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: pyupgrade in openedx/content app (#26890)

Browse Source
This commit is contained in:
M. Zulqarnain
2021-03-17 17:27:48 +05:00
committed by GitHub
parent b578dba357
commit 0d86ebb361
36 changed files with 215 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
openedx/core/djangoapps/content/block_structure/admin.py
View File

@@ -17,7 +17,7 @@ class BlockStructureAdmin(ConfigurationModelAdmin):
"""
Excludes unused 'enabled field from super's list.
"""
displayable_field_names = super(BlockStructureAdmin, self).get_displayable_field_names() # lint-amnesty, pylint: disable=super-with-arguments
displayable_field_names = super().get_displayable_field_names()
displayable_field_names.remove('enabled')
return displayable_field_names

2
openedx/core/djangoapps/content/block_structure/apps.py
View File

@@ -10,7 +10,7 @@ class BlockStructureConfig(AppConfig):
"""
block_structure django app.
"""
name = u'openedx.core.djangoapps.content.block_structure'
name = 'openedx.core.djangoapps.content.block_structure'
def ready(self):
"""

34
openedx/core/djangoapps/content/block_structure/block_structure.py
View File

@@ -14,8 +14,6 @@ from copy import deepcopy
from functools import partial
from logging import getLogger
import six
from openedx.core.lib.graph_traversals import traverse_post_order, traverse_topologically
from .exceptions import TransformerException
@@ -27,7 +25,7 @@ logger = getLogger(__name__) # pylint: disable=invalid-name
TRANSFORMER_VERSION_KEY = '_version'
class _BlockRelations(object):
class _BlockRelations:
"""
Data structure to encapsulate relationships for a single block,
including its children and parents.
@@ -43,7 +41,7 @@ class _BlockRelations(object):
self.children = []
class BlockStructure(object):
class BlockStructure:
"""
Base class for a block structure. BlockStructures are constructed
using the BlockStructureFactory and then used as the currency across
@@ -148,7 +146,7 @@ class BlockStructure(object):
iterator(UsageKey) - An iterator of the usage
keys of all the blocks in the block structure.
"""
return six.iterkeys(self._block_relations)
return iter(self._block_relations.keys())
#--- Block structure traversal methods ---#
@@ -280,7 +278,7 @@ class BlockStructure(object):
block_relations[usage_key] = _BlockRelations()
class FieldData(object):
class FieldData:
"""
Data structure to encapsulate collected fields.
"""
@@ -299,21 +297,21 @@ class FieldData(object):
def __getattr__(self, field_name):
if self._is_own_field(field_name):
return super(FieldData, self).__getattr__(field_name) # lint-amnesty, pylint: disable=no-member, super-with-arguments
return super().__getattr__(field_name) # lint-amnesty, pylint: disable=no-member
try:
return self.fields[field_name]
except KeyError:
raise AttributeError(u"Field {0} does not exist".format(field_name)) # lint-amnesty, pylint: disable=raise-missing-from
raise AttributeError(f"Field {field_name} does not exist") # lint-amnesty, pylint: disable=raise-missing-from
def __setattr__(self, field_name, field_value):
if self._is_own_field(field_name):
return super(FieldData, self).__setattr__(field_name, field_value) # lint-amnesty, pylint: disable=super-with-arguments
return super().__setattr__(field_name, field_value)
else:
self.fields[field_name] = field_value
def __delattr__(self, field_name):
if self._is_own_field(field_name):
return super(FieldData, self).__delattr__(field_name) # lint-amnesty, pylint: disable=super-with-arguments
return super().__delattr__(field_name)
else:
del self.fields[field_name]
@@ -383,10 +381,10 @@ class BlockData(FieldData):
Data structure to encapsulate collected data for a single block.
"""
def class_field_names(self):
return super(BlockData, self).class_field_names() + ['location', 'transformer_data'] # lint-amnesty, pylint: disable=super-with-arguments
return super().class_field_names() + ['location', 'transformer_data']
def __init__(self, usage_key):
super(BlockData, self).__init__() # lint-amnesty, pylint: disable=super-with-arguments
super().__init__()
# Location (or usage key) of the block.
self.location = usage_key
@@ -407,7 +405,7 @@ class BlockStructureBlockData(BlockStructure):
VERSION = 2
def __init__(self, root_block_usage_key):
super(BlockStructureBlockData, self).__init__(root_block_usage_key) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(root_block_usage_key)
# Map of a block's usage key to its collected data, including
# its xBlock fields and block-specific transformer data.
@@ -435,14 +433,14 @@ class BlockStructureBlockData(BlockStructure):
Returns iterator of (UsageKey, BlockData) pairs for all
blocks in the BlockStructure.
"""
return six.iteritems(self._block_data_map)
return iter(self._block_data_map.items())
def itervalues(self):
"""
Returns iterator of BlockData for all blocks in the
BlockStructure.
"""
return six.itervalues(self._block_data_map)
return iter(self._block_data_map.values())
def __getitem__(self, usage_key):
"""
@@ -750,7 +748,7 @@ class BlockStructureBlockData(BlockStructure):
its current version number.
"""
if transformer.WRITE_VERSION == 0:
raise TransformerException(u'Version attributes are not set on transformer {0}.', transformer.name()) # lint-amnesty, pylint: disable=raising-format-tuple
raise TransformerException('Version attributes are not set on transformer {0}.', transformer.name()) # lint-amnesty, pylint: disable=raising-format-tuple
self.set_transformer_data(transformer, TRANSFORMER_VERSION_KEY, transformer.WRITE_VERSION)
def _get_or_create_block(self, usage_key):
@@ -778,7 +776,7 @@ class BlockStructureModulestoreData(BlockStructureBlockData):
interface and implementation of an xBlock.
"""
def __init__(self, root_block_usage_key):
super(BlockStructureModulestoreData, self).__init__(root_block_usage_key) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(root_block_usage_key)
# Map of a block's usage key to its instantiated xBlock.
# dict {UsageKey: XBlock}
@@ -838,7 +836,7 @@ class BlockStructureModulestoreData(BlockStructureBlockData):
Iterates through all instantiated xBlocks that were added and
collects all xBlock fields that were requested.
"""
for xblock_usage_key, xblock in six.iteritems(self._xblock_map):
for xblock_usage_key, xblock in self._xblock_map.items():
block_data = self._get_or_create_block(xblock_usage_key)
for field_name in self._requested_xblock_fields:
self._set_xblock_field(block_data, xblock, field_name)

4
openedx/core/djangoapps/content/block_structure/config/models.py
View File

@@ -18,7 +18,7 @@ class BlockStructureConfiguration(ConfigurationModel):
DEFAULT_PRUNE_KEEP_COUNT = 5
DEFAULT_CACHE_TIMEOUT_IN_SECONDS = 60 * 60 * 24 # 24 hours
class Meta(object):
class Meta:
app_label = 'block_structure'
db_table = 'block_structure_config'
@@ -26,7 +26,7 @@ class BlockStructureConfiguration(ConfigurationModel):
cache_timeout_in_seconds = IntegerField(blank=True, null=True, default=DEFAULT_CACHE_TIMEOUT_IN_SECONDS)
def __str__(self):
return u"BlockStructureConfiguration: num_versions_to_keep: {}, cache_timeout_in_seconds: {}".format(
return "BlockStructureConfiguration: num_versions_to_keep: {}, cache_timeout_in_seconds: {}".format(
self.num_versions_to_keep,
self.cache_timeout_in_seconds,
)

4
openedx/core/djangoapps/content/block_structure/exceptions.py
View File

@@ -37,6 +37,6 @@ class BlockStructureNotFound(BlockStructureException):
Exception for when a Block Structure is not found.
"""
def __init__(self, root_block_usage_key):
super(BlockStructureNotFound, self).__init__( # lint-amnesty, pylint: disable=super-with-arguments
u'Block structure not found; data_usage_key: {}'.format(root_block_usage_key)
super().__init__(
f'Block structure not found; data_usage_key: {root_block_usage_key}'
)

2
openedx/core/djangoapps/content/block_structure/factory.py
View File

@@ -4,7 +4,7 @@ Module for factory class for BlockStructure objects.
from .block_structure import BlockStructureBlockData, BlockStructureModulestoreData
class BlockStructureFactory(object):
class BlockStructureFactory:
"""
Factory class for BlockStructure objects.
"""

40
openedx/core/djangoapps/content/block_structure/management/commands/generate_course_blocks.py
View File

@@ -5,9 +5,7 @@ Command to load course blocks.
import logging
import six
from django.core.management.base import BaseCommand
from six import text_type
import openedx.core.djangoapps.content.block_structure.api as api
import openedx.core.djangoapps.content.block_structure.store as store
@@ -29,8 +27,8 @@ class Command(BaseCommand):
$ ./manage.py lms generate_course_blocks --all_courses --settings=devstack
$ ./manage.py lms generate_course_blocks 'edX/DemoX/Demo_Course' --settings=devstack
"""
args = u'<course_id course_id ...>'
help = u'Generates and stores course blocks for one or more courses.'
args = '<course_id course_id ...>'
help = 'Generates and stores course blocks for one or more courses.'
def add_arguments(self, parser):
"""
@@ -40,46 +38,46 @@ class Command(BaseCommand):
'--courses',
dest='courses',
nargs='+',
help=u'Generate course blocks for the list of courses provided.',
help='Generate course blocks for the list of courses provided.',
)
parser.add_argument(
'--all_courses',
help=u'Generate course blocks for all courses, given the requested start and end indices.',
help='Generate course blocks for all courses, given the requested start and end indices.',
action='store_true',
default=False,
)
parser.add_argument(
'--enqueue_task',
help=u'Enqueue the tasks for asynchronous computation.',
help='Enqueue the tasks for asynchronous computation.',
action='store_true',
default=False,
)
parser.add_argument(
'--routing_key',
dest='routing_key',
help=u'Routing key to use for asynchronous computation.',
help='Routing key to use for asynchronous computation.',
)
parser.add_argument(
'--force_update',
help=u'Force update of the course blocks for the requested courses.',
help='Force update of the course blocks for the requested courses.',
action='store_true',
default=False,
)
parser.add_argument(
'--start_index',
help=u'Starting index of course list.',
help='Starting index of course list.',
default=0,
type=int,
)
parser.add_argument(
'--end_index',
help=u'Ending index of course list.',
help='Ending index of course list.',
default=0,
type=int,
)
parser.add_argument(
'--with_storage',
help=u'Store the course blocks in Storage, overriding value of the storage_backing_for_cache waffle switch',
help='Store the course blocks in Storage, overriding value of the storage_backing_for_cache waffle switch',
action='store_true',
default=False,
)
@@ -101,9 +99,9 @@ class Command(BaseCommand):
self._set_log_levels(options)
log.critical(u'BlockStructure: STARTED generating Course Blocks for %d courses.', len(course_keys))
log.critical('BlockStructure: STARTED generating Course Blocks for %d courses.', len(course_keys))
self._generate_course_blocks(options, course_keys)
log.critical(u'BlockStructure: FINISHED generating Course Blocks for %d courses.', len(course_keys))
log.critical('BlockStructure: FINISHED generating Course Blocks for %d courses.', len(course_keys))
def _set_log_levels(self, options):
"""
@@ -139,9 +137,9 @@ class Command(BaseCommand):
self._generate_for_course(options, course_key)
except Exception as ex: # pylint: disable=broad-except
log.exception(
u'BlockStructure: An error occurred while generating course blocks for %s: %s',
six.text_type(course_key),
text_type(ex),
'BlockStructure: An error occurred while generating course blocks for %s: %s',
str(course_key),
str(ex),
)
def _generate_for_course(self, options, course_key):
@@ -152,12 +150,12 @@ class Command(BaseCommand):
action = tasks.update_course_in_cache_v2 if options.get('force_update') else tasks.get_course_in_cache_v2
task_options = {'routing_key': options['routing_key']} if options.get('routing_key') else {}
result = action.apply_async(
kwargs=dict(course_id=six.text_type(course_key), with_storage=options.get('with_storage')),
kwargs=dict(course_id=str(course_key), with_storage=options.get('with_storage')),
**task_options
)
log.info(u'BlockStructure: ENQUEUED generating for course: %s, task_id: %s.', course_key, result.id)
log.info('BlockStructure: ENQUEUED generating for course: %s, task_id: %s.', course_key, result.id)
else:
log.info(u'BlockStructure: STARTED generating for course: %s.', course_key)
log.info('BlockStructure: STARTED generating for course: %s.', course_key)
action = api.update_course_in_cache if options.get('force_update') else api.get_course_in_cache
action(course_key)
log.info(u'BlockStructure: FINISHED generating for course: %s.', course_key)
log.info('BlockStructure: FINISHED generating for course: %s.', course_key)

12
openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py
View File

@@ -2,14 +2,12 @@
Tests for generate_course_blocks management command.
"""
from unittest.mock import patch
import itertools
import pytest
import ddt
from django.core.management.base import CommandError
from mock import patch
import six
from six.moves import range
from openedx.core.djangoapps.content.block_structure.tests.helpers import (
is_course_in_block_structure_cache,
@@ -32,7 +30,7 @@ class TestGenerateCourseBlocks(ModuleStoreTestCase):
"""
Create courses in modulestore.
"""
super(TestGenerateCourseBlocks, self).setUp() # lint-amnesty, pylint: disable=super-with-arguments
super().setUp()
self.courses = [CourseFactory.create() for _ in range(self.num_courses)]
self.course_keys = [course.id for course in self.courses]
self.command = generate_course_blocks.Command()
@@ -88,13 +86,13 @@ class TestGenerateCourseBlocks(ModuleStoreTestCase):
def test_one_course(self):
self._assert_courses_not_in_block_cache(*self.course_keys)
self.command.handle(courses=[six.text_type(self.course_keys[0])])
self.command.handle(courses=[str(self.course_keys[0])])
self._assert_courses_in_block_cache(self.course_keys[0])
self._assert_courses_not_in_block_cache(*self.course_keys[1:])
self._assert_courses_not_in_block_storage(*self.course_keys)
def test_with_storage(self):
self.command.handle(with_storage=True, courses=[six.text_type(self.course_keys[0])])
self.command.handle(with_storage=True, courses=[str(self.course_keys[0])])
self._assert_courses_in_block_cache(self.course_keys[0])
self._assert_courses_in_block_storage(self.course_keys[0])
self._assert_courses_not_in_block_storage(*self.course_keys[1:])
@@ -170,7 +168,7 @@ class TestGenerateCourseBlocks(ModuleStoreTestCase):
)
@ddt.unpack
def test_dependent_options_error(self, dependent_option, depending_on_option):
expected_error_message = 'Option --{} requires option --{}.'.format(dependent_option, depending_on_option)
expected_error_message = f'Option --{dependent_option} requires option --{depending_on_option}.'
options = {dependent_option: 1, depending_on_option: False, 'courses': ['some/course/key']}
with self.assertRaisesMessage(CommandError, expected_error_message):
self.command.handle(**options)

10
openedx/core/djangoapps/content/block_structure/manager.py
View File

@@ -6,8 +6,6 @@ BlockStructures.
from contextlib import contextmanager
import six
from . import config
from .exceptions import BlockStructureNotFound, TransformerDataIncompatible, UsageKeyNotInBlockStructure
from .factory import BlockStructureFactory
@@ -15,7 +13,7 @@ from .store import BlockStructureStore
from .transformers import BlockStructureTransformers
class BlockStructureManager(object):
class BlockStructureManager:
"""
Top-level class for managing Block Structures.
"""
@@ -72,9 +70,9 @@ class BlockStructureManager(object):
# as part of the transformation.
if starting_block_usage_key not in block_structure:
raise UsageKeyNotInBlockStructure( # lint-amnesty, pylint: disable=raising-format-tuple
u"The requested usage_key '{0}' is not found in the block_structure with root '{1}'",
six.text_type(starting_block_usage_key),
six.text_type(self.root_block_usage_key),
"The requested usage_key '{0}' is not found in the block_structure with root '{1}'",
str(starting_block_usage_key),
str(self.root_block_usage_key),
)
block_structure.set_root_block(starting_block_usage_key)
transformers.transform(block_structure)

3
openedx/core/djangoapps/content/block_structure/migrations/0001_config.py
View File

@@ -1,6 +1,3 @@
# -*- coding: utf-8 -*-
from django.db import migrations, models
import django.db.models.deletion
from django.conf import settings

3
openedx/core/djangoapps/content/block_structure/migrations/0002_blockstructuremodel.py
View File

@@ -1,6 +1,3 @@
# -*- coding: utf-8 -*-
from django.db import migrations, models
import django.utils.timezone
import model_utils.fields

3
openedx/core/djangoapps/content/block_structure/migrations/0003_blockstructuremodel_storage.py
View File

@@ -1,6 +1,3 @@
# -*- coding: utf-8 -*-
from django.db import migrations, models
import openedx.core.djangoapps.content.block_structure.models

3
openedx/core/djangoapps/content/block_structure/migrations/0004_blockstructuremodel_usagekeywithrun.py
View File

@@ -1,6 +1,3 @@
# -*- coding: utf-8 -*-
from django.db import migrations, models
import openedx.core.djangoapps.xmodule_django.models

50
openedx/core/djangoapps/content/block_structure/models.py
View File

@@ -8,8 +8,6 @@ from contextlib import contextmanager
from datetime import datetime
from logging import getLogger
import six
from six.moves import map
from django.conf import settings
from django.core.exceptions import SuspiciousOperation
from django.core.files.base import ContentFile
@@ -30,7 +28,7 @@ def _create_path(directory, filename):
"""
Returns the full path for the given directory and filename.
"""
return '{}/{}'.format(directory, filename)
return f'{directory}/{filename}'
def _directory_name(data_usage_key):
@@ -48,7 +46,7 @@ def _directory_name(data_usage_key):
# replace any '/' in the usage key so they aren't interpreted
# as folder separators.
encoded_usage_key = six.text_type(data_usage_key).replace('/', '_')
encoded_usage_key = str(data_usage_key).replace('/', '_')
return '{}{}'.format(
directory_prefix,
encoded_usage_key,
@@ -102,10 +100,10 @@ class CustomizableFileField(models.FileField):
storage=_bs_model_storage(),
max_length=500, # allocate enough for base path + prefix + usage_key + timestamp in filepath
))
super(CustomizableFileField, self).__init__(*args, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments
super().__init__(*args, **kwargs)
def deconstruct(self): # lint-amnesty, pylint: disable=missing-function-docstring
name, path, args, kwargs = super(CustomizableFileField, self).deconstruct() # lint-amnesty, pylint: disable=super-with-arguments
name, path, args, kwargs = super().deconstruct()
del kwargs['upload_to']
del kwargs['storage']
del kwargs['max_length']
@@ -135,7 +133,7 @@ def _storage_error_handling(bs_model, operation, is_read_operation=False):
try:
yield
except Exception as error: # pylint: disable=broad-except
log.exception(u'BlockStructure: Exception %s on store %s; %s.', error.__class__, operation, bs_model)
log.exception('BlockStructure: Exception %s on store %s; %s.', error.__class__, operation, bs_model)
if isinstance(error, OSError) and error.errno in (errno.EACCES, errno.EPERM): # lint-amnesty, pylint: disable=no-else-raise, no-member
raise
elif is_read_operation and isinstance(error, (IOError, SuspiciousOperation)):
@@ -155,40 +153,40 @@ class BlockStructureModel(TimeStampedModel):
.. no_pii:
"""
VERSION_FIELDS = [
u'data_version',
u'data_edit_timestamp',
u'transformers_schema_version',
u'block_structure_schema_version',
'data_version',
'data_edit_timestamp',
'transformers_schema_version',
'block_structure_schema_version',
]
UNIQUENESS_FIELDS = [u'data_usage_key'] + VERSION_FIELDS
UNIQUENESS_FIELDS = ['data_usage_key'] + VERSION_FIELDS
class Meta(object):
class Meta:
db_table = 'block_structure'
data_usage_key = UsageKeyWithRunField(
u'Identifier of the data being collected.',
'Identifier of the data being collected.',
blank=False,
max_length=255,
unique=True,
)
data_version = models.CharField(
u'Version of the data at the time of collection.',
'Version of the data at the time of collection.',
blank=True,
null=True,
max_length=255,
)
data_edit_timestamp = models.DateTimeField(
u'Edit timestamp of the data at the time of collection.',
'Edit timestamp of the data at the time of collection.',
blank=True,
null=True,
)
transformers_schema_version = models.CharField(
u'Representation of the schema version of the transformers used during collection.',
'Representation of the schema version of the transformers used during collection.',
blank=False,
max_length=255,
)
block_structure_schema_version = models.CharField(
u'Version of the block structure schema at the time of collection.',
'Version of the block structure schema at the time of collection.',
blank=False,
max_length=255,
)
@@ -198,7 +196,7 @@ class BlockStructureModel(TimeStampedModel):
"""
Returns the collected data for this instance.
"""
operation = u'Read'
operation = 'Read'
with _storage_error_handling(self, operation, is_read_operation=True):
serialized_data = self.data.read()
@@ -215,7 +213,7 @@ class BlockStructureModel(TimeStampedModel):
try:
return cls.objects.get(data_usage_key=data_usage_key)
except cls.DoesNotExist:
log.info(u'BlockStructure: Not found in table; %s.', data_usage_key)
log.info('BlockStructure: Not found in table; %s.', data_usage_key)
raise BlockStructureNotFound(data_usage_key) # lint-amnesty, pylint: disable=raise-missing-from
@classmethod
@@ -229,7 +227,7 @@ class BlockStructureModel(TimeStampedModel):
# unless the file is successfully persisted.
with transaction.atomic():
bs_model, created = cls.objects.update_or_create(defaults=kwargs, data_usage_key=data_usage_key)
operation = u'Created' if created else u'Updated'
operation = 'Created' if created else 'Updated'
with _storage_error_handling(bs_model, operation):
bs_model.data.save('', ContentFile(serialized_data))
@@ -245,8 +243,8 @@ class BlockStructureModel(TimeStampedModel):
"""
Returns a string representation of this model.
"""
return u', '.join(
u'{}: {}'.format(field_name, six.text_type(getattr(self, field_name)))
return ', '.join(
'{}: {}'.format(field_name, str(getattr(self, field_name)))
for field_name in self.UNIQUENESS_FIELDS
)
@@ -266,7 +264,7 @@ class BlockStructureModel(TimeStampedModel):
files_to_delete = all_files_by_date[:-num_to_keep] if num_to_keep > 0 else all_files_by_date # lint-amnesty, pylint: disable=invalid-unary-operand-type
cls._delete_files(files_to_delete)
log.info(
u'BlockStructure: Deleted %d out of total %d files in store; data_usage_key: %s, num_to_keep: %d.',
'BlockStructure: Deleted %d out of total %d files in store; data_usage_key: %s, num_to_keep: %d.',
len(files_to_delete),
len(all_files_by_date),
data_usage_key,
@@ -274,7 +272,7 @@ class BlockStructureModel(TimeStampedModel):
)
except Exception: # pylint: disable=broad-except
log.exception(u'BlockStructure: Exception when deleting old files; data_usage_key: %s.', data_usage_key)
log.exception('BlockStructure: Exception when deleting old files; data_usage_key: %s.', data_usage_key)
@classmethod
def _delete_files(cls, files):
@@ -303,7 +301,7 @@ class BlockStructureModel(TimeStampedModel):
Writes log information for the given values.
"""
log.info(
u'BlockStructure: %s in store %s at %s%s; %s, size: %d',
'BlockStructure: %s in store %s at %s%s; %s, size: %d',
operation,
bs_model.data.storage.__class__,
getattr(bs_model.data.storage, 'bucket_name', ''),

5
openedx/core/djangoapps/content/block_structure/signals.py
View File

@@ -4,7 +4,6 @@ Signal handlers for invalidating cached data.
import logging
import six
from django.conf import settings
from django.dispatch.dispatcher import receiver
from opaque_keys.edx.locator import LibraryLocator
@@ -34,12 +33,12 @@ def update_block_structure_on_course_publish(sender, course_key, **kwargs): # p
clear_course_from_cache(course_key)
except BlockStructureNotFound:
log.warning(
u"BlockStructure: %s not found when trying to clear course from cache",
"BlockStructure: %s not found when trying to clear course from cache",
course_key,
)
update_course_in_cache_v2.apply_async(
kwargs=dict(course_id=six.text_type(course_key)),
kwargs=dict(course_id=str(course_key)),
countdown=settings.BLOCK_STRUCTURES_SETTINGS['COURSE_PUBLISH_TASK_DELAY'],
)

23
openedx/core/djangoapps/content/block_structure/store.py
View File

@@ -6,7 +6,6 @@ Module for the Storage of BlockStructure objects.
from logging import getLogger
import six
from django.utils.encoding import python_2_unicode_compatible
from openedx.core.lib.cache_utils import zpickle, zunpickle
@@ -22,7 +21,7 @@ logger = getLogger(__name__) # pylint: disable=C0103
@python_2_unicode_compatible
class StubModel(object):
class StubModel:
"""
Stub model to use when storage backing is disabled.
By using this stub, we eliminate the need for extra
@@ -33,7 +32,7 @@ class StubModel(object):
self.data_usage_key = root_block_usage_key
def __str__(self):
return six.text_type(self.data_usage_key)
return str(self.data_usage_key)
def delete(self):
"""
@@ -42,7 +41,7 @@ class StubModel(object):
pass # lint-amnesty, pylint: disable=unnecessary-pass
class BlockStructureStore(object):
class BlockStructureStore:
"""
Storage for BlockStructure objects.
"""
@@ -116,7 +115,7 @@ class BlockStructureStore(object):
bs_model = self._get_model(root_block_usage_key)
self._cache.delete(self._encode_root_cache_key(bs_model))
bs_model.delete()
logger.info(u"BlockStructure: Deleted from cache and store; %s.", bs_model)
logger.info("BlockStructure: Deleted from cache and store; %s.", bs_model)
def is_up_to_date(self, root_block_usage_key, modulestore):
"""
@@ -165,7 +164,7 @@ class BlockStructureStore(object):
"""
cache_key = self._encode_root_cache_key(bs_model)
self._cache.set(cache_key, serialized_data, timeout=config.cache_timeout_in_seconds())
logger.info(u"BlockStructure: Added to cache; %s, size: %d", bs_model, len(serialized_data))
logger.info("BlockStructure: Added to cache; %s, size: %d", bs_model, len(serialized_data))
def _get_from_cache(self, bs_model):
"""
@@ -178,7 +177,7 @@ class BlockStructureStore(object):
serialized_data = self._cache.get(cache_key)
if not serialized_data:
logger.info(u"BlockStructure: Not found in cache; %s.", bs_model)
logger.info("BlockStructure: Not found in cache; %s.", bs_model)
raise BlockStructureNotFound(bs_model.data_usage_key)
return serialized_data
@@ -215,7 +214,7 @@ class BlockStructureStore(object):
except Exception:
# Somehow failed to de-serialized the data, assume it's corrupt.
bs_model = self._get_model(root_block_usage_key)
logger.exception(u"BlockStructure: Failed to load data from cache for %s", bs_model)
logger.exception("BlockStructure: Failed to load data from cache for %s", bs_model)
raise BlockStructureNotFound(bs_model.data_usage_key) # lint-amnesty, pylint: disable=raise-missing-from
return BlockStructureFactory.create_new(
@@ -232,10 +231,10 @@ class BlockStructureStore(object):
BlockStructureModel or StubModel.
"""
if config.STORAGE_BACKING_FOR_CACHE.is_enabled():
return six.text_type(bs_model)
return str(bs_model)
return "v{version}.root.key.{root_usage_key}".format(
version=six.text_type(BlockStructureBlockData.VERSION),
root_usage_key=six.text_type(bs_model.data_usage_key),
version=str(BlockStructureBlockData.VERSION),
root_usage_key=str(bs_model.data_usage_key),
)
@staticmethod
@@ -248,7 +247,7 @@ class BlockStructureStore(object):
data_version=getattr(root_block, 'course_version', None),
data_edit_timestamp=getattr(root_block, 'subtree_edited_on', None),
transformers_schema_version=TransformerRegistry.get_write_version_hash(),
block_structure_schema_version=six.text_type(BlockStructureBlockData.VERSION),
block_structure_schema_version=str(BlockStructureBlockData.VERSION),
)
@staticmethod

6
openedx/core/djangoapps/content/block_structure/tasks.py
View File

@@ -108,18 +108,18 @@ def _call_and_retry_if_needed(self, api_method, **kwargs):
except NO_RETRY_TASKS:
# Known unrecoverable errors
log.exception(
u"BlockStructure: %s encountered unrecoverable error in course %s, task_id %s",
"BlockStructure: %s encountered unrecoverable error in course %s, task_id %s",
self.__name__,
kwargs.get('course_id'),
self.request.id,
)
raise
except RETRY_TASKS as exc:
log.exception(u"%s encountered expected error, retrying.", self.__name__)
log.exception("%s encountered expected error, retrying.", self.__name__)
raise self.retry(kwargs=kwargs, exc=exc)
except Exception as exc:
log.exception(
u"BlockStructure: %s encountered unknown error in course %s, task_id %s. Retry #%d",
"BlockStructure: %s encountered unknown error in course %s, task_id %s. Retry #%d",
self.__name__,
kwargs.get('course_id'),
self.request.id,

29
openedx/core/djangoapps/content/block_structure/tests/helpers.py
View File

@@ -5,9 +5,8 @@ Common utilities for tests in block_structure module
from contextlib import contextmanager
from uuid import uuid4
from unittest.mock import patch
import six
from mock import patch
from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator
from xmodule.modulestore.exceptions import ItemNotFoundError
@@ -45,7 +44,7 @@ def is_course_in_block_structure_storage(course_key, store):
return False
class MockXBlock(object):
class MockXBlock:
"""
A mock XBlock to be used in unit tests, thereby decoupling the
implementation of the block cache framework from the xBlock
@@ -72,7 +71,7 @@ class MockXBlock(object):
return [self.modulestore.get_item(child) for child in self.children]
class MockModulestore(object):
class MockModulestore:
"""
A mock Modulestore to be used in unit tests, providing only the
minimum methods needed by the block cache framework.
@@ -112,7 +111,7 @@ class MockModulestore(object):
yield
class MockCache(object):
class MockCache:
"""
A mock Cache object, providing only the minimum features needed
by the block cache framework.
@@ -145,7 +144,7 @@ class MockCache(object):
del self.map[key]
class MockModulestoreFactory(object):
class MockModulestoreFactory:
"""
A factory for creating MockModulestore objects.
"""
@@ -228,7 +227,7 @@ def mock_registered_transformers(transformers):
yield
class ChildrenMapTestMixin(object):
class ChildrenMapTestMixin:
"""
A Test Mixin with utility methods for testing with block structures
created and manipulated using children_map and parents_map.
@@ -305,34 +304,34 @@ class ChildrenMapTestMixin(object):
for block_key, children in enumerate(children_map):
# Verify presence
assert (self.block_key_factory(block_key) in block_structure) == (block_key not in missing_blocks),\
u'Expected presence in block_structure for block_key {} to match absence in missing_blocks.'\
.format(six.text_type(block_key))
'Expected presence in block_structure for block_key {} to match absence in missing_blocks.'\
.format(str(block_key))
# Verify children
if block_key not in missing_blocks:
assert set(block_structure.get_children(self.block_key_factory(block_key))) ==\
set((self.block_key_factory(child) for child in children))
{self.block_key_factory(child) for child in children}
# Verify parents
parents_map = self.get_parents_map(children_map)
for block_key, parents in enumerate(parents_map):
if block_key not in missing_blocks:
assert set(block_structure.get_parents(self.block_key_factory(block_key))) ==\
set((self.block_key_factory(parent) for parent in parents))
{self.block_key_factory(parent) for parent in parents}
class UsageKeyFactoryMixin(object):
class UsageKeyFactoryMixin:
"""
Test Mixin that provides a block_key_factory to create OpaqueKey objects
for block_ids rather than simple integers. By default, the children maps in
ChildrenMapTestMixin use integers for block_ids.
"""
def setUp(self):
super(UsageKeyFactoryMixin, self).setUp() # lint-amnesty, pylint: disable=super-with-arguments
self.course_key = CourseLocator('org', 'course', six.text_type(uuid4()))
super().setUp()
self.course_key = CourseLocator('org', 'course', str(uuid4()))
def block_key_factory(self, block_id):
"""
Returns a block key object for the given block_id.
"""
return BlockUsageLocator(course_key=self.course_key, block_type='course', block_id=six.text_type(block_id))
return BlockUsageLocator(course_key=self.course_key, block_type='course', block_id=str(block_id))

6
openedx/core/djangoapps/content/block_structure/tests/test_block_structure.py
View File

@@ -11,8 +11,6 @@ from datetime import datetime
from unittest import TestCase
import ddt
import six
from six.moves import range
from openedx.core.lib.graph_traversals import traverse_post_order
@@ -101,7 +99,7 @@ class TestBlockStructureData(TestCase, ChildrenMapTestMixin):
block_structure._add_transformer(t_info.transformer)
for key, val in t_info.structure_wide_data:
block_structure.set_transformer_data(t_info.transformer, key, val)
for block, block_data in six.iteritems(t_info.block_specific_data):
for block, block_data in t_info.block_specific_data.items():
for key, val in block_data:
block_structure.set_transformer_block_field(block, t_info.transformer, key, val)
@@ -110,7 +108,7 @@ class TestBlockStructureData(TestCase, ChildrenMapTestMixin):
assert block_structure._get_transformer_data_version(t_info.transformer) == MockTransformer.WRITE_VERSION
for key, val in t_info.structure_wide_data:
assert block_structure.get_transformer_data(t_info.transformer, key) == val
for block, block_data in six.iteritems(t_info.block_specific_data):
for block, block_data in t_info.block_specific_data.items():
for key, val in block_data:
assert block_structure.get_transformer_block_field(block, t_info.transformer, key) == val

2
openedx/core/djangoapps/content/block_structure/tests/test_factory.py
View File

@@ -19,7 +19,7 @@ class TestBlockStructureFactory(TestCase, ChildrenMapTestMixin):
"""
def setUp(self):
super(TestBlockStructureFactory, self).setUp() # lint-amnesty, pylint: disable=super-with-arguments
super().setUp()
self.children_map = self.SIMPLE_CHILDREN_MAP
self.modulestore = MockModulestoreFactory.create(self.children_map, self.block_key_factory)

5
openedx/core/djangoapps/content/block_structure/tests/test_manager.py
View File

@@ -4,7 +4,6 @@ Tests for manager.py
import pytest
import ddt
import six
from django.test import TestCase
from edx_toggles.toggles.testutils import override_waffle_switch
@@ -92,7 +91,7 @@ class TestTransformer1(MockTransformer):
Returns a unique deterministic value for the given block key
and data key.
"""
return data_key + 't1.val1.' + six.text_type(block_key)
return data_key + 't1.val1.' + str(block_key)
@ddt.ddt
@@ -102,7 +101,7 @@ class TestBlockStructureManager(UsageKeyFactoryMixin, ChildrenMapTestMixin, Test
"""
def setUp(self):
super(TestBlockStructureManager, self).setUp() # lint-amnesty, pylint: disable=super-with-arguments
super().setUp()
TestTransformer1.collect_call_count = 0
self.registered_transformers = [TestTransformer1()]

29
openedx/core/djangoapps/content/block_structure/tests/test_models.py
View File

@@ -2,20 +2,17 @@
Unit tests for Block Structure models.
"""
# pylint: disable=protected-access
import pytest
import errno
from itertools import product
from uuid import uuid4
from unittest.mock import Mock, patch
import ddt
import six
from six.moves import range
import pytest
from django.conf import settings
from django.core.exceptions import SuspiciousOperation
from django.test import TestCase
from django.utils.timezone import now
from mock import Mock, patch
from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator
from ..exceptions import BlockStructureNotFound
@@ -28,22 +25,22 @@ class BlockStructureModelTestCase(TestCase):
Tests for BlockStructureModel.
"""
def setUp(self):
super(BlockStructureModelTestCase, self).setUp() # lint-amnesty, pylint: disable=super-with-arguments
self.course_key = CourseLocator('org', 'course', six.text_type(uuid4()))
super().setUp()
self.course_key = CourseLocator('org', 'course', str(uuid4()))
self.usage_key = BlockUsageLocator(course_key=self.course_key, block_type='course', block_id='course')
self.params = self._create_bsm_params()
def tearDown(self):
BlockStructureModel._prune_files(self.usage_key, num_to_keep=0)
super(BlockStructureModelTestCase, self).tearDown() # lint-amnesty, pylint: disable=super-with-arguments
super().tearDown()
def _assert_bsm_fields(self, bsm, expected_serialized_data):
"""
Verifies that the field values and serialized data
on the given bsm are as expected.
"""
for field_name, field_value in six.iteritems(self.params):
for field_name, field_value in self.params.items():
assert field_value == getattr(bsm, field_name)
assert bsm.get_serialized_data().decode('utf-8') == expected_serialized_data
@@ -65,7 +62,7 @@ class BlockStructureModelTestCase(TestCase):
data_version='DV',
data_edit_timestamp=now(),
transformers_schema_version='TV',
block_structure_schema_version=six.text_type(1),
block_structure_schema_version=str(1),
)
def _verify_update_or_create_call(self, serialized_data, mock_log=None, expect_created=None):
@@ -139,7 +136,7 @@ class BlockStructureModelTestCase(TestCase):
return_value=prune_keep_count,
):
for x in range(num_prior_edits):
self._verify_update_or_create_call('data_{}'.format(x))
self._verify_update_or_create_call(f'data_{x}')
if num_prior_edits:
self._assert_file_count_equal(min(num_prior_edits, prune_keep_count))
@@ -148,12 +145,12 @@ class BlockStructureModelTestCase(TestCase):
self._assert_file_count_equal(min(num_prior_edits + 1, prune_keep_count))
@ddt.data(
(IOError, errno.ENOENT, u'No such file or directory', BlockStructureNotFound, True),
(IOError, errno.ENOENT, u'No such file or directory', IOError, False),
(IOError, errno.ENOENT, 'No such file or directory', BlockStructureNotFound, True),
(IOError, errno.ENOENT, 'No such file or directory', IOError, False),
(SuspiciousOperation, None, None, BlockStructureNotFound, True),
(SuspiciousOperation, None, None, SuspiciousOperation, False),
(OSError, errno.EACCES, u'Permission denied', OSError, True),
(OSError, errno.EACCES, u'Permission denied', OSError, False),
(OSError, errno.EACCES, 'Permission denied', OSError, True),
(OSError, errno.EACCES, 'Permission denied', OSError, False),
)
@ddt.unpack
def test_error_handling(self, error_raised_in_operation, errno_raised, message_raised,
@@ -168,7 +165,7 @@ class BlockStructureModelTestCase(TestCase):
@patch('openedx.core.djangoapps.content.block_structure.models.log')
def test_old_mongo_keys(self, mock_log):
self.course_key = CourseLocator('org2', 'course2', six.text_type(uuid4()), deprecated=True)
self.course_key = CourseLocator('org2', 'course2', str(uuid4()), deprecated=True)
self.usage_key = BlockUsageLocator(course_key=self.course_key, block_type='course', block_id='course')
serialized_data = 'test data for old course'
self.params['data_usage_key'] = self.usage_key

4
openedx/core/djangoapps/content/block_structure/tests/test_signals.py
View File

@@ -1,11 +1,11 @@
"""
Unit tests for the Course Blocks signals
"""
from unittest.mock import patch
import pytest
import ddt
from edx_toggles.toggles.testutils import override_waffle_switch
from mock import patch
from opaque_keys.edx.locator import CourseLocator, LibraryLocator
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
@@ -25,7 +25,7 @@ class CourseBlocksSignalTest(ModuleStoreTestCase):
ENABLED_SIGNALS = ['course_deleted', 'course_published']
def setUp(self):
super(CourseBlocksSignalTest, self).setUp() # lint-amnesty, pylint: disable=super-with-arguments
super().setUp()
self.course = CourseFactory.create()
self.course_usage_key = self.store.make_course_usage_key(self.course.id)

4
openedx/core/djangoapps/content/block_structure/tests/test_store.py
View File

@@ -23,7 +23,7 @@ class TestBlockStructureStore(UsageKeyFactoryMixin, ChildrenMapTestMixin, CacheI
ENABLED_CACHES = ['default']
def setUp(self):
super(TestBlockStructureStore, self).setUp() # lint-amnesty, pylint: disable=super-with-arguments
super().setUp()
self.children_map = self.SIMPLE_CHILDREN_MAP
self.block_structure = self.create_block_structure(self.children_map)
@@ -43,7 +43,7 @@ class TestBlockStructureStore(UsageKeyFactoryMixin, ChildrenMapTestMixin, CacheI
self.block_key_factory(0),
transformer,
key='test',
value=u'{} val'.format(transformer.name()),
value=f'{transformer.name()} val',
)
@ddt.data(True, False)

2
openedx/core/djangoapps/content/block_structure/tests/test_tasks.py
View File

@@ -3,7 +3,7 @@ Unit tests for the Course Blocks tasks
"""
from mock import patch
from unittest.mock import patch
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase

2
openedx/core/djangoapps/content/block_structure/tests/test_transformer_registry.py
View File

@@ -39,7 +39,7 @@ class TransformerRegistryTestCase(TestCase):
"""
def tearDown(self):
super(TransformerRegistryTestCase, self).tearDown() # lint-amnesty, pylint: disable=super-with-arguments
super().tearDown()
clear_registered_transformers_cache()
@ddt.data(

7
openedx/core/djangoapps/content/block_structure/tests/test_transformers.py
View File

@@ -1,11 +1,10 @@
"""
Tests for transformers.py
"""
from unittest import TestCase
from unittest.mock import MagicMock, patch
import pytest
from unittest import TestCase
from mock import MagicMock, patch
from ..block_structure import BlockStructureModulestoreData
from ..exceptions import TransformerDataIncompatible, TransformerException
@@ -25,7 +24,7 @@ class TestBlockStructureTransformers(ChildrenMapTestMixin, TestCase):
pass # lint-amnesty, pylint: disable=unnecessary-pass
def setUp(self):
super(TestBlockStructureTransformers, self).setUp() # lint-amnesty, pylint: disable=super-with-arguments
super().setUp()
self.transformers = BlockStructureTransformers(usage_info=MagicMock())
self.registered_transformers = [MockTransformer(), MockFilteringTransformer()]

2
openedx/core/djangoapps/content/block_structure/transformer.py
View File

@@ -8,7 +8,7 @@ from abc import abstractmethod
import functools
class BlockStructureTransformer(object):
class BlockStructureTransformer:
"""
Abstract base class for all block structure transformers.
"""

11
openedx/core/djangoapps/content/block_structure/transformer_registry.py
View File

@@ -7,7 +7,6 @@ PluginManager.
from base64 import b64encode
from hashlib import sha1
import six
from edx_django_utils.plugins import PluginManager
from openedx.core.lib.cache_utils import process_cached
@@ -34,7 +33,7 @@ class TransformerRegistry(PluginManager):
registered with the platform's PluginManager.
"""
if cls.USE_PLUGIN_MANAGER:
return set(six.itervalues(cls.get_available_plugins()))
return set(cls.get_available_plugins().values())
else:
return set()
@@ -49,8 +48,8 @@ class TransformerRegistry(PluginManager):
sorted_transformers = sorted(cls.get_registered_transformers(), key=lambda t: t.name())
for transformer in sorted_transformers:
hash_obj.update(six.b(transformer.name()))
hash_obj.update(six.b(str(transformer.WRITE_VERSION)))
hash_obj.update((transformer.name()).encode())
hash_obj.update((str(transformer.WRITE_VERSION)).encode())
return b64encode(hash_obj.digest()).decode('utf-8')
@@ -69,6 +68,6 @@ class TransformerRegistry(PluginManager):
set([string]) - Set of names of a subset of the given
transformers that weren't found in the registry.
"""
registered_transformer_names = set(reg_trans.name() for reg_trans in cls.get_registered_transformers())
requested_transformer_names = set(transformer.name() for transformer in transformers)
registered_transformer_names = {reg_trans.name() for reg_trans in cls.get_registered_transformers()}
requested_transformer_names = {transformer.name() for transformer in transformers}
return requested_transformer_names - registered_transformer_names

6
openedx/core/djangoapps/content/block_structure/transformers.py
View File

@@ -10,7 +10,7 @@ from .transformer_registry import TransformerRegistry
logger = getLogger(__name__) # pylint: disable=C0103
class BlockStructureTransformers(object):
class BlockStructureTransformers:
"""
The BlockStructureTransformers class encapsulates an ordered list of block
structure transformers. It uses the Transformer Registry to verify the
@@ -58,7 +58,7 @@ class BlockStructureTransformers(object):
unregistered_transformers = TransformerRegistry.find_unregistered(transformers)
if unregistered_transformers:
raise TransformerException(
u"The following requested transformers are not registered: {}".format(unregistered_transformers)
f"The following requested transformers are not registered: {unregistered_transformers}"
)
for transformer in transformers:
@@ -98,7 +98,7 @@ class BlockStructureTransformers(object):
if outdated_transformers:
raise TransformerDataIncompatible( # lint-amnesty, pylint: disable=raising-format-tuple
u"Collected Block Structure data for the following transformers is outdated: '%s'.",
"Collected Block Structure data for the following transformers is outdated: '%s'.",
[(transformer.name(), transformer.READ_VERSION) for transformer in outdated_transformers],
)
return True

1
openedx/core/djangoapps/content/course_overviews/api.py
View File

@@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
"""
CourseOverview internal api
"""

23
openedx/core/djangoapps/content/course_overviews/management/commands/backfill_history.py
View File

@@ -6,7 +6,6 @@ import csv
import logging
import os
import time
from io import open
from django.core.management.base import BaseCommand
from django.db import connection, transaction
@@ -40,7 +39,7 @@ class Command(BaseCommand):
]
def add_arguments(self, parser):
super(Command, self).add_arguments(parser)
super().add_arguments(parser)
parser.add_argument(
'--sleep_between',
@@ -85,7 +84,7 @@ class Command(BaseCommand):
history_date = input_filename.rsplit('_')[-1]
with connection.cursor() as cursor:
query = u"""
query = """
SELECT
column_name
FROM information_schema.columns
@@ -97,20 +96,20 @@ class Command(BaseCommand):
if exclude_column in columns:
columns.remove(exclude_column)
with open(file_path, 'r') as input_file:
with open(file_path) as input_file:
reader = csv.DictReader(input_file, delimiter='\x01')
lines = list(reader)
for rows in self.chunks(lines, batch_size):
row_ids = [row['ID'] for row in rows]
if table == 'course_overviews_courseoverview':
ids = ','.join("'{}'".format(id) for id in row_ids)
ids = ','.join(f"'{id}'" for id in row_ids)
else:
ids = ','.join(row_ids)
# Checks for existing historical records
with connection.cursor() as cursor:
query = u"""
query = """
SELECT COUNT(1)
FROM {historical_table}
WHERE ID in ({ids})
@@ -124,12 +123,12 @@ class Command(BaseCommand):
if count == len(rows):
log.info(
u"Initial history records already exist for ids %s..%s - skipping.",
"Initial history records already exist for ids %s..%s - skipping.",
','.join(row_ids[:2]), ','.join(row_ids[-2:])
)
continue
elif count != 0:
raise Exception(u"Database count: %s does not match input count: %s" % (count, len(ids)))
raise Exception("Database count: {} does not match input count: {}".format(count, len(ids)))
values = [[row[column.upper()] for column in columns] for row in rows]
@@ -145,17 +144,17 @@ class Command(BaseCommand):
# Convert to tuple
values = [tuple(value) for value in values]
quoted_columns = ['`{}`'.format(c) for c in columns]
quoted_columns = [f'`{c}`' for c in columns]
with transaction.atomic():
with connection.cursor() as cursor:
log.info(
u"Inserting historical records for %s starting with id %s to %s",
"Inserting historical records for %s starting with id %s to %s",
table,
row_ids[0],
row_ids[-1]
)
query = u"""
query = """
INSERT INTO {historical_table}(
{insert_columns},`history_date`,`history_change_reason`,`history_type`,`history_user_id`
)
@@ -167,5 +166,5 @@ class Command(BaseCommand):
) # noqa
cursor.executemany(query, values)
log.info(u"Sleeping %s seconds...", sleep_between)
log.info("Sleeping %s seconds...", sleep_between)
time.sleep(sleep_between)

11
openedx/core/djangoapps/content/course_overviews/management/commands/generate_course_overview.py
View File

@@ -5,7 +5,6 @@ Command to load course overviews.
import logging
import six
from django.core.management.base import BaseCommand, CommandError
from opaque_keys import InvalidKeyError
@@ -37,25 +36,25 @@ class Command(BaseCommand):
dest='all_courses',
action='store_true',
default=DEFAULT_ALL_COURSES,
help=u'Generate course overview for all courses.',
help='Generate course overview for all courses.',
)
parser.add_argument(
'--force-update', '--force_update',
action='store_true',
default=DEFAULT_FORCE_UPDATE,
help=u'Force update course overviews for the requested courses.',
help='Force update course overviews for the requested courses.',
)
parser.add_argument(
'--chunk-size',
action='store',
type=int,
default=DEFAULT_CHUNK_SIZE,
help=u'The maximum number of courses each task will generate a course overview for.'
help='The maximum number of courses each task will generate a course overview for.'
)
parser.add_argument(
'--routing-key',
dest='routing_key',
help=u'The celery routing key to use.'
help='The celery routing key to use.'
)
def handle(self, *args, **options):
@@ -73,4 +72,4 @@ class Command(BaseCommand):
**kwargs
)
except InvalidKeyError as exc:
raise CommandError(u'Invalid Course Key: ' + six.text_type(exc)) # lint-amnesty, pylint: disable=raise-missing-from
raise CommandError('Invalid Course Key: ' + str(exc)) # lint-amnesty, pylint: disable=raise-missing-from

91
openedx/core/djangoapps/content/course_overviews/management/commands/simulate_publish.py
View File

@@ -20,7 +20,6 @@ import sys
import textwrap
import time
import six
from django.core.management.base import BaseCommand, CommandError
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey
@@ -69,9 +68,9 @@ class Command(BaseCommand):
(+ 17 more)
"""
help = (
u"Simulate course publish signals without actually modifying course "
u"content. This command is useful for triggering various async tasks "
u"that listen for course_published signals."
"Simulate course publish signals without actually modifying course "
"content. This command is useful for triggering various async tasks "
"that listen for course_published signals."
)
# Having this be a class attribute makes it easier to substitute during
@@ -85,16 +84,16 @@ class Command(BaseCommand):
'--show-receivers',
dest='show_receivers',
action='store_true',
help=(u'Display the list of possible receiver functions and exit.')
help=('Display the list of possible receiver functions and exit.')
), # lint-amnesty, pylint: disable=trailing-comma-tuple
parser.add_argument(
'--dry-run',
dest='dry_run',
action='store_true',
help=(
u"Just show a preview of what would happen. This may make an "
u"expensive modulestore query to find courses, but it will "
u"not emit any signals."
"Just show a preview of what would happen. This may make an "
"expensive modulestore query to find courses, but it will "
"not emit any signals."
)
), # lint-amnesty, pylint: disable=trailing-comma-tuple
parser.add_argument(
@@ -103,11 +102,11 @@ class Command(BaseCommand):
action='store',
nargs='+',
help=(
u'Send course_published to specific receivers. If this flag is '
u'not present, course_published will be sent to all receivers. '
u'The CCX receiver is always included unless --skip-ccx is '
u'explicitly passed (otherwise CCX courses would never get '
u'called for any signal).'
'Send course_published to specific receivers. If this flag is '
'not present, course_published will be sent to all receivers. '
'The CCX receiver is always included unless --skip-ccx is '
'explicitly passed (otherwise CCX courses would never get '
'called for any signal).'
)
)
parser.add_argument(
@@ -116,8 +115,8 @@ class Command(BaseCommand):
action='store',
nargs='+',
help=(
u'Send course_published for specific courses. If this flag is '
u'not present, course_published will be sent to all courses.'
'Send course_published for specific courses. If this flag is '
'not present, course_published will be sent to all courses.'
)
)
parser.add_argument(
@@ -127,8 +126,8 @@ class Command(BaseCommand):
type=int,
default=0,
help=(
u"Number of seconds to sleep between emitting course_published "
u"signals, so that we don't flood our queues."
"Number of seconds to sleep between emitting course_published "
"signals, so that we don't flood our queues."
)
)
parser.add_argument(
@@ -136,11 +135,11 @@ class Command(BaseCommand):
dest='force_lms',
action='store_true',
help=(
u"This command should be run under cms (Studio), not LMS. "
u"Regular publishes happen via Studio, and this script will "
u"exit with an error if you attempt to run it in an LMS "
u"process. However, if you know what you're doing and need to "
u"override that behavior, use this flag."
"This command should be run under cms (Studio), not LMS. "
"Regular publishes happen via Studio, and this script will "
"exit with an error if you attempt to run it in an LMS "
"process. However, if you know what you're doing and need to "
"override that behavior, use this flag."
)
), # lint-amnesty, pylint: disable=trailing-comma-tuple
parser.add_argument(
@@ -148,13 +147,13 @@ class Command(BaseCommand):
dest='skip_ccx',
action='store_true',
help=(
u"CCX receivers are special echoing receivers that relay "
u"the course_published signal to all CCX courses derived from "
u"a modulestore-stored course. That means we almost always "
u"want to emit to them (even when using --receivers), or none "
u"of our signals will reach any CCX derived courses. However, "
u"if you know what you're doing, you can disable this behavior "
u"with this flag, so that CCX receivers are omitted."
"CCX receivers are special echoing receivers that relay "
"the course_published signal to all CCX courses derived from "
"a modulestore-stored course. That means we almost always "
"want to emit to them (even when using --receivers), or none "
"of our signals will reach any CCX derived courses. However, "
"if you know what you're doing, you can disable this behavior "
"with this flag, so that CCX receivers are omitted."
)
), # lint-amnesty, pylint: disable=trailing-comma-tuple
parser.add_argument(
@@ -185,7 +184,7 @@ class Command(BaseCommand):
return self.print_show_receivers()
log.info(
u"simulate_publish starting, dry-run=%s, delay=%d seconds",
"simulate_publish starting, dry-run=%s, delay=%d seconds",
options['dry_run'],
options['delay']
)
@@ -195,8 +194,8 @@ class Command(BaseCommand):
log.info("Forcing simulate_publish to run in LMS process.")
else:
log.fatal( # lint-amnesty, pylint: disable=logging-not-lazy
u"simulate_publish should be run as a CMS (Studio) " +
u"command, not %s (override with --force-lms).",
"simulate_publish should be run as a CMS (Studio) " +
"command, not %s (override with --force-lms).",
os.environ.get('SERVICE_VARIANT')
)
sys.exit(1)
@@ -216,7 +215,7 @@ class Command(BaseCommand):
# actual work of emitting signals.
for i, course_key in enumerate(course_keys, start=1):
log.info(
u"Emitting course_published signal (%d of %d) for course %s",
"Emitting course_published signal (%d of %d) for course %s",
i, len(course_keys), course_key
)
if options['delay']:
@@ -237,19 +236,19 @@ class Command(BaseCommand):
unknown_receiver_names = set(receiver_names) - all_receiver_names
if unknown_receiver_names:
log.fatal(
u"The following receivers were specified but not recognized: %s",
u", ".join(sorted(unknown_receiver_names))
"The following receivers were specified but not recognized: %s",
", ".join(sorted(unknown_receiver_names))
)
log.fatal(u"Known receivers: %s", ", ".join(sorted(all_receiver_names)))
log.fatal("Known receivers: %s", ", ".join(sorted(all_receiver_names)))
sys.exit(1)
log.info(u"%d receivers specified: %s", len(receiver_names), ", ".join(receiver_names))
log.info("%d receivers specified: %s", len(receiver_names), ", ".join(receiver_names))
receiver_names_set = set(receiver_names)
for receiver_fn in get_receiver_fns():
if receiver_fn == ccx_receiver_fn and not skip_ccx: # lint-amnesty, pylint: disable=comparison-with-callable
continue
fn_name = name_from_fn(receiver_fn)
if fn_name not in receiver_names_set:
log.info(u"Disconnecting %s", fn_name)
log.info("Disconnecting %s", fn_name)
self.course_published_signal.disconnect(receiver_fn)
def get_course_keys(self, courses):
@@ -265,20 +264,20 @@ class Command(BaseCommand):
# Use specific courses if specified, but fall back to all courses.
course_keys = []
if courses:
log.info(u"%d courses specified: %s", len(courses), ", ".join(courses))
log.info("%d courses specified: %s", len(courses), ", ".join(courses))
for course_id in courses:
try:
course_keys.append(CourseKey.from_string(course_id))
except InvalidKeyError:
log.fatal(u"%s is not a parseable CourseKey", course_id)
log.fatal("%s is not a parseable CourseKey", course_id)
sys.exit(1)
else:
log.info("No courses specified, reading all courses from modulestore...")
course_keys = sorted(
(course.id for course in modulestore().get_course_summaries()),
key=six.text_type # Different types of CourseKeys can't be compared without this.
key=str # Different types of CourseKeys can't be compared without this.
)
log.info(u"%d courses read from modulestore.", len(course_keys))
log.info("%d courses read from modulestore.", len(course_keys))
return course_keys
@@ -307,15 +306,15 @@ class Command(BaseCommand):
for course_key in course_keys[:COURSES_TO_SHOW]:
print(" ", course_key)
if len(course_keys) > COURSES_TO_SHOW:
print(u" (+ {} more)".format(len(course_keys) - COURSES_TO_SHOW))
print(" (+ {} more)".format(len(course_keys) - COURSES_TO_SHOW))
def get_receiver_names():
"""Return an unordered set of receiver names (full.module.path.function)"""
return set(
return {
name_from_fn(fn_ref())
for _, fn_ref in Command.course_published_signal.receivers
)
}
def get_receiver_fns():
@@ -328,4 +327,4 @@ def get_receiver_fns():
def name_from_fn(fn):
"""Human readable module.function name."""
return u"{}.{}".format(fn.__module__, fn.__name__)
return f"{fn.__module__}.{fn.__name__}"

15
openedx/core/djangoapps/content/course_overviews/management/commands/tests/test_generate_course_overview.py
View File

@@ -1,11 +1,10 @@
"""
Tests that the generate_course_overview management command actually generates course overviews.
"""
from unittest.mock import patch
import pytest
import six
from django.core.management.base import CommandError
from mock import patch
from openedx.core.djangoapps.content.course_overviews.management.commands import generate_course_overview
from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
@@ -22,7 +21,7 @@ class TestGenerateCourseOverview(ModuleStoreTestCase):
"""
Create courses in modulestore.
"""
super(TestGenerateCourseOverview, self).setUp() # lint-amnesty, pylint: disable=super-with-arguments
super().setUp()
self.course_key_1 = CourseFactory.create().id
self.course_key_2 = CourseFactory.create().id
self.command = generate_course_overview.Command()
@@ -59,7 +58,7 @@ class TestGenerateCourseOverview(ModuleStoreTestCase):
Test that a specified course is loaded into course overviews.
"""
self._assert_courses_not_in_overview(self.course_key_1, self.course_key_2)
self.command.handle(six.text_type(self.course_key_1), all_courses=False)
self.command.handle(str(self.course_key_1), all_courses=False)
self._assert_courses_in_overview(self.course_key_1)
self._assert_courses_not_in_overview(self.course_key_2)
@@ -67,15 +66,15 @@ class TestGenerateCourseOverview(ModuleStoreTestCase):
self.command.handle(all_courses=True)
# update each course
updated_course_name = u'test_generate_course_overview.course_edit'
updated_course_name = 'test_generate_course_overview.course_edit'
for course_key in (self.course_key_1, self.course_key_2):
course = self.store.get_course(course_key)
course.display_name = updated_course_name
self.store.update_item(course, self.user.id)
# force_update course_key_1, but not course_key_2
self.command.handle(six.text_type(self.course_key_1), all_courses=False, force_update=True)
self.command.handle(six.text_type(self.course_key_2), all_courses=False, force_update=False)
self.command.handle(str(self.course_key_1), all_courses=False, force_update=True)
self.command.handle(str(self.course_key_2), all_courses=False, force_update=False)
assert CourseOverview.get_from_id(self.course_key_1).display_name == updated_course_name
assert CourseOverview.get_from_id(self.course_key_2).display_name != updated_course_name
@@ -107,7 +106,7 @@ class TestGenerateCourseOverview(ModuleStoreTestCase):
self.command.handle(all_courses=True, force_update=True, routing_key='my-routing-key', chunk_size=10000)
called_kwargs = mock_async_task.apply_async.call_args_list[0][1]
assert sorted([six.text_type(self.course_key_1), six.text_type(self.course_key_2)]) ==\
assert sorted([str(self.course_key_1), str(self.course_key_2)]) ==\
sorted(called_kwargs.pop('args'))
assert {'kwargs': {'force_update': True}, 'routing_key': 'my-routing-key'} == called_kwargs
assert 1 == mock_async_task.apply_async.call_count

16
openedx/core/djangoapps/content/course_overviews/management/commands/tests/test_simulate_publish.py
View File

@@ -1,10 +1,6 @@
"""
Tests the simulate_publish management command.
"""
import six
from django.core.management import call_command
from django.core.management.base import CommandError
from testfixtures import LogCapture
@@ -32,7 +28,7 @@ class TestSimulatePublish(SharedModuleStoreTestCase):
Modulestore signals are suppressed by ModuleStoreIsolationMixin, so this
method should not trigger things like CourseOverview creation.
"""
super(TestSimulatePublish, cls).setUpClass()
super().setUpClass()
cls.command = Command()
# org.0/course_0/Run_0
cls.course_key_1 = CourseFactory.create(default_store=ModuleStoreEnum.Type.mongo).id
@@ -47,7 +43,7 @@ class TestSimulatePublish(SharedModuleStoreTestCase):
might look like you can move this to setUpClass, but be very careful if
doing so, to make sure side-effects don't leak out between tests.
"""
super(TestSimulatePublish, self).setUp() # lint-amnesty, pylint: disable=super-with-arguments
super().setUp()
# Instead of using the process global SignalHandler.course_published, we
# create our own SwitchedSignal to manually send to.
@@ -79,7 +75,7 @@ class TestSimulatePublish(SharedModuleStoreTestCase):
)
Command.course_published_signal.disconnect(self.sample_receiver_1)
Command.course_published_signal.disconnect(self.sample_receiver_2)
super(TestSimulatePublish, self).tearDown() # lint-amnesty, pylint: disable=super-with-arguments
super().tearDown()
def options(self, **kwargs):
"""
@@ -112,7 +108,7 @@ class TestSimulatePublish(SharedModuleStoreTestCase):
"""Test sending only to specific courses."""
self.command.handle(
**self.options(
courses=[six.text_type(self.course_key_1), six.text_type(self.course_key_2)]
courses=[str(self.course_key_1), str(self.course_key_2)]
)
)
assert self.course_key_1 in self.received_1
@@ -173,7 +169,7 @@ class TestSimulatePublish(SharedModuleStoreTestCase):
log.check_present(
(
LOGGER_NAME, 'INFO',
u"simulate_publish starting, dry-run={}, delay={} seconds".format('False', '0')
"simulate_publish starting, dry-run={}, delay={} seconds".format('False', '0')
),
)
@@ -183,6 +179,6 @@ class TestSimulatePublish(SharedModuleStoreTestCase):
log.check_present(
(
LOGGER_NAME, 'INFO',
u"simulate_publish starting, dry-run={}, delay={} seconds".format('True', '20')
"simulate_publish starting, dry-run={}, delay={} seconds".format('True', '20')
),
)