Andal.LND/edx-platform
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #14719 from edx/neem/block-structure-waffle-in-task

Browse Source 
Have generate_course_blocks pass with_storage to celery tasks
This commit is contained in:
Nimisha Asthagiri
2017-03-20 17:26:05 -04:00
committed by GitHub
parent bb31724f12 e83182c743
commit 89518d798b
12 changed files with 129 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
openedx/core/djangoapps/content/block_structure/config/__init__.py
View File

@@ -2,12 +2,17 @@
This module contains various configuration settings via
waffle switches for the Block Structure framework.
"""
import logging
from openedx.core.djangolib.waffle_utils import is_switch_enabled
from request_cache.middleware import request_cached
from request_cache.middleware import request_cached, RequestCache, func_call_cache_key
from .models import BlockStructureConfiguration
log = logging.getLogger(__name__)
INVALIDATE_CACHE_ON_PUBLISH = u'invalidate_cache_on_publish'
STORAGE_BACKING_FOR_CACHE = u'storage_backing_for_cache'
RAISE_ERROR_WHEN_NOT_FOUND = u'raise_error_when_not_found'
@@ -23,6 +28,19 @@ def is_enabled(setting_name):
return is_switch_enabled(bs_waffle_name)
def enable_for_current_request(setting_name):
"""
Enables the given block_structure setting for the
duration of the current request.
"""
cache_key = func_call_cache_key(
is_switch_enabled.request_cached_contained_func,
_bs_waffle_switch_name(setting_name),
)
RequestCache.get_request_cache().data[cache_key] = True
log.warning(u'BlockStructure: Config %s is enabled for current request.', setting_name)
@request_cached
def num_versions_to_keep():
"""

0
openedx/core/djangoapps/content/block_structure/management/__init__.py Normal file
View File

0
openedx/core/djangoapps/content/block_structure/management/commands/__init__.py Normal file
View File

158
openedx/core/djangoapps/content/block_structure/management/commands/generate_course_blocks.py Normal file
View File

@@ -0,0 +1,158 @@
"""
Command to load course blocks.
"""
import logging
from django.core.management.base import BaseCommand
from xmodule.modulestore.django import modulestore
import openedx.core.djangoapps.content.block_structure.api as api
from openedx.core.djangoapps.content.block_structure.config import STORAGE_BACKING_FOR_CACHE, enable_for_current_request
import openedx.core.djangoapps.content.block_structure.tasks as tasks
import openedx.core.djangoapps.content.block_structure.store as store
from openedx.core.lib.command_utils import (
get_mutually_exclusive_required_option,
validate_dependent_option,
parse_course_keys,
)
log = logging.getLogger(__name__)
class Command(BaseCommand):
"""
Example usage:
$ ./manage.py lms generate_course_blocks --all --settings=devstack
$ ./manage.py lms generate_course_blocks 'edX/DemoX/Demo_Course' --settings=devstack
"""
args = u'<course_id course_id ...>'
help = u'Generates and stores course blocks for one or more courses.'
def add_arguments(self, parser):
"""
Entry point for subclassed commands to add custom arguments.
"""
parser.add_argument(
'--courses',
dest='courses',
nargs='+',
help=u'Generate course blocks for the list of courses provided.',
)
parser.add_argument(
'--all_courses',
help=u'Generate course blocks for all courses, given the requested start and end indices.',
action='store_true',
default=False,
)
parser.add_argument(
'--enqueue_task',
help=u'Enqueue the tasks for asynchronous computation.',
action='store_true',
default=False,
)
parser.add_argument(
'--routing_key',
dest='routing_key',
help=u'Routing key to use for asynchronous computation.',
)
parser.add_argument(
'--force_update',
help=u'Force update of the course blocks for the requested courses.',
action='store_true',
default=False,
)
parser.add_argument(
'--start_index',
help=u'Starting index of course list.',
default=0,
type=int,
)
parser.add_argument(
'--end_index',
help=u'Ending index of course list.',
default=0,
type=int,
)
parser.add_argument(
'--with_storage',
help=u'Store the course blocks in Storage, overriding value of the storage_backing_for_cache waffle switch',
action='store_true',
default=False,
)
def handle(self, *args, **options):
courses_mode = get_mutually_exclusive_required_option(options, 'courses', 'all_courses')
validate_dependent_option(options, 'routing_key', 'enqueue_task')
validate_dependent_option(options, 'start_index', 'all_courses')
validate_dependent_option(options, 'end_index', 'all_courses')
if courses_mode == 'all_courses':
course_keys = [course.id for course in modulestore().get_course_summaries()]
if options.get('start_index'):
end = options.get('end_index') or len(course_keys)
course_keys = course_keys[options['start_index']:end]
else:
course_keys = parse_course_keys(options['courses'])
self._set_log_levels(options)
log.critical(u'BlockStructure: STARTED generating Course Blocks for %d courses.', len(course_keys))
self._generate_course_blocks(options, course_keys)
log.critical(u'BlockStructure: FINISHED generating Course Blocks for %d courses.', len(course_keys))
def _set_log_levels(self, options):
"""
Sets logging levels for this module and the block structure
cache module, based on the given the options.
"""
if options.get('verbosity') == 0:
log_level = logging.CRITICAL
elif options.get('verbosity') == 1:
log_level = logging.WARNING
else:
log_level = logging.INFO
if options.get('verbosity') < 3:
cache_log_level = logging.CRITICAL
else:
cache_log_level = logging.INFO
log.setLevel(log_level)
store.logger.setLevel(cache_log_level)
def _generate_course_blocks(self, options, course_keys):
"""
Generates course blocks for the given course_keys per the given options.
"""
if options.get('with_storage'):
enable_for_current_request(STORAGE_BACKING_FOR_CACHE)
for course_key in course_keys:
try:
self._generate_for_course(options, course_key)
except Exception as ex: # pylint: disable=broad-except
log.exception(
u'BlockStructure: An error occurred while generating course blocks for %s: %s',
unicode(course_key),
ex.message,
)
def _generate_for_course(self, options, course_key):
"""
Generates course blocks for the given course_key per the given options.
"""
if options.get('enqueue_task'):
action = tasks.update_course_in_cache_v2 if options.get('force_update') else tasks.get_course_in_cache_v2
task_options = {'routing_key': options['routing_key']} if options.get('routing_key') else {}
result = action.apply_async(
kwargs=dict(course_id=unicode(course_key), with_storage=options.get('with_storage')),
**task_options
)
log.info(u'BlockStructure: ENQUEUED generating for course: %s, task_id: %s.', course_key, result.id)
else:
log.info(u'BlockStructure: STARTED generating for course: %s.', course_key)
action = api.update_course_in_cache if options.get('force_update') else api.get_course_in_cache
action(course_key)
log.info(u'BlockStructure: FINISHED generating for course: %s.', course_key)

0
openedx/core/djangoapps/content/block_structure/management/commands/tests/__init__.py Normal file
View File

178
openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py Normal file
View File

@@ -0,0 +1,178 @@
"""
Tests for generate_course_blocks management command.
"""
import ddt
from django.core.management.base import CommandError
import itertools
from mock import patch
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory
from .. import generate_course_blocks
from openedx.core.djangoapps.content.block_structure.tests.helpers import (
is_course_in_block_structure_cache,
is_course_in_block_structure_storage,
)
@ddt.ddt
class TestGenerateCourseBlocks(ModuleStoreTestCase):
"""
Tests generate course blocks management command.
"""
num_courses = 2
def setUp(self):
"""
Create courses in modulestore.
"""
super(TestGenerateCourseBlocks, self).setUp()
self.courses = [CourseFactory.create() for _ in range(self.num_courses)]
self.course_keys = [course.id for course in self.courses]
self.command = generate_course_blocks.Command()
def _assert_courses_not_in_block_cache(self, *course_keys):
"""
Assert courses don't exist in the course block cache.
"""
for course_key in course_keys:
self.assertFalse(is_course_in_block_structure_cache(course_key, self.store))
def _assert_courses_in_block_cache(self, *course_keys):
"""
Assert courses exist in course block cache.
"""
for course_key in course_keys:
self.assertTrue(is_course_in_block_structure_cache(course_key, self.store))
def _assert_courses_not_in_block_storage(self, *course_keys):
"""
Assert courses don't exist in course block storage.
"""
for course_key in course_keys:
self.assertFalse(is_course_in_block_structure_storage(course_key, self.store))
def _assert_courses_in_block_storage(self, *course_keys):
"""
Assert courses exist in course block storage.
"""
for course_key in course_keys:
self.assertTrue(is_course_in_block_structure_storage(course_key, self.store))
def _assert_message_presence_in_logs(self, message, mock_log, expected_presence=True):
"""
Asserts that the logger was called with the given message.
"""
message_present = any([message in call_args[0][0] for call_args in mock_log.warning.call_args_list])
if expected_presence:
self.assertTrue(message_present)
else:
self.assertFalse(message_present)
@ddt.data(True, False)
def test_all_courses(self, force_update):
self._assert_courses_not_in_block_cache(*self.course_keys)
self.command.handle(all_courses=True)
self._assert_courses_in_block_cache(*self.course_keys)
with patch(
'openedx.core.djangoapps.content.block_structure.factory.BlockStructureFactory.create_from_modulestore'
) as mock_update_from_store:
self.command.handle(all_courses=True, force_update=force_update)
self.assertEqual(mock_update_from_store.call_count, self.num_courses if force_update else 0)
def test_one_course(self):
self._assert_courses_not_in_block_cache(*self.course_keys)
self.command.handle(courses=[unicode(self.course_keys[0])])
self._assert_courses_in_block_cache(self.course_keys[0])
self._assert_courses_not_in_block_cache(*self.course_keys[1:])
self._assert_courses_not_in_block_storage(*self.course_keys)
def test_with_storage(self):
self.command.handle(with_storage=True, courses=[unicode(self.course_keys[0])])
self._assert_courses_in_block_cache(self.course_keys[0])
self._assert_courses_in_block_storage(self.course_keys[0])
self._assert_courses_not_in_block_storage(*self.course_keys[1:])
@ddt.data(
*itertools.product(
(True, False),
(True, False),
('route_1', None),
)
)
@ddt.unpack
def test_enqueue(self, enqueue_task, force_update, routing_key):
command_options = dict(all_courses=True, enqueue_task=enqueue_task, force_update=force_update)
if enqueue_task and routing_key:
command_options['routing_key'] = routing_key
with patch(
'openedx.core.djangoapps.content.block_structure.management.commands.generate_course_blocks.tasks'
) as mock_tasks:
with patch(
'openedx.core.djangoapps.content.block_structure.management.commands.generate_course_blocks.api'
) as mock_api:
self.command.handle(**command_options)
self.assertEqual(
mock_tasks.update_course_in_cache_v2.apply_async.call_count,
self.num_courses if enqueue_task and force_update else 0,
)
self.assertEqual(
mock_tasks.get_course_in_cache_v2.apply_async.call_count,
self.num_courses if enqueue_task and not force_update else 0,
)
self.assertEqual(
mock_api.update_course_in_cache.call_count,
self.num_courses if not enqueue_task and force_update else 0,
)
self.assertEqual(
mock_api.get_course_in_cache.call_count,
self.num_courses if not enqueue_task and not force_update else 0,
)
if enqueue_task:
if force_update:
task_action = mock_tasks.update_course_in_cache_v2
else:
task_action = mock_tasks.get_course_in_cache_v2
task_options = task_action.apply_async.call_args[1]
if routing_key:
self.assertEquals(task_options['routing_key'], routing_key)
else:
self.assertNotIn('routing_key', task_options)
@patch('openedx.core.djangoapps.content.block_structure.management.commands.generate_course_blocks.log')
def test_not_found_key(self, mock_log):
self.command.handle(courses=['fake/course/id'])
self.assertTrue(mock_log.exception.called)
def test_invalid_key(self):
with self.assertRaises(CommandError):
self.command.handle(courses=['not/found'])
def test_no_params(self):
with self.assertRaises(CommandError):
self.command.handle(all_courses=False)
def test_no_course_mode(self):
with self.assertRaisesMessage(CommandError, 'Either --courses or --all_courses must be specified.'):
self.command.handle()
def test_both_course_modes(self):
with self.assertRaisesMessage(CommandError, 'Both --courses and --all_courses cannot be specified.'):
self.command.handle(all_courses=True, courses=['some/course/key'])
@ddt.data(
('routing_key', 'enqueue_task'),
('start_index', 'all_courses'),
('end_index', 'all_courses'),
)
@ddt.unpack
def test_dependent_options_error(self, dependent_option, depending_on_option):
expected_error_message = 'Option --{} requires option --{}.'.format(dependent_option, depending_on_option)
options = {dependent_option: 1, depending_on_option: False, 'courses': ['some/course/key']}
with self.assertRaisesMessage(CommandError, expected_error_message):
self.command.handle(**options)

10
openedx/core/djangoapps/content/block_structure/signals.py
View File

@@ -10,11 +10,11 @@ from opaque_keys.edx.locator import LibraryLocator
from . import config
from .api import clear_course_from_cache
from .tasks import update_course_in_cache
from .tasks import update_course_in_cache_v2
@receiver(SignalHandler.course_published)
def _listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable=unused-argument
def _update_block_structure_on_course_publish(sender, course_key, **kwargs): # pylint: disable=unused-argument
"""
Catches the signal that a course has been published in the module
store and creates/updates the corresponding cache entry.
@@ -26,14 +26,14 @@ def _listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable
if config.is_enabled(config.INVALIDATE_CACHE_ON_PUBLISH):
clear_course_from_cache(course_key)
update_course_in_cache.apply_async(
[unicode(course_key)],
update_course_in_cache_v2.apply_async(
kwargs=dict(course_id=unicode(course_key)),
countdown=settings.BLOCK_STRUCTURES_SETTINGS['COURSE_PUBLISH_TASK_DELAY'],
)
@receiver(SignalHandler.course_deleted)
def _listen_for_course_delete(sender, course_key, **kwargs): # pylint: disable=unused-argument
def _delete_block_structure_on_course_delete(sender, course_key, **kwargs): # pylint: disable=unused-argument
"""
Catches the signal that a course has been deleted from the
module store and invalidates the corresponding cache entry if one

101
openedx/core/djangoapps/content/block_structure/tasks.py
View File

@@ -13,6 +13,7 @@ from opaque_keys.edx.keys import CourseKey
from xmodule.modulestore.exceptions import ItemNotFoundError
from openedx.core.djangoapps.content.block_structure import api
from openedx.core.djangoapps.content.block_structure.config import STORAGE_BACKING_FOR_CACHE, enable_for_current_request
log = logging.getLogger('edx.celery.task')
@@ -21,53 +22,101 @@ RETRY_TASKS = (ItemNotFoundError, TypeError, ValInternalError)
NO_RETRY_TASKS = (XMLSyntaxError, LoncapaProblemError, UnicodeEncodeError)
@task(
default_retry_delay=settings.BLOCK_STRUCTURES_SETTINGS['TASK_DEFAULT_RETRY_DELAY'],
max_retries=settings.BLOCK_STRUCTURES_SETTINGS['TASK_MAX_RETRIES'],
bind=True,
)
def block_structure_task(**kwargs):
"""
Decorator for block structure tasks.
"""
return task(
default_retry_delay=settings.BLOCK_STRUCTURES_SETTINGS['TASK_DEFAULT_RETRY_DELAY'],
max_retries=settings.BLOCK_STRUCTURES_SETTINGS['TASK_MAX_RETRIES'],
bind=True,
**kwargs
)
@block_structure_task()
def update_course_in_cache_v2(self, **kwargs):
"""
Updates the course blocks (mongo -> BlockStructure) for the specified course.
Keyword Arguments:
course_id (string) - The string serialized value of the course key.
with_storage (boolean) - Whether or not storage backing should be
enabled for the generated block structure(s).
"""
_update_course_in_cache(self, **kwargs)
@block_structure_task()
def update_course_in_cache(self, course_id):
"""
Updates the course blocks (in the database) for the specified course.
Updates the course blocks (mongo -> BlockStructure) for the specified course.
"""
_call_and_retry_if_needed(course_id, api.update_course_in_cache, update_course_in_cache, self.request.id)
_update_course_in_cache(self, course_id=course_id)
@task(
default_retry_delay=settings.BLOCK_STRUCTURES_SETTINGS['TASK_DEFAULT_RETRY_DELAY'],
max_retries=settings.BLOCK_STRUCTURES_SETTINGS['TASK_MAX_RETRIES'],
bind=True,
)
def _update_course_in_cache(self, **kwargs):
"""
Updates the course blocks (mongo -> BlockStructure) for the specified course.
"""
if kwargs.get('with_storage'):
enable_for_current_request(STORAGE_BACKING_FOR_CACHE)
_call_and_retry_if_needed(self, api.update_course_in_cache, **kwargs)
@block_structure_task()
def get_course_in_cache_v2(self, **kwargs):
"""
Gets the course blocks for the specified course, updating the cache if needed.
Keyword Arguments:
course_id (string) - The string serialized value of the course key.
with_storage (boolean) - Whether or not storage backing should be
enabled for any generated block structure(s).
"""
_get_course_in_cache(self, **kwargs)
@block_structure_task()
def get_course_in_cache(self, course_id):
"""
Gets the course blocks for the specified course, updating the cache if needed.
"""
_call_and_retry_if_needed(course_id, api.get_course_in_cache, get_course_in_cache, self.request.id)
_get_course_in_cache(self, course_id=course_id)
def _call_and_retry_if_needed(course_id, api_method, task_method, task_id):
def _get_course_in_cache(self, **kwargs):
"""
Gets the course blocks for the specified course, updating the cache if needed.
"""
if kwargs.get('with_storage'):
enable_for_current_request(STORAGE_BACKING_FOR_CACHE)
_call_and_retry_if_needed(self, api.get_course_in_cache, **kwargs)
def _call_and_retry_if_needed(self, api_method, **kwargs):
"""
Calls the given api_method with the given course_id, retrying task_method upon failure.
"""
try:
course_key = CourseKey.from_string(course_id)
course_key = CourseKey.from_string(kwargs['course_id'])
api_method(course_key)
except NO_RETRY_TASKS as exc:
except NO_RETRY_TASKS:
# Known unrecoverable errors
log.exception(
"update_course_in_cache encountered unrecoverable error in course {}, task_id {}".format(
course_id,
task_id
)
"BlockStructure: %s encountered unrecoverable error in course %s, task_id %s",
self.__name__,
kwargs.get('course_id'),
self.request.id,
)
raise
except RETRY_TASKS as exc:
log.exception("%s encountered expected error, retrying.", task_method.__name__)
raise task_method.retry(args=[course_id], exc=exc)
log.exception("%s encountered expected error, retrying.", self.__name__)
raise self.retry(kwargs=kwargs, exc=exc)
except Exception as exc: # pylint: disable=broad-except
log.exception(
"%s encountered unknown error. Retry #%d",
task_method.__name__,
task_method.request.retries,
"BlockStructure: %s encountered unknown error in course %s, task_id %s. Retry #%d",
self.__name__,
kwargs.get('course_id'),
self.request.id,
self.request.retries,
)
raise task_method.retry(args=[course_id], exc=exc)
raise self.retry(kwargs=kwargs, exc=exc)

6
openedx/core/djangoapps/content/block_structure/tests/test_signals.py
View File

@@ -11,7 +11,7 @@ from xmodule.modulestore.tests.factories import CourseFactory
from ..api import get_block_structure_manager
from ..config import INVALIDATE_CACHE_ON_PUBLISH
from ..signals import _listen_for_course_publish
from ..signals import _update_block_structure_on_course_publish
from .helpers import is_course_in_block_structure_cache, override_config_setting
@@ -76,7 +76,7 @@ class CourseBlocksSignalTest(ModuleStoreTestCase):
(LibraryLocator(org='org', course='course'), False),
)
@ddt.unpack
@patch('openedx.core.djangoapps.content.block_structure.tasks.update_course_in_cache.apply_async')
@patch('openedx.core.djangoapps.content.block_structure.tasks.update_course_in_cache_v2.apply_async')
def test_update_only_for_courses(self, key, expect_update_called, mock_update):
_listen_for_course_publish(sender=None, course_key=key)
_update_block_structure_on_course_publish(sender=None, course_key=key)
self.assertEqual(mock_update.called, expect_update_called)

6
openedx/core/djangoapps/content/block_structure/tests/test_tasks.py
View File

@@ -6,19 +6,19 @@ from mock import patch
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from ..tasks import update_course_in_cache
from ..tasks import update_course_in_cache_v2
class UpdateCourseInCacheTaskTest(ModuleStoreTestCase):
"""
Ensures that the update_course_in_cache task runs as expected.
"""
@patch('openedx.core.djangoapps.content.block_structure.tasks.update_course_in_cache.retry')
@patch('openedx.core.djangoapps.content.block_structure.tasks.update_course_in_cache_v2.retry')
@patch('openedx.core.djangoapps.content.block_structure.api.update_course_in_cache')
def test_retry_on_error(self, mock_update, mock_retry):
"""
Ensures that tasks will be retried if IntegrityErrors are encountered.
"""
mock_update.side_effect = Exception("WHAMMY")
update_course_in_cache.apply(args=["invalid_course_key raises exception 12345 meow"])
update_course_in_cache_v2.apply(kwargs=dict(course_id="invalid_course_key raises exception 12345 meow"))
self.assertTrue(mock_retry.called)