Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #14617 from edx/neem/command-generate-blocks

Browse Source 
Block Structure Storage management command
This commit is contained in:
Nimisha Asthagiri
2017-03-02 13:04:50 -05:00
committed by GitHub
parent 95e24a4b85 952143b776
commit ed5eda3fd0
3 changed files with 88 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
lms/djangoapps/course_blocks/management/commands/generate_course_blocks.py
View File

@@ -7,6 +7,7 @@ from django.core.management.base import BaseCommand
from xmodule.modulestore.django import modulestore
import openedx.core.djangoapps.content.block_structure.api as api
from openedx.core.djangoapps.content.block_structure.config import _bs_waffle_switch_name, STORAGE_BACKING_FOR_CACHE
import openedx.core.djangoapps.content.block_structure.tasks as tasks
import openedx.core.djangoapps.content.block_structure.store as store
from openedx.core.lib.command_utils import (
@@ -14,6 +15,8 @@ from openedx.core.lib.command_utils import (
validate_dependent_option,
parse_course_keys,
)
from request_cache.middleware import RequestCache, func_call_cache_key
from openedx.core.djangolib.waffle_utils import is_switch_enabled
log = logging.getLogger(__name__)
@@ -25,8 +28,8 @@ class Command(BaseCommand):
$ ./manage.py lms generate_course_blocks --all --settings=devstack
$ ./manage.py lms generate_course_blocks 'edX/DemoX/Demo_Course' --settings=devstack
"""
args = '<course_id course_id ...>'
help = 'Generates and stores course blocks for one or more courses.'
args = u'<course_id course_id ...>'
help = u'Generates and stores course blocks for one or more courses.'
def add_arguments(self, parser):
"""
@@ -36,43 +39,49 @@ class Command(BaseCommand):
'--courses',
dest='courses',
nargs='+',
help='Generate course blocks for the list of courses provided.',
help=u'Generate course blocks for the list of courses provided.',
)
parser.add_argument(
'--all_courses',
help='Generate course blocks for all courses, given the requested start and end indices.',
help=u'Generate course blocks for all courses, given the requested start and end indices.',
action='store_true',
default=False,
)
parser.add_argument(
'--enqueue_task',
help='Enqueue the tasks for asynchronous computation.',
help=u'Enqueue the tasks for asynchronous computation.',
action='store_true',
default=False,
)
parser.add_argument(
'--routing_key',
dest='routing_key',
help='Routing key to use for asynchronous computation.',
help=u'Routing key to use for asynchronous computation.',
)
parser.add_argument(
'--force_update',
help='Force update of the course blocks for the requested courses.',
help=u'Force update of the course blocks for the requested courses.',
action='store_true',
default=False,
)
parser.add_argument(
'--start_index',
help='Starting index of course list.',
help=u'Starting index of course list.',
default=0,
type=int,
)
parser.add_argument(
'--end_index',
help='Ending index of course list.',
help=u'Ending index of course list.',
default=0,
type=int,
)
parser.add_argument(
'--with_storage',
help=u'Store the course blocks in Storage, overriding value of the storage_backing_for_cache waffle switch',
action='store_true',
default=False,
)
def handle(self, *args, **options):
@@ -91,9 +100,9 @@ class Command(BaseCommand):
self._set_log_levels(options)
log.warning('STARTED generating Course Blocks for %d courses.', len(course_keys))
log.critical(u'STARTED generating Course Blocks for %d courses.', len(course_keys))
self._generate_course_blocks(options, course_keys)
log.warning('FINISHED generating Course Blocks for %d courses.', len(course_keys))
log.critical(u'FINISHED generating Course Blocks for %d courses.', len(course_keys))
def _set_log_levels(self, options):
"""
@@ -119,23 +128,40 @@ class Command(BaseCommand):
"""
Generates course blocks for the given course_keys per the given options.
"""
if options.get('with_storage'):
self._enable_storage()
for course_key in course_keys:
try:
log.info('STARTED generating Course Blocks for course: %s.', course_key)
if options.get('enqueue_task'):
action = tasks.update_course_in_cache if options.get('force_update') else tasks.get_course_in_cache
task_options = {'routing_key': options['routing_key']} if options.get('routing_key') else {}
action.apply_async([unicode(course_key)], **task_options)
else:
action = api.update_course_in_cache if options.get('force_update') else api.get_course_in_cache
action(course_key)
log.info('FINISHED generating Course Blocks for course: %s.', course_key)
log.info(u'STARTED generating Course Blocks for course: %s.', course_key)
self._generate_for_course(options, course_key)
log.info(u'FINISHED generating Course Blocks for course: %s.', course_key)
except Exception as ex: # pylint: disable=broad-except
log.exception(
'An error occurred while generating course blocks for %s: %s',
u'An error occurred while generating course blocks for %s: %s',
unicode(course_key),
ex.message,
)
def _generate_for_course(self, options, course_key):
"""
Generates course blocks for the given course_key per the given options.
"""
if options.get('enqueue_task'):
action = tasks.update_course_in_cache if options.get('force_update') else tasks.get_course_in_cache
task_options = {'routing_key': options['routing_key']} if options.get('routing_key') else {}
action.apply_async([unicode(course_key)], **task_options)
else:
action = api.update_course_in_cache if options.get('force_update') else api.get_course_in_cache
action(course_key)
def _enable_storage(self):
"""
Enables storage backing by setting the waffle's cached value to True.
"""
cache_key = func_call_cache_key(
is_switch_enabled.request_cached_contained_func,
_bs_waffle_switch_name(STORAGE_BACKING_FOR_CACHE),
)
RequestCache.get_request_cache().data[cache_key] = True
log.warning(u'STORAGE_BACKING_FOR_CACHE is enabled.')

29
lms/djangoapps/course_blocks/management/commands/tests/test_generate_course_blocks.py
View File

@@ -6,11 +6,13 @@ from django.core.management.base import CommandError
import itertools
from mock import patch
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from xmodule.modulestore.tests.factories import CourseFactory
from .. import generate_course_blocks
from openedx.core.djangoapps.content.block_structure.tests.helpers import is_course_in_block_structure_cache
from openedx.core.djangoapps.content.block_structure.tests.helpers import (
is_course_in_block_structure_cache,
is_course_in_block_structure_storage,
)
@ddt.ddt
@@ -43,6 +45,20 @@ class TestGenerateCourseBlocks(ModuleStoreTestCase):
for course_key in course_keys:
self.assertTrue(is_course_in_block_structure_cache(course_key, self.store))
def _assert_courses_not_in_block_storage(self, *course_keys):
"""
Assert courses don't exist in course block storage.
"""
for course_key in course_keys:
self.assertFalse(is_course_in_block_structure_storage(course_key, self.store))
def _assert_courses_in_block_storage(self, *course_keys):
"""
Assert courses exist in course block storage.
"""
for course_key in course_keys:
self.assertTrue(is_course_in_block_structure_storage(course_key, self.store))
def _assert_message_presence_in_logs(self, message, mock_log, expected_presence=True):
"""
Asserts that the logger was called with the given message.
@@ -69,6 +85,13 @@ class TestGenerateCourseBlocks(ModuleStoreTestCase):
self.command.handle(courses=[unicode(self.course_keys[0])])
self._assert_courses_in_block_cache(self.course_keys[0])
self._assert_courses_not_in_block_cache(*self.course_keys[1:])
self._assert_courses_not_in_block_storage(*self.course_keys)
def test_with_storage(self):
self.command.handle(with_storage=True, courses=[unicode(self.course_keys[0])])
self._assert_courses_in_block_cache(self.course_keys[0])
self._assert_courses_in_block_storage(self.course_keys[0])
self._assert_courses_not_in_block_storage(*self.course_keys[1:])
@ddt.data(
*itertools.product(

13
openedx/core/djangoapps/content/block_structure/tests/helpers.py
View File

@@ -13,6 +13,7 @@ from ..api import get_cache
from ..block_structure import BlockStructureBlockData
from ..config import _bs_waffle_switch_name
from ..exceptions import BlockStructureNotFound
from ..models import BlockStructureModel
from ..store import BlockStructureStore
from ..transformer import BlockStructureTransformer, FilteringTransformerMixin
from ..transformer_registry import TransformerRegistry
@@ -30,6 +31,18 @@ def is_course_in_block_structure_cache(course_key, store):
return False
def is_course_in_block_structure_storage(course_key, store):
"""
Returns whether the given course is in Block Structure storage.
"""
course_usage_key = store.make_course_usage_key(course_key)
try:
BlockStructureModel.get(course_usage_key)
return True
except BlockStructureNotFound:
return False
class override_config_setting(override_switch): # pylint:disable=invalid-name
"""
Subclasses override_switch to use the block structure