diff --git a/lms/djangoapps/course_api/blocks/tests/test_utils.py b/lms/djangoapps/course_api/blocks/tests/helpers.py similarity index 100% rename from lms/djangoapps/course_api/blocks/tests/test_utils.py rename to lms/djangoapps/course_api/blocks/tests/helpers.py diff --git a/lms/djangoapps/course_blocks/__init__.py b/lms/djangoapps/course_blocks/__init__.py index 70663b7dfc..2843a9c480 100644 --- a/lms/djangoapps/course_blocks/__init__.py +++ b/lms/djangoapps/course_blocks/__init__.py @@ -1,6 +1,6 @@ """ The Course Blocks app, built upon the Block Cache framework in -openedx.core.lib.block_cache, is a higher layer django app in LMS that +openedx.core.lib.block_structure, is a higher layer django app in LMS that provides additional context of Courses and Users (via usage_info.py) with implementations for Block Structure Transformers that are related to block structure course access. diff --git a/lms/djangoapps/course_blocks/management/__init__.py b/lms/djangoapps/course_blocks/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lms/djangoapps/course_blocks/management/commands/__init__.py b/lms/djangoapps/course_blocks/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lms/djangoapps/course_blocks/management/commands/generate_course_blocks.py b/lms/djangoapps/course_blocks/management/commands/generate_course_blocks.py new file mode 100644 index 0000000000..590576c9d2 --- /dev/null +++ b/lms/djangoapps/course_blocks/management/commands/generate_course_blocks.py @@ -0,0 +1,88 @@ +""" +Command to load course blocks. +""" +import logging + +from django.core.management.base import BaseCommand, CommandError +from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey +from xmodule.modulestore.django import modulestore + +from ...api import get_course_in_cache + + +log = logging.getLogger(__name__) + + +class Command(BaseCommand): + """ + Example usage: + $ ./manage.py lms generate_course_blocks --all --settings=devstack + $ ./manage.py lms generate_course_blocks 'edX/DemoX/Demo_Course' --settings=devstack + """ + args = '' + help = 'Generates and stores course blocks for one or more courses.' + + def add_arguments(self, parser): + """ + Entry point for subclassed commands to add custom arguments. + """ + parser.add_argument( + '--all', + help='Generate course blocks for all or specified courses.', + action='store_true', + default=False, + ) + parser.add_argument( + '--dags', + help='Find and log DAGs for all or specified courses.', + action='store_true', + default=False, + ) + + def handle(self, *args, **options): + + if options.get('all'): + course_keys = [course.id for course in modulestore().get_course_summaries()] + else: + if len(args) < 1: + raise CommandError('At least one course or --all must be specified.') + try: + course_keys = [CourseKey.from_string(arg) for arg in args] + except InvalidKeyError: + raise CommandError('Invalid key specified.') + + log.info('Generating course blocks for %d courses.', len(course_keys)) + log.debug('Generating course blocks for the following courses: %s', course_keys) + + for course_key in course_keys: + try: + block_structure = get_course_in_cache(course_key) + if options.get('dags'): + self._find_and_log_dags(block_structure, course_key) + except Exception as ex: # pylint: disable=broad-except + log.exception( + 'An error occurred while generating course blocks for %s: %s', + unicode(course_key), + ex.message, + ) + + log.info('Finished generating course blocks.') + + def _find_and_log_dags(self, block_structure, course_key): + """ + Finds all DAGs within the given block structure. + + Arguments: + BlockStructureBlockData - The block structure in which to find DAGs. + """ + log.info('DAG check starting for course %s.', unicode(course_key)) + for block_key in block_structure.get_block_keys(): + parents = block_structure.get_parents(block_key) + if len(parents) > 1: + log.warning( + 'DAG alert - %s has multiple parents: %s.', + unicode(block_key), + [unicode(parent) for parent in parents], + ) + log.info('DAG check complete for course %s.', unicode(course_key)) diff --git a/lms/djangoapps/course_blocks/management/commands/tests/__init__.py b/lms/djangoapps/course_blocks/management/commands/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lms/djangoapps/course_blocks/management/commands/tests/test_generate_course_blocks.py b/lms/djangoapps/course_blocks/management/commands/tests/test_generate_course_blocks.py new file mode 100644 index 0000000000..9c384be15f --- /dev/null +++ b/lms/djangoapps/course_blocks/management/commands/tests/test_generate_course_blocks.py @@ -0,0 +1,81 @@ +""" +Tests for generate_course_blocks management command. +""" +from django.core.management.base import CommandError +from mock import patch + +from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory +from .. import generate_course_blocks +from ....tests.helpers import is_course_in_block_structure_cache + + +class TestGenerateCourseBlocks(ModuleStoreTestCase): + """ + Tests generate course blocks management command. + """ + def setUp(self): + """ + Create courses in modulestore. + """ + super(TestGenerateCourseBlocks, self).setUp() + self.course_1 = CourseFactory.create() + self.course_2 = CourseFactory.create() + self.command = generate_course_blocks.Command() + + def _assert_courses_not_in_block_cache(self, *courses): + """ + Assert courses don't exist in the course block cache. + """ + for course_key in courses: + self.assertFalse(is_course_in_block_structure_cache(course_key, self.store)) + + def _assert_courses_in_block_cache(self, *courses): + """ + Assert courses exist in course block cache. + """ + for course_key in courses: + self.assertTrue(is_course_in_block_structure_cache(course_key, self.store)) + + def test_generate_all(self): + self._assert_courses_not_in_block_cache(self.course_1.id, self.course_2.id) + self.command.handle(all=True) + self._assert_courses_in_block_cache(self.course_1.id, self.course_2.id) + + def test_generate_one(self): + self._assert_courses_not_in_block_cache(self.course_1.id, self.course_2.id) + self.command.handle(unicode(self.course_1.id)) + self._assert_courses_in_block_cache(self.course_1.id) + self._assert_courses_not_in_block_cache(self.course_2.id) + + @patch('lms.djangoapps.course_blocks.management.commands.generate_course_blocks.log') + def test_generate_no_dags(self, mock_log): + self.command.handle(dags=True, all=True) + self.assertEquals(mock_log.warning.call_count, 0) + + @patch('lms.djangoapps.course_blocks.management.commands.generate_course_blocks.log') + def test_generate_with_dags(self, mock_log): + with self.store.branch_setting(ModuleStoreEnum.Branch.draft_preferred): + item1 = ItemFactory.create(parent=self.course_1) + item2 = ItemFactory.create(parent=item1) + item3 = ItemFactory.create(parent=item1) + item2.children.append(item3.location) + self.store.update_item(item2, ModuleStoreEnum.UserID.mgmt_command) + self.store.publish(self.course_1.location, ModuleStoreEnum.UserID.mgmt_command) + + self.command.handle(dags=True, all=True) + self.assertEquals(mock_log.warning.call_count, 1) + + @patch('lms.djangoapps.course_blocks.management.commands.generate_course_blocks.log') + def test_not_found_key(self, mock_log): + self.command.handle('fake/course/id', all=False) + self.assertTrue(mock_log.exception.called) + + def test_invalid_key(self): + with self.assertRaises(CommandError): + self.command.handle('not/found', all=False) + + def test_no_params(self): + with self.assertRaises(CommandError): + self.command.handle(all=False) diff --git a/lms/djangoapps/course_blocks/signals.py b/lms/djangoapps/course_blocks/signals.py index 2767b56a1a..b8e65a723c 100644 --- a/lms/djangoapps/course_blocks/signals.py +++ b/lms/djangoapps/course_blocks/signals.py @@ -6,16 +6,21 @@ from django.dispatch.dispatcher import receiver from xmodule.modulestore.django import SignalHandler from .api import clear_course_from_cache +from .tasks import update_course_in_cache @receiver(SignalHandler.course_published) def _listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable=unused-argument """ Catches the signal that a course has been published in the module - store and invalidates the corresponding cache entry if one exists. + store and creates/updates the corresponding cache entry. """ clear_course_from_cache(course_key) + # The countdown=0 kwarg ensures the call occurs after the signal emitter + # has finished all operations. + update_course_in_cache.apply_async([unicode(course_key)], countdown=0) + @receiver(SignalHandler.course_deleted) def _listen_for_course_delete(sender, course_key, **kwargs): # pylint: disable=unused-argument diff --git a/lms/djangoapps/course_blocks/tasks.py b/lms/djangoapps/course_blocks/tasks.py new file mode 100644 index 0000000000..d86fdf5243 --- /dev/null +++ b/lms/djangoapps/course_blocks/tasks.py @@ -0,0 +1,20 @@ +""" +Asynchronous tasks related to the Course Blocks sub-application. +""" +import logging +from celery.task import task +from opaque_keys.edx.keys import CourseKey + +from . import api + + +log = logging.getLogger('edx.celery.task') + + +@task() +def update_course_in_cache(course_key): + """ + Updates the course blocks (in the database) for the specified course. + """ + course_key = CourseKey.from_string(course_key) + api.update_course_in_cache(course_key) diff --git a/lms/djangoapps/course_blocks/tests/__init__.py b/lms/djangoapps/course_blocks/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lms/djangoapps/course_blocks/tests/test_utils.py b/lms/djangoapps/course_blocks/tests/helpers.py similarity index 100% rename from lms/djangoapps/course_blocks/tests/test_utils.py rename to lms/djangoapps/course_blocks/tests/helpers.py diff --git a/lms/djangoapps/course_blocks/tests/test_signals.py b/lms/djangoapps/course_blocks/tests/test_signals.py new file mode 100644 index 0000000000..8c232a0f49 --- /dev/null +++ b/lms/djangoapps/course_blocks/tests/test_signals.py @@ -0,0 +1,51 @@ +""" +Unit tests for the Course Blocks signals +""" + +from xmodule.modulestore.exceptions import ItemNotFoundError +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory + +from ..api import get_course_blocks, _get_block_structure_manager +from ..transformers.visibility import VisibilityTransformer +from .helpers import is_course_in_block_structure_cache, EnableTransformerRegistryMixin + + +class CourseBlocksSignalTest(EnableTransformerRegistryMixin, ModuleStoreTestCase): + """ + Tests for the Course Blocks signal + """ + + def setUp(self): + super(CourseBlocksSignalTest, self).setUp(create_user=True) + self.course = CourseFactory.create() + self.course_usage_key = self.store.make_course_usage_key(self.course.id) + + def test_course_publish(self): + # course is not visible to staff only + self.assertFalse(self.course.visible_to_staff_only) + orig_block_structure = get_course_blocks(self.user, self.course_usage_key) + self.assertFalse( + VisibilityTransformer.get_visible_to_staff_only(orig_block_structure, self.course_usage_key) + ) + + # course becomes visible to staff only + self.course.visible_to_staff_only = True + self.store.update_item(self.course, self.user.id) + + updated_block_structure = get_course_blocks(self.user, self.course_usage_key) + self.assertTrue( + VisibilityTransformer.get_visible_to_staff_only(updated_block_structure, self.course_usage_key) + ) + + def test_course_delete(self): + get_course_blocks(self.user, self.course_usage_key) + bs_manager = _get_block_structure_manager(self.course.id) + self.assertIsNotNone(bs_manager.get_collected()) + self.assertTrue(is_course_in_block_structure_cache(self.course.id, self.store)) + + self.store.delete_course(self.course.id, self.user.id) + with self.assertRaises(ItemNotFoundError): + bs_manager.get_collected() + + self.assertFalse(is_course_in_block_structure_cache(self.course.id, self.store)) diff --git a/lms/djangoapps/course_blocks/transformers/tests/test_helpers.py b/lms/djangoapps/course_blocks/transformers/tests/helpers.py similarity index 100% rename from lms/djangoapps/course_blocks/transformers/tests/test_helpers.py rename to lms/djangoapps/course_blocks/transformers/tests/helpers.py diff --git a/openedx/core/djangoapps/content/course_overviews/management/commands/generate_course_overview.py b/openedx/core/djangoapps/content/course_overviews/management/commands/generate_course_overview.py index 48ae577092..def0fcdfe7 100644 --- a/openedx/core/djangoapps/content/course_overviews/management/commands/generate_course_overview.py +++ b/openedx/core/djangoapps/content/course_overviews/management/commands/generate_course_overview.py @@ -34,7 +34,7 @@ class Command(BaseCommand): def handle(self, *args, **options): if options['all']: - course_keys = [course.id for course in modulestore().get_courses()] + course_keys = [course.id for course in modulestore().get_course_summaries()] else: if len(args) < 1: raise CommandError('At least one course or --all must be specified.') diff --git a/openedx/core/lib/block_structure/tests/test_utils.py b/openedx/core/lib/block_structure/tests/helpers.py similarity index 100% rename from openedx/core/lib/block_structure/tests/test_utils.py rename to openedx/core/lib/block_structure/tests/helpers.py