Collect Course Blocks on Course Publish and Management Command

MA-1368

lms/djangoapps/course_blocks/__init__.py

@@ -1,6 +1,6 @@
 """
 The Course Blocks app, built upon the Block Cache framework in
-openedx.core.lib.block_cache, is a higher layer django app in LMS that
+openedx.core.lib.block_structure, is a higher layer django app in LMS that
 provides additional context of Courses and Users (via usage_info.py) with
 implementations for Block Structure Transformers that are related to
 block structure course access.

lms/djangoapps/course_blocks/management/commands/generate_course_blocks.py

@@ -0,0 +1,88 @@
+"""
+Command to load course blocks.
+"""
+import logging
+
+from django.core.management.base import BaseCommand, CommandError
+from opaque_keys import InvalidKeyError
+from opaque_keys.edx.keys import CourseKey
+from xmodule.modulestore.django import modulestore
+
+from ...api import get_course_in_cache
+
+
+log = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """
+    Example usage:
+        $ ./manage.py lms generate_course_blocks --all --settings=devstack
+        $ ./manage.py lms generate_course_blocks 'edX/DemoX/Demo_Course' --settings=devstack
+    """
+    args = '<course_id course_id ...>'
+    help = 'Generates and stores course blocks for one or more courses.'
+
+    def add_arguments(self, parser):
+        """
+        Entry point for subclassed commands to add custom arguments.
+        """
+        parser.add_argument(
+            '--all',
+            help='Generate course blocks for all or specified courses.',
+            action='store_true',
+            default=False,
+        )
+        parser.add_argument(
+            '--dags',
+            help='Find and log DAGs for all or specified courses.',
+            action='store_true',
+            default=False,
+        )
+
+    def handle(self, *args, **options):
+
+        if options.get('all'):
+            course_keys = [course.id for course in modulestore().get_course_summaries()]
+        else:
+            if len(args) < 1:
+                raise CommandError('At least one course or --all must be specified.')
+            try:
+                course_keys = [CourseKey.from_string(arg) for arg in args]
+            except InvalidKeyError:
+                raise CommandError('Invalid key specified.')
+
+        log.info('Generating course blocks for %d courses.', len(course_keys))
+        log.debug('Generating course blocks for the following courses: %s', course_keys)
+
+        for course_key in course_keys:
+            try:
+                block_structure = get_course_in_cache(course_key)
+                if options.get('dags'):
+                    self._find_and_log_dags(block_structure, course_key)
+            except Exception as ex:  # pylint: disable=broad-except
+                log.exception(
+                    'An error occurred while generating course blocks for %s: %s',
+                    unicode(course_key),
+                    ex.message,
+                )
+
+        log.info('Finished generating course blocks.')
+
+    def _find_and_log_dags(self, block_structure, course_key):
+        """
+        Finds all DAGs within the given block structure.
+
+        Arguments:
+            BlockStructureBlockData - The block structure in which to find DAGs.
+        """
+        log.info('DAG check starting for course %s.', unicode(course_key))
+        for block_key in block_structure.get_block_keys():
+            parents = block_structure.get_parents(block_key)
+            if len(parents) > 1:
+                log.warning(
+                    'DAG alert - %s has multiple parents: %s.',
+                    unicode(block_key),
+                    [unicode(parent) for parent in parents],
+                )
+        log.info('DAG check complete for course %s.', unicode(course_key))

lms/djangoapps/course_blocks/management/commands/tests/test_generate_course_blocks.py

@@ -0,0 +1,81 @@
+"""
+Tests for generate_course_blocks management command.
+"""
+from django.core.management.base import CommandError
+from mock import patch
+
+from xmodule.modulestore import ModuleStoreEnum
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
+from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
+from .. import generate_course_blocks
+from ....tests.helpers import is_course_in_block_structure_cache
+
+
+class TestGenerateCourseBlocks(ModuleStoreTestCase):
+    """
+    Tests generate course blocks management command.
+    """
+    def setUp(self):
+        """
+        Create courses in modulestore.
+        """
+        super(TestGenerateCourseBlocks, self).setUp()
+        self.course_1 = CourseFactory.create()
+        self.course_2 = CourseFactory.create()
+        self.command = generate_course_blocks.Command()
+
+    def _assert_courses_not_in_block_cache(self, *courses):
+        """
+        Assert courses don't exist in the course block cache.
+        """
+        for course_key in courses:
+            self.assertFalse(is_course_in_block_structure_cache(course_key, self.store))
+
+    def _assert_courses_in_block_cache(self, *courses):
+        """
+        Assert courses exist in course block cache.
+        """
+        for course_key in courses:
+            self.assertTrue(is_course_in_block_structure_cache(course_key, self.store))
+
+    def test_generate_all(self):
+        self._assert_courses_not_in_block_cache(self.course_1.id, self.course_2.id)
+        self.command.handle(all=True)
+        self._assert_courses_in_block_cache(self.course_1.id, self.course_2.id)
+
+    def test_generate_one(self):
+        self._assert_courses_not_in_block_cache(self.course_1.id, self.course_2.id)
+        self.command.handle(unicode(self.course_1.id))
+        self._assert_courses_in_block_cache(self.course_1.id)
+        self._assert_courses_not_in_block_cache(self.course_2.id)
+
+    @patch('lms.djangoapps.course_blocks.management.commands.generate_course_blocks.log')
+    def test_generate_no_dags(self, mock_log):
+        self.command.handle(dags=True, all=True)
+        self.assertEquals(mock_log.warning.call_count, 0)
+
+    @patch('lms.djangoapps.course_blocks.management.commands.generate_course_blocks.log')
+    def test_generate_with_dags(self, mock_log):
+        with self.store.branch_setting(ModuleStoreEnum.Branch.draft_preferred):
+            item1 = ItemFactory.create(parent=self.course_1)
+            item2 = ItemFactory.create(parent=item1)
+            item3 = ItemFactory.create(parent=item1)
+            item2.children.append(item3.location)
+            self.store.update_item(item2, ModuleStoreEnum.UserID.mgmt_command)
+            self.store.publish(self.course_1.location, ModuleStoreEnum.UserID.mgmt_command)
+
+        self.command.handle(dags=True, all=True)
+        self.assertEquals(mock_log.warning.call_count, 1)
+
+    @patch('lms.djangoapps.course_blocks.management.commands.generate_course_blocks.log')
+    def test_not_found_key(self, mock_log):
+        self.command.handle('fake/course/id', all=False)
+        self.assertTrue(mock_log.exception.called)
+
+    def test_invalid_key(self):
+        with self.assertRaises(CommandError):
+            self.command.handle('not/found', all=False)
+
+    def test_no_params(self):
+        with self.assertRaises(CommandError):
+            self.command.handle(all=False)

lms/djangoapps/course_blocks/signals.py

@@ -6,16 +6,21 @@ from django.dispatch.dispatcher import receiver
 from xmodule.modulestore.django import SignalHandler
 
 from .api import clear_course_from_cache
+from .tasks import update_course_in_cache
 
 
 @receiver(SignalHandler.course_published)
 def _listen_for_course_publish(sender, course_key, **kwargs):  # pylint: disable=unused-argument
     """
     Catches the signal that a course has been published in the module
-    store and invalidates the corresponding cache entry if one exists.
+    store and creates/updates the corresponding cache entry.
     """
     clear_course_from_cache(course_key)
 
+    # The countdown=0 kwarg ensures the call occurs after the signal emitter
+    # has finished all operations.
+    update_course_in_cache.apply_async([unicode(course_key)], countdown=0)
+
 
 @receiver(SignalHandler.course_deleted)
 def _listen_for_course_delete(sender, course_key, **kwargs):  # pylint: disable=unused-argument

lms/djangoapps/course_blocks/tasks.py

@@ -0,0 +1,20 @@
+"""
+Asynchronous tasks related to the Course Blocks sub-application.
+"""
+import logging
+from celery.task import task
+from opaque_keys.edx.keys import CourseKey
+
+from . import api
+
+
+log = logging.getLogger('edx.celery.task')
+
+
+@task()
+def update_course_in_cache(course_key):
+    """
+    Updates the course blocks (in the database) for the specified course.
+    """
+    course_key = CourseKey.from_string(course_key)
+    api.update_course_in_cache(course_key)

lms/djangoapps/course_blocks/tests/test_signals.py

@@ -0,0 +1,51 @@
+"""
+Unit tests for the Course Blocks signals
+"""
+
+from xmodule.modulestore.exceptions import ItemNotFoundError
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
+from xmodule.modulestore.tests.factories import CourseFactory
+
+from ..api import get_course_blocks, _get_block_structure_manager
+from ..transformers.visibility import VisibilityTransformer
+from .helpers import is_course_in_block_structure_cache, EnableTransformerRegistryMixin
+
+
+class CourseBlocksSignalTest(EnableTransformerRegistryMixin, ModuleStoreTestCase):
+    """
+    Tests for the Course Blocks signal
+    """
+
+    def setUp(self):
+        super(CourseBlocksSignalTest, self).setUp(create_user=True)
+        self.course = CourseFactory.create()
+        self.course_usage_key = self.store.make_course_usage_key(self.course.id)
+
+    def test_course_publish(self):
+        # course is not visible to staff only
+        self.assertFalse(self.course.visible_to_staff_only)
+        orig_block_structure = get_course_blocks(self.user, self.course_usage_key)
+        self.assertFalse(
+            VisibilityTransformer.get_visible_to_staff_only(orig_block_structure, self.course_usage_key)
+        )
+
+        # course becomes visible to staff only
+        self.course.visible_to_staff_only = True
+        self.store.update_item(self.course, self.user.id)
+
+        updated_block_structure = get_course_blocks(self.user, self.course_usage_key)
+        self.assertTrue(
+            VisibilityTransformer.get_visible_to_staff_only(updated_block_structure, self.course_usage_key)
+        )
+
+    def test_course_delete(self):
+        get_course_blocks(self.user, self.course_usage_key)
+        bs_manager = _get_block_structure_manager(self.course.id)
+        self.assertIsNotNone(bs_manager.get_collected())
+        self.assertTrue(is_course_in_block_structure_cache(self.course.id, self.store))
+
+        self.store.delete_course(self.course.id, self.user.id)
+        with self.assertRaises(ItemNotFoundError):
+            bs_manager.get_collected()
+
+        self.assertFalse(is_course_in_block_structure_cache(self.course.id, self.store))

openedx/core/djangoapps/content/course_overviews/management/commands/generate_course_overview.py

@@ -34,7 +34,7 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
 
         if options['all']:
-            course_keys = [course.id for course in modulestore().get_courses()]
+            course_keys = [course.id for course in modulestore().get_course_summaries()]
         else:
             if len(args) < 1:
                 raise CommandError('At least one course or --all must be specified.')