From 8d1d7b2222086af702ef32f71d550e94453ea8bc Mon Sep 17 00:00:00 2001 From: Matthew Piatetsky Date: Thu, 25 Feb 2021 14:56:21 -0500 Subject: [PATCH] feat: Export highlights to s3 for use by braze Things other than highlights may be exported in the future. The storage class is flexible so backends other than s3 may be used in the future. AA-461 --- .../export_course_metadata/__init__.py | 0 cms/djangoapps/export_course_metadata/apps.py | 18 ++++++ .../export_course_metadata/signals.py | 32 +++++++++++ .../export_course_metadata/storage.py | 20 +++++++ .../export_course_metadata/test_signals.py | 56 +++++++++++++++++++ .../export_course_metadata/toggles.py | 15 +++++ cms/envs/common.py | 5 ++ cms/envs/production.py | 7 +++ .../schedules/content_highlights.py | 16 ++++++ .../tests/test_content_highlights.py | 9 +++ 10 files changed, 178 insertions(+) create mode 100644 cms/djangoapps/export_course_metadata/__init__.py create mode 100644 cms/djangoapps/export_course_metadata/apps.py create mode 100644 cms/djangoapps/export_course_metadata/signals.py create mode 100644 cms/djangoapps/export_course_metadata/storage.py create mode 100644 cms/djangoapps/export_course_metadata/test_signals.py create mode 100644 cms/djangoapps/export_course_metadata/toggles.py diff --git a/cms/djangoapps/export_course_metadata/__init__.py b/cms/djangoapps/export_course_metadata/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cms/djangoapps/export_course_metadata/apps.py b/cms/djangoapps/export_course_metadata/apps.py new file mode 100644 index 0000000000..554c0dddde --- /dev/null +++ b/cms/djangoapps/export_course_metadata/apps.py @@ -0,0 +1,18 @@ +""" +Define the export_course_metadata Django App. +""" + +from django.apps import AppConfig + + +class ExportCourseMetadataConfig(AppConfig): + """ + App for exporting a subset of course metadata + """ + name = 'cms.djangoapps.export_course_metadata' + + def ready(self): + """ + Connect signal handler that exports course metadata + """ + from . import signals # pylint: disable=unused-import diff --git a/cms/djangoapps/export_course_metadata/signals.py b/cms/djangoapps/export_course_metadata/signals.py new file mode 100644 index 0000000000..bd12c868e5 --- /dev/null +++ b/cms/djangoapps/export_course_metadata/signals.py @@ -0,0 +1,32 @@ +""" +This file exports metadata about the course. +""" + +import json + +from django.core.files.base import ContentFile +from django.dispatch import receiver +from openedx.core.djangoapps.schedules.content_highlights import get_all_course_highlights +from xmodule.modulestore.django import SignalHandler + +from .storage import course_metadata_export_storage +from .toggles import EXPORT_COURSE_METADATA_FLAG + + +@receiver(SignalHandler.course_published) +def export_course_metadata(sender, course_key, **kwargs): # pylint: disable=unused-argument + """ + Export course metadata on course publish. + + File format + '{"highlights": [["week1highlight1", "week1highlight2"], ["week1highlight1", "week1highlight2"], [], []]}' + To retrieve highlights for week1, you would need to do + course_metadata['highlights'][0] + + This data is initially being used by Braze Connected Content to include + section highlights in emails, but may be used for other things in the future. + """ + if EXPORT_COURSE_METADATA_FLAG.is_enabled(): + highlights = get_all_course_highlights(course_key) + highlights_content = ContentFile(json.dumps({'highlights': highlights})) + course_metadata_export_storage.save('course_metadata_export/{}.json'.format(course_key), highlights_content) diff --git a/cms/djangoapps/export_course_metadata/storage.py b/cms/djangoapps/export_course_metadata/storage.py new file mode 100644 index 0000000000..d010dd7bd2 --- /dev/null +++ b/cms/djangoapps/export_course_metadata/storage.py @@ -0,0 +1,20 @@ +""" +Storage backend for course metadata export. +""" + + +from django.conf import settings +from django.core.files.storage import get_storage_class +from storages.backends.s3boto import S3BotoStorage + + +class CourseMetadataExportS3Storage(S3BotoStorage): # pylint: disable=abstract-method + """ + S3 backend for course metadata export + """ + + def __init__(self): + bucket = settings.COURSE_METADATA_EXPORT_BUCKET + super().__init__(bucket=bucket, custom_domain=None, querystring_auth=True) + +course_metadata_export_storage = get_storage_class(settings.COURSE_METADATA_EXPORT_STORAGE)() diff --git a/cms/djangoapps/export_course_metadata/test_signals.py b/cms/djangoapps/export_course_metadata/test_signals.py new file mode 100644 index 0000000000..c0da25f9e0 --- /dev/null +++ b/cms/djangoapps/export_course_metadata/test_signals.py @@ -0,0 +1,56 @@ +""" +Tests for signals.py +""" + +from unittest.mock import patch + +from edx_toggles.toggles.testutils import override_waffle_flag +from xmodule.modulestore.django import SignalHandler +from xmodule.modulestore.tests.django_utils import TEST_DATA_SPLIT_MODULESTORE, SharedModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory + +from .signals import export_course_metadata +from .toggles import EXPORT_COURSE_METADATA_FLAG + + +@override_waffle_flag(EXPORT_COURSE_METADATA_FLAG, True) +class TestExportCourseMetadata(SharedModuleStoreTestCase): + """ + Tests for the export_course_metadata function + """ + MODULESTORE = TEST_DATA_SPLIT_MODULESTORE + ENABLED_SIGNALS = ['course_published'] + + def setUp(self): + super().setUp() + SignalHandler.course_published.disconnect(export_course_metadata) + self.course = CourseFactory.create(highlights_enabled_for_messaging=True) + self.course_key = self.course.id + + def tearDown(self): + super().tearDown() + SignalHandler.course_published.disconnect(export_course_metadata) + + def _create_chapter(self, **kwargs): + ItemFactory.create( + parent=self.course, + category='chapter', + **kwargs + ) + + @patch('cms.djangoapps.export_course_metadata.signals.course_metadata_export_storage') + @patch('cms.djangoapps.export_course_metadata.signals.ContentFile') + def test_happy_path(self, patched_content, patched_storage): + """ Ensure we call the storage class with the correct parameters and course metadata """ + all_highlights = [["week1highlight1", "week1highlight2"], ["week1highlight1", "week1highlight2"], [], []] + with self.store.bulk_operations(self.course_key): + for week_highlights in all_highlights: + self._create_chapter(highlights=week_highlights) + SignalHandler.course_published.connect(export_course_metadata) + SignalHandler.course_published.send(sender=None, course_key=self.course_key) + patched_content.assert_called_once_with( + '{"highlights": [["week1highlight1", "week1highlight2"], ["week1highlight1", "week1highlight2"], [], []]}' + ) + patched_storage.save.assert_called_once_with( + 'course_metadata_export/course-v1:org.0+course_0+Run_0.json', patched_content.return_value + ) diff --git a/cms/djangoapps/export_course_metadata/toggles.py b/cms/djangoapps/export_course_metadata/toggles.py new file mode 100644 index 0000000000..8f6474157b --- /dev/null +++ b/cms/djangoapps/export_course_metadata/toggles.py @@ -0,0 +1,15 @@ +""" +Toggles for export_course_metadata app +""" + +from edx_toggles.toggles import WaffleFlag + +# .. toggle_name: export_course_metadata +# .. toggle_implementation: WaffleFlag +# .. toggle_default: False +# .. toggle_description: Export of course metadata (initially to s3 for use by braze) +# .. toggle_use_cases: temporary +# .. toggle_creation_date: 2021-03-01 +# .. toggle_target_removal_date: None +# .. toggle_tickets: AA-461 +EXPORT_COURSE_METADATA_FLAG = WaffleFlag('cms.export_course_metadata', __name__) diff --git a/cms/envs/common.py b/cms/envs/common.py index 746f08a5ae..8afc02b953 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -968,6 +968,8 @@ HTTPS = 'on' ROOT_URLCONF = 'cms.urls' COURSE_IMPORT_EXPORT_BUCKET = '' +COURSE_METADATA_EXPORT_BUCKET = '' + ALTERNATE_WORKER_QUEUES = 'lms' STATIC_URL_BASE = '/static/' @@ -1052,6 +1054,8 @@ derived('LOCALE_PATHS') MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage' COURSE_IMPORT_EXPORT_STORAGE = 'django.core.files.storage.FileSystemStorage' +COURSE_METADATA_EXPORT_STORAGE = 'django.core.files.storage.FileSystemStorage' + ##### EMBARGO ##### EMBARGO_SITE_REDIRECT_URL = None @@ -1389,6 +1393,7 @@ INSTALLED_APPS = [ 'common.djangoapps.student.apps.StudentConfig', # misleading name due to sharing with lms 'openedx.core.djangoapps.course_groups', # not used in cms (yet), but tests run 'cms.djangoapps.xblock_config.apps.XBlockConfig', + 'cms.djangoapps.export_course_metadata.apps.ExportCourseMetadataConfig', # New (Blockstore-based) XBlock runtime 'openedx.core.djangoapps.xblock.apps.StudioXBlockAppConfig', diff --git a/cms/envs/production.py b/cms/envs/production.py index 818395b707..8cdde951e6 100644 --- a/cms/envs/production.py +++ b/cms/envs/production.py @@ -350,6 +350,13 @@ else: USER_TASKS_ARTIFACT_STORAGE = COURSE_IMPORT_EXPORT_STORAGE +COURSE_METADATA_EXPORT_BUCKET = ENV_TOKENS.get('COURSE_METADATA_EXPORT_BUCKET', '') + +if COURSE_METADATA_EXPORT_BUCKET: + COURSE_METADATA_EXPORT_STORAGE = 'cms.djangoapps.export_course_metadata.storage.CourseMetadataExportS3Storage' +else: + COURSE_METADATA_EXPORT_STORAGE = DEFAULT_FILE_STORAGE + DATABASES = AUTH_TOKENS['DATABASES'] # The normal database user does not have enough permissions to run migrations. diff --git a/openedx/core/djangoapps/schedules/content_highlights.py b/openedx/core/djangoapps/schedules/content_highlights.py index 4d0eac8bcb..0734972497 100644 --- a/openedx/core/djangoapps/schedules/content_highlights.py +++ b/openedx/core/djangoapps/schedules/content_highlights.py @@ -14,6 +14,22 @@ from xmodule.modulestore.django import modulestore log = logging.getLogger(__name__) +def get_all_course_highlights(course_key): + """ + This ignores access checks, since highlights may be lurking in currently + inaccessible content. + Returns a list of all the section highlights in the course + """ + try: + course = _get_course_with_highlights(course_key) + + except CourseUpdateDoesNotExist: + return [] + else: + highlights = [section.highlights for section in course.get_children() if not section.hide_from_toc] + return highlights + + def course_has_highlights(course): """ Does the course have any highlights for any section/week in it? diff --git a/openedx/core/djangoapps/schedules/tests/test_content_highlights.py b/openedx/core/djangoapps/schedules/tests/test_content_highlights.py index adb3f63243..fb2210cdb9 100644 --- a/openedx/core/djangoapps/schedules/tests/test_content_highlights.py +++ b/openedx/core/djangoapps/schedules/tests/test_content_highlights.py @@ -5,6 +5,7 @@ from unittest.mock import patch import pytest from openedx.core.djangoapps.schedules.content_highlights import ( course_has_highlights_from_store, + get_all_course_highlights, get_next_section_highlights, get_week_highlights ) @@ -58,6 +59,14 @@ class TestContentHighlights(ModuleStoreTestCase): # lint-amnesty, pylint: disab assert course_has_highlights_from_store(self.course_key) assert get_week_highlights(self.user, self.course_key, week_num=1) == highlights + def test_get_all_course_highlights(self): + all_highlights = [["week1highlight1", "week1highlight2"], ["week1highlight1", "week1highlight2"], [], []] + with self.store.bulk_operations(self.course_key): + for week_highlights in all_highlights: + self._create_chapter(highlights=week_highlights) + + assert get_all_course_highlights(self.course_key) == all_highlights + def test_highlights_disabled_for_messaging(self): highlights = ['A test highlight.'] with self.store.bulk_operations(self.course_key):