diff --git a/lms/djangoapps/discussion/tests/test_tasks.py b/lms/djangoapps/discussion/tests/test_tasks.py index 556b206c5c..c7a7a119fe 100644 --- a/lms/djangoapps/discussion/tests/test_tasks.py +++ b/lms/djangoapps/discussion/tests/test_tasks.py @@ -15,6 +15,7 @@ from django_comment_common.models import ForumsConfig from django_comment_common.signals import comment_created from edx_ace.recipient import Recipient from edx_ace.renderers import EmailRenderer +from edx_ace.channel import ChannelType, get_channel_for_message from edx_ace.utils import date from lms.djangoapps.discussion.signals.handlers import ENABLE_FORUM_NOTIFICATIONS_FOR_SITE_KEY from lms.djangoapps.discussion.tasks import _should_send_message, _track_notification_sent @@ -230,12 +231,12 @@ class TaskTestCase(ModuleStoreTestCase): with emulate_http_request( site=message.context['site'], user=self.thread_author ): - rendered_email = EmailRenderer().render(message) - self.assertTrue(self.comment['body'] in rendered_email.body_html) - self.assertTrue(self.comment_author.username in rendered_email.body_html) - self.assertTrue(self.thread_author.username in rendered_email.body_html) - self.assertTrue(self.mock_permalink in rendered_email.body_html) - self.assertTrue(message.context['site'].domain in rendered_email.body_html) + # pylint: disable=unsupported-membership-test + rendered_email = EmailRenderer().render(get_channel_for_message(ChannelType.EMAIL, message), message) + assert self.comment['body'] in rendered_email.body_html + assert self.comment_author.username in rendered_email.body_html + assert self.mock_permalink.return_value in rendered_email.body_html + assert message.context['site'].domain in rendered_email.body_html def run_should_not_send_email_test(self, thread, comment_dict): """ diff --git a/lms/djangoapps/experiments/utils.py b/lms/djangoapps/experiments/utils.py index f76ff516ec..47d911e039 100644 --- a/lms/djangoapps/experiments/utils.py +++ b/lms/djangoapps/experiments/utils.py @@ -2,6 +2,8 @@ Utilities to facilitate experimentation """ +import hashlib +import re from student.models import CourseEnrollment from django_comment_common.models import Role from course_modes.models import get_cosmetic_verified_display_price @@ -202,3 +204,23 @@ def get_experiment_dashboard_metadata_context(enrollments): """ return {str(enrollment.course): enrollment.course_price for enrollment in enrollments} #TODO END: Clean up REVEM-205 + + +def stable_bucketing_hash_group(group_name, group_count, username): + """ + Return the bucket that a user should be in for a given stable bucketing assignment. + + This function has been verified to return the same values as the stable bucketing + functions in javascript and the master experiments table. + + Arguments: + group_name: The name of the grouping/experiment. + group_count: How many groups to bucket users into. + username: The username of the user being bucketed. + """ + hasher = hashlib.md5() + hasher.update(group_name.encode('utf-8')) + hasher.update(username.encode('utf-8')) + hash_str = hasher.hexdigest() + + return int(re.sub('[8-9a-f]', '1', re.sub('[0-7]', '0', hash_str)), 2) % group_count diff --git a/openedx/core/lib/celery/task_utils.py b/openedx/core/lib/celery/task_utils.py index 224f03273f..10f45c2cb2 100644 --- a/openedx/core/lib/celery/task_utils.py +++ b/openedx/core/lib/celery/task_utils.py @@ -45,6 +45,7 @@ def emulate_http_request(site=None, user=None, middleware_classes=None): except Exception as exc: for middleware in reversed(middleware_instances): _run_method_if_implemented(middleware, 'process_exception', request, exc) + raise else: for middleware in reversed(middleware_instances): _run_method_if_implemented(middleware, 'process_response', request, response) diff --git a/openedx/features/course_duration_limits/management/__init__.py b/openedx/features/course_duration_limits/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openedx/features/course_duration_limits/management/commands/__init__.py b/openedx/features/course_duration_limits/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openedx/features/course_duration_limits/management/commands/cdl_setup_models_to_send_test_emails.py b/openedx/features/course_duration_limits/management/commands/cdl_setup_models_to_send_test_emails.py new file mode 100644 index 0000000000..d1fa37e882 --- /dev/null +++ b/openedx/features/course_duration_limits/management/commands/cdl_setup_models_to_send_test_emails.py @@ -0,0 +1,55 @@ +""" +A managment command that can be used to set up Schedules with various configurations for testing. +""" + +import datetime +import pytz +import factory + +from django.core.management.base import BaseCommand +from django.contrib.sites.models import Site +from course_modes.models import CourseMode +from course_modes.tests.factories import CourseModeFactory +from openedx.core.djangoapps.content.course_overviews.models import CourseOverview +from openedx.core.djangoapps.schedules.tests.factories import ( + ScheduleFactory, ScheduleConfigFactory, +) +from xmodule.modulestore.tests.factories import CourseFactory, XMODULE_FACTORY_LOCK +from xmodule.modulestore.django import modulestore + + +class CourseDurationLimitExpirySchedule(ScheduleFactory): + """ + A ScheduleFactory that creates a Schedule set up for Course Duration Limit expiry + """ + start = factory.Faker('date_time_between', start_date='-21d', end_date='-21d', tzinfo=pytz.UTC) + + +class Command(BaseCommand): + """ + A management command that generates schedule objects for all expected course duration limit + email types, so that it is easy to generate test emails of all available types. + """ + + def handle(self, *args, **options): + courses = modulestore().get_courses() + + # Find the largest auto-generated course, and pick the next sequence id to generate the next + # course with. + max_org_sequence_id = max([0] + [int(course.org[4:]) for course in courses if course.org.startswith('org.')]) + + XMODULE_FACTORY_LOCK.enable() + CourseFactory.reset_sequence(max_org_sequence_id + 1, force=True) + course = CourseFactory.create( + start=datetime.datetime.today() - datetime.timedelta(days=30), + end=datetime.datetime.today() + datetime.timedelta(days=30), + number=factory.Sequence('schedules_test_course_{}'.format), + display_name=factory.Sequence(u'Schedules Test Course {}'.format), + ) + XMODULE_FACTORY_LOCK.disable() + course_overview = CourseOverview.load_from_module_store(course.id) + CourseModeFactory.create(course_id=course_overview.id, mode_slug=CourseMode.AUDIT) + CourseModeFactory.create(course_id=course_overview.id, mode_slug=CourseMode.VERIFIED) + CourseDurationLimitExpirySchedule.create_batch(20, enrollment__course=course_overview) + + ScheduleConfigFactory.create(site=Site.objects.get(name='example.com')) diff --git a/openedx/features/course_duration_limits/management/commands/send_access_expiry_reminder.py b/openedx/features/course_duration_limits/management/commands/send_access_expiry_reminder.py new file mode 100644 index 0000000000..43ee616b55 --- /dev/null +++ b/openedx/features/course_duration_limits/management/commands/send_access_expiry_reminder.py @@ -0,0 +1,15 @@ +""" +./manage.py lms send_access_expiry_reminder + +Send out reminder emails for any students who will lose access to course content in 7 days. +""" + +from openedx.core.djangoapps.schedules.management.commands import SendEmailBaseCommand +from openedx.features.course_duration_limits.tasks import CourseDurationLimitExpiryReminder +from ... import resolvers + + +class Command(SendEmailBaseCommand): + async_send_task = CourseDurationLimitExpiryReminder + log_prefix = resolvers.EXPIRY_REMINDER_LOG_PREFIX + offsets = (7,) # Days until Course Duration Limit expiry diff --git a/openedx/features/course_duration_limits/message_types.py b/openedx/features/course_duration_limits/message_types.py new file mode 100644 index 0000000000..3e6a77f52d --- /dev/null +++ b/openedx/features/course_duration_limits/message_types.py @@ -0,0 +1,13 @@ +""" +Message types used for ACE communication by the course_duration_limits app. +""" +import logging + +from openedx.core.djangoapps.ace_common.message import BaseMessageType +from openedx.core.djangoapps.schedules.config import DEBUG_MESSAGE_WAFFLE_FLAG + + +class ExpiryReminder(BaseMessageType): + def __init__(self, *args, **kwargs): + super(ExpiryReminder, self).__init__(*args, **kwargs) + self.log_level = logging.DEBUG if DEBUG_MESSAGE_WAFFLE_FLAG.is_enabled() else None diff --git a/openedx/features/course_duration_limits/resolvers.py b/openedx/features/course_duration_limits/resolvers.py new file mode 100644 index 0000000000..c5402e5480 --- /dev/null +++ b/openedx/features/course_duration_limits/resolvers.py @@ -0,0 +1,159 @@ +""" +Resolvers used to find users for course_duration_limit message +""" + +import logging +from datetime import datetime, timedelta + +from course_modes.models import CourseMode +from django.contrib.staticfiles.templatetags.staticfiles import static +from django.db.models import Q +from django.utils.timesince import timeuntil +from django.utils.translation import ugettext as _ +from eventtracking import tracker +from lms.djangoapps.experiments.utils import stable_bucketing_hash_group +from openedx.core.djangoapps.catalog.utils import get_course_run_details +from openedx.core.djangoapps.schedules.resolvers import ( + BinnedSchedulesBaseResolver, + InvalidContextError, + _get_trackable_course_home_url, + _get_upsell_information_for_schedule +) +from track import segment + +from .access import MAX_DURATION, MIN_DURATION, get_user_course_expiration_date +from .models import CourseDurationLimitConfig + +LOG = logging.getLogger(__name__) + +DEFAULT_NUM_BINS = 24 +EXPIRY_REMINDER_NUM_BINS = 24 +EXPIRY_REMINDER_LOG_PREFIX = 'FBE Expiry Reminder' + + +class ExpiryReminderResolver(BinnedSchedulesBaseResolver): + """ + Send a message to all users whose course duration limit expiration date + is at ``self.current_date`` + ``day_offset``. + """ + log_prefix = EXPIRY_REMINDER_LOG_PREFIX + schedule_date_field = 'start' + num_bins = EXPIRY_REMINDER_NUM_BINS + + def __init__( + self, + async_send_task, + site, + course_key, + target_datetime, + day_offset, + bin_num, + override_recipient_email=None + ): + access_duration = MIN_DURATION + discovery_course_details = get_course_run_details(course_key, ['weeks_to_complete']) + expected_weeks = discovery_course_details.get('weeks_to_complete') + if expected_weeks: + access_duration = timedelta(weeks=expected_weeks) + + access_duration = max(MIN_DURATION, min(MAX_DURATION, access_duration)) + + self.course_key = course_key + + super(ExpiryReminderResolver, self).__init__( + async_send_task, + site, + target_datetime - access_duration, + day_offset - access_duration.days, + bin_num, + override_recipient_email, + ) + + # TODO: This isn't named well, given the purpose we're using it for. That's ok for now, + # this is just a test. + @property + def experience_filter(self): + return Q(enrollment__course_id=self.course_key, enrollment__mode=CourseMode.AUDIT) + + def get_template_context(self, user, user_schedules): + course_id_strs = [] + course_links = [] + first_valid_upsell_context = None + first_schedule = None + first_expiration_date = None + + # Experiment code: Skip users who are in the control bucket + hash_bucket = stable_bucketing_hash_group('fbe_access_expiry_reminder', 2, user.username) + properties = { + 'site': self.site.domain, # pylint: disable=no-member + 'app_label': 'course_duration_limits', + 'nonInteraction': 1, + 'bucket': hash_bucket, + 'experiment': 'REVMI-95', + } + course_ids = course_id_strs + properties['num_courses'] = len(course_ids) + if course_ids: + properties['course_ids'] = course_ids[:10] + properties['primary_course_id'] = course_ids[0] + + tracking_context = { + 'host': self.site.domain, # pylint: disable=no-member + 'path': '/', # make up a value, in order to allow the host to be passed along. + } + # I wonder if the user of this event should be the recipient, as they are not the ones + # who took an action. Rather, the system is acting, and they are the object. + # Admittedly that may be what 'nonInteraction' is meant to address. But sessionization may + # get confused by these events if they're attributed in this way, because there's no way for + # this event to get context that would match with what the user might be doing at the moment. + # But the events do show up in GA being joined up with existing sessions (i.e. within a half + # hour in the past), so they don't always break sessions. Not sure what happens after these. + # We can put the recipient_user_id into the properties, and then export as a custom dimension. + with tracker.get_tracker().context('course_duration_limits', tracking_context): + segment.track( + user_id=user.id, + event_name='edx.bi.experiment.user.bucketed', + properties=properties, + ) + if hash_bucket == 0: + raise InvalidContextError() + + for schedule in user_schedules: + upsell_context = _get_upsell_information_for_schedule(user, schedule) + if not upsell_context['show_upsell']: + continue + + if not CourseDurationLimitConfig.enabled_for_enrollment(enrollment=schedule.enrollment): + LOG.info(u"course duration limits not enabled for %s", schedule.enrollment) + continue + + expiration_date = get_user_course_expiration_date(user, schedule.enrollment.course) + if expiration_date is None: + LOG.info(u"No course expiration date for %s", schedule.enrollment.course) + continue + + if first_valid_upsell_context is None: + first_schedule = schedule + first_valid_upsell_context = upsell_context + first_expiration_date = expiration_date + course_id_str = str(schedule.enrollment.course_id) + course_id_strs.append(course_id_str) + course_links.append({ + 'url': _get_trackable_course_home_url(schedule.enrollment.course_id), + 'name': schedule.enrollment.course.display_name + }) + + if first_schedule is None: + self.log_debug('No courses eligible for upgrade for user.') + raise InvalidContextError() + + context = { + 'course_links': course_links, + 'first_course_name': first_schedule.enrollment.course.display_name, + 'cert_image': static('course_experience/images/verified-cert.png'), + 'course_ids': course_id_strs, + 'first_course_expiration_date': first_expiration_date.strftime(_(u"%b. %d, %Y")), + 'time_until_expiration': timeuntil(first_expiration_date.date(), now=datetime.utcnow().date()) + } + context.update(first_valid_upsell_context) + return context diff --git a/openedx/features/course_duration_limits/tasks.py b/openedx/features/course_duration_limits/tasks.py new file mode 100644 index 0000000000..89cedd288d --- /dev/null +++ b/openedx/features/course_duration_limits/tasks.py @@ -0,0 +1,175 @@ +""" +Tasks requiring asynchronous handling for course_duration_limits +""" + +import datetime +import logging + +import waffle +from celery import task +from celery_utils.logged_task import LoggedTask +from django.conf import settings +from django.contrib.auth.models import User +from django.contrib.sites.models import Site +from edx_ace import ace +from edx_ace.message import Message +from edx_ace.utils.date import deserialize, serialize +from edx_django_utils.monitoring import set_custom_metric +from openedx.core.djangoapps.schedules.tasks import ( + _annonate_send_task_for_monitoring, + _track_message_sent +) +from openedx.core.djangoapps.schedules.resolvers import _get_datetime_beginning_of_day +from openedx.core.lib.celery.task_utils import emulate_http_request + +from . import message_types, resolvers +from .models import CourseDurationLimitConfig + +LOG = logging.getLogger(__name__) +ROUTING_KEY = getattr(settings, 'ACE_ROUTING_KEY', None) + + +class CourseDurationLimitMessageBaseTask(LoggedTask): + """ + Base class for top-level Schedule tasks that create subtasks + for each Bin. + """ + ignore_result = True + routing_key = ROUTING_KEY + num_bins = resolvers.DEFAULT_NUM_BINS + enqueue_config_var = None # define in subclass + log_prefix = None + resolver = None # define in subclass + async_send_task = None # define in subclass + + @classmethod + def log_debug(cls, message, *args, **kwargs): + """ + Wrapper around LOG.debug that prefixes the message. + """ + LOG.debug(cls.log_prefix + ': ' + message, *args, **kwargs) + + @classmethod + def log_info(cls, message, *args, **kwargs): + """ + Wrapper around LOG.info that prefixes the message. + """ + LOG.info(cls.log_prefix + ': ' + message, *args, **kwargs) + + @classmethod + def enqueue(cls, site, current_date, day_offset, override_recipient_email=None): + current_date = _get_datetime_beginning_of_day(current_date) + + # for course_key, config in CourseDurationLimitConfig.all_current_course_configs().items(): + for course_key, _ in CourseDurationLimitConfig.all_current_course_configs().items(): + # enqueue_enabled, _ = config[cls.enqueue_config_var] + # TODO: Switch over to a model where enqueing is based in CourseDurationLimitConfig + enqueue_enabled = waffle.switch_is_active('course_duration_limits.enqueue_enabled') + + if not enqueue_enabled: + cls.log_info(u'Message queuing disabled for course_key %s', course_key) + return + + target_date = current_date + datetime.timedelta(days=day_offset) + for bin_num in range(cls.num_bins): + task_args = ( + site.id, + course_key, + serialize(target_date), + day_offset, + bin_num, + override_recipient_email, + ) + cls().apply_async( + task_args, + retry=False, + ) + + def run( # pylint: disable=arguments-differ + self, site_id, course_key, target_day_str, day_offset, bin_num, override_recipient_email=None, + ): + try: + site = Site.objects.select_related('configuration').get(id=site_id) + with emulate_http_request(site=site): + msg_type = self.make_message_type(day_offset) + _annotate_for_monitoring(msg_type, course_key, bin_num, target_day_str, day_offset) + return self.resolver( # pylint: disable=not-callable + self.async_send_task, + site, + course_key, + deserialize(target_day_str), + day_offset, + bin_num, + override_recipient_email=override_recipient_email, + ).send(msg_type) + except Exception: # pylint: disable=broad-except + LOG.exception("Task failed") + + def make_message_type(self, day_offset): + raise NotImplementedError + + +@task(base=LoggedTask, ignore_result=True, routing_key=ROUTING_KEY) +def _expiry_reminder_schedule_send(site_id, msg_str): + _schedule_send( + msg_str, + site_id, + 'deliver_expiry_reminder', + resolvers.EXPIRY_REMINDER_LOG_PREFIX, + ) + + +class CourseDurationLimitExpiryReminder(CourseDurationLimitMessageBaseTask): + """ + Task to send out a reminder that a users access to course content is expiring soon. + """ + num_bins = resolvers.EXPIRY_REMINDER_NUM_BINS + enqueue_config_var = 'enqueue_expiry_reminder' + log_prefix = resolvers.EXPIRY_REMINDER_LOG_PREFIX + resolver = resolvers.ExpiryReminderResolver + async_send_task = _expiry_reminder_schedule_send + + def make_message_type(self, day_offset): + return message_types.ExpiryReminder() + + +def _schedule_send(msg_str, site_id, delivery_config_var, log_prefix): + site = Site.objects.select_related('configuration').get(pk=site_id) + if _is_delivery_enabled(site, delivery_config_var, log_prefix): + msg = Message.from_string(msg_str) + + user = User.objects.get(username=msg.recipient.username) # pylint: disable=no-member + with emulate_http_request(site=site, user=user): + _annonate_send_task_for_monitoring(msg) + LOG.debug(u'%s: Sending message = %s', log_prefix, msg_str) + ace.send(msg) + _track_message_sent(site, user, msg) + + +def _is_delivery_enabled(site, delivery_config_var, log_prefix): # pylint: disable=unused-argument + # Experiment TODO: when this is going to prod, switch over to a config-model backed solution + #if getattr(CourseDurationLimitConfig.current(site=site), delivery_config_var, False): + if waffle.switch_is_active('course_duration_limits.delivery_enabled'): + return True + else: + LOG.info(u'%s: Message delivery disabled for site %s', log_prefix, site.domain) + return False + + +def _annotate_for_monitoring(message_type, course_key, bin_num, target_day_str, day_offset): + """ + Set custom metrics in monitoring to make it easier to identify what messages are being sent and why. + """ + # This identifies the type of message being sent, for example: schedules.recurring_nudge3. + set_custom_metric('message_name', '{0}.{1}'.format(message_type.app_label, message_type.name)) + # The domain name of the site we are sending the message for. + set_custom_metric('course_key', course_key) + # This is the "bin" of data being processed. We divide up the work into chunks so that we don't tie up celery + # workers for too long. This could help us identify particular bins that are problematic. + set_custom_metric('bin', bin_num) + # The date we are processing data for. + set_custom_metric('target_day', target_day_str) + # The number of days relative to the current date to process data for. + set_custom_metric('day_offset', day_offset) + # A unique identifier for this batch of messages being sent. + set_custom_metric('send_uuid', message_type.uuid) diff --git a/openedx/features/course_duration_limits/templates/course_duration_limits/edx_ace/expiryreminder/email/body.html b/openedx/features/course_duration_limits/templates/course_duration_limits/edx_ace/expiryreminder/email/body.html new file mode 100644 index 0000000000..900044b8fa --- /dev/null +++ b/openedx/features/course_duration_limits/templates/course_duration_limits/edx_ace/expiryreminder/email/body.html @@ -0,0 +1,47 @@ +{% extends 'ace_common/edx_ace/common/base_body.html' %} +{% load i18n %} +{% load ace %} + +{% block preview_text %} +{% blocktrans trimmed %} +We hope you have enjoyed {{first_course_name}}! You lose all access to this course in {{time_until_expiration}}. +{% endblocktrans %} +{% endblock %} + +{% block content %} + + + + +
+

+ {% blocktrans trimmed %} + We hope you have enjoyed {{first_course_name}}! You lose all access to this course, including your progress, on + {{ first_course_expiration_date }} ({{time_until_expiration}}). + {% endblocktrans %} +

+

+ {% blocktrans trimmed %} + Upgrade now to get unlimited access and for the chance to earn a verified certificate. + {% endblocktrans %} +

+ + {% if show_upsell %} +

+ + {% trans "Upgrade Now" %} + +

+ {% endif %} +
+{% endblock %} diff --git a/openedx/features/course_duration_limits/templates/course_duration_limits/edx_ace/expiryreminder/email/body.txt b/openedx/features/course_duration_limits/templates/course_duration_limits/edx_ace/expiryreminder/email/body.txt new file mode 100644 index 0000000000..d6839e070a --- /dev/null +++ b/openedx/features/course_duration_limits/templates/course_duration_limits/edx_ace/expiryreminder/email/body.txt @@ -0,0 +1,13 @@ +{% autoescape off %} +{% load i18n %} +{% load ace %} + +{% blocktrans trimmed %} +We hope you have enjoyed {{first_course_name}}! You lose all access to this course, including your progress, on {{ first_course_expiration_date }} ({{time_until_expiration}}). + +Upgrade now to get unlimited access and for the chance to earn a verified certificate. +{% endblocktrans %} +{% if show_upsell %} +{% trans "Upgrade Now" %} <{% with_link_tracking upsell_link %}> +{% endif %} +{% endautoescape %} diff --git a/openedx/features/course_duration_limits/templates/course_duration_limits/edx_ace/expiryreminder/email/from_name.txt b/openedx/features/course_duration_limits/templates/course_duration_limits/edx_ace/expiryreminder/email/from_name.txt new file mode 100644 index 0000000000..039cc99c9a --- /dev/null +++ b/openedx/features/course_duration_limits/templates/course_duration_limits/edx_ace/expiryreminder/email/from_name.txt @@ -0,0 +1,3 @@ +{% autoescape off %} +{{ first_course_name }} +{% endautoescape %} diff --git a/openedx/features/course_duration_limits/templates/course_duration_limits/edx_ace/expiryreminder/email/head.html b/openedx/features/course_duration_limits/templates/course_duration_limits/edx_ace/expiryreminder/email/head.html new file mode 100644 index 0000000000..366ada7ad9 --- /dev/null +++ b/openedx/features/course_duration_limits/templates/course_duration_limits/edx_ace/expiryreminder/email/head.html @@ -0,0 +1 @@ +{% extends 'ace_common/edx_ace/common/base_head.html' %} diff --git a/openedx/features/course_duration_limits/templates/course_duration_limits/edx_ace/expiryreminder/email/subject.txt b/openedx/features/course_duration_limits/templates/course_duration_limits/edx_ace/expiryreminder/email/subject.txt new file mode 100644 index 0000000000..6e3dccbcd2 --- /dev/null +++ b/openedx/features/course_duration_limits/templates/course_duration_limits/edx_ace/expiryreminder/email/subject.txt @@ -0,0 +1,5 @@ +{% autoescape off %} +{% load i18n %} + +{% blocktrans trimmed %}Upgrade to keep your access to {{first_course_name}}{% endblocktrans %} +{% endautoescape %}