diff --git a/openedx/core/djangoapps/schedules/admin.py b/openedx/core/djangoapps/schedules/admin.py index e6cb1bfc12..b16e8555f9 100644 --- a/openedx/core/djangoapps/schedules/admin.py +++ b/openedx/core/djangoapps/schedules/admin.py @@ -1,4 +1,5 @@ from django.contrib import admin +from django import forms from django.utils.translation import ugettext_lazy as _ from . import models @@ -32,6 +33,15 @@ class ScheduleAdmin(admin.ModelAdmin): return qs +class ScheduleConfigAdminForm(forms.ModelForm): + + def clean_hold_back_ratio(self): + hold_back_ratio = self.cleaned_data["hold_back_ratio"] + if hold_back_ratio < 0 or hold_back_ratio > 1: + raise forms.ValidationError("Invalid hold back ratio, the value must be between 0 and 1.") + return hold_back_ratio + + @admin.register(models.ScheduleConfig) class ScheduleConfigAdmin(admin.ModelAdmin): search_fields = ('site',) @@ -40,4 +50,6 @@ class ScheduleConfigAdmin(admin.ModelAdmin): 'enqueue_recurring_nudge', 'deliver_recurring_nudge', 'enqueue_upgrade_reminder', 'deliver_upgrade_reminder', 'enqueue_course_update', 'deliver_course_update', + 'hold_back_ratio', ) + form = ScheduleConfigAdminForm diff --git a/openedx/core/djangoapps/schedules/migrations/0007_scheduleconfig_hold_back_ratio.py b/openedx/core/djangoapps/schedules/migrations/0007_scheduleconfig_hold_back_ratio.py new file mode 100644 index 0000000000..41ee03d634 --- /dev/null +++ b/openedx/core/djangoapps/schedules/migrations/0007_scheduleconfig_hold_back_ratio.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('schedules', '0006_scheduleexperience'), + ] + + operations = [ + migrations.AddField( + model_name='scheduleconfig', + name='hold_back_ratio', + field=models.FloatField(default=0), + ), + ] diff --git a/openedx/core/djangoapps/schedules/models.py b/openedx/core/djangoapps/schedules/models.py index b60ba955fc..2ca8c5f422 100644 --- a/openedx/core/djangoapps/schedules/models.py +++ b/openedx/core/djangoapps/schedules/models.py @@ -46,6 +46,7 @@ class ScheduleConfig(ConfigurationModel): deliver_upgrade_reminder = models.BooleanField(default=False) enqueue_course_update = models.BooleanField(default=False) deliver_course_update = models.BooleanField(default=False) + hold_back_ratio = models.FloatField(default=0) class ScheduleExperience(models.Model): diff --git a/openedx/core/djangoapps/schedules/signals.py b/openedx/core/djangoapps/schedules/signals.py index 7ba0985606..d94966a4b4 100644 --- a/openedx/core/djangoapps/schedules/signals.py +++ b/openedx/core/djangoapps/schedules/signals.py @@ -1,6 +1,8 @@ import datetime import logging +import random +import analytics from django.db.models.signals import post_save from django.dispatch import receiver from django.utils import timezone @@ -55,17 +57,26 @@ def create_schedule(sender, **kwargs): upgrade_deadline = _calculate_upgrade_deadline(enrollment.course_id, content_availability_date) + if course_has_highlights(enrollment.course_id): + experience_type = ScheduleExperience.EXPERIENCES.course_updates + else: + experience_type = ScheduleExperience.EXPERIENCES.default + + if _should_randomly_suppress_schedule_creation( + schedule_config, + enrollment, + upgrade_deadline, + experience_type, + content_availability_date, + ): + return + schedule = Schedule.objects.create( enrollment=enrollment, start=content_availability_date, upgrade_deadline=upgrade_deadline ) - if course_has_highlights(enrollment.course_id): - experience_type = ScheduleExperience.EXPERIENCES.course_updates - else: - experience_type = ScheduleExperience.EXPERIENCES.default - ScheduleExperience(schedule=schedule, experience_type=experience_type).save() log.debug('Schedules: created a new schedule starting at %s with an upgrade deadline of %s and experience type: %s', @@ -138,3 +149,36 @@ def _get_upgrade_deadline_delta_setting(course_id): delta = None return delta + + +def _should_randomly_suppress_schedule_creation( + schedule_config, + enrollment, + upgrade_deadline, + experience_type, + content_availability_date, +): + # The hold back ratio is always between 0 and 1. A value of 0 indicates that schedules should be created for all + # schedules. A value of 1 indicates that no schedules should be created for any enrollments. A value of 0.2 would + # mean that 20% of enrollments should *not* be given schedules. + + # This allows us to measure the impact of the dynamic schedule experience by comparing this "control" group that + # does not receive any of benefits of the feature against the group that does. + if random.random() < schedule_config.hold_back_ratio: + log.debug('Schedules: Enrollment held back from dynamic schedule experiences.') + upgrade_deadline_str = None + if upgrade_deadline: + upgrade_deadline_str = upgrade_deadline.isoformat() + analytics.track( + 'edx.bi.schedule.suppressed', + { + 'user_id': enrollment.user.id, + 'course_id': unicode(enrollment.course_id), + 'experience_type': experience_type, + 'upgrade_deadline': upgrade_deadline_str, + 'content_availability_date': content_availability_date.isoformat(), + } + ) + return True + + return False diff --git a/openedx/core/djangoapps/schedules/tests/factories.py b/openedx/core/djangoapps/schedules/tests/factories.py index 4b54c712f2..981da04a8a 100644 --- a/openedx/core/djangoapps/schedules/tests/factories.py +++ b/openedx/core/djangoapps/schedules/tests/factories.py @@ -35,3 +35,4 @@ class ScheduleConfigFactory(factory.DjangoModelFactory): deliver_upgrade_reminder = True enqueue_course_update = True deliver_course_update = True + hold_back_ratio = 0 diff --git a/openedx/core/djangoapps/schedules/tests/test_signals.py b/openedx/core/djangoapps/schedules/tests/test_signals.py index 20d45d6e39..e4eb10579a 100644 --- a/openedx/core/djangoapps/schedules/tests/test_signals.py +++ b/openedx/core/djangoapps/schedules/tests/test_signals.py @@ -20,6 +20,7 @@ from ..models import Schedule from ..tests.factories import ScheduleConfigFactory +@ddt.ddt @patch('openedx.core.djangoapps.schedules.signals.get_current_site') @skip_unless_lms class CreateScheduleTests(SharedModuleStoreTestCase): @@ -94,6 +95,33 @@ class CreateScheduleTests(SharedModuleStoreTestCase): mock_get_current_site.return_value = site self.assert_schedule_created(experience_type=ScheduleExperience.EXPERIENCES.course_updates) + @override_waffle_flag(CREATE_SCHEDULE_WAFFLE_FLAG, True) + @patch('analytics.track') + @patch('random.random') + @ddt.data( + (0, True), + (0.1, True), + (0.3, False), + ) + @ddt.unpack + def test_create_schedule_hold_backs( + self, + hold_back_ratio, + expect_schedule_created, + mock_random, + mock_track, + mock_get_current_site + ): + mock_random.return_value = 0.2 + schedule_config = ScheduleConfigFactory.create(enabled=True, hold_back_ratio=hold_back_ratio) + mock_get_current_site.return_value = schedule_config.site + if expect_schedule_created: + self.assert_schedule_created() + self.assertFalse(mock_track.called) + else: + self.assert_schedule_not_created() + mock_track.assert_called_once() + @ddt.ddt @skip_unless_lms