Merge pull request #16441 from edx/mulby/hold-back-some-users

hold back some users from dynamic pacing features

openedx/core/djangoapps/schedules/admin.py

@@ -1,4 +1,5 @@
 from django.contrib import admin
+from django import forms
 from django.utils.translation import ugettext_lazy as _
 
 from . import models
@@ -32,6 +33,15 @@ class ScheduleAdmin(admin.ModelAdmin):
         return qs
 
 
+class ScheduleConfigAdminForm(forms.ModelForm):
+
+    def clean_hold_back_ratio(self):
+        hold_back_ratio = self.cleaned_data["hold_back_ratio"]
+        if hold_back_ratio < 0 or hold_back_ratio > 1:
+            raise forms.ValidationError("Invalid hold back ratio, the value must be between 0 and 1.")
+        return hold_back_ratio
+
+
 @admin.register(models.ScheduleConfig)
 class ScheduleConfigAdmin(admin.ModelAdmin):
     search_fields = ('site',)
@@ -40,4 +50,6 @@ class ScheduleConfigAdmin(admin.ModelAdmin):
         'enqueue_recurring_nudge', 'deliver_recurring_nudge',
         'enqueue_upgrade_reminder', 'deliver_upgrade_reminder',
         'enqueue_course_update', 'deliver_course_update',
+        'hold_back_ratio',
     )
+    form = ScheduleConfigAdminForm

openedx/core/djangoapps/schedules/migrations/0007_scheduleconfig_hold_back_ratio.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('schedules', '0006_scheduleexperience'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='scheduleconfig',
+            name='hold_back_ratio',
+            field=models.FloatField(default=0),
+        ),
+    ]

openedx/core/djangoapps/schedules/models.py

@@ -46,6 +46,7 @@ class ScheduleConfig(ConfigurationModel):
     deliver_upgrade_reminder = models.BooleanField(default=False)
     enqueue_course_update = models.BooleanField(default=False)
     deliver_course_update = models.BooleanField(default=False)
+    hold_back_ratio = models.FloatField(default=0)
 
 
 class ScheduleExperience(models.Model):

openedx/core/djangoapps/schedules/signals.py

@@ -1,6 +1,8 @@
 import datetime
 import logging
+import random
 
+import analytics
 from django.db.models.signals import post_save
 from django.dispatch import receiver
 from django.utils import timezone
@@ -55,17 +57,26 @@ def create_schedule(sender, **kwargs):
 
     upgrade_deadline = _calculate_upgrade_deadline(enrollment.course_id, content_availability_date)
 
+    if course_has_highlights(enrollment.course_id):
+        experience_type = ScheduleExperience.EXPERIENCES.course_updates
+    else:
+        experience_type = ScheduleExperience.EXPERIENCES.default
+
+    if _should_randomly_suppress_schedule_creation(
+        schedule_config,
+        enrollment,
+        upgrade_deadline,
+        experience_type,
+        content_availability_date,
+    ):
+        return
+
     schedule = Schedule.objects.create(
         enrollment=enrollment,
         start=content_availability_date,
         upgrade_deadline=upgrade_deadline
     )
 
-    if course_has_highlights(enrollment.course_id):
-        experience_type = ScheduleExperience.EXPERIENCES.course_updates
-    else:
-        experience_type = ScheduleExperience.EXPERIENCES.default
-
     ScheduleExperience(schedule=schedule, experience_type=experience_type).save()
 
     log.debug('Schedules: created a new schedule starting at %s with an upgrade deadline of %s and experience type: %s',
@@ -138,3 +149,36 @@ def _get_upgrade_deadline_delta_setting(course_id):
         delta = None
 
     return delta
+
+
+def _should_randomly_suppress_schedule_creation(
+    schedule_config,
+    enrollment,
+    upgrade_deadline,
+    experience_type,
+    content_availability_date,
+):
+    # The hold back ratio is always between 0 and 1. A value of 0 indicates that schedules should be created for all
+    # schedules. A value of 1 indicates that no schedules should be created for any enrollments. A value of 0.2 would
+    # mean that 20% of enrollments should *not* be given schedules.
+
+    # This allows us to measure the impact of the dynamic schedule experience by comparing this "control" group that
+    # does not receive any of benefits of the feature against the group that does.
+    if random.random() < schedule_config.hold_back_ratio:
+        log.debug('Schedules: Enrollment held back from dynamic schedule experiences.')
+        upgrade_deadline_str = None
+        if upgrade_deadline:
+            upgrade_deadline_str = upgrade_deadline.isoformat()
+        analytics.track(
+            'edx.bi.schedule.suppressed',
+            {
+                'user_id': enrollment.user.id,
+                'course_id': unicode(enrollment.course_id),
+                'experience_type': experience_type,
+                'upgrade_deadline': upgrade_deadline_str,
+                'content_availability_date': content_availability_date.isoformat(),
+            }
+        )
+        return True
+
+    return False

openedx/core/djangoapps/schedules/tests/factories.py

@@ -35,3 +35,4 @@ class ScheduleConfigFactory(factory.DjangoModelFactory):
     deliver_upgrade_reminder = True
     enqueue_course_update = True
     deliver_course_update = True
+    hold_back_ratio = 0

openedx/core/djangoapps/schedules/tests/test_signals.py

@@ -20,6 +20,7 @@ from ..models import Schedule
 from ..tests.factories import ScheduleConfigFactory
 
 
+@ddt.ddt
 @patch('openedx.core.djangoapps.schedules.signals.get_current_site')
 @skip_unless_lms
 class CreateScheduleTests(SharedModuleStoreTestCase):
@@ -94,6 +95,33 @@ class CreateScheduleTests(SharedModuleStoreTestCase):
         mock_get_current_site.return_value = site
         self.assert_schedule_created(experience_type=ScheduleExperience.EXPERIENCES.course_updates)
 
+    @override_waffle_flag(CREATE_SCHEDULE_WAFFLE_FLAG, True)
+    @patch('analytics.track')
+    @patch('random.random')
+    @ddt.data(
+        (0, True),
+        (0.1, True),
+        (0.3, False),
+    )
+    @ddt.unpack
+    def test_create_schedule_hold_backs(
+        self,
+        hold_back_ratio,
+        expect_schedule_created,
+        mock_random,
+        mock_track,
+        mock_get_current_site
+    ):
+        mock_random.return_value = 0.2
+        schedule_config = ScheduleConfigFactory.create(enabled=True, hold_back_ratio=hold_back_ratio)
+        mock_get_current_site.return_value = schedule_config.site
+        if expect_schedule_created:
+            self.assert_schedule_created()
+            self.assertFalse(mock_track.called)
+        else:
+            self.assert_schedule_not_created()
+            mock_track.assert_called_once()
+
 
 @ddt.ddt
 @skip_unless_lms