Merge remote-tracking branch 'origin/release' into adam/merge-release-into-master

Conflicts:
	common/djangoapps/student/models.py
	lms/djangoapps/bulk_email/tasks.py

lms/djangoapps/instructor_task/subtasks.py

@@ -71,7 +71,7 @@ def track_memory_usage(metric, course_id):
 
 
 def _generate_items_for_subtask(
-    item_queryset,
+    item_querysets,
     item_fields,
     total_num_items,
     items_per_task,
@@ -82,10 +82,10 @@ def _generate_items_for_subtask(
     Generates a chunk of "items" that should be passed into a subtask.
 
     Arguments:
-        `item_queryset` : a query set that defines the "items" that should be passed to subtasks.
+        `item_querysets` : a list of query sets, each of which defines the "items" that should be passed to subtasks.
         `item_fields` : the fields that should be included in the dict that is returned.
             These are in addition to the 'pk' field.
-        `total_num_items` : the result of item_queryset.count().
+        `total_num_items` : the result of summing the count of each queryset in `item_querysets`.
         `items_per_query` : size of chunks to break the query operation into.
         `items_per_task` : maximum size of chunks to break each query chunk into for use by a subtask.
         `course_id` : course_id of the course. Only needed for the track_memory_usage context manager.
@@ -102,13 +102,14 @@ def _generate_items_for_subtask(
     items_for_task = []
 
     with track_memory_usage('course_email.subtask_generation.memory', course_id):
-        for item in item_queryset.values(*all_item_fields).iterator():
-            if len(items_for_task) == items_per_task and num_subtasks < total_num_subtasks - 1:
-                yield items_for_task
-                num_items_queued += items_per_task
-                items_for_task = []
-                num_subtasks += 1
-            items_for_task.append(item)
+        for queryset in item_querysets:
+            for item in queryset.values(*all_item_fields).iterator():
+                if len(items_for_task) == items_per_task and num_subtasks < total_num_subtasks - 1:
+                    yield items_for_task
+                    num_items_queued += items_per_task
+                    items_for_task = []
+                    num_subtasks += 1
+                items_for_task.append(item)
 
         # yield remainder items for task, if any
         if items_for_task:
@@ -275,7 +276,7 @@ def initialize_subtask_info(entry, action_name, total_num, subtask_id_list):
     return task_progress
 
 
-def queue_subtasks_for_query(entry, action_name, create_subtask_fcn, item_queryset, item_fields, items_per_task):
+def queue_subtasks_for_query(entry, action_name, create_subtask_fcn, item_querysets, item_fields, items_per_task):
     """
     Generates and queues subtasks to each execute a chunk of "items" generated by a queryset.
 
@@ -285,7 +286,7 @@ def queue_subtasks_for_query(entry, action_name, create_subtask_fcn, item_querys
         `create_subtask_fcn` : a function of two arguments that constructs the desired kind of subtask object.
             Arguments are the list of items to be processed by this subtask, and a SubtaskStatus
             object reflecting initial status (and containing the subtask's id).
-        `item_queryset` : a query set that defines the "items" that should be passed to subtasks.
+        `item_querysets` : a list of query sets that define the "items" that should be passed to subtasks.
         `item_fields` : the fields that should be included in the dict that is returned.
             These are in addition to the 'pk' field.
         `items_per_task` : maximum size of chunks to break each query chunk into for use by a subtask.
@@ -294,7 +295,7 @@ def queue_subtasks_for_query(entry, action_name, create_subtask_fcn, item_querys
 
     """
     task_id = entry.task_id
-    total_num_items = item_queryset.count()
+    total_num_items = sum([item_queryset.count() for item_queryset in item_querysets])
 
     # Calculate the number of tasks that will be created, and create a list of ids for each task.
     total_num_subtasks = _get_number_of_subtasks(total_num_items, items_per_task)
@@ -313,7 +314,7 @@ def queue_subtasks_for_query(entry, action_name, create_subtask_fcn, item_querys
     # Construct a generator that will return the recipients to use for each subtask.
     # Pass in the desired fields to fetch for each recipient.
     item_list_generator = _generate_items_for_subtask(
-        item_queryset,
+        item_querysets,
         item_fields,
         total_num_items,
         items_per_task,

lms/djangoapps/instructor_task/tests/test_subtasks.py

@@ -38,7 +38,7 @@ class TestSubtasks(InstructorTaskCourseTestCase):
         )
 
         self._enroll_students_in_course(self.course.id, initial_count)
-        task_queryset = CourseEnrollment.objects.filter(course_id=self.course.id)
+        task_querysets = [CourseEnrollment.objects.filter(course_id=self.course.id)]
 
         def initialize_subtask_info(*args):  # pylint: disable=unused-argument
             """Instead of initializing subtask info enroll some more students into course."""
@@ -51,7 +51,7 @@ class TestSubtasks(InstructorTaskCourseTestCase):
                 entry=instructor_task,
                 action_name='action_name',
                 create_subtask_fcn=create_subtask_fcn,
-                item_queryset=task_queryset,
+                item_querysets=task_querysets,
                 item_fields=[],
                 items_per_task=items_per_task,
             )