Remove the use of celery.group from bulk email subtasks.

lms/djangoapps/instructor_task/subtasks.py

@@ -3,6 +3,8 @@ This module contains celery task functions for handling the management of subtas
 """
 from time import time
 import json
+from uuid import uuid4
+import math
 
 from celery.utils.log import get_task_logger
 from celery.states import SUCCESS, READY_STATES, RETRY
@@ -23,6 +25,68 @@ class DuplicateTaskException(Exception):
     pass
 
 
+def create_subtask_ids(total_num_items, items_per_query, items_per_task):
+    """
+    Determines number of subtasks that need to be generated, and provides a list of id values to use.
+
+    This needs to be calculated before a query is executed so that the list of all subtasks can be
+    stored in the InstructorTask before any subtasks are started.
+
+    The number of subtask_id values returned by this should match the number of chunks returned
+    by the generate_items_for_subtask generator.
+    """
+    total_num_tasks = 0
+    num_queries = int(math.ceil(float(total_num_items) / float(items_per_query)))
+    num_items_remaining = total_num_items
+    for _ in range(num_queries):
+        num_items_this_query = min(num_items_remaining, items_per_query)
+        num_items_remaining -= num_items_this_query
+        num_tasks_this_query = int(math.ceil(float(num_items_this_query) / float(items_per_task)))
+        total_num_tasks += num_tasks_this_query
+
+    # Now that the number of tasks is known, return a list of ids for each task.
+    return [str(uuid4()) for _ in range(total_num_tasks)]
+
+
+def generate_items_for_subtask(item_queryset, item_fields, total_num_items, items_per_query, items_per_task):
+    """
+    Generates a chunk of "items" that should be passed into a subtask.
+
+    Arguments:
+        `item_queryset` : a query set that defines the "items" that should be passed to subtasks.
+        `item_fields` : the fields that should be included in the dict that is returned.
+            These are in addition to the 'pk' field.
+        `total_num_items` : the result of item_queryset.count().
+        `items_per_query` : size of chunks to break the query operation into.
+        `items_per_task` : maximum size of chunks to break each query chunk into for use by a subtask.
+
+    Returns:  yields a list of dicts, where each dict contains the fields in `item_fields`, plus the 'pk' field.
+
+    """
+    num_queries = int(math.ceil(float(total_num_items) / float(items_per_query)))
+    last_pk = item_queryset[0].pk - 1
+    num_items_queued = 0
+    all_item_fields = list(item_fields)
+    all_item_fields.append('pk')
+    for _ in range(num_queries):
+        item_sublist = list(item_queryset.order_by('pk').filter(pk__gt=last_pk).values(*all_item_fields)[:items_per_query])
+        last_pk = item_sublist[-1]['pk']
+        num_items_this_query = len(item_sublist)
+        num_tasks_this_query = int(math.ceil(float(num_items_this_query) / float(items_per_task)))
+        chunk = int(math.ceil(float(num_items_this_query) / float(num_tasks_this_query)))
+        for i in range(num_tasks_this_query):
+            items_for_task = item_sublist[i * chunk:i * chunk + chunk]
+            yield items_for_task
+
+        num_items_queued += num_items_this_query
+
+    # Sanity check: we expect the chunking to be properly summing to the original count:
+    if num_items_queued != total_num_items:
+        error_msg = "Task {}: number of items generated by chunking {} not equal to original total {}".format(num_items_queued, total_num_items)
+        TASK_LOG.error(error_msg)
+        raise ValueError(error_msg)
+
+
 def create_subtask_status(task_id, succeeded=0, failed=0, skipped=0, retried_nomax=0, retried_withmax=0, state=None):
     """
     Create and return a dict for tracking the status of a subtask.