Add new instructor task for weighted problems

lms/djangoapps/instructor_task/api.py

@@ -19,6 +19,7 @@ from instructor_task.tasks import (
     delete_problem_state,
     send_bulk_course_email,
     calculate_grades_csv,
+    calculate_problem_grade_report,
     calculate_students_features_csv,
     cohort_students,
 )
@@ -334,6 +335,18 @@ def submit_calculate_grades_csv(request, course_key):
     return submit_task(request, task_type, task_class, course_key, task_input, task_key)
 
 
+def submit_problem_grade_report(request, course_key):
+    """
+    Submits a task to generate a CSV grade report containing weighted problem
+    values.
+    """
+    task_type = 'grade_problems'
+    task_class = calculate_problem_grade_report
+    task_input = {}
+    task_key = ""
+    return submit_task(request, task_type, task_class, course_key, task_input, task_key)
+
+
 def submit_calculate_students_features_csv(request, course_key, features):
     """
     Submits a task to generate a CSV containing student profile info.

lms/djangoapps/instructor_task/tasks.py

@@ -35,6 +35,7 @@ from instructor_task.tasks_helper import (
     reset_attempts_module_state,
     delete_problem_module_state,
     upload_grades_csv,
+    upload_problem_grade_report,
     upload_students_csv,
     cohort_students_and_upload
 )
@@ -155,6 +156,25 @@ def calculate_grades_csv(entry_id, xmodule_instance_args):
     return run_main_task(entry_id, task_fn, action_name)
 
 
+# TODO: GRADES_DOWNLOAD_ROUTING_KEY is the high mem queue.  Do we know we need it?
+@task(base=BaseInstructorTask, routing_key=settings.GRADES_DOWNLOAD_ROUTING_KEY)  # pylint: disable=not-callable
+def calculate_problem_grade_report(entry_id, xmodule_instance_args):
+    """
+    Generate a CSV for a course containing all students' weighted problem
+    grades and push the results to an S3 bucket for download.
+    """
+    # Translators: This is a past-tense verb that is inserted into task progress messages as {action}.
+    # TODO: can this be the same as the `calculate_grades_csv` action_name?
+    action_name = ugettext_noop('graded')
+    TASK_LOG.info(
+        u"Task: %s, InstructorTask ID: %s, Task type: %s, Preparing for task execution",
+        xmodule_instance_args.get('task_id'), entry_id, action_name
+    )
+
+    task_fn = partial(upload_problem_grade_report, xmodule_instance_args)
+    return run_main_task(entry_id, task_fn, action_name)
+
+
 @task(base=BaseInstructorTask, routing_key=settings.GRADES_DOWNLOAD_ROUTING_KEY)  # pylint: disable=not-callable
 def calculate_students_features_csv(entry_id, xmodule_instance_args):
     """

lms/djangoapps/instructor_task/tasks_helper.py

@@ -4,7 +4,10 @@ running state of a course.
 
 """
 import json
+from collections import OrderedDict
 from datetime import datetime
+from eventtracking import tracker
+from itertools import chain
 from time import time
 import unicodecsv
 import logging
@@ -34,6 +37,7 @@ from instructor_task.models import ReportStore, InstructorTask, PROGRESS
 from lms.djangoapps.lms_xblock.runtime import LmsPartitionService
 from openedx.core.djangoapps.course_groups.cohorts import get_cohort
 from openedx.core.djangoapps.course_groups.models import CourseUserGroup
+from openedx.core.djangoapps.content.course_structures.models import CourseStructure
 from opaque_keys.edx.keys import UsageKey
 from openedx.core.djangoapps.course_groups.cohorts import add_user_to_cohort, is_course_cohorted
 from student.models import CourseEnrollment
@@ -705,6 +709,105 @@ def upload_grades_csv(_xmodule_instance_args, _entry_id, course_id, _task_input,
     return task_progress.update_task_state(extra_meta=current_step)
 
 
+def _order_problems(blocks):
+    """
+    Sort the problems by the assignment type and assignment that it belongs to.
+    """
+    problems = OrderedDict()
+    assignments = dict()
+    # First, sort out all the blocks into their correct assignments and all the
+    # assignments into their correct types.
+    for block in blocks:
+        # Put the assignments in order into the assignments list.
+        if blocks[block]['block_type'] == 'sequential':
+            block_format = blocks[block]['format']
+            if block_format not in assignments:
+                assignments[block_format] = OrderedDict()
+            assignments[block_format][block] = list()
+
+        # Put the problems into the correct order within their assignment.
+        if blocks[block]['block_type'] == 'problem' and blocks[block]['graded'] is True:
+            current = blocks[block]['parent']
+            # crawl up the tree for the sequential block
+            while blocks[current]['block_type'] != 'sequential':
+                current = blocks[current]['parent']
+
+            current_format = blocks[current]['format']
+            assignments[current_format][current].append(block)
+
+    # Now that we have a sorting and an order for the assignments and problems,
+    # iterate through them in order to generate the header row.
+    for assignment_type in assignments:
+        for assignment_index, assignment in enumerate(assignments[assignment_type].keys(), start=1):
+            for problem in assignments[assignment_type][assignment]:
+                header_name = "{assignment_type} {assignment_index}: {assignment_name} - {block}".format(
+                    block=blocks[problem]['display_name'],
+                    assignment_type=assignment_type,
+                    assignment_index=assignment_index,
+                    assignment_name=blocks[assignment]['display_name']
+                )
+                problems[problem] = [header_name + " (Earned)", header_name + " (Possible)"]
+
+    return problems
+
+
+def upload_problem_grade_report(_xmodule_instance_args, _entry_id, course_id, _task_input, action_name):
+    """
+    Generate a CSV containing all students' problem grades within a given
+    `course_id`.
+    """
+    start_time = time()
+    start_date = datetime.now(UTC)
+    status_interval = 100
+    enrolled_students = CourseEnrollment.users_enrolled_in(course_id)
+    task_progress = TaskProgress(action_name, enrolled_students.count(), start_time)
+
+    # This struct encapsulates both the display names of each static item in
+    # the header row as values as well as the django User field names of those
+    # items as the keys.  It is structured in this way to keep the values
+    # related.
+    header_row = OrderedDict([('id', 'Student ID'), ('email', 'Email'), ('username', 'Username')])
+
+    try:
+        course_structure = CourseStructure.objects.get(course_id=course_id)
+        blocks = course_structure.ordered_blocks
+        problems = _order_problems(blocks)
+    except CourseStructure.DoesNotExist:
+        return task_progress.update_task_state(extra_meta={'step': 'Generating course structure. Please refresh and try again.'})
+
+    # Just generate the static fields for now.
+    rows = [list(header_row.values()) + ['Final Grade'] + list(chain.from_iterable(problems.values()))]
+    current_step = {'step': 'Calculating Grades'}
+
+    for student, gradeset, err_msg in iterate_grades_for(course_id, enrolled_students, keep_raw_scores=True):
+        student_fields = [getattr(student, field_name) for field_name in header_row]
+        final_grade = gradeset['percent']
+        # Only consider graded problems
+        problem_scores = {unicode(score.module_id): score for score in gradeset['raw_scores'] if score.graded}
+        earned_possible_values = list()
+        for problem_id in problems:
+            try:
+                problem_score = problem_scores[problem_id]
+                earned_possible_values.append([problem_score.earned, problem_score.possible])
+            except KeyError:
+                # The student has not been graded on this problem.  For example,
+                # iterate_grades_for skips problems that students have never
+                # seen in order to speed up report generation.  It could also be
+                # the case that the student does not have access to it (e.g. A/B
+                # test or cohorted courseware).
+                earned_possible_values.append(['N/A', 'N/A'])
+        rows.append(student_fields + [final_grade] + list(chain.from_iterable(earned_possible_values)))
+
+        task_progress.attempted += 1
+        task_progress.succeeded += 1
+        if task_progress.attempted % status_interval == 0:
+            task_progress.update_task_state(extra_meta=current_step)
+
+    # Perform the upload
+    upload_csv_to_report_store(rows, 'problem_grade_report', course_id, start_date)
+    return task_progress.update_task_state(extra_meta={'step': 'Uploading CSV'})
+
+
 def upload_students_csv(_xmodule_instance_args, _entry_id, course_id, task_input, action_name):
     """
     For a given `course_id`, generate a CSV file containing profile

lms/djangoapps/instructor_task/tests/test_base.py

@@ -127,7 +127,9 @@ class InstructorTaskCourseTestCase(LoginEnrollmentTestCase, ModuleStoreTestCase)
         if course_factory_kwargs is not None:
             course_args.update(course_factory_kwargs)
         self.course = CourseFactory.create(**course_args)
+        self.add_course_content()
 
+    def add_course_content(self):
         # Add a chapter to the course
         chapter = ItemFactory.create(parent_location=self.course.location,
                                      display_name=TEST_SECTION_NAME)
@@ -141,12 +143,13 @@ class InstructorTaskCourseTestCase(LoginEnrollmentTestCase, ModuleStoreTestCase)
     @staticmethod
     def get_user_email(username):
         """Generate email address based on username"""
-        return '{0}@test.com'.format(username)
+        return u'{0}@test.com'.format(username)
 
     def login_username(self, username):
         """Login the user, given the `username`."""
         if self.current_user != username:
-            self.login(InstructorTaskCourseTestCase.get_user_email(username), "test")
+            user_email = User.objects.get(username=username).email
+            self.login(user_email, "test")
             self.current_user = username
 
     def _create_user(self, username, email=None, is_staff=False, mode='honor'):
@@ -190,16 +193,18 @@ class InstructorTaskModuleTestCase(InstructorTaskCourseTestCase):
     the setup of a course and problem in order to access StudentModule state.
     """
     @staticmethod
-    def problem_location(problem_url_name):
+    def problem_location(problem_url_name, course_key=None):
         """
         Create an internal location for a test problem.
         """
         if "i4x:" in problem_url_name:
             return Location.from_deprecated_string(problem_url_name)
+        elif course_key:
+            return course_key.make_usage_key('problem', problem_url_name)
         else:
             return TEST_COURSE_KEY.make_usage_key('problem', problem_url_name)
 
-    def define_option_problem(self, problem_url_name, parent=None):
+    def define_option_problem(self, problem_url_name, parent=None, **kwargs):
         """Create the problem definition so the answer is Option 1"""
         if parent is None:
             parent = self.problem_section
@@ -213,7 +218,8 @@ class InstructorTaskModuleTestCase(InstructorTaskCourseTestCase):
                            parent=parent,
                            category="problem",
                            display_name=str(problem_url_name),
-                           data=problem_xml)
+                           data=problem_xml,
+                           **kwargs)
 
     def redefine_option_problem(self, problem_url_name):
         """Change the problem definition so the answer is Option 2"""
@@ -249,8 +255,9 @@ class InstructorTaskModuleTestCase(InstructorTaskCourseTestCase):
             # Note that this is a capa-specific convention.  The form is a version of the problem's
             # URL, modified so that it can be easily stored in html, prepended with "input-" and
             # appended with a sequence identifier for the particular response the input goes to.
-            return 'input_i4x-{0}-{1}-problem-{2}_{3}'.format(TEST_COURSE_ORG.lower(),
-                                                              TEST_COURSE_NUMBER.replace('.', '_'),
+            course_key = self.course.id
+            return 'input_i4x-{0}-{1}-problem-{2}_{3}'.format(course_key.org,
+                                                              course_key.course.replace('.', '_'),
                                                               problem_url_name, response_id)
 
         # make sure that the requested user is logged in, so that the ajax call works
@@ -260,7 +267,7 @@ class InstructorTaskModuleTestCase(InstructorTaskCourseTestCase):
         modx_url = reverse('xblock_handler', kwargs={
             'course_id': self.course.id.to_deprecated_string(),
             'usage_id': quote_slashes(
-                InstructorTaskModuleTestCase.problem_location(problem_url_name).to_deprecated_string()
+                InstructorTaskModuleTestCase.problem_location(problem_url_name, self.course.id).to_deprecated_string()
             ),
             'handler': 'xmodule_handler',
             'suffix': 'problem_check',

lms/djangoapps/instructor_task/tests/test_integration.py

@@ -28,7 +28,7 @@ from instructor_task.api import (submit_rescore_problem_for_all_students,
                                  submit_reset_problem_attempts_for_all_students,
                                  submit_delete_problem_state_for_all_students)
 from instructor_task.models import InstructorTask
-from instructor_task.tasks_helper import upload_grades_csv
+from instructor_task.tasks_helper import upload_grades_csv, upload_problem_grade_report
 from instructor_task.tests.test_base import (InstructorTaskModuleTestCase, TestReportMixin, TEST_COURSE_ORG,
                                              TEST_COURSE_NUMBER, OPTION_1, OPTION_2)
 from capa.responsetypes import StudentInputError

lms/djangoapps/instructor_task/tests/test_tasks_helper.py

@@ -19,6 +19,7 @@ from instructor_task.tasks_helper import cohort_students_and_upload, upload_grad
 from instructor_task.tests.test_base import InstructorTaskCourseTestCase, TestReportMixin, InstructorTaskModuleTestCase
 from openedx.core.djangoapps.course_groups.models import CourseUserGroupPartitionGroup
 from openedx.core.djangoapps.course_groups.tests.helpers import CohortFactory
+from openedx.core.djangoapps.user_api.tests.factories import UserCourseTagFactory
 import openedx.core.djangoapps.user_api.course_tag.api as course_tag_api
 from openedx.core.djangoapps.user_api.partition_schemes import RandomUserPartitionScheme
 from student.tests.factories import UserFactory
@@ -26,6 +27,13 @@ from student.models import CourseEnrollment
 from verify_student.tests.factories import SoftwareSecurePhotoVerificationFactory
 from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
 from xmodule.partitions.partitions import Group, UserPartition
+from instructor_task.models import ReportStore
+from instructor_task.tasks_helper import (
+    cohort_students_and_upload, upload_grades_csv, upload_problem_grade_report, upload_students_csv
+)
+from instructor_task.tests.test_base import InstructorTaskCourseTestCase, TestReportMixin
+from instructor_task.tests.test_integration import TestGradeReportConditionalContent
+from django_comment_client.tests.utils import ContentGroupTestCase
 
 
 @ddt.ddt
@@ -261,6 +269,164 @@ class TestInstructorGradeReport(TestReportMixin, InstructorTaskCourseTestCase):
         self.assertDictContainsSubset({'attempted': 1, 'succeeded': 1, 'failed': 0}, result)
 
 
+class TestProblemGradeReport(TestReportMixin, InstructorTaskModuleTestCase):
+    """
+    Test that the weighted problem CSV generation works.
+    """
+    def setUp(self):
+        super(TestProblemGradeReport, self).setUp()
+        self.maxDiff = None
+        self.initialize_course()
+        # Add unicode data to CSV even though unicode usernames aren't
+        # technically possible in openedx.
+        self.student_1 = self.create_student(u'üser_1')
+        self.student_2 = self.create_student(u'üser_2')
+        self.csv_header_row = [u'Student ID', u'Email', u'Username', u'Final Grade']
+
+    @patch('instructor_task.tasks_helper._get_current_task')
+    def test_no_problems(self, _get_current_task):
+        """
+        Verify that we see no grade information for a  course with no graded
+        problems.
+        """
+        result = upload_problem_grade_report(None, None, self.course.id, None, 'graded')
+        self.assertDictContainsSubset({'action_name': 'graded', 'attempted': 2, 'succeeded': 2, 'failed': 0}, result)
+        self.verify_rows_in_csv([
+            dict(zip(self.csv_header_row, [unicode(self.student_1.id), self.student_1.email, self.student_1.username, '0.0'])),
+            dict(zip(self.csv_header_row, [unicode(self.student_2.id), self.student_2.email, self.student_2.username, '0.0']))
+        ])
+
+    @patch('instructor_task.tasks_helper._get_current_task')
+    def test_single_problem(self, _get_current_task):
+        vertical = ItemFactory.create(
+            parent_location=self.problem_section.location,
+            category='vertical',
+            metadata={'graded': True},
+            display_name='Problem Vertical'
+        )
+        self.define_option_problem('Problem1', parent=vertical)
+        # generate the course structure
+
+        self.submit_student_answer(self.student_1.username, 'Problem1', ['Option 1'])
+        result = upload_problem_grade_report(None, None, self.course.id, None, 'graded')
+        self.assertDictContainsSubset({'action_name': 'graded', 'attempted': 2, 'succeeded': 2, 'failed': 0}, result)
+        problem_name = 'Homework 1: Problem - Problem1'
+        header_row = self.csv_header_row + [problem_name + ' (Earned)', problem_name + ' (Possible)']
+        self.verify_rows_in_csv([
+            dict(zip(
+                header_row,
+                [unicode(self.student_1.id), self.student_1.email, self.student_1.username, '0.01', '1.0', '2.0']
+            )),
+            dict(zip(
+                header_row,
+                [unicode(self.student_2.id), self.student_2.email, self.student_2.username, '0.0', 'N/A', 'N/A']
+            ))
+        ])
+
+
+class TestProblemReportSplitTestContent(TestGradeReportConditionalContent):
+    OPTION_1 = 'Option 1'
+    OPTION_2 = 'Option 2'
+
+    def setUp(self):
+        super(TestProblemReportSplitTestContent, self).setUp()
+        self.problem_a_url = 'problem_a_url'
+        self.problem_b_url = 'problem_b_url'
+        self.define_option_problem(self.problem_a_url, parent=self.vertical_a)
+        self.define_option_problem(self.problem_b_url, parent=self.vertical_b)
+
+    def test_problem_grade_report(self):
+        """
+        Test problems that exist in a problem grade report.
+        """
+        # student A will get 100%, student B will get 50% because
+        # OPTION_1 is the correct option, and OPTION_2 is the
+        # incorrect option
+        self.submit_student_answer(self.student_a.username, self.problem_a_url, [self.OPTION_1, self.OPTION_1])
+        self.submit_student_answer(self.student_a.username, self.problem_b_url, [self.OPTION_1, self.OPTION_1])
+
+        self.submit_student_answer(self.student_b.username, self.problem_a_url, [self.OPTION_1, self.OPTION_2])
+        self.submit_student_answer(self.student_b.username, self.problem_b_url, [self.OPTION_1, self.OPTION_2])
+
+        with patch('instructor_task.tasks_helper._get_current_task'):
+            result = upload_problem_grade_report(None, None, self.course.id, None, 'graded')
+            self.verify_csv_task_success(result)
+
+        problem_names = ['Homework 1: Problem - problem_a_url', 'Homework 1: Problem - problem_b_url']
+        header_row = [u'Student ID', u'Email', u'Username', u'Final Grade']
+        for problem in problem_names:
+            header_row += [problem + ' (Earned)', problem + ' (Possible)']
+
+        self.verify_rows_in_csv([
+            dict(zip(
+                header_row,
+                [unicode(self.student_a.id), self.student_a.email, self.student_a.username, u'1.0', u'2.0', u'2.0', u'N/A', u'N/A']
+            )),
+            dict(zip(
+                header_row,
+                [unicode(self.student_b.id), self.student_b.email, self.student_b.username, u'0.5', u'N/A', u'N/A', u'1.0', u'2.0']
+            ))
+        ])
+
+
+class TestProblemReportCohortedContent(TestReportMixin, ContentGroupTestCase, InstructorTaskModuleTestCase):
+    def setUp(self):
+        super(TestProblemReportCohortedContent, self).setUp()
+        # contstruct cohorted problems to work on.
+        self.add_course_content()
+        vertical = ItemFactory.create(
+            parent_location=self.problem_section.location,
+            category='vertical',
+            metadata={'graded': True},
+            display_name='Problem Vertical'
+        )
+        print self.course.user_partitions
+        self.define_option_problem(
+            "Problem0",
+            parent=vertical,
+            group_access={self.course.user_partitions[0].id: [self.course.user_partitions[0].groups[0].id]}
+        )
+        self.define_option_problem(
+            "Problem1",
+            parent=vertical,
+            group_access={self.course.user_partitions[0].id: [self.course.user_partitions[0].groups[1].id]}
+        )
+
+        self.submit_student_answer(self.alpha_user.username, 'Problem0', ['Option 1', 'Option 1'])
+        self.submit_student_answer(self.alpha_user.username, 'Problem1', ['Option 1', 'Option 1'])
+        self.submit_student_answer(self.beta_user.username, 'Problem0', ['Option 1', 'Option 2'])
+        self.submit_student_answer(self.beta_user.username, 'Problem1', ['Option 1', 'Option 2'])
+
+    def test_cohort_content(self):
+        with patch('instructor_task.tasks_helper._get_current_task'):
+            result = upload_problem_grade_report(None, None, self.course.id, None, 'graded')
+            self.assertDictContainsSubset({'action_name': 'graded', 'attempted': 4, 'succeeded': 4, 'failed': 0}, result)
+
+        problem_names = ['Homework 1: Problem - Problem0', 'Homework 1: Problem - Problem1']
+        header_row = [u'Student ID', u'Email', u'Username', u'Final Grade']
+        for problem in problem_names:
+            header_row += [problem + ' (Earned)', problem + ' (Possible)']
+
+        self.verify_rows_in_csv([
+            dict(zip(
+                header_row,
+                [unicode(self.staff_user.id), self.staff_user.email, self.staff_user.username, u'0.0', u'N/A', u'N/A', u'N/A', u'N/A']
+            )),
+            dict(zip(
+                header_row,
+                [unicode(self.alpha_user.id), self.alpha_user.email, self.alpha_user.username, u'1.0', u'2.0', u'2.0', u'N/A', u'N/A']
+            )),
+            dict(zip(
+                header_row,
+                [unicode(self.beta_user.id), self.beta_user.email, self.beta_user.username, u'0.5', u'N/A', u'N/A', u'1.0', u'2.0']
+            )),
+            dict(zip(
+                header_row,
+                [unicode(self.non_cohorted_user.id), self.non_cohorted_user.email, self.non_cohorted_user.username, u'0.0', u'N/A', u'N/A', u'N/A', u'N/A']
+            )),
+        ])
+
+
 @ddt.ddt
 class TestStudentReport(TestReportMixin, InstructorTaskCourseTestCase):
     """