diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e8da97d124..a6c335db14 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,17 @@ These are notable changes in edx-platform. This is a rolling list of changes, in roughly chronological order, most recent first. Add your entries at or near the top. Include a label indicating the component affected. +LMS: Add feature for providing background grade report generation via Celery + instructor task, with reports uploaded to S3. Feature is visible on the beta + instructor dashboard. LMS-58 + +LMS: Beta-tester status is now set on a per-course-run basis, rather than being + valid across all runs with the same course name. Old group membership will + still work across runs, but new beta-testers will only be added to a single + course run. + +Blades: Enabled several Video Jasmine tests. BLD-463. + Studio: Continued modification of Studio pages to follow a RESTful framework. includes Settings pages, edit page for Subsection and Unit, and interfaces for updating xblocks (xmodules) and getting their editing HTML. @@ -19,7 +30,8 @@ Blades: Fix transcripts 500 error in studio (BLD-530) LMS: Add error recovery when a user loads or switches pages in an inline discussion. -Blades: Allow multiple strings as the correct answer to a string response question. BLD-474. +Blades: Allow multiple strings as the correct answer to a string response +question. BLD-474. Blades: a11y - Videos will alert screenreaders when the video is over. @@ -27,6 +39,7 @@ LMS: Trap focus on the loading element when a user loads more threads in the forum sidebar to improve accessibility. LMS: Add error recovery when a user loads more threads in the forum sidebar. +>>>>>>> origin/master LMS: Add a user-visible alert modal when a forums AJAX request fails. @@ -47,7 +60,8 @@ text like with bold or italics. (BLD-449) LMS: Beta instructor dashboard will only count actively enrolled students for course enrollment numbers. -Blades: Fix speed menu that is not rendered correctly when YouTube is unavailable. (BLD-457). +Blades: Fix speed menu that is not rendered correctly when YouTube is +unavailable. (BLD-457). LMS: Users with is_staff=True no longer have the STAFF label appear on their forum posts. diff --git a/common/djangoapps/student/models.py b/common/djangoapps/student/models.py index d7a918b569..6774632634 100644 --- a/common/djangoapps/student/models.py +++ b/common/djangoapps/student/models.py @@ -769,7 +769,7 @@ class CourseEnrollment(models.Model): activation_changed = True mode_changed = False - # if mode is None, the call to update_enrollment didn't specify a new + # if mode is None, the call to update_enrollment didn't specify a new # mode, so leave as-is if self.mode != mode and mode is not None: self.mode = mode @@ -967,6 +967,14 @@ class CourseEnrollment(models.Model): def enrollments_for_user(cls, user): return CourseEnrollment.objects.filter(user=user, is_active=1) + @classmethod + def users_enrolled_in(cls, course_id): + """Return a queryset of User for every user enrolled in the course.""" + return User.objects.filter( + courseenrollment__course_id=course_id, + courseenrollment__is_active=True + ) + def activate(self): """Makes this `CourseEnrollment` record active. Saves immediately.""" self.update_enrollment(is_active=True) diff --git a/common/lib/xmodule/xmodule/abtest_module.py b/common/lib/xmodule/xmodule/abtest_module.py index c0a53d048f..06d4e0b2d2 100644 --- a/common/lib/xmodule/xmodule/abtest_module.py +++ b/common/lib/xmodule/xmodule/abtest_module.py @@ -45,7 +45,7 @@ class ABTestModule(ABTestFields, XModule): """ def __init__(self, *args, **kwargs): - XModule.__init__(self, *args, **kwargs) + super(ABTestModule, self).__init__(*args, **kwargs) if self.group is None: self.group = group_from_value( diff --git a/common/lib/xmodule/xmodule/annotatable_module.py b/common/lib/xmodule/xmodule/annotatable_module.py index ca85065577..fbc175b5b9 100644 --- a/common/lib/xmodule/xmodule/annotatable_module.py +++ b/common/lib/xmodule/xmodule/annotatable_module.py @@ -50,7 +50,7 @@ class AnnotatableModule(AnnotatableFields, XModule): icon_class = 'annotatable' def __init__(self, *args, **kwargs): - XModule.__init__(self, *args, **kwargs) + super(AnnotatableModule, self).__init__(*args, **kwargs) xmltree = etree.fromstring(self.data) diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py index 6d664acbf6..cf6c2e3dce 100644 --- a/common/lib/xmodule/xmodule/capa_module.py +++ b/common/lib/xmodule/xmodule/capa_module.py @@ -190,7 +190,7 @@ class CapaModule(CapaFields, XModule): """ Accepts the same arguments as xmodule.x_module:XModule.__init__ """ - XModule.__init__(self, *args, **kwargs) + super(CapaModule, self).__init__(*args, **kwargs) due_date = self.due diff --git a/common/lib/xmodule/xmodule/combined_open_ended_module.py b/common/lib/xmodule/xmodule/combined_open_ended_module.py index 6dc2ac045b..68a0d65617 100644 --- a/common/lib/xmodule/xmodule/combined_open_ended_module.py +++ b/common/lib/xmodule/xmodule/combined_open_ended_module.py @@ -412,7 +412,7 @@ class CombinedOpenEndedModule(CombinedOpenEndedFields, XModule): See DEFAULT_DATA for a sample. """ - XModule.__init__(self, *args, **kwargs) + super(CombinedOpenEndedModule, self).__init__(*args, **kwargs) self.system.set('location', self.location) diff --git a/common/lib/xmodule/xmodule/crowdsource_hinter.py b/common/lib/xmodule/xmodule/crowdsource_hinter.py index 62bfe5b586..5a1091f6fb 100644 --- a/common/lib/xmodule/xmodule/crowdsource_hinter.py +++ b/common/lib/xmodule/xmodule/crowdsource_hinter.py @@ -75,7 +75,7 @@ class CrowdsourceHinterModule(CrowdsourceHinterFields, XModule): js_module_name = "Hinter" def __init__(self, *args, **kwargs): - XModule.__init__(self, *args, **kwargs) + super(CrowdsourceHinterModule, self).__init__(*args, **kwargs) # We need to know whether we are working with a FormulaResponse problem. try: responder = self.get_display_items()[0].lcp.responders.values()[0] diff --git a/common/lib/xmodule/xmodule/foldit_module.py b/common/lib/xmodule/xmodule/foldit_module.py index e1714ff96b..655ff1911a 100644 --- a/common/lib/xmodule/xmodule/foldit_module.py +++ b/common/lib/xmodule/xmodule/foldit_module.py @@ -39,7 +39,7 @@ class FolditModule(FolditFields, XModule): required_sublevel_half_credit="3" show_leaderboard="false"/> """ - XModule.__init__(self, *args, **kwargs) + super(FolditModule, self).__init__(*args, **kwargs) self.due_time = self.due def is_complete(self): diff --git a/common/lib/xmodule/xmodule/open_ended_grading_classes/combined_open_ended_modulev1.py b/common/lib/xmodule/xmodule/open_ended_grading_classes/combined_open_ended_modulev1.py index 4a1715c48d..72915eb7b3 100644 --- a/common/lib/xmodule/xmodule/open_ended_grading_classes/combined_open_ended_modulev1.py +++ b/common/lib/xmodule/xmodule/open_ended_grading_classes/combined_open_ended_modulev1.py @@ -93,7 +93,6 @@ class CombinedOpenEndedV1Module(): Definition file should have one or many task blocks, a rubric block, and a prompt block. See DEFAULT_DATA in combined_open_ended_module for a sample. """ - self.instance_state = instance_state self.display_name = instance_state.get('display_name', "Open Ended") diff --git a/common/lib/xmodule/xmodule/randomize_module.py b/common/lib/xmodule/xmodule/randomize_module.py index 00baf3f140..71d23012d1 100644 --- a/common/lib/xmodule/xmodule/randomize_module.py +++ b/common/lib/xmodule/xmodule/randomize_module.py @@ -39,7 +39,7 @@ class RandomizeModule(RandomizeFields, XModule): modules. """ def __init__(self, *args, **kwargs): - XModule.__init__(self, *args, **kwargs) + super(RandomizeModule, self).__init__(*args, **kwargs) # NOTE: calling self.get_children() creates a circular reference-- # it calls get_child_descriptors() internally, but that doesn't work until diff --git a/common/lib/xmodule/xmodule/seq_module.py b/common/lib/xmodule/xmodule/seq_module.py index 291d7a1ea1..62e93cb90e 100644 --- a/common/lib/xmodule/xmodule/seq_module.py +++ b/common/lib/xmodule/xmodule/seq_module.py @@ -38,7 +38,7 @@ class SequenceModule(SequenceFields, XModule): def __init__(self, *args, **kwargs): - XModule.__init__(self, *args, **kwargs) + super(SequenceModule, self).__init__(*args, **kwargs) # if position is specified in system, then use that instead if getattr(self.system, 'position', None) is not None: diff --git a/common/lib/xmodule/xmodule/tests/test_capa_module.py b/common/lib/xmodule/xmodule/tests/test_capa_module.py index ce8d7489d5..b7b7d8b6f4 100644 --- a/common/lib/xmodule/xmodule/tests/test_capa_module.py +++ b/common/lib/xmodule/xmodule/tests/test_capa_module.py @@ -143,7 +143,6 @@ class CapaFactory(object): DictFieldData(field_data), ScopeIds(None, None, location, location), ) - system.xmodule_instance = module if correct: # TODO: probably better to actually set the internal state properly, but... diff --git a/common/lib/xmodule/xmodule/tests/test_combined_open_ended.py b/common/lib/xmodule/xmodule/tests/test_combined_open_ended.py index b72eeab66b..e1b2a4ebe2 100644 --- a/common/lib/xmodule/xmodule/tests/test_combined_open_ended.py +++ b/common/lib/xmodule/xmodule/tests/test_combined_open_ended.py @@ -664,7 +664,6 @@ class CombinedOpenEndedModuleTest(unittest.TestCase): static_data=self.static_data, metadata=self.metadata, instance_state=instance_state) - self.test_system.xmodule_instance = module return combinedoe def ai_state_reset(self, task_state, task_number=None): @@ -717,6 +716,7 @@ class CombinedOpenEndedModuleTest(unittest.TestCase): def test_state_pe_single(self): self.ai_state_success(TEST_STATE_PE_SINGLE, iscore=0, tasks=[self.task_xml2]) + class OpenEndedModuleXmlTest(unittest.TestCase, DummyModulestore): """ Test the student flow in the combined open ended xmodule @@ -726,31 +726,42 @@ class OpenEndedModuleXmlTest(unittest.TestCase, DummyModulestore): assessment = [0, 1] hint = "blah" - def setUp(self): - self.test_system = get_test_system() - self.test_system.open_ended_grading_interface = None - self.test_system.xqueue['interface'] = Mock( + def get_module_system(self, descriptor): + test_system = get_test_system() + test_system.open_ended_grading_interface = None + test_system.xqueue['interface'] = Mock( send_to_queue=Mock(side_effect=[1, "queued"]) ) + + return test_system + + def setUp(self): self.setup_modulestore(COURSE) + def _handle_ajax(self, dispatch, content): + # Load the module from persistence + module = self._module() + + # Call handle_ajax on the module + result = module.handle_ajax(dispatch, content) + + # Persist the state + module.save() + + return result + + def _module(self): + return self.get_module_from_location(self.problem_location, COURSE) + def test_open_ended_load_and_save(self): """ See if we can load the module and save an answer @return: """ - # Load the module - module = self.get_module_from_location(self.problem_location, COURSE) - # Try saving an answer - module.handle_ajax("save_answer", {"student_answer": self.answer}) - # Save our modifications to the underlying KeyValueStore so they can be persisted - module.save() - task_one_json = json.loads(module.task_states[0]) - self.assertEqual(task_one_json['child_history'][0]['answer'], self.answer) + self._handle_ajax("save_answer", {"student_answer": self.answer}) - module = self.get_module_from_location(self.problem_location, COURSE) - task_one_json = json.loads(module.task_states[0]) + task_one_json = json.loads(self._module().task_states[0]) self.assertEqual(task_one_json['child_history'][0]['answer'], self.answer) def test_open_ended_flow_reset(self): @@ -759,42 +770,37 @@ class OpenEndedModuleXmlTest(unittest.TestCase, DummyModulestore): @return: """ assessment = [0, 1] - module = self.get_module_from_location(self.problem_location, COURSE) # Simulate a student saving an answer - html = module.handle_ajax("get_html", {}) - module.save() - module.handle_ajax("save_answer", {"student_answer": self.answer}) - module.save() - html = module.handle_ajax("get_html", {}) - module.save() + self._handle_ajax("get_html", {}) + self._handle_ajax("save_answer", {"student_answer": self.answer}) + self._handle_ajax("get_html", {}) # Mock a student submitting an assessment assessment_dict = MultiDict({'assessment': sum(assessment)}) assessment_dict.extend(('score_list[]', val) for val in assessment) - module.handle_ajax("save_assessment", assessment_dict) - module.save() - task_one_json = json.loads(module.task_states[0]) + self._handle_ajax("save_assessment", assessment_dict) + + task_one_json = json.loads(self._module().task_states[0]) self.assertEqual(json.loads(task_one_json['child_history'][0]['post_assessment']), assessment) - rubric = module.handle_ajax("get_combined_rubric", {}) - module.save() + + self._handle_ajax("get_combined_rubric", {}) # Move to the next step in the problem - module.handle_ajax("next_problem", {}) - module.save() - self.assertEqual(module.current_task_number, 0) + self._handle_ajax("next_problem", {}) + self.assertEqual(self._module().current_task_number, 0) - html = module.render('student_view').content + html = self._module().render('student_view').content self.assertIsInstance(html, basestring) - rubric = module.handle_ajax("get_combined_rubric", {}) - module.save() + rubric = self._handle_ajax("get_combined_rubric", {}) self.assertIsInstance(rubric, basestring) - self.assertEqual(module.state, "assessing") - module.handle_ajax("reset", {}) - module.save() - self.assertEqual(module.current_task_number, 0) + + self.assertEqual(self._module().state, "assessing") + + self._handle_ajax("reset", {}) + self.assertEqual(self._module().current_task_number, 0) def test_open_ended_flow_correct(self): """ @@ -803,42 +809,36 @@ class OpenEndedModuleXmlTest(unittest.TestCase, DummyModulestore): @return: """ assessment = [1, 1] - # Load the module - module = self.get_module_from_location(self.problem_location, COURSE) # Simulate a student saving an answer - module.handle_ajax("save_answer", {"student_answer": self.answer}) - module.save() - status = module.handle_ajax("get_status", {}) - module.save() + self._handle_ajax("save_answer", {"student_answer": self.answer}) + status = self._handle_ajax("get_status", {}) self.assertIsInstance(status, basestring) # Mock a student submitting an assessment assessment_dict = MultiDict({'assessment': sum(assessment)}) assessment_dict.extend(('score_list[]', val) for val in assessment) - module.handle_ajax("save_assessment", assessment_dict) - module.save() - task_one_json = json.loads(module.task_states[0]) + self._handle_ajax("save_assessment", assessment_dict) + + task_one_json = json.loads(self._module().task_states[0]) self.assertEqual(json.loads(task_one_json['child_history'][0]['post_assessment']), assessment) # Move to the next step in the problem try: - module.handle_ajax("next_problem", {}) - module.save() + self._handle_ajax("next_problem", {}) except GradingServiceError: # This error is okay. We don't have a grading service to connect to! pass - self.assertEqual(module.current_task_number, 1) + self.assertEqual(self._module().current_task_number, 1) try: - module.render('student_view') + self._module().render('student_view') except GradingServiceError: # This error is okay. We don't have a grading service to connect to! pass # Try to get the rubric from the module - module.handle_ajax("get_combined_rubric", {}) - module.save() + self._handle_ajax("get_combined_rubric", {}) # Make a fake reply from the queue queue_reply = { @@ -856,29 +856,26 @@ class OpenEndedModuleXmlTest(unittest.TestCase, DummyModulestore): }) } - module.handle_ajax("check_for_score", {}) - module.save() + self._handle_ajax("check_for_score", {}) # Update the module with the fake queue reply - module.handle_ajax("score_update", queue_reply) - module.save() + self._handle_ajax("score_update", queue_reply) + + module = self._module() self.assertFalse(module.ready_to_reset) self.assertEqual(module.current_task_number, 1) # Get html and other data client will request module.render('student_view') - module.handle_ajax("skip_post_assessment", {}) - module.save() + self._handle_ajax("skip_post_assessment", {}) # Get all results - module.handle_ajax("get_combined_rubric", {}) - module.save() + self._handle_ajax("get_combined_rubric", {}) # reset the problem - module.handle_ajax("reset", {}) - module.save() - self.assertEqual(module.state, "initial") + self._handle_ajax("reset", {}) + self.assertEqual(self._module().state, "initial") class OpenEndedModuleXmlAttemptTest(unittest.TestCase, DummyModulestore): @@ -890,14 +887,32 @@ class OpenEndedModuleXmlAttemptTest(unittest.TestCase, DummyModulestore): assessment = [0, 1] hint = "blah" - def setUp(self): - self.test_system = get_test_system() - self.test_system.open_ended_grading_interface = None - self.test_system.xqueue['interface'] = Mock( + def get_module_system(self, descriptor): + test_system = get_test_system() + test_system.open_ended_grading_interface = None + test_system.xqueue['interface'] = Mock( send_to_queue=Mock(side_effect=[1, "queued"]) ) + return test_system + + def setUp(self): self.setup_modulestore(COURSE) + def _handle_ajax(self, dispatch, content): + # Load the module from persistence + module = self._module() + + # Call handle_ajax on the module + result = module.handle_ajax(dispatch, content) + + # Persist the state + module.save() + + return result + + def _module(self): + return self.get_module_from_location(self.problem_location, COURSE) + def test_reset_fail(self): """ Test the flow of the module if we complete the self assessment step and then reset @@ -905,39 +920,32 @@ class OpenEndedModuleXmlAttemptTest(unittest.TestCase, DummyModulestore): @return: """ assessment = [0, 1] - module = self.get_module_from_location(self.problem_location, COURSE) - module.save() # Simulate a student saving an answer - module.handle_ajax("save_answer", {"student_answer": self.answer}) - module.save() + self._handle_ajax("save_answer", {"student_answer": self.answer}) # Mock a student submitting an assessment assessment_dict = MultiDict({'assessment': sum(assessment)}) assessment_dict.extend(('score_list[]', val) for val in assessment) - module.handle_ajax("save_assessment", assessment_dict) - module.save() - task_one_json = json.loads(module.task_states[0]) + self._handle_ajax("save_assessment", assessment_dict) + task_one_json = json.loads(self._module().task_states[0]) self.assertEqual(json.loads(task_one_json['child_history'][0]['post_assessment']), assessment) # Move to the next step in the problem - module.handle_ajax("next_problem", {}) - module.save() - self.assertEqual(module.current_task_number, 0) + self._handle_ajax("next_problem", {}) + self.assertEqual(self._module().current_task_number, 0) - html = module.render('student_view').content + html = self._module().render('student_view').content self.assertIsInstance(html, basestring) # Module should now be done - rubric = module.handle_ajax("get_combined_rubric", {}) - module.save() + rubric = self._handle_ajax("get_combined_rubric", {}) self.assertIsInstance(rubric, basestring) - self.assertEqual(module.state, "done") + self.assertEqual(self._module().state, "done") # Try to reset, should fail because only 1 attempt is allowed - reset_data = json.loads(module.handle_ajax("reset", {})) - module.save() + reset_data = json.loads(self._handle_ajax("reset", {})) self.assertEqual(reset_data['success'], False) class OpenEndedModuleXmlImageUploadTest(unittest.TestCase, DummyModulestore): @@ -951,13 +959,16 @@ class OpenEndedModuleXmlImageUploadTest(unittest.TestCase, DummyModulestore): answer_link = "http://www.edx.org" autolink_tag = "'.format(self.id) @@ -762,7 +763,7 @@ class XModuleDescriptor(XModuleMixin, HTMLSnippet, ResourceTemplates, XBlock): assert self.xmodule_runtime.error_descriptor_class is not None if self.xmodule_runtime.xmodule_instance is None: try: - self.xmodule_runtime.xmodule_instance = self.xmodule_runtime.construct_xblock_from_class( + self.xmodule_runtime.construct_xblock_from_class( self.module_class, descriptor=self, scope_ids=self.scope_ids, @@ -770,6 +771,10 @@ class XModuleDescriptor(XModuleMixin, HTMLSnippet, ResourceTemplates, XBlock): ) self.xmodule_runtime.xmodule_instance.save() except Exception: # pylint: disable=broad-except + # xmodule_instance is set by the XModule.__init__. If we had an error after that, + # we need to clean it out so that we can set up the ErrorModule instead + self.xmodule_runtime.xmodule_instance = None + if isinstance(self, self.xmodule_runtime.error_descriptor_class): log.exception('Error creating an ErrorModule from an ErrorDescriptor') raise @@ -1066,6 +1071,7 @@ class ModuleSystem(ConfigurableFragmentWrapper, Runtime): # pylint: disable=abs """ The url prefix to be used by XModules to call into handle_ajax """ + assert self.xmodule_instance is not None return self.handler_url(self.xmodule_instance, 'xmodule_handler', '', '').rstrip('/?') diff --git a/lms/djangoapps/courseware/access.py b/lms/djangoapps/courseware/access.py index 4b50c559de..8c4fb662f9 100644 --- a/lms/djangoapps/courseware/access.py +++ b/lms/djangoapps/courseware/access.py @@ -241,7 +241,11 @@ def _has_access_descriptor(user, descriptor, action, course_context=None): # Check start date if descriptor.start is not None: now = datetime.now(UTC()) - effective_start = _adjust_start_date_for_beta_testers(user, descriptor) + effective_start = _adjust_start_date_for_beta_testers( + user, + descriptor, + course_context=course_context + ) if now > effective_start: # after start date, everyone can see it debug("Allow: now > effective start date") @@ -337,7 +341,7 @@ def _dispatch(table, action, user, obj): type(obj), action)) -def _adjust_start_date_for_beta_testers(user, descriptor): +def _adjust_start_date_for_beta_testers(user, descriptor, course_context=None): """ If user is in a beta test group, adjust the start date by the appropriate number of days. @@ -364,7 +368,7 @@ def _adjust_start_date_for_beta_testers(user, descriptor): # bail early if no beta testing is set up return descriptor.start - if CourseBetaTesterRole(descriptor.location).has_user(user): + if CourseBetaTesterRole(descriptor.location, course_context=course_context).has_user(user): debug("Adjust start time: user in beta role for %s", descriptor) delta = timedelta(descriptor.days_early_for_beta) effective = descriptor.start - delta diff --git a/lms/djangoapps/courseware/grades.py b/lms/djangoapps/courseware/grades.py index 49d68cc453..f17a3486d5 100644 --- a/lms/djangoapps/courseware/grades.py +++ b/lms/djangoapps/courseware/grades.py @@ -1,6 +1,6 @@ # Compute grades using real division, with no integer truncation from __future__ import division - +from collections import defaultdict import random import logging @@ -9,14 +9,18 @@ from collections import defaultdict from django.conf import settings from django.contrib.auth.models import User from django.db import transaction +from django.test.client import RequestFactory -from courseware.model_data import FieldDataCache, DjangoKeyValueStore +from dogapi import dog_stats_api + +from courseware import courses +from courseware.model_data import FieldDataCache from xblock.fields import Scope -from .module_render import get_module, get_module_for_descriptor from xmodule import graders from xmodule.capa_module import CapaModule from xmodule.graders import Score from .models import StudentModule +from .module_render import get_module, get_module_for_descriptor log = logging.getLogger("mitx.courseware") @@ -331,7 +335,6 @@ def _progress_summary(student, request, course): module_creator = section_module.xmodule_runtime.get_module for module_descriptor in yield_dynamic_descriptor_descendents(section_module, module_creator): - course_id = course.id (correct, total) = get_score(course_id, student, module_descriptor, module_creator) if correct is None and total is None: @@ -447,3 +450,52 @@ def manual_transaction(): raise else: transaction.commit() + + +def iterate_grades_for(course_id, students): + """Given a course_id and an iterable of students (User), yield a tuple of: + + (student, gradeset, err_msg) for every student enrolled in the course. + + If an error occured, gradeset will be an empty dict and err_msg will be an + exception message. If there was no error, err_msg is an empty string. + + The gradeset is a dictionary with the following fields: + + - grade : A final letter grade. + - percent : The final percent for the class (rounded up). + - section_breakdown : A breakdown of each section that makes + up the grade. (For display) + - grade_breakdown : A breakdown of the major components that + make up the final grade. (For display) + - raw_scores: contains scores for every graded module + """ + course = courses.get_course_by_id(course_id) + + # We make a fake request because grading code expects to be able to look at + # the request. We have to attach the correct user to the request before + # grading that student. + request = RequestFactory().get('/') + + for student in students: + with dog_stats_api.timer('lms.grades.iterate_grades_for', tags=['action:{}'.format(course_id)]): + try: + request.user = student + # Grading calls problem rendering, which calls masquerading, + # which checks session vars -- thus the empty session dict below. + # It's not pretty, but untangling that is currently beyond the + # scope of this feature. + request.session = {} + gradeset = grade(student, request, course) + yield student, gradeset, "" + except Exception as exc: # pylint: disable=broad-except + # Keep marching on even if this student couldn't be graded for + # some reason, but log it for future reference. + log.exception( + 'Cannot grade student %s (%s) in course %s because of exception: %s', + student.username, + student.id, + course_id, + exc.message + ) + yield student, {}, exc.message diff --git a/lms/djangoapps/courseware/roles.py b/lms/djangoapps/courseware/roles.py index 1643dd5051..110bb9f362 100644 --- a/lms/djangoapps/courseware/roles.py +++ b/lms/djangoapps/courseware/roles.py @@ -187,6 +187,6 @@ class OrgStaffRole(OrgRole): class OrgInstructorRole(OrgRole): - """An organization staff member""" + """An organization instructor""" def __init__(self, *args, **kwargs): - super(OrgInstructorRole, self).__init__('staff', *args, **kwargs) + super(OrgInstructorRole, self).__init__('instructor', *args, **kwargs) diff --git a/lms/djangoapps/courseware/tests/factories.py b/lms/djangoapps/courseware/tests/factories.py index 91f91f2617..38ad365011 100644 --- a/lms/djangoapps/courseware/tests/factories.py +++ b/lms/djangoapps/courseware/tests/factories.py @@ -14,7 +14,14 @@ from student.tests.factories import RegistrationFactory # Imported to re-export from student.tests.factories import UserProfileFactory as StudentUserProfileFactory from courseware.models import StudentModule, XModuleUserStateSummaryField from courseware.models import XModuleStudentInfoField, XModuleStudentPrefsField -from courseware.roles import CourseInstructorRole, CourseStaffRole +from courseware.roles import ( + CourseInstructorRole, + CourseStaffRole, + CourseBetaTesterRole, + GlobalStaff, + OrgStaffRole, + OrgInstructorRole, +) from xmodule.modulestore import Location @@ -54,6 +61,59 @@ class StaffFactory(UserFactory): CourseStaffRole(extracted).add_users(self) +class BetaTesterFactory(UserFactory): + """ + Given a course Location, returns a User object with beta-tester + permissions for `course`. + """ + last_name = "Beta-Tester" + + @post_generation + def course(self, create, extracted, **kwargs): + if extracted is None: + raise ValueError("Must specify a course location for a beta-tester user") + CourseBetaTesterRole(extracted).add_users(self) + + +class OrgStaffFactory(UserFactory): + """ + Given a course Location, returns a User object with org-staff + permissions for `course`. + """ + last_name = "Org-Staff" + + @post_generation + def course(self, create, extracted, **kwargs): + if extracted is None: + raise ValueError("Must specify a course location for an org-staff user") + OrgStaffRole(extracted).add_users(self) + + +class OrgInstructorFactory(UserFactory): + """ + Given a course Location, returns a User object with org-instructor + permissions for `course`. + """ + last_name = "Org-Instructor" + + @post_generation + def course(self, create, extracted, **kwargs): + if extracted is None: + raise ValueError("Must specify a course location for an org-instructor user") + OrgInstructorRole(extracted).add_users(self) + + +class GlobalStaffFactory(UserFactory): + """ + Returns a User object with global staff access + """ + last_name = "GlobalStaff" + + @post_generation + def set_staff(self, create, extracted, **kwargs): + GlobalStaff().add_users(self) + + class StudentModuleFactory(DjangoModelFactory): FACTORY_FOR = StudentModule diff --git a/lms/djangoapps/courseware/tests/test_view_authentication.py b/lms/djangoapps/courseware/tests/test_view_authentication.py index 3639d07afa..9ed0dea8e5 100644 --- a/lms/djangoapps/courseware/tests/test_view_authentication.py +++ b/lms/djangoapps/courseware/tests/test_view_authentication.py @@ -9,14 +9,23 @@ from django.test.utils import override_settings # Need access to internal func to put users in the right group from courseware.access import has_access -from courseware.roles import CourseBetaTesterRole, CourseInstructorRole, CourseStaffRole, GlobalStaff from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory +from student.tests.factories import UserFactory, CourseEnrollmentFactory + from courseware.tests.helpers import LoginEnrollmentTestCase, check_for_get_code from courseware.tests.modulestore_config import TEST_DATA_MIXED_MODULESTORE +from courseware.tests.factories import ( + BetaTesterFactory, + StaffFactory, + GlobalStaffFactory, + InstructorFactory, + OrgStaffFactory, + OrgInstructorFactory, +) @override_settings(MODULESTORE=TEST_DATA_MIXED_MODULESTORE) @@ -89,13 +98,16 @@ class TestViewAuth(ModuleStoreTestCase, LoginEnrollmentTestCase): # user (the student), and the requesting user (the prof) url = reverse('student_progress', kwargs={'course_id': course.id, - 'student_id': User.objects.get(email=self.ACCOUNT_INFO[0][0]).id}) + 'student_id': self.enrolled_user.id}) check_for_get_code(self, 404, url) # The courseware url should redirect, not 200 url = self._reverse_urls(['courseware'], course)[0] check_for_get_code(self, 302, url) + def login(self, user): + return super(TestViewAuth, self).login(user.email, 'test') + def setUp(self): self.course = CourseFactory.create(number='999', display_name='Robot_Super_Course') @@ -103,26 +115,37 @@ class TestViewAuth(ModuleStoreTestCase, LoginEnrollmentTestCase): self.courseware_chapter = ItemFactory.create(display_name='courseware') self.test_course = CourseFactory.create(number='666', display_name='Robot_Sub_Course') - self.sub_courseware_chapter = ItemFactory.create(parent_location=self.test_course.location, - display_name='courseware') - self.sub_overview_chapter = ItemFactory.create(parent_location=self.sub_courseware_chapter.location, - display_name='Overview') - self.welcome_section = ItemFactory.create(parent_location=self.overview_chapter.location, - display_name='Welcome') + self.other_org_course = CourseFactory.create(org='Other_Org_Course') + self.sub_courseware_chapter = ItemFactory.create( + parent_location=self.test_course.location, display_name='courseware' + ) + self.sub_overview_chapter = ItemFactory.create( + parent_location=self.sub_courseware_chapter.location, + display_name='Overview' + ) + self.welcome_section = ItemFactory.create( + parent_location=self.overview_chapter.location, + display_name='Welcome' + ) - # Create two accounts and activate them. - for i in range(len(self.ACCOUNT_INFO)): - username, email, password = 'u{0}'.format(i), self.ACCOUNT_INFO[i][0], self.ACCOUNT_INFO[i][1] - self.create_account(username, email, password) - self.activate_user(email) + self.unenrolled_user = UserFactory(last_name="Unenrolled") + + self.enrolled_user = UserFactory(last_name="Enrolled") + CourseEnrollmentFactory(user=self.enrolled_user, course_id=self.course.id) + CourseEnrollmentFactory(user=self.enrolled_user, course_id=self.test_course.id) + + self.staff_user = StaffFactory(course=self.course.location) + self.instructor_user = InstructorFactory(course=self.course.location) + self.org_staff_user = OrgStaffFactory(course=self.course.location) + self.org_instructor_user = OrgInstructorFactory(course=self.course.location) + self.global_staff_user = GlobalStaffFactory() def test_redirection_unenrolled(self): """ Verify unenrolled student is redirected to the 'about' section of the chapter instead of the 'Welcome' section after clicking on the courseware tab. """ - email, password = self.ACCOUNT_INFO[0] - self.login(email, password) + self.login(self.unenrolled_user) response = self.client.get(reverse('courseware', kwargs={'course_id': self.course.id})) self.assertRedirects(response, @@ -134,9 +157,7 @@ class TestViewAuth(ModuleStoreTestCase, LoginEnrollmentTestCase): Verify enrolled student is redirected to the 'Welcome' section of the chapter after clicking on the courseware tab. """ - email, password = self.ACCOUNT_INFO[0] - self.login(email, password) - self.enroll(self.course) + self.login(self.enrolled_user) response = self.client.get(reverse('courseware', kwargs={'course_id': self.course.id})) @@ -152,11 +173,7 @@ class TestViewAuth(ModuleStoreTestCase, LoginEnrollmentTestCase): Verify non-staff cannot load the instructor dashboard, the grade views, and student profile pages. """ - email, password = self.ACCOUNT_INFO[0] - self.login(email, password) - - self.enroll(self.course) - self.enroll(self.test_course) + self.login(self.enrolled_user) urls = [reverse('instructor_dashboard', kwargs={'course_id': self.course.id}), reverse('instructor_dashboard', kwargs={'course_id': self.test_course.id})] @@ -165,17 +182,12 @@ class TestViewAuth(ModuleStoreTestCase, LoginEnrollmentTestCase): for url in urls: check_for_get_code(self, 404, url) - def test_instructor_course_access(self): + def test_staff_course_access(self): """ - Verify instructor can load the instructor dashboard, the grade views, + Verify staff can load the staff dashboard, the grade views, and student profile pages for their course. """ - email, password = self.ACCOUNT_INFO[1] - - # Make the instructor staff in self.course - CourseInstructorRole(self.course.location).add_users(User.objects.get(email=email)) - - self.login(email, password) + self.login(self.staff_user) # Now should be able to get to self.course, but not self.test_course url = reverse('instructor_dashboard', kwargs={'course_id': self.course.id}) @@ -184,18 +196,55 @@ class TestViewAuth(ModuleStoreTestCase, LoginEnrollmentTestCase): url = reverse('instructor_dashboard', kwargs={'course_id': self.test_course.id}) check_for_get_code(self, 404, url) - def test_instructor_as_staff_access(self): + def test_instructor_course_access(self): """ - Verify the instructor can load staff pages if he is given - staff permissions. + Verify instructor can load the instructor dashboard, the grade views, + and student profile pages for their course. """ - email, password = self.ACCOUNT_INFO[1] - self.login(email, password) + self.login(self.instructor_user) - # now make the instructor also staff - instructor = User.objects.get(email=email) - instructor.is_staff = True - instructor.save() + # Now should be able to get to self.course, but not self.test_course + url = reverse('instructor_dashboard', kwargs={'course_id': self.course.id}) + check_for_get_code(self, 200, url) + + url = reverse('instructor_dashboard', kwargs={'course_id': self.test_course.id}) + check_for_get_code(self, 404, url) + + def test_org_staff_access(self): + """ + Verify org staff can load the instructor dashboard, the grade views, + and student profile pages for course in their org. + """ + self.login(self.org_staff_user) + url = reverse('instructor_dashboard', kwargs={'course_id': self.course.id}) + check_for_get_code(self, 200, url) + + url = reverse('instructor_dashboard', kwargs={'course_id': self.test_course.id}) + check_for_get_code(self, 200, url) + + url = reverse('instructor_dashboard', kwargs={'course_id': self.other_org_course.id}) + check_for_get_code(self, 404, url) + + def test_org_instructor_access(self): + """ + Verify org instructor can load the instructor dashboard, the grade views, + and student profile pages for course in their org. + """ + self.login(self.org_instructor_user) + url = reverse('instructor_dashboard', kwargs={'course_id': self.course.id}) + check_for_get_code(self, 200, url) + + url = reverse('instructor_dashboard', kwargs={'course_id': self.test_course.id}) + check_for_get_code(self, 200, url) + + url = reverse('instructor_dashboard', kwargs={'course_id': self.other_org_course.id}) + check_for_get_code(self, 404, url) + + def test_global_staff_access(self): + """ + Verify the global staff user can access any course. + """ + self.login(self.global_staff_user) # and now should be able to load both urls = [reverse('instructor_dashboard', kwargs={'course_id': self.course.id}), @@ -211,8 +260,6 @@ class TestViewAuth(ModuleStoreTestCase, LoginEnrollmentTestCase): pages. """ - student_email, student_password = self.ACCOUNT_INFO[0] - # Make courses start in the future now = datetime.datetime.now(pytz.UTC) tomorrow = now + datetime.timedelta(days=1) @@ -225,9 +272,7 @@ class TestViewAuth(ModuleStoreTestCase, LoginEnrollmentTestCase): self.assertFalse(self.test_course.has_started()) # First, try with an enrolled student - self.login(student_email, student_password) - self.enroll(self.course, True) - self.enroll(self.test_course, True) + self.login(self.enrolled_user) # shouldn't be able to get to anything except the light pages self._check_non_staff_light(self.course) @@ -241,8 +286,6 @@ class TestViewAuth(ModuleStoreTestCase, LoginEnrollmentTestCase): Make sure that before course start instructors can access the page for their course. """ - instructor_email, instructor_password = self.ACCOUNT_INFO[1] - now = datetime.datetime.now(pytz.UTC) tomorrow = now + datetime.timedelta(days=1) course_data = {'start': tomorrow} @@ -250,11 +293,7 @@ class TestViewAuth(ModuleStoreTestCase, LoginEnrollmentTestCase): self.course = self.update_course(self.course, course_data) self.test_course = self.update_course(self.test_course, test_course_data) - # Make the instructor staff in self.course - CourseStaffRole(self.course.location).add_users(User.objects.get(email=instructor_email)) - - self.logout() - self.login(instructor_email, instructor_password) + self.login(self.instructor_user) # Enroll in the classes---can't see courseware otherwise. self.enroll(self.course, True) self.enroll(self.test_course, True) @@ -265,13 +304,11 @@ class TestViewAuth(ModuleStoreTestCase, LoginEnrollmentTestCase): self._check_staff(self.course) @patch.dict('courseware.access.settings.MITX_FEATURES', {'DISABLE_START_DATES': False}) - def test_dark_launch_staff(self): + def test_dark_launch_global_staff(self): """ Make sure that before course start staff can access course pages. """ - instructor_email, instructor_password = self.ACCOUNT_INFO[1] - now = datetime.datetime.now(pytz.UTC) tomorrow = now + datetime.timedelta(days=1) course_data = {'start': tomorrow} @@ -279,15 +316,10 @@ class TestViewAuth(ModuleStoreTestCase, LoginEnrollmentTestCase): self.course = self.update_course(self.course, course_data) self.test_course = self.update_course(self.test_course, test_course_data) - self.login(instructor_email, instructor_password) + self.login(self.global_staff_user) self.enroll(self.course, True) self.enroll(self.test_course, True) - # now also make the instructor staff - instructor = User.objects.get(email=instructor_email) - instructor.is_staff = True - instructor.save() - # and now should be able to load both self._check_staff(self.course) self._check_staff(self.test_course) @@ -297,9 +329,6 @@ class TestViewAuth(ModuleStoreTestCase, LoginEnrollmentTestCase): """ Check that enrollment periods work. """ - student_email, student_password = self.ACCOUNT_INFO[0] - instructor_email, instructor_password = self.ACCOUNT_INFO[1] - # Make courses start in the future now = datetime.datetime.now(pytz.UTC) tomorrow = now + datetime.timedelta(days=1) @@ -315,52 +344,53 @@ class TestViewAuth(ModuleStoreTestCase, LoginEnrollmentTestCase): self.test_course = self.update_course(self.test_course, test_course_data) # First, try with an enrolled student - self.login(student_email, student_password) + self.login(self.unenrolled_user) self.assertFalse(self.enroll(self.course)) self.assertTrue(self.enroll(self.test_course)) - # Make the instructor staff in the self.course - instructor_role = CourseInstructorRole(self.course.location) - instructor_role.add_users(User.objects.get(email=instructor_email)) - self.logout() - self.login(instructor_email, instructor_password) + self.login(self.instructor_user) self.assertTrue(self.enroll(self.course)) - # now make the instructor global staff, but not in the instructor group - instructor_role.remove_users(User.objects.get(email=instructor_email)) - GlobalStaff().add_users(User.objects.get(email=instructor_email)) - # unenroll and try again - self.unenroll(self.course) + self.login(self.global_staff_user) self.assertTrue(self.enroll(self.course)) - @patch.dict('courseware.access.settings.MITX_FEATURES', {'DISABLE_START_DATES': False}) - def test_beta_period(self): - """ - Check that beta-test access works. - """ - student_email, student_password = self.ACCOUNT_INFO[0] - instructor_email, instructor_password = self.ACCOUNT_INFO[1] - # Make courses start in the future +@override_settings(MODULESTORE=TEST_DATA_MIXED_MODULESTORE) +class TestBetatesterAccess(ModuleStoreTestCase): + + def setUp(self): + now = datetime.datetime.now(pytz.UTC) tomorrow = now + datetime.timedelta(days=1) - course_data = {'start': tomorrow} - # self.course's hasn't started - self.course = self.update_course(self.course, course_data) + self.course = CourseFactory(days_early_for_beta=2, start=tomorrow) + self.content = ItemFactory(parent=self.course) + + self.normal_student = UserFactory() + self.beta_tester = BetaTesterFactory(course=self.course.location) + + @patch.dict('courseware.access.settings.MITX_FEATURES', {'DISABLE_START_DATES': False}) + def test_course_beta_period(self): + """ + Check that beta-test access works for courses. + """ self.assertFalse(self.course.has_started()) - # but should be accessible for beta testers - self.course.days_early_for_beta = 2 - # student user shouldn't see it - student_user = User.objects.get(email=student_email) - self.assertFalse(has_access(student_user, self.course, 'load')) - - # now add the student to the beta test group - CourseBetaTesterRole(self.course.location).add_users(student_user) + self.assertFalse(has_access(self.normal_student, self.course, 'load')) # now the student should see it - self.assertTrue(has_access(student_user, self.course, 'load')) + self.assertTrue(has_access(self.beta_tester, self.course, 'load')) + + @patch.dict('courseware.access.settings.MITX_FEATURES', {'DISABLE_START_DATES': False}) + def test_content_beta_period(self): + """ + Check that beta-test access works for content. + """ + # student user shouldn't see it + self.assertFalse(has_access(self.normal_student, self.content, 'load', self.course.id)) + + # now the student should see it + self.assertTrue(has_access(self.beta_tester, self.content, 'load', self.course.id)) diff --git a/lms/djangoapps/instructor/tests/test_api.py b/lms/djangoapps/instructor/tests/test_api.py index 6744031c6f..016b5bc39c 100644 --- a/lms/djangoapps/instructor/tests/test_api.py +++ b/lms/djangoapps/instructor/tests/test_api.py @@ -29,7 +29,6 @@ from courseware.models import StudentModule # modules which are mocked in test cases. import instructor_task.api - from instructor.access import allow_access import instructor.views.api from instructor.views.api import _split_input_list, _msk_from_problem_urlname, common_exceptions_400 @@ -128,6 +127,8 @@ class TestInstructorAPIDenyLevels(ModuleStoreTestCase, LoginEnrollmentTestCase): ('send_email', {'send_to': 'staff', 'subject': 'test', 'message': 'asdf'}), ('list_instructor_tasks', {}), ('list_background_email_tasks', {}), + ('list_grade_downloads', {}), + ('calculate_grades_csv', {}), ] # Endpoints that only Instructors can access self.instructor_level_endpoints = [ @@ -790,6 +791,50 @@ class TestInstructorAPILevelsDataDump(ModuleStoreTestCase, LoginEnrollmentTestCa self.assertTrue(body.startswith('"User ID","Anonymized user ID"\n"2","42"\n')) self.assertTrue(body.endswith('"7","42"\n')) + def test_list_grade_downloads(self): + url = reverse('list_grade_downloads', kwargs={'course_id': self.course.id}) + with patch('instructor_task.models.LocalFSGradesStore.links_for') as mock_links_for: + mock_links_for.return_value = [ + ('mock_file_name_1', 'https://1.mock.url'), + ('mock_file_name_2', 'https://2.mock.url'), + ] + response = self.client.get(url, {}) + + expected_response = { + "downloads": [ + { + "url": "https://1.mock.url", + "link": "mock_file_name_1", + "name": "mock_file_name_1" + }, + { + "url": "https://2.mock.url", + "link": "mock_file_name_2", + "name": "mock_file_name_2" + } + ] + } + res_json = json.loads(response.content) + self.assertEqual(res_json, expected_response) + + def test_calculate_grades_csv_success(self): + url = reverse('calculate_grades_csv', kwargs={'course_id': self.course.id}) + + with patch('instructor_task.api.submit_calculate_grades_csv') as mock_cal_grades: + mock_cal_grades.return_value = True + response = self.client.get(url, {}) + success_status = "Your grade report is being generated! You can view the status of the generation task in the 'Pending Instructor Tasks' section." + self.assertIn(success_status, response.content) + + def test_calculate_grades_csv_already_running(self): + url = reverse('calculate_grades_csv', kwargs={'course_id': self.course.id}) + + with patch('instructor_task.api.submit_calculate_grades_csv') as mock_cal_grades: + mock_cal_grades.side_effect = AlreadyRunningError() + response = self.client.get(url, {}) + already_running_status = "A grade report generation task is already in progress. Check the 'Pending Instructor Tasks' table for the status of the task. When completed, the report will be available for download in the table below." + self.assertIn(already_running_status, response.content) + def test_get_students_features_csv(self): """ Test that some minimum of information is formatted @@ -802,7 +847,7 @@ class TestInstructorAPILevelsDataDump(ModuleStoreTestCase, LoginEnrollmentTestCa def test_get_distribution_no_feature(self): """ Test that get_distribution lists available features - when supplied no feature quparameter. + when supplied no feature parameter. """ url = reverse('get_distribution', kwargs={'course_id': self.course.id}) response = self.client.get(url) diff --git a/lms/djangoapps/instructor/views/api.py b/lms/djangoapps/instructor/views/api.py index 5596ba9c41..215f775d77 100644 --- a/lms/djangoapps/instructor/views/api.py +++ b/lms/djangoapps/instructor/views/api.py @@ -32,6 +32,7 @@ from student.models import unique_id_for_user import instructor_task.api from instructor_task.api_helper import AlreadyRunningError from instructor_task.views import get_task_completion_info +from instructor_task.models import GradesStore import instructor.enrollment as enrollment from instructor.enrollment import enroll_email, unenroll_email, get_email_params from instructor.views.tools import strip_if_string, get_student_from_identifier @@ -750,6 +751,42 @@ def list_instructor_tasks(request, course_id): return JsonResponse(response_payload) +@ensure_csrf_cookie +@cache_control(no_cache=True, no_store=True, must_revalidate=True) +@require_level('staff') +def list_grade_downloads(_request, course_id): + """ + List grade CSV files that are available for download for this course. + """ + grades_store = GradesStore.from_config() + + response_payload = { + 'downloads': [ + dict(name=name, url=url, link='{}'.format(url, name)) + for name, url in grades_store.links_for(course_id) + ] + } + return JsonResponse(response_payload) + + +@ensure_csrf_cookie +@cache_control(no_cache=True, no_store=True, must_revalidate=True) +@require_level('staff') +def calculate_grades_csv(request, course_id): + """ + AlreadyRunningError is raised if the course's grades are already being updated. + """ + try: + instructor_task.api.submit_calculate_grades_csv(request, course_id) + success_status = _("Your grade report is being generated! You can view the status of the generation task in the 'Pending Instructor Tasks' section.") + return JsonResponse({"status": success_status}) + except AlreadyRunningError: + already_running_status = _("A grade report generation task is already in progress. Check the 'Pending Instructor Tasks' table for the status of the task. When completed, the report will be available for download in the table below.") + return JsonResponse({ + "status": already_running_status + }) + + @ensure_csrf_cookie @cache_control(no_cache=True, no_store=True, must_revalidate=True) @require_level('staff') diff --git a/lms/djangoapps/instructor/views/api_urls.py b/lms/djangoapps/instructor/views/api_urls.py index 3f05f1cc70..f016b9820f 100644 --- a/lms/djangoapps/instructor/views/api_urls.py +++ b/lms/djangoapps/instructor/views/api_urls.py @@ -37,4 +37,10 @@ urlpatterns = patterns('', # nopep8 'instructor.views.api.proxy_legacy_analytics', name="proxy_legacy_analytics"), url(r'^send_email$', 'instructor.views.api.send_email', name="send_email"), + + # Grade downloads... + url(r'^list_grade_downloads$', + 'instructor.views.api.list_grade_downloads', name="list_grade_downloads"), + url(r'calculate_grades_csv$', + 'instructor.views.api.calculate_grades_csv', name="calculate_grades_csv"), ) diff --git a/lms/djangoapps/instructor/views/instructor_dashboard.py b/lms/djangoapps/instructor/views/instructor_dashboard.py index 7c3fd32a0a..de14df2940 100644 --- a/lms/djangoapps/instructor/views/instructor_dashboard.py +++ b/lms/djangoapps/instructor/views/instructor_dashboard.py @@ -171,6 +171,8 @@ def _section_data_download(course_id, access): 'get_students_features_url': reverse('get_students_features', kwargs={'course_id': course_id}), 'get_anon_ids_url': reverse('get_anon_ids', kwargs={'course_id': course_id}), 'list_instructor_tasks_url': reverse('list_instructor_tasks', kwargs={'course_id': course_id}), + 'list_grade_downloads_url': reverse('list_grade_downloads', kwargs={'course_id': course_id}), + 'calculate_grades_csv_url': reverse('calculate_grades_csv', kwargs={'course_id': course_id}), } return section_data diff --git a/lms/djangoapps/instructor_task/api.py b/lms/djangoapps/instructor_task/api.py index 7521a8eb3a..73af3e4378 100644 --- a/lms/djangoapps/instructor_task/api.py +++ b/lms/djangoapps/instructor_task/api.py @@ -16,7 +16,8 @@ from instructor_task.models import InstructorTask from instructor_task.tasks import (rescore_problem, reset_problem_attempts, delete_problem_state, - send_bulk_course_email) + send_bulk_course_email, + calculate_grades_csv) from instructor_task.api_helper import (check_arguments_for_rescoring, encode_problem_and_student_input, @@ -206,3 +207,15 @@ def submit_bulk_course_email(request, course_id, email_id): # create the key value by using MD5 hash: task_key = hashlib.md5(task_key_stub).hexdigest() return submit_task(request, task_type, task_class, course_id, task_input, task_key) + + +def submit_calculate_grades_csv(request, course_id): + """ + AlreadyRunningError is raised if the course's grades are already being updated. + """ + task_type = 'grade_course' + task_class = calculate_grades_csv + task_input = {} + task_key = "" + + return submit_task(request, task_type, task_class, course_id, task_input, task_key) diff --git a/lms/djangoapps/instructor_task/models.py b/lms/djangoapps/instructor_task/models.py index 8d6376fae3..8df6a95c71 100644 --- a/lms/djangoapps/instructor_task/models.py +++ b/lms/djangoapps/instructor_task/models.py @@ -12,9 +12,20 @@ file and check it in at the same time as your model changes. To do that, ASSUMPTIONS: modules have unique IDs, even across different module_types """ +from cStringIO import StringIO +from gzip import GzipFile from uuid import uuid4 +import csv import json +import hashlib +import os +import os.path +import urllib +from boto.s3.connection import S3Connection +from boto.s3.key import Key + +from django.conf import settings from django.contrib.auth.models import User from django.db import models, transaction @@ -176,3 +187,220 @@ class InstructorTask(models.Model): def create_output_for_revoked(): """Creates standard message to store in output format for revoked tasks.""" return json.dumps({'message': 'Task revoked before running'}) + + +class GradesStore(object): + """ + Simple abstraction layer that can fetch and store CSV files for grades + download. Should probably refactor later to create a GradesFile object that + can simply be appended to for the sake of memory efficiency, rather than + passing in the whole dataset. Doing that for now just because it's simpler. + """ + @classmethod + def from_config(cls): + """ + Return one of the GradesStore subclasses depending on django + configuration. Look at subclasses for expected configuration. + """ + storage_type = settings.GRADES_DOWNLOAD.get("STORAGE_TYPE") + if storage_type.lower() == "s3": + return S3GradesStore.from_config() + elif storage_type.lower() == "localfs": + return LocalFSGradesStore.from_config() + + +class S3GradesStore(GradesStore): + """ + Grades store backed by S3. The directory structure we use to store things + is:: + + `{bucket}/{root_path}/{sha1 hash of course_id}/filename` + + We might later use subdirectories or metadata to do more intelligent + grouping and querying, but right now it simply depends on its own + conventions on where files are stored to know what to display. Clients using + this class can name the final file whatever they want. + """ + def __init__(self, bucket_name, root_path): + self.root_path = root_path + + conn = S3Connection( + settings.AWS_ACCESS_KEY_ID, + settings.AWS_SECRET_ACCESS_KEY + ) + self.bucket = conn.get_bucket(bucket_name) + + @classmethod + def from_config(cls): + """ + The expected configuration for an `S3GradesStore` is to have a + `GRADES_DOWNLOAD` dict in settings with the following fields:: + + STORAGE_TYPE : "s3" + BUCKET : Your bucket name, e.g. "grades-bucket" + ROOT_PATH : The path you want to store all course files under. Do not + use a leading or trailing slash. e.g. "staging" or + "staging/2013", not "/staging", or "/staging/" + + Since S3 access relies on boto, you must also define `AWS_ACCESS_KEY_ID` + and `AWS_SECRET_ACCESS_KEY` in settings. + """ + return cls( + settings.GRADES_DOWNLOAD['BUCKET'], + settings.GRADES_DOWNLOAD['ROOT_PATH'] + ) + + def key_for(self, course_id, filename): + """Return the S3 key we would use to store and retrive the data for the + given filename.""" + hashed_course_id = hashlib.sha1(course_id) + + key = Key(self.bucket) + key.key = "{}/{}/{}".format( + self.root_path, + hashed_course_id.hexdigest(), + filename + ) + + return key + + def store(self, course_id, filename, buff): + """ + Store the contents of `buff` in a directory determined by hashing + `course_id`, and name the file `filename`. `buff` is typically a + `StringIO`, but can be anything that implements `.getvalue()`. + + This method assumes that the contents of `buff` are gzip-encoded (it + will add the appropriate headers to S3 to make the decompression + transparent via the browser). Filenames should end in whatever + suffix makes sense for the original file, so `.txt` instead of `.gz` + """ + key = self.key_for(course_id, filename) + + data = buff.getvalue() + key.size = len(data) + key.content_encoding = "gzip" + key.content_type = "text/csv" + + # Just setting the content encoding and type above should work + # according to the docs, but when experimenting, this was necessary for + # it to actually take. + key.set_contents_from_string( + data, + headers={ + "Content-Encoding": "gzip", + "Content-Length": len(data), + "Content-Type": "text/csv", + } + ) + + def store_rows(self, course_id, filename, rows): + """ + Given a `course_id`, `filename`, and `rows` (each row is an iterable of + strings), create a buffer that is a gzip'd csv file, and then `store()` + that buffer. + + Even though we store it in gzip format, browsers will transparently + download and decompress it. Filenames should end in `.csv`, not `.gz`. + """ + output_buffer = StringIO() + gzip_file = GzipFile(fileobj=output_buffer, mode="wb") + csv.writer(gzip_file).writerows(rows) + gzip_file.close() + + self.store(course_id, filename, output_buffer) + + def links_for(self, course_id): + """ + For a given `course_id`, return a list of `(filename, url)` tuples. `url` + can be plugged straight into an href + """ + course_dir = self.key_for(course_id, '') + return sorted( + [ + (key.key.split("/")[-1], key.generate_url(expires_in=300)) + for key in self.bucket.list(prefix=course_dir.key) + ], + reverse=True + ) + + +class LocalFSGradesStore(GradesStore): + """ + LocalFS implementation of a GradesStore. This is meant for debugging + purposes and is *absolutely not for production use*. Use S3GradesStore for + that. We use this in tests and for local development. When it generates + links, it will make file:/// style links. That means you actually have to + copy them and open them in a separate browser window, for security reasons. + This lets us do the cheap thing locally for debugging without having to open + up a separate URL that would only be used to send files in dev. + """ + def __init__(self, root_path): + """ + Initialize with root_path where we're going to store our files. We + will build a directory structure under this for each course. + """ + self.root_path = root_path + if not os.path.exists(root_path): + os.makedirs(root_path) + + @classmethod + def from_config(cls): + """ + Generate an instance of this object from Django settings. It assumes + that there is a dict in settings named GRADES_DOWNLOAD and that it has + a ROOT_PATH that maps to an absolute file path that the web app has + write permissions to. `LocalFSGradesStore` will create any intermediate + directories as needed. Example:: + + STORAGE_TYPE : "localfs" + ROOT_PATH : /tmp/edx/grade-downloads/ + """ + return cls(settings.GRADES_DOWNLOAD['ROOT_PATH']) + + def path_to(self, course_id, filename): + """Return the full path to a given file for a given course.""" + return os.path.join(self.root_path, urllib.quote(course_id, safe=''), filename) + + def store(self, course_id, filename, buff): + """ + Given the `course_id` and `filename`, store the contents of `buff` in + that file. Overwrite anything that was there previously. `buff` is + assumed to be a StringIO objecd (or anything that can flush its contents + to string using `.getvalue()`). + """ + full_path = self.path_to(course_id, filename) + directory = os.path.dirname(full_path) + if not os.path.exists(directory): + os.mkdir(directory) + + with open(full_path, "wb") as f: + f.write(buff.getvalue()) + + def store_rows(self, course_id, filename, rows): + """ + Given a course_id, filename, and rows (each row is an iterable of strings), + write this data out. + """ + output_buffer = StringIO() + csv.writer(output_buffer).writerows(rows) + self.store(course_id, filename, output_buffer) + + def links_for(self, course_id): + """ + For a given `course_id`, return a list of `(filename, url)` tuples. `url` + can be plugged straight into an href. Note that `LocalFSGradesStore` + will generate `file://` type URLs, so you'll need to copy the URL and + open it in a new browser window. Again, this class is only meant for + local development. + """ + course_dir = self.path_to(course_id, '') + if not os.path.exists(course_dir): + return [] + return sorted( + [ + (filename, ("file://" + urllib.quote(os.path.join(course_dir, filename)))) + for filename in os.listdir(course_dir) + ], + reverse=True + ) diff --git a/lms/djangoapps/instructor_task/tasks.py b/lms/djangoapps/instructor_task/tasks.py index f30ffe3af2..69e8ee1865 100644 --- a/lms/djangoapps/instructor_task/tasks.py +++ b/lms/djangoapps/instructor_task/tasks.py @@ -19,6 +19,7 @@ a problem URL and optionally a student. These are used to set up the initial va of the query for traversing StudentModule objects. """ +from django.conf import settings from django.utils.translation import ugettext_noop from celery import task from functools import partial @@ -29,6 +30,7 @@ from instructor_task.tasks_helper import ( rescore_problem_module_state, reset_attempts_module_state, delete_problem_module_state, + push_grades_to_s3, ) from bulk_email.tasks import perform_delegate_email_batches @@ -127,3 +129,13 @@ def send_bulk_course_email(entry_id, _xmodule_instance_args): action_name = ugettext_noop('emailed') visit_fcn = perform_delegate_email_batches return run_main_task(entry_id, visit_fcn, action_name) + + +@task(base=BaseInstructorTask, routing_key=settings.GRADES_DOWNLOAD_ROUTING_KEY) # pylint: disable=E1102 +def calculate_grades_csv(entry_id, xmodule_instance_args): + """ + Grade a course and push the results to an S3 bucket for download. + """ + action_name = ugettext_noop('graded') + task_fn = partial(push_grades_to_s3, xmodule_instance_args) + return run_main_task(entry_id, task_fn, action_name) diff --git a/lms/djangoapps/instructor_task/tasks_helper.py b/lms/djangoapps/instructor_task/tasks_helper.py index cf828edb5b..06941e7fdf 100644 --- a/lms/djangoapps/instructor_task/tasks_helper.py +++ b/lms/djangoapps/instructor_task/tasks_helper.py @@ -4,24 +4,27 @@ running state of a course. """ import json +import urllib +from datetime import datetime from time import time from celery import Task, current_task from celery.utils.log import get_task_logger from celery.states import SUCCESS, FAILURE - from django.contrib.auth.models import User from django.db import transaction, reset_queries from dogapi import dog_stats_api +from pytz import UTC from xmodule.modulestore.django import modulestore - from track.views import task_track +from courseware.grades import iterate_grades_for from courseware.models import StudentModule from courseware.model_data import FieldDataCache from courseware.module_render import get_module_for_descriptor_internal -from instructor_task.models import InstructorTask, PROGRESS +from instructor_task.models import GradesStore, InstructorTask, PROGRESS +from student.models import CourseEnrollment # define different loggers for use within tasks and on client side TASK_LOG = get_task_logger(__name__) @@ -465,3 +468,106 @@ def delete_problem_module_state(xmodule_instance_args, _module_descriptor, stude track_function = _get_track_function_for_task(student_module.student, xmodule_instance_args) track_function('problem_delete_state', {}) return UPDATE_STATUS_SUCCEEDED + + +def push_grades_to_s3(_xmodule_instance_args, _entry_id, course_id, _task_input, action_name): + """ + For a given `course_id`, generate a grades CSV file for all students that + are enrolled, and store using a `GradesStore`. Once created, the files can + be accessed by instantiating another `GradesStore` (via + `GradesStore.from_config()`) and calling `link_for()` on it. Writes are + buffered, so we'll never write part of a CSV file to S3 -- i.e. any files + that are visible in GradesStore will be complete ones. + + As we start to add more CSV downloads, it will probably be worthwhile to + make a more general CSVDoc class instead of building out the rows like we + do here. + """ + start_time = datetime.now(UTC) + status_interval = 100 + + enrolled_students = CourseEnrollment.users_enrolled_in(course_id) + num_total = enrolled_students.count() + num_attempted = 0 + num_succeeded = 0 + num_failed = 0 + curr_step = "Calculating Grades" + + def update_task_progress(): + """Return a dict containing info about current task""" + current_time = datetime.now(UTC) + progress = { + 'action_name': action_name, + 'attempted': num_attempted, + 'succeeded': num_succeeded, + 'failed': num_failed, + 'total': num_total, + 'duration_ms': int((current_time - start_time).total_seconds() * 1000), + 'step': curr_step, + } + _get_current_task().update_state(state=PROGRESS, meta=progress) + + return progress + + # Loop over all our students and build our CSV lists in memory + header = None + rows = [] + err_rows = [["id", "username", "error_msg"]] + for student, gradeset, err_msg in iterate_grades_for(course_id, enrolled_students): + # Periodically update task status (this is a cache write) + if num_attempted % status_interval == 0: + update_task_progress() + num_attempted += 1 + + if gradeset: + # We were able to successfully grade this student for this course. + num_succeeded += 1 + if not header: + header = [section['label'] for section in gradeset[u'section_breakdown']] + rows.append(["id", "email", "username", "grade"] + header) + + percents = { + section['label']: section.get('percent', 0.0) + for section in gradeset[u'section_breakdown'] + if 'label' in section + } + + # Not everybody has the same gradable items. If the item is not + # found in the user's gradeset, just assume it's a 0. The aggregated + # grades for their sections and overall course will be calculated + # without regard for the item they didn't have access to, so it's + # possible for a student to have a 0.0 show up in their row but + # still have 100% for the course. + row_percents = [percents.get(label, 0.0) for label in header] + rows.append([student.id, student.email, student.username, gradeset['percent']] + row_percents) + else: + # An empty gradeset means we failed to grade a student. + num_failed += 1 + err_rows.append([student.id, student.username, err_msg]) + + # By this point, we've got the rows we're going to stuff into our CSV files. + curr_step = "Uploading CSVs" + update_task_progress() + + # Generate parts of the file name + timestamp_str = start_time.strftime("%Y-%m-%d-%H%M") + course_id_prefix = urllib.quote(course_id.replace("/", "_")) + + # Perform the actual upload + grades_store = GradesStore.from_config() + grades_store.store_rows( + course_id, + "{}_grade_report_{}.csv".format(course_id_prefix, timestamp_str), + rows + ) + + # If there are any error rows (don't count the header), write them out as well + if len(err_rows) > 1: + grades_store.store_rows( + course_id, + "{}_grade_report_{}_err.csv".format(course_id_prefix, timestamp_str), + err_rows + ) + + # One last update before we close out... + return update_task_progress() diff --git a/lms/djangoapps/instructor_task/views.py b/lms/djangoapps/instructor_task/views.py index 9a23841425..a8e7eade0b 100644 --- a/lms/djangoapps/instructor_task/views.py +++ b/lms/djangoapps/instructor_task/views.py @@ -75,7 +75,6 @@ def instructor_task_status(request): 'traceback': optional, returned if task failed and produced a traceback. """ - output = {} if 'task_id' in request.REQUEST: task_id = request.REQUEST['task_id'] diff --git a/lms/envs/aws.py b/lms/envs/aws.py index 0827e91b6d..315b99e346 100644 --- a/lms/envs/aws.py +++ b/lms/envs/aws.py @@ -86,6 +86,7 @@ CELERY_DEFAULT_EXCHANGE = 'edx.{0}core'.format(QUEUE_VARIANT) HIGH_PRIORITY_QUEUE = 'edx.{0}core.high'.format(QUEUE_VARIANT) DEFAULT_PRIORITY_QUEUE = 'edx.{0}core.default'.format(QUEUE_VARIANT) LOW_PRIORITY_QUEUE = 'edx.{0}core.low'.format(QUEUE_VARIANT) +HIGH_MEM_QUEUE = 'edx.{0}core.high_mem'.format(QUEUE_VARIANT) CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE @@ -93,9 +94,19 @@ CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE CELERY_QUEUES = { HIGH_PRIORITY_QUEUE: {}, LOW_PRIORITY_QUEUE: {}, - DEFAULT_PRIORITY_QUEUE: {} + DEFAULT_PRIORITY_QUEUE: {}, + HIGH_MEM_QUEUE: {}, } +# If we're a worker on the high_mem queue, set ourselves to die after processing +# one request to avoid having memory leaks take down the worker server. This env +# var is set in /etc/init/edx-workers.conf -- this should probably be replaced +# with some celery API call to see what queue we started listening to, but I +# don't know what that call is or if it's active at this point in the code. +if os.environ.get('QUEUE') == 'high_mem': + CELERYD_MAX_TASKS_PER_CHILD = 1 + + ########################## NON-SECURE ENV CONFIG ############################## # Things like server locations, ports, etc. @@ -318,3 +329,8 @@ TRACKING_BACKENDS.update(AUTH_TOKENS.get("TRACKING_BACKENDS", {})) # Student identity verification settings VERIFY_STUDENT = AUTH_TOKENS.get("VERIFY_STUDENT", VERIFY_STUDENT) + +# Grades download +GRADES_DOWNLOAD_ROUTING_KEY = HIGH_MEM_QUEUE + +GRADES_DOWNLOAD = ENV_TOKENS.get("GRADES_DOWNLOAD", GRADES_DOWNLOAD) diff --git a/lms/envs/common.py b/lms/envs/common.py index c698ce24be..2fdfe3325d 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -192,6 +192,14 @@ MITX_FEATURES = { # Disable instructor dash buttons for downloading course data # when enrollment exceeds this number 'MAX_ENROLLMENT_INSTR_BUTTONS': 200, + + # Grade calculation started from the new instructor dashboard will write + # grades CSV files to S3 and give links for downloads. + 'ENABLE_S3_GRADE_DOWNLOADS': False, + + # Give course staff unrestricted access to grade downloads (if set to False, + # only edX superusers can perform the downloads) + 'ALLOW_COURSE_STAFF_GRADE_DOWNLOADS': False, } # Used for A/B testing @@ -846,6 +854,7 @@ CELERY_DEFAULT_EXCHANGE_TYPE = 'direct' HIGH_PRIORITY_QUEUE = 'edx.core.high' DEFAULT_PRIORITY_QUEUE = 'edx.core.default' LOW_PRIORITY_QUEUE = 'edx.core.low' +HIGH_MEM_QUEUE = 'edx.core.high_mem' CELERY_QUEUE_HA_POLICY = 'all' @@ -857,7 +866,8 @@ CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE CELERY_QUEUES = { HIGH_PRIORITY_QUEUE: {}, LOW_PRIORITY_QUEUE: {}, - DEFAULT_PRIORITY_QUEUE: {} + DEFAULT_PRIORITY_QUEUE: {}, + HIGH_MEM_QUEUE: {}, } # let logging work as configured: @@ -1061,3 +1071,12 @@ REGISTRATION_OPTIONAL_FIELDS = set([ 'mailing_address', 'goals', ]) + +###################### Grade Downloads ###################### +GRADES_DOWNLOAD_ROUTING_KEY = HIGH_MEM_QUEUE + +GRADES_DOWNLOAD = { + 'STORAGE_TYPE': 'localfs', + 'BUCKET': 'edx-grades', + 'ROOT_PATH': '/tmp/edx-s3/grades', +} diff --git a/lms/envs/dev.py b/lms/envs/dev.py index 1fe7fc330c..8fe5e7a19c 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -40,6 +40,7 @@ MITX_FEATURES['ENABLE_INSTRUCTOR_BETA_DASHBOARD'] = True MITX_FEATURES['MULTIPLE_ENROLLMENT_ROLES'] = True MITX_FEATURES['ENABLE_SHOPPING_CART'] = True MITX_FEATURES['AUTOMATIC_VERIFY_STUDENT_IDENTITY_FOR_TESTING'] = True +MITX_FEATURES['ENABLE_S3_GRADE_DOWNLOADS'] = True FEEDBACK_SUBMISSION_EMAIL = "dummy@example.com" diff --git a/lms/static/coffee/src/instructor_dashboard/data_download.coffee b/lms/static/coffee/src/instructor_dashboard/data_download.coffee index e6108b1055..22bda4151d 100644 --- a/lms/static/coffee/src/instructor_dashboard/data_download.coffee +++ b/lms/static/coffee/src/instructor_dashboard/data_download.coffee @@ -17,14 +17,23 @@ class DataDownload # this object to call event handlers like 'onClickTitle' @$section.data 'wrapper', @ # gather elements - @$display = @$section.find '.data-display' - @$display_text = @$display.find '.data-display-text' - @$display_table = @$display.find '.data-display-table' - @$request_response_error = @$display.find '.request-response-error' @$list_studs_btn = @$section.find("input[name='list-profiles']'") @$list_anon_btn = @$section.find("input[name='list-anon-ids']'") @$grade_config_btn = @$section.find("input[name='dump-gradeconf']'") + @$calculate_grades_csv_btn = @$section.find("input[name='calculate-grades-csv']'") + + # response areas + @$download = @$section.find '.data-download-container' + @$download_display_text = @$download.find '.data-display-text' + @$download_display_table = @$download.find '.data-display-table' + @$download_request_response_error = @$download.find '.request-response-error' + @$grades = @$section.find '.grades-download-container' + @$grades_request_response = @$grades.find '.request-response' + @$grades_request_response_error = @$grades.find '.request-response-error' + + @grade_downloads = new GradeDownloads(@$section) @instructor_tasks = new (PendingInstructorTasks()) @$section + @clear_display() # attach click handlers # The list-anon case is always CSV @@ -43,8 +52,9 @@ class DataDownload url += '/csv' location.href = url else + # Dynamically generate slickgrid table for displaying student profile information @clear_display() - @$display_table.text 'Loading...' + @$download_display_table.text gettext('Loading...') # fetch user list $.ajax @@ -52,7 +62,7 @@ class DataDownload url: url error: std_ajax_err => @clear_display() - @$request_response_error.text "Error getting student list." + @$download_request_response_error.text gettext("Error getting student list.") success: (data) => @clear_display() @@ -61,12 +71,13 @@ class DataDownload enableCellNavigation: true enableColumnReorder: false forceFitColumns: true + rowHeight: 35 columns = ({id: feature, field: feature, name: feature} for feature in data.queried_features) grid_data = data.students $table_placeholder = $ '
', class: 'slickgrid' - @$display_table.append $table_placeholder + @$download_display_table.append $table_placeholder grid = new Slick.Grid($table_placeholder, grid_data, columns, options) # grid.autosizeColumns() @@ -78,21 +89,103 @@ class DataDownload url: url error: std_ajax_err => @clear_display() - @$request_response_error.text "Error getting grading configuration." + @$download_request_response_error.text gettext("Error retrieving grading configuration.") success: (data) => @clear_display() - @$display_text.html data['grading_config_summary'] + @$download_display_text.html data['grading_config_summary'] + + @$calculate_grades_csv_btn.click (e) => + # Clear any CSS styling from the request-response areas + #$(".msg-confirm").css({"display":"none"}) + #$(".msg-error").css({"display":"none"}) + @clear_display() + url = @$calculate_grades_csv_btn.data 'endpoint' + $.ajax + dataType: 'json' + url: url + error: std_ajax_err => + @$grades_request_response_error.text gettext("Error generating grades. Please try again.") + $(".msg-error").css({"display":"block"}) + success: (data) => + @$grades_request_response.text data['status'] + $(".msg-confirm").css({"display":"block"}) # handler for when the section title is clicked. - onClickTitle: -> @instructor_tasks.task_poller.start() + onClickTitle: -> + # Clear display of anything that was here before + @clear_display() + @instructor_tasks.task_poller.start() + @grade_downloads.downloads_poller.start() # handler for when the section is closed - onExit: -> @instructor_tasks.task_poller.stop() + onExit: -> + @instructor_tasks.task_poller.stop() + @grade_downloads.downloads_poller.stop() clear_display: -> - @$display_text.empty() - @$display_table.empty() - @$request_response_error.empty() + # Clear any generated tables, warning messages, etc. + @$download_display_text.empty() + @$download_display_table.empty() + @$download_request_response_error.empty() + @$grades_request_response.empty() + @$grades_request_response_error.empty() + # Clear any CSS styling from the request-response areas + $(".msg-confirm").css({"display":"none"}) + $(".msg-error").css({"display":"none"}) + + +class GradeDownloads + ### Grade Downloads -- links expire quickly, so we refresh every 5 mins #### + constructor: (@$section) -> + + + @$grades = @$section.find '.grades-download-container' + @$grades_request_response = @$grades.find '.request-response' + @$grades_request_response_error = @$grades.find '.request-response-error' + @$grade_downloads_table = @$grades.find ".grade-downloads-table" + + POLL_INTERVAL = 1000 * 60 * 5 # 5 minutes in ms + @downloads_poller = new window.InstructorDashboard.util.IntervalManager( + POLL_INTERVAL, => @reload_grade_downloads() + ) + + reload_grade_downloads: -> + endpoint = @$grade_downloads_table.data 'endpoint' + $.ajax + dataType: 'json' + url: endpoint + success: (data) => + if data.downloads.length + @create_grade_downloads_table data.downloads + else + console.log "No grade CSVs ready for download" + error: std_ajax_err => console.error "Error finding grade download CSVs" + + create_grade_downloads_table: (grade_downloads_data) -> + @$grade_downloads_table.empty() + + options = + enableCellNavigation: true + enableColumnReorder: false + rowHeight: 30 + forceFitColumns: true + + columns = [ + id: 'link' + field: 'link' + name: gettext('File Name (Newest First)') + toolTip: gettext("Links are generated on demand and expire within 5 minutes due to the sensitive nature of student grade information.") + sortable: false + minWidth: 150 + cssClass: "file-download-link" + formatter: (row, cell, value, columnDef, dataContext) -> + '' + dataContext['name'] + '' + ] + + $table_placeholder = $ '
', class: 'slickgrid' + @$grade_downloads_table.append $table_placeholder + grid = new Slick.Grid($table_placeholder, grade_downloads_data, columns, options) + grid.autosizeColumns() # export for use diff --git a/lms/static/coffee/src/instructor_dashboard/util.coffee b/lms/static/coffee/src/instructor_dashboard/util.coffee index af5b99f419..14a2f2b8ca 100644 --- a/lms/static/coffee/src/instructor_dashboard/util.coffee +++ b/lms/static/coffee/src/instructor_dashboard/util.coffee @@ -43,7 +43,7 @@ create_task_list_table = ($table_tasks, tasks_data) -> id: 'task_type' field: 'task_type' name: 'Task Type' - minWidth: 100 + minWidth: 102 , id: 'task_input' field: 'task_input' diff --git a/lms/static/sass/course/instructor/_instructor_2.scss b/lms/static/sass/course/instructor/_instructor_2.scss index d1a04ce185..2f8eb8947e 100644 --- a/lms/static/sass/course/instructor/_instructor_2.scss +++ b/lms/static/sass/course/instructor/_instructor_2.scss @@ -26,6 +26,13 @@ @include font-size(16); } + .file-download-link a { + font-size: 15px; + color: $link-color; + text-decoration: underline; + padding: 5px; + } + // system feedback - messages .msg { border-radius: 1px; @@ -117,7 +124,7 @@ section.instructor-dashboard-content-2 { .slickgrid { margin-left: 1px; color:#333333; - font-size:11px; + font-size:12px; font-family: verdana,arial,sans-serif; .slick-header-column { @@ -428,13 +435,26 @@ section.instructor-dashboard-content-2 { line-height: 1.3em; } - .data-display { + .data-download-container { .data-display-table { .slickgrid { height: 400px; } } } + + .grades-download-container { + .grade-downloads-table { + .slickgrid { + height: 300px; + padding: 5px; + } + // Disable horizontal scroll bar when grid only has 1 column. Remove this CSS class when more columns added. + .slick-viewport { + overflow-x: hidden !important; + } + } + } } diff --git a/lms/templates/instructor/instructor_dashboard_2/data_download.html b/lms/templates/instructor/instructor_dashboard_2/data_download.html index b57fd7b30e..cbbc2a871b 100644 --- a/lms/templates/instructor/instructor_dashboard_2/data_download.html +++ b/lms/templates/instructor/instructor_dashboard_2/data_download.html @@ -2,23 +2,49 @@ <%page args="section_data"/> -

${_("Data Download")}

+
+

${_("Data Download")}

+
- - -
-## -## -##
-## -##
- - +

${_("The following button displays a list of all students enrolled in this course, along with profile information such as email address and username. The data can also be downloaded as a CSV file.")}

-
-
+

+

-
+ +
+

${_("Displays the grading configuration for the course. The grading configuration is the breakdown of graded subsections of the course (such as exams and problem sets), and can be changed on the 'Grading' page (under 'Settings') in Studio.")}

+

+ +
+
+ +

${_("Download a CSV of anonymized student IDs by clicking this button.")}

+

+
+ +%if settings.MITX_FEATURES.get('ENABLE_S3_GRADE_DOWNLOADS'): +
+
+

${_("Grade Reports")}

+ + %if settings.MITX_FEATURES.get('ALLOW_COURSE_STAFF_GRADE_DOWNLOADS') or section_data['access']['admin']: +

${_("The following button will generate a CSV grade report for all currently enrolled students. For large courses, generating this report may take a few hours.")}

+ +

${_("The report is generated in the background, meaning it is OK to navigate away from this page while your report is generating. Generated reports appear in a table below and can be downloaded.")}

+ +
+
+
+ +

+ %endif + +

${_("Reports Available for Download")}

+

${_("File links are generated on demand and expire within 5 minutes due to the sensitive nature of student grade information. Please note that the report filename contains a timestamp that represents when your file was generated; this timestamp is UTC, not your local timezone.")}


+
+
+%endif %if settings.MITX_FEATURES.get('ENABLE_INSTRUCTOR_BACKGROUND_TASKS'):
@@ -26,9 +52,6 @@

${_("Pending Instructor Tasks")}

${_("The status for any active tasks appears in a table below.")}


-
- %endif -
diff --git a/lms/templates/instructor/instructor_dashboard_2/student_admin.html b/lms/templates/instructor/instructor_dashboard_2/student_admin.html index 4d60810008..2762ba4899 100644 --- a/lms/templates/instructor/instructor_dashboard_2/student_admin.html +++ b/lms/templates/instructor/instructor_dashboard_2/student_admin.html @@ -5,7 +5,6 @@

${_("Student-specific grade inspection")}

- ${_("Specify the {platform_name} email address or username of a student here:").format(platform_name=settings.PLATFORM_NAME)}

@@ -26,7 +25,6 @@

${_("Student-specific grade adjustment")}

- ${_("Specify the {platform_name} email address or username of a student here:").format(platform_name=settings.PLATFORM_NAME)}

@@ -60,18 +58,18 @@

%if section_data['access']['instructor']:

${_('You may also delete the entire state of a student for the specified problem:')}

- +

%endif

%if settings.MITX_FEATURES.get('ENABLE_INSTRUCTOR_BACKGROUND_TASKS') and section_data['access']['instructor']:

- ${_("Rescoring runs in the background, and status for active tasks will appear in a table on the Course Info tab. " + ${_("Rescoring runs in the background, and status for active tasks will appear in the 'Pending Instructor Tasks' table. " "To see status for all tasks submitted for this problem and student, click on this button:")}

- +

%endif
@@ -101,10 +99,10 @@

- ${_("These actions run in the background, and status for active tasks will appear in a table on the Course Info tab. " + ${_("The above actions run in the background, and status for active tasks will appear in a table on the Course Info tab. " "To see status for all tasks submitted for this problem, click on this button")}:

- +