diff --git a/lms/djangoapps/grades/management/commands/get_grades.py b/lms/djangoapps/grades/management/commands/get_grades.py deleted file mode 100644 index 2cfe4a8cf6..0000000000 --- a/lms/djangoapps/grades/management/commands/get_grades.py +++ /dev/null @@ -1,136 +0,0 @@ -""" -Management command to generate a list of grades for -all students that are enrolled in a course. -""" -import csv -import datetime -import os -from optparse import make_option - -from django.contrib.auth.models import User -from django.core.handlers.base import BaseHandler -from django.core.management.base import BaseCommand, CommandError -from django.test.client import RequestFactory -from opaque_keys.edx.keys import CourseKey - -from lms.djangoapps.certificates.models import GeneratedCertificate -from lms.djangoapps.courseware import courses -from lms.djangoapps.grades.new.course_grade_factory import CourseGradeFactory - - -class RequestMock(RequestFactory): - """ - Class to create a mock request. - """ - def request(self, **request): - "Construct a generic request object." - request = RequestFactory.request(self, **request) - handler = BaseHandler() - handler.load_middleware() - for middleware_method in handler._request_middleware: # pylint: disable=protected-access - if middleware_method(request): - raise Exception("Couldn't create request mock object - " - "request middleware returned a response") - return request - - -class Command(BaseCommand): - """ - Management command for get_grades - """ - - help = """ - Generate a list of grades for all students - that are enrolled in a course. - - CSV will include the following: - - username - - email - - grade in the certificate table if it exists - - computed grade - - grade breakdown - - Outputs grades to a csv file. - - Example: - sudo -u www-data SERVICE_VARIANT=lms /opt/edx/bin/django-admin.py get_grades \ - -c MITx/Chi6.00intro/A_Taste_of_Python_Programming -o /tmp/20130813-6.00x.csv \ - --settings=lms.envs.aws --pythonpath=/opt/wwc/edx-platform - """ - - option_list = BaseCommand.option_list + ( - make_option('-c', '--course', - metavar='COURSE_ID', - dest='course', - default=False, - help='Course ID for grade distribution'), - make_option('-o', '--output', - metavar='FILE', - dest='output', - default=False, - help='Filename for grade output')) - - def handle(self, *args, **options): - if os.path.exists(options['output']): - raise CommandError("File {0} already exists".format( - options['output'])) - - status_interval = 100 - - # parse out the course into a coursekey - if options['course']: - course_key = CourseKey.from_string(options['course']) - - print "Fetching enrolled students for {0}".format(course_key) - enrolled_students = User.objects.filter( - courseenrollment__course_id=course_key - ) - factory = RequestMock() - request = factory.get('/') - - total = enrolled_students.count() - print "Total enrolled: {0}".format(total) - course = courses.get_course_by_id(course_key) - total = enrolled_students.count() - start = datetime.datetime.now() - rows = [] - header = None - print "Fetching certificate data" - cert_grades = { - cert.user.username: cert.grade - for cert in list( - GeneratedCertificate.objects.filter( # pylint: disable=no-member - course_id=course_key - ).prefetch_related('user') - ) - } - print "Grading students" - for count, student in enumerate(enrolled_students): - count += 1 - if count % status_interval == 0: - # Print a status update with an approximation of - # how much time is left based on how long the last - # interval took - diff = datetime.datetime.now() - start - timeleft = diff * (total - count) / status_interval - hours, remainder = divmod(timeleft.seconds, 3600) - minutes, __ = divmod(remainder, 60) - print "{0}/{1} completed ~{2:02}:{3:02}m remaining".format( - count, total, hours, minutes) - start = datetime.datetime.now() - request.user = student - grade = CourseGradeFactory().create(student, course) - if not header: - header = [section['label'] for section in grade.summary[u'section_breakdown']] - rows.append(["email", "username", "certificate-grade", "grade"] + header) - percents = {section['label']: section['percent'] for section in grade.summary[u'section_breakdown']} - row_percents = [percents[label] for label in header] - if student.username in cert_grades: - rows.append( - [student.email, student.username, cert_grades[student.username], grade.percent] + row_percents, - ) - else: - rows.append([student.email, student.username, "N/A", grade.percent] + row_percents) - with open(options['output'], 'wb') as f: - writer = csv.writer(f) - writer.writerows(rows) diff --git a/lms/djangoapps/grades/management/commands/reset_grades.py b/lms/djangoapps/grades/management/commands/reset_grades.py deleted file mode 100644 index b22a100c1f..0000000000 --- a/lms/djangoapps/grades/management/commands/reset_grades.py +++ /dev/null @@ -1,144 +0,0 @@ -""" -Reset persistent grades for learners. -""" -import logging -from datetime import datetime -from textwrap import dedent - -from django.core.management.base import BaseCommand, CommandError -from django.db.models import Count -from pytz import utc - -from lms.djangoapps.grades.models import PersistentCourseGrade, PersistentSubsectionGrade -from openedx.core.lib.command_utils import get_mutually_exclusive_required_option, parse_course_keys - -log = logging.getLogger(__name__) - - -DATE_FORMAT = "%Y-%m-%d %H:%M" - - -class Command(BaseCommand): - """ - Reset persistent grades for learners. - """ - help = dedent(__doc__).strip() - - def add_arguments(self, parser): - """ - Add arguments to the command parser. - """ - parser.add_argument( - '--dry_run', - action='store_true', - default=False, - dest='dry_run', - help="Output what we're going to do, but don't actually do it. To actually delete, use --delete instead." - ) - parser.add_argument( - '--delete', - action='store_true', - default=False, - dest='delete', - help="Actually perform the deletions of the course. For a Dry Run, use --dry_run instead." - ) - parser.add_argument( - '--courses', - dest='courses', - nargs='+', - help='Reset persistent grades for the list of courses provided.', - ) - parser.add_argument( - '--all_courses', - action='store_true', - dest='all_courses', - default=False, - help='Reset persistent grades for all courses.', - ) - parser.add_argument( - '--modified_start', - dest='modified_start', - help='Starting range for modified date (inclusive): e.g. "2016-08-23 16:43"; expected in UTC.', - ) - parser.add_argument( - '--modified_end', - dest='modified_end', - help='Ending range for modified date (inclusive): e.g. "2016-12-23 16:43"; expected in UTC.', - ) - parser.add_argument( - '--db_table', - dest='db_table', - help='Specify "subsection" to reset subsection grades or "course" to reset course grades. If absent, both ' - 'are reset.', - ) - - def handle(self, *args, **options): - course_keys = None - modified_start = None - modified_end = None - - run_mode = get_mutually_exclusive_required_option(options, 'delete', 'dry_run') - courses_mode = get_mutually_exclusive_required_option(options, 'courses', 'all_courses') - db_table = options.get('db_table') - if db_table not in {'subsection', 'course', None}: - raise CommandError('Invalid value for db_table. Valid options are "subsection" or "course" only.') - - if options.get('modified_start'): - modified_start = utc.localize(datetime.strptime(options['modified_start'], DATE_FORMAT)) - - if options.get('modified_end'): - if not modified_start: - raise CommandError('Optional value for modified_end provided without a value for modified_start.') - modified_end = utc.localize(datetime.strptime(options['modified_end'], DATE_FORMAT)) - - if courses_mode == 'courses': - course_keys = parse_course_keys(options['courses']) - - log.info("reset_grade: Started in %s mode!", run_mode) - - operation = self._query_grades if run_mode == 'dry_run' else self._delete_grades - - if db_table == 'subsection' or db_table is None: - operation(PersistentSubsectionGrade, course_keys, modified_start, modified_end) - - if db_table == 'course' or db_table is None: - operation(PersistentCourseGrade, course_keys, modified_start, modified_end) - - log.info("reset_grade: Finished in %s mode!", run_mode) - - def _delete_grades(self, grade_model_class, *args, **kwargs): - """ - Deletes the requested grades in the given model, filtered by the provided args and kwargs. - """ - grades_query_set = grade_model_class.query_grades(*args, **kwargs) - num_rows_to_delete = grades_query_set.count() - - log.info("reset_grade: Deleting %s: %d row(s).", grade_model_class.__name__, num_rows_to_delete) - - grade_model_class.delete_grades(*args, **kwargs) - - log.info("reset_grade: Deleted %s: %d row(s).", grade_model_class.__name__, num_rows_to_delete) - - def _query_grades(self, grade_model_class, *args, **kwargs): - """ - Queries the requested grades in the given model, filtered by the provided args and kwargs. - """ - total_for_all_courses = 0 - - grades_query_set = grade_model_class.query_grades(*args, **kwargs) - grades_stats = grades_query_set.values('course_id').order_by().annotate(total=Count('course_id')) - - for stat in grades_stats: - total_for_all_courses += stat['total'] - log.info( - "reset_grade: Would delete %s for COURSE %s: %d row(s).", - grade_model_class.__name__, - stat['course_id'], - stat['total'], - ) - - log.info( - "reset_grade: Would delete %s in TOTAL: %d row(s).", - grade_model_class.__name__, - total_for_all_courses, - ) diff --git a/lms/djangoapps/grades/management/commands/tests/test_reset_grades.py b/lms/djangoapps/grades/management/commands/tests/test_reset_grades.py deleted file mode 100644 index 3917944b63..0000000000 --- a/lms/djangoapps/grades/management/commands/tests/test_reset_grades.py +++ /dev/null @@ -1,319 +0,0 @@ -""" -Tests for reset_grades management command. -""" -from datetime import datetime, timedelta - -import ddt -from django.core.management.base import CommandError -from django.test import TestCase -from freezegun import freeze_time -from mock import MagicMock, patch -from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator - -from lms.djangoapps.grades.management.commands import reset_grades -from lms.djangoapps.grades.models import PersistentCourseGrade, PersistentSubsectionGrade - - -@ddt.ddt -class TestResetGrades(TestCase): - """ - Tests generate course blocks management command. - """ - num_users = 3 - num_courses = 5 - num_subsections = 7 - - def setUp(self): - super(TestResetGrades, self).setUp() - self.command = reset_grades.Command() - - self.user_ids = [user_id for user_id in range(self.num_users)] - - self.course_keys = [] - for course_index in range(self.num_courses): - self.course_keys.append( - CourseLocator( - org='some_org', - course='some_course', - run=unicode(course_index), - ) - ) - - self.subsection_keys_by_course = {} - for course_key in self.course_keys: - subsection_keys_in_course = [] - for subsection_index in range(self.num_subsections): - subsection_keys_in_course.append( - BlockUsageLocator( - course_key=course_key, - block_type='sequential', - block_id=unicode(subsection_index), - ) - ) - self.subsection_keys_by_course[course_key] = subsection_keys_in_course - - def _update_or_create_grades(self, courses_keys=None): - """ - Creates grades for all courses and subsections. - """ - if courses_keys is None: - courses_keys = self.course_keys - - course_grade_params = { - "course_version": "JoeMcEwing", - "course_edited_timestamp": datetime( - year=2016, - month=8, - day=1, - hour=18, - minute=53, - second=24, - microsecond=354741, - ), - "percent_grade": 77.7, - "letter_grade": "Great job", - "passed": True, - } - subsection_grade_params = { - "course_version": "deadbeef", - "subtree_edited_timestamp": "2016-08-01 18:53:24.354741", - "earned_all": 6.0, - "possible_all": 12.0, - "earned_graded": 6.0, - "possible_graded": 8.0, - "visible_blocks": MagicMock(), - "first_attempted": datetime.now(), - } - - for course_key in courses_keys: - for user_id in self.user_ids: - course_grade_params['user_id'] = user_id - course_grade_params['course_id'] = course_key - PersistentCourseGrade.update_or_create(**course_grade_params) - for subsection_key in self.subsection_keys_by_course[course_key]: - subsection_grade_params['user_id'] = user_id - subsection_grade_params['usage_key'] = subsection_key - PersistentSubsectionGrade.update_or_create_grade(**subsection_grade_params) - - def _assert_grades_exist_for_courses(self, course_keys, db_table=None): - """ - Assert grades for given courses exist. - """ - for course_key in course_keys: - if db_table == "course" or db_table is None: - self.assertIsNotNone(PersistentCourseGrade.read(self.user_ids[0], course_key)) - if db_table == "subsection" or db_table is None: - for subsection_key in self.subsection_keys_by_course[course_key]: - self.assertIsNotNone(PersistentSubsectionGrade.read_grade(self.user_ids[0], subsection_key)) - - def _assert_grades_absent_for_courses(self, course_keys, db_table=None): - """ - Assert grades for given courses do not exist. - """ - for course_key in course_keys: - if db_table == "course" or db_table is None: - with self.assertRaises(PersistentCourseGrade.DoesNotExist): - PersistentCourseGrade.read(self.user_ids[0], course_key) - - if db_table == "subsection" or db_table is None: - for subsection_key in self.subsection_keys_by_course[course_key]: - with self.assertRaises(PersistentSubsectionGrade.DoesNotExist): - PersistentSubsectionGrade.read_grade(self.user_ids[0], subsection_key) - - def _assert_stat_logged(self, mock_log, num_rows, grade_model_class, message_substring, log_offset): - self.assertIn('reset_grade: ' + message_substring, mock_log.info.call_args_list[log_offset][0][0]) - self.assertEqual(grade_model_class.__name__, mock_log.info.call_args_list[log_offset][0][1]) - self.assertEqual(num_rows, mock_log.info.call_args_list[log_offset][0][2]) - - def _assert_course_delete_stat_logged(self, mock_log, num_rows): - self._assert_stat_logged(mock_log, num_rows, PersistentCourseGrade, 'Deleted', log_offset=4) - - def _assert_subsection_delete_stat_logged(self, mock_log, num_rows): - self._assert_stat_logged(mock_log, num_rows, PersistentSubsectionGrade, 'Deleted', log_offset=2) - - def _assert_course_query_stat_logged(self, mock_log, num_rows, num_courses=None): - if num_courses is None: - num_courses = self.num_courses - log_offset = num_courses + 1 + num_courses + 1 - self._assert_stat_logged(mock_log, num_rows, PersistentCourseGrade, 'Would delete', log_offset) - - def _assert_subsection_query_stat_logged(self, mock_log, num_rows, num_courses=None): - if num_courses is None: - num_courses = self.num_courses - log_offset = num_courses + 1 - self._assert_stat_logged(mock_log, num_rows, PersistentSubsectionGrade, 'Would delete', log_offset) - - def _date_from_now(self, days=None): - return datetime.now() + timedelta(days=days) - - def _date_str_from_now(self, days=None): - future_date = self._date_from_now(days=days) - return future_date.strftime(reset_grades.DATE_FORMAT) - - @patch('lms.djangoapps.grades.management.commands.reset_grades.log') - def test_reset_all_courses(self, mock_log): - self._update_or_create_grades() - self._assert_grades_exist_for_courses(self.course_keys) - - with self.assertNumQueries(7): - self.command.handle(delete=True, all_courses=True) - - self._assert_grades_absent_for_courses(self.course_keys) - self._assert_subsection_delete_stat_logged( - mock_log, - num_rows=self.num_users * self.num_courses * self.num_subsections, - ) - self._assert_course_delete_stat_logged( - mock_log, - num_rows=self.num_users * self.num_courses, - ) - - @patch('lms.djangoapps.grades.management.commands.reset_grades.log') - @ddt.data(1, 2, 3) - def test_reset_some_courses(self, num_courses_to_reset, mock_log): - self._update_or_create_grades() - self._assert_grades_exist_for_courses(self.course_keys) - - with self.assertNumQueries(6): - self.command.handle( - delete=True, - courses=[unicode(course_key) for course_key in self.course_keys[:num_courses_to_reset]] - ) - - self._assert_grades_absent_for_courses(self.course_keys[:num_courses_to_reset]) - self._assert_grades_exist_for_courses(self.course_keys[num_courses_to_reset:]) - self._assert_subsection_delete_stat_logged( - mock_log, - num_rows=self.num_users * num_courses_to_reset * self.num_subsections, - ) - self._assert_course_delete_stat_logged( - mock_log, - num_rows=self.num_users * num_courses_to_reset, - ) - - def test_reset_by_modified_start_date(self): - self._update_or_create_grades() - self._assert_grades_exist_for_courses(self.course_keys) - - num_courses_with_updated_grades = 2 - with freeze_time(self._date_from_now(days=4)): - self._update_or_create_grades(self.course_keys[:num_courses_with_updated_grades]) - - with self.assertNumQueries(6): - self.command.handle(delete=True, modified_start=self._date_str_from_now(days=2), all_courses=True) - - self._assert_grades_absent_for_courses(self.course_keys[:num_courses_with_updated_grades]) - self._assert_grades_exist_for_courses(self.course_keys[num_courses_with_updated_grades:]) - - def test_reset_by_modified_start_end_date(self): - self._update_or_create_grades() - self._assert_grades_exist_for_courses(self.course_keys) - - with freeze_time(self._date_from_now(days=3)): - self._update_or_create_grades(self.course_keys[:2]) - with freeze_time(self._date_from_now(days=5)): - self._update_or_create_grades(self.course_keys[2:4]) - - with self.assertNumQueries(6): - self.command.handle( - delete=True, - modified_start=self._date_str_from_now(days=2), - modified_end=self._date_str_from_now(days=4), - all_courses=True, - ) - - # Only grades for courses modified within the 2->4 days - # should be deleted. - self._assert_grades_absent_for_courses(self.course_keys[:2]) - self._assert_grades_exist_for_courses(self.course_keys[2:]) - - @ddt.data('subsection', 'course') - def test_specify_db_table(self, db_table): - self._update_or_create_grades() - self._assert_grades_exist_for_courses(self.course_keys) - self.command.handle(delete=True, all_courses=True, db_table=db_table) - self._assert_grades_absent_for_courses(self.course_keys, db_table=db_table) - if db_table == "subsection": - self._assert_grades_exist_for_courses(self.course_keys, db_table='course') - else: - self._assert_grades_exist_for_courses(self.course_keys, db_table='subsection') - - @patch('lms.djangoapps.grades.management.commands.reset_grades.log') - def test_dry_run_all_courses(self, mock_log): - self._update_or_create_grades() - self._assert_grades_exist_for_courses(self.course_keys) - - with self.assertNumQueries(2): - self.command.handle(dry_run=True, all_courses=True) - - self._assert_grades_exist_for_courses(self.course_keys) - self._assert_subsection_query_stat_logged( - mock_log, - num_rows=self.num_users * self.num_courses * self.num_subsections, - ) - self._assert_course_query_stat_logged( - mock_log, - num_rows=self.num_users * self.num_courses, - ) - - @patch('lms.djangoapps.grades.management.commands.reset_grades.log') - @ddt.data(1, 2, 3) - def test_dry_run_some_courses(self, num_courses_to_query, mock_log): - self._update_or_create_grades() - self._assert_grades_exist_for_courses(self.course_keys) - - with self.assertNumQueries(2): - self.command.handle( - dry_run=True, - courses=[unicode(course_key) for course_key in self.course_keys[:num_courses_to_query]] - ) - - self._assert_grades_exist_for_courses(self.course_keys) - self._assert_subsection_query_stat_logged( - mock_log, - num_rows=self.num_users * num_courses_to_query * self.num_subsections, - num_courses=num_courses_to_query, - ) - self._assert_course_query_stat_logged( - mock_log, - num_rows=self.num_users * num_courses_to_query, - num_courses=num_courses_to_query, - ) - - @patch('lms.djangoapps.grades.management.commands.reset_grades.log') - def test_reset_no_existing_grades(self, mock_log): - self._assert_grades_absent_for_courses(self.course_keys) - - with self.assertNumQueries(4): - self.command.handle(delete=True, all_courses=True) - - self._assert_grades_absent_for_courses(self.course_keys) - self._assert_subsection_delete_stat_logged(mock_log, num_rows=0) - self._assert_course_delete_stat_logged(mock_log, num_rows=0) - - def test_invalid_key(self): - with self.assertRaisesRegexp(CommandError, 'Invalid key specified.*invalid/key'): - self.command.handle(dry_run=True, courses=['invalid/key']) - - def test_invalid_db_table(self): - with self.assertRaisesMessage( - CommandError, - 'Invalid value for db_table. Valid options are "subsection" or "course" only.' - ): - self.command.handle(delete=True, all_courses=True, db_table="not course or subsection") - - def test_no_run_mode(self): - with self.assertRaisesMessage(CommandError, 'Must specify exactly one of --delete, --dry_run'): - self.command.handle(all_courses=True) - - def test_both_run_modes(self): - with self.assertRaisesMessage(CommandError, 'Must specify exactly one of --delete, --dry_run'): - self.command.handle(all_courses=True, dry_run=True, delete=True) - - def test_no_course_mode(self): - with self.assertRaisesMessage(CommandError, 'Must specify exactly one of --courses, --all_courses'): - self.command.handle(dry_run=True) - - def test_both_course_modes(self): - with self.assertRaisesMessage(CommandError, 'Must specify exactly one of --courses, --all_courses'): - self.command.handle(dry_run=True, all_courses=True, courses=['some/course/key']) diff --git a/lms/djangoapps/grades/models.py b/lms/djangoapps/grades/models.py index d11a6bdb35..9dbe7a22a9 100644 --- a/lms/djangoapps/grades/models.py +++ b/lms/djangoapps/grades/models.py @@ -39,38 +39,6 @@ BLOCK_RECORD_LIST_VERSION = 1 BlockRecord = namedtuple('BlockRecord', ['locator', 'weight', 'raw_possible', 'graded']) -class DeleteGradesMixin(object): - """ - A Mixin class that provides functionality to delete grades. - """ - - @classmethod - def query_grades(cls, course_ids=None, modified_start=None, modified_end=None): - """ - Queries all the grades in the table, filtered by the provided arguments. - """ - kwargs = {} - - if course_ids: - kwargs['course_id__in'] = [course_id for course_id in course_ids] - - if modified_start: - if modified_end: - kwargs['modified__range'] = (modified_start, modified_end) - else: - kwargs['modified__gt'] = modified_start - - return cls.objects.filter(**kwargs) - - @classmethod - def delete_grades(cls, *args, **kwargs): - """ - Deletes all the grades in the table, filtered by the provided arguments. - """ - query = cls.query_grades(*args, **kwargs) - query.delete() - - class BlockRecordList(tuple): """ An immutable ordered list of BlockRecord objects. @@ -285,7 +253,7 @@ class VisibleBlocks(models.Model): return u"visible_blocks_cache.{}".format(course_key) -class PersistentSubsectionGrade(DeleteGradesMixin, TimeStampedModel): +class PersistentSubsectionGrade(TimeStampedModel): """ A django model tracking persistent grades at the subsection level. """ @@ -546,7 +514,7 @@ class PersistentSubsectionGrade(DeleteGradesMixin, TimeStampedModel): ) -class PersistentCourseGrade(DeleteGradesMixin, TimeStampedModel): +class PersistentCourseGrade(TimeStampedModel): """ A django model tracking persistent course grades. """