From 10c623d1335507ba9c0f54f4ba03cd370666fb40 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Wed, 14 Nov 2012 21:41:17 -0500 Subject: [PATCH 1/4] Django commands for certificates * Removing the command to find unicode names since it was only used for testing * ungenerated_certs will now by default run through all courses that have ended. a course option can be used to run on one course * gen_cert_report added which will summarize certificate states for all courses that have ended --- .../management/commands/find_unicode_certs.py | 38 -------- .../management/commands/gen_cert_report.py | 95 +++++++++++++++++++ .../management/commands/ungenerated_certs.py | 93 +++++++++++++++--- 3 files changed, 173 insertions(+), 53 deletions(-) delete mode 100644 lms/djangoapps/certificates/management/commands/find_unicode_certs.py create mode 100644 lms/djangoapps/certificates/management/commands/gen_cert_report.py diff --git a/lms/djangoapps/certificates/management/commands/find_unicode_certs.py b/lms/djangoapps/certificates/management/commands/find_unicode_certs.py deleted file mode 100644 index 49497622c6..0000000000 --- a/lms/djangoapps/certificates/management/commands/find_unicode_certs.py +++ /dev/null @@ -1,38 +0,0 @@ -# -*- coding: utf-8 -*- - -from django.core.management.base import BaseCommand -from certificates.models import certificate_status_for_student -from certificates.queue import XQueueCertInterface -from django.contrib.auth.models import User -from student.models import UserProfile - - -class Command(BaseCommand): - - help = """ - Looks for names that have unicode characters - and queues them up for a certificate request - """ - - def handle(self, *args, **options): - - # TODO this is only temporary for CS169 certs - - course_id = 'BerkeleyX/CS169.1x/2012_Fall' - - enrolled_students = User.objects.filter( - courseenrollment__course_id=course_id).prefetch_related( - "groups").order_by('username') - xq = XQueueCertInterface() - print "Looking for unusual names.." - for student in enrolled_students: - if certificate_status_for_student( - student, course_id)['status'] == 'unavailable': - continue - name = UserProfile.objects.get(user=student).name - for c in name: - if ord(c) >= 0x200: - ret = xq.add_cert(student, course_id) - if ret == 'generating': - print 'generating for {0}'.format(student) - break diff --git a/lms/djangoapps/certificates/management/commands/gen_cert_report.py b/lms/djangoapps/certificates/management/commands/gen_cert_report.py new file mode 100644 index 0000000000..6c175dcc4c --- /dev/null +++ b/lms/djangoapps/certificates/management/commands/gen_cert_report.py @@ -0,0 +1,95 @@ +from django.core.management.base import BaseCommand +from certificates.models import certificate_status_for_student +from django.contrib.auth.models import User +from optparse import make_option +from django.conf import settings +from xmodule.course_module import CourseDescriptor +from xmodule.modulestore.django import modulestore +from collections import Counter + + +class Command(BaseCommand): + + help = """ + + Generate a certificate status report for all courses that have ended. + This command does not do anything other than report the current + certificate status. + + unavailable - A student is not eligible for a certificate. + generating - A request has been made to generate a certificate, + but it has not been generated yet. + regenerating - A request has been made to regenerate a certificate, + but it has not been generated yet. + deleting - A request has been made to delete a certificate. + + deleted - The certificate has been deleted. + downloadable - The certificate is available for download. + notpassing - The student was graded but is not passing + + """ + + option_list = BaseCommand.option_list + ( + make_option('-c', '--course', + metavar='COURSE_ID', + dest='course', + default=None, + help='Only generate for COURSE_ID'), + ) + + def handle(self, *args, **options): + + # Find all courses that have ended + + if options['course']: + ended_courses = [options['course']] + else: + ended_courses = [] + for course_id in [course # all courses in COURSE_LISTINGS + for sub in settings.COURSE_LISTINGS + for course in settings.COURSE_LISTINGS[sub]]: + + course_loc = CourseDescriptor.id_to_location(course_id) + course = modulestore().get_instance(course_id, course_loc) + if course.has_ended(): + ended_courses.append(course_id) + + total_enrolled = {} + cert_statuses = {} + + for course_id in ended_courses: + + # find students who are enrolled + print "Looking up certificate states for {0}".format(course_id) + enrolled_students = User.objects.filter( + courseenrollment__course_id=course_id).prefetch_related( + "groups").order_by('username') + total_enrolled[course_id] = enrolled_students.count() + + # tally up certificate statuses for every student + # enrolled in the course + cert_statuses[course_id] = Counter( + [certificate_status_for_student( + student, course_id)['status'] + for student in enrolled_students]) + + # all states we have seen far all courses + status_headings = set( + [status for course in cert_statuses + for status in cert_statuses[course]]) + + # print the heading for the report + print "{0:>20}{1:>10}".format("course ID", "enrolled"), + print ' '.join(["{:>12}".format(heading) + for heading in status_headings]) + + # print the report + for course_id in total_enrolled: + print "{0:>20}{1:>10}".format( + course_id[0:18], total_enrolled[course_id]), + for heading in status_headings: + if heading in cert_statuses[course_id]: + print "{:>12}".format(cert_statuses[course_id][heading]), + else: + print " " * 12, + print diff --git a/lms/djangoapps/certificates/management/commands/ungenerated_certs.py b/lms/djangoapps/certificates/management/commands/ungenerated_certs.py index 82e86c2097..080918c0cc 100644 --- a/lms/djangoapps/certificates/management/commands/ungenerated_certs.py +++ b/lms/djangoapps/certificates/management/commands/ungenerated_certs.py @@ -2,29 +2,92 @@ from django.core.management.base import BaseCommand from certificates.models import certificate_status_for_student from certificates.queue import XQueueCertInterface from django.contrib.auth.models import User +from optparse import make_option +from django.conf import settings +from xmodule.course_module import CourseDescriptor +from xmodule.modulestore.django import modulestore +from certificates.models import CertificateStatuses +import datetime class Command(BaseCommand): help = """ - Find all students that have need certificates - and put certificate requests on the queue + Find all students that need certificates + for courses that have finished and + put their cert requests on the queue - This is only for BerkeleyX/CS169.1x/2012_Fall + Use the --noop option to test without actually + putting certificates on the queue to be generated. """ + option_list = BaseCommand.option_list + ( + make_option('-n', '--noop', + action='store_true', + dest='noop', + default=False, + help="Don't add certificate requests to the queue"), + make_option('-c', '--course', + metavar='COURSE_ID', + dest='course', + default=False, + help='Grade and generate certificates for a specific course'), + + ) + def handle(self, *args, **options): - # TODO This is only temporary for CS169 certs + # Will only generate a certificate if the current + # status is in this state - course_id = 'BerkeleyX/CS169.1x/2012_Fall' - enrolled_students = User.objects.filter( - courseenrollment__course_id=course_id).prefetch_related( - "groups").order_by('username') - xq = XQueueCertInterface() - for student in enrolled_students: - if certificate_status_for_student( - student, course_id)['status'] == 'unavailable': - ret = xq.add_cert(student, course_id) - if ret == 'generating': - print 'generating for {0}'.format(student) + VALID_STATUSES = [ + CertificateStatuses.unavailable + ] + + # Print update after this many students + + STATUS_INTERVAL = 500 + + if options['course']: + ended_courses = [options['course']] + else: + # Find all courses that have ended + ended_courses = [] + for course_id in [course # all courses in COURSE_LISTINGS + for sub in settings.COURSE_LISTINGS + for course in settings.COURSE_LISTINGS[sub]]: + course_loc = CourseDescriptor.id_to_location(course_id) + course = modulestore().get_instance(course_id, course_loc) + if course.has_ended(): + ended_courses.append(course_id) + + for course_id in ended_courses: + print "Fetching enrolled students for {0}".format(course_id) + enrolled_students = User.objects.filter( + courseenrollment__course_id=course_id).prefetch_related( + "groups").order_by('username') + xq = XQueueCertInterface() + total = enrolled_students.count() + count = 0 + start = datetime.datetime.now() + for student in enrolled_students: + count += 1 + if count % STATUS_INTERVAL == 0: + # Print a status update with an approximation of + # how much time is left based on how long the last + # interval took + diff = datetime.datetime.now() - start + timeleft = diff * (total - count) / STATUS_INTERVAL + hours, remainder = divmod(timeleft.seconds, 3600) + minutes, seconds = divmod(remainder, 60) + print "{0}/{1} completed ~{2:02}:{3:02}m remaining".format( + count, total, hours, minutes) + start = datetime.datetime.now() + + if certificate_status_for_student( + student, course_id)['status'] in VALID_STATUSES: + if not options['noop']: + # Add the certificate request to the queue + ret = xq.add_cert(student, course_id) + if ret == 'generating': + print '{0} - {1}'.format(student, ret) From 626e6c7e0849fb4236e9ecb104cf1ef47692bcca Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Thu, 15 Nov 2012 19:05:28 -0500 Subject: [PATCH 2/4] Addresses review comments * List of courses that have ended is now a generator * Using db-aggregation to speed up query for tallies --- .../management/commands/gen_cert_report.py | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/lms/djangoapps/certificates/management/commands/gen_cert_report.py b/lms/djangoapps/certificates/management/commands/gen_cert_report.py index 6c175dcc4c..8cb6879d34 100644 --- a/lms/djangoapps/certificates/management/commands/gen_cert_report.py +++ b/lms/djangoapps/certificates/management/commands/gen_cert_report.py @@ -1,5 +1,6 @@ from django.core.management.base import BaseCommand from certificates.models import certificate_status_for_student +from certificates.models import CertificateStatuses from django.contrib.auth.models import User from optparse import make_option from django.conf import settings @@ -37,6 +38,15 @@ class Command(BaseCommand): help='Only generate for COURSE_ID'), ) + def _ended_courses(self): + for course_id in [course # all courses in COURSE_LISTINGS + for sub in settings.COURSE_LISTINGS + for course in settings.COURSE_LISTINGS[sub]]: + course_loc = CourseDescriptor.id_to_location(course_id) + course = modulestore().get_instance(course_id, course_loc) + if course.has_ended(): + yield course_id + def handle(self, *args, **options): # Find all courses that have ended @@ -44,15 +54,7 @@ class Command(BaseCommand): if options['course']: ended_courses = [options['course']] else: - ended_courses = [] - for course_id in [course # all courses in COURSE_LISTINGS - for sub in settings.COURSE_LISTINGS - for course in settings.COURSE_LISTINGS[sub]]: - - course_loc = CourseDescriptor.id_to_location(course_id) - course = modulestore().get_instance(course_id, course_loc) - if course.has_ended(): - ended_courses.append(course_id) + ended_courses = self._ended_courses() total_enrolled = {} cert_statuses = {} @@ -66,12 +68,14 @@ class Command(BaseCommand): "groups").order_by('username') total_enrolled[course_id] = enrolled_students.count() + cert_statuses = [attr # all possible certificate statuses + for attr in dir(CertificateStatuses()) + if not callable(attr) and not attr.startswith("__")] + # tally up certificate statuses for every student # enrolled in the course - cert_statuses[course_id] = Counter( - [certificate_status_for_student( - student, course_id)['status'] - for student in enrolled_students]) + cert_statuses = {status:GeneratedCertificate.objects.filter(course_id__exact=course_id, status=status).count() for status in cert_statuses} + # all states we have seen far all courses status_headings = set( From efaf39ea7657bddf7f0c54a8dcdc0d1eed5331be Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Thu, 15 Nov 2012 19:42:47 -0500 Subject: [PATCH 3/4] cleanup --- .../management/commands/gen_cert_report.py | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/lms/djangoapps/certificates/management/commands/gen_cert_report.py b/lms/djangoapps/certificates/management/commands/gen_cert_report.py index 8cb6879d34..4fda0078e1 100644 --- a/lms/djangoapps/certificates/management/commands/gen_cert_report.py +++ b/lms/djangoapps/certificates/management/commands/gen_cert_report.py @@ -1,12 +1,11 @@ from django.core.management.base import BaseCommand -from certificates.models import certificate_status_for_student -from certificates.models import CertificateStatuses +from certificates.models import GeneratedCertificate from django.contrib.auth.models import User from optparse import make_option from django.conf import settings from xmodule.course_module import CourseDescriptor from xmodule.modulestore.django import modulestore -from collections import Counter +from django.db.models import Count class Command(BaseCommand): @@ -56,8 +55,7 @@ class Command(BaseCommand): else: ended_courses = self._ended_courses() - total_enrolled = {} - cert_statuses = {} + cert_data = {} for course_id in ended_courses: @@ -66,34 +64,35 @@ class Command(BaseCommand): enrolled_students = User.objects.filter( courseenrollment__course_id=course_id).prefetch_related( "groups").order_by('username') - total_enrolled[course_id] = enrolled_students.count() - - cert_statuses = [attr # all possible certificate statuses - for attr in dir(CertificateStatuses()) - if not callable(attr) and not attr.startswith("__")] - - # tally up certificate statuses for every student - # enrolled in the course - cert_statuses = {status:GeneratedCertificate.objects.filter(course_id__exact=course_id, status=status).count() for status in cert_statuses} + unavailable_count = enrolled_students.count() - \ + GeneratedCertificate.objects.filter( + course_id__exact=course_id).count() + cert_data[course_id] = {'enrolled': enrolled_students.count()} + cert_data[course_id].update({'unavailable': unavailable_count}) + tallies = GeneratedCertificate.objects.values( + 'status').annotate(dcount=Count('status')) + cert_data[course_id].update( + {status['status']: status['dcount'] + for status in tallies}) + print cert_data # all states we have seen far all courses status_headings = set( - [status for course in cert_statuses - for status in cert_statuses[course]]) + [status for course in cert_data + for status in cert_data[course]]) # print the heading for the report - print "{0:>20}{1:>10}".format("course ID", "enrolled"), + print "{:>20}".format("course ID"), print ' '.join(["{:>12}".format(heading) for heading in status_headings]) # print the report - for course_id in total_enrolled: - print "{0:>20}{1:>10}".format( - course_id[0:18], total_enrolled[course_id]), + for course_id in cert_data: + print "{0:>20}".format(course_id[0:18]), for heading in status_headings: - if heading in cert_statuses[course_id]: - print "{:>12}".format(cert_statuses[course_id][heading]), + if heading in cert_data[course_id]: + print "{:>12}".format(cert_data[course_id][heading]), else: print " " * 12, print From 9a12435f6b464071e7f4d915df0c33714ba2ef6b Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Thu, 15 Nov 2012 19:47:33 -0500 Subject: [PATCH 4/4] Removing print --- .../certificates/management/commands/gen_cert_report.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lms/djangoapps/certificates/management/commands/gen_cert_report.py b/lms/djangoapps/certificates/management/commands/gen_cert_report.py index 4fda0078e1..21e1775665 100644 --- a/lms/djangoapps/certificates/management/commands/gen_cert_report.py +++ b/lms/djangoapps/certificates/management/commands/gen_cert_report.py @@ -75,7 +75,6 @@ class Command(BaseCommand): cert_data[course_id].update( {status['status']: status['dcount'] for status in tallies}) - print cert_data # all states we have seen far all courses status_headings = set(