From d3277c99b490bd10df1a79bb58bac78024e78215 Mon Sep 17 00:00:00 2001 From: Julia Hansbrough Date: Wed, 4 Dec 2013 15:25:32 +0000 Subject: [PATCH 01/14] Initial PR for verified cert reporting Refactored from code in Stanford's itemized revenue reports into a new Report class used for all revenue-related reporting. Models for all cert types complete; test coverage for half of cert types complete. If this architecture is deemed solid, the next steps are to add a reasonable UI folks will use to select reports to download, allow them to restrict based on dates/universities, and of course complete test coverage. --- lms/djangoapps/shoppingcart/models.py | 262 ++++++++++++++---- .../shoppingcart/tests/test_models.py | 175 +++++++++++- .../shoppingcart/tests/test_views.py | 8 +- lms/djangoapps/shoppingcart/views.py | 11 +- lms/djangoapps/verify_student/models.py | 2 +- lms/djangoapps/verify_student/views.py | 1 + 6 files changed, 387 insertions(+), 72 deletions(-) diff --git a/lms/djangoapps/shoppingcart/models.py b/lms/djangoapps/shoppingcart/models.py index 7647b2c487..2013067360 100644 --- a/lms/djangoapps/shoppingcart/models.py +++ b/lms/djangoapps/shoppingcart/models.py @@ -23,6 +23,7 @@ from xmodule.course_module import CourseDescriptor from xmodule.modulestore.exceptions import ItemNotFoundError from course_modes.models import CourseMode +from courseware.courses import get_course_by_id from edxmako.shortcuts import render_to_string from student.views import course_from_id from student.models import CourseEnrollment, unenroll_done @@ -259,66 +260,6 @@ class OrderItem(models.Model): """ return self.pk_with_subclass, set([]) - @classmethod - def purchased_items_btw_dates(cls, start_date, end_date): - """ - Returns a QuerySet of the purchased items between start_date and end_date inclusive. - """ - return cls.objects.filter( - status="purchased", - fulfilled_time__gte=start_date, - fulfilled_time__lt=end_date, - ) - - @classmethod - def csv_purchase_report_btw_dates(cls, filelike, start_date, end_date): - """ - Outputs a CSV report into "filelike" (a file-like python object, such as an actual file, an HttpRequest, - or sys.stdout) of purchased items between start_date and end_date inclusive. - Opening and closing filelike (if applicable) should be taken care of by the caller - """ - items = cls.purchased_items_btw_dates(start_date, end_date).order_by("fulfilled_time") - - writer = unicodecsv.writer(filelike, encoding="utf-8") - writer.writerow(OrderItem.csv_report_header_row()) - - for item in items: - writer.writerow(item.csv_report_row) - - @classmethod - def csv_report_header_row(cls): - """ - Returns the "header" row for a csv report of purchases - """ - return [ - "Purchase Time", - "Order ID", - "Status", - "Quantity", - "Unit Cost", - "Total Cost", - "Currency", - "Description", - "Comments" - ] - - @property - def csv_report_row(self): - """ - Returns an array which can be fed into csv.writer to write out one csv row - """ - return [ - self.fulfilled_time, - self.order_id, # pylint: disable=no-member - self.status, - self.qty, - self.unit_cost, - self.line_cost, - self.currency, - self.line_desc, - self.report_comments, - ] - @property def pk_with_subclass(self): """ @@ -625,3 +566,204 @@ class CertificateItem(OrderItem): "Please include your order number in your e-mail. " "Please do NOT include your credit card information.").format( billing_email=settings.PAYMENT_SUPPORT_EMAIL) + +class Report(models.Model): + """ + Base class for making CSV reports related to revenue, enrollments, etc + + To make a different type of report, write a new subclass that implements + the methods get_query, csv_report_header_row, and csv_report_row. + """ + + @classmethod + def initialize_report(cls, report_type): + if report_type == "refund_report": + return RefundReport() + elif report_type == "itemized_purchase_report": + return ItemizedPurchaseReport() + elif report_type == "university_revenue_share": + return UniversityRevenueShareReport() + elif report_type == "certificate_status": + return CertificateStatusReport() + else: + return # TODO return an error + + def get_query(self, start_date, end_date): + raise NotImplementedError + + def csv_report_header_row(self, start_date, end_date): + raise NotImplementedError + + def csv_report_row(self): + raise NotImplementedError + + @classmethod + def make_report(cls, report_type, filelike, start_date, end_date): + report = cls.initialize_report(report_type) + items = report.get_query(start_date, end_date) + writer = unicodecsv.writer(filelike, encoding="utf-8") + writer.writerow(report.csv_report_header_row()) + for item in items: + writer.writerow(report.csv_report_row(item)) + + +class RefundReport(Report): + def get_query(self, start_date, end_date): + return CertificateItem.objects.filter( + status="refunded", + ) + + def csv_report_header_row(self): + return [ + "Order Number", + "Customer Name", + "Date of Original Transaction", + "Date of Refund", + "Amount of Refund", + "Service Fees (if any)", + ] + + def csv_report_row(self, item): + return [ + item.order_id, + item.user.get_full_name(), + item.fulfilled_time, + item.refund_requested_time, # actually may need to use refund_fulfilled here + item.line_cost, + 0, # TODO: determine if service_fees field is necessary; if so, add + ] + +class ItemizedPurchaseReport(Report): + def get_query(self, start_date, end_date): + return OrderItem.objects.filter( + status="purchased", + fulfilled_time__gte=start_date, + fulfilled_time__lt=end_date, + ).order_by("fulfilled_time") + + def csv_report_header_row(self): + return [ + "Purchase Time", + "Order ID", + "Status", + "Quantity", + "Unit Cost", + "Total Cost", + "Currency", + "Description", + "Comments" + ] + + def csv_report_row(self, item): + return [ + item.fulfilled_time, + item.order_id, # pylint: disable=no-member + item.status, + item.qty, + item.unit_cost, + item.line_cost, + item.currency, + item.line_desc, + item.report_comments, + ] + + +class CertificateStatusReport(Report): + def get_query(self, start_date, send_date): + results = [] + for course_id in settings.COURSE_LISTINGS: + cur_course = get_course_by_id(course) + university = cur_course.org + course = cur_course.number + " " + course.display_name #TODO add term (i.e. Fall 2013)? + enrollments = CourseEnrollment.objects.filter(course_id=course_id) + total_enrolled = enrollments.objects.count() + audit_enrolled = enrollments.objects.filter(mode="audit").count() + honor_enrolled = enrollments.objects.filter(mode="honor").count() + verified_enrollments = enrollments.objects.filter(mode="verified") + verified_enrolled = verified_enrollments.objects.count() + gross_rev = CertificateItem.objects.filter(course_id=course_id, mode="verified").aggregate(Sum('unit_cost')) + gross_rev_over_min = gross_rev - (CourseMode.objects.get('course_id').min_price * verified_enrollments) + num_verified_over_min = 0 # TODO clarify with billing what exactly this means + refunded_enrollments = CertificateItem.objects.filter(course_id='course_id', mode="refunded") + number_of_refunds = refunded_enrollments.objects.count() + dollars_refunded = refunded_enrollments.objects.aggregate(Sum('unit_cost')) + + result = [ + university, + course, + total_enrolled, + audit_enrolled, + honor_enrolled, + verified_enrolled, + gross_rev, + gross_rev_over_min, + num_verified_over_min, + number_of_refunds, + dollars_refunded + ] + + results.append(result) + return results + + def csv_report_header_row(self): + return [ + "University", + "Course", + "Total Enrolled", + "Audit Enrollment", + "Honor Code Enrollment", + "Verified Enrollment", + "Gross Revenue", + "Gross Revenue over the Minimum", + "Number of Verified over the Minimum", + "Number of Refunds", + "Dollars Refunded", + ] + + def csv_report_row(self, item): + return item + + +class UniversityRevenueShareReport(Report): + def get_query(self, start_date, end_date): + results = [] + for course_id in settings.COURSE_LISTINGS: + cur_course = get_course_by_id(course) + university = cur_course.org + course = cur_course.number + " " + course.display_name + num_transactions = 0 # TODO clarify with building what transactions are included in this (purchases? refunds? etc) + total_payments_collected = CertificateItem.objects.filter(course_id=course_id, mode="verified").aggregate(Sum('unit_cost')) + #note: we're assuming certitems are the only way to make money right now + service_fees = 0 # TODO add an actual service fees field, in case needed in future + refunded_enrollments = CertificateItem.objects.filter(course_id='course_id', mode="refunded") + num_refunds = refunded_enrollments.objects.count() + amount_refunds = refunded_enrollments.objects.aggregate(Sum('unit_cost')) + + result = [ + university, + course, + num_transactions, + total_payments_collected, + service_fees, + refunded_enrollments, + num_refunds, + amount_refunds + ] + results.append(result) + + return results + + def csv_report_header_row(self): + return [ + "University", + "Course", + "Number of Transactions", + "Total Payments Collected", + "Service Fees (if any)", + "Number of Successful Refunds", + "Total Amount of Refunds", + # note this is restricted by a date range + ] + + def csv_report_row(self, item): + return item diff --git a/lms/djangoapps/shoppingcart/tests/test_models.py b/lms/djangoapps/shoppingcart/tests/test_models.py index 5a52ca0680..aeb92740a6 100644 --- a/lms/djangoapps/shoppingcart/tests/test_models.py +++ b/lms/djangoapps/shoppingcart/tests/test_models.py @@ -17,7 +17,7 @@ from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE from shoppingcart.models import (Order, OrderItem, CertificateItem, InvalidCartItem, PaidCourseRegistration, - OrderItemSubclassPK, PaidCourseRegistrationAnnotation) + OrderItemSubclassPK, PaidCourseRegistrationAnnotation, Report) from student.tests.factories import UserFactory from student.models import CourseEnrollment from course_modes.models import CourseMode @@ -324,7 +324,82 @@ class PaidCourseRegistrationTest(ModuleStoreTestCase): @override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) -class PurchaseReportTest(ModuleStoreTestCase): +class RefundReportTest(ModuleStoreTestCase): + FIVE_MINS = datetime.timedelta(minutes=5) + + def setUp(self): + self.user = UserFactory.create() + self.user.first_name = "John" + self.user.last_name = "Doe" + self.user.save() + self.course_id = "MITx/999/Robot_Super_Course" + self.cost = 40 + self.course = CourseFactory.create(org='MITx', number='999', display_name=u'Robot Super Course') + course_mode = CourseMode(course_id=self.course_id, + mode_slug="honor", + mode_display_name="honor cert", + min_price=self.cost) + course_mode.save() + + course_mode2 = CourseMode(course_id=self.course_id, + mode_slug="verified", + mode_display_name="verified cert", + min_price=self.cost) + course_mode2.save() + + self.cart = Order.get_cart_for_user(self.user) + CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') + self.cart.purchase() + + # should auto-refund the relevant cert + CourseEnrollment.unenroll(self.user, self.course_id) + + self.cert_item = CertificateItem.objects.get(user=self.user, course_id=self.course_id) + + self.now = datetime.datetime.now(pytz.UTC) + + def test_get_query(self): + report_type = "refund_report" + report = Report.initialize_report(report_type) + refunded_certs = report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + self.assertEqual(len(refunded_certs), 1) + self.assertIn(self.cert_item, refunded_certs) + # TODO no time restrictions yet + + test_time = datetime.datetime.now(pytz.UTC) + + CORRECT_CSV = dedent(""" + Order Number,Customer Name,Date of Original Transaction,Date of Refund,Amount of Refund,Service Fees (if any) + 1,John Doe,{time_str},{time_str},40,0 + """.format(time_str=str(test_time))) + + def test_purchased_csv(self): + """ + Tests that a generated purchase report CSV is as we expect + """ + # coerce the purchase times to self.test_time so that the test can match. + # It's pretty hard to patch datetime.datetime b/c it's a python built-in, which is immutable, so we + # make the times match this way + # TODO test multiple report types + report_type = "refund_report" + report = Report.initialize_report(report_type) + for item in report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS): + item.fulfilled_time = self.test_time + item.refund_requested_time = self.test_time #hm do we want to make these different + item.save() + + # add annotation to the + csv_file = StringIO.StringIO() + Report.make_report(report_type, csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + csv = csv_file.getvalue() + csv_file.close() + # Using excel mode csv, which automatically ends lines with \r\n, so need to convert to \n + self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_CSV.strip()) + + + +@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) +class ItemizedPurchaseReportTest(ModuleStoreTestCase): FIVE_MINS = datetime.timedelta(minutes=5) TEST_ANNOTATION = u'Ba\xfc\u5305' @@ -353,11 +428,14 @@ class PurchaseReportTest(ModuleStoreTestCase): self.now = datetime.datetime.now(pytz.UTC) def test_purchased_items_btw_dates(self): - purchases = OrderItem.purchased_items_btw_dates(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + # TODO test multiple report types + report_type = "itemized_purchase_report" + report = Report.initialize_report(report_type) + purchases = report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) self.assertEqual(len(purchases), 2) self.assertIn(self.reg.orderitem_ptr, purchases) self.assertIn(self.cert_item.orderitem_ptr, purchases) - no_purchases = OrderItem.purchased_items_btw_dates(self.now + self.FIVE_MINS, + no_purchases = report.get_query(self.now + self.FIVE_MINS, self.now + self.FIVE_MINS + self.FIVE_MINS) self.assertFalse(no_purchases) @@ -376,13 +454,16 @@ class PurchaseReportTest(ModuleStoreTestCase): # coerce the purchase times to self.test_time so that the test can match. # It's pretty hard to patch datetime.datetime b/c it's a python built-in, which is immutable, so we # make the times match this way - for item in OrderItem.purchased_items_btw_dates(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS): + # TODO test multiple report types + report_type = "itemized_purchase_report" + report = Report.initialize_report(report_type) + for item in report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS): item.fulfilled_time = self.test_time item.save() # add annotation to the csv_file = StringIO.StringIO() - OrderItem.csv_purchase_report_btw_dates(csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + Report.make_report(report_type, csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) csv = csv_file.getvalue() csv_file.close() # Using excel mode csv, which automatically ends lines with \r\n, so need to convert to \n @@ -403,6 +484,88 @@ class PurchaseReportTest(ModuleStoreTestCase): """ self.assertEqual(unicode(self.annotation), u'{} : {}'.format(self.course_id, self.TEST_ANNOTATION)) +# TODO: finish this test class +@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) +class CertificateStatusReportTest(ModuleStoreTestCase): + FIVE_MINS = datetime.timedelta(minutes=5) + + def setUp(self): + self.user = UserFactory.create() + self.user.first_name = "John" + self.user.last_name = "Doe" + self.user.save() + self.course_id = "MITx/999/Robot_Super_Course" + self.cost = 40 + self.course = CourseFactory.create(org='MITx', number='999', display_name=u'Robot Super Course') + course_mode = CourseMode(course_id=self.course_id, + mode_slug="honor", + mode_display_name="honor cert", + min_price=self.cost) + course_mode.save() + + course_mode2 = CourseMode(course_id=self.course_id, + mode_slug="verified", + mode_display_name="verified cert", + min_price=self.cost) + course_mode2.save() + + self.cart = Order.get_cart_for_user(self.user) + CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') + self.cart.purchase() + + self.now = datetime.datetime.now(pytz.UTC) + + # TODO finish these tests. This is just a basic test to start with, making sure the regular + # flow doesn't throw any strange errors while running + def test_basic(self): + report_type = "certificate_status" + report = Report.initialize_report(report_type) + refunded_certs = report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + csv_file = StringIO.StringIO() + report.make_report(report_type, csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + # TODO no time restrictions yet + +# TODO: finish this test class +@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) +class UniversityRevenueShareReportTest(ModuleStoreTestCase): + FIVE_MINS = datetime.timedelta(minutes=5) + + def setUp(self): + self.user = UserFactory.create() + self.user.first_name = "John" + self.user.last_name = "Doe" + self.user.save() + self.course_id = "MITx/999/Robot_Super_Course" + self.cost = 40 + self.course = CourseFactory.create(org='MITx', number='999', display_name=u'Robot Super Course') + course_mode = CourseMode(course_id=self.course_id, + mode_slug="honor", + mode_display_name="honor cert", + min_price=self.cost) + course_mode.save() + + course_mode2 = CourseMode(course_id=self.course_id, + mode_slug="verified", + mode_display_name="verified cert", + min_price=self.cost) + course_mode2.save() + + self.cart = Order.get_cart_for_user(self.user) + CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') + self.cart.purchase() + + self.now = datetime.datetime.now(pytz.UTC) + + # TODO finish these tests. This is just a basic test to start with, making sure the regular + # flow doesn't throw any strange errors while running + def test_basic(self): + report_type = "university_revenue_share" + report = Report.initialize_report(report_type) + refunded_certs = report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + csv_file = StringIO.StringIO() + report.make_report(report_type, csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + # TODO no time restrictions yet + @override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) class CertificateItemTest(ModuleStoreTestCase): diff --git a/lms/djangoapps/shoppingcart/tests/test_views.py b/lms/djangoapps/shoppingcart/tests/test_views.py index d1ab9ab24f..0658d56ab9 100644 --- a/lms/djangoapps/shoppingcart/tests/test_views.py +++ b/lms/djangoapps/shoppingcart/tests/test_views.py @@ -14,7 +14,7 @@ from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE from shoppingcart.views import _can_download_report, _get_date_from_str -from shoppingcart.models import Order, CertificateItem, PaidCourseRegistration, OrderItem +from shoppingcart.models import Order, CertificateItem, PaidCourseRegistration, OrderItem, Report from student.tests.factories import UserFactory from student.models import CourseEnrollment from course_modes.models import CourseMode @@ -379,6 +379,9 @@ class CSVReportViewsTest(ModuleStoreTestCase): CORRECT_CSV_NO_DATE = ",1,purchased,1,40,40,usd,Registration for Course: Robot Super Course," def test_report_csv(self): + # TODO test multiple types + report_type = "itemized_purchase_report" + PaidCourseRegistration.add_to_order(self.cart, self.course_id) self.cart.purchase() self.login_user() @@ -386,7 +389,8 @@ class CSVReportViewsTest(ModuleStoreTestCase): response = self.client.post(reverse('payment_csv_report'), {'start_date': '1970-01-01', 'end_date': '2100-01-01'}) self.assertEqual(response['Content-Type'], 'text/csv') - self.assertIn(",".join(OrderItem.csv_report_header_row()), response.content) + report = Report.initialize_report(report_type) + self.assertIn(",".join(report.csv_report_header_row()), response.content) self.assertIn(self.CORRECT_CSV_NO_DATE, response.content) diff --git a/lms/djangoapps/shoppingcart/views.py b/lms/djangoapps/shoppingcart/views.py index 22bccbf28c..907c0dcbce 100644 --- a/lms/djangoapps/shoppingcart/views.py +++ b/lms/djangoapps/shoppingcart/views.py @@ -11,8 +11,8 @@ from django.core.urlresolvers import reverse from django.views.decorators.csrf import csrf_exempt from django.contrib.auth.decorators import login_required from edxmako.shortcuts import render_to_response -from .models import Order, PaidCourseRegistration, OrderItem from student.models import CourseEnrollment +from .models import Order, PaidCourseRegistration, OrderItem, Report from .processors import process_postpay_callback, render_purchase_form_html from .exceptions import ItemAlreadyInCartException, AlreadyEnrolledInCourseException, CourseDoesNotExistException @@ -174,6 +174,9 @@ def csv_report(request): """ Downloads csv reporting of orderitems """ + # TODO: change this to something modular later + report_type = "itemized_purchase_report" + if not _can_download_report(request.user): return HttpResponseForbidden(_('You do not have permission to view this page.')) @@ -187,7 +190,8 @@ def csv_report(request): # Error case: there was a badly formatted user-input date string return _render_report_form(start_str, end_str, date_fmt_error=True) - items = OrderItem.purchased_items_btw_dates(start_date, end_date) + report = Report.initialize_report(report_type) + items = report.get_query(start_date, end_date) if items.count() > settings.PAYMENT_REPORT_MAX_ITEMS: # Error case: too many items would be generated in the report and we're at risk of timeout return _render_report_form(start_str, end_str, total_count_error=True) @@ -195,7 +199,8 @@ def csv_report(request): response = HttpResponse(mimetype='text/csv') filename = "purchases_report_{}.csv".format(datetime.datetime.now(pytz.UTC).strftime("%Y-%m-%d-%H-%M-%S")) response['Content-Disposition'] = 'attachment; filename="{}"'.format(filename) - OrderItem.csv_purchase_report_btw_dates(response, start_date, end_date) + # this flos is a little odd; what's up with report_type being called twice? check later + report.make_report(report_type, response, start_date, end_date) return response elif request.method == 'GET': diff --git a/lms/djangoapps/verify_student/models.py b/lms/djangoapps/verify_student/models.py index dccbdb430a..52ec2b3cd3 100644 --- a/lms/djangoapps/verify_student/models.py +++ b/lms/djangoapps/verify_student/models.py @@ -477,7 +477,7 @@ class SoftwareSecurePhotoVerification(PhotoVerification): # developing and aren't interested in working on student identity # verification functionality. If you do want to work on it, you have to # explicitly enable these in your private settings. - if settings.FEATURES.get('AUTOMATIC_VERIFY_STUDENT_IDENTITY_FOR_TESTING'): + if settings.MITX_FEATURES.get('AUTOMATIC_VERIFY_STUDENT_IDENTITY_FOR_TESTING'): return aes_key_str = settings.VERIFY_STUDENT["SOFTWARE_SECURE"]["FACE_IMAGE_AES_KEY"] diff --git a/lms/djangoapps/verify_student/views.py b/lms/djangoapps/verify_student/views.py index 2ac798bbf2..1821116574 100644 --- a/lms/djangoapps/verify_student/views.py +++ b/lms/djangoapps/verify_student/views.py @@ -132,6 +132,7 @@ def create_order(request): """ Submit PhotoVerification and create a new Order for this verified cert """ + from nose.tools import set_trace; set_trace() if not SoftwareSecurePhotoVerification.user_has_valid_or_pending(request.user): attempt = SoftwareSecurePhotoVerification(user=request.user) b64_face_image = request.POST['face_image'].split(",")[1] From 1981ee5078a70549cca68f1a5792670df6959762 Mon Sep 17 00:00:00 2001 From: Julia Hansbrough Date: Mon, 9 Dec 2013 18:36:04 +0000 Subject: [PATCH 02/14] Fixed CertificateStatusReportModel --- lms/djangoapps/shoppingcart/models.py | 36 ++++++---- .../shoppingcart/tests/test_models.py | 69 ++++++++++++++++--- 2 files changed, 83 insertions(+), 22 deletions(-) diff --git a/lms/djangoapps/shoppingcart/models.py b/lms/djangoapps/shoppingcart/models.py index 2013067360..39fe69a0cc 100644 --- a/lms/djangoapps/shoppingcart/models.py +++ b/lms/djangoapps/shoppingcart/models.py @@ -10,6 +10,7 @@ from boto.exception import BotoServerError # this is a super-class of SESError from django.dispatch import receiver from django.db import models +from django.db.models import Sum from django.conf import settings from django.core.exceptions import ObjectDoesNotExist from django.core.mail import send_mail @@ -18,6 +19,8 @@ from django.utils.translation import ugettext as _ from django.db import transaction from django.core.urlresolvers import reverse +from decimal import Decimal + from xmodule.modulestore.django import modulestore from xmodule.course_module import CourseDescriptor from xmodule.modulestore.exceptions import ItemNotFoundError @@ -671,22 +674,27 @@ class ItemizedPurchaseReport(Report): class CertificateStatusReport(Report): def get_query(self, start_date, send_date): results = [] - for course_id in settings.COURSE_LISTINGS: - cur_course = get_course_by_id(course) + for course_id in settings.COURSE_LISTINGS['default']: + cur_course = get_course_by_id(course_id) university = cur_course.org - course = cur_course.number + " " + course.display_name #TODO add term (i.e. Fall 2013)? + course = cur_course.number + " " + cur_course.display_name #TODO add term (i.e. Fall 2013)? enrollments = CourseEnrollment.objects.filter(course_id=course_id) - total_enrolled = enrollments.objects.count() - audit_enrolled = enrollments.objects.filter(mode="audit").count() - honor_enrolled = enrollments.objects.filter(mode="honor").count() - verified_enrollments = enrollments.objects.filter(mode="verified") - verified_enrolled = verified_enrollments.objects.count() - gross_rev = CertificateItem.objects.filter(course_id=course_id, mode="verified").aggregate(Sum('unit_cost')) - gross_rev_over_min = gross_rev - (CourseMode.objects.get('course_id').min_price * verified_enrollments) + total_enrolled = enrollments.count() + audit_enrolled = enrollments.filter(mode="audit").count() + honor_enrolled = enrollments.filter(mode="honor").count() + verified_enrollments = enrollments.filter(mode="verified") + verified_enrolled = verified_enrollments.count() + gross_rev_temp = CertificateItem.objects.filter(course_id=course_id, mode="verified").aggregate(Sum('unit_cost')) + gross_rev = gross_rev_temp['unit_cost__sum'] + gross_rev_over_min = gross_rev - (CourseMode.objects.get(course_id=course_id,mode_slug="verified").min_price * verified_enrolled) num_verified_over_min = 0 # TODO clarify with billing what exactly this means refunded_enrollments = CertificateItem.objects.filter(course_id='course_id', mode="refunded") - number_of_refunds = refunded_enrollments.objects.count() - dollars_refunded = refunded_enrollments.objects.aggregate(Sum('unit_cost')) + number_of_refunds = refunded_enrollments.count() + dollars_refunded_temp = refunded_enrollments.aggregate(Sum('unit_cost')) + if dollars_refunded_temp['unit_cost__sum'] is None: + dollars_refunded = Decimal(0.00) + else: + dollars_refunded = dollars_refunded_temp['unit_cost__sum'] result = [ university, @@ -727,8 +735,8 @@ class CertificateStatusReport(Report): class UniversityRevenueShareReport(Report): def get_query(self, start_date, end_date): results = [] - for course_id in settings.COURSE_LISTINGS: - cur_course = get_course_by_id(course) + for course_id in settings.COURSE_LISTINGS['default']: + cur_course = get_course_by_id(course_id) university = cur_course.org course = cur_course.number + " " + course.display_name num_transactions = 0 # TODO clarify with building what transactions are included in this (purchases? refunds? etc) diff --git a/lms/djangoapps/shoppingcart/tests/test_models.py b/lms/djangoapps/shoppingcart/tests/test_models.py index aeb92740a6..7edf06b06f 100644 --- a/lms/djangoapps/shoppingcart/tests/test_models.py +++ b/lms/djangoapps/shoppingcart/tests/test_models.py @@ -490,11 +490,41 @@ class CertificateStatusReportTest(ModuleStoreTestCase): FIVE_MINS = datetime.timedelta(minutes=5) def setUp(self): - self.user = UserFactory.create() - self.user.first_name = "John" - self.user.last_name = "Doe" - self.user.save() + # Need to make a *lot* of users for this one + self.user1 = UserFactory.create() + self.user1.first_name = "John" + self.user1.last_name = "Doe" + self.user1.save() + + self.user2 = UserFactory.create() + self.user2.first_name = "Jane" + self.user2.last_name = "Deer" + self.user2.save() + + self.user3 = UserFactory.create() + self.user3.first_name = "Joe" + self.user3.last_name = "Miller" + self.user3.save() + + self.user4 = UserFactory.create() + self.user4.first_name = "Simon" + self.user4.last_name = "Blackquill" + self.user4.save() + + self.user5 = UserFactory.create() + self.user5.first_name = "Super" + self.user5.last_name = "Mario" + self.user5.save() + + self.user6 = UserFactory.create() + self.user6.first_name = "Princess" + self.user6.last_name = "Peach" + self.user6.save() + + # Two are verified, three are audit, one honor + self.course_id = "MITx/999/Robot_Super_Course" + settings.COURSE_LISTINGS['default'] = [self.course_id] self.cost = 40 self.course = CourseFactory.create(org='MITx', number='999', display_name=u'Robot Super Course') course_mode = CourseMode(course_id=self.course_id, @@ -509,12 +539,33 @@ class CertificateStatusReportTest(ModuleStoreTestCase): min_price=self.cost) course_mode2.save() - self.cart = Order.get_cart_for_user(self.user) - CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') - self.cart.purchase() + # User 1 & 2 will be verified + self.cart1 = Order.get_cart_for_user(self.user1) + CertificateItem.add_to_order(self.cart1, self.course_id, self.cost, 'verified') + self.cart1.purchase() + + self.cart2 = Order.get_cart_for_user(self.user2) + CertificateItem.add_to_order(self.cart2, self.course_id, self.cost, 'verified') + self.cart2.purchase() + + # Users 3, 4, and 5 are audit + CourseEnrollment.get_or_create_enrollment(self.user3, self.course_id).update_enrollment(mode="audit") + CourseEnrollment.get_or_create_enrollment(self.user4, self.course_id).update_enrollment(mode="audit") + CourseEnrollment.get_or_create_enrollment(self.user5, self.course_id).update_enrollment(mode="audit") + + # User 6 is honor + CourseEnrollment.get_or_create_enrollment(self.user6, self.course_id).update_enrollment(mode="honor") self.now = datetime.datetime.now(pytz.UTC) + # bluh need to test some refunds + + test_time = datetime.datetime.now(pytz.UTC) + CORRECT_CSV = dedent(""" + University,Course,Total Enrolled,Audit Enrollment,Honor Code Enrollment,Verified Enrollment,Gross Revenue,Gross Revenue over the Minimum,Number of Verified over the Minimum,Number of Refunds,Dollars Refunded + MITx,999 Robot Super Course,6,3,1,2,80.00,0.00,0,0,0 + """.format(time_str=str(test_time))) + # TODO finish these tests. This is just a basic test to start with, making sure the regular # flow doesn't throw any strange errors while running def test_basic(self): @@ -523,7 +574,9 @@ class CertificateStatusReportTest(ModuleStoreTestCase): refunded_certs = report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) csv_file = StringIO.StringIO() report.make_report(report_type, csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) - # TODO no time restrictions yet + csv = csv_file.getvalue() + self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_CSV.strip()) + # TODO no time restrictions ye # TODO: finish this test class @override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) From fa87793e9b8e22e8d4148aa12b407fc1a08f50dc Mon Sep 17 00:00:00 2001 From: Julia Hansbrough Date: Mon, 9 Dec 2013 19:10:07 +0000 Subject: [PATCH 03/14] Fixed UniversityRevenueShare model --- lms/djangoapps/shoppingcart/exceptions.py | 8 ++ lms/djangoapps/shoppingcart/models.py | 111 +++++++++++++----- .../shoppingcart/tests/test_models.py | 81 ++++++++----- .../shoppingcart/tests/test_views.py | 2 +- lms/djangoapps/verify_student/views.py | 1 - 5 files changed, 137 insertions(+), 66 deletions(-) diff --git a/lms/djangoapps/shoppingcart/exceptions.py b/lms/djangoapps/shoppingcart/exceptions.py index a40c2e9feb..b6f826040b 100644 --- a/lms/djangoapps/shoppingcart/exceptions.py +++ b/lms/djangoapps/shoppingcart/exceptions.py @@ -26,3 +26,11 @@ class AlreadyEnrolledInCourseException(InvalidCartItem): class CourseDoesNotExistException(InvalidCartItem): pass + + +class ReportException(Exception): + pass + + +class ReportTypeDoesNotExistException(ReportException): + pass diff --git a/lms/djangoapps/shoppingcart/models.py b/lms/djangoapps/shoppingcart/models.py index 39fe69a0cc..dcc4c22538 100644 --- a/lms/djangoapps/shoppingcart/models.py +++ b/lms/djangoapps/shoppingcart/models.py @@ -34,7 +34,8 @@ from student.models import CourseEnrollment, unenroll_done from verify_student.models import SoftwareSecurePhotoVerification from .exceptions import (InvalidCartItem, PurchasedCallbackException, ItemAlreadyInCartException, - AlreadyEnrolledInCourseException, CourseDoesNotExistException) + AlreadyEnrolledInCourseException, CourseDoesNotExistException, ReportException, + ReportTypeDoesNotExistException) log = logging.getLogger("shoppingcart") @@ -214,6 +215,7 @@ class OrderItem(models.Model): currency = models.CharField(default="usd", max_length=8) # lower case ISO currency codes fulfilled_time = models.DateTimeField(null=True) refund_requested_time = models.DateTimeField(null=True) + service_fee = models.DecimalField(default=0.0, decimal_places=2, max_digits=30) # general purpose field, not user-visible. Used for reporting report_comments = models.TextField(default="") @@ -570,6 +572,7 @@ class CertificateItem(OrderItem): "Please do NOT include your credit card information.").format( billing_email=settings.PAYMENT_SUPPORT_EMAIL) + class Report(models.Model): """ Base class for making CSV reports related to revenue, enrollments, etc @@ -580,6 +583,9 @@ class Report(models.Model): @classmethod def initialize_report(cls, report_type): + """ + Creates the appropriate type of Report object based on the string report_type. + """ if report_type == "refund_report": return RefundReport() elif report_type == "itemized_purchase_report": @@ -589,19 +595,32 @@ class Report(models.Model): elif report_type == "certificate_status": return CertificateStatusReport() else: - return # TODO return an error + raise ReportTypeDoesNotExistException def get_query(self, start_date, end_date): + """ + Performs any database queries necessary to obtain the data for the report. + """ raise NotImplementedError - def csv_report_header_row(self, start_date, end_date): + def csv_report_header_row(self): + """ + Returns the appropriate header based on the report type. + """ raise NotImplementedError - def csv_report_row(self): + def csv_report_row(self, item): + """ + Given the results of the query from get_query, this function generates a single row of a csv. + """ raise NotImplementedError @classmethod def make_report(cls, report_type, filelike, start_date, end_date): + """ + Given the string report_type, a file object to write to, and start/end date bounds, + generates a CSV report of the appropriate type. + """ report = cls.initialize_report(report_type) items = report.get_query(start_date, end_date) writer = unicodecsv.writer(filelike, encoding="utf-8") @@ -611,10 +630,13 @@ class Report(models.Model): class RefundReport(Report): + """ + Subclass of Report, used to generate Refund Reports for finance purposes. + """ def get_query(self, start_date, end_date): return CertificateItem.objects.filter( - status="refunded", - ) + status="refunded", + ) def csv_report_header_row(self): return [ @@ -631,18 +653,22 @@ class RefundReport(Report): item.order_id, item.user.get_full_name(), item.fulfilled_time, - item.refund_requested_time, # actually may need to use refund_fulfilled here + item.refund_requested_time, # TODO actually may need to use refund_fulfilled here item.line_cost, - 0, # TODO: determine if service_fees field is necessary; if so, add + item.service_fee, ] + class ItemizedPurchaseReport(Report): + """ + Subclass of Report, used to generate itemized purchase reports. + """ def get_query(self, start_date, end_date): return OrderItem.objects.filter( - status="purchased", - fulfilled_time__gte=start_date, - fulfilled_time__lt=end_date, - ).order_by("fulfilled_time") + status="purchased", + fulfilled_time__gte=start_date, + fulfilled_time__lt=end_date, + ).order_by("fulfilled_time") def csv_report_header_row(self): return [ @@ -672,13 +698,16 @@ class ItemizedPurchaseReport(Report): class CertificateStatusReport(Report): - def get_query(self, start_date, send_date): + """ + Subclass of Report, used to generate Certificate Status Reports for ed services. + """ + def get_query(self, start_date, end_date): results = [] for course_id in settings.COURSE_LISTINGS['default']: cur_course = get_course_by_id(course_id) university = cur_course.org - course = cur_course.number + " " + cur_course.display_name #TODO add term (i.e. Fall 2013)? - enrollments = CourseEnrollment.objects.filter(course_id=course_id) + course = cur_course.number + " " + cur_course.display_name # TODO add term (i.e. Fall 2013)? + enrollments = CourseEnrollment.objects.filter(course_id=course_id) total_enrolled = enrollments.count() audit_enrolled = enrollments.filter(mode="audit").count() honor_enrolled = enrollments.filter(mode="honor").count() @@ -686,8 +715,8 @@ class CertificateStatusReport(Report): verified_enrolled = verified_enrollments.count() gross_rev_temp = CertificateItem.objects.filter(course_id=course_id, mode="verified").aggregate(Sum('unit_cost')) gross_rev = gross_rev_temp['unit_cost__sum'] - gross_rev_over_min = gross_rev - (CourseMode.objects.get(course_id=course_id,mode_slug="verified").min_price * verified_enrolled) - num_verified_over_min = 0 # TODO clarify with billing what exactly this means + gross_rev_over_min = gross_rev - (CourseMode.objects.get(course_id=course_id, mode_slug="verified").min_price * verified_enrolled) + num_verified_over_min = 0 # TODO clarify with billing what exactly this means refunded_enrollments = CertificateItem.objects.filter(course_id='course_id', mode="refunded") number_of_refunds = refunded_enrollments.count() dollars_refunded_temp = refunded_enrollments.aggregate(Sum('unit_cost')) @@ -697,10 +726,10 @@ class CertificateStatusReport(Report): dollars_refunded = dollars_refunded_temp['unit_cost__sum'] result = [ - university, - course, - total_enrolled, - audit_enrolled, + university, + course, + total_enrolled, + audit_enrolled, honor_enrolled, verified_enrolled, gross_rev, @@ -733,19 +762,39 @@ class CertificateStatusReport(Report): class UniversityRevenueShareReport(Report): + """ + Subclass of Report, used to generate University Revenue Share Reports for finance purposes. + """ def get_query(self, start_date, end_date): results = [] for course_id in settings.COURSE_LISTINGS['default']: cur_course = get_course_by_id(course_id) university = cur_course.org - course = cur_course.number + " " + course.display_name - num_transactions = 0 # TODO clarify with building what transactions are included in this (purchases? refunds? etc) - total_payments_collected = CertificateItem.objects.filter(course_id=course_id, mode="verified").aggregate(Sum('unit_cost')) - #note: we're assuming certitems are the only way to make money right now - service_fees = 0 # TODO add an actual service fees field, in case needed in future - refunded_enrollments = CertificateItem.objects.filter(course_id='course_id', mode="refunded") - num_refunds = refunded_enrollments.objects.count() - amount_refunds = refunded_enrollments.objects.aggregate(Sum('unit_cost')) + course = cur_course.number + " " + cur_course.display_name + num_transactions = 0 # TODO clarify with building what transactions are included in this (purchases? refunds? etc) + + all_paid_certs = CertificateItem.objects.filter(course_id=course_id, status="purchased") + + total_payments_collected_temp = all_paid_certs.aggregate(Sum('unit_cost')) + if total_payments_collected_temp['unit_cost__sum'] is None: + total_payments_collected = Decimal(0.00) + else: + total_payments_collected = total_payments_collected_temp['unit_cost__sum'] + + total_service_fees_temp = all_paid_certs.aggregate(Sum('service_fee')) + if total_service_fees_temp['service_fee__sum'] is None: + service_fees = Decimal(0.00) + else: + service_fees = total_service_fees_temp['service_fee__sum'] + + refunded_enrollments = CertificateItem.objects.filter(course_id=course_id, status="refunded") + num_refunds = refunded_enrollments.count() + + amount_refunds_temp = refunded_enrollments.aggregate(Sum('unit_cost')) + if amount_refunds_temp['unit_cost__sum'] is None: + amount_refunds = Decimal(0.00) + else: + amount_refunds = amount_refunds_temp['unit_cost__sum'] result = [ university, @@ -753,7 +802,6 @@ class UniversityRevenueShareReport(Report): num_transactions, total_payments_collected, service_fees, - refunded_enrollments, num_refunds, amount_refunds ] @@ -765,12 +813,11 @@ class UniversityRevenueShareReport(Report): return [ "University", "Course", - "Number of Transactions", + "Number of Transactions", "Total Payments Collected", "Service Fees (if any)", "Number of Successful Refunds", "Total Amount of Refunds", - # note this is restricted by a date range ] def csv_report_row(self, item): diff --git a/lms/djangoapps/shoppingcart/tests/test_models.py b/lms/djangoapps/shoppingcart/tests/test_models.py index 7edf06b06f..8ff8284b9c 100644 --- a/lms/djangoapps/shoppingcart/tests/test_models.py +++ b/lms/djangoapps/shoppingcart/tests/test_models.py @@ -364,7 +364,6 @@ class RefundReportTest(ModuleStoreTestCase): refunded_certs = report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) self.assertEqual(len(refunded_certs), 1) self.assertIn(self.cert_item, refunded_certs) - # TODO no time restrictions yet test_time = datetime.datetime.now(pytz.UTC) @@ -377,18 +376,13 @@ class RefundReportTest(ModuleStoreTestCase): """ Tests that a generated purchase report CSV is as we expect """ - # coerce the purchase times to self.test_time so that the test can match. - # It's pretty hard to patch datetime.datetime b/c it's a python built-in, which is immutable, so we - # make the times match this way - # TODO test multiple report types report_type = "refund_report" report = Report.initialize_report(report_type) for item in report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS): item.fulfilled_time = self.test_time - item.refund_requested_time = self.test_time #hm do we want to make these different + item.refund_requested_time = self.test_time # hm do we want to make these different item.save() - # add annotation to the csv_file = StringIO.StringIO() Report.make_report(report_type, csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) csv = csv_file.getvalue() @@ -397,10 +391,11 @@ class RefundReportTest(ModuleStoreTestCase): self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_CSV.strip()) - @override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) class ItemizedPurchaseReportTest(ModuleStoreTestCase): - + """ + Tests for the models used to generate itemized purchase reports + """ FIVE_MINS = datetime.timedelta(minutes=5) TEST_ANNOTATION = u'Ba\xfc\u5305' @@ -428,15 +423,13 @@ class ItemizedPurchaseReportTest(ModuleStoreTestCase): self.now = datetime.datetime.now(pytz.UTC) def test_purchased_items_btw_dates(self): - # TODO test multiple report types report_type = "itemized_purchase_report" report = Report.initialize_report(report_type) purchases = report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) self.assertEqual(len(purchases), 2) self.assertIn(self.reg.orderitem_ptr, purchases) self.assertIn(self.cert_item.orderitem_ptr, purchases) - no_purchases = report.get_query(self.now + self.FIVE_MINS, - self.now + self.FIVE_MINS + self.FIVE_MINS) + no_purchases = report.get_query(self.now + self.FIVE_MINS, self.now + self.FIVE_MINS + self.FIVE_MINS) self.assertFalse(no_purchases) test_time = datetime.datetime.now(pytz.UTC) @@ -451,17 +444,12 @@ class ItemizedPurchaseReportTest(ModuleStoreTestCase): """ Tests that a generated purchase report CSV is as we expect """ - # coerce the purchase times to self.test_time so that the test can match. - # It's pretty hard to patch datetime.datetime b/c it's a python built-in, which is immutable, so we - # make the times match this way - # TODO test multiple report types report_type = "itemized_purchase_report" report = Report.initialize_report(report_type) for item in report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS): item.fulfilled_time = self.test_time item.save() - # add annotation to the csv_file = StringIO.StringIO() Report.make_report(report_type, csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) csv = csv_file.getvalue() @@ -484,9 +472,12 @@ class ItemizedPurchaseReportTest(ModuleStoreTestCase): """ self.assertEqual(unicode(self.annotation), u'{} : {}'.format(self.course_id, self.TEST_ANNOTATION)) -# TODO: finish this test class + @override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) class CertificateStatusReportTest(ModuleStoreTestCase): + """ + Tests for the models used to generate certificate status reports + """ FIVE_MINS = datetime.timedelta(minutes=5) def setUp(self): @@ -566,28 +557,38 @@ class CertificateStatusReportTest(ModuleStoreTestCase): MITx,999 Robot Super Course,6,3,1,2,80.00,0.00,0,0,0 """.format(time_str=str(test_time))) - # TODO finish these tests. This is just a basic test to start with, making sure the regular - # flow doesn't throw any strange errors while running def test_basic(self): report_type = "certificate_status" report = Report.initialize_report(report_type) - refunded_certs = report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) csv_file = StringIO.StringIO() report.make_report(report_type, csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) csv = csv_file.getvalue() self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_CSV.strip()) - # TODO no time restrictions ye -# TODO: finish this test class + @override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) class UniversityRevenueShareReportTest(ModuleStoreTestCase): + """ + Tests for the models used to generate university revenue share reports + """ FIVE_MINS = datetime.timedelta(minutes=5) def setUp(self): - self.user = UserFactory.create() - self.user.first_name = "John" - self.user.last_name = "Doe" - self.user.save() + self.user1 = UserFactory.create() + self.user1.first_name = "John" + self.user1.last_name = "Doe" + self.user1.save() + + self.user2 = UserFactory.create() + self.user2.first_name = "Jane" + self.user2.last_name = "Deer" + self.user2.save() + + self.user3 = UserFactory.create() + self.user3.first_name = "Simon" + self.user3.last_name = "Blackquill" + self.user3.save() + self.course_id = "MITx/999/Robot_Super_Course" self.cost = 40 self.course = CourseFactory.create(org='MITx', number='999', display_name=u'Robot Super Course') @@ -603,21 +604,37 @@ class UniversityRevenueShareReportTest(ModuleStoreTestCase): min_price=self.cost) course_mode2.save() - self.cart = Order.get_cart_for_user(self.user) + # user1 is a verified purchase + self.cart = Order.get_cart_for_user(self.user1) CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') self.cart.purchase() + # user2 & user3 are refunded purchases + self.cart = Order.get_cart_for_user(self.user2) + CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') + self.cart.purchase() + CourseEnrollment.unenroll(self.user2, self.course_id) + + self.cart = Order.get_cart_for_user(self.user3) + CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') + self.cart.purchase() + CourseEnrollment.unenroll(self.user3, self.course_id) + self.now = datetime.datetime.now(pytz.UTC) - # TODO finish these tests. This is just a basic test to start with, making sure the regular - # flow doesn't throw any strange errors while running + test_time = datetime.datetime.now(pytz.UTC) + CORRECT_CSV = dedent(""" + University,Course,Number of Transactions,Total Payments Collected,Service Fees (if any),Number of Successful Refunds,Total Amount of Refunds + MITx,999 Robot Super Course,0,40.00,0,2,80.00 + """.format(time_str=str(test_time))) + def test_basic(self): report_type = "university_revenue_share" report = Report.initialize_report(report_type) - refunded_certs = report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) csv_file = StringIO.StringIO() report.make_report(report_type, csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) - # TODO no time restrictions yet + csv = csv_file.getvalue() + self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_CSV.strip()) @override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) diff --git a/lms/djangoapps/shoppingcart/tests/test_views.py b/lms/djangoapps/shoppingcart/tests/test_views.py index 0658d56ab9..d0e59eb680 100644 --- a/lms/djangoapps/shoppingcart/tests/test_views.py +++ b/lms/djangoapps/shoppingcart/tests/test_views.py @@ -14,7 +14,7 @@ from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE from shoppingcart.views import _can_download_report, _get_date_from_str -from shoppingcart.models import Order, CertificateItem, PaidCourseRegistration, OrderItem, Report +from shoppingcart.models import Order, CertificateItem, PaidCourseRegistration, Report from student.tests.factories import UserFactory from student.models import CourseEnrollment from course_modes.models import CourseMode diff --git a/lms/djangoapps/verify_student/views.py b/lms/djangoapps/verify_student/views.py index 1821116574..2ac798bbf2 100644 --- a/lms/djangoapps/verify_student/views.py +++ b/lms/djangoapps/verify_student/views.py @@ -132,7 +132,6 @@ def create_order(request): """ Submit PhotoVerification and create a new Order for this verified cert """ - from nose.tools import set_trace; set_trace() if not SoftwareSecurePhotoVerification.user_has_valid_or_pending(request.user): attempt = SoftwareSecurePhotoVerification(user=request.user) b64_face_image = request.POST['face_image'].split(",")[1] From cf5745ce59d38e6a85d73e7e60bc15c56b6c1210 Mon Sep 17 00:00:00 2001 From: Julia Hansbrough Date: Mon, 9 Dec 2013 21:31:36 +0000 Subject: [PATCH 04/14] Refactored tests --- lms/djangoapps/shoppingcart/models.py | 9 +- .../shoppingcart/tests/test_models.py | 217 ++++++------------ 2 files changed, 76 insertions(+), 150 deletions(-) diff --git a/lms/djangoapps/shoppingcart/models.py b/lms/djangoapps/shoppingcart/models.py index dcc4c22538..de7443c3bb 100644 --- a/lms/djangoapps/shoppingcart/models.py +++ b/lms/djangoapps/shoppingcart/models.py @@ -707,17 +707,18 @@ class CertificateStatusReport(Report): cur_course = get_course_by_id(course_id) university = cur_course.org course = cur_course.number + " " + cur_course.display_name # TODO add term (i.e. Fall 2013)? - enrollments = CourseEnrollment.objects.filter(course_id=course_id) + enrollments = CourseEnrollment.objects.filter(course_id=course_id,is_active=True) total_enrolled = enrollments.count() audit_enrolled = enrollments.filter(mode="audit").count() honor_enrolled = enrollments.filter(mode="honor").count() - verified_enrollments = enrollments.filter(mode="verified") + # Since every verified enrollment has 1 and only 1 cert item, let's just query those + verified_enrollments = CertificateItem.objects.filter(course_id=course_id, mode="verified", status="purchased") verified_enrolled = verified_enrollments.count() - gross_rev_temp = CertificateItem.objects.filter(course_id=course_id, mode="verified").aggregate(Sum('unit_cost')) + gross_rev_temp = CertificateItem.objects.filter(course_id=course_id, mode="verified", status="purchased").aggregate(Sum('unit_cost')) gross_rev = gross_rev_temp['unit_cost__sum'] gross_rev_over_min = gross_rev - (CourseMode.objects.get(course_id=course_id, mode_slug="verified").min_price * verified_enrolled) num_verified_over_min = 0 # TODO clarify with billing what exactly this means - refunded_enrollments = CertificateItem.objects.filter(course_id='course_id', mode="refunded") + refunded_enrollments = CertificateItem.objects.filter(course_id='course_id', mode="verified", status="refunded") number_of_refunds = refunded_enrollments.count() dollars_refunded_temp = refunded_enrollments.aggregate(Sum('unit_cost')) if dollars_refunded_temp['unit_cost__sum'] is None: diff --git a/lms/djangoapps/shoppingcart/tests/test_models.py b/lms/djangoapps/shoppingcart/tests/test_models.py index 8ff8284b9c..cf4ab5dbe7 100644 --- a/lms/djangoapps/shoppingcart/tests/test_models.py +++ b/lms/djangoapps/shoppingcart/tests/test_models.py @@ -323,74 +323,6 @@ class PaidCourseRegistrationTest(ModuleStoreTestCase): self.assertFalse(CourseEnrollment.is_enrolled(self.user, self.course_id)) -@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) -class RefundReportTest(ModuleStoreTestCase): - FIVE_MINS = datetime.timedelta(minutes=5) - - def setUp(self): - self.user = UserFactory.create() - self.user.first_name = "John" - self.user.last_name = "Doe" - self.user.save() - self.course_id = "MITx/999/Robot_Super_Course" - self.cost = 40 - self.course = CourseFactory.create(org='MITx', number='999', display_name=u'Robot Super Course') - course_mode = CourseMode(course_id=self.course_id, - mode_slug="honor", - mode_display_name="honor cert", - min_price=self.cost) - course_mode.save() - - course_mode2 = CourseMode(course_id=self.course_id, - mode_slug="verified", - mode_display_name="verified cert", - min_price=self.cost) - course_mode2.save() - - self.cart = Order.get_cart_for_user(self.user) - CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') - self.cart.purchase() - - # should auto-refund the relevant cert - CourseEnrollment.unenroll(self.user, self.course_id) - - self.cert_item = CertificateItem.objects.get(user=self.user, course_id=self.course_id) - - self.now = datetime.datetime.now(pytz.UTC) - - def test_get_query(self): - report_type = "refund_report" - report = Report.initialize_report(report_type) - refunded_certs = report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) - self.assertEqual(len(refunded_certs), 1) - self.assertIn(self.cert_item, refunded_certs) - - test_time = datetime.datetime.now(pytz.UTC) - - CORRECT_CSV = dedent(""" - Order Number,Customer Name,Date of Original Transaction,Date of Refund,Amount of Refund,Service Fees (if any) - 1,John Doe,{time_str},{time_str},40,0 - """.format(time_str=str(test_time))) - - def test_purchased_csv(self): - """ - Tests that a generated purchase report CSV is as we expect - """ - report_type = "refund_report" - report = Report.initialize_report(report_type) - for item in report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS): - item.fulfilled_time = self.test_time - item.refund_requested_time = self.test_time # hm do we want to make these different - item.save() - - csv_file = StringIO.StringIO() - Report.make_report(report_type, csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) - csv = csv_file.getvalue() - csv_file.close() - # Using excel mode csv, which automatically ends lines with \r\n, so need to convert to \n - self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_CSV.strip()) - - @override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) class ItemizedPurchaseReportTest(ModuleStoreTestCase): """ @@ -474,7 +406,7 @@ class ItemizedPurchaseReportTest(ModuleStoreTestCase): @override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) -class CertificateStatusReportTest(ModuleStoreTestCase): +class ReportTypeTests(ModuleStoreTestCase): """ Tests for the models used to generate certificate status reports """ @@ -512,6 +444,16 @@ class CertificateStatusReportTest(ModuleStoreTestCase): self.user6.last_name = "Peach" self.user6.save() + self.user7 = UserFactory.create() + self.user7.first_name = "King" + self.user7.last_name = "Bowser" + self.user7.save() + + self.user8 = UserFactory.create() + self.user8.first_name = "Susan" + self.user8.last_name = "Smith" + self.user8.save() + # Two are verified, three are audit, one honor self.course_id = "MITx/999/Robot_Super_Course" @@ -540,101 +482,84 @@ class CertificateStatusReportTest(ModuleStoreTestCase): self.cart2.purchase() # Users 3, 4, and 5 are audit - CourseEnrollment.get_or_create_enrollment(self.user3, self.course_id).update_enrollment(mode="audit") - CourseEnrollment.get_or_create_enrollment(self.user4, self.course_id).update_enrollment(mode="audit") - CourseEnrollment.get_or_create_enrollment(self.user5, self.course_id).update_enrollment(mode="audit") + CourseEnrollment.enroll(self.user3, self.course_id, "audit") + CourseEnrollment.enroll(self.user4, self.course_id, "audit") + CourseEnrollment.enroll(self.user5, self.course_id, "audit") # User 6 is honor - CourseEnrollment.get_or_create_enrollment(self.user6, self.course_id).update_enrollment(mode="honor") + CourseEnrollment.enroll(self.user6, self.course_id, "honor") self.now = datetime.datetime.now(pytz.UTC) - # bluh need to test some refunds + # Users 7 & 8 are refunds + self.cart = Order.get_cart_for_user(self.user7) + CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') + self.cart.purchase() + CourseEnrollment.unenroll(self.user7, self.course_id) - test_time = datetime.datetime.now(pytz.UTC) - CORRECT_CSV = dedent(""" - University,Course,Total Enrolled,Audit Enrollment,Honor Code Enrollment,Verified Enrollment,Gross Revenue,Gross Revenue over the Minimum,Number of Verified over the Minimum,Number of Refunds,Dollars Refunded - MITx,999 Robot Super Course,6,3,1,2,80.00,0.00,0,0,0 - """.format(time_str=str(test_time))) + self.cart = Order.get_cart_for_user(self.user8) + CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') + self.cart.purchase(self.user8, self.course_id) + CourseEnrollment.unenroll(self.user8, self.course_id) - def test_basic(self): + self.test_time = datetime.datetime.now(pytz.UTC) + self.CORRECT_REFUND_REPORT_CSV = dedent(""" + Order Number,Customer Name,Date of Original Transaction,Date of Refund,Amount of Refund,Service Fees (if any) + 3,King Bowser,{time_str},{time_str},40,0 + 4,Susan Smith,{time_str},{time_str},40,0 + """.format(time_str=str(self.test_time))) + + self.CORRECT_CERT_STATUS_CSV = dedent(""" + University,Course,Total Enrolled,Audit Enrollment,Honor Code Enrollment,Verified Enrollment,Gross Revenue,Gross Revenue over the Minimum,Number of Verified over the Minimum,Number of Refunds,Dollars Refunded + MITx,999 Robot Super Course,6,3,1,2,80.00,0.00,0,0,0 + """.format(time_str=str(self.test_time))) + + self.CORRECT_UNI_REVENUE_SHARE_CSV = dedent(""" + University,Course,Number of Transactions,Total Payments Collected,Service Fees (if any),Number of Successful Refunds,Total Amount of Refunds + MITx,999 Robot Super Course,0,80.00,0.00,2,80.00 + """.format(time_str=str(self.test_time))) + + def test_refund_report_get_query(self): + report_type = "refund_report" + report = Report.initialize_report(report_type) + refunded_certs = report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + self.assertEqual(len(refunded_certs), 2) + self.assertTrue(CertificateItem.objects.get(user=self.user7, course_id=self.course_id)) + self.assertTrue(CertificateItem.objects.get(user=self.user8, course_id=self.course_id)) + + def test_refund_report_purchased_csv(self): + """ + Tests that a generated purchase report CSV is as we expect + """ + report_type = "refund_report" + report = Report.initialize_report(report_type) + for item in report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS): + item.fulfilled_time = self.test_time + item.refund_requested_time = self.test_time # hm do we want to make these different + item.save() + + csv_file = StringIO.StringIO() + Report.make_report(report_type, csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + csv = csv_file.getvalue() + csv_file.close() + # Using excel mode csv, which automatically ends lines with \r\n, so need to convert to \n + self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_REFUND_REPORT_CSV.strip()) + + def test_basic_cert_status_csv(self): report_type = "certificate_status" report = Report.initialize_report(report_type) csv_file = StringIO.StringIO() report.make_report(report_type, csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) csv = csv_file.getvalue() - self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_CSV.strip()) + self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_CERT_STATUS_CSV.strip()) - -@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) -class UniversityRevenueShareReportTest(ModuleStoreTestCase): - """ - Tests for the models used to generate university revenue share reports - """ - FIVE_MINS = datetime.timedelta(minutes=5) - - def setUp(self): - self.user1 = UserFactory.create() - self.user1.first_name = "John" - self.user1.last_name = "Doe" - self.user1.save() - - self.user2 = UserFactory.create() - self.user2.first_name = "Jane" - self.user2.last_name = "Deer" - self.user2.save() - - self.user3 = UserFactory.create() - self.user3.first_name = "Simon" - self.user3.last_name = "Blackquill" - self.user3.save() - - self.course_id = "MITx/999/Robot_Super_Course" - self.cost = 40 - self.course = CourseFactory.create(org='MITx', number='999', display_name=u'Robot Super Course') - course_mode = CourseMode(course_id=self.course_id, - mode_slug="honor", - mode_display_name="honor cert", - min_price=self.cost) - course_mode.save() - - course_mode2 = CourseMode(course_id=self.course_id, - mode_slug="verified", - mode_display_name="verified cert", - min_price=self.cost) - course_mode2.save() - - # user1 is a verified purchase - self.cart = Order.get_cart_for_user(self.user1) - CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') - self.cart.purchase() - - # user2 & user3 are refunded purchases - self.cart = Order.get_cart_for_user(self.user2) - CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') - self.cart.purchase() - CourseEnrollment.unenroll(self.user2, self.course_id) - - self.cart = Order.get_cart_for_user(self.user3) - CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') - self.cart.purchase() - CourseEnrollment.unenroll(self.user3, self.course_id) - - self.now = datetime.datetime.now(pytz.UTC) - - test_time = datetime.datetime.now(pytz.UTC) - CORRECT_CSV = dedent(""" - University,Course,Number of Transactions,Total Payments Collected,Service Fees (if any),Number of Successful Refunds,Total Amount of Refunds - MITx,999 Robot Super Course,0,40.00,0,2,80.00 - """.format(time_str=str(test_time))) - - def test_basic(self): + def test_basic_uni_revenue_share_csv(self): report_type = "university_revenue_share" report = Report.initialize_report(report_type) csv_file = StringIO.StringIO() report.make_report(report_type, csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) csv = csv_file.getvalue() - self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_CSV.strip()) + self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_UNI_REVENUE_SHARE_CSV.strip()) @override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) From e3114a54b9ff42e2606f62638b92dcbfc3c5b778 Mon Sep 17 00:00:00 2001 From: Julia Hansbrough Date: Mon, 9 Dec 2013 22:15:59 +0000 Subject: [PATCH 05/14] Migrations --- ...ort__add_certificatestatusreport__add_i.py | 189 ++++++++++++++++++ lms/djangoapps/verify_student/models.py | 2 +- 2 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 lms/djangoapps/shoppingcart/migrations/0007_auto__add_refundreport__add_report__add_certificatestatusreport__add_i.py diff --git a/lms/djangoapps/shoppingcart/migrations/0007_auto__add_refundreport__add_report__add_certificatestatusreport__add_i.py b/lms/djangoapps/shoppingcart/migrations/0007_auto__add_refundreport__add_report__add_certificatestatusreport__add_i.py new file mode 100644 index 0000000000..f2461e7885 --- /dev/null +++ b/lms/djangoapps/shoppingcart/migrations/0007_auto__add_refundreport__add_report__add_certificatestatusreport__add_i.py @@ -0,0 +1,189 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'RefundReport' + db.create_table('shoppingcart_refundreport', ( + ('report_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['shoppingcart.Report'], unique=True, primary_key=True)), + )) + db.send_create_signal('shoppingcart', ['RefundReport']) + + # Adding model 'Report' + db.create_table('shoppingcart_report', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + )) + db.send_create_signal('shoppingcart', ['Report']) + + # Adding model 'CertificateStatusReport' + db.create_table('shoppingcart_certificatestatusreport', ( + ('report_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['shoppingcart.Report'], unique=True, primary_key=True)), + )) + db.send_create_signal('shoppingcart', ['CertificateStatusReport']) + + # Adding model 'ItemizedPurchaseReport' + db.create_table('shoppingcart_itemizedpurchasereport', ( + ('report_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['shoppingcart.Report'], unique=True, primary_key=True)), + )) + db.send_create_signal('shoppingcart', ['ItemizedPurchaseReport']) + + # Adding model 'UniversityRevenueShareReport' + db.create_table('shoppingcart_universityrevenuesharereport', ( + ('report_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['shoppingcart.Report'], unique=True, primary_key=True)), + )) + db.send_create_signal('shoppingcart', ['UniversityRevenueShareReport']) + + # Adding field 'OrderItem.service_fee' + db.add_column('shoppingcart_orderitem', 'service_fee', + self.gf('django.db.models.fields.DecimalField')(default=0.0, max_digits=30, decimal_places=2), + keep_default=False) + + + def backwards(self, orm): + # Deleting model 'RefundReport' + db.delete_table('shoppingcart_refundreport') + + # Deleting model 'Report' + db.delete_table('shoppingcart_report') + + # Deleting model 'CertificateStatusReport' + db.delete_table('shoppingcart_certificatestatusreport') + + # Deleting model 'ItemizedPurchaseReport' + db.delete_table('shoppingcart_itemizedpurchasereport') + + # Deleting model 'UniversityRevenueShareReport' + db.delete_table('shoppingcart_universityrevenuesharereport') + + # Deleting field 'OrderItem.service_fee' + db.delete_column('shoppingcart_orderitem', 'service_fee') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'shoppingcart.certificateitem': { + 'Meta': {'object_name': 'CertificateItem', '_ormbases': ['shoppingcart.OrderItem']}, + 'course_enrollment': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['student.CourseEnrollment']"}), + 'course_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'mode': ('django.db.models.fields.SlugField', [], {'max_length': '50'}), + 'orderitem_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['shoppingcart.OrderItem']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'shoppingcart.certificatestatusreport': { + 'Meta': {'object_name': 'CertificateStatusReport', '_ormbases': ['shoppingcart.Report']}, + 'report_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['shoppingcart.Report']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'shoppingcart.itemizedpurchasereport': { + 'Meta': {'object_name': 'ItemizedPurchaseReport', '_ormbases': ['shoppingcart.Report']}, + 'report_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['shoppingcart.Report']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'shoppingcart.order': { + 'Meta': {'object_name': 'Order'}, + 'bill_to_cardtype': ('django.db.models.fields.CharField', [], {'max_length': '32', 'blank': 'True'}), + 'bill_to_ccnum': ('django.db.models.fields.CharField', [], {'max_length': '8', 'blank': 'True'}), + 'bill_to_city': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}), + 'bill_to_country': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}), + 'bill_to_first': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}), + 'bill_to_last': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}), + 'bill_to_postalcode': ('django.db.models.fields.CharField', [], {'max_length': '16', 'blank': 'True'}), + 'bill_to_state': ('django.db.models.fields.CharField', [], {'max_length': '8', 'blank': 'True'}), + 'bill_to_street1': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}), + 'bill_to_street2': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}), + 'currency': ('django.db.models.fields.CharField', [], {'default': "'usd'", 'max_length': '8'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'processor_reply_dump': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'purchase_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'refunded_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'cart'", 'max_length': '32'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'shoppingcart.orderitem': { + 'Meta': {'object_name': 'OrderItem'}, + 'currency': ('django.db.models.fields.CharField', [], {'default': "'usd'", 'max_length': '8'}), + 'fulfilled_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'line_desc': ('django.db.models.fields.CharField', [], {'default': "'Misc. Item'", 'max_length': '1024'}), + 'order': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['shoppingcart.Order']"}), + 'qty': ('django.db.models.fields.IntegerField', [], {'default': '1'}), + 'refund_requested_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'report_comments': ('django.db.models.fields.TextField', [], {'default': "''"}), + 'service_fee': ('django.db.models.fields.DecimalField', [], {'default': '0.0', 'max_digits': '30', 'decimal_places': '2'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'cart'", 'max_length': '32'}), + 'unit_cost': ('django.db.models.fields.DecimalField', [], {'default': '0.0', 'max_digits': '30', 'decimal_places': '2'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'shoppingcart.paidcourseregistration': { + 'Meta': {'object_name': 'PaidCourseRegistration', '_ormbases': ['shoppingcart.OrderItem']}, + 'course_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'mode': ('django.db.models.fields.SlugField', [], {'default': "'honor'", 'max_length': '50'}), + 'orderitem_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['shoppingcart.OrderItem']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'shoppingcart.paidcourseregistrationannotation': { + 'Meta': {'object_name': 'PaidCourseRegistrationAnnotation'}, + 'annotation': ('django.db.models.fields.TextField', [], {'null': 'True'}), + 'course_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + 'shoppingcart.refundreport': { + 'Meta': {'object_name': 'RefundReport', '_ormbases': ['shoppingcart.Report']}, + 'report_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['shoppingcart.Report']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'shoppingcart.report': { + 'Meta': {'object_name': 'Report'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + 'shoppingcart.universityrevenuesharereport': { + 'Meta': {'object_name': 'UniversityRevenueShareReport', '_ormbases': ['shoppingcart.Report']}, + 'report_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['shoppingcart.Report']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'student.courseenrollment': { + 'Meta': {'ordering': "('user', 'course_id')", 'unique_together': "(('user', 'course_id'),)", 'object_name': 'CourseEnrollment'}, + 'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'null': 'True', 'db_index': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'mode': ('django.db.models.fields.CharField', [], {'default': "'honor'", 'max_length': '100'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + } + } + + complete_apps = ['shoppingcart'] \ No newline at end of file diff --git a/lms/djangoapps/verify_student/models.py b/lms/djangoapps/verify_student/models.py index 52ec2b3cd3..dccbdb430a 100644 --- a/lms/djangoapps/verify_student/models.py +++ b/lms/djangoapps/verify_student/models.py @@ -477,7 +477,7 @@ class SoftwareSecurePhotoVerification(PhotoVerification): # developing and aren't interested in working on student identity # verification functionality. If you do want to work on it, you have to # explicitly enable these in your private settings. - if settings.MITX_FEATURES.get('AUTOMATIC_VERIFY_STUDENT_IDENTITY_FOR_TESTING'): + if settings.FEATURES.get('AUTOMATIC_VERIFY_STUDENT_IDENTITY_FOR_TESTING'): return aes_key_str = settings.VERIFY_STUDENT["SOFTWARE_SECURE"]["FACE_IMAGE_AES_KEY"] From 68174dc33daf80d8159a6ecec5e4a5f403dc1d50 Mon Sep 17 00:00:00 2001 From: Julia Hansbrough Date: Tue, 10 Dec 2013 20:10:48 +0000 Subject: [PATCH 06/14] Initial view --- lms/djangoapps/shoppingcart/models.py | 12 ++++----- .../shoppingcart/tests/test_models.py | 4 +-- .../shoppingcart/tests/test_views.py | 25 ++++++++++++------- lms/djangoapps/shoppingcart/urls.py | 3 ++- lms/djangoapps/shoppingcart/views.py | 13 +++++----- .../shoppingcart/download_report.html | 13 +++++++--- 6 files changed, 42 insertions(+), 28 deletions(-) diff --git a/lms/djangoapps/shoppingcart/models.py b/lms/djangoapps/shoppingcart/models.py index de7443c3bb..3063a0b98d 100644 --- a/lms/djangoapps/shoppingcart/models.py +++ b/lms/djangoapps/shoppingcart/models.py @@ -636,6 +636,8 @@ class RefundReport(Report): def get_query(self, start_date, end_date): return CertificateItem.objects.filter( status="refunded", + refund_requested_time__gte=start_date, + refund_requested_time__lt=end_date, ) def csv_report_header_row(self): @@ -653,7 +655,7 @@ class RefundReport(Report): item.order_id, item.user.get_full_name(), item.fulfilled_time, - item.refund_requested_time, # TODO actually may need to use refund_fulfilled here + item.refund_requested_time, # TODO Change this torefund_fulfilled once we start recording that value item.line_cost, item.service_fee, ] @@ -707,7 +709,8 @@ class CertificateStatusReport(Report): cur_course = get_course_by_id(course_id) university = cur_course.org course = cur_course.number + " " + cur_course.display_name # TODO add term (i.e. Fall 2013)? - enrollments = CourseEnrollment.objects.filter(course_id=course_id,is_active=True) + enrollments = CourseEnrollment.objects.filter(course_id=course_id, + is_active=True,) total_enrolled = enrollments.count() audit_enrolled = enrollments.filter(mode="audit").count() honor_enrolled = enrollments.filter(mode="honor").count() @@ -717,7 +720,6 @@ class CertificateStatusReport(Report): gross_rev_temp = CertificateItem.objects.filter(course_id=course_id, mode="verified", status="purchased").aggregate(Sum('unit_cost')) gross_rev = gross_rev_temp['unit_cost__sum'] gross_rev_over_min = gross_rev - (CourseMode.objects.get(course_id=course_id, mode_slug="verified").min_price * verified_enrolled) - num_verified_over_min = 0 # TODO clarify with billing what exactly this means refunded_enrollments = CertificateItem.objects.filter(course_id='course_id', mode="verified", status="refunded") number_of_refunds = refunded_enrollments.count() dollars_refunded_temp = refunded_enrollments.aggregate(Sum('unit_cost')) @@ -735,7 +737,6 @@ class CertificateStatusReport(Report): verified_enrolled, gross_rev, gross_rev_over_min, - num_verified_over_min, number_of_refunds, dollars_refunded ] @@ -753,7 +754,6 @@ class CertificateStatusReport(Report): "Verified Enrollment", "Gross Revenue", "Gross Revenue over the Minimum", - "Number of Verified over the Minimum", "Number of Refunds", "Dollars Refunded", ] @@ -772,7 +772,7 @@ class UniversityRevenueShareReport(Report): cur_course = get_course_by_id(course_id) university = cur_course.org course = cur_course.number + " " + cur_course.display_name - num_transactions = 0 # TODO clarify with building what transactions are included in this (purchases? refunds? etc) + num_transactions = 0 # TODO clarify with billing what transactions are included in this (purchases? refunds? etc) all_paid_certs = CertificateItem.objects.filter(course_id=course_id, status="purchased") diff --git a/lms/djangoapps/shoppingcart/tests/test_models.py b/lms/djangoapps/shoppingcart/tests/test_models.py index cf4ab5dbe7..80dd5a1423 100644 --- a/lms/djangoapps/shoppingcart/tests/test_models.py +++ b/lms/djangoapps/shoppingcart/tests/test_models.py @@ -510,8 +510,8 @@ class ReportTypeTests(ModuleStoreTestCase): """.format(time_str=str(self.test_time))) self.CORRECT_CERT_STATUS_CSV = dedent(""" - University,Course,Total Enrolled,Audit Enrollment,Honor Code Enrollment,Verified Enrollment,Gross Revenue,Gross Revenue over the Minimum,Number of Verified over the Minimum,Number of Refunds,Dollars Refunded - MITx,999 Robot Super Course,6,3,1,2,80.00,0.00,0,0,0 + University,Course,Total Enrolled,Audit Enrollment,Honor Code Enrollment,Verified Enrollment,Gross Revenue,Gross Revenue over the Minimum,Number of Refunds,Dollars Refunded + MITx,999 Robot Super Course,6,3,1,2,80.00,0.00,0,0 """.format(time_str=str(self.test_time))) self.CORRECT_UNI_REVENUE_SHARE_CSV = dedent(""" diff --git a/lms/djangoapps/shoppingcart/tests/test_views.py b/lms/djangoapps/shoppingcart/tests/test_views.py index d0e59eb680..a08decd05f 100644 --- a/lms/djangoapps/shoppingcart/tests/test_views.py +++ b/lms/djangoapps/shoppingcart/tests/test_views.py @@ -304,6 +304,11 @@ class CSVReportViewsTest(ModuleStoreTestCase): mode_display_name="honor cert", min_price=self.cost) self.course_mode.save() + self.course_mode2 = CourseMode(course_id=self.course_id, + mode_slug="verified", + mode_display_name="verified cert", + min_price=self.cost) + self.course_mode2.save() self.verified_course_id = 'org/test/Test_Course' CourseFactory.create(org='org', number='test', run='course1', display_name='Test Course') self.cart = Order.get_cart_for_user(self.user) @@ -343,13 +348,14 @@ class CSVReportViewsTest(ModuleStoreTestCase): self.assertEqual(template, 'shoppingcart/download_report.html') self.assertFalse(context['total_count_error']) self.assertFalse(context['date_fmt_error']) - self.assertIn(_("Download Purchase Report"), response.content) + self.assertIn(_("Download CSV Reports"), response.content) @patch('shoppingcart.views.render_to_response', render_mock) def test_report_csv_bad_date(self): self.login_user() self.add_to_download_group(self.user) - response = self.client.post(reverse('payment_csv_report'), {'start_date': 'BAD', 'end_date': 'BAD'}) + report_type = "itemized_purchase_report" + response = self.client.post(reverse('payment_csv_report'), {'start_date': 'BAD', 'end_date': 'BAD', 'requested_report': 'itemized_purchase_report'}) ((template, context), unused_kwargs) = render_mock.call_args self.assertEqual(template, 'shoppingcart/download_report.html') @@ -366,7 +372,8 @@ class CSVReportViewsTest(ModuleStoreTestCase): self.login_user() self.add_to_download_group(self.user) response = self.client.post(reverse('payment_csv_report'), {'start_date': '1970-01-01', - 'end_date': '2100-01-01'}) + 'end_date': '2100-01-01', + 'requested_report': 'itemized_purchase_report'}) ((template, context), unused_kwargs) = render_mock.call_args self.assertEqual(template, 'shoppingcart/download_report.html') @@ -376,22 +383,22 @@ class CSVReportViewsTest(ModuleStoreTestCase): # just going to ignored the date in this test, since we already deal with date testing # in test_models.py - CORRECT_CSV_NO_DATE = ",1,purchased,1,40,40,usd,Registration for Course: Robot Super Course," + + CORRECT_CSV_NO_DATE_ITEMIZED_PURCHASE = ",1,purchased,1,40,40,usd,Registration for Course: Robot Super Course," def test_report_csv(self): - # TODO test multiple types - report_type = "itemized_purchase_report" - + report_type = 'itemized_purchase_report' PaidCourseRegistration.add_to_order(self.cart, self.course_id) self.cart.purchase() self.login_user() self.add_to_download_group(self.user) response = self.client.post(reverse('payment_csv_report'), {'start_date': '1970-01-01', - 'end_date': '2100-01-01'}) + 'end_date': '2100-01-01', + 'requested_report': report_type}) self.assertEqual(response['Content-Type'], 'text/csv') report = Report.initialize_report(report_type) self.assertIn(",".join(report.csv_report_header_row()), response.content) - self.assertIn(self.CORRECT_CSV_NO_DATE, response.content) + self.assertIn(self.CORRECT_CSV_NO_DATE_ITEMIZED_PURCHASE, response.content) class UtilFnsTest(TestCase): diff --git a/lms/djangoapps/shoppingcart/urls.py b/lms/djangoapps/shoppingcart/urls.py index b9797e9a5b..ba4119aef7 100644 --- a/lms/djangoapps/shoppingcart/urls.py +++ b/lms/djangoapps/shoppingcart/urls.py @@ -5,6 +5,7 @@ urlpatterns = patterns('shoppingcart.views', # nopep8 url(r'^postpay_callback/$', 'postpay_callback'), # Both the ~accept and ~reject callback pages are handled here url(r'^receipt/(?P[0-9]*)/$', 'show_receipt'), ) + if settings.FEATURES['ENABLE_SHOPPING_CART']: urlpatterns += patterns( 'shoppingcart.views', @@ -19,5 +20,5 @@ if settings.FEATURES.get('ENABLE_PAYMENT_FAKE'): from shoppingcart.tests.payment_fake import PaymentFakeView urlpatterns += patterns( 'shoppingcart.tests.payment_fake', - url(r'^payment_fake', PaymentFakeView.as_view()) + url(r'^payment_fake', PaymentFakeView.as_view()), ) diff --git a/lms/djangoapps/shoppingcart/views.py b/lms/djangoapps/shoppingcart/views.py index 907c0dcbce..b19e2244f7 100644 --- a/lms/djangoapps/shoppingcart/views.py +++ b/lms/djangoapps/shoppingcart/views.py @@ -95,7 +95,6 @@ def postpay_callback(request): return render_to_response('shoppingcart/error.html', {'order': result['order'], 'error_html': result['error_html']}) - @login_required def show_receipt(request, ordernum): """ @@ -156,7 +155,7 @@ def _get_date_from_str(date_input): return datetime.datetime.strptime(date_input.strip(), "%Y-%m-%d").replace(tzinfo=pytz.UTC) -def _render_report_form(start_str, end_str, total_count_error=False, date_fmt_error=False): +def _render_report_form(start_str, end_str, report_type, total_count_error=False, date_fmt_error=False): """ Helper function that renders the purchase form. Reduces repetition """ @@ -165,6 +164,7 @@ def _render_report_form(start_str, end_str, total_count_error=False, date_fmt_er 'date_fmt_error': date_fmt_error, 'start_date': start_str, 'end_date': end_str, + 'requested_report': report_type, } return render_to_response('shoppingcart/download_report.html', context) @@ -174,8 +174,6 @@ def csv_report(request): """ Downloads csv reporting of orderitems """ - # TODO: change this to something modular later - report_type = "itemized_purchase_report" if not _can_download_report(request.user): return HttpResponseForbidden(_('You do not have permission to view this page.')) @@ -183,18 +181,19 @@ def csv_report(request): if request.method == 'POST': start_str = request.POST.get('start_date', '') end_str = request.POST.get('end_date', '') + report_type = request.POST.get('requested_report', '') try: start_date = _get_date_from_str(start_str) end_date = _get_date_from_str(end_str) + datetime.timedelta(days=1) except ValueError: # Error case: there was a badly formatted user-input date string - return _render_report_form(start_str, end_str, date_fmt_error=True) + return _render_report_form(start_str, end_str, report_type, date_fmt_error=True) report = Report.initialize_report(report_type) items = report.get_query(start_date, end_date) if items.count() > settings.PAYMENT_REPORT_MAX_ITEMS: # Error case: too many items would be generated in the report and we're at risk of timeout - return _render_report_form(start_str, end_str, total_count_error=True) + return _render_report_form(start_str, end_str, report_type, total_count_error=True) response = HttpResponse(mimetype='text/csv') filename = "purchases_report_{}.csv".format(datetime.datetime.now(pytz.UTC).strftime("%Y-%m-%d-%H-%M-%S")) @@ -206,7 +205,7 @@ def csv_report(request): elif request.method == 'GET': end_date = datetime.datetime.now(pytz.UTC) start_date = end_date - datetime.timedelta(days=30) - return _render_report_form(start_date.strftime("%Y-%m-%d"), end_date.strftime("%Y-%m-%d")) + return _render_report_form(start_date.strftime("%Y-%m-%d"), end_date.strftime("%Y-%m-%d"), report_type="") else: return HttpResponseBadRequest("HTTP Method Not Supported") diff --git a/lms/templates/shoppingcart/download_report.html b/lms/templates/shoppingcart/download_report.html index 838b07f145..67cdbce913 100644 --- a/lms/templates/shoppingcart/download_report.html +++ b/lms/templates/shoppingcart/download_report.html @@ -2,11 +2,11 @@ <%! from django.core.urlresolvers import reverse %> <%inherit file="../main.html" /> -<%block name="title">${_("Download Purchase Report")} +<%block name="title">${_("Download CSV Reports")}
-

${_("Download CSV of purchase data")}

+

${_("Download CSV Data")}

% if date_fmt_error:
${_("There was an error in your date input. It should be formatted as YYYY-MM-DD")} @@ -24,6 +24,13 @@ - +
+ +
+ +
+ +
+
From edd0b5436e76347ebbad0a2978f38d71758d16a4 Mon Sep 17 00:00:00 2001 From: Julia Hansbrough Date: Wed, 18 Dec 2013 21:09:41 +0000 Subject: [PATCH 07/14] Response to CR --- common/djangoapps/student/models.py | 5 + lms/djangoapps/shoppingcart/models.py | 261 +----------------- lms/djangoapps/shoppingcart/reports.py | 249 +++++++++++++++++ .../shoppingcart/tests/test_models.py | 192 ++----------- .../shoppingcart/tests/test_reports.py | 179 ++++++++++++ .../shoppingcart/tests/test_views.py | 23 +- lms/djangoapps/shoppingcart/views.py | 53 +++- .../shoppingcart/download_report.html | 4 + 8 files changed, 525 insertions(+), 441 deletions(-) create mode 100644 lms/djangoapps/shoppingcart/reports.py create mode 100644 lms/djangoapps/shoppingcart/tests/test_reports.py diff --git a/common/djangoapps/student/models.py b/common/djangoapps/student/models.py index 935abbdd2e..9e98bfd965 100644 --- a/common/djangoapps/student/models.py +++ b/common/djangoapps/student/models.py @@ -580,6 +580,11 @@ class CourseEnrollment(models.Model): courseenrollment__is_active=True ) + @classmethod + def enrollments_in(cls, course_id): + """Return a queryset of CourseEnrollment for every active enrollment in the course.""" + return cls.objects.filter(course_id=course_id, is_active=True,) + def activate(self): """Makes this `CourseEnrollment` record active. Saves immediately.""" self.update_enrollment(is_active=True) diff --git a/lms/djangoapps/shoppingcart/models.py b/lms/djangoapps/shoppingcart/models.py index 3063a0b98d..685f320e4d 100644 --- a/lms/djangoapps/shoppingcart/models.py +++ b/lms/djangoapps/shoppingcart/models.py @@ -10,7 +10,6 @@ from boto.exception import BotoServerError # this is a super-class of SESError from django.dispatch import receiver from django.db import models -from django.db.models import Sum from django.conf import settings from django.core.exceptions import ObjectDoesNotExist from django.core.mail import send_mail @@ -19,14 +18,11 @@ from django.utils.translation import ugettext as _ from django.db import transaction from django.core.urlresolvers import reverse -from decimal import Decimal - from xmodule.modulestore.django import modulestore from xmodule.course_module import CourseDescriptor from xmodule.modulestore.exceptions import ItemNotFoundError from course_modes.models import CourseMode -from courseware.courses import get_course_by_id from edxmako.shortcuts import render_to_string from student.views import course_from_id from student.models import CourseEnrollment, unenroll_done @@ -34,8 +30,7 @@ from student.models import CourseEnrollment, unenroll_done from verify_student.models import SoftwareSecurePhotoVerification from .exceptions import (InvalidCartItem, PurchasedCallbackException, ItemAlreadyInCartException, - AlreadyEnrolledInCourseException, CourseDoesNotExistException, ReportException, - ReportTypeDoesNotExistException) + AlreadyEnrolledInCourseException, CourseDoesNotExistException, ReportException) log = logging.getLogger("shoppingcart") @@ -45,6 +40,8 @@ ORDER_STATUSES = ( ('refunded', 'refunded'), ) + + # we need a tuple to represent the primary key of various OrderItem subclasses OrderItemSubclassPK = namedtuple('OrderItemSubclassPK', ['cls', 'pk']) # pylint: disable=C0103 @@ -571,255 +568,3 @@ class CertificateItem(OrderItem): "Please include your order number in your e-mail. " "Please do NOT include your credit card information.").format( billing_email=settings.PAYMENT_SUPPORT_EMAIL) - - -class Report(models.Model): - """ - Base class for making CSV reports related to revenue, enrollments, etc - - To make a different type of report, write a new subclass that implements - the methods get_query, csv_report_header_row, and csv_report_row. - """ - - @classmethod - def initialize_report(cls, report_type): - """ - Creates the appropriate type of Report object based on the string report_type. - """ - if report_type == "refund_report": - return RefundReport() - elif report_type == "itemized_purchase_report": - return ItemizedPurchaseReport() - elif report_type == "university_revenue_share": - return UniversityRevenueShareReport() - elif report_type == "certificate_status": - return CertificateStatusReport() - else: - raise ReportTypeDoesNotExistException - - def get_query(self, start_date, end_date): - """ - Performs any database queries necessary to obtain the data for the report. - """ - raise NotImplementedError - - def csv_report_header_row(self): - """ - Returns the appropriate header based on the report type. - """ - raise NotImplementedError - - def csv_report_row(self, item): - """ - Given the results of the query from get_query, this function generates a single row of a csv. - """ - raise NotImplementedError - - @classmethod - def make_report(cls, report_type, filelike, start_date, end_date): - """ - Given the string report_type, a file object to write to, and start/end date bounds, - generates a CSV report of the appropriate type. - """ - report = cls.initialize_report(report_type) - items = report.get_query(start_date, end_date) - writer = unicodecsv.writer(filelike, encoding="utf-8") - writer.writerow(report.csv_report_header_row()) - for item in items: - writer.writerow(report.csv_report_row(item)) - - -class RefundReport(Report): - """ - Subclass of Report, used to generate Refund Reports for finance purposes. - """ - def get_query(self, start_date, end_date): - return CertificateItem.objects.filter( - status="refunded", - refund_requested_time__gte=start_date, - refund_requested_time__lt=end_date, - ) - - def csv_report_header_row(self): - return [ - "Order Number", - "Customer Name", - "Date of Original Transaction", - "Date of Refund", - "Amount of Refund", - "Service Fees (if any)", - ] - - def csv_report_row(self, item): - return [ - item.order_id, - item.user.get_full_name(), - item.fulfilled_time, - item.refund_requested_time, # TODO Change this torefund_fulfilled once we start recording that value - item.line_cost, - item.service_fee, - ] - - -class ItemizedPurchaseReport(Report): - """ - Subclass of Report, used to generate itemized purchase reports. - """ - def get_query(self, start_date, end_date): - return OrderItem.objects.filter( - status="purchased", - fulfilled_time__gte=start_date, - fulfilled_time__lt=end_date, - ).order_by("fulfilled_time") - - def csv_report_header_row(self): - return [ - "Purchase Time", - "Order ID", - "Status", - "Quantity", - "Unit Cost", - "Total Cost", - "Currency", - "Description", - "Comments" - ] - - def csv_report_row(self, item): - return [ - item.fulfilled_time, - item.order_id, # pylint: disable=no-member - item.status, - item.qty, - item.unit_cost, - item.line_cost, - item.currency, - item.line_desc, - item.report_comments, - ] - - -class CertificateStatusReport(Report): - """ - Subclass of Report, used to generate Certificate Status Reports for ed services. - """ - def get_query(self, start_date, end_date): - results = [] - for course_id in settings.COURSE_LISTINGS['default']: - cur_course = get_course_by_id(course_id) - university = cur_course.org - course = cur_course.number + " " + cur_course.display_name # TODO add term (i.e. Fall 2013)? - enrollments = CourseEnrollment.objects.filter(course_id=course_id, - is_active=True,) - total_enrolled = enrollments.count() - audit_enrolled = enrollments.filter(mode="audit").count() - honor_enrolled = enrollments.filter(mode="honor").count() - # Since every verified enrollment has 1 and only 1 cert item, let's just query those - verified_enrollments = CertificateItem.objects.filter(course_id=course_id, mode="verified", status="purchased") - verified_enrolled = verified_enrollments.count() - gross_rev_temp = CertificateItem.objects.filter(course_id=course_id, mode="verified", status="purchased").aggregate(Sum('unit_cost')) - gross_rev = gross_rev_temp['unit_cost__sum'] - gross_rev_over_min = gross_rev - (CourseMode.objects.get(course_id=course_id, mode_slug="verified").min_price * verified_enrolled) - refunded_enrollments = CertificateItem.objects.filter(course_id='course_id', mode="verified", status="refunded") - number_of_refunds = refunded_enrollments.count() - dollars_refunded_temp = refunded_enrollments.aggregate(Sum('unit_cost')) - if dollars_refunded_temp['unit_cost__sum'] is None: - dollars_refunded = Decimal(0.00) - else: - dollars_refunded = dollars_refunded_temp['unit_cost__sum'] - - result = [ - university, - course, - total_enrolled, - audit_enrolled, - honor_enrolled, - verified_enrolled, - gross_rev, - gross_rev_over_min, - number_of_refunds, - dollars_refunded - ] - - results.append(result) - return results - - def csv_report_header_row(self): - return [ - "University", - "Course", - "Total Enrolled", - "Audit Enrollment", - "Honor Code Enrollment", - "Verified Enrollment", - "Gross Revenue", - "Gross Revenue over the Minimum", - "Number of Refunds", - "Dollars Refunded", - ] - - def csv_report_row(self, item): - return item - - -class UniversityRevenueShareReport(Report): - """ - Subclass of Report, used to generate University Revenue Share Reports for finance purposes. - """ - def get_query(self, start_date, end_date): - results = [] - for course_id in settings.COURSE_LISTINGS['default']: - cur_course = get_course_by_id(course_id) - university = cur_course.org - course = cur_course.number + " " + cur_course.display_name - num_transactions = 0 # TODO clarify with billing what transactions are included in this (purchases? refunds? etc) - - all_paid_certs = CertificateItem.objects.filter(course_id=course_id, status="purchased") - - total_payments_collected_temp = all_paid_certs.aggregate(Sum('unit_cost')) - if total_payments_collected_temp['unit_cost__sum'] is None: - total_payments_collected = Decimal(0.00) - else: - total_payments_collected = total_payments_collected_temp['unit_cost__sum'] - - total_service_fees_temp = all_paid_certs.aggregate(Sum('service_fee')) - if total_service_fees_temp['service_fee__sum'] is None: - service_fees = Decimal(0.00) - else: - service_fees = total_service_fees_temp['service_fee__sum'] - - refunded_enrollments = CertificateItem.objects.filter(course_id=course_id, status="refunded") - num_refunds = refunded_enrollments.count() - - amount_refunds_temp = refunded_enrollments.aggregate(Sum('unit_cost')) - if amount_refunds_temp['unit_cost__sum'] is None: - amount_refunds = Decimal(0.00) - else: - amount_refunds = amount_refunds_temp['unit_cost__sum'] - - result = [ - university, - course, - num_transactions, - total_payments_collected, - service_fees, - num_refunds, - amount_refunds - ] - results.append(result) - - return results - - def csv_report_header_row(self): - return [ - "University", - "Course", - "Number of Transactions", - "Total Payments Collected", - "Service Fees (if any)", - "Number of Successful Refunds", - "Total Amount of Refunds", - ] - - def csv_report_row(self, item): - return item diff --git a/lms/djangoapps/shoppingcart/reports.py b/lms/djangoapps/shoppingcart/reports.py new file mode 100644 index 0000000000..b4047b22d5 --- /dev/null +++ b/lms/djangoapps/shoppingcart/reports.py @@ -0,0 +1,249 @@ +from shoppingcart.models import CertificateItem, OrderItem +from django.db import models +from django.db.models import Sum +import unicodecsv +from django.conf import settings +from courseware.courses import get_course_by_id +from student.models import CourseEnrollment +from course_modes.models import CourseMode +from decimal import Decimal + + +class Report(models.Model): + """ + Base class for making CSV reports related to revenue, enrollments, etc + + To make a different type of report, write a new subclass that implements + the methods get_report_data, csv_report_header_row, and csv_report_row. + """ + + def get_report_data(self, start_date, end_date, start_letter=None, end_letter=None): + """ + Performs database queries necessary for the report. May return either a query result + or a list of lists, depending on the particular type of report--see Report subclasses + for sample implementations. + """ + raise NotImplementedError + + def csv_report_header_row(self): + """ + Returns the appropriate header based on the report type. + """ + raise NotImplementedError + + def csv_report_row(self, item): + """ + Given the results of get_report_data, this function generates a single row of a csv. + """ + raise NotImplementedError + + def make_report(self, filelike, start_date, end_date, start_letter=None, end_letter=None): + """ + Given the string report_type, a file object to write to, and start/end date bounds, + generates a CSV report of the appropriate type. + """ + items = self.get_report_data(start_date, end_date, start_letter, end_letter) + writer = unicodecsv.writer(filelike, encoding="utf-8") + writer.writerow(self.csv_report_header_row()) + for item in items: + writer.writerow(self.csv_report_row(item)) + + +class RefundReport(Report): + """ + Subclass of Report, used to generate Refund Reports for finance purposes. + """ + def get_report_data(self, start_date, end_date, start_letter=None, end_letter=None): + return CertificateItem.objects.filter( + status="refunded", + refund_requested_time__gte=start_date, + refund_requested_time__lt=end_date, + ) + + def csv_report_header_row(self): + return [ + "Order Number", + "Customer Name", + "Date of Original Transaction", + "Date of Refund", + "Amount of Refund", + "Service Fees (if any)", + ] + + def csv_report_row(self, item): + return [ + item.order_id, + item.user.get_full_name(), + item.fulfilled_time, + item.refund_requested_time, # TODO Change this torefund_fulfilled once we start recording that value + item.line_cost, + item.service_fee, + ] + + +class ItemizedPurchaseReport(Report): + """ + Subclass of Report, used to generate itemized purchase reports. + """ + def get_report_data(self, start_date, end_date, start_letter=None, end_letter=None): + return OrderItem.objects.filter( + status="purchased", + fulfilled_time__gte=start_date, + fulfilled_time__lt=end_date, + ).order_by("fulfilled_time") + + def csv_report_header_row(self): + return [ + "Purchase Time", + "Order ID", + "Status", + "Quantity", + "Unit Cost", + "Total Cost", + "Currency", + "Description", + "Comments" + ] + + def csv_report_row(self, item): + return [ + item.fulfilled_time, + item.order_id, # pylint: disable=no-member + item.status, + item.qty, + item.unit_cost, + item.line_cost, + item.currency, + item.line_desc, + item.report_comments, + ] + + +class CertificateStatusReport(Report): + """ + Subclass of Report, used to generate Certificate Status Reports for ed services. + """ + def get_report_data(self, start_date, end_date, start_letter=None, end_letter=None): + results = [] + for course_id in settings.COURSE_LISTINGS['default']: + if (start_letter.lower() <= course_id.lower()) and (end_letter.lower() >= course_id.lower()) and (get_course_by_id(course_id) is not None): + cur_course = get_course_by_id(course_id) + university = cur_course.org + course = cur_course.number + " " + cur_course.display_name # TODO add term (i.e. Fall 2013)? + enrollments = CourseEnrollment.enrollments_in(course_id) + total_enrolled = enrollments.count() + audit_enrolled = enrollments.filter(mode="audit").count() + honor_enrolled = enrollments.filter(mode="honor").count() + # Since every verified enrollment has 1 and only 1 cert item, let's just query those + verified_enrollments = CertificateItem.objects.filter(course_id=course_id, mode="verified", status="purchased") + verified_enrolled = verified_enrollments.count() + gross_rev_temp = CertificateItem.objects.filter(course_id=course_id, mode="verified", status="purchased").aggregate(Sum('unit_cost')) + gross_rev = gross_rev_temp['unit_cost__sum'] + gross_rev_over_min = gross_rev - (CourseMode.objects.get(course_id=course_id, mode_slug="verified").min_price * verified_enrolled) + refunded_enrollments = CertificateItem.objects.filter(course_id='course_id', mode="verified", status="refunded") + number_of_refunds = refunded_enrollments.count() + dollars_refunded_temp = refunded_enrollments.aggregate(Sum('unit_cost')) + if dollars_refunded_temp['unit_cost__sum'] is None: + dollars_refunded = Decimal(0.00) + else: + dollars_refunded = dollars_refunded_temp['unit_cost__sum'] + + result = [ + university, + course, + total_enrolled, + audit_enrolled, + honor_enrolled, + verified_enrolled, + gross_rev, + gross_rev_over_min, + number_of_refunds, + dollars_refunded + ] + + results.append(result) + return results + + def csv_report_header_row(self): + return [ + "University", + "Course", + "Total Enrolled", + "Audit Enrollment", + "Honor Code Enrollment", + "Verified Enrollment", + "Gross Revenue", + "Gross Revenue over the Minimum", + "Number of Refunds", + "Dollars Refunded", + ] + + def csv_report_row(self, item): + return item + + +class UniversityRevenueShareReport(Report): + """ + Subclass of Report, used to generate University Revenue Share Reports for finance purposes. + """ + def get_report_data(self, start_date, end_date, start_letter=None, end_letter=None): + results = [] + for course_id in settings.COURSE_LISTINGS['default']: + if (start_letter.lower() <= course_id.lower()) and (end_letter.lower() >= course_id.lower()): + try: + cur_course = get_course_by_id(course_id) + except: + break + university = cur_course.org + course = cur_course.number + " " + cur_course.display_name + num_transactions = 0 # TODO clarify with billing what transactions are included in this (purchases? refunds? etc) + + all_paid_certs = CertificateItem.objects.filter(course_id=course_id, status="purchased") + + total_payments_collected_temp = all_paid_certs.aggregate(Sum('unit_cost')) + if total_payments_collected_temp['unit_cost__sum'] is None: + total_payments_collected = Decimal(0.00) + else: + total_payments_collected = total_payments_collected_temp['unit_cost__sum'] + + total_service_fees_temp = all_paid_certs.aggregate(Sum('service_fee')) + if total_service_fees_temp['service_fee__sum'] is None: + service_fees = Decimal(0.00) + else: + service_fees = total_service_fees_temp['service_fee__sum'] + + refunded_enrollments = CertificateItem.objects.filter(course_id=course_id, status="refunded") + num_refunds = refunded_enrollments.count() + + amount_refunds_temp = refunded_enrollments.aggregate(Sum('unit_cost')) + if amount_refunds_temp['unit_cost__sum'] is None: + amount_refunds = Decimal(0.00) + else: + amount_refunds = amount_refunds_temp['unit_cost__sum'] + + result = [ + university, + course, + num_transactions, + total_payments_collected, + service_fees, + num_refunds, + amount_refunds + ] + results.append(result) + + return results + + def csv_report_header_row(self): + return [ + "University", + "Course", + "Number of Transactions", + "Total Payments Collected", + "Service Fees (if any)", + "Number of Successful Refunds", + "Total Amount of Refunds", + ] + + def csv_report_row(self, item): + return item diff --git a/lms/djangoapps/shoppingcart/tests/test_models.py b/lms/djangoapps/shoppingcart/tests/test_models.py index 80dd5a1423..b368837b14 100644 --- a/lms/djangoapps/shoppingcart/tests/test_models.py +++ b/lms/djangoapps/shoppingcart/tests/test_models.py @@ -17,14 +17,31 @@ from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE from shoppingcart.models import (Order, OrderItem, CertificateItem, InvalidCartItem, PaidCourseRegistration, - OrderItemSubclassPK, PaidCourseRegistrationAnnotation, Report) + OrderItemSubclassPK, PaidCourseRegistrationAnnotation) +from shoppingcart.reports import ItemizedPurchaseReport, CertificateStatusReport, UniversityRevenueShareReport, RefundReport from student.tests.factories import UserFactory from student.models import CourseEnrollment from course_modes.models import CourseMode -from shoppingcart.exceptions import PurchasedCallbackException +from shoppingcart.exceptions import PurchasedCallbackException, ReportTypeDoesNotExistException import pytz import datetime +REPORT_TYPES = [ + ("refund_report", RefundReport), + ("itemized_purchase_report", ItemizedPurchaseReport), + ("university_revenue_share", UniversityRevenueShareReport), + ("certificate_status", CertificateStatusReport), +] + + +def initialize_report(report_type): + """ + Creates the appropriate type of Report object based on the string report_type. + """ + for item in REPORT_TYPES: + if report_type in item: + return item[1]() + raise ReportTypeDoesNotExistException @override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) class OrderTest(ModuleStoreTestCase): @@ -355,13 +372,12 @@ class ItemizedPurchaseReportTest(ModuleStoreTestCase): self.now = datetime.datetime.now(pytz.UTC) def test_purchased_items_btw_dates(self): - report_type = "itemized_purchase_report" - report = Report.initialize_report(report_type) - purchases = report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + report = initialize_report("itemized_purchase_report") + purchases = report.get_report_data(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) self.assertEqual(len(purchases), 2) self.assertIn(self.reg.orderitem_ptr, purchases) self.assertIn(self.cert_item.orderitem_ptr, purchases) - no_purchases = report.get_query(self.now + self.FIVE_MINS, self.now + self.FIVE_MINS + self.FIVE_MINS) + no_purchases = report.get_report_data(self.now + self.FIVE_MINS, self.now + self.FIVE_MINS + self.FIVE_MINS) self.assertFalse(no_purchases) test_time = datetime.datetime.now(pytz.UTC) @@ -376,14 +392,13 @@ class ItemizedPurchaseReportTest(ModuleStoreTestCase): """ Tests that a generated purchase report CSV is as we expect """ - report_type = "itemized_purchase_report" - report = Report.initialize_report(report_type) - for item in report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS): + report = initialize_report("itemized_purchase_report") + for item in report.get_report_data(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS): item.fulfilled_time = self.test_time item.save() csv_file = StringIO.StringIO() - Report.make_report(report_type, csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + report.make_report(csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) csv = csv_file.getvalue() csv_file.close() # Using excel mode csv, which automatically ends lines with \r\n, so need to convert to \n @@ -405,163 +420,6 @@ class ItemizedPurchaseReportTest(ModuleStoreTestCase): self.assertEqual(unicode(self.annotation), u'{} : {}'.format(self.course_id, self.TEST_ANNOTATION)) -@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) -class ReportTypeTests(ModuleStoreTestCase): - """ - Tests for the models used to generate certificate status reports - """ - FIVE_MINS = datetime.timedelta(minutes=5) - - def setUp(self): - # Need to make a *lot* of users for this one - self.user1 = UserFactory.create() - self.user1.first_name = "John" - self.user1.last_name = "Doe" - self.user1.save() - - self.user2 = UserFactory.create() - self.user2.first_name = "Jane" - self.user2.last_name = "Deer" - self.user2.save() - - self.user3 = UserFactory.create() - self.user3.first_name = "Joe" - self.user3.last_name = "Miller" - self.user3.save() - - self.user4 = UserFactory.create() - self.user4.first_name = "Simon" - self.user4.last_name = "Blackquill" - self.user4.save() - - self.user5 = UserFactory.create() - self.user5.first_name = "Super" - self.user5.last_name = "Mario" - self.user5.save() - - self.user6 = UserFactory.create() - self.user6.first_name = "Princess" - self.user6.last_name = "Peach" - self.user6.save() - - self.user7 = UserFactory.create() - self.user7.first_name = "King" - self.user7.last_name = "Bowser" - self.user7.save() - - self.user8 = UserFactory.create() - self.user8.first_name = "Susan" - self.user8.last_name = "Smith" - self.user8.save() - - # Two are verified, three are audit, one honor - - self.course_id = "MITx/999/Robot_Super_Course" - settings.COURSE_LISTINGS['default'] = [self.course_id] - self.cost = 40 - self.course = CourseFactory.create(org='MITx', number='999', display_name=u'Robot Super Course') - course_mode = CourseMode(course_id=self.course_id, - mode_slug="honor", - mode_display_name="honor cert", - min_price=self.cost) - course_mode.save() - - course_mode2 = CourseMode(course_id=self.course_id, - mode_slug="verified", - mode_display_name="verified cert", - min_price=self.cost) - course_mode2.save() - - # User 1 & 2 will be verified - self.cart1 = Order.get_cart_for_user(self.user1) - CertificateItem.add_to_order(self.cart1, self.course_id, self.cost, 'verified') - self.cart1.purchase() - - self.cart2 = Order.get_cart_for_user(self.user2) - CertificateItem.add_to_order(self.cart2, self.course_id, self.cost, 'verified') - self.cart2.purchase() - - # Users 3, 4, and 5 are audit - CourseEnrollment.enroll(self.user3, self.course_id, "audit") - CourseEnrollment.enroll(self.user4, self.course_id, "audit") - CourseEnrollment.enroll(self.user5, self.course_id, "audit") - - # User 6 is honor - CourseEnrollment.enroll(self.user6, self.course_id, "honor") - - self.now = datetime.datetime.now(pytz.UTC) - - # Users 7 & 8 are refunds - self.cart = Order.get_cart_for_user(self.user7) - CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') - self.cart.purchase() - CourseEnrollment.unenroll(self.user7, self.course_id) - - self.cart = Order.get_cart_for_user(self.user8) - CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') - self.cart.purchase(self.user8, self.course_id) - CourseEnrollment.unenroll(self.user8, self.course_id) - - self.test_time = datetime.datetime.now(pytz.UTC) - self.CORRECT_REFUND_REPORT_CSV = dedent(""" - Order Number,Customer Name,Date of Original Transaction,Date of Refund,Amount of Refund,Service Fees (if any) - 3,King Bowser,{time_str},{time_str},40,0 - 4,Susan Smith,{time_str},{time_str},40,0 - """.format(time_str=str(self.test_time))) - - self.CORRECT_CERT_STATUS_CSV = dedent(""" - University,Course,Total Enrolled,Audit Enrollment,Honor Code Enrollment,Verified Enrollment,Gross Revenue,Gross Revenue over the Minimum,Number of Refunds,Dollars Refunded - MITx,999 Robot Super Course,6,3,1,2,80.00,0.00,0,0 - """.format(time_str=str(self.test_time))) - - self.CORRECT_UNI_REVENUE_SHARE_CSV = dedent(""" - University,Course,Number of Transactions,Total Payments Collected,Service Fees (if any),Number of Successful Refunds,Total Amount of Refunds - MITx,999 Robot Super Course,0,80.00,0.00,2,80.00 - """.format(time_str=str(self.test_time))) - - def test_refund_report_get_query(self): - report_type = "refund_report" - report = Report.initialize_report(report_type) - refunded_certs = report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) - self.assertEqual(len(refunded_certs), 2) - self.assertTrue(CertificateItem.objects.get(user=self.user7, course_id=self.course_id)) - self.assertTrue(CertificateItem.objects.get(user=self.user8, course_id=self.course_id)) - - def test_refund_report_purchased_csv(self): - """ - Tests that a generated purchase report CSV is as we expect - """ - report_type = "refund_report" - report = Report.initialize_report(report_type) - for item in report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS): - item.fulfilled_time = self.test_time - item.refund_requested_time = self.test_time # hm do we want to make these different - item.save() - - csv_file = StringIO.StringIO() - Report.make_report(report_type, csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) - csv = csv_file.getvalue() - csv_file.close() - # Using excel mode csv, which automatically ends lines with \r\n, so need to convert to \n - self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_REFUND_REPORT_CSV.strip()) - - def test_basic_cert_status_csv(self): - report_type = "certificate_status" - report = Report.initialize_report(report_type) - csv_file = StringIO.StringIO() - report.make_report(report_type, csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) - csv = csv_file.getvalue() - self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_CERT_STATUS_CSV.strip()) - - def test_basic_uni_revenue_share_csv(self): - report_type = "university_revenue_share" - report = Report.initialize_report(report_type) - csv_file = StringIO.StringIO() - report.make_report(report_type, csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) - csv = csv_file.getvalue() - self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_UNI_REVENUE_SHARE_CSV.strip()) - - @override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) class CertificateItemTest(ModuleStoreTestCase): """ diff --git a/lms/djangoapps/shoppingcart/tests/test_reports.py b/lms/djangoapps/shoppingcart/tests/test_reports.py new file mode 100644 index 0000000000..e5312ec594 --- /dev/null +++ b/lms/djangoapps/shoppingcart/tests/test_reports.py @@ -0,0 +1,179 @@ +""" +Tests for the Shopping Cart Models +""" +import StringIO +from textwrap import dedent + +from django.conf import settings +from django.test.utils import override_settings +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory +from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE +from shoppingcart.models import (Order, CertificateItem) +from shoppingcart.reports import ItemizedPurchaseReport, CertificateStatusReport, UniversityRevenueShareReport, RefundReport +from student.tests.factories import UserFactory +from student.models import CourseEnrollment +from course_modes.models import CourseMode +from shoppingcart.views import initialize_report +import pytz +import datetime + +REPORT_TYPES = [ + ("refund_report", RefundReport), + ("itemized_purchase_report", ItemizedPurchaseReport), + ("university_revenue_share", UniversityRevenueShareReport), + ("certificate_status", CertificateStatusReport), +] + + +@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) +class ReportTypeTests(ModuleStoreTestCase): + """ + Tests for the models used to generate certificate status reports + """ + FIVE_MINS = datetime.timedelta(minutes=5) + + def setUp(self): + # Need to make a *lot* of users for this one + self.user1 = UserFactory.create() + self.user1.first_name = "John" + self.user1.last_name = "Doe" + self.user1.save() + + self.user2 = UserFactory.create() + self.user2.first_name = "Jane" + self.user2.last_name = "Deer" + self.user2.save() + + self.user3 = UserFactory.create() + self.user3.first_name = "Joe" + self.user3.last_name = "Miller" + self.user3.save() + + self.user4 = UserFactory.create() + self.user4.first_name = "Simon" + self.user4.last_name = "Blackquill" + self.user4.save() + + self.user5 = UserFactory.create() + self.user5.first_name = "Super" + self.user5.last_name = "Mario" + self.user5.save() + + self.user6 = UserFactory.create() + self.user6.first_name = "Princess" + self.user6.last_name = "Peach" + self.user6.save() + + self.user7 = UserFactory.create() + self.user7.first_name = "King" + self.user7.last_name = "Bowser" + self.user7.save() + + self.user8 = UserFactory.create() + self.user8.first_name = "Susan" + self.user8.last_name = "Smith" + self.user8.save() + + # Two are verified, three are audit, one honor + + self.course_id = "MITx/999/Robot_Super_Course" + settings.COURSE_LISTINGS['default'] = [self.course_id] + self.cost = 40 + self.course = CourseFactory.create(org='MITx', number='999', display_name=u'Robot Super Course') + course_mode = CourseMode(course_id=self.course_id, + mode_slug="honor", + mode_display_name="honor cert", + min_price=self.cost) + course_mode.save() + + course_mode2 = CourseMode(course_id=self.course_id, + mode_slug="verified", + mode_display_name="verified cert", + min_price=self.cost) + course_mode2.save() + + # User 1 & 2 will be verified + self.cart1 = Order.get_cart_for_user(self.user1) + CertificateItem.add_to_order(self.cart1, self.course_id, self.cost, 'verified') + self.cart1.purchase() + + self.cart2 = Order.get_cart_for_user(self.user2) + CertificateItem.add_to_order(self.cart2, self.course_id, self.cost, 'verified') + self.cart2.purchase() + + # Users 3, 4, and 5 are audit + CourseEnrollment.enroll(self.user3, self.course_id, "audit") + CourseEnrollment.enroll(self.user4, self.course_id, "audit") + CourseEnrollment.enroll(self.user5, self.course_id, "audit") + + # User 6 is honor + CourseEnrollment.enroll(self.user6, self.course_id, "honor") + + self.now = datetime.datetime.now(pytz.UTC) + + # Users 7 & 8 are refunds + self.cart = Order.get_cart_for_user(self.user7) + CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') + self.cart.purchase() + CourseEnrollment.unenroll(self.user7, self.course_id) + + self.cart = Order.get_cart_for_user(self.user8) + CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') + self.cart.purchase(self.user8, self.course_id) + CourseEnrollment.unenroll(self.user8, self.course_id) + + self.test_time = datetime.datetime.now(pytz.UTC) + self.CORRECT_REFUND_REPORT_CSV = dedent(""" + Order Number,Customer Name,Date of Original Transaction,Date of Refund,Amount of Refund,Service Fees (if any) + 3,King Bowser,{time_str},{time_str},40,0 + 4,Susan Smith,{time_str},{time_str},40,0 + """.format(time_str=str(self.test_time))) + + self.CORRECT_CERT_STATUS_CSV = dedent(""" + University,Course,Total Enrolled,Audit Enrollment,Honor Code Enrollment,Verified Enrollment,Gross Revenue,Gross Revenue over the Minimum,Number of Refunds,Dollars Refunded + MITx,999 Robot Super Course,6,3,1,2,80.00,0.00,0,0 + """.format(time_str=str(self.test_time))) + + self.CORRECT_UNI_REVENUE_SHARE_CSV = dedent(""" + University,Course,Number of Transactions,Total Payments Collected,Service Fees (if any),Number of Successful Refunds,Total Amount of Refunds + MITx,999 Robot Super Course,0,80.00,0.00,2,80.00 + """.format(time_str=str(self.test_time))) + + def test_refund_report_get_report_data(self): + report = initialize_report("refund_report") + refunded_certs = report.get_report_data(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + self.assertEqual(len(refunded_certs), 2) + self.assertTrue(CertificateItem.objects.get(user=self.user7, course_id=self.course_id)) + self.assertTrue(CertificateItem.objects.get(user=self.user8, course_id=self.course_id)) + + def test_refund_report_purchased_csv(self): + """ + Tests that a generated purchase report CSV is as we expect + """ + report = initialize_report("refund_report") + for item in report.get_report_data(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS): + item.fulfilled_time = self.test_time + item.refund_requested_time = self.test_time # hm do we want to make these different + item.save() + + csv_file = StringIO.StringIO() + report.make_report(csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + csv = csv_file.getvalue() + csv_file.close() + # Using excel mode csv, which automatically ends lines with \r\n, so need to convert to \n + self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_REFUND_REPORT_CSV.strip()) + + def test_basic_cert_status_csv(self): + report = initialize_report("certificate_status") + csv_file = StringIO.StringIO() + report.make_report(csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS, 'A', 'Z') + csv = csv_file.getvalue() + self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_CERT_STATUS_CSV.strip()) + + def test_basic_uni_revenue_share_csv(self): + report = initialize_report("university_revenue_share") + csv_file = StringIO.StringIO() + report.make_report(csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS, 'A', 'Z') + csv = csv_file.getvalue() + self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_UNI_REVENUE_SHARE_CSV.strip()) diff --git a/lms/djangoapps/shoppingcart/tests/test_views.py b/lms/djangoapps/shoppingcart/tests/test_views.py index a08decd05f..949a570a0b 100644 --- a/lms/djangoapps/shoppingcart/tests/test_views.py +++ b/lms/djangoapps/shoppingcart/tests/test_views.py @@ -14,13 +14,15 @@ from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE from shoppingcart.views import _can_download_report, _get_date_from_str -from shoppingcart.models import Order, CertificateItem, PaidCourseRegistration, Report +from shoppingcart.models import Order, CertificateItem, PaidCourseRegistration from student.tests.factories import UserFactory from student.models import CourseEnrollment from course_modes.models import CourseMode from edxmako.shortcuts import render_to_response from shoppingcart.processors import render_purchase_form_html from mock import patch, Mock, sentinel +from shoppingcart.reports import ItemizedPurchaseReport +from shoppingcart.views import initialize_report def mock_render_purchase_form_html(*args, **kwargs): @@ -354,7 +356,6 @@ class CSVReportViewsTest(ModuleStoreTestCase): def test_report_csv_bad_date(self): self.login_user() self.add_to_download_group(self.user) - report_type = "itemized_purchase_report" response = self.client.post(reverse('payment_csv_report'), {'start_date': 'BAD', 'end_date': 'BAD', 'requested_report': 'itemized_purchase_report'}) ((template, context), unused_kwargs) = render_mock.call_args @@ -386,7 +387,7 @@ class CSVReportViewsTest(ModuleStoreTestCase): CORRECT_CSV_NO_DATE_ITEMIZED_PURCHASE = ",1,purchased,1,40,40,usd,Registration for Course: Robot Super Course," - def test_report_csv(self): + def test_report_csv_itemized(self): report_type = 'itemized_purchase_report' PaidCourseRegistration.add_to_order(self.cart, self.course_id) self.cart.purchase() @@ -396,10 +397,24 @@ class CSVReportViewsTest(ModuleStoreTestCase): 'end_date': '2100-01-01', 'requested_report': report_type}) self.assertEqual(response['Content-Type'], 'text/csv') - report = Report.initialize_report(report_type) + report = initialize_report(report_type) self.assertIn(",".join(report.csv_report_header_row()), response.content) self.assertIn(self.CORRECT_CSV_NO_DATE_ITEMIZED_PURCHASE, response.content) + def test_report_csv_university_revenue_share(self): + report_type = 'university_revenue_share' + self.login_user() + self.add_to_download_group(self.user) + response = self.client.post(reverse('payment_csv_report'), {'start_date': '1970-01-01', + 'end_date': '2100-01-01', + 'start_letter': 'A', + 'end_letter': 'Z', + 'requested_report': report_type}) + self.assertEqual(response['Content-Type'], 'text/csv') + report = initialize_report(report_type) + self.assertIn(",".join(report.csv_report_header_row()), response.content) + # TODO add another test here + class UtilFnsTest(TestCase): """ diff --git a/lms/djangoapps/shoppingcart/views.py b/lms/djangoapps/shoppingcart/views.py index b19e2244f7..617cd2c58a 100644 --- a/lms/djangoapps/shoppingcart/views.py +++ b/lms/djangoapps/shoppingcart/views.py @@ -12,14 +12,31 @@ from django.views.decorators.csrf import csrf_exempt from django.contrib.auth.decorators import login_required from edxmako.shortcuts import render_to_response from student.models import CourseEnrollment -from .models import Order, PaidCourseRegistration, OrderItem, Report +from shoppingcart.reports import RefundReport, ItemizedPurchaseReport, UniversityRevenueShareReport, CertificateStatusReport +from .models import Order, PaidCourseRegistration, OrderItem from .processors import process_postpay_callback, render_purchase_form_html -from .exceptions import ItemAlreadyInCartException, AlreadyEnrolledInCourseException, CourseDoesNotExistException +from .exceptions import ItemAlreadyInCartException, AlreadyEnrolledInCourseException, CourseDoesNotExistException, ReportTypeDoesNotExistException log = logging.getLogger("shoppingcart") EVENT_NAME_USER_UPGRADED = 'edx.course.enrollment.upgrade.succeeded' +REPORT_TYPES = [ + ("refund_report", RefundReport), + ("itemized_purchase_report", ItemizedPurchaseReport), + ("university_revenue_share", UniversityRevenueShareReport), + ("certificate_status", CertificateStatusReport), +] + + +def initialize_report(report_type): + """ + Creates the appropriate type of Report object based on the string report_type. + """ + for item in REPORT_TYPES: + if report_type in item: + return item[1]() + raise ReportTypeDoesNotExistException @require_POST def add_course_to_cart(request, course_id): @@ -155,7 +172,7 @@ def _get_date_from_str(date_input): return datetime.datetime.strptime(date_input.strip(), "%Y-%m-%d").replace(tzinfo=pytz.UTC) -def _render_report_form(start_str, end_str, report_type, total_count_error=False, date_fmt_error=False): +def _render_report_form(start_str, end_str, start_letter, end_letter, report_type, total_count_error=False, date_fmt_error=False): """ Helper function that renders the purchase form. Reduces repetition """ @@ -164,6 +181,8 @@ def _render_report_form(start_str, end_str, report_type, total_count_error=False 'date_fmt_error': date_fmt_error, 'start_date': start_str, 'end_date': end_str, + 'start_letter': start_letter, + 'end_letter': end_letter, 'requested_report': report_type, } return render_to_response('shoppingcart/download_report.html', context) @@ -178,34 +197,44 @@ def csv_report(request): if not _can_download_report(request.user): return HttpResponseForbidden(_('You do not have permission to view this page.')) + # TODO temp filler for start letter, end letter + if request.method == 'POST': start_str = request.POST.get('start_date', '') end_str = request.POST.get('end_date', '') + start_letter = request.POST.get('start_letter', '') + end_letter = request.POST.get('end_letter', '') report_type = request.POST.get('requested_report', '') try: - start_date = _get_date_from_str(start_str) + start_date = _get_date_from_str(start_str) + datetime.timedelta(days=0) end_date = _get_date_from_str(end_str) + datetime.timedelta(days=1) except ValueError: # Error case: there was a badly formatted user-input date string - return _render_report_form(start_str, end_str, report_type, date_fmt_error=True) + return _render_report_form(start_str, end_str, start_letter, end_letter, report_type, date_fmt_error=True) - report = Report.initialize_report(report_type) - items = report.get_query(start_date, end_date) - if items.count() > settings.PAYMENT_REPORT_MAX_ITEMS: + report = initialize_report(report_type) + items = report.get_report_data(start_date, end_date, start_letter, end_letter) + + # TODO add this back later as a query-est function or something + try: + if items.count() > settings.PAYMENT_REPORT_MAX_ITEMS: # Error case: too many items would be generated in the report and we're at risk of timeout - return _render_report_form(start_str, end_str, report_type, total_count_error=True) + return _render_report_form(start_str, end_str, start_letter, end_letter, report_type, total_count_error=True) + except: + pass response = HttpResponse(mimetype='text/csv') filename = "purchases_report_{}.csv".format(datetime.datetime.now(pytz.UTC).strftime("%Y-%m-%d-%H-%M-%S")) response['Content-Disposition'] = 'attachment; filename="{}"'.format(filename) - # this flos is a little odd; what's up with report_type being called twice? check later - report.make_report(report_type, response, start_date, end_date) + report.make_report(response, start_date, end_date, start_letter, end_letter) return response elif request.method == 'GET': end_date = datetime.datetime.now(pytz.UTC) start_date = end_date - datetime.timedelta(days=30) - return _render_report_form(start_date.strftime("%Y-%m-%d"), end_date.strftime("%Y-%m-%d"), report_type="") + start_letter = "" + end_letter = "" + return _render_report_form(start_date.strftime("%Y-%m-%d"), end_date.strftime("%Y-%m-%d"), start_letter, end_letter, report_type="") else: return HttpResponseBadRequest("HTTP Method Not Supported") diff --git a/lms/templates/shoppingcart/download_report.html b/lms/templates/shoppingcart/download_report.html index 67cdbce913..9dc5548895 100644 --- a/lms/templates/shoppingcart/download_report.html +++ b/lms/templates/shoppingcart/download_report.html @@ -23,6 +23,10 @@ + + + +
From 893acc57b69a48307ff5fd6df77ca2ffdd78eca8 Mon Sep 17 00:00:00 2001 From: Julia Hansbrough Date: Thu, 19 Dec 2013 23:04:07 +0000 Subject: [PATCH 08/14] Template changes --- .../shoppingcart/tests/test_models.py | 17 +---------------- .../shoppingcart/tests/test_reports.py | 9 +-------- lms/templates/shoppingcart/download_report.html | 13 +++++++++---- 3 files changed, 11 insertions(+), 28 deletions(-) diff --git a/lms/djangoapps/shoppingcart/tests/test_models.py b/lms/djangoapps/shoppingcart/tests/test_models.py index b368837b14..37b73ae506 100644 --- a/lms/djangoapps/shoppingcart/tests/test_models.py +++ b/lms/djangoapps/shoppingcart/tests/test_models.py @@ -18,6 +18,7 @@ from xmodule.modulestore.tests.factories import CourseFactory from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE from shoppingcart.models import (Order, OrderItem, CertificateItem, InvalidCartItem, PaidCourseRegistration, OrderItemSubclassPK, PaidCourseRegistrationAnnotation) +from shoppingcart.views import initialize_report, REPORT_TYPES from shoppingcart.reports import ItemizedPurchaseReport, CertificateStatusReport, UniversityRevenueShareReport, RefundReport from student.tests.factories import UserFactory from student.models import CourseEnrollment @@ -26,22 +27,6 @@ from shoppingcart.exceptions import PurchasedCallbackException, ReportTypeDoesNo import pytz import datetime -REPORT_TYPES = [ - ("refund_report", RefundReport), - ("itemized_purchase_report", ItemizedPurchaseReport), - ("university_revenue_share", UniversityRevenueShareReport), - ("certificate_status", CertificateStatusReport), -] - - -def initialize_report(report_type): - """ - Creates the appropriate type of Report object based on the string report_type. - """ - for item in REPORT_TYPES: - if report_type in item: - return item[1]() - raise ReportTypeDoesNotExistException @override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) class OrderTest(ModuleStoreTestCase): diff --git a/lms/djangoapps/shoppingcart/tests/test_reports.py b/lms/djangoapps/shoppingcart/tests/test_reports.py index e5312ec594..3b80ba61ec 100644 --- a/lms/djangoapps/shoppingcart/tests/test_reports.py +++ b/lms/djangoapps/shoppingcart/tests/test_reports.py @@ -14,17 +14,10 @@ from shoppingcart.reports import ItemizedPurchaseReport, CertificateStatusReport from student.tests.factories import UserFactory from student.models import CourseEnrollment from course_modes.models import CourseMode -from shoppingcart.views import initialize_report +from shoppingcart.views import initialize_report, REPORT_TYPES import pytz import datetime -REPORT_TYPES = [ - ("refund_report", RefundReport), - ("itemized_purchase_report", ItemizedPurchaseReport), - ("university_revenue_share", UniversityRevenueShareReport), - ("certificate_status", CertificateStatusReport), -] - @override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) class ReportTypeTests(ModuleStoreTestCase): diff --git a/lms/templates/shoppingcart/download_report.html b/lms/templates/shoppingcart/download_report.html index 9dc5548895..e6fda982b2 100644 --- a/lms/templates/shoppingcart/download_report.html +++ b/lms/templates/shoppingcart/download_report.html @@ -19,19 +19,24 @@
% endif
+

${_("These reports are delimited by start and end dates.")}

+
+ +
+ + +

+

${_("These reports are delimited alphabetically by university name. i.e., generating a report with 'Start Letter' A and 'End Letter' C will generate reports for all universities starting with A, B, and C.")}

-
- -
- +

From 99e8596feab176b7e3e4a4ce01aa777e9d82d989 Mon Sep 17 00:00:00 2001 From: Julia Hansbrough Date: Mon, 23 Dec 2013 19:18:09 +0000 Subject: [PATCH 09/14] Response to CR --- common/djangoapps/student/models.py | 12 +- lms/djangoapps/shoppingcart/models.py | 11 +- lms/djangoapps/shoppingcart/reports.py | 176 ++++++++++-------- .../shoppingcart/tests/test_models.py | 47 +++-- .../shoppingcart/tests/test_reports.py | 80 ++++---- lms/djangoapps/shoppingcart/views.py | 2 +- 6 files changed, 189 insertions(+), 139 deletions(-) diff --git a/common/djangoapps/student/models.py b/common/djangoapps/student/models.py index 9e98bfd965..7e348177b5 100644 --- a/common/djangoapps/student/models.py +++ b/common/djangoapps/student/models.py @@ -581,9 +581,15 @@ class CourseEnrollment(models.Model): ) @classmethod - def enrollments_in(cls, course_id): - """Return a queryset of CourseEnrollment for every active enrollment in the course.""" - return cls.objects.filter(course_id=course_id, is_active=True,) + def enrollments_in(cls, course_id, mode=None): + """ + Return a queryset of CourseEnrollment for every active enrollment in the course course_id. + Returns only CourseEnrollments with the given mode, if a mode is supplied by the caller. + """ + if mode is None: + return cls.objects.filter(course_id=course_id, is_active=True,) + else: + return cls.objects.filter(course_id=course_id, is_active=True, mode=mode) def activate(self): """Makes this `CourseEnrollment` record active. Saves immediately.""" diff --git a/lms/djangoapps/shoppingcart/models.py b/lms/djangoapps/shoppingcart/models.py index 685f320e4d..cf22b1756c 100644 --- a/lms/djangoapps/shoppingcart/models.py +++ b/lms/djangoapps/shoppingcart/models.py @@ -205,13 +205,13 @@ class OrderItem(models.Model): # this is denormalized, but convenient for SQL queries for reports, etc. user should always be = order.user user = models.ForeignKey(User, db_index=True) # this is denormalized, but convenient for SQL queries for reports, etc. status should always be = order.status - status = models.CharField(max_length=32, default='cart', choices=ORDER_STATUSES) + status = models.CharField(max_length=32, default='cart', choices=ORDER_STATUSES, db_index=True) qty = models.IntegerField(default=1) unit_cost = models.DecimalField(default=0.0, decimal_places=2, max_digits=30) line_desc = models.CharField(default="Misc. Item", max_length=1024) currency = models.CharField(default="usd", max_length=8) # lower case ISO currency codes - fulfilled_time = models.DateTimeField(null=True) - refund_requested_time = models.DateTimeField(null=True) + fulfilled_time = models.DateTimeField(null=True, db_index=True) + refund_requested_time = models.DateTimeField(null=True, db_index=True) service_fee = models.DecimalField(default=0.0, decimal_places=2, max_digits=30) # general purpose field, not user-visible. Used for reporting report_comments = models.TextField(default="") @@ -568,3 +568,8 @@ class CertificateItem(OrderItem): "Please include your order number in your e-mail. " "Please do NOT include your credit card information.").format( billing_email=settings.PAYMENT_SUPPORT_EMAIL) + + @classmethod + def verified_certificates_in(cls, course_id, status): + """Return a queryset of CertificateItem for every verified enrollment in course_id with the given status.""" + return CertificateItem.objects.filter(course_id=course_id, mode='verified', status=status) diff --git a/lms/djangoapps/shoppingcart/reports.py b/lms/djangoapps/shoppingcart/reports.py index b4047b22d5..360797f595 100644 --- a/lms/djangoapps/shoppingcart/reports.py +++ b/lms/djangoapps/shoppingcart/reports.py @@ -9,31 +9,25 @@ from course_modes.models import CourseMode from decimal import Decimal -class Report(models.Model): +class Report(Object): """ Base class for making CSV reports related to revenue, enrollments, etc To make a different type of report, write a new subclass that implements - the methods get_report_data, csv_report_header_row, and csv_report_row. + the methods report_row_generator and csv_report_header_row. """ - def get_report_data(self, start_date, end_date, start_letter=None, end_letter=None): + def report_row_generator(self, start_date, end_date, start_letter=None, end_letter=None): """ - Performs database queries necessary for the report. May return either a query result - or a list of lists, depending on the particular type of report--see Report subclasses - for sample implementations. + Performs database queries necessary for the report. Returns an generator of + lists, in which each list is a separate row of the report. """ raise NotImplementedError def csv_report_header_row(self): """ - Returns the appropriate header based on the report type. - """ - raise NotImplementedError - - def csv_report_row(self, item): - """ - Given the results of get_report_data, this function generates a single row of a csv. + Returns the appropriate header based on the report type, in the form of a + list of strings. """ raise NotImplementedError @@ -42,23 +36,36 @@ class Report(models.Model): Given the string report_type, a file object to write to, and start/end date bounds, generates a CSV report of the appropriate type. """ - items = self.get_report_data(start_date, end_date, start_letter, end_letter) + items = self.report_row_generator(start_date, end_date, start_letter, end_letter) writer = unicodecsv.writer(filelike, encoding="utf-8") writer.writerow(self.csv_report_header_row()) for item in items: - writer.writerow(self.csv_report_row(item)) + writer.writerow(item) class RefundReport(Report): """ Subclass of Report, used to generate Refund Reports for finance purposes. + + For each refund between a given start_date and end_date, we find the relevant + order number, customer name, date of transaction, date of refund, and any service + fees. """ - def get_report_data(self, start_date, end_date, start_letter=None, end_letter=None): - return CertificateItem.objects.filter( + def report_row_generator(self, start_date, end_date, start_letter=None, end_letter=None): + query = CertificateItem.objects.select_related('user__profile').filter( status="refunded", refund_requested_time__gte=start_date, refund_requested_time__lt=end_date, - ) + ).order_by('refund_requested_time') + for item in query: + yield [ + item.order_id, + item.user.profile.name, + item.fulfilled_time, + item.refund_requested_time, + item.line_cost, + item.service_fee, + ] def csv_report_header_row(self): return [ @@ -70,28 +77,35 @@ class RefundReport(Report): "Service Fees (if any)", ] - def csv_report_row(self, item): - return [ - item.order_id, - item.user.get_full_name(), - item.fulfilled_time, - item.refund_requested_time, # TODO Change this torefund_fulfilled once we start recording that value - item.line_cost, - item.service_fee, - ] - class ItemizedPurchaseReport(Report): """ Subclass of Report, used to generate itemized purchase reports. + + For all purchases (verified certificates, paid course registrations, etc) between + a given start_date and end_date, we find that purchase's time, order ID, status, + quantity, unit cost, total cost, currency, description, and related comments. """ - def get_report_data(self, start_date, end_date, start_letter=None, end_letter=None): - return OrderItem.objects.filter( + def report_row_generator(self, start_date, end_date, start_letter=None, end_letter=None): + query = OrderItem.objects.filter( status="purchased", fulfilled_time__gte=start_date, fulfilled_time__lt=end_date, ).order_by("fulfilled_time") + for item in query: + yield [ + item.fulfilled_time, + item.order_id, # pylint: disable=no-member + item.status, + item.qty, + item.unit_cost, + item.line_cost, + item.currency, + item.line_desc, + item.report_comments, + ] + def csv_report_header_row(self): return [ "Purchase Time", @@ -105,47 +119,50 @@ class ItemizedPurchaseReport(Report): "Comments" ] - def csv_report_row(self, item): - return [ - item.fulfilled_time, - item.order_id, # pylint: disable=no-member - item.status, - item.qty, - item.unit_cost, - item.line_cost, - item.currency, - item.line_desc, - item.report_comments, - ] - class CertificateStatusReport(Report): """ - Subclass of Report, used to generate Certificate Status Reports for ed services. + Subclass of Report, used to generate Certificate Status Reports for Ed Services. + + For each course in each university whose name is within the range start_letter and end_letter, + inclusive, (i.e., the letter range H-J includes both Ithaca College and Harvard University), we + calculate the total enrollment, audit enrollment, honor enrollment, verified enrollment, total + gross revenue, gross revenue over the minimum, and total dollars refunded. """ - def get_report_data(self, start_date, end_date, start_letter=None, end_letter=None): + def report_row_generator(self, start_date, end_date, start_letter=None, end_letter=None): results = [] for course_id in settings.COURSE_LISTINGS['default']: + # If the first letter of the university is between start_letter and end_letter, then we include + # it in the report. These comparisons are unicode-safe. if (start_letter.lower() <= course_id.lower()) and (end_letter.lower() >= course_id.lower()) and (get_course_by_id(course_id) is not None): cur_course = get_course_by_id(course_id) university = cur_course.org - course = cur_course.number + " " + cur_course.display_name # TODO add term (i.e. Fall 2013)? + course = cur_course.number + " " + cur_course.display_name_with_default # TODO add term (i.e. Fall 2013)? enrollments = CourseEnrollment.enrollments_in(course_id) total_enrolled = enrollments.count() - audit_enrolled = enrollments.filter(mode="audit").count() - honor_enrolled = enrollments.filter(mode="honor").count() + audit_enrolled = CourseEnrollment.enrollments_in(course_id, "audit").count() + honor_enrolled = CourseEnrollment.enrollments_in(course_id, "honor").count() + # Since every verified enrollment has 1 and only 1 cert item, let's just query those - verified_enrollments = CertificateItem.objects.filter(course_id=course_id, mode="verified", status="purchased") - verified_enrolled = verified_enrollments.count() - gross_rev_temp = CertificateItem.objects.filter(course_id=course_id, mode="verified", status="purchased").aggregate(Sum('unit_cost')) - gross_rev = gross_rev_temp['unit_cost__sum'] - gross_rev_over_min = gross_rev - (CourseMode.objects.get(course_id=course_id, mode_slug="verified").min_price * verified_enrolled) - refunded_enrollments = CertificateItem.objects.filter(course_id='course_id', mode="verified", status="refunded") - number_of_refunds = refunded_enrollments.count() - dollars_refunded_temp = refunded_enrollments.aggregate(Sum('unit_cost')) - if dollars_refunded_temp['unit_cost__sum'] is None: + verified_enrollments = CertificateItem.verified_certificates_in(course_id, 'purchased') + if verified_enrollments is None: + verified_enrolled = 0 + gross_rev = Decimal(0.00) + gross_rev_over_min = Decimal(0.00) + else: + verified_enrolled = verified_enrollments.count() + gross_rev_temp = verified_enrollments.aggregate(Sum('unit_cost')) + gross_rev = gross_rev_temp['unit_cost__sum'] + gross_rev_over_min = gross_rev - (CourseMode.objects.get(course_id=course_id, mode_slug="verified").min_price * verified_enrolled) + + # should I be worried about is_active here? + refunded_enrollments = CertificateItem.verified_certificates_in(course_id, 'refunded') + if refunded_enrollments is None: + number_of_refunds = 0 dollars_refunded = Decimal(0.00) else: + number_of_refunds = refunded_enrollments.count() + dollars_refunded_temp = refunded_enrollments.aggregate(Sum('unit_cost')) dollars_refunded = dollars_refunded_temp['unit_cost__sum'] result = [ @@ -162,7 +179,8 @@ class CertificateStatusReport(Report): ] results.append(result) - return results + for item in results: + yield item def csv_report_header_row(self): return [ @@ -178,41 +196,43 @@ class CertificateStatusReport(Report): "Dollars Refunded", ] - def csv_report_row(self, item): - return item - class UniversityRevenueShareReport(Report): """ Subclass of Report, used to generate University Revenue Share Reports for finance purposes. + + For each course in each university whose name is within the range start_letter and end_letter, + inclusive, (i.e., the letter range H-J includes both Ithaca College and Harvard University), we calculate + the total revenue generated by that particular course. This includes the number of transactions, + total payments collected, service fees, number of refunds, and total amount of refunds. """ - def get_report_data(self, start_date, end_date, start_letter=None, end_letter=None): + def report_row_generator(self, start_date, end_date, start_letter=None, end_letter=None): results = [] for course_id in settings.COURSE_LISTINGS['default']: + # If the first letter of the university is between start_letter and end_letter, then we include + # it in the report. These comparisons are unicode-safe. if (start_letter.lower() <= course_id.lower()) and (end_letter.lower() >= course_id.lower()): try: cur_course = get_course_by_id(course_id) except: break university = cur_course.org - course = cur_course.number + " " + cur_course.display_name + course = cur_course.number + " " + cur_course.display_name_with_default num_transactions = 0 # TODO clarify with billing what transactions are included in this (purchases? refunds? etc) - all_paid_certs = CertificateItem.objects.filter(course_id=course_id, status="purchased") - - total_payments_collected_temp = all_paid_certs.aggregate(Sum('unit_cost')) - if total_payments_collected_temp['unit_cost__sum'] is None: - total_payments_collected = Decimal(0.00) - else: + all_paid_certs = CertificateItem.verified_certificates_in(course_id, "purchased") + try: + total_payments_collected_temp = all_paid_certs.aggregate(Sum('unit_cost')) total_payments_collected = total_payments_collected_temp['unit_cost__sum'] - - total_service_fees_temp = all_paid_certs.aggregate(Sum('service_fee')) - if total_service_fees_temp['service_fee__sum'] is None: - service_fees = Decimal(0.00) - else: + except: + total_payments_collected = Decimal(0.00) + try: + total_service_fees_temp = all_paid_certs.aggregate(Sum('service_fee')) service_fees = total_service_fees_temp['service_fee__sum'] + except: + service_fees = Decimal(0.00) - refunded_enrollments = CertificateItem.objects.filter(course_id=course_id, status="refunded") + refunded_enrollments = CertificateItem.verified_certificates_in(course_id, "refunded") num_refunds = refunded_enrollments.count() amount_refunds_temp = refunded_enrollments.aggregate(Sum('unit_cost')) @@ -232,7 +252,8 @@ class UniversityRevenueShareReport(Report): ] results.append(result) - return results + for item in results: + yield item def csv_report_header_row(self): return [ @@ -244,6 +265,3 @@ class UniversityRevenueShareReport(Report): "Number of Successful Refunds", "Total Amount of Refunds", ] - - def csv_report_row(self, item): - return item diff --git a/lms/djangoapps/shoppingcart/tests/test_models.py b/lms/djangoapps/shoppingcart/tests/test_models.py index 37b73ae506..a9b0ede4d5 100644 --- a/lms/djangoapps/shoppingcart/tests/test_models.py +++ b/lms/djangoapps/shoppingcart/tests/test_models.py @@ -356,32 +356,47 @@ class ItemizedPurchaseReportTest(ModuleStoreTestCase): self.cart.purchase() self.now = datetime.datetime.now(pytz.UTC) + # We can't modify the values returned by report_row_generator directly, since it's a generator, but + # we need the times on CORRECT_CSV and the generated report to match. So, we extract the times from + # the report_row_generator and place them in CORRECT_CSV. + self.time_str = {} + report = initialize_report("itemized_purchase_report") + purchases = report.report_row_generator(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + num_of_item = 0 + for item in purchases: + num_of_item += 1 + self.time_str[num_of_item] = item[0] + + self.CORRECT_CSV = dedent(""" + Purchase Time,Order ID,Status,Quantity,Unit Cost,Total Cost,Currency,Description,Comments + {time_str1},1,purchased,1,40,40,usd,Registration for Course: Robot Super Course,Ba\xc3\xbc\xe5\x8c\x85 + {time_str2},1,purchased,1,40,40,usd,"Certificate of Achievement, verified cert for course Robot Super Course", + """.format(time_str1=str(self.time_str[1]), time_str2=str(self.time_str[2]))) + def test_purchased_items_btw_dates(self): report = initialize_report("itemized_purchase_report") - purchases = report.get_report_data(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) - self.assertEqual(len(purchases), 2) - self.assertIn(self.reg.orderitem_ptr, purchases) - self.assertIn(self.cert_item.orderitem_ptr, purchases) - no_purchases = report.get_report_data(self.now + self.FIVE_MINS, self.now + self.FIVE_MINS + self.FIVE_MINS) - self.assertFalse(no_purchases) + purchases = report.report_row_generator(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) - test_time = datetime.datetime.now(pytz.UTC) + # since there's not many purchases, just run through the generator to make sure we've got the right number + num_purchases = 0 + for item in purchases: + num_purchases += 1 + self.assertEqual(num_purchases, 2) + #self.assertIn(self.reg.orderitem_ptr, purchases) + #self.assertIn(self.cert_item.orderitem_ptr, purchases) - CORRECT_CSV = dedent(""" - Purchase Time,Order ID,Status,Quantity,Unit Cost,Total Cost,Currency,Description,Comments - {time_str},1,purchased,1,40,40,usd,Registration for Course: Robot Super Course,Ba\xc3\xbc\xe5\x8c\x85 - {time_str},1,purchased,1,40,40,usd,"Certificate of Achievement, verified cert for course Robot Super Course", - """.format(time_str=str(test_time))) + no_purchases = report.report_row_generator(self.now + self.FIVE_MINS, self.now + self.FIVE_MINS + self.FIVE_MINS) + + num_purchases = 0 + for item in no_purchases: + num_purchases +=1 + self.assertEqual(num_purchases, 0) def test_purchased_csv(self): """ Tests that a generated purchase report CSV is as we expect """ report = initialize_report("itemized_purchase_report") - for item in report.get_report_data(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS): - item.fulfilled_time = self.test_time - item.save() - csv_file = StringIO.StringIO() report.make_report(csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) csv = csv_file.getvalue() diff --git a/lms/djangoapps/shoppingcart/tests/test_reports.py b/lms/djangoapps/shoppingcart/tests/test_reports.py index 3b80ba61ec..6c90cd8af8 100644 --- a/lms/djangoapps/shoppingcart/tests/test_reports.py +++ b/lms/djangoapps/shoppingcart/tests/test_reports.py @@ -29,44 +29,36 @@ class ReportTypeTests(ModuleStoreTestCase): def setUp(self): # Need to make a *lot* of users for this one self.user1 = UserFactory.create() - self.user1.first_name = "John" - self.user1.last_name = "Doe" - self.user1.save() + self.user1.profile.name = "John Doe" + self.user1.profile.save() self.user2 = UserFactory.create() - self.user2.first_name = "Jane" - self.user2.last_name = "Deer" - self.user2.save() + self.user2.profile.name = "Jane Deer" + self.user2.profile.save() self.user3 = UserFactory.create() - self.user3.first_name = "Joe" - self.user3.last_name = "Miller" - self.user3.save() + self.user3.profile.name = "Joe Miller" + self.user3.profile.save() self.user4 = UserFactory.create() - self.user4.first_name = "Simon" - self.user4.last_name = "Blackquill" - self.user4.save() + self.user4.profile.name = "Simon Blackquill" + self.user4.profile.save() self.user5 = UserFactory.create() - self.user5.first_name = "Super" - self.user5.last_name = "Mario" - self.user5.save() + self.user5.profile.name = "Super Mario" + self.user5.profile.save() self.user6 = UserFactory.create() - self.user6.first_name = "Princess" - self.user6.last_name = "Peach" - self.user6.save() + self.user6.profile.name = "Princess Peach" + self.user6.profile.save() self.user7 = UserFactory.create() - self.user7.first_name = "King" - self.user7.last_name = "Bowser" - self.user7.save() + self.user7.profile.name = "King Bowser" + self.user7.profile.save() self.user8 = UserFactory.create() - self.user8.first_name = "Susan" - self.user8.last_name = "Smith" - self.user8.save() + self.user8.profile.name = "Susan Smith" + self.user8.profile.save() # Two are verified, three are audit, one honor @@ -116,16 +108,29 @@ class ReportTypeTests(ModuleStoreTestCase): self.cart.purchase(self.user8, self.course_id) CourseEnrollment.unenroll(self.user8, self.course_id) - self.test_time = datetime.datetime.now(pytz.UTC) + # We can't modify the values returned by report_row_generator directly, since it's a generator, but + # we need the times on CORRECT_CSV and the generated report to match. So, we extract the times from + # the report_row_generator and place them in CORRECT_CSV. + self.time_str = {} + report = initialize_report("refund_report") + refunds = report.report_row_generator(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + time_index = 0 + for item in refunds: + self.time_str[time_index] = item[2] + time_index += 1 + self.time_str[time_index] = item[3] + time_index += 1 self.CORRECT_REFUND_REPORT_CSV = dedent(""" Order Number,Customer Name,Date of Original Transaction,Date of Refund,Amount of Refund,Service Fees (if any) - 3,King Bowser,{time_str},{time_str},40,0 - 4,Susan Smith,{time_str},{time_str},40,0 - """.format(time_str=str(self.test_time))) + 3,King Bowser,{time_str0},{time_str1},40,0 + 4,Susan Smith,{time_str2},{time_str3},40,0 + """.format(time_str0=str(self.time_str[0]), time_str1=str(self.time_str[1]), time_str2=str(self.time_str[2]), time_str3=str(self.time_str[3]))) + + self.test_time = datetime.datetime.now(pytz.UTC) self.CORRECT_CERT_STATUS_CSV = dedent(""" University,Course,Total Enrolled,Audit Enrollment,Honor Code Enrollment,Verified Enrollment,Gross Revenue,Gross Revenue over the Minimum,Number of Refunds,Dollars Refunded - MITx,999 Robot Super Course,6,3,1,2,80.00,0.00,0,0 + MITx,999 Robot Super Course,6,3,1,2,80.00,0.00,2,80.00 """.format(time_str=str(self.test_time))) self.CORRECT_UNI_REVENUE_SHARE_CSV = dedent(""" @@ -133,10 +138,16 @@ class ReportTypeTests(ModuleStoreTestCase): MITx,999 Robot Super Course,0,80.00,0.00,2,80.00 """.format(time_str=str(self.test_time))) - def test_refund_report_get_report_data(self): + def test_refund_report_report_row_generator(self): report = initialize_report("refund_report") - refunded_certs = report.get_report_data(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) - self.assertEqual(len(refunded_certs), 2) + refunded_certs = report.report_row_generator(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + + # check that we have the right number + num_certs = 0 + for cert in refunded_certs: + num_certs += 1 + self.assertEqual(num_certs, 2) + self.assertTrue(CertificateItem.objects.get(user=self.user7, course_id=self.course_id)) self.assertTrue(CertificateItem.objects.get(user=self.user8, course_id=self.course_id)) @@ -145,11 +156,6 @@ class ReportTypeTests(ModuleStoreTestCase): Tests that a generated purchase report CSV is as we expect """ report = initialize_report("refund_report") - for item in report.get_report_data(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS): - item.fulfilled_time = self.test_time - item.refund_requested_time = self.test_time # hm do we want to make these different - item.save() - csv_file = StringIO.StringIO() report.make_report(csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) csv = csv_file.getvalue() diff --git a/lms/djangoapps/shoppingcart/views.py b/lms/djangoapps/shoppingcart/views.py index 617cd2c58a..eac4684253 100644 --- a/lms/djangoapps/shoppingcart/views.py +++ b/lms/djangoapps/shoppingcart/views.py @@ -213,7 +213,7 @@ def csv_report(request): return _render_report_form(start_str, end_str, start_letter, end_letter, report_type, date_fmt_error=True) report = initialize_report(report_type) - items = report.get_report_data(start_date, end_date, start_letter, end_letter) + items = report.report_row_generator(start_date, end_date, start_letter, end_letter) # TODO add this back later as a query-est function or something try: From 95affba643e56d0e35b26a119ba6ab15f80e2c1e Mon Sep 17 00:00:00 2001 From: Julia Hansbrough Date: Tue, 31 Dec 2013 15:15:44 +0000 Subject: [PATCH 10/14] More response to CR --- common/djangoapps/student/models.py | 27 +- common/djangoapps/util/query.py | 4 + ..._auto__add_field_orderitem_service_fee.py} | 83 ++---- lms/djangoapps/shoppingcart/models.py | 33 ++- lms/djangoapps/shoppingcart/reports.py | 239 ++++++++---------- .../shoppingcart/tests/test_models.py | 31 ++- .../shoppingcart/tests/test_reports.py | 131 +++++----- .../shoppingcart/tests/test_views.py | 24 +- lms/djangoapps/shoppingcart/views.py | 16 +- lms/envs/aws.py | 1 - lms/envs/common.py | 2 - .../shoppingcart/download_report.html | 6 - 12 files changed, 259 insertions(+), 338 deletions(-) create mode 100644 common/djangoapps/util/query.py rename lms/djangoapps/shoppingcart/migrations/{0007_auto__add_refundreport__add_report__add_certificatestatusreport__add_i.py => 0007_auto__add_field_orderitem_service_fee.py} (71%) diff --git a/common/djangoapps/student/models.py b/common/djangoapps/student/models.py index 7e348177b5..f92e26bf5c 100644 --- a/common/djangoapps/student/models.py +++ b/common/djangoapps/student/models.py @@ -10,10 +10,12 @@ file and check it in at the same time as your model changes. To do that, 2. ./manage.py lms schemamigration student --auto description_of_your_change 3. Add the migration file created in edx-platform/common/djangoapps/student/migrations/ """ +import crum from datetime import datetime import hashlib import json import logging +from pytz import UTC import uuid from django.conf import settings @@ -25,16 +27,13 @@ from django.dispatch import receiver, Signal import django.dispatch from django.forms import ModelForm, forms from django.core.exceptions import ObjectDoesNotExist - -from course_modes.models import CourseMode -import lms.lib.comment_client as cc -from pytz import UTC -import crum - from track import contexts from track.views import server_track from eventtracking import tracker +from course_modes.models import CourseMode +import lms.lib.comment_client as cc + unenroll_done = Signal(providing_args=["course_enrollment"]) log = logging.getLogger(__name__) AUDIT_LOG = logging.getLogger("audit") @@ -581,15 +580,17 @@ class CourseEnrollment(models.Model): ) @classmethod - def enrollments_in(cls, course_id, mode=None): + def enrollment_counts(cls, course_id): """ - Return a queryset of CourseEnrollment for every active enrollment in the course course_id. - Returns only CourseEnrollments with the given mode, if a mode is supplied by the caller. + Returns a dictionary that stores the total enrollment count for a course, as well as the + enrollment count for each individual mode. """ - if mode is None: - return cls.objects.filter(course_id=course_id, is_active=True,) - else: - return cls.objects.filter(course_id=course_id, is_active=True, mode=mode) + d = {} + d['total'] = cls.objects.filter(course_id=course_id, is_active=True).count() + d['honor'] = cls.objects.filter(course_id=course_id, is_active=True, mode='honor').count() + d['audit'] = cls.objects.filter(course_id=course_id, is_active=True, mode='audit').count() + d['verified'] = cls.objects.filter(course_id=course_id, is_active=True, mode='verified').count() + return d def activate(self): """Makes this `CourseEnrollment` record active. Saves immediately.""" diff --git a/common/djangoapps/util/query.py b/common/djangoapps/util/query.py new file mode 100644 index 0000000000..cff97078a3 --- /dev/null +++ b/common/djangoapps/util/query.py @@ -0,0 +1,4 @@ +from django.conf import settings + +def use_read_replica_if_available(queryset): + return queryset.using("read_replica") if "read_replica" in settings.DATABASES else queryset \ No newline at end of file diff --git a/lms/djangoapps/shoppingcart/migrations/0007_auto__add_refundreport__add_report__add_certificatestatusreport__add_i.py b/lms/djangoapps/shoppingcart/migrations/0007_auto__add_field_orderitem_service_fee.py similarity index 71% rename from lms/djangoapps/shoppingcart/migrations/0007_auto__add_refundreport__add_report__add_certificatestatusreport__add_i.py rename to lms/djangoapps/shoppingcart/migrations/0007_auto__add_field_orderitem_service_fee.py index f2461e7885..227b8ccc0f 100644 --- a/lms/djangoapps/shoppingcart/migrations/0007_auto__add_refundreport__add_report__add_certificatestatusreport__add_i.py +++ b/lms/djangoapps/shoppingcart/migrations/0007_auto__add_field_orderitem_service_fee.py @@ -8,57 +8,30 @@ from django.db import models class Migration(SchemaMigration): def forwards(self, orm): - # Adding model 'RefundReport' - db.create_table('shoppingcart_refundreport', ( - ('report_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['shoppingcart.Report'], unique=True, primary_key=True)), - )) - db.send_create_signal('shoppingcart', ['RefundReport']) - - # Adding model 'Report' - db.create_table('shoppingcart_report', ( - ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), - )) - db.send_create_signal('shoppingcart', ['Report']) - - # Adding model 'CertificateStatusReport' - db.create_table('shoppingcart_certificatestatusreport', ( - ('report_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['shoppingcart.Report'], unique=True, primary_key=True)), - )) - db.send_create_signal('shoppingcart', ['CertificateStatusReport']) - - # Adding model 'ItemizedPurchaseReport' - db.create_table('shoppingcart_itemizedpurchasereport', ( - ('report_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['shoppingcart.Report'], unique=True, primary_key=True)), - )) - db.send_create_signal('shoppingcart', ['ItemizedPurchaseReport']) - - # Adding model 'UniversityRevenueShareReport' - db.create_table('shoppingcart_universityrevenuesharereport', ( - ('report_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['shoppingcart.Report'], unique=True, primary_key=True)), - )) - db.send_create_signal('shoppingcart', ['UniversityRevenueShareReport']) - # Adding field 'OrderItem.service_fee' db.add_column('shoppingcart_orderitem', 'service_fee', self.gf('django.db.models.fields.DecimalField')(default=0.0, max_digits=30, decimal_places=2), keep_default=False) + # Adding index on 'OrderItem', fields ['status'] + db.create_index('shoppingcart_orderitem', ['status']) + + # Adding index on 'OrderItem', fields ['fulfilled_time'] + db.create_index('shoppingcart_orderitem', ['fulfilled_time']) + + # Adding index on 'OrderItem', fields ['refund_requested_time'] + db.create_index('shoppingcart_orderitem', ['refund_requested_time']) + def backwards(self, orm): - # Deleting model 'RefundReport' - db.delete_table('shoppingcart_refundreport') + # Removing index on 'OrderItem', fields ['refund_requested_time'] + db.delete_index('shoppingcart_orderitem', ['refund_requested_time']) - # Deleting model 'Report' - db.delete_table('shoppingcart_report') + # Removing index on 'OrderItem', fields ['fulfilled_time'] + db.delete_index('shoppingcart_orderitem', ['fulfilled_time']) - # Deleting model 'CertificateStatusReport' - db.delete_table('shoppingcart_certificatestatusreport') - - # Deleting model 'ItemizedPurchaseReport' - db.delete_table('shoppingcart_itemizedpurchasereport') - - # Deleting model 'UniversityRevenueShareReport' - db.delete_table('shoppingcart_universityrevenuesharereport') + # Removing index on 'OrderItem', fields ['status'] + db.delete_index('shoppingcart_orderitem', ['status']) # Deleting field 'OrderItem.service_fee' db.delete_column('shoppingcart_orderitem', 'service_fee') @@ -108,14 +81,6 @@ class Migration(SchemaMigration): 'mode': ('django.db.models.fields.SlugField', [], {'max_length': '50'}), 'orderitem_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['shoppingcart.OrderItem']", 'unique': 'True', 'primary_key': 'True'}) }, - 'shoppingcart.certificatestatusreport': { - 'Meta': {'object_name': 'CertificateStatusReport', '_ormbases': ['shoppingcart.Report']}, - 'report_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['shoppingcart.Report']", 'unique': 'True', 'primary_key': 'True'}) - }, - 'shoppingcart.itemizedpurchasereport': { - 'Meta': {'object_name': 'ItemizedPurchaseReport', '_ormbases': ['shoppingcart.Report']}, - 'report_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['shoppingcart.Report']", 'unique': 'True', 'primary_key': 'True'}) - }, 'shoppingcart.order': { 'Meta': {'object_name': 'Order'}, 'bill_to_cardtype': ('django.db.models.fields.CharField', [], {'max_length': '32', 'blank': 'True'}), @@ -139,15 +104,15 @@ class Migration(SchemaMigration): 'shoppingcart.orderitem': { 'Meta': {'object_name': 'OrderItem'}, 'currency': ('django.db.models.fields.CharField', [], {'default': "'usd'", 'max_length': '8'}), - 'fulfilled_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'fulfilled_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}), 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), 'line_desc': ('django.db.models.fields.CharField', [], {'default': "'Misc. Item'", 'max_length': '1024'}), 'order': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['shoppingcart.Order']"}), 'qty': ('django.db.models.fields.IntegerField', [], {'default': '1'}), - 'refund_requested_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'refund_requested_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}), 'report_comments': ('django.db.models.fields.TextField', [], {'default': "''"}), 'service_fee': ('django.db.models.fields.DecimalField', [], {'default': '0.0', 'max_digits': '30', 'decimal_places': '2'}), - 'status': ('django.db.models.fields.CharField', [], {'default': "'cart'", 'max_length': '32'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'cart'", 'max_length': '32', 'db_index': 'True'}), 'unit_cost': ('django.db.models.fields.DecimalField', [], {'default': '0.0', 'max_digits': '30', 'decimal_places': '2'}), 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) }, @@ -163,18 +128,6 @@ class Migration(SchemaMigration): 'course_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'db_index': 'True'}), 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) }, - 'shoppingcart.refundreport': { - 'Meta': {'object_name': 'RefundReport', '_ormbases': ['shoppingcart.Report']}, - 'report_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['shoppingcart.Report']", 'unique': 'True', 'primary_key': 'True'}) - }, - 'shoppingcart.report': { - 'Meta': {'object_name': 'Report'}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) - }, - 'shoppingcart.universityrevenuesharereport': { - 'Meta': {'object_name': 'UniversityRevenueShareReport', '_ormbases': ['shoppingcart.Report']}, - 'report_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['shoppingcart.Report']", 'unique': 'True', 'primary_key': 'True'}) - }, 'student.courseenrollment': { 'Meta': {'ordering': "('user', 'course_id')", 'unique_together': "(('user', 'course_id'),)", 'object_name': 'CourseEnrollment'}, 'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), diff --git a/lms/djangoapps/shoppingcart/models.py b/lms/djangoapps/shoppingcart/models.py index cf22b1756c..9afa82aa6f 100644 --- a/lms/djangoapps/shoppingcart/models.py +++ b/lms/djangoapps/shoppingcart/models.py @@ -1,13 +1,12 @@ +from collections import namedtuple from datetime import datetime +from decimal import Decimal import pytz import logging import smtplib import unicodecsv -from model_utils.managers import InheritanceManager -from collections import namedtuple from boto.exception import BotoServerError # this is a super-class of SESError and catches connection errors - from django.dispatch import receiver from django.db import models from django.conf import settings @@ -16,7 +15,9 @@ from django.core.mail import send_mail from django.contrib.auth.models import User from django.utils.translation import ugettext as _ from django.db import transaction +from django.db.models import Sum from django.core.urlresolvers import reverse +from model_utils.managers import InheritanceManager from xmodule.modulestore.django import modulestore from xmodule.course_module import CourseDescriptor @@ -26,6 +27,7 @@ from course_modes.models import CourseMode from edxmako.shortcuts import render_to_string from student.views import course_from_id from student.models import CourseEnrollment, unenroll_done +from util.query import use_read_replica_if_available from verify_student.models import SoftwareSecurePhotoVerification @@ -40,8 +42,6 @@ ORDER_STATUSES = ( ('refunded', 'refunded'), ) - - # we need a tuple to represent the primary key of various OrderItem subclasses OrderItemSubclassPK = namedtuple('OrderItemSubclassPK', ['cls', 'pk']) # pylint: disable=C0103 @@ -570,6 +570,25 @@ class CertificateItem(OrderItem): billing_email=settings.PAYMENT_SUPPORT_EMAIL) @classmethod - def verified_certificates_in(cls, course_id, status): + def verified_certificates_count(cls, course_id, status): """Return a queryset of CertificateItem for every verified enrollment in course_id with the given status.""" - return CertificateItem.objects.filter(course_id=course_id, mode='verified', status=status) + return use_read_replica_if_available( + CertificateItem.objects.filter(course_id=course_id, mode='verified', status=status).count()) + + # TODO combine these three methods into one + @classmethod + def verified_certificates_monetary_field_sum(cls, course_id, status, field_to_aggregate): + """ + Returns a Decimal indicating the total sum of field_to_aggregate for all verified certificates with a particular status. + + Sample usages: + - status 'refunded' and field_to_aggregate 'unit_cost' will give the total amount of money refunded for course_id + - status 'purchased' and field_to_aggregate 'service_fees' gives the sum of all service fees for purchased certificates + etc + """ + query = use_read_replica_if_available( + CertificateItem.objects.filter(course_id=course_id, mode='verified', status='purchased').aggregate(Sum(field_to_aggregate)))[field_to_aggregate + '__sum'] + if query is None: + return Decimal(0.00) + else: + return query diff --git a/lms/djangoapps/shoppingcart/reports.py b/lms/djangoapps/shoppingcart/reports.py index 360797f595..2dc14160f2 100644 --- a/lms/djangoapps/shoppingcart/reports.py +++ b/lms/djangoapps/shoppingcart/reports.py @@ -1,44 +1,49 @@ -from shoppingcart.models import CertificateItem, OrderItem -from django.db import models -from django.db.models import Sum -import unicodecsv -from django.conf import settings -from courseware.courses import get_course_by_id -from student.models import CourseEnrollment -from course_modes.models import CourseMode from decimal import Decimal +import unicodecsv + +from django.db import models +from django.conf import settings + +from courseware.courses import get_course_by_id +from course_modes.models import CourseMode +from shoppingcart.models import CertificateItem, OrderItem +from student.models import CourseEnrollment +from util.query import use_read_replica_if_available -class Report(Object): +class Report(object): """ Base class for making CSV reports related to revenue, enrollments, etc To make a different type of report, write a new subclass that implements - the methods report_row_generator and csv_report_header_row. + the methods rows and header. """ - def report_row_generator(self, start_date, end_date, start_letter=None, end_letter=None): + def rows(self, start_date, end_date, start_word=None, end_word=None): """ - Performs database queries necessary for the report. Returns an generator of + Performs database queries necessary for the report and eturns an generator of lists, in which each list is a separate row of the report. + + Arguments are start_date (datetime), end_date (datetime), start_word (str), + and end_word (str). Date comparisons are start_date <= [date of item] < end_date. """ raise NotImplementedError - def csv_report_header_row(self): + def header(self): """ Returns the appropriate header based on the report type, in the form of a list of strings. """ raise NotImplementedError - def make_report(self, filelike, start_date, end_date, start_letter=None, end_letter=None): + def write_csv(self, filelike, start_date, end_date, start_word=None, end_word=None): """ - Given the string report_type, a file object to write to, and start/end date bounds, + Given a file object to write to and {start/end date, start/end letter} bounds, generates a CSV report of the appropriate type. """ - items = self.report_row_generator(start_date, end_date, start_letter, end_letter) + items = self.rows(start_date, end_date, start_word, end_word) writer = unicodecsv.writer(filelike, encoding="utf-8") - writer.writerow(self.csv_report_header_row()) + writer.writerow(self.header()) for item in items: writer.writerow(item) @@ -51,12 +56,14 @@ class RefundReport(Report): order number, customer name, date of transaction, date of refund, and any service fees. """ - def report_row_generator(self, start_date, end_date, start_letter=None, end_letter=None): - query = CertificateItem.objects.select_related('user__profile').filter( - status="refunded", - refund_requested_time__gte=start_date, - refund_requested_time__lt=end_date, - ).order_by('refund_requested_time') + def rows(self, start_date, end_date, start_word=None, end_word=None): + query = use_read_replica_if_available( + CertificateItem.objects.select_related('user__profile').filter( + status="refunded", + refund_requested_time__gte=start_date, + refund_requested_time__lt=end_date, + ).order_by('refund_requested_time')) + for item in query: yield [ item.order_id, @@ -67,7 +74,7 @@ class RefundReport(Report): item.service_fee, ] - def csv_report_header_row(self): + def header(self): return [ "Order Number", "Customer Name", @@ -86,12 +93,13 @@ class ItemizedPurchaseReport(Report): a given start_date and end_date, we find that purchase's time, order ID, status, quantity, unit cost, total cost, currency, description, and related comments. """ - def report_row_generator(self, start_date, end_date, start_letter=None, end_letter=None): - query = OrderItem.objects.filter( - status="purchased", - fulfilled_time__gte=start_date, - fulfilled_time__lt=end_date, - ).order_by("fulfilled_time") + def rows(self, start_date, end_date, start_word=None, end_word=None): + query = use_read_replica_if_available( + OrderItem.objects.filter( + status="purchased", + fulfilled_time__gte=start_date, + fulfilled_time__lt=end_date, + ).order_by("fulfilled_time")) for item in query: yield [ @@ -106,7 +114,7 @@ class ItemizedPurchaseReport(Report): item.report_comments, ] - def csv_report_header_row(self): + def header(self): return [ "Purchase Time", "Order ID", @@ -124,65 +132,55 @@ class CertificateStatusReport(Report): """ Subclass of Report, used to generate Certificate Status Reports for Ed Services. - For each course in each university whose name is within the range start_letter and end_letter, + For each course in each university whose name is within the range start_word and end_word, inclusive, (i.e., the letter range H-J includes both Ithaca College and Harvard University), we calculate the total enrollment, audit enrollment, honor enrollment, verified enrollment, total gross revenue, gross revenue over the minimum, and total dollars refunded. """ - def report_row_generator(self, start_date, end_date, start_letter=None, end_letter=None): + def rows(self, start_date, end_date, start_word=None, end_word=None): results = [] - for course_id in settings.COURSE_LISTINGS['default']: - # If the first letter of the university is between start_letter and end_letter, then we include + for course_id in course_ids_between(start_word, end_word): + # If the first letter of the university is between start_word and end_word, then we include # it in the report. These comparisons are unicode-safe. - if (start_letter.lower() <= course_id.lower()) and (end_letter.lower() >= course_id.lower()) and (get_course_by_id(course_id) is not None): - cur_course = get_course_by_id(course_id) - university = cur_course.org - course = cur_course.number + " " + cur_course.display_name_with_default # TODO add term (i.e. Fall 2013)? - enrollments = CourseEnrollment.enrollments_in(course_id) - total_enrolled = enrollments.count() - audit_enrolled = CourseEnrollment.enrollments_in(course_id, "audit").count() - honor_enrolled = CourseEnrollment.enrollments_in(course_id, "honor").count() + cur_course = get_course_by_id(course_id) + university = cur_course.org + course = cur_course.number + " " + cur_course.display_name_with_default # TODO add term (i.e. Fall 2013)? + counts = CourseEnrollment.enrollment_counts(course_id) + total_enrolled = counts['total'] + audit_enrolled = counts['audit'] + honor_enrolled = counts['honor'] - # Since every verified enrollment has 1 and only 1 cert item, let's just query those - verified_enrollments = CertificateItem.verified_certificates_in(course_id, 'purchased') - if verified_enrollments is None: - verified_enrolled = 0 - gross_rev = Decimal(0.00) - gross_rev_over_min = Decimal(0.00) - else: - verified_enrolled = verified_enrollments.count() - gross_rev_temp = verified_enrollments.aggregate(Sum('unit_cost')) - gross_rev = gross_rev_temp['unit_cost__sum'] - gross_rev_over_min = gross_rev - (CourseMode.objects.get(course_id=course_id, mode_slug="verified").min_price * verified_enrolled) + if counts['verified'] == 0: + verified_enrolled = 0 + gross_rev = Decimal(0.00) + gross_rev_over_min = Decimal(0.00) + else: + verified_enrolled = counts['verified'] + gross_rev = CertificateItem.verified_certificates_monetary_field_sum(course_id, 'purchased', 'unit_cost') + gross_rev_over_min = gross_rev - (CourseMode.min_course_price_for_currency(course_id, 'usd') * verified_enrolled) - # should I be worried about is_active here? - refunded_enrollments = CertificateItem.verified_certificates_in(course_id, 'refunded') - if refunded_enrollments is None: - number_of_refunds = 0 - dollars_refunded = Decimal(0.00) - else: - number_of_refunds = refunded_enrollments.count() - dollars_refunded_temp = refunded_enrollments.aggregate(Sum('unit_cost')) - dollars_refunded = dollars_refunded_temp['unit_cost__sum'] + # should I be worried about is_active here? + number_of_refunds = CertificateItem.verified_certificates_count(course_id, 'refunded') + if number_of_refunds == 0: + dollars_refunded = Decimal(0.00) + else: + dollars_refunded = CertificateItem.verified_certificates_monetary_field_sum(course_id, 'refunded', 'unit_cost') - result = [ - university, - course, - total_enrolled, - audit_enrolled, - honor_enrolled, - verified_enrolled, - gross_rev, - gross_rev_over_min, - number_of_refunds, - dollars_refunded - ] + result = [ + university, + course, + total_enrolled, + audit_enrolled, + honor_enrolled, + verified_enrolled, + gross_rev, + gross_rev_over_min, + number_of_refunds, + dollars_refunded + ] + yield result - results.append(result) - for item in results: - yield item - - def csv_report_header_row(self): + def header(self): return [ "University", "Course", @@ -201,61 +199,35 @@ class UniversityRevenueShareReport(Report): """ Subclass of Report, used to generate University Revenue Share Reports for finance purposes. - For each course in each university whose name is within the range start_letter and end_letter, + For each course in each university whose name is within the range start_word and end_word, inclusive, (i.e., the letter range H-J includes both Ithaca College and Harvard University), we calculate the total revenue generated by that particular course. This includes the number of transactions, total payments collected, service fees, number of refunds, and total amount of refunds. """ - def report_row_generator(self, start_date, end_date, start_letter=None, end_letter=None): + def rows(self, start_date, end_date, start_word=None, end_word=None): results = [] - for course_id in settings.COURSE_LISTINGS['default']: - # If the first letter of the university is between start_letter and end_letter, then we include - # it in the report. These comparisons are unicode-safe. - if (start_letter.lower() <= course_id.lower()) and (end_letter.lower() >= course_id.lower()): - try: - cur_course = get_course_by_id(course_id) - except: - break - university = cur_course.org - course = cur_course.number + " " + cur_course.display_name_with_default - num_transactions = 0 # TODO clarify with billing what transactions are included in this (purchases? refunds? etc) + for course_id in course_ids_between(start_word, end_word): + cur_course = get_course_by_id(course_id) + university = cur_course.org + course = cur_course.number + " " + cur_course.display_name_with_default + num_transactions = 0 # TODO clarify with billing what transactions are included in this (purchases? refunds? etc) + total_payments_collected = CertificateItem.verified_certificates_monetary_field_sum(course_id, 'purchased', 'unit_cost') + service_fees = CertificateItem.verified_certificates_monetary_field_sum(course_id, 'purchased', 'service_fee') + num_refunds = CertificateItem.verified_certificates_count(course_id, "refunded") + amount_refunds = CertificateItem.verified_certificates_monetary_field_sum(course_id, 'refunded', 'unit_cost') - all_paid_certs = CertificateItem.verified_certificates_in(course_id, "purchased") - try: - total_payments_collected_temp = all_paid_certs.aggregate(Sum('unit_cost')) - total_payments_collected = total_payments_collected_temp['unit_cost__sum'] - except: - total_payments_collected = Decimal(0.00) - try: - total_service_fees_temp = all_paid_certs.aggregate(Sum('service_fee')) - service_fees = total_service_fees_temp['service_fee__sum'] - except: - service_fees = Decimal(0.00) + result = [ + university, + course, + num_transactions, + total_payments_collected, + service_fees, + num_refunds, + amount_refunds + ] + yield result - refunded_enrollments = CertificateItem.verified_certificates_in(course_id, "refunded") - num_refunds = refunded_enrollments.count() - - amount_refunds_temp = refunded_enrollments.aggregate(Sum('unit_cost')) - if amount_refunds_temp['unit_cost__sum'] is None: - amount_refunds = Decimal(0.00) - else: - amount_refunds = amount_refunds_temp['unit_cost__sum'] - - result = [ - university, - course, - num_transactions, - total_payments_collected, - service_fees, - num_refunds, - amount_refunds - ] - results.append(result) - - for item in results: - yield item - - def csv_report_header_row(self): + def header(self): return [ "University", "Course", @@ -265,3 +237,14 @@ class UniversityRevenueShareReport(Report): "Number of Successful Refunds", "Total Amount of Refunds", ] + +def course_ids_between(start_word, end_word): + """ + Returns a list of all valid course_ids that fall alphabetically between start_word and end_word. + These comparisons are unicode-safe. + """ + valid_courses = [] + for course_id in settings.COURSE_LISTINGS['default']: + if (start_word.lower() <= course_id.lower()) and (end_word.lower() >= course_id.lower()) and (get_course_by_id(course_id) is not None): + valid_courses.append(course_id) + return valid_courses diff --git a/lms/djangoapps/shoppingcart/tests/test_models.py b/lms/djangoapps/shoppingcart/tests/test_models.py index a9b0ede4d5..d666a7e518 100644 --- a/lms/djangoapps/shoppingcart/tests/test_models.py +++ b/lms/djangoapps/shoppingcart/tests/test_models.py @@ -356,26 +356,25 @@ class ItemizedPurchaseReportTest(ModuleStoreTestCase): self.cart.purchase() self.now = datetime.datetime.now(pytz.UTC) - # We can't modify the values returned by report_row_generator directly, since it's a generator, but - # we need the times on CORRECT_CSV and the generated report to match. So, we extract the times from - # the report_row_generator and place them in CORRECT_CSV. - self.time_str = {} - report = initialize_report("itemized_purchase_report") - purchases = report.report_row_generator(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) - num_of_item = 0 - for item in purchases: - num_of_item += 1 - self.time_str[num_of_item] = item[0] + paid_reg = PaidCourseRegistration.objects.get(course_id=self.course_id, user=self.user) + paid_reg.fulfilled_time = self.now + paid_reg.refund_requested_time = self.now + paid_reg.save() + + cert = CertificateItem.objects.get(course_id=self.course_id, user=self.user) + cert.fulfilled_time = self.now + cert.refund_requested_time = self.now + cert.save() self.CORRECT_CSV = dedent(""" Purchase Time,Order ID,Status,Quantity,Unit Cost,Total Cost,Currency,Description,Comments - {time_str1},1,purchased,1,40,40,usd,Registration for Course: Robot Super Course,Ba\xc3\xbc\xe5\x8c\x85 - {time_str2},1,purchased,1,40,40,usd,"Certificate of Achievement, verified cert for course Robot Super Course", - """.format(time_str1=str(self.time_str[1]), time_str2=str(self.time_str[2]))) + {time_str},1,purchased,1,40,40,usd,Registration for Course: Robot Super Course,Ba\xc3\xbc\xe5\x8c\x85 + {time_str},1,purchased,1,40,40,usd,"Certificate of Achievement, verified cert for course Robot Super Course", + """.format(time_str=str(self.now))) def test_purchased_items_btw_dates(self): report = initialize_report("itemized_purchase_report") - purchases = report.report_row_generator(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + purchases = report.rows(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) # since there's not many purchases, just run through the generator to make sure we've got the right number num_purchases = 0 @@ -385,7 +384,7 @@ class ItemizedPurchaseReportTest(ModuleStoreTestCase): #self.assertIn(self.reg.orderitem_ptr, purchases) #self.assertIn(self.cert_item.orderitem_ptr, purchases) - no_purchases = report.report_row_generator(self.now + self.FIVE_MINS, self.now + self.FIVE_MINS + self.FIVE_MINS) + no_purchases = report.rows(self.now + self.FIVE_MINS, self.now + self.FIVE_MINS + self.FIVE_MINS) num_purchases = 0 for item in no_purchases: @@ -398,7 +397,7 @@ class ItemizedPurchaseReportTest(ModuleStoreTestCase): """ report = initialize_report("itemized_purchase_report") csv_file = StringIO.StringIO() - report.make_report(csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + report.write_csv(csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) csv = csv_file.getvalue() csv_file.close() # Using excel mode csv, which automatically ends lines with \r\n, so need to convert to \n diff --git a/lms/djangoapps/shoppingcart/tests/test_reports.py b/lms/djangoapps/shoppingcart/tests/test_reports.py index 6c90cd8af8..8301fa96b3 100644 --- a/lms/djangoapps/shoppingcart/tests/test_reports.py +++ b/lms/djangoapps/shoppingcart/tests/test_reports.py @@ -3,20 +3,21 @@ Tests for the Shopping Cart Models """ import StringIO from textwrap import dedent +import pytz +import datetime from django.conf import settings from django.test.utils import override_settings -from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase -from xmodule.modulestore.tests.factories import CourseFactory + +from course_modes.models import CourseMode from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE from shoppingcart.models import (Order, CertificateItem) from shoppingcart.reports import ItemizedPurchaseReport, CertificateStatusReport, UniversityRevenueShareReport, RefundReport +from shoppingcart.views import initialize_report, REPORT_TYPES from student.tests.factories import UserFactory from student.models import CourseEnrollment -from course_modes.models import CourseMode -from shoppingcart.views import initialize_report, REPORT_TYPES -import pytz -import datetime +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory @override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) @@ -28,37 +29,37 @@ class ReportTypeTests(ModuleStoreTestCase): def setUp(self): # Need to make a *lot* of users for this one - self.user1 = UserFactory.create() - self.user1.profile.name = "John Doe" - self.user1.profile.save() + self.first_verified_user = UserFactory.create() + self.first_verified_user.profile.name = "John Doe" + self.first_verified_user.profile.save() - self.user2 = UserFactory.create() - self.user2.profile.name = "Jane Deer" - self.user2.profile.save() + self.second_verified_user = UserFactory.create() + self.second_verified_user.profile.name = "Jane Deer" + self.second_verified_user.profile.save() - self.user3 = UserFactory.create() - self.user3.profile.name = "Joe Miller" - self.user3.profile.save() + self.first_audit_user = UserFactory.create() + self.first_audit_user.profile.name = "Joe Miller" + self.first_audit_user.profile.save() - self.user4 = UserFactory.create() - self.user4.profile.name = "Simon Blackquill" - self.user4.profile.save() + self.second_audit_user = UserFactory.create() + self.second_audit_user.profile.name = "Simon Blackquill" + self.second_audit_user.profile.save() - self.user5 = UserFactory.create() - self.user5.profile.name = "Super Mario" - self.user5.profile.save() + self.third_audit_user = UserFactory.create() + self.third_audit_user.profile.name = "Super Mario" + self.third_audit_user.profile.save() - self.user6 = UserFactory.create() - self.user6.profile.name = "Princess Peach" - self.user6.profile.save() + self.honor_user = UserFactory.create() + self.honor_user.profile.name = "Princess Peach" + self.honor_user.profile.save() - self.user7 = UserFactory.create() - self.user7.profile.name = "King Bowser" - self.user7.profile.save() + self.first_refund_user = UserFactory.create() + self.first_refund_user.profile.name = "King Bowser" + self.first_refund_user.profile.save() - self.user8 = UserFactory.create() - self.user8.profile.name = "Susan Smith" - self.user8.profile.save() + self.second_refund_user = UserFactory.create() + self.second_refund_user.profile.name = "Susan Smith" + self.second_refund_user.profile.save() # Two are verified, three are audit, one honor @@ -79,55 +80,53 @@ class ReportTypeTests(ModuleStoreTestCase): course_mode2.save() # User 1 & 2 will be verified - self.cart1 = Order.get_cart_for_user(self.user1) + self.cart1 = Order.get_cart_for_user(self.first_verified_user) CertificateItem.add_to_order(self.cart1, self.course_id, self.cost, 'verified') self.cart1.purchase() - self.cart2 = Order.get_cart_for_user(self.user2) + self.cart2 = Order.get_cart_for_user(self.second_verified_user) CertificateItem.add_to_order(self.cart2, self.course_id, self.cost, 'verified') self.cart2.purchase() # Users 3, 4, and 5 are audit - CourseEnrollment.enroll(self.user3, self.course_id, "audit") - CourseEnrollment.enroll(self.user4, self.course_id, "audit") - CourseEnrollment.enroll(self.user5, self.course_id, "audit") + CourseEnrollment.enroll(self.first_audit_user, self.course_id, "audit") + CourseEnrollment.enroll(self.second_audit_user, self.course_id, "audit") + CourseEnrollment.enroll(self.third_audit_user, self.course_id, "audit") # User 6 is honor - CourseEnrollment.enroll(self.user6, self.course_id, "honor") + CourseEnrollment.enroll(self.honor_user, self.course_id, "honor") self.now = datetime.datetime.now(pytz.UTC) # Users 7 & 8 are refunds - self.cart = Order.get_cart_for_user(self.user7) + self.cart = Order.get_cart_for_user(self.first_refund_user) CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') self.cart.purchase() - CourseEnrollment.unenroll(self.user7, self.course_id) + CourseEnrollment.unenroll(self.first_refund_user, self.course_id) - self.cart = Order.get_cart_for_user(self.user8) + self.cart = Order.get_cart_for_user(self.second_refund_user) CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') - self.cart.purchase(self.user8, self.course_id) - CourseEnrollment.unenroll(self.user8, self.course_id) - - # We can't modify the values returned by report_row_generator directly, since it's a generator, but - # we need the times on CORRECT_CSV and the generated report to match. So, we extract the times from - # the report_row_generator and place them in CORRECT_CSV. - self.time_str = {} - report = initialize_report("refund_report") - refunds = report.report_row_generator(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) - time_index = 0 - for item in refunds: - self.time_str[time_index] = item[2] - time_index += 1 - self.time_str[time_index] = item[3] - time_index += 1 - self.CORRECT_REFUND_REPORT_CSV = dedent(""" - Order Number,Customer Name,Date of Original Transaction,Date of Refund,Amount of Refund,Service Fees (if any) - 3,King Bowser,{time_str0},{time_str1},40,0 - 4,Susan Smith,{time_str2},{time_str3},40,0 - """.format(time_str0=str(self.time_str[0]), time_str1=str(self.time_str[1]), time_str2=str(self.time_str[2]), time_str3=str(self.time_str[3]))) + self.cart.purchase(self.second_refund_user, self.course_id) + CourseEnrollment.unenroll(self.second_refund_user, self.course_id) self.test_time = datetime.datetime.now(pytz.UTC) + first_refund = CertificateItem.objects.get(id=3) + first_refund.fulfilled_time = self.test_time + first_refund.refund_requested_time = self.test_time + first_refund.save() + + second_refund = CertificateItem.objects.get(id=4) + second_refund.fulfilled_time = self.test_time + second_refund.refund_requested_time = self.test_time + second_refund.save() + + self.CORRECT_REFUND_REPORT_CSV = dedent(""" + Order Number,Customer Name,Date of Original Transaction,Date of Refund,Amount of Refund,Service Fees (if any) + 3,King Bowser,{time_str},{time_str},40,0 + 4,Susan Smith,{time_str},{time_str},40,0 + """.format(time_str=str(self.test_time))) + self.CORRECT_CERT_STATUS_CSV = dedent(""" University,Course,Total Enrolled,Audit Enrollment,Honor Code Enrollment,Verified Enrollment,Gross Revenue,Gross Revenue over the Minimum,Number of Refunds,Dollars Refunded MITx,999 Robot Super Course,6,3,1,2,80.00,0.00,2,80.00 @@ -138,9 +137,9 @@ class ReportTypeTests(ModuleStoreTestCase): MITx,999 Robot Super Course,0,80.00,0.00,2,80.00 """.format(time_str=str(self.test_time))) - def test_refund_report_report_row_generator(self): + def test_refund_report_rows(self): report = initialize_report("refund_report") - refunded_certs = report.report_row_generator(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + refunded_certs = report.rows(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) # check that we have the right number num_certs = 0 @@ -148,8 +147,8 @@ class ReportTypeTests(ModuleStoreTestCase): num_certs += 1 self.assertEqual(num_certs, 2) - self.assertTrue(CertificateItem.objects.get(user=self.user7, course_id=self.course_id)) - self.assertTrue(CertificateItem.objects.get(user=self.user8, course_id=self.course_id)) + self.assertTrue(CertificateItem.objects.get(user=self.first_refund_user, course_id=self.course_id)) + self.assertTrue(CertificateItem.objects.get(user=self.second_refund_user, course_id=self.course_id)) def test_refund_report_purchased_csv(self): """ @@ -157,7 +156,7 @@ class ReportTypeTests(ModuleStoreTestCase): """ report = initialize_report("refund_report") csv_file = StringIO.StringIO() - report.make_report(csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + report.write_csv(csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) csv = csv_file.getvalue() csv_file.close() # Using excel mode csv, which automatically ends lines with \r\n, so need to convert to \n @@ -166,13 +165,13 @@ class ReportTypeTests(ModuleStoreTestCase): def test_basic_cert_status_csv(self): report = initialize_report("certificate_status") csv_file = StringIO.StringIO() - report.make_report(csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS, 'A', 'Z') + report.write_csv(csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS, 'A', 'Z') csv = csv_file.getvalue() self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_CERT_STATUS_CSV.strip()) def test_basic_uni_revenue_share_csv(self): report = initialize_report("university_revenue_share") csv_file = StringIO.StringIO() - report.make_report(csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS, 'A', 'Z') + report.write_csv(csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS, 'A', 'Z') csv = csv_file.getvalue() self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_UNI_REVENUE_SHARE_CSV.strip()) diff --git a/lms/djangoapps/shoppingcart/tests/test_views.py b/lms/djangoapps/shoppingcart/tests/test_views.py index 949a570a0b..879e98f780 100644 --- a/lms/djangoapps/shoppingcart/tests/test_views.py +++ b/lms/djangoapps/shoppingcart/tests/test_views.py @@ -365,26 +365,6 @@ class CSVReportViewsTest(ModuleStoreTestCase): self.assertIn(_("There was an error in your date input. It should be formatted as YYYY-MM-DD"), response.content) - @patch('shoppingcart.views.render_to_response', render_mock) - @override_settings(PAYMENT_REPORT_MAX_ITEMS=0) - def test_report_csv_too_long(self): - PaidCourseRegistration.add_to_order(self.cart, self.course_id) - self.cart.purchase() - self.login_user() - self.add_to_download_group(self.user) - response = self.client.post(reverse('payment_csv_report'), {'start_date': '1970-01-01', - 'end_date': '2100-01-01', - 'requested_report': 'itemized_purchase_report'}) - - ((template, context), unused_kwargs) = render_mock.call_args - self.assertEqual(template, 'shoppingcart/download_report.html') - self.assertTrue(context['total_count_error']) - self.assertFalse(context['date_fmt_error']) - self.assertIn(_("There are too many results in your report.") + " (>0)", response.content) - - # just going to ignored the date in this test, since we already deal with date testing - # in test_models.py - CORRECT_CSV_NO_DATE_ITEMIZED_PURCHASE = ",1,purchased,1,40,40,usd,Registration for Course: Robot Super Course," def test_report_csv_itemized(self): @@ -398,7 +378,7 @@ class CSVReportViewsTest(ModuleStoreTestCase): 'requested_report': report_type}) self.assertEqual(response['Content-Type'], 'text/csv') report = initialize_report(report_type) - self.assertIn(",".join(report.csv_report_header_row()), response.content) + self.assertIn(",".join(report.header()), response.content) self.assertIn(self.CORRECT_CSV_NO_DATE_ITEMIZED_PURCHASE, response.content) def test_report_csv_university_revenue_share(self): @@ -412,7 +392,7 @@ class CSVReportViewsTest(ModuleStoreTestCase): 'requested_report': report_type}) self.assertEqual(response['Content-Type'], 'text/csv') report = initialize_report(report_type) - self.assertIn(",".join(report.csv_report_header_row()), response.content) + self.assertIn(",".join(report.header()), response.content) # TODO add another test here diff --git a/lms/djangoapps/shoppingcart/views.py b/lms/djangoapps/shoppingcart/views.py index eac4684253..4a5a728bae 100644 --- a/lms/djangoapps/shoppingcart/views.py +++ b/lms/djangoapps/shoppingcart/views.py @@ -11,11 +11,11 @@ from django.core.urlresolvers import reverse from django.views.decorators.csrf import csrf_exempt from django.contrib.auth.decorators import login_required from edxmako.shortcuts import render_to_response -from student.models import CourseEnrollment from shoppingcart.reports import RefundReport, ItemizedPurchaseReport, UniversityRevenueShareReport, CertificateStatusReport +from student.models import CourseEnrollment +from .exceptions import ItemAlreadyInCartException, AlreadyEnrolledInCourseException, CourseDoesNotExistException, ReportTypeDoesNotExistException from .models import Order, PaidCourseRegistration, OrderItem from .processors import process_postpay_callback, render_purchase_form_html -from .exceptions import ItemAlreadyInCartException, AlreadyEnrolledInCourseException, CourseDoesNotExistException, ReportTypeDoesNotExistException log = logging.getLogger("shoppingcart") @@ -213,20 +213,12 @@ def csv_report(request): return _render_report_form(start_str, end_str, start_letter, end_letter, report_type, date_fmt_error=True) report = initialize_report(report_type) - items = report.report_row_generator(start_date, end_date, start_letter, end_letter) - - # TODO add this back later as a query-est function or something - try: - if items.count() > settings.PAYMENT_REPORT_MAX_ITEMS: - # Error case: too many items would be generated in the report and we're at risk of timeout - return _render_report_form(start_str, end_str, start_letter, end_letter, report_type, total_count_error=True) - except: - pass + items = report.rows(start_date, end_date, start_letter, end_letter) response = HttpResponse(mimetype='text/csv') filename = "purchases_report_{}.csv".format(datetime.datetime.now(pytz.UTC).strftime("%Y-%m-%d-%H-%M-%S")) response['Content-Disposition'] = 'attachment; filename="{}"'.format(filename) - report.make_report(response, start_date, end_date, start_letter, end_letter) + report.write_csv(response, start_date, end_date, start_letter, end_letter) return response elif request.method == 'GET': diff --git a/lms/envs/aws.py b/lms/envs/aws.py index 8f39618fcb..dfa3e81394 100644 --- a/lms/envs/aws.py +++ b/lms/envs/aws.py @@ -170,7 +170,6 @@ PAID_COURSE_REGISTRATION_CURRENCY = ENV_TOKENS.get('PAID_COURSE_REGISTRATION_CUR # Payment Report Settings PAYMENT_REPORT_GENERATOR_GROUP = ENV_TOKENS.get('PAYMENT_REPORT_GENERATOR_GROUP', PAYMENT_REPORT_GENERATOR_GROUP) -PAYMENT_REPORT_MAX_ITEMS = ENV_TOKENS.get('PAYMENT_REPORT_MAX_ITEMS', PAYMENT_REPORT_MAX_ITEMS) # Bulk Email overrides BULK_EMAIL_DEFAULT_FROM_EMAIL = ENV_TOKENS.get('BULK_EMAIL_DEFAULT_FROM_EMAIL', BULK_EMAIL_DEFAULT_FROM_EMAIL) diff --git a/lms/envs/common.py b/lms/envs/common.py index e8dc187401..8512d8f160 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -555,8 +555,6 @@ PAID_COURSE_REGISTRATION_CURRENCY = ['usd', '$'] # Members of this group are allowed to generate payment reports PAYMENT_REPORT_GENERATOR_GROUP = 'shoppingcart_report_access' -# Maximum number of rows the report can contain -PAYMENT_REPORT_MAX_ITEMS = 10000 ################################# open ended grading config ##################### diff --git a/lms/templates/shoppingcart/download_report.html b/lms/templates/shoppingcart/download_report.html index e6fda982b2..57698f20dc 100644 --- a/lms/templates/shoppingcart/download_report.html +++ b/lms/templates/shoppingcart/download_report.html @@ -12,12 +12,6 @@ ${_("There was an error in your date input. It should be formatted as YYYY-MM-DD")} % endif - % if total_count_error: -
- ${_("There are too many results in your report.")} (>${settings.PAYMENT_REPORT_MAX_ITEMS}). - ${_("Try making the date range smaller.")} -
- % endif

${_("These reports are delimited by start and end dates.")}

From f2b0394211d68e7d09ea0eae233c7900fcfc4414 Mon Sep 17 00:00:00 2001 From: Julia Hansbrough Date: Mon, 6 Jan 2014 13:27:59 +0000 Subject: [PATCH 11/14] CR response --- common/djangoapps/course_modes/models.py | 8 + common/djangoapps/student/models.py | 15 +- lms/djangoapps/shoppingcart/models.py | 11 +- lms/djangoapps/shoppingcart/reports.py | 144 +++++++++++------- .../shoppingcart/tests/test_models.py | 11 +- .../shoppingcart/tests/test_reports.py | 24 +-- .../shoppingcart/tests/test_views.py | 22 ++- lms/djangoapps/shoppingcart/urls.py | 2 +- lms/djangoapps/shoppingcart/views.py | 21 ++- 9 files changed, 159 insertions(+), 99 deletions(-) diff --git a/common/djangoapps/course_modes/models.py b/common/djangoapps/course_modes/models.py index 4cf60c0e1e..9ed64e048f 100644 --- a/common/djangoapps/course_modes/models.py +++ b/common/djangoapps/course_modes/models.py @@ -95,6 +95,14 @@ class CourseMode(models.Model): else: return None + @classmethod + def min_course_price_for_verified_for_currency(cls, course_id, currency): + modes = cls.modes_for_course(course_id) + for mode in modes: + if (mode.currency == currency) and (mode.slug == 'verified'): + return mode.min_price + return 0 + @classmethod def min_course_price_for_currency(cls, course_id, currency): """ diff --git a/common/djangoapps/student/models.py b/common/djangoapps/student/models.py index f92e26bf5c..6ec263fed1 100644 --- a/common/djangoapps/student/models.py +++ b/common/djangoapps/student/models.py @@ -17,11 +17,13 @@ import json import logging from pytz import UTC import uuid +from collections import defaultdict from django.conf import settings from django.contrib.auth.models import User from django.contrib.auth.signals import user_logged_in, user_logged_out from django.db import models, IntegrityError +from django.db.models import Sum, Count from django.db.models.signals import post_save from django.dispatch import receiver, Signal import django.dispatch @@ -585,11 +587,14 @@ class CourseEnrollment(models.Model): Returns a dictionary that stores the total enrollment count for a course, as well as the enrollment count for each individual mode. """ - d = {} - d['total'] = cls.objects.filter(course_id=course_id, is_active=True).count() - d['honor'] = cls.objects.filter(course_id=course_id, is_active=True, mode='honor').count() - d['audit'] = cls.objects.filter(course_id=course_id, is_active=True, mode='audit').count() - d['verified'] = cls.objects.filter(course_id=course_id, is_active=True, mode='verified').count() + # Unfortunately, Django's "group by"-style queries look super-awkward + query = cls.objects.filter(course_id=course_id, is_active=True).values('mode').order_by().annotate(Count('mode')) + total = 0 + d = defaultdict(int) + for item in query: + d[item['mode']] = item['mode__count'] + total += item['mode__count'] + d['total'] = total return d def activate(self): diff --git a/lms/djangoapps/shoppingcart/models.py b/lms/djangoapps/shoppingcart/models.py index 9afa82aa6f..d0b20e547f 100644 --- a/lms/djangoapps/shoppingcart/models.py +++ b/lms/djangoapps/shoppingcart/models.py @@ -587,8 +587,17 @@ class CertificateItem(OrderItem): etc """ query = use_read_replica_if_available( - CertificateItem.objects.filter(course_id=course_id, mode='verified', status='purchased').aggregate(Sum(field_to_aggregate)))[field_to_aggregate + '__sum'] + CertificateItem.objects.filter(course_id=course_id, mode='verified', status=status).aggregate(Sum(field_to_aggregate)))[field_to_aggregate + '__sum'] if query is None: return Decimal(0.00) else: return query + + @classmethod + def verified_certificates_contributing_more_than_minimum(cls, course_id): + return use_read_replica_if_available( + CertificateItem.objects.filter( + course_id=course_id, + mode='verified', + status='purchased', + unit_cost__gt=(CourseMode.min_course_price_for_verified_for_currency(course_id, 'usd'))).count()) diff --git a/lms/djangoapps/shoppingcart/reports.py b/lms/djangoapps/shoppingcart/reports.py index 2dc14160f2..1c46ddb7c9 100644 --- a/lms/djangoapps/shoppingcart/reports.py +++ b/lms/djangoapps/shoppingcart/reports.py @@ -1,14 +1,18 @@ from decimal import Decimal import unicodecsv +import logging from django.db import models from django.conf import settings +from django.utils.translation import ugettext as _ from courseware.courses import get_course_by_id from course_modes.models import CourseMode from shoppingcart.models import CertificateItem, OrderItem from student.models import CourseEnrollment from util.query import use_read_replica_if_available +from xmodule.error_module import ErrorDescriptor +from xmodule.modulestore.django import modulestore class Report(object): @@ -18,8 +22,13 @@ class Report(object): To make a different type of report, write a new subclass that implements the methods rows and header. """ + def __init__(self, start_date, end_date, start_word=None, end_word=None): + self.start_date = start_date + self.end_date = end_date + self.start_word = start_word + self.end_word = end_word - def rows(self, start_date, end_date, start_word=None, end_word=None): + def rows(self): """ Performs database queries necessary for the report and eturns an generator of lists, in which each list is a separate row of the report. @@ -36,12 +45,12 @@ class Report(object): """ raise NotImplementedError - def write_csv(self, filelike, start_date, end_date, start_word=None, end_word=None): + def write_csv(self, filelike): """ Given a file object to write to and {start/end date, start/end letter} bounds, generates a CSV report of the appropriate type. """ - items = self.rows(start_date, end_date, start_word, end_word) + items = self.rows() writer = unicodecsv.writer(filelike, encoding="utf-8") writer.writerow(self.header()) for item in items: @@ -56,13 +65,20 @@ class RefundReport(Report): order number, customer name, date of transaction, date of refund, and any service fees. """ - def rows(self, start_date, end_date, start_word=None, end_word=None): - query = use_read_replica_if_available( + def rows(self): + query1 = use_read_replica_if_available( CertificateItem.objects.select_related('user__profile').filter( status="refunded", - refund_requested_time__gte=start_date, - refund_requested_time__lt=end_date, + refund_requested_time__gte=self.start_date, + refund_requested_time__lt=self.end_date, ).order_by('refund_requested_time')) + query2 = use_read_replica_if_available( + CertificateItem.objects.select_related('user__profile').filter( + status="refunded", + refund_requested_time=None, + )) + + query = query1 | query2 for item in query: yield [ @@ -76,12 +92,12 @@ class RefundReport(Report): def header(self): return [ - "Order Number", - "Customer Name", - "Date of Original Transaction", - "Date of Refund", - "Amount of Refund", - "Service Fees (if any)", + _("Order Number"), + _("Customer Name"), + _("Date of Original Transaction"), + _("Date of Refund"), + _("Amount of Refund"), + _("Service Fees (if any)"), ] @@ -93,12 +109,12 @@ class ItemizedPurchaseReport(Report): a given start_date and end_date, we find that purchase's time, order ID, status, quantity, unit cost, total cost, currency, description, and related comments. """ - def rows(self, start_date, end_date, start_word=None, end_word=None): + def rows(self): query = use_read_replica_if_available( OrderItem.objects.filter( status="purchased", - fulfilled_time__gte=start_date, - fulfilled_time__lt=end_date, + fulfilled_time__gte=self.start_date, + fulfilled_time__lt=self.end_date, ).order_by("fulfilled_time")) for item in query: @@ -116,15 +132,15 @@ class ItemizedPurchaseReport(Report): def header(self): return [ - "Purchase Time", - "Order ID", - "Status", - "Quantity", - "Unit Cost", - "Total Cost", - "Currency", - "Description", - "Comments" + _("Purchase Time"), + _("Order ID"), + _("Status"), + _("Quantity"), + _("Unit Cost"), + _("Total Cost"), + _("Currency"), + _("Description"), + _("Comments") ] @@ -137,9 +153,8 @@ class CertificateStatusReport(Report): calculate the total enrollment, audit enrollment, honor enrollment, verified enrollment, total gross revenue, gross revenue over the minimum, and total dollars refunded. """ - def rows(self, start_date, end_date, start_word=None, end_word=None): - results = [] - for course_id in course_ids_between(start_word, end_word): + def rows(self): + for course_id in course_ids_between(self.start_word, self.end_word): # If the first letter of the university is between start_word and end_word, then we include # it in the report. These comparisons are unicode-safe. cur_course = get_course_by_id(course_id) @@ -157,7 +172,9 @@ class CertificateStatusReport(Report): else: verified_enrolled = counts['verified'] gross_rev = CertificateItem.verified_certificates_monetary_field_sum(course_id, 'purchased', 'unit_cost') - gross_rev_over_min = gross_rev - (CourseMode.min_course_price_for_currency(course_id, 'usd') * verified_enrolled) + gross_rev_over_min = gross_rev - (CourseMode.min_course_price_for_verified_for_currency(course_id, 'usd') * verified_enrolled) + + num_verified_over_the_minimum = CertificateItem.verified_certificates_contributing_more_than_minimum(course_id) # should I be worried about is_active here? number_of_refunds = CertificateItem.verified_certificates_count(course_id, 'refunded') @@ -166,32 +183,46 @@ class CertificateStatusReport(Report): else: dollars_refunded = CertificateItem.verified_certificates_monetary_field_sum(course_id, 'refunded', 'unit_cost') - result = [ + course_announce_date = "" + course_reg_start_date = "" + course_reg_close_date = "" + registration_period = "" + + yield [ university, course, + "", + "", + "", + "", total_enrolled, audit_enrolled, honor_enrolled, verified_enrolled, gross_rev, gross_rev_over_min, + num_verified_over_the_minimum, number_of_refunds, dollars_refunded ] - yield result def header(self): return [ - "University", - "Course", - "Total Enrolled", - "Audit Enrollment", - "Honor Code Enrollment", - "Verified Enrollment", - "Gross Revenue", - "Gross Revenue over the Minimum", - "Number of Refunds", - "Dollars Refunded", + _("University"), + _("Course"), + _("Course Announce Date"), + _("Course Start Date"), + _("Course Registration Close Date"), + _("Course Registration Period"), + _("Total Enrolled"), + _("Audit Enrollment"), + _("Honor Code Enrollment"), + _("Verified Enrollment"), + _("Gross Revenue"), + _("Gross Revenue over the Minimum"), + _("Number of Verified Students Contributing More than the Minimum"), + _("Number of Refunds"), + _("Dollars Refunded"), ] @@ -204,19 +235,18 @@ class UniversityRevenueShareReport(Report): the total revenue generated by that particular course. This includes the number of transactions, total payments collected, service fees, number of refunds, and total amount of refunds. """ - def rows(self, start_date, end_date, start_word=None, end_word=None): - results = [] - for course_id in course_ids_between(start_word, end_word): + def rows(self): + for course_id in course_ids_between(self.start_word, self.end_word): cur_course = get_course_by_id(course_id) university = cur_course.org course = cur_course.number + " " + cur_course.display_name_with_default - num_transactions = 0 # TODO clarify with billing what transactions are included in this (purchases? refunds? etc) total_payments_collected = CertificateItem.verified_certificates_monetary_field_sum(course_id, 'purchased', 'unit_cost') service_fees = CertificateItem.verified_certificates_monetary_field_sum(course_id, 'purchased', 'service_fee') num_refunds = CertificateItem.verified_certificates_count(course_id, "refunded") amount_refunds = CertificateItem.verified_certificates_monetary_field_sum(course_id, 'refunded', 'unit_cost') + num_transactions = (num_refunds * 2) + CertificateItem.verified_certificates_count(course_id, "purchased") - result = [ + yield [ university, course, num_transactions, @@ -225,17 +255,16 @@ class UniversityRevenueShareReport(Report): num_refunds, amount_refunds ] - yield result def header(self): return [ - "University", - "Course", - "Number of Transactions", - "Total Payments Collected", - "Service Fees (if any)", - "Number of Successful Refunds", - "Total Amount of Refunds", + _("University"), + _("Course"), + _("Number of Transactions"), + _("Total Payments Collected"), + _("Service Fees (if any)"), + _("Number of Successful Refunds"), + _("Total Amount of Refunds"), ] def course_ids_between(start_word, end_word): @@ -243,8 +272,9 @@ def course_ids_between(start_word, end_word): Returns a list of all valid course_ids that fall alphabetically between start_word and end_word. These comparisons are unicode-safe. """ + valid_courses = [] - for course_id in settings.COURSE_LISTINGS['default']: - if (start_word.lower() <= course_id.lower()) and (end_word.lower() >= course_id.lower()) and (get_course_by_id(course_id) is not None): - valid_courses.append(course_id) + for course in modulestore().get_courses(): + if (start_word.lower() <= course.id.lower() <= course.id.lower()) and (get_course_by_id(course.id) is not None): + valid_courses.append(course.id) return valid_courses diff --git a/lms/djangoapps/shoppingcart/tests/test_models.py b/lms/djangoapps/shoppingcart/tests/test_models.py index d666a7e518..ea5bb4d34d 100644 --- a/lms/djangoapps/shoppingcart/tests/test_models.py +++ b/lms/djangoapps/shoppingcart/tests/test_models.py @@ -373,8 +373,8 @@ class ItemizedPurchaseReportTest(ModuleStoreTestCase): """.format(time_str=str(self.now))) def test_purchased_items_btw_dates(self): - report = initialize_report("itemized_purchase_report") - purchases = report.rows(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + report = initialize_report("itemized_purchase_report", self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + purchases = report.rows() # since there's not many purchases, just run through the generator to make sure we've got the right number num_purchases = 0 @@ -384,7 +384,8 @@ class ItemizedPurchaseReportTest(ModuleStoreTestCase): #self.assertIn(self.reg.orderitem_ptr, purchases) #self.assertIn(self.cert_item.orderitem_ptr, purchases) - no_purchases = report.rows(self.now + self.FIVE_MINS, self.now + self.FIVE_MINS + self.FIVE_MINS) + report = initialize_report("itemized_purchase_report", self.now + self.FIVE_MINS, self.now + self.FIVE_MINS + self.FIVE_MINS) + no_purchases = report.rows() num_purchases = 0 for item in no_purchases: @@ -395,9 +396,9 @@ class ItemizedPurchaseReportTest(ModuleStoreTestCase): """ Tests that a generated purchase report CSV is as we expect """ - report = initialize_report("itemized_purchase_report") + report = initialize_report("itemized_purchase_report", self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) csv_file = StringIO.StringIO() - report.write_csv(csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + report.write_csv(csv_file) csv = csv_file.getvalue() csv_file.close() # Using excel mode csv, which automatically ends lines with \r\n, so need to convert to \n diff --git a/lms/djangoapps/shoppingcart/tests/test_reports.py b/lms/djangoapps/shoppingcart/tests/test_reports.py index 8301fa96b3..a6f9b0aa8a 100644 --- a/lms/djangoapps/shoppingcart/tests/test_reports.py +++ b/lms/djangoapps/shoppingcart/tests/test_reports.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + """ Tests for the Shopping Cart Models """ @@ -128,18 +130,18 @@ class ReportTypeTests(ModuleStoreTestCase): """.format(time_str=str(self.test_time))) self.CORRECT_CERT_STATUS_CSV = dedent(""" - University,Course,Total Enrolled,Audit Enrollment,Honor Code Enrollment,Verified Enrollment,Gross Revenue,Gross Revenue over the Minimum,Number of Refunds,Dollars Refunded - MITx,999 Robot Super Course,6,3,1,2,80.00,0.00,2,80.00 + University,Course,Course Announce Date,Course Start Date,Course Registration Close Date,Course Registration Period,Total Enrolled,Audit Enrollment,Honor Code Enrollment,Verified Enrollment,Gross Revenue,Gross Revenue over the Minimum,Number of Verified Students Contributing More than the Minimum,Number of Refunds,Dollars Refunded + MITx,999 Robot Super Course,,,,,6,3,1,2,80.00,0.00,0,2,80.00 """.format(time_str=str(self.test_time))) self.CORRECT_UNI_REVENUE_SHARE_CSV = dedent(""" University,Course,Number of Transactions,Total Payments Collected,Service Fees (if any),Number of Successful Refunds,Total Amount of Refunds - MITx,999 Robot Super Course,0,80.00,0.00,2,80.00 + MITx,999 Robot Super Course,6,80.00,0.00,2,80.00 """.format(time_str=str(self.test_time))) def test_refund_report_rows(self): - report = initialize_report("refund_report") - refunded_certs = report.rows(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + report = initialize_report("refund_report", self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + refunded_certs = report.rows() # check that we have the right number num_certs = 0 @@ -154,24 +156,24 @@ class ReportTypeTests(ModuleStoreTestCase): """ Tests that a generated purchase report CSV is as we expect """ - report = initialize_report("refund_report") + report = initialize_report("refund_report", self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) csv_file = StringIO.StringIO() - report.write_csv(csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + report.write_csv(csv_file) csv = csv_file.getvalue() csv_file.close() # Using excel mode csv, which automatically ends lines with \r\n, so need to convert to \n self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_REFUND_REPORT_CSV.strip()) def test_basic_cert_status_csv(self): - report = initialize_report("certificate_status") + report = initialize_report("certificate_status", self.now - self.FIVE_MINS, self.now + self.FIVE_MINS, 'A', 'Z') csv_file = StringIO.StringIO() - report.write_csv(csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS, 'A', 'Z') + report.write_csv(csv_file) csv = csv_file.getvalue() self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_CERT_STATUS_CSV.strip()) def test_basic_uni_revenue_share_csv(self): - report = initialize_report("university_revenue_share") + report = initialize_report("university_revenue_share", self.now - self.FIVE_MINS, self.now + self.FIVE_MINS, 'A', 'Z') csv_file = StringIO.StringIO() - report.write_csv(csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS, 'A', 'Z') + report.write_csv(csv_file) csv = csv_file.getvalue() self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_UNI_REVENUE_SHARE_CSV.strip()) diff --git a/lms/djangoapps/shoppingcart/tests/test_views.py b/lms/djangoapps/shoppingcart/tests/test_views.py index 879e98f780..cafe402ec2 100644 --- a/lms/djangoapps/shoppingcart/tests/test_views.py +++ b/lms/djangoapps/shoppingcart/tests/test_views.py @@ -369,29 +369,35 @@ class CSVReportViewsTest(ModuleStoreTestCase): def test_report_csv_itemized(self): report_type = 'itemized_purchase_report' + start_date = '1970-01-01' + end_date = '2100-01-01' PaidCourseRegistration.add_to_order(self.cart, self.course_id) self.cart.purchase() self.login_user() self.add_to_download_group(self.user) - response = self.client.post(reverse('payment_csv_report'), {'start_date': '1970-01-01', - 'end_date': '2100-01-01', + response = self.client.post(reverse('payment_csv_report'), {'start_date': start_date, + 'end_date': end_date, 'requested_report': report_type}) self.assertEqual(response['Content-Type'], 'text/csv') - report = initialize_report(report_type) + report = initialize_report(report_type, start_date, end_date) self.assertIn(",".join(report.header()), response.content) self.assertIn(self.CORRECT_CSV_NO_DATE_ITEMIZED_PURCHASE, response.content) def test_report_csv_university_revenue_share(self): report_type = 'university_revenue_share' + start_date = '1970-01-01' + end_date = '2100-01-01' + start_letter = 'A' + end_letter = 'Z' self.login_user() self.add_to_download_group(self.user) - response = self.client.post(reverse('payment_csv_report'), {'start_date': '1970-01-01', - 'end_date': '2100-01-01', - 'start_letter': 'A', - 'end_letter': 'Z', + response = self.client.post(reverse('payment_csv_report'), {'start_date': start_date, + 'end_date': end_date, + 'start_letter': start_letter, + 'end_letter': end_letter, 'requested_report': report_type}) self.assertEqual(response['Content-Type'], 'text/csv') - report = initialize_report(report_type) + report = initialize_report(report_type, start_date, end_date, start_letter, end_letter) self.assertIn(",".join(report.header()), response.content) # TODO add another test here diff --git a/lms/djangoapps/shoppingcart/urls.py b/lms/djangoapps/shoppingcart/urls.py index ba4119aef7..f37e72be8b 100644 --- a/lms/djangoapps/shoppingcart/urls.py +++ b/lms/djangoapps/shoppingcart/urls.py @@ -4,6 +4,7 @@ from django.conf import settings urlpatterns = patterns('shoppingcart.views', # nopep8 url(r'^postpay_callback/$', 'postpay_callback'), # Both the ~accept and ~reject callback pages are handled here url(r'^receipt/(?P[0-9]*)/$', 'show_receipt'), + url(r'^csv_report/$', 'csv_report', name='payment_csv_report'), ) if settings.FEATURES['ENABLE_SHOPPING_CART']: @@ -13,7 +14,6 @@ if settings.FEATURES['ENABLE_SHOPPING_CART']: url(r'^clear/$', 'clear_cart'), url(r'^remove_item/$', 'remove_item'), url(r'^add/course/(?P[^/]+/[^/]+/[^/]+)/$', 'add_course_to_cart', name='add_course_to_cart'), - url(r'^csv_report/$', 'csv_report', name='payment_csv_report'), ) if settings.FEATURES.get('ENABLE_PAYMENT_FAKE'): diff --git a/lms/djangoapps/shoppingcart/views.py b/lms/djangoapps/shoppingcart/views.py index 4a5a728bae..3dbf7d0378 100644 --- a/lms/djangoapps/shoppingcart/views.py +++ b/lms/djangoapps/shoppingcart/views.py @@ -29,13 +29,13 @@ REPORT_TYPES = [ ] -def initialize_report(report_type): +def initialize_report(report_type, start_date, end_date, start_letter=None, end_letter=None): """ Creates the appropriate type of Report object based on the string report_type. """ for item in REPORT_TYPES: if report_type in item: - return item[1]() + return item[1](start_date, end_date, start_letter, end_letter) raise ReportTypeDoesNotExistException @require_POST @@ -193,32 +193,31 @@ def csv_report(request): """ Downloads csv reporting of orderitems """ - if not _can_download_report(request.user): return HttpResponseForbidden(_('You do not have permission to view this page.')) # TODO temp filler for start letter, end letter if request.method == 'POST': - start_str = request.POST.get('start_date', '') - end_str = request.POST.get('end_date', '') + start_date = request.POST.get('start_date', '') + end_date = request.POST.get('end_date', '') start_letter = request.POST.get('start_letter', '') end_letter = request.POST.get('end_letter', '') report_type = request.POST.get('requested_report', '') try: - start_date = _get_date_from_str(start_str) + datetime.timedelta(days=0) - end_date = _get_date_from_str(end_str) + datetime.timedelta(days=1) + start_date = _get_date_from_str(start_date) + datetime.timedelta(days=0) + end_date = _get_date_from_str(end_date) + datetime.timedelta(days=1) except ValueError: # Error case: there was a badly formatted user-input date string - return _render_report_form(start_str, end_str, start_letter, end_letter, report_type, date_fmt_error=True) + return _render_report_form(start_date, end_date, start_letter, end_letter, report_type, date_fmt_error=True) - report = initialize_report(report_type) - items = report.rows(start_date, end_date, start_letter, end_letter) + report = initialize_report(report_type, start_date, end_date, start_letter, end_letter) + items = report.rows() response = HttpResponse(mimetype='text/csv') filename = "purchases_report_{}.csv".format(datetime.datetime.now(pytz.UTC).strftime("%Y-%m-%d-%H-%M-%S")) response['Content-Disposition'] = 'attachment; filename="{}"'.format(filename) - report.write_csv(response, start_date, end_date, start_letter, end_letter) + report.write_csv(response) return response elif request.method == 'GET': From a77647609856ee370bd1a83eaccdbd9e8d94ab91 Mon Sep 17 00:00:00 2001 From: Julia Hansbrough Date: Mon, 13 Jan 2014 16:39:47 +0000 Subject: [PATCH 12/14] Response to CR 1-13 --- lms/djangoapps/shoppingcart/reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/djangoapps/shoppingcart/reports.py b/lms/djangoapps/shoppingcart/reports.py index 1c46ddb7c9..4a8501eaf7 100644 --- a/lms/djangoapps/shoppingcart/reports.py +++ b/lms/djangoapps/shoppingcart/reports.py @@ -275,6 +275,6 @@ def course_ids_between(start_word, end_word): valid_courses = [] for course in modulestore().get_courses(): - if (start_word.lower() <= course.id.lower() <= course.id.lower()) and (get_course_by_id(course.id) is not None): + if (start_word.lower() <= course.id.lower() <= end_word.lower()) and (get_course_by_id(course.id) is not None): valid_courses.append(course.id) return valid_courses From ea0ae111719c1427a5c5695dd4f2eb918a4362c7 Mon Sep 17 00:00:00 2001 From: Julia Hansbrough Date: Tue, 14 Jan 2014 20:27:23 +0000 Subject: [PATCH 13/14] Response to CR 1-14 --- common/djangoapps/course_modes/models.py | 9 ++ common/djangoapps/student/models.py | 5 +- common/djangoapps/util/query.py | 5 + lms/djangoapps/shoppingcart/models.py | 4 +- lms/djangoapps/shoppingcart/reports.py | 19 ++-- .../shoppingcart/tests/test_models.py | 99 +---------------- .../shoppingcart/tests/test_reports.py | 104 +++++++++++++++++- .../shoppingcart/tests/test_views.py | 1 - lms/envs/aws.py | 2 + lms/envs/common.py | 2 + lms/envs/dev.py | 2 + .../shoppingcart/download_report.html | 21 +++- 12 files changed, 152 insertions(+), 121 deletions(-) diff --git a/common/djangoapps/course_modes/models.py b/common/djangoapps/course_modes/models.py index 9ed64e048f..ae9fac5ed4 100644 --- a/common/djangoapps/course_modes/models.py +++ b/common/djangoapps/course_modes/models.py @@ -97,6 +97,15 @@ class CourseMode(models.Model): @classmethod def min_course_price_for_verified_for_currency(cls, course_id, currency): + """ + Returns the minimum price of the course int he appropriate currency over all the + course's *verified*, non-expired modes. + + Assuming all verified courses have a minimum price of >0, this value should always + be >0. + + If no verified mode is found, 0 is returned. + """ modes = cls.modes_for_course(course_id) for mode in modes: if (mode.currency == currency) and (mode.slug == 'verified'): diff --git a/common/djangoapps/student/models.py b/common/djangoapps/student/models.py index 6ec263fed1..a074d61cb9 100644 --- a/common/djangoapps/student/models.py +++ b/common/djangoapps/student/models.py @@ -23,7 +23,7 @@ from django.conf import settings from django.contrib.auth.models import User from django.contrib.auth.signals import user_logged_in, user_logged_out from django.db import models, IntegrityError -from django.db.models import Sum, Count +from django.db.models import Count from django.db.models.signals import post_save from django.dispatch import receiver, Signal import django.dispatch @@ -35,6 +35,7 @@ from eventtracking import tracker from course_modes.models import CourseMode import lms.lib.comment_client as cc +from util.query import use_read_replica_if_available unenroll_done = Signal(providing_args=["course_enrollment"]) log = logging.getLogger(__name__) @@ -588,7 +589,7 @@ class CourseEnrollment(models.Model): enrollment count for each individual mode. """ # Unfortunately, Django's "group by"-style queries look super-awkward - query = cls.objects.filter(course_id=course_id, is_active=True).values('mode').order_by().annotate(Count('mode')) + query = use_read_replica_if_available(cls.objects.filter(course_id=course_id, is_active=True).values('mode').order_by().annotate(Count('mode'))) total = 0 d = defaultdict(int) for item in query: diff --git a/common/djangoapps/util/query.py b/common/djangoapps/util/query.py index cff97078a3..6800bc45ab 100644 --- a/common/djangoapps/util/query.py +++ b/common/djangoapps/util/query.py @@ -1,4 +1,9 @@ +""" Utility functions related to database queries """ from django.conf import settings + def use_read_replica_if_available(queryset): + """ + If there is a database called 'read_replica', use that database for the queryset. + """ return queryset.using("read_replica") if "read_replica" in settings.DATABASES else queryset \ No newline at end of file diff --git a/lms/djangoapps/shoppingcart/models.py b/lms/djangoapps/shoppingcart/models.py index d0b20e547f..9ca6eeada2 100644 --- a/lms/djangoapps/shoppingcart/models.py +++ b/lms/djangoapps/shoppingcart/models.py @@ -1,3 +1,5 @@ +""" Models for the shopping cart and assorted purchase types """ + from collections import namedtuple from datetime import datetime from decimal import Decimal @@ -581,7 +583,7 @@ class CertificateItem(OrderItem): """ Returns a Decimal indicating the total sum of field_to_aggregate for all verified certificates with a particular status. - Sample usages: + Sample usages: - status 'refunded' and field_to_aggregate 'unit_cost' will give the total amount of money refunded for course_id - status 'purchased' and field_to_aggregate 'service_fees' gives the sum of all service fees for purchased certificates etc diff --git a/lms/djangoapps/shoppingcart/reports.py b/lms/djangoapps/shoppingcart/reports.py index 4a8501eaf7..1eaee921f2 100644 --- a/lms/djangoapps/shoppingcart/reports.py +++ b/lms/djangoapps/shoppingcart/reports.py @@ -1,9 +1,8 @@ +""" Objects and functions related to generating CSV reports """ + from decimal import Decimal import unicodecsv -import logging -from django.db import models -from django.conf import settings from django.utils.translation import ugettext as _ from courseware.courses import get_course_by_id @@ -11,7 +10,6 @@ from course_modes.models import CourseMode from shoppingcart.models import CertificateItem, OrderItem from student.models import CourseEnrollment from util.query import use_read_replica_if_available -from xmodule.error_module import ErrorDescriptor from xmodule.modulestore.django import modulestore @@ -191,10 +189,10 @@ class CertificateStatusReport(Report): yield [ university, course, - "", - "", - "", - "", + course_announce_date, + course_reg_start_date, + course_reg_close_date, + registration_period, total_enrolled, audit_enrolled, honor_enrolled, @@ -267,10 +265,11 @@ class UniversityRevenueShareReport(Report): _("Total Amount of Refunds"), ] + def course_ids_between(start_word, end_word): - """ + """ Returns a list of all valid course_ids that fall alphabetically between start_word and end_word. - These comparisons are unicode-safe. + These comparisons are unicode-safe. """ valid_courses = [] diff --git a/lms/djangoapps/shoppingcart/tests/test_models.py b/lms/djangoapps/shoppingcart/tests/test_models.py index ea5bb4d34d..c130ecf632 100644 --- a/lms/djangoapps/shoppingcart/tests/test_models.py +++ b/lms/djangoapps/shoppingcart/tests/test_models.py @@ -18,12 +18,10 @@ from xmodule.modulestore.tests.factories import CourseFactory from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE from shoppingcart.models import (Order, OrderItem, CertificateItem, InvalidCartItem, PaidCourseRegistration, OrderItemSubclassPK, PaidCourseRegistrationAnnotation) -from shoppingcart.views import initialize_report, REPORT_TYPES -from shoppingcart.reports import ItemizedPurchaseReport, CertificateStatusReport, UniversityRevenueShareReport, RefundReport from student.tests.factories import UserFactory from student.models import CourseEnrollment from course_modes.models import CourseMode -from shoppingcart.exceptions import PurchasedCallbackException, ReportTypeDoesNotExistException +from shoppingcart.exceptions import PurchasedCallbackException import pytz import datetime @@ -325,101 +323,6 @@ class PaidCourseRegistrationTest(ModuleStoreTestCase): self.assertFalse(CourseEnrollment.is_enrolled(self.user, self.course_id)) -@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) -class ItemizedPurchaseReportTest(ModuleStoreTestCase): - """ - Tests for the models used to generate itemized purchase reports - """ - FIVE_MINS = datetime.timedelta(minutes=5) - TEST_ANNOTATION = u'Ba\xfc\u5305' - - def setUp(self): - self.user = UserFactory.create() - self.course_id = "MITx/999/Robot_Super_Course" - self.cost = 40 - self.course = CourseFactory.create(org='MITx', number='999', display_name=u'Robot Super Course') - course_mode = CourseMode(course_id=self.course_id, - mode_slug="honor", - mode_display_name="honor cert", - min_price=self.cost) - course_mode.save() - course_mode2 = CourseMode(course_id=self.course_id, - mode_slug="verified", - mode_display_name="verified cert", - min_price=self.cost) - course_mode2.save() - self.annotation = PaidCourseRegistrationAnnotation(course_id=self.course_id, annotation=self.TEST_ANNOTATION) - self.annotation.save() - self.cart = Order.get_cart_for_user(self.user) - self.reg = PaidCourseRegistration.add_to_order(self.cart, self.course_id) - self.cert_item = CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') - self.cart.purchase() - self.now = datetime.datetime.now(pytz.UTC) - - paid_reg = PaidCourseRegistration.objects.get(course_id=self.course_id, user=self.user) - paid_reg.fulfilled_time = self.now - paid_reg.refund_requested_time = self.now - paid_reg.save() - - cert = CertificateItem.objects.get(course_id=self.course_id, user=self.user) - cert.fulfilled_time = self.now - cert.refund_requested_time = self.now - cert.save() - - self.CORRECT_CSV = dedent(""" - Purchase Time,Order ID,Status,Quantity,Unit Cost,Total Cost,Currency,Description,Comments - {time_str},1,purchased,1,40,40,usd,Registration for Course: Robot Super Course,Ba\xc3\xbc\xe5\x8c\x85 - {time_str},1,purchased,1,40,40,usd,"Certificate of Achievement, verified cert for course Robot Super Course", - """.format(time_str=str(self.now))) - - def test_purchased_items_btw_dates(self): - report = initialize_report("itemized_purchase_report", self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) - purchases = report.rows() - - # since there's not many purchases, just run through the generator to make sure we've got the right number - num_purchases = 0 - for item in purchases: - num_purchases += 1 - self.assertEqual(num_purchases, 2) - #self.assertIn(self.reg.orderitem_ptr, purchases) - #self.assertIn(self.cert_item.orderitem_ptr, purchases) - - report = initialize_report("itemized_purchase_report", self.now + self.FIVE_MINS, self.now + self.FIVE_MINS + self.FIVE_MINS) - no_purchases = report.rows() - - num_purchases = 0 - for item in no_purchases: - num_purchases +=1 - self.assertEqual(num_purchases, 0) - - def test_purchased_csv(self): - """ - Tests that a generated purchase report CSV is as we expect - """ - report = initialize_report("itemized_purchase_report", self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) - csv_file = StringIO.StringIO() - report.write_csv(csv_file) - csv = csv_file.getvalue() - csv_file.close() - # Using excel mode csv, which automatically ends lines with \r\n, so need to convert to \n - self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_CSV.strip()) - - def test_csv_report_no_annotation(self): - """ - Fill in gap in test coverage. csv_report_comments for PaidCourseRegistration instance with no - matching annotation - """ - # delete the matching annotation - self.annotation.delete() - self.assertEqual(u"", self.reg.csv_report_comments) - - def test_paidcourseregistrationannotation_unicode(self): - """ - Fill in gap in test coverage. __unicode__ method of PaidCourseRegistrationAnnotation - """ - self.assertEqual(unicode(self.annotation), u'{} : {}'.format(self.course_id, self.TEST_ANNOTATION)) - - @override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) class CertificateItemTest(ModuleStoreTestCase): """ diff --git a/lms/djangoapps/shoppingcart/tests/test_reports.py b/lms/djangoapps/shoppingcart/tests/test_reports.py index a6f9b0aa8a..206aa393d2 100644 --- a/lms/djangoapps/shoppingcart/tests/test_reports.py +++ b/lms/djangoapps/shoppingcart/tests/test_reports.py @@ -13,8 +13,7 @@ from django.test.utils import override_settings from course_modes.models import CourseMode from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE -from shoppingcart.models import (Order, CertificateItem) -from shoppingcart.reports import ItemizedPurchaseReport, CertificateStatusReport, UniversityRevenueShareReport, RefundReport +from shoppingcart.models import (Order, CertificateItem, PaidCourseRegistration, PaidCourseRegistrationAnnotation) from shoppingcart.views import initialize_report, REPORT_TYPES from student.tests.factories import UserFactory from student.models import CourseEnrollment @@ -56,11 +55,11 @@ class ReportTypeTests(ModuleStoreTestCase): self.honor_user.profile.save() self.first_refund_user = UserFactory.create() - self.first_refund_user.profile.name = "King Bowser" + self.first_refund_user.profile.name = u"King Bowsér" self.first_refund_user.profile.save() self.second_refund_user = UserFactory.create() - self.second_refund_user.profile.name = "Susan Smith" + self.second_refund_user.profile.name = u"Súsan Smith" self.second_refund_user.profile.save() # Two are verified, three are audit, one honor @@ -125,8 +124,8 @@ class ReportTypeTests(ModuleStoreTestCase): self.CORRECT_REFUND_REPORT_CSV = dedent(""" Order Number,Customer Name,Date of Original Transaction,Date of Refund,Amount of Refund,Service Fees (if any) - 3,King Bowser,{time_str},{time_str},40,0 - 4,Susan Smith,{time_str},{time_str},40,0 + 3,King Bowsér,{time_str},{time_str},40,0 + 4,Súsan Smith,{time_str},{time_str},40,0 """.format(time_str=str(self.test_time))) self.CORRECT_CERT_STATUS_CSV = dedent(""" @@ -177,3 +176,96 @@ class ReportTypeTests(ModuleStoreTestCase): report.write_csv(csv_file) csv = csv_file.getvalue() self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_UNI_REVENUE_SHARE_CSV.strip()) + + +@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) +class ItemizedPurchaseReportTest(ModuleStoreTestCase): + """ + Tests for the models used to generate itemized purchase reports + """ + FIVE_MINS = datetime.timedelta(minutes=5) + TEST_ANNOTATION = u'Ba\xfc\u5305' + + def setUp(self): + self.user = UserFactory.create() + self.course_id = "MITx/999/Robot_Super_Course" + self.cost = 40 + self.course = CourseFactory.create(org='MITx', number='999', display_name=u'Robot Super Course') + course_mode = CourseMode(course_id=self.course_id, + mode_slug="honor", + mode_display_name="honor cert", + min_price=self.cost) + course_mode.save() + course_mode2 = CourseMode(course_id=self.course_id, + mode_slug="verified", + mode_display_name="verified cert", + min_price=self.cost) + course_mode2.save() + self.annotation = PaidCourseRegistrationAnnotation(course_id=self.course_id, annotation=self.TEST_ANNOTATION) + self.annotation.save() + self.cart = Order.get_cart_for_user(self.user) + self.reg = PaidCourseRegistration.add_to_order(self.cart, self.course_id) + self.cert_item = CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') + self.cart.purchase() + self.now = datetime.datetime.now(pytz.UTC) + + paid_reg = PaidCourseRegistration.objects.get(course_id=self.course_id, user=self.user) + paid_reg.fulfilled_time = self.now + paid_reg.refund_requested_time = self.now + paid_reg.save() + + cert = CertificateItem.objects.get(course_id=self.course_id, user=self.user) + cert.fulfilled_time = self.now + cert.refund_requested_time = self.now + cert.save() + + self.CORRECT_CSV = dedent(""" + Purchase Time,Order ID,Status,Quantity,Unit Cost,Total Cost,Currency,Description,Comments + {time_str},1,purchased,1,40,40,usd,Registration for Course: Robot Super Course,Ba\xc3\xbc\xe5\x8c\x85 + {time_str},1,purchased,1,40,40,usd,"Certificate of Achievement, verified cert for course Robot Super Course", + """.format(time_str=str(self.now))) + + def test_purchased_items_btw_dates(self): + report = initialize_report("itemized_purchase_report", self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + purchases = report.rows() + + # since there's not many purchases, just run through the generator to make sure we've got the right number + num_purchases = 0 + for item in purchases: + num_purchases += 1 + self.assertEqual(num_purchases, 2) + + report = initialize_report("itemized_purchase_report", self.now + self.FIVE_MINS, self.now + self.FIVE_MINS + self.FIVE_MINS) + no_purchases = report.rows() + + num_purchases = 0 + for item in no_purchases: + num_purchases += 1 + self.assertEqual(num_purchases, 0) + + def test_purchased_csv(self): + """ + Tests that a generated purchase report CSV is as we expect + """ + report = initialize_report("itemized_purchase_report", self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + csv_file = StringIO.StringIO() + report.write_csv(csv_file) + csv = csv_file.getvalue() + csv_file.close() + # Using excel mode csv, which automatically ends lines with \r\n, so need to convert to \n + self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_CSV.strip()) + + def test_csv_report_no_annotation(self): + """ + Fill in gap in test coverage. csv_report_comments for PaidCourseRegistration instance with no + matching annotation + """ + # delete the matching annotation + self.annotation.delete() + self.assertEqual(u"", self.reg.csv_report_comments) + + def test_paidcourseregistrationannotation_unicode(self): + """ + Fill in gap in test coverage. __unicode__ method of PaidCourseRegistrationAnnotation + """ + self.assertEqual(unicode(self.annotation), u'{} : {}'.format(self.course_id, self.TEST_ANNOTATION)) diff --git a/lms/djangoapps/shoppingcart/tests/test_views.py b/lms/djangoapps/shoppingcart/tests/test_views.py index cafe402ec2..519dfe90be 100644 --- a/lms/djangoapps/shoppingcart/tests/test_views.py +++ b/lms/djangoapps/shoppingcart/tests/test_views.py @@ -21,7 +21,6 @@ from course_modes.models import CourseMode from edxmako.shortcuts import render_to_response from shoppingcart.processors import render_purchase_form_html from mock import patch, Mock, sentinel -from shoppingcart.reports import ItemizedPurchaseReport from shoppingcart.views import initialize_report diff --git a/lms/envs/aws.py b/lms/envs/aws.py index dfa3e81394..cc47336ea6 100644 --- a/lms/envs/aws.py +++ b/lms/envs/aws.py @@ -285,6 +285,8 @@ if AWS_SECRET_ACCESS_KEY == "": AWS_STORAGE_BUCKET_NAME = AUTH_TOKENS.get('AWS_STORAGE_BUCKET_NAME', 'edxuploads') +# If there is a database called 'read_replica', you can use the use_read_replica_if_available +# function in util/query.py, which is useful for very large database reads DATABASES = AUTH_TOKENS['DATABASES'] XQUEUE_INTERFACE = AUTH_TOKENS['XQUEUE_INTERFACE'] diff --git a/lms/envs/common.py b/lms/envs/common.py index 8512d8f160..d65e0715ad 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -202,6 +202,8 @@ FEATURES = { # Give course staff unrestricted access to grade downloads (if set to False, # only edX superusers can perform the downloads) 'ALLOW_COURSE_STAFF_GRADE_DOWNLOADS': False, + + 'ENABLED_PAYMENT_REPORTS': [ "refund_report", "itemized_purchase_report", "university_revenue_share", "certificate_status"], } # Used for A/B testing diff --git a/lms/envs/dev.py b/lms/envs/dev.py index 30053d3da3..bf861bab04 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -52,6 +52,8 @@ LOGGING = get_logger_config(ENV_ROOT / "log", dev_env=True, debug=True) +# If there is a database called 'read_replica', you can use the use_read_replica_if_available +# function in util/query.py, which is useful for very large database reads DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', diff --git a/lms/templates/shoppingcart/download_report.html b/lms/templates/shoppingcart/download_report.html index 57698f20dc..df618ff61b 100644 --- a/lms/templates/shoppingcart/download_report.html +++ b/lms/templates/shoppingcart/download_report.html @@ -1,5 +1,6 @@ <%! from django.utils.translation import ugettext as _ %> <%! from django.core.urlresolvers import reverse %> +<%! from django.conf import settings %> <%inherit file="../main.html" /> <%block name="title">${_("Download CSV Reports")} @@ -19,21 +20,35 @@
+ + %if "itemized_purchase_report" in settings.FEATURES['ENABLED_PAYMENT_REPORTS']:
- + %endif + + %if "refund_report" in settings.FEATURES['ENABLED_PAYMENT_REPORTS']: + +
+ %endif + +
-

${_("These reports are delimited alphabetically by university name. i.e., generating a report with 'Start Letter' A and 'End Letter' C will generate reports for all universities starting with A, B, and C.")}

-
+ + %if "university_revenue_share" in settings.FEATURES['ENABLED_PAYMENT_REPORTS']:
+ %endif + + %if "certificate_status" in settings.FEATURES['ENABLED_PAYMENT_REPORTS']: +
+ %endif From 1664452a146f452918f3fb725aa9ae826d7db835 Mon Sep 17 00:00:00 2001 From: Julia Hansbrough Date: Wed, 15 Jan 2014 13:02:33 +0000 Subject: [PATCH 14/14] Final iteration --- .../shoppingcart/tests/test_reports.py | 2 +- .../shoppingcart/tests/test_views.py | 1 - lms/djangoapps/shoppingcart/views.py | 2 -- lms/envs/common.py | 2 +- .../shoppingcart/download_report.html | 22 +++++++++++-------- 5 files changed, 15 insertions(+), 14 deletions(-) diff --git a/lms/djangoapps/shoppingcart/tests/test_reports.py b/lms/djangoapps/shoppingcart/tests/test_reports.py index 206aa393d2..d0dccd0acd 100644 --- a/lms/djangoapps/shoppingcart/tests/test_reports.py +++ b/lms/djangoapps/shoppingcart/tests/test_reports.py @@ -14,7 +14,7 @@ from django.test.utils import override_settings from course_modes.models import CourseMode from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE from shoppingcart.models import (Order, CertificateItem, PaidCourseRegistration, PaidCourseRegistrationAnnotation) -from shoppingcart.views import initialize_report, REPORT_TYPES +from shoppingcart.views import initialize_report from student.tests.factories import UserFactory from student.models import CourseEnrollment from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase diff --git a/lms/djangoapps/shoppingcart/tests/test_views.py b/lms/djangoapps/shoppingcart/tests/test_views.py index 519dfe90be..b18ebe6e8c 100644 --- a/lms/djangoapps/shoppingcart/tests/test_views.py +++ b/lms/djangoapps/shoppingcart/tests/test_views.py @@ -398,7 +398,6 @@ class CSVReportViewsTest(ModuleStoreTestCase): self.assertEqual(response['Content-Type'], 'text/csv') report = initialize_report(report_type, start_date, end_date, start_letter, end_letter) self.assertIn(",".join(report.header()), response.content) - # TODO add another test here class UtilFnsTest(TestCase): diff --git a/lms/djangoapps/shoppingcart/views.py b/lms/djangoapps/shoppingcart/views.py index 3dbf7d0378..fd1470588d 100644 --- a/lms/djangoapps/shoppingcart/views.py +++ b/lms/djangoapps/shoppingcart/views.py @@ -196,8 +196,6 @@ def csv_report(request): if not _can_download_report(request.user): return HttpResponseForbidden(_('You do not have permission to view this page.')) - # TODO temp filler for start letter, end letter - if request.method == 'POST': start_date = request.POST.get('start_date', '') end_date = request.POST.get('end_date', '') diff --git a/lms/envs/common.py b/lms/envs/common.py index d65e0715ad..39a7324da5 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -203,7 +203,7 @@ FEATURES = { # only edX superusers can perform the downloads) 'ALLOW_COURSE_STAFF_GRADE_DOWNLOADS': False, - 'ENABLED_PAYMENT_REPORTS': [ "refund_report", "itemized_purchase_report", "university_revenue_share", "certificate_status"], + 'ENABLED_PAYMENT_REPORTS': ["refund_report", "itemized_purchase_report", "university_revenue_share", "certificate_status"], } # Used for A/B testing diff --git a/lms/templates/shoppingcart/download_report.html b/lms/templates/shoppingcart/download_report.html index df618ff61b..584469f49d 100644 --- a/lms/templates/shoppingcart/download_report.html +++ b/lms/templates/shoppingcart/download_report.html @@ -14,6 +14,7 @@ % endif
+ %if ("itemized_purchase_report" or "refund_report") in settings.FEATURES['ENABLED_PAYMENT_REPORTS']:

${_("These reports are delimited by start and end dates.")}

@@ -22,17 +23,19 @@
%if "itemized_purchase_report" in settings.FEATURES['ENABLED_PAYMENT_REPORTS']: - -
+ +
%endif %if "refund_report" in settings.FEATURES['ENABLED_PAYMENT_REPORTS']: - -
+ +
%endif
- + %endif + + %if ("certificate_status" or "university_revenue_share") in settings.FEATURES['ENABLED_PAYMENT_REPORTS']:

${_("These reports are delimited alphabetically by university name. i.e., generating a report with 'Start Letter' A and 'End Letter' C will generate reports for all universities starting with A, B, and C.")}

@@ -42,13 +45,14 @@
%if "university_revenue_share" in settings.FEATURES['ENABLED_PAYMENT_REPORTS']: - -
+ +
%endif %if "certificate_status" in settings.FEATURES['ENABLED_PAYMENT_REPORTS']: - -
+ +
%endif + %endif