diff --git a/common/djangoapps/course_modes/models.py b/common/djangoapps/course_modes/models.py index 4cf60c0e1e..ae9fac5ed4 100644 --- a/common/djangoapps/course_modes/models.py +++ b/common/djangoapps/course_modes/models.py @@ -95,6 +95,23 @@ class CourseMode(models.Model): else: return None + @classmethod + def min_course_price_for_verified_for_currency(cls, course_id, currency): + """ + Returns the minimum price of the course int he appropriate currency over all the + course's *verified*, non-expired modes. + + Assuming all verified courses have a minimum price of >0, this value should always + be >0. + + If no verified mode is found, 0 is returned. + """ + modes = cls.modes_for_course(course_id) + for mode in modes: + if (mode.currency == currency) and (mode.slug == 'verified'): + return mode.min_price + return 0 + @classmethod def min_course_price_for_currency(cls, course_id, currency): """ diff --git a/common/djangoapps/student/models.py b/common/djangoapps/student/models.py index 935abbdd2e..a074d61cb9 100644 --- a/common/djangoapps/student/models.py +++ b/common/djangoapps/student/models.py @@ -10,31 +10,33 @@ file and check it in at the same time as your model changes. To do that, 2. ./manage.py lms schemamigration student --auto description_of_your_change 3. Add the migration file created in edx-platform/common/djangoapps/student/migrations/ """ +import crum from datetime import datetime import hashlib import json import logging +from pytz import UTC import uuid +from collections import defaultdict from django.conf import settings from django.contrib.auth.models import User from django.contrib.auth.signals import user_logged_in, user_logged_out from django.db import models, IntegrityError +from django.db.models import Count from django.db.models.signals import post_save from django.dispatch import receiver, Signal import django.dispatch from django.forms import ModelForm, forms from django.core.exceptions import ObjectDoesNotExist - -from course_modes.models import CourseMode -import lms.lib.comment_client as cc -from pytz import UTC -import crum - from track import contexts from track.views import server_track from eventtracking import tracker +from course_modes.models import CourseMode +import lms.lib.comment_client as cc +from util.query import use_read_replica_if_available + unenroll_done = Signal(providing_args=["course_enrollment"]) log = logging.getLogger(__name__) AUDIT_LOG = logging.getLogger("audit") @@ -580,6 +582,22 @@ class CourseEnrollment(models.Model): courseenrollment__is_active=True ) + @classmethod + def enrollment_counts(cls, course_id): + """ + Returns a dictionary that stores the total enrollment count for a course, as well as the + enrollment count for each individual mode. + """ + # Unfortunately, Django's "group by"-style queries look super-awkward + query = use_read_replica_if_available(cls.objects.filter(course_id=course_id, is_active=True).values('mode').order_by().annotate(Count('mode'))) + total = 0 + d = defaultdict(int) + for item in query: + d[item['mode']] = item['mode__count'] + total += item['mode__count'] + d['total'] = total + return d + def activate(self): """Makes this `CourseEnrollment` record active. Saves immediately.""" self.update_enrollment(is_active=True) diff --git a/common/djangoapps/util/query.py b/common/djangoapps/util/query.py new file mode 100644 index 0000000000..6800bc45ab --- /dev/null +++ b/common/djangoapps/util/query.py @@ -0,0 +1,9 @@ +""" Utility functions related to database queries """ +from django.conf import settings + + +def use_read_replica_if_available(queryset): + """ + If there is a database called 'read_replica', use that database for the queryset. + """ + return queryset.using("read_replica") if "read_replica" in settings.DATABASES else queryset \ No newline at end of file diff --git a/lms/djangoapps/shoppingcart/exceptions.py b/lms/djangoapps/shoppingcart/exceptions.py index a40c2e9feb..b6f826040b 100644 --- a/lms/djangoapps/shoppingcart/exceptions.py +++ b/lms/djangoapps/shoppingcart/exceptions.py @@ -26,3 +26,11 @@ class AlreadyEnrolledInCourseException(InvalidCartItem): class CourseDoesNotExistException(InvalidCartItem): pass + + +class ReportException(Exception): + pass + + +class ReportTypeDoesNotExistException(ReportException): + pass diff --git a/lms/djangoapps/shoppingcart/migrations/0007_auto__add_field_orderitem_service_fee.py b/lms/djangoapps/shoppingcart/migrations/0007_auto__add_field_orderitem_service_fee.py new file mode 100644 index 0000000000..227b8ccc0f --- /dev/null +++ b/lms/djangoapps/shoppingcart/migrations/0007_auto__add_field_orderitem_service_fee.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'OrderItem.service_fee' + db.add_column('shoppingcart_orderitem', 'service_fee', + self.gf('django.db.models.fields.DecimalField')(default=0.0, max_digits=30, decimal_places=2), + keep_default=False) + + # Adding index on 'OrderItem', fields ['status'] + db.create_index('shoppingcart_orderitem', ['status']) + + # Adding index on 'OrderItem', fields ['fulfilled_time'] + db.create_index('shoppingcart_orderitem', ['fulfilled_time']) + + # Adding index on 'OrderItem', fields ['refund_requested_time'] + db.create_index('shoppingcart_orderitem', ['refund_requested_time']) + + + def backwards(self, orm): + # Removing index on 'OrderItem', fields ['refund_requested_time'] + db.delete_index('shoppingcart_orderitem', ['refund_requested_time']) + + # Removing index on 'OrderItem', fields ['fulfilled_time'] + db.delete_index('shoppingcart_orderitem', ['fulfilled_time']) + + # Removing index on 'OrderItem', fields ['status'] + db.delete_index('shoppingcart_orderitem', ['status']) + + # Deleting field 'OrderItem.service_fee' + db.delete_column('shoppingcart_orderitem', 'service_fee') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'shoppingcart.certificateitem': { + 'Meta': {'object_name': 'CertificateItem', '_ormbases': ['shoppingcart.OrderItem']}, + 'course_enrollment': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['student.CourseEnrollment']"}), + 'course_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'mode': ('django.db.models.fields.SlugField', [], {'max_length': '50'}), + 'orderitem_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['shoppingcart.OrderItem']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'shoppingcart.order': { + 'Meta': {'object_name': 'Order'}, + 'bill_to_cardtype': ('django.db.models.fields.CharField', [], {'max_length': '32', 'blank': 'True'}), + 'bill_to_ccnum': ('django.db.models.fields.CharField', [], {'max_length': '8', 'blank': 'True'}), + 'bill_to_city': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}), + 'bill_to_country': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}), + 'bill_to_first': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}), + 'bill_to_last': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}), + 'bill_to_postalcode': ('django.db.models.fields.CharField', [], {'max_length': '16', 'blank': 'True'}), + 'bill_to_state': ('django.db.models.fields.CharField', [], {'max_length': '8', 'blank': 'True'}), + 'bill_to_street1': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}), + 'bill_to_street2': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}), + 'currency': ('django.db.models.fields.CharField', [], {'default': "'usd'", 'max_length': '8'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'processor_reply_dump': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'purchase_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'refunded_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'cart'", 'max_length': '32'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'shoppingcart.orderitem': { + 'Meta': {'object_name': 'OrderItem'}, + 'currency': ('django.db.models.fields.CharField', [], {'default': "'usd'", 'max_length': '8'}), + 'fulfilled_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'line_desc': ('django.db.models.fields.CharField', [], {'default': "'Misc. Item'", 'max_length': '1024'}), + 'order': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['shoppingcart.Order']"}), + 'qty': ('django.db.models.fields.IntegerField', [], {'default': '1'}), + 'refund_requested_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}), + 'report_comments': ('django.db.models.fields.TextField', [], {'default': "''"}), + 'service_fee': ('django.db.models.fields.DecimalField', [], {'default': '0.0', 'max_digits': '30', 'decimal_places': '2'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'cart'", 'max_length': '32', 'db_index': 'True'}), + 'unit_cost': ('django.db.models.fields.DecimalField', [], {'default': '0.0', 'max_digits': '30', 'decimal_places': '2'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'shoppingcart.paidcourseregistration': { + 'Meta': {'object_name': 'PaidCourseRegistration', '_ormbases': ['shoppingcart.OrderItem']}, + 'course_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'mode': ('django.db.models.fields.SlugField', [], {'default': "'honor'", 'max_length': '50'}), + 'orderitem_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['shoppingcart.OrderItem']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'shoppingcart.paidcourseregistrationannotation': { + 'Meta': {'object_name': 'PaidCourseRegistrationAnnotation'}, + 'annotation': ('django.db.models.fields.TextField', [], {'null': 'True'}), + 'course_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + 'student.courseenrollment': { + 'Meta': {'ordering': "('user', 'course_id')", 'unique_together': "(('user', 'course_id'),)", 'object_name': 'CourseEnrollment'}, + 'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'null': 'True', 'db_index': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'mode': ('django.db.models.fields.CharField', [], {'default': "'honor'", 'max_length': '100'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + } + } + + complete_apps = ['shoppingcart'] \ No newline at end of file diff --git a/lms/djangoapps/shoppingcart/models.py b/lms/djangoapps/shoppingcart/models.py index 7647b2c487..9ca6eeada2 100644 --- a/lms/djangoapps/shoppingcart/models.py +++ b/lms/djangoapps/shoppingcart/models.py @@ -1,13 +1,14 @@ +""" Models for the shopping cart and assorted purchase types """ + +from collections import namedtuple from datetime import datetime +from decimal import Decimal import pytz import logging import smtplib import unicodecsv -from model_utils.managers import InheritanceManager -from collections import namedtuple from boto.exception import BotoServerError # this is a super-class of SESError and catches connection errors - from django.dispatch import receiver from django.db import models from django.conf import settings @@ -16,7 +17,9 @@ from django.core.mail import send_mail from django.contrib.auth.models import User from django.utils.translation import ugettext as _ from django.db import transaction +from django.db.models import Sum from django.core.urlresolvers import reverse +from model_utils.managers import InheritanceManager from xmodule.modulestore.django import modulestore from xmodule.course_module import CourseDescriptor @@ -26,11 +29,12 @@ from course_modes.models import CourseMode from edxmako.shortcuts import render_to_string from student.views import course_from_id from student.models import CourseEnrollment, unenroll_done +from util.query import use_read_replica_if_available from verify_student.models import SoftwareSecurePhotoVerification from .exceptions import (InvalidCartItem, PurchasedCallbackException, ItemAlreadyInCartException, - AlreadyEnrolledInCourseException, CourseDoesNotExistException) + AlreadyEnrolledInCourseException, CourseDoesNotExistException, ReportException) log = logging.getLogger("shoppingcart") @@ -203,13 +207,14 @@ class OrderItem(models.Model): # this is denormalized, but convenient for SQL queries for reports, etc. user should always be = order.user user = models.ForeignKey(User, db_index=True) # this is denormalized, but convenient for SQL queries for reports, etc. status should always be = order.status - status = models.CharField(max_length=32, default='cart', choices=ORDER_STATUSES) + status = models.CharField(max_length=32, default='cart', choices=ORDER_STATUSES, db_index=True) qty = models.IntegerField(default=1) unit_cost = models.DecimalField(default=0.0, decimal_places=2, max_digits=30) line_desc = models.CharField(default="Misc. Item", max_length=1024) currency = models.CharField(default="usd", max_length=8) # lower case ISO currency codes - fulfilled_time = models.DateTimeField(null=True) - refund_requested_time = models.DateTimeField(null=True) + fulfilled_time = models.DateTimeField(null=True, db_index=True) + refund_requested_time = models.DateTimeField(null=True, db_index=True) + service_fee = models.DecimalField(default=0.0, decimal_places=2, max_digits=30) # general purpose field, not user-visible. Used for reporting report_comments = models.TextField(default="") @@ -259,66 +264,6 @@ class OrderItem(models.Model): """ return self.pk_with_subclass, set([]) - @classmethod - def purchased_items_btw_dates(cls, start_date, end_date): - """ - Returns a QuerySet of the purchased items between start_date and end_date inclusive. - """ - return cls.objects.filter( - status="purchased", - fulfilled_time__gte=start_date, - fulfilled_time__lt=end_date, - ) - - @classmethod - def csv_purchase_report_btw_dates(cls, filelike, start_date, end_date): - """ - Outputs a CSV report into "filelike" (a file-like python object, such as an actual file, an HttpRequest, - or sys.stdout) of purchased items between start_date and end_date inclusive. - Opening and closing filelike (if applicable) should be taken care of by the caller - """ - items = cls.purchased_items_btw_dates(start_date, end_date).order_by("fulfilled_time") - - writer = unicodecsv.writer(filelike, encoding="utf-8") - writer.writerow(OrderItem.csv_report_header_row()) - - for item in items: - writer.writerow(item.csv_report_row) - - @classmethod - def csv_report_header_row(cls): - """ - Returns the "header" row for a csv report of purchases - """ - return [ - "Purchase Time", - "Order ID", - "Status", - "Quantity", - "Unit Cost", - "Total Cost", - "Currency", - "Description", - "Comments" - ] - - @property - def csv_report_row(self): - """ - Returns an array which can be fed into csv.writer to write out one csv row - """ - return [ - self.fulfilled_time, - self.order_id, # pylint: disable=no-member - self.status, - self.qty, - self.unit_cost, - self.line_cost, - self.currency, - self.line_desc, - self.report_comments, - ] - @property def pk_with_subclass(self): """ @@ -625,3 +570,36 @@ class CertificateItem(OrderItem): "Please include your order number in your e-mail. " "Please do NOT include your credit card information.").format( billing_email=settings.PAYMENT_SUPPORT_EMAIL) + + @classmethod + def verified_certificates_count(cls, course_id, status): + """Return a queryset of CertificateItem for every verified enrollment in course_id with the given status.""" + return use_read_replica_if_available( + CertificateItem.objects.filter(course_id=course_id, mode='verified', status=status).count()) + + # TODO combine these three methods into one + @classmethod + def verified_certificates_monetary_field_sum(cls, course_id, status, field_to_aggregate): + """ + Returns a Decimal indicating the total sum of field_to_aggregate for all verified certificates with a particular status. + + Sample usages: + - status 'refunded' and field_to_aggregate 'unit_cost' will give the total amount of money refunded for course_id + - status 'purchased' and field_to_aggregate 'service_fees' gives the sum of all service fees for purchased certificates + etc + """ + query = use_read_replica_if_available( + CertificateItem.objects.filter(course_id=course_id, mode='verified', status=status).aggregate(Sum(field_to_aggregate)))[field_to_aggregate + '__sum'] + if query is None: + return Decimal(0.00) + else: + return query + + @classmethod + def verified_certificates_contributing_more_than_minimum(cls, course_id): + return use_read_replica_if_available( + CertificateItem.objects.filter( + course_id=course_id, + mode='verified', + status='purchased', + unit_cost__gt=(CourseMode.min_course_price_for_verified_for_currency(course_id, 'usd'))).count()) diff --git a/lms/djangoapps/shoppingcart/reports.py b/lms/djangoapps/shoppingcart/reports.py new file mode 100644 index 0000000000..1eaee921f2 --- /dev/null +++ b/lms/djangoapps/shoppingcart/reports.py @@ -0,0 +1,279 @@ +""" Objects and functions related to generating CSV reports """ + +from decimal import Decimal +import unicodecsv + +from django.utils.translation import ugettext as _ + +from courseware.courses import get_course_by_id +from course_modes.models import CourseMode +from shoppingcart.models import CertificateItem, OrderItem +from student.models import CourseEnrollment +from util.query import use_read_replica_if_available +from xmodule.modulestore.django import modulestore + + +class Report(object): + """ + Base class for making CSV reports related to revenue, enrollments, etc + + To make a different type of report, write a new subclass that implements + the methods rows and header. + """ + def __init__(self, start_date, end_date, start_word=None, end_word=None): + self.start_date = start_date + self.end_date = end_date + self.start_word = start_word + self.end_word = end_word + + def rows(self): + """ + Performs database queries necessary for the report and eturns an generator of + lists, in which each list is a separate row of the report. + + Arguments are start_date (datetime), end_date (datetime), start_word (str), + and end_word (str). Date comparisons are start_date <= [date of item] < end_date. + """ + raise NotImplementedError + + def header(self): + """ + Returns the appropriate header based on the report type, in the form of a + list of strings. + """ + raise NotImplementedError + + def write_csv(self, filelike): + """ + Given a file object to write to and {start/end date, start/end letter} bounds, + generates a CSV report of the appropriate type. + """ + items = self.rows() + writer = unicodecsv.writer(filelike, encoding="utf-8") + writer.writerow(self.header()) + for item in items: + writer.writerow(item) + + +class RefundReport(Report): + """ + Subclass of Report, used to generate Refund Reports for finance purposes. + + For each refund between a given start_date and end_date, we find the relevant + order number, customer name, date of transaction, date of refund, and any service + fees. + """ + def rows(self): + query1 = use_read_replica_if_available( + CertificateItem.objects.select_related('user__profile').filter( + status="refunded", + refund_requested_time__gte=self.start_date, + refund_requested_time__lt=self.end_date, + ).order_by('refund_requested_time')) + query2 = use_read_replica_if_available( + CertificateItem.objects.select_related('user__profile').filter( + status="refunded", + refund_requested_time=None, + )) + + query = query1 | query2 + + for item in query: + yield [ + item.order_id, + item.user.profile.name, + item.fulfilled_time, + item.refund_requested_time, + item.line_cost, + item.service_fee, + ] + + def header(self): + return [ + _("Order Number"), + _("Customer Name"), + _("Date of Original Transaction"), + _("Date of Refund"), + _("Amount of Refund"), + _("Service Fees (if any)"), + ] + + +class ItemizedPurchaseReport(Report): + """ + Subclass of Report, used to generate itemized purchase reports. + + For all purchases (verified certificates, paid course registrations, etc) between + a given start_date and end_date, we find that purchase's time, order ID, status, + quantity, unit cost, total cost, currency, description, and related comments. + """ + def rows(self): + query = use_read_replica_if_available( + OrderItem.objects.filter( + status="purchased", + fulfilled_time__gte=self.start_date, + fulfilled_time__lt=self.end_date, + ).order_by("fulfilled_time")) + + for item in query: + yield [ + item.fulfilled_time, + item.order_id, # pylint: disable=no-member + item.status, + item.qty, + item.unit_cost, + item.line_cost, + item.currency, + item.line_desc, + item.report_comments, + ] + + def header(self): + return [ + _("Purchase Time"), + _("Order ID"), + _("Status"), + _("Quantity"), + _("Unit Cost"), + _("Total Cost"), + _("Currency"), + _("Description"), + _("Comments") + ] + + +class CertificateStatusReport(Report): + """ + Subclass of Report, used to generate Certificate Status Reports for Ed Services. + + For each course in each university whose name is within the range start_word and end_word, + inclusive, (i.e., the letter range H-J includes both Ithaca College and Harvard University), we + calculate the total enrollment, audit enrollment, honor enrollment, verified enrollment, total + gross revenue, gross revenue over the minimum, and total dollars refunded. + """ + def rows(self): + for course_id in course_ids_between(self.start_word, self.end_word): + # If the first letter of the university is between start_word and end_word, then we include + # it in the report. These comparisons are unicode-safe. + cur_course = get_course_by_id(course_id) + university = cur_course.org + course = cur_course.number + " " + cur_course.display_name_with_default # TODO add term (i.e. Fall 2013)? + counts = CourseEnrollment.enrollment_counts(course_id) + total_enrolled = counts['total'] + audit_enrolled = counts['audit'] + honor_enrolled = counts['honor'] + + if counts['verified'] == 0: + verified_enrolled = 0 + gross_rev = Decimal(0.00) + gross_rev_over_min = Decimal(0.00) + else: + verified_enrolled = counts['verified'] + gross_rev = CertificateItem.verified_certificates_monetary_field_sum(course_id, 'purchased', 'unit_cost') + gross_rev_over_min = gross_rev - (CourseMode.min_course_price_for_verified_for_currency(course_id, 'usd') * verified_enrolled) + + num_verified_over_the_minimum = CertificateItem.verified_certificates_contributing_more_than_minimum(course_id) + + # should I be worried about is_active here? + number_of_refunds = CertificateItem.verified_certificates_count(course_id, 'refunded') + if number_of_refunds == 0: + dollars_refunded = Decimal(0.00) + else: + dollars_refunded = CertificateItem.verified_certificates_monetary_field_sum(course_id, 'refunded', 'unit_cost') + + course_announce_date = "" + course_reg_start_date = "" + course_reg_close_date = "" + registration_period = "" + + yield [ + university, + course, + course_announce_date, + course_reg_start_date, + course_reg_close_date, + registration_period, + total_enrolled, + audit_enrolled, + honor_enrolled, + verified_enrolled, + gross_rev, + gross_rev_over_min, + num_verified_over_the_minimum, + number_of_refunds, + dollars_refunded + ] + + def header(self): + return [ + _("University"), + _("Course"), + _("Course Announce Date"), + _("Course Start Date"), + _("Course Registration Close Date"), + _("Course Registration Period"), + _("Total Enrolled"), + _("Audit Enrollment"), + _("Honor Code Enrollment"), + _("Verified Enrollment"), + _("Gross Revenue"), + _("Gross Revenue over the Minimum"), + _("Number of Verified Students Contributing More than the Minimum"), + _("Number of Refunds"), + _("Dollars Refunded"), + ] + + +class UniversityRevenueShareReport(Report): + """ + Subclass of Report, used to generate University Revenue Share Reports for finance purposes. + + For each course in each university whose name is within the range start_word and end_word, + inclusive, (i.e., the letter range H-J includes both Ithaca College and Harvard University), we calculate + the total revenue generated by that particular course. This includes the number of transactions, + total payments collected, service fees, number of refunds, and total amount of refunds. + """ + def rows(self): + for course_id in course_ids_between(self.start_word, self.end_word): + cur_course = get_course_by_id(course_id) + university = cur_course.org + course = cur_course.number + " " + cur_course.display_name_with_default + total_payments_collected = CertificateItem.verified_certificates_monetary_field_sum(course_id, 'purchased', 'unit_cost') + service_fees = CertificateItem.verified_certificates_monetary_field_sum(course_id, 'purchased', 'service_fee') + num_refunds = CertificateItem.verified_certificates_count(course_id, "refunded") + amount_refunds = CertificateItem.verified_certificates_monetary_field_sum(course_id, 'refunded', 'unit_cost') + num_transactions = (num_refunds * 2) + CertificateItem.verified_certificates_count(course_id, "purchased") + + yield [ + university, + course, + num_transactions, + total_payments_collected, + service_fees, + num_refunds, + amount_refunds + ] + + def header(self): + return [ + _("University"), + _("Course"), + _("Number of Transactions"), + _("Total Payments Collected"), + _("Service Fees (if any)"), + _("Number of Successful Refunds"), + _("Total Amount of Refunds"), + ] + + +def course_ids_between(start_word, end_word): + """ + Returns a list of all valid course_ids that fall alphabetically between start_word and end_word. + These comparisons are unicode-safe. + """ + + valid_courses = [] + for course in modulestore().get_courses(): + if (start_word.lower() <= course.id.lower() <= end_word.lower()) and (get_course_by_id(course.id) is not None): + valid_courses.append(course.id) + return valid_courses diff --git a/lms/djangoapps/shoppingcart/tests/test_models.py b/lms/djangoapps/shoppingcart/tests/test_models.py index 5a52ca0680..c130ecf632 100644 --- a/lms/djangoapps/shoppingcart/tests/test_models.py +++ b/lms/djangoapps/shoppingcart/tests/test_models.py @@ -323,87 +323,6 @@ class PaidCourseRegistrationTest(ModuleStoreTestCase): self.assertFalse(CourseEnrollment.is_enrolled(self.user, self.course_id)) -@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) -class PurchaseReportTest(ModuleStoreTestCase): - - FIVE_MINS = datetime.timedelta(minutes=5) - TEST_ANNOTATION = u'Ba\xfc\u5305' - - def setUp(self): - self.user = UserFactory.create() - self.course_id = "MITx/999/Robot_Super_Course" - self.cost = 40 - self.course = CourseFactory.create(org='MITx', number='999', display_name=u'Robot Super Course') - course_mode = CourseMode(course_id=self.course_id, - mode_slug="honor", - mode_display_name="honor cert", - min_price=self.cost) - course_mode.save() - course_mode2 = CourseMode(course_id=self.course_id, - mode_slug="verified", - mode_display_name="verified cert", - min_price=self.cost) - course_mode2.save() - self.annotation = PaidCourseRegistrationAnnotation(course_id=self.course_id, annotation=self.TEST_ANNOTATION) - self.annotation.save() - self.cart = Order.get_cart_for_user(self.user) - self.reg = PaidCourseRegistration.add_to_order(self.cart, self.course_id) - self.cert_item = CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') - self.cart.purchase() - self.now = datetime.datetime.now(pytz.UTC) - - def test_purchased_items_btw_dates(self): - purchases = OrderItem.purchased_items_btw_dates(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) - self.assertEqual(len(purchases), 2) - self.assertIn(self.reg.orderitem_ptr, purchases) - self.assertIn(self.cert_item.orderitem_ptr, purchases) - no_purchases = OrderItem.purchased_items_btw_dates(self.now + self.FIVE_MINS, - self.now + self.FIVE_MINS + self.FIVE_MINS) - self.assertFalse(no_purchases) - - test_time = datetime.datetime.now(pytz.UTC) - - CORRECT_CSV = dedent(""" - Purchase Time,Order ID,Status,Quantity,Unit Cost,Total Cost,Currency,Description,Comments - {time_str},1,purchased,1,40,40,usd,Registration for Course: Robot Super Course,Ba\xc3\xbc\xe5\x8c\x85 - {time_str},1,purchased,1,40,40,usd,"Certificate of Achievement, verified cert for course Robot Super Course", - """.format(time_str=str(test_time))) - - def test_purchased_csv(self): - """ - Tests that a generated purchase report CSV is as we expect - """ - # coerce the purchase times to self.test_time so that the test can match. - # It's pretty hard to patch datetime.datetime b/c it's a python built-in, which is immutable, so we - # make the times match this way - for item in OrderItem.purchased_items_btw_dates(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS): - item.fulfilled_time = self.test_time - item.save() - - # add annotation to the - csv_file = StringIO.StringIO() - OrderItem.csv_purchase_report_btw_dates(csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) - csv = csv_file.getvalue() - csv_file.close() - # Using excel mode csv, which automatically ends lines with \r\n, so need to convert to \n - self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_CSV.strip()) - - def test_csv_report_no_annotation(self): - """ - Fill in gap in test coverage. csv_report_comments for PaidCourseRegistration instance with no - matching annotation - """ - # delete the matching annotation - self.annotation.delete() - self.assertEqual(u"", self.reg.csv_report_comments) - - def test_paidcourseregistrationannotation_unicode(self): - """ - Fill in gap in test coverage. __unicode__ method of PaidCourseRegistrationAnnotation - """ - self.assertEqual(unicode(self.annotation), u'{} : {}'.format(self.course_id, self.TEST_ANNOTATION)) - - @override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) class CertificateItemTest(ModuleStoreTestCase): """ diff --git a/lms/djangoapps/shoppingcart/tests/test_reports.py b/lms/djangoapps/shoppingcart/tests/test_reports.py new file mode 100644 index 0000000000..d0dccd0acd --- /dev/null +++ b/lms/djangoapps/shoppingcart/tests/test_reports.py @@ -0,0 +1,271 @@ +# -*- coding: utf-8 -*- + +""" +Tests for the Shopping Cart Models +""" +import StringIO +from textwrap import dedent +import pytz +import datetime + +from django.conf import settings +from django.test.utils import override_settings + +from course_modes.models import CourseMode +from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE +from shoppingcart.models import (Order, CertificateItem, PaidCourseRegistration, PaidCourseRegistrationAnnotation) +from shoppingcart.views import initialize_report +from student.tests.factories import UserFactory +from student.models import CourseEnrollment +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory + + +@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) +class ReportTypeTests(ModuleStoreTestCase): + """ + Tests for the models used to generate certificate status reports + """ + FIVE_MINS = datetime.timedelta(minutes=5) + + def setUp(self): + # Need to make a *lot* of users for this one + self.first_verified_user = UserFactory.create() + self.first_verified_user.profile.name = "John Doe" + self.first_verified_user.profile.save() + + self.second_verified_user = UserFactory.create() + self.second_verified_user.profile.name = "Jane Deer" + self.second_verified_user.profile.save() + + self.first_audit_user = UserFactory.create() + self.first_audit_user.profile.name = "Joe Miller" + self.first_audit_user.profile.save() + + self.second_audit_user = UserFactory.create() + self.second_audit_user.profile.name = "Simon Blackquill" + self.second_audit_user.profile.save() + + self.third_audit_user = UserFactory.create() + self.third_audit_user.profile.name = "Super Mario" + self.third_audit_user.profile.save() + + self.honor_user = UserFactory.create() + self.honor_user.profile.name = "Princess Peach" + self.honor_user.profile.save() + + self.first_refund_user = UserFactory.create() + self.first_refund_user.profile.name = u"King Bowsér" + self.first_refund_user.profile.save() + + self.second_refund_user = UserFactory.create() + self.second_refund_user.profile.name = u"Súsan Smith" + self.second_refund_user.profile.save() + + # Two are verified, three are audit, one honor + + self.course_id = "MITx/999/Robot_Super_Course" + settings.COURSE_LISTINGS['default'] = [self.course_id] + self.cost = 40 + self.course = CourseFactory.create(org='MITx', number='999', display_name=u'Robot Super Course') + course_mode = CourseMode(course_id=self.course_id, + mode_slug="honor", + mode_display_name="honor cert", + min_price=self.cost) + course_mode.save() + + course_mode2 = CourseMode(course_id=self.course_id, + mode_slug="verified", + mode_display_name="verified cert", + min_price=self.cost) + course_mode2.save() + + # User 1 & 2 will be verified + self.cart1 = Order.get_cart_for_user(self.first_verified_user) + CertificateItem.add_to_order(self.cart1, self.course_id, self.cost, 'verified') + self.cart1.purchase() + + self.cart2 = Order.get_cart_for_user(self.second_verified_user) + CertificateItem.add_to_order(self.cart2, self.course_id, self.cost, 'verified') + self.cart2.purchase() + + # Users 3, 4, and 5 are audit + CourseEnrollment.enroll(self.first_audit_user, self.course_id, "audit") + CourseEnrollment.enroll(self.second_audit_user, self.course_id, "audit") + CourseEnrollment.enroll(self.third_audit_user, self.course_id, "audit") + + # User 6 is honor + CourseEnrollment.enroll(self.honor_user, self.course_id, "honor") + + self.now = datetime.datetime.now(pytz.UTC) + + # Users 7 & 8 are refunds + self.cart = Order.get_cart_for_user(self.first_refund_user) + CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') + self.cart.purchase() + CourseEnrollment.unenroll(self.first_refund_user, self.course_id) + + self.cart = Order.get_cart_for_user(self.second_refund_user) + CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') + self.cart.purchase(self.second_refund_user, self.course_id) + CourseEnrollment.unenroll(self.second_refund_user, self.course_id) + + self.test_time = datetime.datetime.now(pytz.UTC) + + first_refund = CertificateItem.objects.get(id=3) + first_refund.fulfilled_time = self.test_time + first_refund.refund_requested_time = self.test_time + first_refund.save() + + second_refund = CertificateItem.objects.get(id=4) + second_refund.fulfilled_time = self.test_time + second_refund.refund_requested_time = self.test_time + second_refund.save() + + self.CORRECT_REFUND_REPORT_CSV = dedent(""" + Order Number,Customer Name,Date of Original Transaction,Date of Refund,Amount of Refund,Service Fees (if any) + 3,King Bowsér,{time_str},{time_str},40,0 + 4,Súsan Smith,{time_str},{time_str},40,0 + """.format(time_str=str(self.test_time))) + + self.CORRECT_CERT_STATUS_CSV = dedent(""" + University,Course,Course Announce Date,Course Start Date,Course Registration Close Date,Course Registration Period,Total Enrolled,Audit Enrollment,Honor Code Enrollment,Verified Enrollment,Gross Revenue,Gross Revenue over the Minimum,Number of Verified Students Contributing More than the Minimum,Number of Refunds,Dollars Refunded + MITx,999 Robot Super Course,,,,,6,3,1,2,80.00,0.00,0,2,80.00 + """.format(time_str=str(self.test_time))) + + self.CORRECT_UNI_REVENUE_SHARE_CSV = dedent(""" + University,Course,Number of Transactions,Total Payments Collected,Service Fees (if any),Number of Successful Refunds,Total Amount of Refunds + MITx,999 Robot Super Course,6,80.00,0.00,2,80.00 + """.format(time_str=str(self.test_time))) + + def test_refund_report_rows(self): + report = initialize_report("refund_report", self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + refunded_certs = report.rows() + + # check that we have the right number + num_certs = 0 + for cert in refunded_certs: + num_certs += 1 + self.assertEqual(num_certs, 2) + + self.assertTrue(CertificateItem.objects.get(user=self.first_refund_user, course_id=self.course_id)) + self.assertTrue(CertificateItem.objects.get(user=self.second_refund_user, course_id=self.course_id)) + + def test_refund_report_purchased_csv(self): + """ + Tests that a generated purchase report CSV is as we expect + """ + report = initialize_report("refund_report", self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + csv_file = StringIO.StringIO() + report.write_csv(csv_file) + csv = csv_file.getvalue() + csv_file.close() + # Using excel mode csv, which automatically ends lines with \r\n, so need to convert to \n + self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_REFUND_REPORT_CSV.strip()) + + def test_basic_cert_status_csv(self): + report = initialize_report("certificate_status", self.now - self.FIVE_MINS, self.now + self.FIVE_MINS, 'A', 'Z') + csv_file = StringIO.StringIO() + report.write_csv(csv_file) + csv = csv_file.getvalue() + self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_CERT_STATUS_CSV.strip()) + + def test_basic_uni_revenue_share_csv(self): + report = initialize_report("university_revenue_share", self.now - self.FIVE_MINS, self.now + self.FIVE_MINS, 'A', 'Z') + csv_file = StringIO.StringIO() + report.write_csv(csv_file) + csv = csv_file.getvalue() + self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_UNI_REVENUE_SHARE_CSV.strip()) + + +@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) +class ItemizedPurchaseReportTest(ModuleStoreTestCase): + """ + Tests for the models used to generate itemized purchase reports + """ + FIVE_MINS = datetime.timedelta(minutes=5) + TEST_ANNOTATION = u'Ba\xfc\u5305' + + def setUp(self): + self.user = UserFactory.create() + self.course_id = "MITx/999/Robot_Super_Course" + self.cost = 40 + self.course = CourseFactory.create(org='MITx', number='999', display_name=u'Robot Super Course') + course_mode = CourseMode(course_id=self.course_id, + mode_slug="honor", + mode_display_name="honor cert", + min_price=self.cost) + course_mode.save() + course_mode2 = CourseMode(course_id=self.course_id, + mode_slug="verified", + mode_display_name="verified cert", + min_price=self.cost) + course_mode2.save() + self.annotation = PaidCourseRegistrationAnnotation(course_id=self.course_id, annotation=self.TEST_ANNOTATION) + self.annotation.save() + self.cart = Order.get_cart_for_user(self.user) + self.reg = PaidCourseRegistration.add_to_order(self.cart, self.course_id) + self.cert_item = CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified') + self.cart.purchase() + self.now = datetime.datetime.now(pytz.UTC) + + paid_reg = PaidCourseRegistration.objects.get(course_id=self.course_id, user=self.user) + paid_reg.fulfilled_time = self.now + paid_reg.refund_requested_time = self.now + paid_reg.save() + + cert = CertificateItem.objects.get(course_id=self.course_id, user=self.user) + cert.fulfilled_time = self.now + cert.refund_requested_time = self.now + cert.save() + + self.CORRECT_CSV = dedent(""" + Purchase Time,Order ID,Status,Quantity,Unit Cost,Total Cost,Currency,Description,Comments + {time_str},1,purchased,1,40,40,usd,Registration for Course: Robot Super Course,Ba\xc3\xbc\xe5\x8c\x85 + {time_str},1,purchased,1,40,40,usd,"Certificate of Achievement, verified cert for course Robot Super Course", + """.format(time_str=str(self.now))) + + def test_purchased_items_btw_dates(self): + report = initialize_report("itemized_purchase_report", self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + purchases = report.rows() + + # since there's not many purchases, just run through the generator to make sure we've got the right number + num_purchases = 0 + for item in purchases: + num_purchases += 1 + self.assertEqual(num_purchases, 2) + + report = initialize_report("itemized_purchase_report", self.now + self.FIVE_MINS, self.now + self.FIVE_MINS + self.FIVE_MINS) + no_purchases = report.rows() + + num_purchases = 0 + for item in no_purchases: + num_purchases += 1 + self.assertEqual(num_purchases, 0) + + def test_purchased_csv(self): + """ + Tests that a generated purchase report CSV is as we expect + """ + report = initialize_report("itemized_purchase_report", self.now - self.FIVE_MINS, self.now + self.FIVE_MINS) + csv_file = StringIO.StringIO() + report.write_csv(csv_file) + csv = csv_file.getvalue() + csv_file.close() + # Using excel mode csv, which automatically ends lines with \r\n, so need to convert to \n + self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_CSV.strip()) + + def test_csv_report_no_annotation(self): + """ + Fill in gap in test coverage. csv_report_comments for PaidCourseRegistration instance with no + matching annotation + """ + # delete the matching annotation + self.annotation.delete() + self.assertEqual(u"", self.reg.csv_report_comments) + + def test_paidcourseregistrationannotation_unicode(self): + """ + Fill in gap in test coverage. __unicode__ method of PaidCourseRegistrationAnnotation + """ + self.assertEqual(unicode(self.annotation), u'{} : {}'.format(self.course_id, self.TEST_ANNOTATION)) diff --git a/lms/djangoapps/shoppingcart/tests/test_views.py b/lms/djangoapps/shoppingcart/tests/test_views.py index d1ab9ab24f..b18ebe6e8c 100644 --- a/lms/djangoapps/shoppingcart/tests/test_views.py +++ b/lms/djangoapps/shoppingcart/tests/test_views.py @@ -14,13 +14,14 @@ from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE from shoppingcart.views import _can_download_report, _get_date_from_str -from shoppingcart.models import Order, CertificateItem, PaidCourseRegistration, OrderItem +from shoppingcart.models import Order, CertificateItem, PaidCourseRegistration from student.tests.factories import UserFactory from student.models import CourseEnrollment from course_modes.models import CourseMode from edxmako.shortcuts import render_to_response from shoppingcart.processors import render_purchase_form_html from mock import patch, Mock, sentinel +from shoppingcart.views import initialize_report def mock_render_purchase_form_html(*args, **kwargs): @@ -304,6 +305,11 @@ class CSVReportViewsTest(ModuleStoreTestCase): mode_display_name="honor cert", min_price=self.cost) self.course_mode.save() + self.course_mode2 = CourseMode(course_id=self.course_id, + mode_slug="verified", + mode_display_name="verified cert", + min_price=self.cost) + self.course_mode2.save() self.verified_course_id = 'org/test/Test_Course' CourseFactory.create(org='org', number='test', run='course1', display_name='Test Course') self.cart = Order.get_cart_for_user(self.user) @@ -343,13 +349,13 @@ class CSVReportViewsTest(ModuleStoreTestCase): self.assertEqual(template, 'shoppingcart/download_report.html') self.assertFalse(context['total_count_error']) self.assertFalse(context['date_fmt_error']) - self.assertIn(_("Download Purchase Report"), response.content) + self.assertIn(_("Download CSV Reports"), response.content) @patch('shoppingcart.views.render_to_response', render_mock) def test_report_csv_bad_date(self): self.login_user() self.add_to_download_group(self.user) - response = self.client.post(reverse('payment_csv_report'), {'start_date': 'BAD', 'end_date': 'BAD'}) + response = self.client.post(reverse('payment_csv_report'), {'start_date': 'BAD', 'end_date': 'BAD', 'requested_report': 'itemized_purchase_report'}) ((template, context), unused_kwargs) = render_mock.call_args self.assertEqual(template, 'shoppingcart/download_report.html') @@ -358,36 +364,40 @@ class CSVReportViewsTest(ModuleStoreTestCase): self.assertIn(_("There was an error in your date input. It should be formatted as YYYY-MM-DD"), response.content) - @patch('shoppingcart.views.render_to_response', render_mock) - @override_settings(PAYMENT_REPORT_MAX_ITEMS=0) - def test_report_csv_too_long(self): + CORRECT_CSV_NO_DATE_ITEMIZED_PURCHASE = ",1,purchased,1,40,40,usd,Registration for Course: Robot Super Course," + + def test_report_csv_itemized(self): + report_type = 'itemized_purchase_report' + start_date = '1970-01-01' + end_date = '2100-01-01' PaidCourseRegistration.add_to_order(self.cart, self.course_id) self.cart.purchase() self.login_user() self.add_to_download_group(self.user) - response = self.client.post(reverse('payment_csv_report'), {'start_date': '1970-01-01', - 'end_date': '2100-01-01'}) - - ((template, context), unused_kwargs) = render_mock.call_args - self.assertEqual(template, 'shoppingcart/download_report.html') - self.assertTrue(context['total_count_error']) - self.assertFalse(context['date_fmt_error']) - self.assertIn(_("There are too many results in your report.") + " (>0)", response.content) - - # just going to ignored the date in this test, since we already deal with date testing - # in test_models.py - CORRECT_CSV_NO_DATE = ",1,purchased,1,40,40,usd,Registration for Course: Robot Super Course," - - def test_report_csv(self): - PaidCourseRegistration.add_to_order(self.cart, self.course_id) - self.cart.purchase() - self.login_user() - self.add_to_download_group(self.user) - response = self.client.post(reverse('payment_csv_report'), {'start_date': '1970-01-01', - 'end_date': '2100-01-01'}) + response = self.client.post(reverse('payment_csv_report'), {'start_date': start_date, + 'end_date': end_date, + 'requested_report': report_type}) self.assertEqual(response['Content-Type'], 'text/csv') - self.assertIn(",".join(OrderItem.csv_report_header_row()), response.content) - self.assertIn(self.CORRECT_CSV_NO_DATE, response.content) + report = initialize_report(report_type, start_date, end_date) + self.assertIn(",".join(report.header()), response.content) + self.assertIn(self.CORRECT_CSV_NO_DATE_ITEMIZED_PURCHASE, response.content) + + def test_report_csv_university_revenue_share(self): + report_type = 'university_revenue_share' + start_date = '1970-01-01' + end_date = '2100-01-01' + start_letter = 'A' + end_letter = 'Z' + self.login_user() + self.add_to_download_group(self.user) + response = self.client.post(reverse('payment_csv_report'), {'start_date': start_date, + 'end_date': end_date, + 'start_letter': start_letter, + 'end_letter': end_letter, + 'requested_report': report_type}) + self.assertEqual(response['Content-Type'], 'text/csv') + report = initialize_report(report_type, start_date, end_date, start_letter, end_letter) + self.assertIn(",".join(report.header()), response.content) class UtilFnsTest(TestCase): diff --git a/lms/djangoapps/shoppingcart/urls.py b/lms/djangoapps/shoppingcart/urls.py index b9797e9a5b..f37e72be8b 100644 --- a/lms/djangoapps/shoppingcart/urls.py +++ b/lms/djangoapps/shoppingcart/urls.py @@ -4,7 +4,9 @@ from django.conf import settings urlpatterns = patterns('shoppingcart.views', # nopep8 url(r'^postpay_callback/$', 'postpay_callback'), # Both the ~accept and ~reject callback pages are handled here url(r'^receipt/(?P[0-9]*)/$', 'show_receipt'), + url(r'^csv_report/$', 'csv_report', name='payment_csv_report'), ) + if settings.FEATURES['ENABLE_SHOPPING_CART']: urlpatterns += patterns( 'shoppingcart.views', @@ -12,12 +14,11 @@ if settings.FEATURES['ENABLE_SHOPPING_CART']: url(r'^clear/$', 'clear_cart'), url(r'^remove_item/$', 'remove_item'), url(r'^add/course/(?P[^/]+/[^/]+/[^/]+)/$', 'add_course_to_cart', name='add_course_to_cart'), - url(r'^csv_report/$', 'csv_report', name='payment_csv_report'), ) if settings.FEATURES.get('ENABLE_PAYMENT_FAKE'): from shoppingcart.tests.payment_fake import PaymentFakeView urlpatterns += patterns( 'shoppingcart.tests.payment_fake', - url(r'^payment_fake', PaymentFakeView.as_view()) + url(r'^payment_fake', PaymentFakeView.as_view()), ) diff --git a/lms/djangoapps/shoppingcart/views.py b/lms/djangoapps/shoppingcart/views.py index 22bccbf28c..fd1470588d 100644 --- a/lms/djangoapps/shoppingcart/views.py +++ b/lms/djangoapps/shoppingcart/views.py @@ -11,15 +11,32 @@ from django.core.urlresolvers import reverse from django.views.decorators.csrf import csrf_exempt from django.contrib.auth.decorators import login_required from edxmako.shortcuts import render_to_response -from .models import Order, PaidCourseRegistration, OrderItem +from shoppingcart.reports import RefundReport, ItemizedPurchaseReport, UniversityRevenueShareReport, CertificateStatusReport from student.models import CourseEnrollment +from .exceptions import ItemAlreadyInCartException, AlreadyEnrolledInCourseException, CourseDoesNotExistException, ReportTypeDoesNotExistException +from .models import Order, PaidCourseRegistration, OrderItem from .processors import process_postpay_callback, render_purchase_form_html -from .exceptions import ItemAlreadyInCartException, AlreadyEnrolledInCourseException, CourseDoesNotExistException log = logging.getLogger("shoppingcart") EVENT_NAME_USER_UPGRADED = 'edx.course.enrollment.upgrade.succeeded' +REPORT_TYPES = [ + ("refund_report", RefundReport), + ("itemized_purchase_report", ItemizedPurchaseReport), + ("university_revenue_share", UniversityRevenueShareReport), + ("certificate_status", CertificateStatusReport), +] + + +def initialize_report(report_type, start_date, end_date, start_letter=None, end_letter=None): + """ + Creates the appropriate type of Report object based on the string report_type. + """ + for item in REPORT_TYPES: + if report_type in item: + return item[1](start_date, end_date, start_letter, end_letter) + raise ReportTypeDoesNotExistException @require_POST def add_course_to_cart(request, course_id): @@ -95,7 +112,6 @@ def postpay_callback(request): return render_to_response('shoppingcart/error.html', {'order': result['order'], 'error_html': result['error_html']}) - @login_required def show_receipt(request, ordernum): """ @@ -156,7 +172,7 @@ def _get_date_from_str(date_input): return datetime.datetime.strptime(date_input.strip(), "%Y-%m-%d").replace(tzinfo=pytz.UTC) -def _render_report_form(start_str, end_str, total_count_error=False, date_fmt_error=False): +def _render_report_form(start_str, end_str, start_letter, end_letter, report_type, total_count_error=False, date_fmt_error=False): """ Helper function that renders the purchase form. Reduces repetition """ @@ -165,6 +181,9 @@ def _render_report_form(start_str, end_str, total_count_error=False, date_fmt_er 'date_fmt_error': date_fmt_error, 'start_date': start_str, 'end_date': end_str, + 'start_letter': start_letter, + 'end_letter': end_letter, + 'requested_report': report_type, } return render_to_response('shoppingcart/download_report.html', context) @@ -178,30 +197,33 @@ def csv_report(request): return HttpResponseForbidden(_('You do not have permission to view this page.')) if request.method == 'POST': - start_str = request.POST.get('start_date', '') - end_str = request.POST.get('end_date', '') + start_date = request.POST.get('start_date', '') + end_date = request.POST.get('end_date', '') + start_letter = request.POST.get('start_letter', '') + end_letter = request.POST.get('end_letter', '') + report_type = request.POST.get('requested_report', '') try: - start_date = _get_date_from_str(start_str) - end_date = _get_date_from_str(end_str) + datetime.timedelta(days=1) + start_date = _get_date_from_str(start_date) + datetime.timedelta(days=0) + end_date = _get_date_from_str(end_date) + datetime.timedelta(days=1) except ValueError: # Error case: there was a badly formatted user-input date string - return _render_report_form(start_str, end_str, date_fmt_error=True) + return _render_report_form(start_date, end_date, start_letter, end_letter, report_type, date_fmt_error=True) - items = OrderItem.purchased_items_btw_dates(start_date, end_date) - if items.count() > settings.PAYMENT_REPORT_MAX_ITEMS: - # Error case: too many items would be generated in the report and we're at risk of timeout - return _render_report_form(start_str, end_str, total_count_error=True) + report = initialize_report(report_type, start_date, end_date, start_letter, end_letter) + items = report.rows() response = HttpResponse(mimetype='text/csv') filename = "purchases_report_{}.csv".format(datetime.datetime.now(pytz.UTC).strftime("%Y-%m-%d-%H-%M-%S")) response['Content-Disposition'] = 'attachment; filename="{}"'.format(filename) - OrderItem.csv_purchase_report_btw_dates(response, start_date, end_date) + report.write_csv(response) return response elif request.method == 'GET': end_date = datetime.datetime.now(pytz.UTC) start_date = end_date - datetime.timedelta(days=30) - return _render_report_form(start_date.strftime("%Y-%m-%d"), end_date.strftime("%Y-%m-%d")) + start_letter = "" + end_letter = "" + return _render_report_form(start_date.strftime("%Y-%m-%d"), end_date.strftime("%Y-%m-%d"), start_letter, end_letter, report_type="") else: return HttpResponseBadRequest("HTTP Method Not Supported") diff --git a/lms/envs/aws.py b/lms/envs/aws.py index ad6f9463d8..a2f2633baa 100644 --- a/lms/envs/aws.py +++ b/lms/envs/aws.py @@ -170,7 +170,6 @@ PAID_COURSE_REGISTRATION_CURRENCY = ENV_TOKENS.get('PAID_COURSE_REGISTRATION_CUR # Payment Report Settings PAYMENT_REPORT_GENERATOR_GROUP = ENV_TOKENS.get('PAYMENT_REPORT_GENERATOR_GROUP', PAYMENT_REPORT_GENERATOR_GROUP) -PAYMENT_REPORT_MAX_ITEMS = ENV_TOKENS.get('PAYMENT_REPORT_MAX_ITEMS', PAYMENT_REPORT_MAX_ITEMS) # Bulk Email overrides BULK_EMAIL_DEFAULT_FROM_EMAIL = ENV_TOKENS.get('BULK_EMAIL_DEFAULT_FROM_EMAIL', BULK_EMAIL_DEFAULT_FROM_EMAIL) @@ -278,6 +277,8 @@ if AWS_SECRET_ACCESS_KEY == "": AWS_STORAGE_BUCKET_NAME = AUTH_TOKENS.get('AWS_STORAGE_BUCKET_NAME', 'edxuploads') +# If there is a database called 'read_replica', you can use the use_read_replica_if_available +# function in util/query.py, which is useful for very large database reads DATABASES = AUTH_TOKENS['DATABASES'] XQUEUE_INTERFACE = AUTH_TOKENS['XQUEUE_INTERFACE'] diff --git a/lms/envs/common.py b/lms/envs/common.py index 7c47e2b75e..73e848f98b 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -205,6 +205,8 @@ FEATURES = { # Give course staff unrestricted access to grade downloads (if set to False, # only edX superusers can perform the downloads) 'ALLOW_COURSE_STAFF_GRADE_DOWNLOADS': False, + + 'ENABLED_PAYMENT_REPORTS': ["refund_report", "itemized_purchase_report", "university_revenue_share", "certificate_status"], } # Used for A/B testing @@ -566,8 +568,6 @@ PAID_COURSE_REGISTRATION_CURRENCY = ['usd', '$'] # Members of this group are allowed to generate payment reports PAYMENT_REPORT_GENERATOR_GROUP = 'shoppingcart_report_access' -# Maximum number of rows the report can contain -PAYMENT_REPORT_MAX_ITEMS = 10000 ################################# open ended grading config ##################### diff --git a/lms/envs/dev.py b/lms/envs/dev.py index 5cdece6927..282daa0432 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -51,6 +51,8 @@ LOGGING = get_logger_config(ENV_ROOT / "log", dev_env=True, debug=True) +# If there is a database called 'read_replica', you can use the use_read_replica_if_available +# function in util/query.py, which is useful for very large database reads DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', diff --git a/lms/templates/shoppingcart/download_report.html b/lms/templates/shoppingcart/download_report.html index 838b07f145..584469f49d 100644 --- a/lms/templates/shoppingcart/download_report.html +++ b/lms/templates/shoppingcart/download_report.html @@ -1,29 +1,58 @@ <%! from django.utils.translation import ugettext as _ %> <%! from django.core.urlresolvers import reverse %> +<%! from django.conf import settings %> <%inherit file="../main.html" /> -<%block name="title">${_("Download Purchase Report")} +<%block name="title">${_("Download CSV Reports")}
-

${_("Download CSV of purchase data")}

+

${_("Download CSV Data")}

% if date_fmt_error:
${_("There was an error in your date input. It should be formatted as YYYY-MM-DD")}
% endif - % if total_count_error: -
- ${_("There are too many results in your report.")} (>${settings.PAYMENT_REPORT_MAX_ITEMS}). - ${_("Try making the date range smaller.")} -
- % endif
+ %if ("itemized_purchase_report" or "refund_report") in settings.FEATURES['ENABLED_PAYMENT_REPORTS']: +

${_("These reports are delimited by start and end dates.")}

+
+ + %if "itemized_purchase_report" in settings.FEATURES['ENABLED_PAYMENT_REPORTS']: + +
+ %endif + + %if "refund_report" in settings.FEATURES['ENABLED_PAYMENT_REPORTS']: + +
+ %endif + +
+ %endif + + %if ("certificate_status" or "university_revenue_share") in settings.FEATURES['ENABLED_PAYMENT_REPORTS']: +

${_("These reports are delimited alphabetically by university name. i.e., generating a report with 'Start Letter' A and 'End Letter' C will generate reports for all universities starting with A, B, and C.")}

+ + + + - +
+ + %if "university_revenue_share" in settings.FEATURES['ENABLED_PAYMENT_REPORTS']: + +
+ %endif + + %if "certificate_status" in settings.FEATURES['ENABLED_PAYMENT_REPORTS']: + +
+ %endif + %endif