From b99977a3cb66d4b2203931a818f767ea3da3b018 Mon Sep 17 00:00:00 2001 From: HammadAhmadWaqas Date: Mon, 24 Feb 2020 16:08:08 +0500 Subject: [PATCH] added management command to push old enrollments to ecommerce --- ...rs_for_old_enterprise_course_enrollment.py | 279 ++++++++++++++++++ ...rs_for_old_enterprise_course_enrollmnet.py | 97 ++++++ .../commerce/management/__init__.py | 0 .../commerce/management/commands/__init__.py | 0 .../enterprise_support/tests/factories.py | 1 + 5 files changed, 377 insertions(+) create mode 100644 lms/djangoapps/commerce/management/commands/create_orders_for_old_enterprise_course_enrollment.py create mode 100644 lms/djangoapps/commerce/management/commands/tests/test_create_orders_for_old_enterprise_course_enrollmnet.py create mode 100644 openedx/core/djangoapps/commerce/management/__init__.py create mode 100644 openedx/core/djangoapps/commerce/management/commands/__init__.py diff --git a/lms/djangoapps/commerce/management/commands/create_orders_for_old_enterprise_course_enrollment.py b/lms/djangoapps/commerce/management/commands/create_orders_for_old_enterprise_course_enrollment.py new file mode 100644 index 0000000000..49add56505 --- /dev/null +++ b/lms/djangoapps/commerce/management/commands/create_orders_for_old_enterprise_course_enrollment.py @@ -0,0 +1,279 @@ +""" +Management command to +./manage.py lms create_orders_for_old_enterprise_course_enrollment +./manage.py lms create_orders_for_old_enterprise_course_enrollment --start-index=0 --end-index=100 +./manage.py lms create_orders_for_old_enterprise_course_enrollment --start-index=0 --end-index=100 --batch-size=20 +""" + +import traceback +from textwrap import dedent + +from django.conf import settings +from django.contrib.auth import get_user_model + +from django.core.management.base import BaseCommand, CommandError +from opaque_keys.edx.keys import CourseKey +from requests import Timeout +from slumber.exceptions import HttpServerError, SlumberBaseException + +from student.models import CourseEnrollment +from openedx.core.djangoapps.commerce.utils import ecommerce_api_client +from enterprise.models import EnterpriseCourseEnrollment + +from util.query import use_read_replica_if_available + +User = get_user_model() + + +class Command(BaseCommand): + """ + Command to back-populate orders(in e-commerce) for the enterprise_course_enrollments. + """ + help = dedent(__doc__).strip() + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + service_user = User.objects.get(username=settings.ECOMMERCE_SERVICE_WORKER_USERNAME) + self.client = ecommerce_api_client(service_user) + + def _get_enrollments_queryset(self, start_index, end_index): + """ + + Args: + start_index: start index or None + end_index: end index or None + + Returns: + EnterpriseCourseEnrollments Queryset + + """ + self.stdout.write(u'Getting enrollments from {start} to {end} index (as per command params)' + .format(start=start_index or 'start', end=end_index or 'end')) + enrollments_qs = EnterpriseCourseEnrollment.objects.filter( + source__isnull=True + ).order_by('id')[start_index:end_index] + return use_read_replica_if_available(enrollments_qs) + + def _create_manual_enrollment_orders(self, enrollments): + """ + Calls ecommerce to create orders for the manual enrollments passed in. + + Returns (success_count, fail_count) + """ + try: + order_response = self.client.manual_course_enrollment_order.post( + { + "enrollments": enrollments + } + ) + except (SlumberBaseException, ConnectionError, Timeout, HttpServerError) as exc: + self.stderr.write( + "\t\t\tFailed to create order for manual enrollments for the following enrollments: {}. Reason: {}" + .format(enrollments, exc) + ) + return 0, 0, len(enrollments), [] + + order_creations = order_response["orders"] + + successful_creations = [] + failed_creations = [] + new_creations = [] + new_creation_order_numbers = [] + for order in order_creations: + if order["status"] == "failure": + failed_creations.append(order) + elif order["status"] == "success": + successful_creations.append(order) + if order["new_order_created"]: + new_creations.append(order) + new_creation_order_numbers.append(order["detail"]) + + if failed_creations: + self.stderr.write( + "\t\t\tFailed to created orders for the following manual enrollments. %s", + failed_creations + ) + return len(successful_creations), len(new_creations), len(failed_creations), new_creation_order_numbers + + def _is_paid_mode_course_enrollment(self, username, course_id): + """ + Returns True if mode of the enrollment is paid + """ + paid_modes = ['verified', 'professional'] + course_key = CourseKey.from_string(course_id) + enrollment = CourseEnrollment.objects.get( + user__username=username, course_id=course_key + ) + return enrollment.mode in paid_modes + + def _get_batched_enrollments(self, enrollments_queryset, offset, batch_size): + """ + Args: + enrollments_queryset: enrollments_queryset to slice + batch_size: slice size + + Returns: enrollments + + """ + + self.stdout.write( + u'\tFetching Enrollments from {start} to {end}'.format(start=offset, end=offset + batch_size) + ) + enrollments = enrollments_queryset.select_related( + 'enterprise_customer_user', 'enterprise_customer_user__enterprise_customer' + )[offset: offset + batch_size] + return enrollments + + def _sync_with_ecommerce(self, enrollments_batch): + """ + Sync batch of enrollments with ecommerce + """ + enrollments_payload = [] + + non_paid = 0 + invalid = 0 + + self.stdout.write( + u'\t\tProcessing Total : {},'.format(len(enrollments_batch)) + ) + + for enrollment in enrollments_batch: + try: + enterprise_customer_user = enrollment.enterprise_customer_user + user = enterprise_customer_user.user + enterprise_customer = enterprise_customer_user.enterprise_customer + username = user.username + course_id = enrollment.course_id + if not self._is_paid_mode_course_enrollment(username, course_id): + # we want to skip this enrollment, as its not paid + non_paid += 1 + continue + enrollment_payload = { + "enterprise_enrollment_id": enrollment.id, + "lms_user_id": user.id, + "username": username, + "email": user.email, + "date_placed": enrollment.created.isoformat(), + "course_run_key": course_id, + "enterprise_customer_name": enterprise_customer.name, + "enterprise_customer_uuid": str(enterprise_customer.uuid), + } + except AttributeError as ex: + self.stderr.write(u'\t\tskipping enrollment {} due to invalid data. {}'.format(enrollment, ex)) + invalid += 1 + continue + except CourseEnrollment.DoesNotExist: + self.stderr.write(u'\t\tskipping enrollment {} because CourseEnrollment not found'.format(enrollment)) + invalid += 1 + continue + enrollments_payload.append(enrollment_payload) + + self.stdout.write(u'\t\tFound {count} Paid enrollments to sync'.format(count=len(enrollments_payload))) + if not enrollments_payload: + return 0, 0, 0, invalid, non_paid, [] + + self.stdout.write(u'\t\tSyncing started...') + success, new, failed, order_numbers = self._create_manual_enrollment_orders(enrollments_payload) + self.stdout.write( + u'\t\tSuccess: {} , New: {}, Failed: {}, Invalid:{} , Non-Paid: {}'.format( + success, new, failed, invalid, non_paid, + ) + ) + return success, new, failed, invalid, non_paid, order_numbers + + def _sync(self, enrollments_queryset, enrollments_count, enrollments_batch_size): + """ + Syncs a single site + """ + self.stdout.write(u'Syncing process started.') + + offset = 0 + enrollments_queue = [] + enrollments_query_batch_size = 5000 + successfully_synced_enrollments = 0 + new_created_orders = 0 + new_created_order_numbers = [] + failed_to_synced_enrollments = 0 + invalid_enrollments = 0 + non_paid_enrollments = 0 + + while offset < enrollments_count: + is_last_iteration = (offset + enrollments_query_batch_size) >= enrollments_count + self.stdout.write( + u'\tSyncing enrollments batch from {start} to {end}.'.format( + start=offset, end=offset + enrollments_query_batch_size + ) + ) + enrollments_queue += self._get_batched_enrollments( + enrollments_queryset, + offset, + enrollments_query_batch_size + ) + while len(enrollments_queue) >= enrollments_batch_size \ + or (is_last_iteration and enrollments_queue): # for last iteration need to empty enrollments_queue + enrollments_batch = enrollments_queue[:enrollments_batch_size] + del enrollments_queue[:enrollments_batch_size] + success, new, failed, invalid, non_paid, order_numbers = self._sync_with_ecommerce(enrollments_batch) + successfully_synced_enrollments += success + new_created_orders += new + failed_to_synced_enrollments += failed + invalid_enrollments += invalid + non_paid_enrollments += non_paid + new_created_order_numbers += order_numbers + self.stdout.write( + u'\tSuccessfully synced enrollments batch from {start} to {end}'.format( + start=offset, end=offset + enrollments_query_batch_size, + ) + ) + offset += enrollments_query_batch_size + + self.stdout.write( + u'[Final Summary] Enrollments Success: {}, New: {}, Failed: {}, Invalid: {} , Non-Paid: {}'.format( + successfully_synced_enrollments, new_created_orders, failed_to_synced_enrollments, invalid_enrollments, + non_paid_enrollments + ) + ) + self.stdout.write('New created order numbers {}'.format(new_created_order_numbers)) + + def add_arguments(self, parser): + """ + Definition of arguments this command accepts + """ + parser.add_argument( + '--start-index', + dest='start_index', + type=int, + help='Staring index for enrollments', + ) + parser.add_argument( + '--end-index', + dest='end_index', + type=int, + help='Ending index for enrollments', + ) + parser.add_argument( + '--batch-size', + default=25, + dest='batch_size', + type=int, + help='Size of enrollments batch to be sent to ecommerce', + ) + + def handle(self, *args, **options): + """ + Main command handler + """ + start_index = options['start_index'] + end_index = options['end_index'] + batch_size = options['batch_size'] + + try: + self.stdout.write(u'Command execution started with options = {}.'.format(options)) + enrollments_queryset = self._get_enrollments_queryset(start_index, end_index) + enrollments_count = enrollments_queryset.count() + self.stdout.write(u'Total Enrollments count to process: {count}'.format(count=enrollments_count)) + self._sync(enrollments_queryset, enrollments_count, batch_size) + + except Exception as ex: + traceback.print_exc() + raise CommandError(u'Command failed with traceback %s' % str(ex)) diff --git a/lms/djangoapps/commerce/management/commands/tests/test_create_orders_for_old_enterprise_course_enrollmnet.py b/lms/djangoapps/commerce/management/commands/tests/test_create_orders_for_old_enterprise_course_enrollmnet.py new file mode 100644 index 0000000000..d6621184cf --- /dev/null +++ b/lms/djangoapps/commerce/management/commands/tests/test_create_orders_for_old_enterprise_course_enrollmnet.py @@ -0,0 +1,97 @@ +""" +Test the create_orders_for_old_enterprise_course_enrollment management command +""" + +import re + +from django.core.management import call_command +from django.test import TestCase, override_settings +from django.utils.six import StringIO +from mock import patch +from six.moves import range + +from course_modes.models import CourseMode +from student.tests.factories import UserFactory, CourseEnrollmentFactory +from openedx.core.djangoapps.credit.tests.test_api import TEST_ECOMMERCE_WORKER +from openedx.core.djangolib.testing.utils import skip_unless_lms +from openedx.features.enterprise_support.tests.factories import ( + EnterpriseCourseEnrollmentFactory, EnterpriseCustomerUserFactory +) + + +@skip_unless_lms +@override_settings(ECOMMERCE_SERVICE_WORKER_USERNAME=TEST_ECOMMERCE_WORKER) +class TestEnterpriseCourseEnrollmentCreateOldOrder(TestCase): + """ + Test create_orders_for_old_enterprise_course_enrollment management command. + """ + + @classmethod + def setUpClass(cls): + super(TestEnterpriseCourseEnrollmentCreateOldOrder, cls).setUpClass() + UserFactory(username=TEST_ECOMMERCE_WORKER) + cls.enrollment_count = 30 + cls._create_enterprise_course_enrollments(30) + + @classmethod + def _create_enterprise_course_enrollments(cls, count): + """ + Creates `count` test enrollments plus 1 invalid and 1 Audit enrollment + """ + for _ in range(count): + user = UserFactory() + course_enrollment = CourseEnrollmentFactory(mode=CourseMode.VERIFIED, user=user) + course = course_enrollment.course + enterprise_customer_user = EnterpriseCustomerUserFactory(user_id=user.id) + EnterpriseCourseEnrollmentFactory(enterprise_customer_user=enterprise_customer_user, course_id=course.id) + + # creating audit enrollment + user = UserFactory() + course_enrollment = CourseEnrollmentFactory(mode=CourseMode.AUDIT, user=user) + course = course_enrollment.course + enterprise_customer_user = EnterpriseCustomerUserFactory(user_id=user.id) + EnterpriseCourseEnrollmentFactory(enterprise_customer_user=enterprise_customer_user, course_id=course.id) + + # creating invalid enrollment (with no CourseEnrollment) + user = UserFactory() + enterprise_customer_user = EnterpriseCustomerUserFactory(user_id=user.id) + EnterpriseCourseEnrollmentFactory(enterprise_customer_user=enterprise_customer_user, course_id=course.id) + + @patch('lms.djangoapps.commerce.management.commands.create_orders_for_old_enterprise_course_enrollment' + '.Command._create_manual_enrollment_orders') + def test_command(self, mock_create_manual_enrollment_orders): + """ + Test command with batch size + """ + mock_create_manual_enrollment_orders.return_value = (0, 0, 0, []) # not correct return value, just fixes unpack + out = StringIO() + call_command('create_orders_for_old_enterprise_course_enrollment', '--batch-size=10', stdout=out) + output = out.getvalue() + self.assertIn("Total Enrollments count to process: 32", output) # 30 + 1 + 1 + self.assertTrue( + re.search( + r'\[Final Summary\] Enrollments Success: \d+, New: \d+, Failed: 0, Invalid: 1 , Non-Paid: 1', + output + ) + ) + self.assertEqual(mock_create_manual_enrollment_orders.call_count, 4) # batch of 4 (10, 10, 10, 2) + + @patch('lms.djangoapps.commerce.management.commands.create_orders_for_old_enterprise_course_enrollment' + '.Command._create_manual_enrollment_orders') + def test_command_start_and_end_index(self, mock_create_manual_enrollment_orders): + """ + Test command with batch size + """ + mock_create_manual_enrollment_orders.return_value = (0, 0, 0, []) # not correct return value, just fixes unpack + out = StringIO() + call_command( + 'create_orders_for_old_enterprise_course_enrollment', + '--start-index=5', + '--end-index=20', + '--batch-size=10', + stdout=out + ) + output = out.getvalue() + self.assertIn("Total Enrollments count to process: 15", output) + self.assertIn('[Final Summary] Enrollments Success: ', output) + self.assertEqual(mock_create_manual_enrollment_orders.call_count, 2) # batch of 2 (10, 5) diff --git a/openedx/core/djangoapps/commerce/management/__init__.py b/openedx/core/djangoapps/commerce/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openedx/core/djangoapps/commerce/management/commands/__init__.py b/openedx/core/djangoapps/commerce/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openedx/features/enterprise_support/tests/factories.py b/openedx/features/enterprise_support/tests/factories.py index 582ca316d7..0b815e44d5 100644 --- a/openedx/features/enterprise_support/tests/factories.py +++ b/openedx/features/enterprise_support/tests/factories.py @@ -31,6 +31,7 @@ class EnterpriseCustomerFactory(factory.django.DjangoModelFactory): uuid = factory.LazyAttribute(lambda x: UUID(FAKER.uuid4())) # pylint: disable=no-member name = factory.LazyAttribute(lambda x: FAKER.company()) # pylint: disable=no-member + slug = factory.LazyAttribute(lambda x: FAKER.slug()) # pylint: disable=no-member active = True site = factory.SubFactory(SiteFactory) enable_data_sharing_consent = True