diff --git a/common/djangoapps/student/management/commands/populate_is_marketable_user_attribute.py b/common/djangoapps/student/management/commands/populate_is_marketable_user_attribute.py deleted file mode 100644 index 51fef174b4..0000000000 --- a/common/djangoapps/student/management/commands/populate_is_marketable_user_attribute.py +++ /dev/null @@ -1,173 +0,0 @@ -""" Management command to back-populate marketing emails opt-in for the user accounts. """ - -import time - -from django.contrib.auth import get_user_model -from django.core.management.base import BaseCommand -from django.db import IntegrityError -from django.db.models import Exists, OuterRef - -from common.djangoapps.student.models import UserAttribute -from common.djangoapps.util.query import use_read_replica_if_available - -OLD_USER_ATTRIBUTE_NAME = 'marketing_emails_opt_in' -NEW_USER_ATTRIBUTE_NAME = 'is_marketable' - - -class Command(BaseCommand): - """ - Example usage: - $ ./manage.py lms populate_is_marketable_user_attribute - """ - help = """ - Creates a row in the UserAttribute table for all users in the platform. - This command back-populates the 'is_marketable' attribute in the - UserAttribute table for the user accounts. - """ - - def add_arguments(self, parser): - parser.add_argument( - '--batch-delay', - type=float, - dest='batch_delay', - default=0.5, - help='Time delay in each iteration' - ) - parser.add_argument( - '--batch-size', - type=int, - dest='batch_size', - default=10000, - help='Batch size' - ) - parser.add_argument( - '--start-user-id', - type=int, - dest='start_user_id', - default=10000, - help='Starting user ID' - ) - parser.add_argument( - '--start-userattribute-id', - type=int, - dest='start_userattribute_id', - default=10000, - help='Starting user attribute ID' - ) - parser.add_argument( - '--backfill-only', - type=str, - dest='backfill_only', - default=None, - help='Only backfill user attribute, renaming attribute is not required' - ) - - def _update_user_attribute(self, start_id, end_id): - """ Updates the user attributes in batches. """ - self.stdout.write( - f'Updating user attribute starting from ID {start_id} till {end_id}.' - ) - return UserAttribute.objects.filter( - id__gte=start_id, - id__lt=end_id, - name=OLD_USER_ATTRIBUTE_NAME - ).order_by('id').update(name=NEW_USER_ATTRIBUTE_NAME) - - def _get_old_users_queryset(self, batch_size, marketing_opt_in_start_user_id, user_id=0): - """ - Fetches all the old users in batches, in ascending order of id, that exist before a specified user id. - Returns queryset of tuples with 'id' and 'is_active' field values. - """ - self.stdout.write(f'Fetching old users in batch starting from ID {user_id} with batch size {batch_size}.') - query_set = get_user_model().objects.filter( - id__gt=user_id, - id__lt=marketing_opt_in_start_user_id - ).values_list('id', 'is_active').order_by('id')[:batch_size] - return use_read_replica_if_available(query_set) - - def _get_recent_users_queryset(self, batch_size, user_id): - """ - Fetches all the recent users in batches, in ascending order of id, that exist after a specified user id - and does not have 'is_marketable' user attribute set. - Returns queryset of tuples with 'id' and 'is_active' field values. - """ - self.stdout.write(f'Fetching recent users in batch starting from ID {user_id} with batch size {batch_size}.') - user_attribute_qs = UserAttribute.objects.filter(user=OuterRef('pk'), name=NEW_USER_ATTRIBUTE_NAME) - user_query_set = get_user_model().objects.filter( - ~Exists(user_attribute_qs), - id__gt=user_id, - ).values_list('id', 'is_active').order_by('id')[:batch_size] - return use_read_replica_if_available(user_query_set) - - def _bulk_create_user_attributes(self, users): - """ Creates the UserAttribute records in bulk. """ - last_user_id = 0 - user_attributes = [] - for user in users: - user_attributes.append(UserAttribute( - user_id=user[0], - name=NEW_USER_ATTRIBUTE_NAME, - value=str(user[1]).lower() - )) - last_user_id = user[0] - try: - UserAttribute.objects.bulk_create(user_attributes) - except IntegrityError: - # A UserAttribute object was already created. This could only happen if we try to create 'is_marketable' - # user attribute that is already created. Ignore it if it does. - self.stdout.write(f'IntegrityError raised during bulk_create having last user id: {last_user_id}.') - return last_user_id - - def _backfill_old_users_attribute(self, batch_size, batch_delay, marketing_opt_in_start_user_id): - """ - Backfills the is_marketable user attribute. Fetches all the old user accounts, in ascending order of id, - that were created before a specified user id. All the fetched users do not have 'is_marketable' - user attribute set. - """ - users = self._get_old_users_queryset(batch_size, marketing_opt_in_start_user_id) - while users: - last_user_id = self._bulk_create_user_attributes(users) - time.sleep(batch_delay) - users = self._get_old_users_queryset(batch_size, marketing_opt_in_start_user_id, last_user_id) - - def _backfill_recent_users_attribute(self, batch_size, batch_delay, start_user_id): - """ - Backfills the is_marketable user attribute. Fetches all the recent user accounts, in ascending order of id, - that were created after a specified user id (start_user_id). - This method handles the backfill of all those users that have missing user attribute even after enabling - the MARKETING_EMAILS_OPT_IN flag. - """ - users = self._get_recent_users_queryset(batch_size, start_user_id) - while users: - last_user_id = self._bulk_create_user_attributes(users) - time.sleep(batch_delay) - users = self._get_recent_users_queryset(batch_size, last_user_id) - - def _rename_user_attribute_name(self, batch_size, batch_delay, start_user_attribute_id): - """ Renames the old user attribute 'marketing_emails_opt_in' to 'is_marketable'. """ - updated_records_count = 0 - start_id = start_user_attribute_id - end_id = start_id + batch_size - total_user_attribute_count = UserAttribute.objects.filter(name=OLD_USER_ATTRIBUTE_NAME).count() - - while updated_records_count < total_user_attribute_count: - updated_records_count += self._update_user_attribute(start_id, end_id) - start_id = end_id - end_id = start_id + batch_size - time.sleep(batch_delay) - - def handle(self, *args, **options): - """ - This command back-populates the 'is_marketable' attribute for all existing users who do not already - have the attribute set. - """ - batch_delay = options['batch_delay'] - batch_size = options['batch_size'] - self.stdout.write(f'Command execution started with options: {options}.') - - if not options['backfill_only']: - self._rename_user_attribute_name(batch_size, batch_delay, options['start_userattribute_id']) - self._backfill_old_users_attribute(batch_size, batch_delay, options['start_user_id']) - self._backfill_recent_users_attribute(batch_size, batch_delay, options['start_user_id']) - - self.stdout.write('Command executed successfully.') diff --git a/common/djangoapps/student/management/commands/populate_user_data_on_braze.py b/common/djangoapps/student/management/commands/populate_user_data_on_braze.py deleted file mode 100644 index f8fdbd8bf3..0000000000 --- a/common/djangoapps/student/management/commands/populate_user_data_on_braze.py +++ /dev/null @@ -1,164 +0,0 @@ -""" Management command to add user data on Braze. """ -import logging -import time - -from braze.client import BrazeClient -from braze.exceptions import BrazeClientError -from django.conf import settings -from django.contrib.auth import get_user_model -from django.core.management.base import BaseCommand - -from common.djangoapps.student.models import UserAttribute -from common.djangoapps.util.query import use_read_replica_if_available - -User = get_user_model() - -MARKETING_EMAIL_ATTRIBUTE_NAME = 'is_marketable' -TRACK_USER_COMPONENT_CHUNK_SIZE = 75 - -logger = logging.getLogger(__name__) - - -class Command(BaseCommand): - """ - Command to add is_marketable field on Braze against a user_id. If an account - with user_id doesn't exist on Braze, it automatically creates one. - - Example usage: - $ ./manage.py lms backfill_user_data_on_braze - """ - help = """ - Creates user accounts on Braze for missing users and adds the is_marketable - user attribute to all user accounts on Braze. - """ - - def add_arguments(self, parser): - """ - Function to get command arguments - """ - parser.add_argument( - '--batch-delay', - type=float, - dest='batch_delay', - default=0.5, - help='Time delay in seconds between each iteration' - ) - parser.add_argument( - '--batch-size', - type=int, - dest='batch_size', - default=10000, - help='Batch size' - ) - parser.add_argument( - '--starting-user-id', - type=int, - dest='starting_user_id', - default=0, - help='Starting user id to process a specific batch of users. ' - 'Both start and end id should be provided.', - ) - parser.add_argument( - '--ending-user-id', - type=int, - dest='ending_user_id', - default=0, - help='Ending user id (inclusive) to process a specific batch of users. ' - 'Both start and end id should be provided.', - ) - - def __init__(self): - super().__init__() - self.braze_client = BrazeClient( - api_key=settings.EDX_BRAZE_API_KEY, - api_url=settings.EDX_BRAZE_API_SERVER, - app_id='', - ) - - @staticmethod - def _chunks(users, chunk_size=TRACK_USER_COMPONENT_CHUNK_SIZE): - """ - Yields successive chunks of users. The size of each chunk is determined by - TRACK_USER_COMPONENT_CHUNK_SIZE which is set to 75. - - Reference: https://www.braze.com/docs/api/endpoints/user_data/post_user_track/ - """ - for index in range(0, len(users), chunk_size): - yield users[index:index + chunk_size] - - def _get_user_batch(self, batch_start_id, batch_end_id): - """ - This returns the batch of users. - """ - query = User.objects.filter( - id__gte=batch_start_id, id__lt=batch_end_id, - ).select_related('profile').values_list( - 'id', 'email', 'profile__name', 'username', named=True, - ).order_by('id') - - return use_read_replica_if_available(query) - - def _get_marketable_users(self, batch_start_id, batch_end_id): - """ - For a given batch of users it gives the marketable users list - """ - get_marketable_users_query = UserAttribute.objects.filter( - user_id__gte=batch_start_id, - user_id__lt=batch_end_id, - name=MARKETING_EMAIL_ATTRIBUTE_NAME, - value__in=['True', 'true', '1'], - ).values_list('user_id', flat=True).order_by('user_id') - return set(use_read_replica_if_available(get_marketable_users_query)) - - def _update_braze_attributes(self, users, marketable_users): - """ - Sends Braze API request to update/create user account. - - Fields sent using the API include: - - external_id (user_id) - - email - - name - - username - - is_marketable - """ - attributes = [] - for user in users: - attributes.append( - { - "external_id": user.id, - "email": user.email, - "name": user.profile__name, - "username": user.username, - "is_marketable": user.id in marketable_users, - } - ) - - try: - self.braze_client.track_user(attributes=attributes) - except BrazeClientError as error: - logger.error(f'Failed to update attributes. Error: {error}') - - def handle(self, *args, **options): - """ - Handler to run the command. - """ - sleep_time = options['batch_delay'] - batch_size = options['batch_size'] - starting_user_id = options['starting_user_id'] - ending_user_id = options['ending_user_id'] - - all_users_query = use_read_replica_if_available(User.objects) - total_users_count = ending_user_id if ending_user_id else all_users_query.count() - - for index in range(starting_user_id, total_users_count + 1, batch_size): - users = self._get_user_batch(index, index + batch_size) - logger.info(f'Processing users with user ids in {index} - {(index + batch_size) - 1} range') - - # Force evaluating the query to avoid multiple hits to db - # when we evaluate the chunks. - evaluated_users = list(users) - marketable_users = self._get_marketable_users(index, index + batch_size) - for user_chunk in self._chunks(evaluated_users): - self._update_braze_attributes(user_chunk, marketable_users) - - time.sleep(sleep_time) diff --git a/common/djangoapps/student/management/tests/test_populate_is_marketable_user_attribute.py b/common/djangoapps/student/management/tests/test_populate_is_marketable_user_attribute.py deleted file mode 100644 index f692a09353..0000000000 --- a/common/djangoapps/student/management/tests/test_populate_is_marketable_user_attribute.py +++ /dev/null @@ -1,75 +0,0 @@ -""" -Unittests for populate_marketing_opt_in_user_attribute management command. -""" - -import pytest -from django.contrib.auth.models import User # lint-amnesty, pylint: disable=imported-auth-user -from django.core.management import call_command -from django.test import TransactionTestCase - -from common.djangoapps.student.models import UserAttribute -from common.djangoapps.student.tests.factories import UserFactory -from openedx.core.djangolib.testing.utils import skip_unless_lms - -IS_MARKETABLE = 'is_marketable' - - -@skip_unless_lms -class TestPopulateMarketingOptInUserAttribute(TransactionTestCase): - """ - Test populate_is_marketable_user_attribute management command. - """ - - def setUp(self): - super().setUp() - self.existing_user = UserFactory() - - def test_command_with_existing_users(self): - """ - Test population of is_marketable attribute with an existing user. - """ - assert UserAttribute.objects.count() == 0 - call_command( - 'populate_is_marketable_user_attribute', - start_user_id=1, - start_userattribute_id=1 - ) - assert UserAttribute.objects.filter(name=IS_MARKETABLE).count() == User.objects.count() - - def test_command_with_new_user(self): - """ - Test population of is_marketable attribute with a new user. - """ - user = UserFactory() - call_command( - 'populate_is_marketable_user_attribute', - start_user_id=0, - start_userattribute_id=1 - ) - assert UserAttribute.objects.filter(name=IS_MARKETABLE).count() == User.objects.count() - - def test_command_rename_to_new_attribute(self): - """ - Test renaming of marketing_emails_opt_in to is_marketable attribute. - """ - user = UserFactory() - UserAttribute.objects.create(user=user, name='marketing_emails_opt_in', value='true') - call_command( - 'populate_is_marketable_user_attribute', - start_user_id=1, - start_userattribute_id=1 - ) - assert UserAttribute.objects.filter(name='marketing_emails_opt_in').count() == 0 - assert UserAttribute.get_user_attribute(user, IS_MARKETABLE) == 'true' - - def test_command_with_invalid_argument(self): - """ - Test management command raises TypeError on wrong data type value for '--batch-size' argument. - """ - with pytest.raises(TypeError): - call_command( - 'populate_is_marketable_user_attribute', - batch_size='1000', - start_user_id=1, - start_userattribute_id=1 - ) diff --git a/common/djangoapps/student/management/tests/test_populate_user_data_on_braze.py b/common/djangoapps/student/management/tests/test_populate_user_data_on_braze.py deleted file mode 100644 index 9fe0b990c7..0000000000 --- a/common/djangoapps/student/management/tests/test_populate_user_data_on_braze.py +++ /dev/null @@ -1,93 +0,0 @@ -""" -Unittests for populate_marketing_opt_in_user_attribute management command. -""" -from unittest.mock import patch, MagicMock - -from braze.exceptions import BrazeClientError -from django.core.management import call_command -from django.test import TestCase -from testfixtures import LogCapture - -from common.djangoapps.student.management.commands.populate_user_data_on_braze import MARKETING_EMAIL_ATTRIBUTE_NAME -from common.djangoapps.student.tests.factories import UserFactory, UserAttributeFactory -from openedx.core.djangolib.testing.utils import skip_unless_lms - -LOGGER_NAME = 'common.djangoapps.student.management.commands.populate_user_data_on_braze' - - -@skip_unless_lms -class TestPopulateUserDataOnBraze(TestCase): - """ - Tests for PopulateUserDataOnBraze management command. - """ - - @classmethod - def setUpClass(cls): - super().setUpClass() - for index in range(15): - user = UserFactory() - UserAttributeFactory(user=user, name=MARKETING_EMAIL_ATTRIBUTE_NAME, value=(index % 2 == 0)) - - @patch('common.djangoapps.student.management.commands.populate_user_data_on_braze.BrazeClient.track_user') - def test_command_updates_users_on_braze(self, track_user): - """ - Test that Braze API is called successfully for all users - """ - track_user.return_value = MagicMock() - call_command('populate_user_data_on_braze', batch_delay=0) - assert track_user.called - - @patch('common.djangoapps.student.management.commands.populate_user_data_on_braze.BrazeClient.track_user') - def test_logs_for_success(self, track_user): - """ - Test logs for a successful run of updating user accounts on Braze - """ - track_user.return_value = MagicMock() - with LogCapture(LOGGER_NAME) as log: - call_command( - 'populate_user_data_on_braze', - batch_size=5, - batch_delay=0, - ) - log.check( - (LOGGER_NAME, 'INFO', 'Processing users with user ids in 0 - 4 range'), - (LOGGER_NAME, 'INFO', 'Processing users with user ids in 5 - 9 range'), - (LOGGER_NAME, 'INFO', 'Processing users with user ids in 10 - 14 range'), - (LOGGER_NAME, 'INFO', 'Processing users with user ids in 15 - 19 range'), - ) - - @patch('common.djangoapps.student.management.commands.populate_user_data_on_braze.BrazeClient.track_user') - def test_logs_for_failure(self, track_user): - """ - Test logs for when the update to Braze fails - """ - track_user.side_effect = BrazeClientError('Update to attributes failed.') - with LogCapture(LOGGER_NAME) as log: - call_command( - 'populate_user_data_on_braze', - batch_size=5, - batch_delay=0, - ) - log.check_present( - (LOGGER_NAME, 'ERROR', 'Failed to update attributes. Error: Update to attributes failed.'), - ) - - @patch('common.djangoapps.student.management.commands.populate_user_data_on_braze.BrazeClient.track_user') - def test_running_a_specific_batch(self, track_user): - """ - Test running command for a specific batch of users - """ - track_user.return_value = MagicMock() - with LogCapture(LOGGER_NAME) as log: - call_command( - 'populate_user_data_on_braze', - batch_size=5, - batch_delay=0, - starting_user_id=2, - ending_user_id=13, - ) - log.check( - (LOGGER_NAME, 'INFO', 'Processing users with user ids in 2 - 6 range'), - (LOGGER_NAME, 'INFO', 'Processing users with user ids in 7 - 11 range'), - (LOGGER_NAME, 'INFO', 'Processing users with user ids in 12 - 16 range'), - )