diff --git a/common/djangoapps/student/management/commands/populate_is_marketable_user_attribute.py b/common/djangoapps/student/management/commands/populate_is_marketable_user_attribute.py new file mode 100644 index 0000000000..51fef174b4 --- /dev/null +++ b/common/djangoapps/student/management/commands/populate_is_marketable_user_attribute.py @@ -0,0 +1,173 @@ +""" Management command to back-populate marketing emails opt-in for the user accounts. """ + +import time + +from django.contrib.auth import get_user_model +from django.core.management.base import BaseCommand +from django.db import IntegrityError +from django.db.models import Exists, OuterRef + +from common.djangoapps.student.models import UserAttribute +from common.djangoapps.util.query import use_read_replica_if_available + +OLD_USER_ATTRIBUTE_NAME = 'marketing_emails_opt_in' +NEW_USER_ATTRIBUTE_NAME = 'is_marketable' + + +class Command(BaseCommand): + """ + Example usage: + $ ./manage.py lms populate_is_marketable_user_attribute + """ + help = """ + Creates a row in the UserAttribute table for all users in the platform. + This command back-populates the 'is_marketable' attribute in the + UserAttribute table for the user accounts. + """ + + def add_arguments(self, parser): + parser.add_argument( + '--batch-delay', + type=float, + dest='batch_delay', + default=0.5, + help='Time delay in each iteration' + ) + parser.add_argument( + '--batch-size', + type=int, + dest='batch_size', + default=10000, + help='Batch size' + ) + parser.add_argument( + '--start-user-id', + type=int, + dest='start_user_id', + default=10000, + help='Starting user ID' + ) + parser.add_argument( + '--start-userattribute-id', + type=int, + dest='start_userattribute_id', + default=10000, + help='Starting user attribute ID' + ) + parser.add_argument( + '--backfill-only', + type=str, + dest='backfill_only', + default=None, + help='Only backfill user attribute, renaming attribute is not required' + ) + + def _update_user_attribute(self, start_id, end_id): + """ Updates the user attributes in batches. """ + self.stdout.write( + f'Updating user attribute starting from ID {start_id} till {end_id}.' + ) + return UserAttribute.objects.filter( + id__gte=start_id, + id__lt=end_id, + name=OLD_USER_ATTRIBUTE_NAME + ).order_by('id').update(name=NEW_USER_ATTRIBUTE_NAME) + + def _get_old_users_queryset(self, batch_size, marketing_opt_in_start_user_id, user_id=0): + """ + Fetches all the old users in batches, in ascending order of id, that exist before a specified user id. + Returns queryset of tuples with 'id' and 'is_active' field values. + """ + self.stdout.write(f'Fetching old users in batch starting from ID {user_id} with batch size {batch_size}.') + query_set = get_user_model().objects.filter( + id__gt=user_id, + id__lt=marketing_opt_in_start_user_id + ).values_list('id', 'is_active').order_by('id')[:batch_size] + return use_read_replica_if_available(query_set) + + def _get_recent_users_queryset(self, batch_size, user_id): + """ + Fetches all the recent users in batches, in ascending order of id, that exist after a specified user id + and does not have 'is_marketable' user attribute set. + Returns queryset of tuples with 'id' and 'is_active' field values. + """ + self.stdout.write(f'Fetching recent users in batch starting from ID {user_id} with batch size {batch_size}.') + user_attribute_qs = UserAttribute.objects.filter(user=OuterRef('pk'), name=NEW_USER_ATTRIBUTE_NAME) + user_query_set = get_user_model().objects.filter( + ~Exists(user_attribute_qs), + id__gt=user_id, + ).values_list('id', 'is_active').order_by('id')[:batch_size] + return use_read_replica_if_available(user_query_set) + + def _bulk_create_user_attributes(self, users): + """ Creates the UserAttribute records in bulk. """ + last_user_id = 0 + user_attributes = [] + for user in users: + user_attributes.append(UserAttribute( + user_id=user[0], + name=NEW_USER_ATTRIBUTE_NAME, + value=str(user[1]).lower() + )) + last_user_id = user[0] + try: + UserAttribute.objects.bulk_create(user_attributes) + except IntegrityError: + # A UserAttribute object was already created. This could only happen if we try to create 'is_marketable' + # user attribute that is already created. Ignore it if it does. + self.stdout.write(f'IntegrityError raised during bulk_create having last user id: {last_user_id}.') + return last_user_id + + def _backfill_old_users_attribute(self, batch_size, batch_delay, marketing_opt_in_start_user_id): + """ + Backfills the is_marketable user attribute. Fetches all the old user accounts, in ascending order of id, + that were created before a specified user id. All the fetched users do not have 'is_marketable' + user attribute set. + """ + users = self._get_old_users_queryset(batch_size, marketing_opt_in_start_user_id) + while users: + last_user_id = self._bulk_create_user_attributes(users) + time.sleep(batch_delay) + users = self._get_old_users_queryset(batch_size, marketing_opt_in_start_user_id, last_user_id) + + def _backfill_recent_users_attribute(self, batch_size, batch_delay, start_user_id): + """ + Backfills the is_marketable user attribute. Fetches all the recent user accounts, in ascending order of id, + that were created after a specified user id (start_user_id). + This method handles the backfill of all those users that have missing user attribute even after enabling + the MARKETING_EMAILS_OPT_IN flag. + """ + users = self._get_recent_users_queryset(batch_size, start_user_id) + while users: + last_user_id = self._bulk_create_user_attributes(users) + time.sleep(batch_delay) + users = self._get_recent_users_queryset(batch_size, last_user_id) + + def _rename_user_attribute_name(self, batch_size, batch_delay, start_user_attribute_id): + """ Renames the old user attribute 'marketing_emails_opt_in' to 'is_marketable'. """ + updated_records_count = 0 + start_id = start_user_attribute_id + end_id = start_id + batch_size + total_user_attribute_count = UserAttribute.objects.filter(name=OLD_USER_ATTRIBUTE_NAME).count() + + while updated_records_count < total_user_attribute_count: + updated_records_count += self._update_user_attribute(start_id, end_id) + start_id = end_id + end_id = start_id + batch_size + time.sleep(batch_delay) + + def handle(self, *args, **options): + """ + This command back-populates the 'is_marketable' attribute for all existing users who do not already + have the attribute set. + """ + batch_delay = options['batch_delay'] + batch_size = options['batch_size'] + self.stdout.write(f'Command execution started with options: {options}.') + + if not options['backfill_only']: + self._rename_user_attribute_name(batch_size, batch_delay, options['start_userattribute_id']) + self._backfill_old_users_attribute(batch_size, batch_delay, options['start_user_id']) + self._backfill_recent_users_attribute(batch_size, batch_delay, options['start_user_id']) + + self.stdout.write('Command executed successfully.') diff --git a/common/djangoapps/student/management/tests/test_populate_is_marketable_user_attribute.py b/common/djangoapps/student/management/tests/test_populate_is_marketable_user_attribute.py new file mode 100644 index 0000000000..f692a09353 --- /dev/null +++ b/common/djangoapps/student/management/tests/test_populate_is_marketable_user_attribute.py @@ -0,0 +1,75 @@ +""" +Unittests for populate_marketing_opt_in_user_attribute management command. +""" + +import pytest +from django.contrib.auth.models import User # lint-amnesty, pylint: disable=imported-auth-user +from django.core.management import call_command +from django.test import TransactionTestCase + +from common.djangoapps.student.models import UserAttribute +from common.djangoapps.student.tests.factories import UserFactory +from openedx.core.djangolib.testing.utils import skip_unless_lms + +IS_MARKETABLE = 'is_marketable' + + +@skip_unless_lms +class TestPopulateMarketingOptInUserAttribute(TransactionTestCase): + """ + Test populate_is_marketable_user_attribute management command. + """ + + def setUp(self): + super().setUp() + self.existing_user = UserFactory() + + def test_command_with_existing_users(self): + """ + Test population of is_marketable attribute with an existing user. + """ + assert UserAttribute.objects.count() == 0 + call_command( + 'populate_is_marketable_user_attribute', + start_user_id=1, + start_userattribute_id=1 + ) + assert UserAttribute.objects.filter(name=IS_MARKETABLE).count() == User.objects.count() + + def test_command_with_new_user(self): + """ + Test population of is_marketable attribute with a new user. + """ + user = UserFactory() + call_command( + 'populate_is_marketable_user_attribute', + start_user_id=0, + start_userattribute_id=1 + ) + assert UserAttribute.objects.filter(name=IS_MARKETABLE).count() == User.objects.count() + + def test_command_rename_to_new_attribute(self): + """ + Test renaming of marketing_emails_opt_in to is_marketable attribute. + """ + user = UserFactory() + UserAttribute.objects.create(user=user, name='marketing_emails_opt_in', value='true') + call_command( + 'populate_is_marketable_user_attribute', + start_user_id=1, + start_userattribute_id=1 + ) + assert UserAttribute.objects.filter(name='marketing_emails_opt_in').count() == 0 + assert UserAttribute.get_user_attribute(user, IS_MARKETABLE) == 'true' + + def test_command_with_invalid_argument(self): + """ + Test management command raises TypeError on wrong data type value for '--batch-size' argument. + """ + with pytest.raises(TypeError): + call_command( + 'populate_is_marketable_user_attribute', + batch_size='1000', + start_user_id=1, + start_userattribute_id=1 + ) diff --git a/openedx/core/djangoapps/user_authn/views/register.py b/openedx/core/djangoapps/user_authn/views/register.py index e522f8a0ba..fb06eca0c0 100644 --- a/openedx/core/djangoapps/user_authn/views/register.py +++ b/openedx/core/djangoapps/user_authn/views/register.py @@ -101,7 +101,7 @@ REGISTRATION_UTM_PARAMETERS = { 'utm_content': 'registration_utm_content', } REGISTRATION_UTM_CREATED_AT = 'registration_utm_created_at' -MARKETING_EMAILS_OPT_IN = 'marketing_emails_opt_in' +IS_MARKETABLE = 'is_marketable' # used to announce a registration # providing_args=["user", "registration"] REGISTER_USER = Signal() @@ -253,7 +253,8 @@ def create_account_with_params(request, params): # pylint: disable=too-many-sta except Exception: # pylint: disable=broad-except log.exception(f"Enable discussion notifications failed for user {user.id}.") - _track_user_registration(user, profile, params, third_party_provider, registration) + is_marketable = params.get('marketing_emails_opt_in') in ['true', '1'] + _track_user_registration(user, profile, params, third_party_provider, registration, is_marketable) # Announce registration REGISTER_USER.send(sender=None, user=user, registration=registration) @@ -275,7 +276,7 @@ def create_account_with_params(request, params): # pylint: disable=too-many-sta try: _record_registration_attributions(request, new_user) - _record_marketing_emails_opt_in_attribute(params.get('marketing_emails_opt_in'), new_user) + _record_is_marketable_attribute(is_marketable, new_user) # Don't prevent a user from registering due to attribution errors. except Exception: # pylint: disable=broad-except log.exception('Error while attributing cookies to user registration.') @@ -347,10 +348,9 @@ def _link_user_to_third_party_provider( return third_party_provider, running_pipeline -def _track_user_registration(user, profile, params, third_party_provider, registration): +def _track_user_registration(user, profile, params, third_party_provider, registration, is_marketable): """ Track the user's registration. """ if hasattr(settings, 'LMS_SEGMENT_KEY') and settings.LMS_SEGMENT_KEY: - is_marketable = params.get('marketing_emails_opt_in') in ['true', '1'] traits = { 'email': user.email, 'username': user.username, @@ -461,12 +461,12 @@ def _skip_activation_email(user, running_pipeline, third_party_provider): ) -def _record_marketing_emails_opt_in_attribute(opt_in, user): +def _record_is_marketable_attribute(is_marketable, user): """ Attribute this user's registration based on form data """ - if settings.MARKETING_EMAILS_OPT_IN and user and opt_in: - UserAttribute.set_user_attribute(user, MARKETING_EMAILS_OPT_IN, opt_in) + if settings.MARKETING_EMAILS_OPT_IN and user: + UserAttribute.set_user_attribute(user, IS_MARKETABLE, str(is_marketable).lower()) def _record_registration_attributions(request, user):