Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: remove unused management commands (#30861)

Browse Source 
Removed two commands:
- populate_is_marketable_user_attribute
- populate_user_data_on_braze

VAN-971
This commit is contained in:
Zainab Amir
2022-08-16 15:10:20 +05:00
committed by GitHub
parent 7313803e39
commit f6e886b027
4 changed files with 0 additions and 505 deletions
Download Patch File Download Diff File Expand all files Collapse all files

173
common/djangoapps/student/management/commands/populate_is_marketable_user_attribute.py
View File

@@ -1,173 +0,0 @@
""" Management command to back-populate marketing emails opt-in for the user accounts. """
import time
from django.contrib.auth import get_user_model
from django.core.management.base import BaseCommand
from django.db import IntegrityError
from django.db.models import Exists, OuterRef
from common.djangoapps.student.models import UserAttribute
from common.djangoapps.util.query import use_read_replica_if_available
OLD_USER_ATTRIBUTE_NAME = 'marketing_emails_opt_in'
NEW_USER_ATTRIBUTE_NAME = 'is_marketable'
class Command(BaseCommand):
"""
Example usage:
$ ./manage.py lms populate_is_marketable_user_attribute
"""
help = """
Creates a row in the UserAttribute table for all users in the platform.
This command back-populates the 'is_marketable' attribute in the
UserAttribute table for the user accounts.
"""
def add_arguments(self, parser):
parser.add_argument(
'--batch-delay',
type=float,
dest='batch_delay',
default=0.5,
help='Time delay in each iteration'
)
parser.add_argument(
'--batch-size',
type=int,
dest='batch_size',
default=10000,
help='Batch size'
)
parser.add_argument(
'--start-user-id',
type=int,
dest='start_user_id',
default=10000,
help='Starting user ID'
)
parser.add_argument(
'--start-userattribute-id',
type=int,
dest='start_userattribute_id',
default=10000,
help='Starting user attribute ID'
)
parser.add_argument(
'--backfill-only',
type=str,
dest='backfill_only',
default=None,
help='Only backfill user attribute, renaming attribute is not required'
)
def _update_user_attribute(self, start_id, end_id):
""" Updates the user attributes in batches. """
self.stdout.write(
f'Updating user attribute starting from ID {start_id} till {end_id}.'
)
return UserAttribute.objects.filter(
id__gte=start_id,
id__lt=end_id,
name=OLD_USER_ATTRIBUTE_NAME
).order_by('id').update(name=NEW_USER_ATTRIBUTE_NAME)
def _get_old_users_queryset(self, batch_size, marketing_opt_in_start_user_id, user_id=0):
"""
Fetches all the old users in batches, in ascending order of id, that exist before a specified user id.
Returns queryset of tuples with 'id' and 'is_active' field values.
"""
self.stdout.write(f'Fetching old users in batch starting from ID {user_id} with batch size {batch_size}.')
query_set = get_user_model().objects.filter(
id__gt=user_id,
id__lt=marketing_opt_in_start_user_id
).values_list('id', 'is_active').order_by('id')[:batch_size]
return use_read_replica_if_available(query_set)
def _get_recent_users_queryset(self, batch_size, user_id):
"""
Fetches all the recent users in batches, in ascending order of id, that exist after a specified user id
and does not have 'is_marketable' user attribute set.
Returns queryset of tuples with 'id' and 'is_active' field values.
"""
self.stdout.write(f'Fetching recent users in batch starting from ID {user_id} with batch size {batch_size}.')
user_attribute_qs = UserAttribute.objects.filter(user=OuterRef('pk'), name=NEW_USER_ATTRIBUTE_NAME)
user_query_set = get_user_model().objects.filter(
~Exists(user_attribute_qs),
id__gt=user_id,
).values_list('id', 'is_active').order_by('id')[:batch_size]
return use_read_replica_if_available(user_query_set)
def _bulk_create_user_attributes(self, users):
""" Creates the UserAttribute records in bulk. """
last_user_id = 0
user_attributes = []
for user in users:
user_attributes.append(UserAttribute(
user_id=user[0],
name=NEW_USER_ATTRIBUTE_NAME,
value=str(user[1]).lower()
))
last_user_id = user[0]
try:
UserAttribute.objects.bulk_create(user_attributes)
except IntegrityError:
# A UserAttribute object was already created. This could only happen if we try to create 'is_marketable'
# user attribute that is already created. Ignore it if it does.
self.stdout.write(f'IntegrityError raised during bulk_create having last user id: {last_user_id}.')
return last_user_id
def _backfill_old_users_attribute(self, batch_size, batch_delay, marketing_opt_in_start_user_id):
"""
Backfills the is_marketable user attribute. Fetches all the old user accounts, in ascending order of id,
that were created before a specified user id. All the fetched users do not have 'is_marketable'
user attribute set.
"""
users = self._get_old_users_queryset(batch_size, marketing_opt_in_start_user_id)
while users:
last_user_id = self._bulk_create_user_attributes(users)
time.sleep(batch_delay)
users = self._get_old_users_queryset(batch_size, marketing_opt_in_start_user_id, last_user_id)
def _backfill_recent_users_attribute(self, batch_size, batch_delay, start_user_id):
"""
Backfills the is_marketable user attribute. Fetches all the recent user accounts, in ascending order of id,
that were created after a specified user id (start_user_id).
This method handles the backfill of all those users that have missing user attribute even after enabling
the MARKETING_EMAILS_OPT_IN flag.
"""
users = self._get_recent_users_queryset(batch_size, start_user_id)
while users:
last_user_id = self._bulk_create_user_attributes(users)
time.sleep(batch_delay)
users = self._get_recent_users_queryset(batch_size, last_user_id)
def _rename_user_attribute_name(self, batch_size, batch_delay, start_user_attribute_id):
""" Renames the old user attribute 'marketing_emails_opt_in' to 'is_marketable'. """
updated_records_count = 0
start_id = start_user_attribute_id
end_id = start_id + batch_size
total_user_attribute_count = UserAttribute.objects.filter(name=OLD_USER_ATTRIBUTE_NAME).count()
while updated_records_count < total_user_attribute_count:
updated_records_count += self._update_user_attribute(start_id, end_id)
start_id = end_id
end_id = start_id + batch_size
time.sleep(batch_delay)
def handle(self, *args, **options):
"""
This command back-populates the 'is_marketable' attribute for all existing users who do not already
have the attribute set.
"""
batch_delay = options['batch_delay']
batch_size = options['batch_size']
self.stdout.write(f'Command execution started with options: {options}.')
if not options['backfill_only']:
self._rename_user_attribute_name(batch_size, batch_delay, options['start_userattribute_id'])
self._backfill_old_users_attribute(batch_size, batch_delay, options['start_user_id'])
self._backfill_recent_users_attribute(batch_size, batch_delay, options['start_user_id'])
self.stdout.write('Command executed successfully.')

164
common/djangoapps/student/management/commands/populate_user_data_on_braze.py
View File

@@ -1,164 +0,0 @@
""" Management command to add user data on Braze. """
import logging
import time
from braze.client import BrazeClient
from braze.exceptions import BrazeClientError
from django.conf import settings
from django.contrib.auth import get_user_model
from django.core.management.base import BaseCommand
from common.djangoapps.student.models import UserAttribute
from common.djangoapps.util.query import use_read_replica_if_available
User = get_user_model()
MARKETING_EMAIL_ATTRIBUTE_NAME = 'is_marketable'
TRACK_USER_COMPONENT_CHUNK_SIZE = 75
logger = logging.getLogger(__name__)
class Command(BaseCommand):
"""
Command to add is_marketable field on Braze against a user_id. If an account
with user_id doesn't exist on Braze, it automatically creates one.
Example usage:
$ ./manage.py lms backfill_user_data_on_braze
"""
help = """
Creates user accounts on Braze for missing users and adds the is_marketable
user attribute to all user accounts on Braze.
"""
def add_arguments(self, parser):
"""
Function to get command arguments
"""
parser.add_argument(
'--batch-delay',
type=float,
dest='batch_delay',
default=0.5,
help='Time delay in seconds between each iteration'
)
parser.add_argument(
'--batch-size',
type=int,
dest='batch_size',
default=10000,
help='Batch size'
)
parser.add_argument(
'--starting-user-id',
type=int,
dest='starting_user_id',
default=0,
help='Starting user id to process a specific batch of users. '
'Both start and end id should be provided.',
)
parser.add_argument(
'--ending-user-id',
type=int,
dest='ending_user_id',
default=0,
help='Ending user id (inclusive) to process a specific batch of users. '
'Both start and end id should be provided.',
)
def __init__(self):
super().__init__()
self.braze_client = BrazeClient(
api_key=settings.EDX_BRAZE_API_KEY,
api_url=settings.EDX_BRAZE_API_SERVER,
app_id='',
)
@staticmethod
def _chunks(users, chunk_size=TRACK_USER_COMPONENT_CHUNK_SIZE):
"""
Yields successive chunks of users. The size of each chunk is determined by
TRACK_USER_COMPONENT_CHUNK_SIZE which is set to 75.
Reference: https://www.braze.com/docs/api/endpoints/user_data/post_user_track/
"""
for index in range(0, len(users), chunk_size):
yield users[index:index + chunk_size]
def _get_user_batch(self, batch_start_id, batch_end_id):
"""
This returns the batch of users.
"""
query = User.objects.filter(
id__gte=batch_start_id, id__lt=batch_end_id,
).select_related('profile').values_list(
'id', 'email', 'profile__name', 'username', named=True,
).order_by('id')
return use_read_replica_if_available(query)
def _get_marketable_users(self, batch_start_id, batch_end_id):
"""
For a given batch of users it gives the marketable users list
"""
get_marketable_users_query = UserAttribute.objects.filter(
user_id__gte=batch_start_id,
user_id__lt=batch_end_id,
name=MARKETING_EMAIL_ATTRIBUTE_NAME,
value__in=['True', 'true', '1'],
).values_list('user_id', flat=True).order_by('user_id')
return set(use_read_replica_if_available(get_marketable_users_query))
def _update_braze_attributes(self, users, marketable_users):
"""
Sends Braze API request to update/create user account.
Fields sent using the API include:
- external_id (user_id)
- email
- name
- username
- is_marketable
"""
attributes = []
for user in users:
attributes.append(
{
"external_id": user.id,
"email": user.email,
"name": user.profile__name,
"username": user.username,
"is_marketable": user.id in marketable_users,
}
)
try:
self.braze_client.track_user(attributes=attributes)
except BrazeClientError as error:
logger.error(f'Failed to update attributes. Error: {error}')
def handle(self, *args, **options):
"""
Handler to run the command.
"""
sleep_time = options['batch_delay']
batch_size = options['batch_size']
starting_user_id = options['starting_user_id']
ending_user_id = options['ending_user_id']
all_users_query = use_read_replica_if_available(User.objects)
total_users_count = ending_user_id if ending_user_id else all_users_query.count()
for index in range(starting_user_id, total_users_count + 1, batch_size):
users = self._get_user_batch(index, index + batch_size)
logger.info(f'Processing users with user ids in {index} - {(index + batch_size) - 1} range')
# Force evaluating the query to avoid multiple hits to db
# when we evaluate the chunks.
evaluated_users = list(users)
marketable_users = self._get_marketable_users(index, index + batch_size)
for user_chunk in self._chunks(evaluated_users):
self._update_braze_attributes(user_chunk, marketable_users)
time.sleep(sleep_time)

75
common/djangoapps/student/management/tests/test_populate_is_marketable_user_attribute.py
View File

@@ -1,75 +0,0 @@
"""
Unittests for populate_marketing_opt_in_user_attribute management command.
"""
import pytest
from django.contrib.auth.models import User # lint-amnesty, pylint: disable=imported-auth-user
from django.core.management import call_command
from django.test import TransactionTestCase
from common.djangoapps.student.models import UserAttribute
from common.djangoapps.student.tests.factories import UserFactory
from openedx.core.djangolib.testing.utils import skip_unless_lms
IS_MARKETABLE = 'is_marketable'
@skip_unless_lms
class TestPopulateMarketingOptInUserAttribute(TransactionTestCase):
"""
Test populate_is_marketable_user_attribute management command.
"""
def setUp(self):
super().setUp()
self.existing_user = UserFactory()
def test_command_with_existing_users(self):
"""
Test population of is_marketable attribute with an existing user.
"""
assert UserAttribute.objects.count() == 0
call_command(
'populate_is_marketable_user_attribute',
start_user_id=1,
start_userattribute_id=1
)
assert UserAttribute.objects.filter(name=IS_MARKETABLE).count() == User.objects.count()
def test_command_with_new_user(self):
"""
Test population of is_marketable attribute with a new user.
"""
user = UserFactory()
call_command(
'populate_is_marketable_user_attribute',
start_user_id=0,
start_userattribute_id=1
)
assert UserAttribute.objects.filter(name=IS_MARKETABLE).count() == User.objects.count()
def test_command_rename_to_new_attribute(self):
"""
Test renaming of marketing_emails_opt_in to is_marketable attribute.
"""
user = UserFactory()
UserAttribute.objects.create(user=user, name='marketing_emails_opt_in', value='true')
call_command(
'populate_is_marketable_user_attribute',
start_user_id=1,
start_userattribute_id=1
)
assert UserAttribute.objects.filter(name='marketing_emails_opt_in').count() == 0
assert UserAttribute.get_user_attribute(user, IS_MARKETABLE) == 'true'
def test_command_with_invalid_argument(self):
"""
Test management command raises TypeError on wrong data type value for '--batch-size' argument.
"""
with pytest.raises(TypeError):
call_command(
'populate_is_marketable_user_attribute',
batch_size='1000',
start_user_id=1,
start_userattribute_id=1
)

93
common/djangoapps/student/management/tests/test_populate_user_data_on_braze.py
View File

@@ -1,93 +0,0 @@
"""
Unittests for populate_marketing_opt_in_user_attribute management command.
"""
from unittest.mock import patch, MagicMock
from braze.exceptions import BrazeClientError
from django.core.management import call_command
from django.test import TestCase
from testfixtures import LogCapture
from common.djangoapps.student.management.commands.populate_user_data_on_braze import MARKETING_EMAIL_ATTRIBUTE_NAME
from common.djangoapps.student.tests.factories import UserFactory, UserAttributeFactory
from openedx.core.djangolib.testing.utils import skip_unless_lms
LOGGER_NAME = 'common.djangoapps.student.management.commands.populate_user_data_on_braze'
@skip_unless_lms
class TestPopulateUserDataOnBraze(TestCase):
"""
Tests for PopulateUserDataOnBraze management command.
"""
@classmethod
def setUpClass(cls):
super().setUpClass()
for index in range(15):
user = UserFactory()
UserAttributeFactory(user=user, name=MARKETING_EMAIL_ATTRIBUTE_NAME, value=(index % 2 == 0))
@patch('common.djangoapps.student.management.commands.populate_user_data_on_braze.BrazeClient.track_user')
def test_command_updates_users_on_braze(self, track_user):
"""
Test that Braze API is called successfully for all users
"""
track_user.return_value = MagicMock()
call_command('populate_user_data_on_braze', batch_delay=0)
assert track_user.called
@patch('common.djangoapps.student.management.commands.populate_user_data_on_braze.BrazeClient.track_user')
def test_logs_for_success(self, track_user):
"""
Test logs for a successful run of updating user accounts on Braze
"""
track_user.return_value = MagicMock()
with LogCapture(LOGGER_NAME) as log:
call_command(
'populate_user_data_on_braze',
batch_size=5,
batch_delay=0,
)
log.check(
(LOGGER_NAME, 'INFO', 'Processing users with user ids in 0 - 4 range'),
(LOGGER_NAME, 'INFO', 'Processing users with user ids in 5 - 9 range'),
(LOGGER_NAME, 'INFO', 'Processing users with user ids in 10 - 14 range'),
(LOGGER_NAME, 'INFO', 'Processing users with user ids in 15 - 19 range'),
)
@patch('common.djangoapps.student.management.commands.populate_user_data_on_braze.BrazeClient.track_user')
def test_logs_for_failure(self, track_user):
"""
Test logs for when the update to Braze fails
"""
track_user.side_effect = BrazeClientError('Update to attributes failed.')
with LogCapture(LOGGER_NAME) as log:
call_command(
'populate_user_data_on_braze',
batch_size=5,
batch_delay=0,
)
log.check_present(
(LOGGER_NAME, 'ERROR', 'Failed to update attributes. Error: Update to attributes failed.'),
)
@patch('common.djangoapps.student.management.commands.populate_user_data_on_braze.BrazeClient.track_user')
def test_running_a_specific_batch(self, track_user):
"""
Test running command for a specific batch of users
"""
track_user.return_value = MagicMock()
with LogCapture(LOGGER_NAME) as log:
call_command(
'populate_user_data_on_braze',
batch_size=5,
batch_delay=0,
starting_user_id=2,
ending_user_id=13,
)
log.check(
(LOGGER_NAME, 'INFO', 'Processing users with user ids in 2 - 6 range'),
(LOGGER_NAME, 'INFO', 'Processing users with user ids in 7 - 11 range'),
(LOGGER_NAME, 'INFO', 'Processing users with user ids in 12 - 16 range'),
)