Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: batch verifications when creating in bulk (#29119)

Browse Source 
batch verifications when creating in bulk
This commit is contained in:
Zachary Hancock
2021-10-27 10:12:02 -04:00
committed by GitHub
parent 09778cf9f9
commit 249f4e6fbf
2 changed files with 32 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
lms/djangoapps/verify_student/management/commands/manual_verifications.py
View File

@@ -39,6 +39,12 @@ class Command(BaseCommand):
help='Single email to verify one user',
type=str,
)
parser.add_argument(
'--batch-size',
default=None,
help='(Optional) Maximum records to write in one transaction.',
type=int,
)
def handle(self, *args, **options):
@@ -51,12 +57,13 @@ class Command(BaseCommand):
return
email_ids_file = options['email_ids_file']
batch_size = options['batch_size']
if email_ids_file:
if not os.path.exists(email_ids_file):
raise CommandError('Pass the correct absolute path to email ids file as --email-ids-file argument.')
total_emails, failed_emails = self._generate_manual_verification_from_file(email_ids_file)
total_emails, failed_emails = self._generate_manual_verification_from_file(email_ids_file, batch_size)
if failed_emails:
log.error('Completed manual verification. {} of {} failed.'.format(
@@ -67,28 +74,42 @@ class Command(BaseCommand):
else:
log.info(f'Successfully generated manual verification for {total_emails} emails.')
def _generate_manual_verification_from_file(self, email_ids_file):
def _generate_manual_verification_from_file(self, email_ids_file, batch_size=None):
"""
Generate manual verification for the emails provided in the email ids file.
Arguments:
email_ids_file (str): path of the file containing email ids.
batch_size (int): limits the number of verifications written to db at once
Returns:
(total_emails, failed_emails): a tuple containing count of emails processed and a list containing
emails whose verifications could not be processed.
"""
failed_emails = []
with open(email_ids_file) as file_handler:
email_ids = file_handler.readlines()
email_ids = [line.rstrip() for line in file_handler]
total_emails = len(email_ids)
log.info(f'Creating manual verification for {total_emails} emails.')
for email_id in email_ids:
successfully_verified = self._add_user_to_manual_verification(email_id)
if successfully_verified is False:
failed_emails.append(email_id)
return total_emails, failed_emails
log.info(f'Creating manual verification for {total_emails} emails.')
verifications_to_create = []
users = User.objects.filter(email__in=email_ids)
user_existing_verification = {v.user.id for v in ManualVerification.objects.filter(
user__in=users,
status='approved',
created_at__gte=earliest_allowed_verification_date(),
)}
for user in users:
if user.id not in user_existing_verification:
verifications_to_create.append(ManualVerification(
user=user,
name=user.profile.name,
status='approved',
))
else:
log.info(f'Skipping email {user.email}, existing verification found.')
ManualVerification.objects.bulk_create(verifications_to_create, batch_size)
failed_emails = set(email_ids) - set(users.values_list('email', flat=True))
return total_emails, list(failed_emails)
def _add_user_to_manual_verification(self, email_id):
"""
@@ -101,7 +122,6 @@ class Command(BaseCommand):
(success): boolean to show if the user has been successfully verified.
"""
try:
email_id = email_id.strip()
user = User.objects.get(email=email_id)
ManualVerification.objects.get_or_create(
user=user,

4
lms/djangoapps/verify_student/management/commands/tests/test_manual_verify_student.py
View File

@@ -89,10 +89,6 @@ class TestVerifyStudentCommand(TestCase):
'INFO',
'Creating manual verification for 4 emails.'
),
(LOGGER_NAME,
'ERROR',
'Tried to verify email unknown@unknown.com, but user not found'
),
(LOGGER_NAME,
'ERROR',
'Completed manual verification. 1 of 4 failed.'