From d0ecae30d7edaa64ae24b383c532a34faebd03d3 Mon Sep 17 00:00:00 2001 From: Ashley Penney Date: Thu, 17 Jan 2013 16:24:44 -0500 Subject: [PATCH] Fix datadog logging to use .name on the file objects and add an additional logging line. --- .../commands/pearson_import_conf_zip.py | 26 ++++++++++--------- .../management/commands/pearson_transfer.py | 21 +++++++++------ 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/common/djangoapps/student/management/commands/pearson_import_conf_zip.py b/common/djangoapps/student/management/commands/pearson_import_conf_zip.py index a491a73868..fa9741dc68 100644 --- a/common/djangoapps/student/management/commands/pearson_import_conf_zip.py +++ b/common/djangoapps/student/management/commands/pearson_import_conf_zip.py @@ -33,7 +33,9 @@ class Command(BaseCommand): source_zip = args[0] if not is_zipfile(source_zip): - raise CommandError("Input file is not a zipfile: \"{}\"".format(source_zip)) + error = "Input file is not a zipfile: \"{}\"".format(source_zip) + Command.datadog_error(error, source_zip) + raise CommandError(error) # loop through all files in zip, and process them based on filename prefix: with ZipFile(source_zip, 'r') as zipfile: @@ -55,9 +57,9 @@ class Command(BaseCommand): client_authorization_id = row['ClientAuthorizationID'] if not client_authorization_id: if row['Status'] == 'Error': - Command.datadog_error("Error in EAD file processing ({}): {}".format(row['Date'], row['Message']), eacfile) + Command.datadog_error("Error in EAD file processing ({}): {}".format(row['Date'], row['Message']), eacfile.name) else: - Command.datadog_error("Encountered bad record: {}".format(row), eacfile) + Command.datadog_error("Encountered bad record: {}".format(row), eacfile.name) else: try: registration = TestCenterRegistration.objects.get(client_authorization_id=client_authorization_id) @@ -68,18 +70,18 @@ class Command(BaseCommand): try: registration.processed_at = strftime('%Y-%m-%d %H:%M:%S', strptime(row['Date'], '%Y/%m/%d %H:%M:%S')) except ValueError as ve: - Command.datadog_error("Bad Date value found for {}: message {}".format(client_authorization_id, ve), eacfile) + Command.datadog_error("Bad Date value found for {}: message {}".format(client_authorization_id, ve), eacfile.name) # store the authorization Id if one is provided. (For debugging) if row['AuthorizationID']: try: registration.authorization_id = int(row['AuthorizationID']) except ValueError as ve: - Command.datadog_error("Bad AuthorizationID value found for {}: message {}".format(client_authorization_id, ve), eacfile) + Command.datadog_error("Bad AuthorizationID value found for {}: message {}".format(client_authorization_id, ve), eacfile.name) registration.confirmed_at = datetime.utcnow() registration.save() except TestCenterRegistration.DoesNotExist: - Command.datadog_error("Failed to find record for client_auth_id {}".format(client_authorization_id), eacfile) + Command.datadog_error("Failed to find record for client_auth_id {}".format(client_authorization_id), eacfile.name) def process_vcdc(self, vcdcfile): print "processing vcdc" @@ -88,28 +90,28 @@ class Command(BaseCommand): client_candidate_id = row['ClientCandidateID'] if not client_candidate_id: if row['Status'] == 'Error': - Command.datadog_error("Error in CDD file processing ({}): {}".format(row['Date'], row['Message']), vcdcfile) + Command.datadog_error("Error in CDD file processing ({}): {}".format(row['Date'], row['Message']), vcdcfile.name) else: - Command.datadog_error("Encountered bad record: {}".format(row), vcdcfile) + Command.datadog_error("Encountered bad record: {}".format(row), vcdcfile.name) else: try: tcuser = TestCenterUser.objects.get(client_candidate_id=client_candidate_id) - Command.datadog_error("Found demographics record for user {}".format(tcuser.user.username), vcdcfile) + Command.datadog_error("Found demographics record for user {}".format(tcuser.user.username), vcdcfile.name) # now update the record: tcuser.upload_status = row['Status'] tcuser.upload_error_message = row['Message'] try: tcuser.processed_at = strftime('%Y-%m-%d %H:%M:%S', strptime(row['Date'], '%Y/%m/%d %H:%M:%S')) except ValueError as ve: - Command.datadog_error("Bad Date value found for {}: message {}".format(client_candidate_id, ve), vcdcfile) + Command.datadog_error("Bad Date value found for {}: message {}".format(client_candidate_id, ve), vcdcfile.name) # store the candidate Id if one is provided. (For debugging) if row['CandidateID']: try: tcuser.candidate_id = int(row['CandidateID']) except ValueError as ve: - Command.datadog_error("Bad CandidateID value found for {}: message {}".format(client_candidate_id, ve), vcdcfile) + Command.datadog_error("Bad CandidateID value found for {}: message {}".format(client_candidate_id, ve), vcdcfile.name) tcuser.confirmed_at = datetime.utcnow() tcuser.save() except TestCenterUser.DoesNotExist: - Command.datadog_error(" Failed to find record for client_candidate_id {}".format(client_candidate_id), vcdcfile) + Command.datadog_error(" Failed to find record for client_candidate_id {}".format(client_candidate_id), vcdcfile.name) diff --git a/common/djangoapps/student/management/commands/pearson_transfer.py b/common/djangoapps/student/management/commands/pearson_transfer.py index aede8fe92d..5f126a24f0 100644 --- a/common/djangoapps/student/management/commands/pearson_transfer.py +++ b/common/djangoapps/student/management/commands/pearson_transfer.py @@ -39,14 +39,18 @@ class Command(BaseCommand): '(env/auth.json) for {0}'.format(value)) def import_pearson(): - sftp(settings.PEARSON['SFTP_IMPORT'], - settings.PEARSON['LOCAL_IMPORT'], options['mode']) - s3(settings.PEARSON['LOCAL_IMPORT'], - settings.PEARSON['BUCKET'], options['mode']) - for file in os.listdir(settings.PEARSON['LOCAL_IMPORT']): - call_command('pearson_import_conf_zip', - settings.PEARSON['LOCAL_IMPORT'] + '/' + file) - os.remove(file) + try: + sftp(settings.PEARSON['SFTP_IMPORT'], + settings.PEARSON['LOCAL_IMPORT'], options['mode']) + s3(settings.PEARSON['LOCAL_IMPORT'], + settings.PEARSON['BUCKET'], options['mode']) + except Exception as e: + dog_http_api.event('Pearson Import failure', str(e)) + else: + for file in os.listdir(settings.PEARSON['LOCAL_IMPORT']): + call_command('pearson_import_conf_zip', + settings.PEARSON['LOCAL_IMPORT'] + '/' + file) + os.remove(file) def export_pearson(): call_command('pearson_export_cdd', 'dest_from_settings') @@ -79,6 +83,7 @@ class Command(BaseCommand): for filename in sftp.listdir(files_from): sftp.get(files_from + '/' + filename, files_to + '/' + filename) + sftp.remove(files_from + '/' + filename) t.close() except: dog_http_api.event('pearson {0}'.format(mode),