Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix datadog logging to use .name on the file objects and add an additional logging line.

Browse Source
This commit is contained in:
Ashley Penney
2013-01-17 16:24:44 -05:00
parent 274cb8d865
commit d0ecae30d7
2 changed files with 27 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
common/djangoapps/student/management/commands/pearson_import_conf_zip.py
View File

@@ -33,7 +33,9 @@ class Command(BaseCommand):
source_zip = args[0]
if not is_zipfile(source_zip):
raise CommandError("Input file is not a zipfile: \"{}\"".format(source_zip))
error = "Input file is not a zipfile: \"{}\"".format(source_zip)
Command.datadog_error(error, source_zip)
raise CommandError(error)
# loop through all files in zip, and process them based on filename prefix:
with ZipFile(source_zip, 'r') as zipfile:
@@ -55,9 +57,9 @@ class Command(BaseCommand):
client_authorization_id = row['ClientAuthorizationID']
if not client_authorization_id:
if row['Status'] == 'Error':
Command.datadog_error("Error in EAD file processing ({}): {}".format(row['Date'], row['Message']), eacfile)
Command.datadog_error("Error in EAD file processing ({}): {}".format(row['Date'], row['Message']), eacfile.name)
else:
Command.datadog_error("Encountered bad record: {}".format(row), eacfile)
Command.datadog_error("Encountered bad record: {}".format(row), eacfile.name)
else:
try:
registration = TestCenterRegistration.objects.get(client_authorization_id=client_authorization_id)
@@ -68,18 +70,18 @@ class Command(BaseCommand):
try:
registration.processed_at = strftime('%Y-%m-%d %H:%M:%S', strptime(row['Date'], '%Y/%m/%d %H:%M:%S'))
except ValueError as ve:
Command.datadog_error("Bad Date value found for {}: message {}".format(client_authorization_id, ve), eacfile)
Command.datadog_error("Bad Date value found for {}: message {}".format(client_authorization_id, ve), eacfile.name)
# store the authorization Id if one is provided. (For debugging)
if row['AuthorizationID']:
try:
registration.authorization_id = int(row['AuthorizationID'])
except ValueError as ve:
Command.datadog_error("Bad AuthorizationID value found for {}: message {}".format(client_authorization_id, ve), eacfile)
Command.datadog_error("Bad AuthorizationID value found for {}: message {}".format(client_authorization_id, ve), eacfile.name)
registration.confirmed_at = datetime.utcnow()
registration.save()
except TestCenterRegistration.DoesNotExist:
Command.datadog_error("Failed to find record for client_auth_id {}".format(client_authorization_id), eacfile)
Command.datadog_error("Failed to find record for client_auth_id {}".format(client_authorization_id), eacfile.name)
def process_vcdc(self, vcdcfile):
print "processing vcdc"
@@ -88,28 +90,28 @@ class Command(BaseCommand):
client_candidate_id = row['ClientCandidateID']
if not client_candidate_id:
if row['Status'] == 'Error':
Command.datadog_error("Error in CDD file processing ({}): {}".format(row['Date'], row['Message']), vcdcfile)
Command.datadog_error("Error in CDD file processing ({}): {}".format(row['Date'], row['Message']), vcdcfile.name)
else:
Command.datadog_error("Encountered bad record: {}".format(row), vcdcfile)
Command.datadog_error("Encountered bad record: {}".format(row), vcdcfile.name)
else:
try:
tcuser = TestCenterUser.objects.get(client_candidate_id=client_candidate_id)
Command.datadog_error("Found demographics record for user {}".format(tcuser.user.username), vcdcfile)
Command.datadog_error("Found demographics record for user {}".format(tcuser.user.username), vcdcfile.name)
# now update the record:
tcuser.upload_status = row['Status']
tcuser.upload_error_message = row['Message']
try:
tcuser.processed_at = strftime('%Y-%m-%d %H:%M:%S', strptime(row['Date'], '%Y/%m/%d %H:%M:%S'))
except ValueError as ve:
Command.datadog_error("Bad Date value found for {}: message {}".format(client_candidate_id, ve), vcdcfile)
Command.datadog_error("Bad Date value found for {}: message {}".format(client_candidate_id, ve), vcdcfile.name)
# store the candidate Id if one is provided. (For debugging)
if row['CandidateID']:
try:
tcuser.candidate_id = int(row['CandidateID'])
except ValueError as ve:
Command.datadog_error("Bad CandidateID value found for {}: message {}".format(client_candidate_id, ve), vcdcfile)
Command.datadog_error("Bad CandidateID value found for {}: message {}".format(client_candidate_id, ve), vcdcfile.name)
tcuser.confirmed_at = datetime.utcnow()
tcuser.save()
except TestCenterUser.DoesNotExist:
Command.datadog_error(" Failed to find record for client_candidate_id {}".format(client_candidate_id), vcdcfile)
Command.datadog_error(" Failed to find record for client_candidate_id {}".format(client_candidate_id), vcdcfile.name)

21
common/djangoapps/student/management/commands/pearson_transfer.py
View File

@@ -39,14 +39,18 @@ class Command(BaseCommand):
'(env/auth.json) for {0}'.format(value))
def import_pearson():
sftp(settings.PEARSON['SFTP_IMPORT'],
settings.PEARSON['LOCAL_IMPORT'], options['mode'])
s3(settings.PEARSON['LOCAL_IMPORT'],
settings.PEARSON['BUCKET'], options['mode'])
for file in os.listdir(settings.PEARSON['LOCAL_IMPORT']):
call_command('pearson_import_conf_zip',
settings.PEARSON['LOCAL_IMPORT'] + '/' + file)
os.remove(file)
try:
sftp(settings.PEARSON['SFTP_IMPORT'],
settings.PEARSON['LOCAL_IMPORT'], options['mode'])
s3(settings.PEARSON['LOCAL_IMPORT'],
settings.PEARSON['BUCKET'], options['mode'])
except Exception as e:
dog_http_api.event('Pearson Import failure', str(e))
else:
for file in os.listdir(settings.PEARSON['LOCAL_IMPORT']):
call_command('pearson_import_conf_zip',
settings.PEARSON['LOCAL_IMPORT'] + '/' + file)
os.remove(file)
def export_pearson():
call_command('pearson_export_cdd', 'dest_from_settings')
@@ -79,6 +83,7 @@ class Command(BaseCommand):
for filename in sftp.listdir(files_from):
sftp.get(files_from + '/' + filename,
files_to + '/' + filename)
sftp.remove(files_from + '/' + filename)
t.close()
except:
dog_http_api.event('pearson {0}'.format(mode),