diff --git a/common/djangoapps/student/management/commands/pearson_export_cdd.py b/common/djangoapps/student/management/commands/pearson_export_cdd.py index 2f68ccc7d1..14c652f11e 100644 --- a/common/djangoapps/student/management/commands/pearson_export_cdd.py +++ b/common/djangoapps/student/management/commands/pearson_export_cdd.py @@ -1,15 +1,16 @@ import csv +import os from collections import OrderedDict from datetime import datetime -from os.path import isdir from optparse import make_option from django.core.management.base import BaseCommand from student.models import TestCenterUser + class Command(BaseCommand): - + CSV_TO_MODEL_FIELDS = OrderedDict([ # Skipping optional field CandidateID ("ClientCandidateID", "client_candidate_id"), @@ -34,7 +35,7 @@ class Command(BaseCommand): ("FAXCountryCode", "fax_country_code"), ("CompanyName", "company_name"), # Skipping optional field CustomQuestion - ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store + ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store ]) option_list = BaseCommand.option_list + ( @@ -55,29 +56,28 @@ class Command(BaseCommand): # field uploaded_at = datetime.utcnow() - # if specified destination is an existing directory, then + # if specified destination is an existing directory, then # create a filename for it automatically. If it doesn't exist, # then we will create the directory. # Name will use timestamp -- this is UTC, so it will look funny, - # but it should at least be consistent with the other timestamps + # but it should at least be consistent with the other timestamps # used in the system. if 'dest-from-settings' in options: - if LOCAL_EXPORT in settings.PEARSON: - dest = settings.PEARSON[LOCAL_EXPORT] + if 'LOCAL_EXPORT' in settings.PEARSON: + dest = settings.PEARSON['LOCAL_EXPORT'] else: raise CommandError('--dest-from-settings was enabled but the' - 'PEARSON[LOCAL_EXPORT] setting was not set.') + 'PEARSON[LOCAL_EXPORT] setting was not set.') elif 'destination' in options: dest = options['destination'] else: raise CommandError('--destination or --dest-from-settings must be used') - if not os.path.isdir(dest): os.makedirs(dest) destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat")) - + # strings must be in latin-1 format. CSV parser will # otherwise convert unicode objects to ascii. def ensure_encoding(value): @@ -85,8 +85,8 @@ class Command(BaseCommand): return value.encode('iso-8859-1') else: return value - - dump_all = kwargs['dump_all'] + + dump_all = options['dump_all'] with open(destfile, "wb") as outfile: writer = csv.DictWriter(outfile, @@ -104,6 +104,3 @@ class Command(BaseCommand): writer.writerow(record) tcu.uploaded_at = uploaded_at tcu.save() - - - diff --git a/common/djangoapps/student/management/commands/pearson_export_ead.py b/common/djangoapps/student/management/commands/pearson_export_ead.py index 0f81cb1df9..9368ac5ddf 100644 --- a/common/djangoapps/student/management/commands/pearson_export_ead.py +++ b/common/djangoapps/student/management/commands/pearson_export_ead.py @@ -1,15 +1,16 @@ import csv +import os from collections import OrderedDict from datetime import datetime -from os.path import isdir, join from optparse import make_option from django.core.management.base import BaseCommand from student.models import TestCenterRegistration + class Command(BaseCommand): - + CSV_TO_MODEL_FIELDS = OrderedDict([ ('AuthorizationTransactionType', 'authorization_transaction_type'), ('AuthorizationID', 'authorization_id'), @@ -20,7 +21,7 @@ class Command(BaseCommand): ('Accommodations', 'accommodation_code'), ('EligibilityApptDateFirst', 'eligibility_appointment_date_first'), ('EligibilityApptDateLast', 'eligibility_appointment_date_last'), - ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store + ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store ]) option_list = BaseCommand.option_list + ( @@ -37,11 +38,11 @@ class Command(BaseCommand): make_option('--dump_all', action='store_true', dest='dump_all', - ), + ), make_option('--force_add', action='store_true', dest='force_add', - ), + ), ) def handle(self, **options): @@ -56,12 +57,12 @@ class Command(BaseCommand): # but it should at least be consistent with the other timestamps # used in the system. if 'dest-from-settings' in options: - if LOCAL_EXPORT in settings.PEARSON: - dest = settings.PEARSON[LOCAL_EXPORT] + if 'LOCAL_EXPORT' in settings.PEARSON: + dest = settings.PEARSON['LOCAL_EXPORT'] else: raise CommandError('--dest-from-settings was enabled but the' - 'PEARSON[LOCAL_EXPORT] setting was not set.') - elif destinations in options: + 'PEARSON[LOCAL_EXPORT] setting was not set.') + elif 'destinations' in options: dest = options['destination'] else: raise CommandError('--destination or --dest-from-settings must be used') @@ -71,7 +72,7 @@ class Command(BaseCommand): destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat")) - dump_all = kwargs['dump_all'] + dump_all = options['dump_all'] with open(destfile, "wb") as outfile: writer = csv.DictWriter(outfile, @@ -88,13 +89,9 @@ class Command(BaseCommand): record["LastUpdate"] = record["LastUpdate"].strftime("%Y/%m/%d %H:%M:%S") record["EligibilityApptDateFirst"] = record["EligibilityApptDateFirst"].strftime("%Y/%m/%d") record["EligibilityApptDateLast"] = record["EligibilityApptDateLast"].strftime("%Y/%m/%d") - if kwargs['force_add']: + if options['force_add']: record['AuthorizationTransactionType'] = 'Add' writer.writerow(record) tcr.uploaded_at = uploaded_at tcr.save() - - - - diff --git a/common/djangoapps/student/management/commands/pearson_transfer.py b/common/djangoapps/student/management/commands/pearson_transfer.py index d07f75d011..8183bf3c05 100644 --- a/common/djangoapps/student/management/commands/pearson_transfer.py +++ b/common/djangoapps/student/management/commands/pearson_transfer.py @@ -1,11 +1,10 @@ from optparse import make_option -from django.contrib.auth.models import User from django.core.management.base import BaseCommand, CommandError -import re from dogapi import dog_http_api, dog_stats_api import paramiko import boto +import os dog_http_api.api_key = settings.DATADOG_API @@ -25,16 +24,26 @@ class Command(BaseCommand): if not settings.PEARSON: raise CommandError('No PEARSON entries in auth/env.json.') + for value in ['LOCAL_IMPORT', 'SFTP_IMPORT', 'BUCKET', 'LOCAL_EXPORT', + 'SFTP_EXPORT']: + if value not in settings.PEARSON: + raise CommandError('No entry in the PEARSON settings' + '(env/auth.json) for {0}'.format(value)) + def import_pearson(): - sftp(settings.PEARSON[SFTP_IMPORT], settings.PEARSON[LOCAL_IMPORT]) - s3(settings.PEARSON[LOCAL_IMPORT], settings.PEARSON[BUCKET]) + sftp(settings.PEARSON['SFTP_IMPORT'], + settings.PEARSON['LOCAL_IMPORT'], options['mode']) + s3(settings.PEARSON['LOCAL_IMPORT'], + settings.PEARSON['BUCKET'], options['mode']) call_command('pearson_import', 'dest_from_settings') def export_pearson(): call_command('pearson_export_ccd', 'dest_from_settings') call_command('pearson_export_ead', 'dest_from_settings') - sftp(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[SFTP_EXPORT]) - s3(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[BUCKET]) + sftp(settings.PEARSON['LOCAL_EXPORT'], + settings.PEARSON['SFTP_EXPORT'], options['mode']) + s3(settings.PEARSON['LOCAL_EXPORT'], + settings.PEARSON['BUCKET'], options['mode']) if options['mode'] == 'export': export_pearson() @@ -44,44 +53,44 @@ class Command(BaseCommand): export_pearson() import_pearson() - - def sftp(files_from, files_to): + def sftp(files_from, files_to, mode): with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'): try: - t = paramiko.Transport((hostname, 22)) - t.connect(username=settings.PEARSON[SFTP_USERNAME], - password=settings.PEARSON[SFTP_PASSWORD]) + t = paramiko.Transport((settings.PEARSON['SFTP_HOSTNAME'], 22)) + t.connect(username=settings.PEARSON['SFTP_USERNAME'], + password=settings.PEARSON['SFTP_PASSWORD']) sftp = paramiko.SFTPClient.from_transport(t) if os.path.isdir(files_from): for filename in os.listdir(files_from): - sftp.put(files_from+'/'+filename, - files_to+'/'+filename) + sftp.put(files_from + '/' + filename, + files_to + '/' + filename) else: for filename in sftp.listdir(files_from): - sftp.get(files_from+'/'+filename, - files_to+'/'+filename) + sftp.get(files_from + '/' + filename, + files_to + '/' + filename) t.close() except: dog_http_api.event('pearson {0}'.format(mode), - 'sftp uploading failed', alert_type='error') + 'sftp uploading failed', + alert_type='error') raise - def s3(files_from, bucket): + def s3(files_from, bucket, mode): with dog_stats_api.timer('pearson.{0}'.format(mode), tags='s3'): try: - for filename in os.listdir(files): - upload_file_to_s3(bucket, files_from+'/'+filename) + for filename in os.listdir(files_from): + upload_file_to_s3(bucket, files_from + '/' + filename) except: - dog_http_api.event('pearson {0}'.format(mode), 's3 archiving failed') + dog_http_api.event('pearson {0}'.format(mode), + 's3 archiving failed') raise - def upload_file_to_s3(bucket, filename): """ Upload file to S3 """ s3 = boto.connect_s3(settings.AWS_ACCESS_KEY_ID, - settings.AWS_SECRET_ACCESS_KEY) + settings.AWS_SECRET_ACCESS_KEY) from boto.s3.key import Key b = s3.get_bucket(bucket) k = Key(b)