diff --git a/common/djangoapps/student/management/commands/pearson.py b/common/djangoapps/student/management/commands/pearson.py new file mode 100644 index 0000000000..2a8b02928c --- /dev/null +++ b/common/djangoapps/student/management/commands/pearson.py @@ -0,0 +1,74 @@ +from optparse import make_option + +from django.contrib.auth.models import User +from django.core.management.base import BaseCommand, CommandError +import re +from dogapi import dog_http_api, dog_stats_api +import paramiko + +dog_http_api.api_key = settings.DATADOG_API + + +class Command(BaseCommand): + + option_list = BaseCommand.option_list + args = '' + help = """ + Mode should be import or export depending on if you're fetching from pearson or + sending to them. + """ + + def handle(self, *args): + if len(args) < 1: + raise CommandError('Usage is pearson {0}'.format(self.args)) + + for mode in args: + if mode == 'export': + sftp(settings.PEARSON_LOCAL_IMPORT, settings.PEARSON_SFTP_IMPORT) + s3(settings.PEARSON_LOCAL, settings.PEARSON_BUCKET) + elif mode == 'import': + sftp(settings.PEARSON_SFTP_EXPORT, settings.PEARSON_LOCAL_EXPORT) + s3(settings.PEARSON_LOCAL_EXPORT, settings.PEARSON_BUCKET) + else: + print("ERROR: Mode must be export or import.") + + def sftp(files_from, files_to): + with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'): + try: + t = paramiko.Transport((hostname, 22)) + t.connect(username=settings.PEARSON_SFTP_USERNAME, + password=settings.PEARSON_SFTP_PASSWORD) + sftp = paramiko.SFTPClient.from_transport(t) + if os.path.isdir(files_from): + for file in os.listdir(files_from): + sftp.put(files_from+'/'+filename, + files_to+'/'+filename) + else: + for file in sftp.listdir(files_from): + sftp.get(files_from+'/'+filename, + files_to+'/'+filename) + except: + dog_http_api.event('pearson {0}'.format(mode), + 'sftp uploading failed', alert_type='error') + raise + + def s3(files_from, bucket): + with dog_stats_api.timer('pearson.{0}'.format(mode), tags='s3'): + try: + for filename in os.listdir(files): + upload_file_to_s3(bucket, files_from+'/'+filename) + except: + dog_http_api.event('pearson {0}'.format(mode), 's3 archiving failed') + raise + + + def upload_file_to_s3(bucket, filename): + """ + Upload file to S3 + """ + s3 = boto.connect_s3() + from boto.s3.key import Key + b = s3.get_bucket(bucket) + k = Key(b) + k.key = "{filename}".format(filename=filename) + k.set_contents_from_filename(filename) diff --git a/common/djangoapps/student/management/commands/pearson_export_cdd.py b/common/djangoapps/student/management/commands/pearson_export_cdd.py index 67230c7f74..cebc098080 100644 --- a/common/djangoapps/student/management/commands/pearson_export_cdd.py +++ b/common/djangoapps/student/management/commands/pearson_export_cdd.py @@ -37,39 +37,25 @@ class Command(BaseCommand): ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store ]) - option_list = BaseCommand.option_list + ( - make_option( - '--dump_all', - action='store_true', - dest='dump_all', - ), - ) - - args = '' - help = """ - Export user demographic information from TestCenterUser model into a tab delimited - text file with a format that Pearson expects. - """ - def handle(self, *args, **kwargs): - if len(args) < 1: - print Command.help - return - + def handle(self, **kwargs): # update time should use UTC in order to be comparable to the user_updated_at # field uploaded_at = datetime.utcnow() # if specified destination is an existing directory, then # create a filename for it automatically. If it doesn't exist, - # or exists as a file, then we will just write to it. + # then we will create the directory. # Name will use timestamp -- this is UTC, so it will look funny, # but it should at least be consistent with the other timestamps # used in the system. - dest = args[0] - if isdir(dest): - destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat")) + if not os.path.isdir(settings.PEARSON_LOCAL_EXPORT): + os.makedirs(settings.PEARSON_LOCAL_EXPORT) + destfile = os.path.join(settings.PEARSON_LOCAL_EXPORT, + uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat")) else: - destfile = dest + destfile = os.path.join(settings.PEARSON_LOCAL_EXPORT, + uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat")) + # strings must be in latin-1 format. CSV parser will # otherwise convert unicode objects to ascii. diff --git a/common/djangoapps/student/management/commands/pearson_export_ead.py b/common/djangoapps/student/management/commands/pearson_export_ead.py index de3bfc04ee..5423282346 100644 --- a/common/djangoapps/student/management/commands/pearson_export_ead.py +++ b/common/djangoapps/student/management/commands/pearson_export_ead.py @@ -23,11 +23,6 @@ class Command(BaseCommand): ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store ]) - args = '' - help = """ - Export user registration information from TestCenterRegistration model into a tab delimited - text file with a format that Pearson expects. - """ option_list = BaseCommand.option_list + ( make_option( @@ -43,26 +38,25 @@ class Command(BaseCommand): ) - def handle(self, *args, **kwargs): - if len(args) < 1: - print Command.help - return + def handle(self, **kwargs): # update time should use UTC in order to be comparable to the user_updated_at # field uploaded_at = datetime.utcnow() - # if specified destination is an existing directory, then + # if specified destination is an existing directory, then # create a filename for it automatically. If it doesn't exist, - # or exists as a file, then we will just write to it. + # then we will create the directory. # Name will use timestamp -- this is UTC, so it will look funny, - # but it should at least be consistent with the other timestamps + # but it should at least be consistent with the other timestamps # used in the system. - dest = args[0] - if isdir(dest): - destfile = join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat")) + if not os.path.isdir(settings.PEARSON_LOCAL_EXPORT): + os.makedirs(settings.PEARSON_LOCAL_EXPORT) + destfile = os.path.join(settings.PEARSON_LOCAL_EXPORT, + uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat")) else: - destfile = dest + destfile = os.path.join(settings.PEARSON_LOCAL_EXPORT, + uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat")) dump_all = kwargs['dump_all']