From 378b1ff0c35c809558bb081f5acc09be9a516db4 Mon Sep 17 00:00:00 2001 From: Ashley Penney Date: Wed, 16 Jan 2013 11:26:39 -0500 Subject: [PATCH] Various changes thanks to feedback from Brian to make the existing export commands handle --dest-from-settings and --destination and fail unless one is provided as well as rename pearson.py to pearson_transfer and allow is to call the import/export commands directly. I've set it to die in pearson_transfer.py if the django PEARSON settings aren't available. I don't want to try and provide defaults, these must exist or it simply fails. --- .../management/commands/pearson_export_cdd.py | 37 ++++++++++--- .../management/commands/pearson_export_ead.py | 52 ++++++++++++------- .../{pearson.py => pearson_transfer.py} | 50 +++++++++++------- 3 files changed, 95 insertions(+), 44 deletions(-) rename common/djangoapps/student/management/commands/{pearson.py => pearson_transfer.py} (63%) diff --git a/common/djangoapps/student/management/commands/pearson_export_cdd.py b/common/djangoapps/student/management/commands/pearson_export_cdd.py index dcb9f5cd97..c2c13916ab 100644 --- a/common/djangoapps/student/management/commands/pearson_export_cdd.py +++ b/common/djangoapps/student/management/commands/pearson_export_cdd.py @@ -37,7 +37,20 @@ class Command(BaseCommand): ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store ]) - def handle(self, **kwargs): + option_list = BaseCommand.option_list + ( + make_option('--dest-from-settings', + action='store_true', + dest='dest-from-settings', + default=False, + help='Retrieve the destination to export to from django? True/False'), + make_option('--destination', + action='store_true', + dest='destination', + default=None, + help='Where to store the exported files') + ) + + def handle(self, **options): # update time should use UTC in order to be comparable to the user_updated_at # field uploaded_at = datetime.utcnow() @@ -48,13 +61,23 @@ class Command(BaseCommand): # Name will use timestamp -- this is UTC, so it will look funny, # but it should at least be consistent with the other timestamps # used in the system. - if not os.path.isdir(settings.PEARSON[LOCAL_EXPORT]): - os.makedirs(settings.PEARSON[LOCAL_EXPORT]) - destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT], - uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat")) + if options['dest-from-settings'] is True: + if settings.PEARSON[LOCAL_EXPORT]: + dest = settings.PEARSON[LOCAL_EXPORT] + else: + raise CommandError('--dest-from-settings was enabled but the' + 'PEARSON[LOCAL_EXPORT] setting was not set.') + elif options['destination']: + dest = options['destination'] else: - destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT], - uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat")) + raise ComamndError('--destination or --dest-from-settings must be used') + + + if not os.path.isdir(dest): + os.makedirs(dest) + destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat")) + else: + destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat")) # strings must be in latin-1 format. CSV parser will diff --git a/common/djangoapps/student/management/commands/pearson_export_ead.py b/common/djangoapps/student/management/commands/pearson_export_ead.py index 9520e9d013..6e3149778d 100644 --- a/common/djangoapps/student/management/commands/pearson_export_ead.py +++ b/common/djangoapps/student/management/commands/pearson_export_ead.py @@ -23,24 +23,29 @@ class Command(BaseCommand): ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store ]) - option_list = BaseCommand.option_list + ( - make_option( - '--dump_all', - action='store_true', - dest='dump_all', + make_option('--dest-from-settings', + action='store_true', + dest='dest-from-settings', + default=False, + help='Retrieve the destination to export to from django? True/False'), + make_option('--destination', + action='store_true', + dest='destination', + default=None, + help='Where to store the exported files'), + make_option('--dump_all', + action='store_true', + dest='dump_all', ), - make_option( - '--force_add', - action='store_true', - dest='force_add', + make_option('--force_add', + action='store_true', + dest='force_add', ), ) - - - def handle(self, **kwargs): - # update time should use UTC in order to be comparable to the user_updated_at + def handle(self, **options): + # update time should use UTC in order to be comparable to the user_updated_at # field uploaded_at = datetime.utcnow() @@ -50,13 +55,22 @@ class Command(BaseCommand): # Name will use timestamp -- this is UTC, so it will look funny, # but it should at least be consistent with the other timestamps # used in the system. - if not os.path.isdir(settings.PEARSON[LOCAL_EXPORT]): - os.makedirs(settings.PEARSON[LOCAL_EXPORT]) - destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT], - uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat")) + if options['dest-from-settings'] is True: + if settings.PEARSON[LOCAL_EXPORT]: + dest = settings.PEARSON[LOCAL_EXPORT] + else: + raise CommandError('--dest-from-settings was enabled but the' + 'PEARSON[LOCAL_EXPORT] setting was not set.') + elif options['destination']: + dest = options['destination'] else: - destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT], - uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat")) + raise ComamndError('--destination or --dest-from-settings must be used') + + if not os.path.isdir(dest): + os.makedirs(dest) + destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat")) + else: + destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat")) dump_all = kwargs['dump_all'] diff --git a/common/djangoapps/student/management/commands/pearson.py b/common/djangoapps/student/management/commands/pearson_transfer.py similarity index 63% rename from common/djangoapps/student/management/commands/pearson.py rename to common/djangoapps/student/management/commands/pearson_transfer.py index 0752aea8be..1d04936216 100644 --- a/common/djangoapps/student/management/commands/pearson.py +++ b/common/djangoapps/student/management/commands/pearson_transfer.py @@ -12,26 +12,40 @@ dog_http_api.api_key = settings.DATADOG_API class Command(BaseCommand): - option_list = BaseCommand.option_list - args = '' - help = """ - Mode should be import or export depending on if you're fetching from pearson or - sending to them. - """ + option_list = BaseCommand.option_list + ( + make_option('--mode', + action='store_true', + dest='mode', + default='both', + help='mode is import, export, or both'), + ) - def handle(self, *args): - if len(args) < 1: - raise CommandError('Usage is pearson {0}'.format(self.args)) + def handle(self, **options): + + if not settings.PEARSON: + raise CommandError('No PEARSON entries in auth/env.json.') + + def import_pearson(): + sftp(settings.PEARSON[SFTP_IMPORT], settings.PEARSON[LOCAL_IMPORT]) + s3(settings.PEARSON[LOCAL_IMPORT], settings.PEARSON[BUCKET]) + call_command('pearson_import', 'dest_from_settings=True') + + def export_pearson(): + call_command('pearson_export_ccd', 'dest_from_settings=True') + call_command('pearson_export_ead', 'dest_from_settings=True') + sftp(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[SFTP_EXPORT]) + s3(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[BUCKET]) + + if options['mode'] == 'both': + export_pearson() + import_pearson() + elif options['mode'] == 'export': + export_pearson() + elif options['mode'] == 'import': + import_pearson() + else: + print("ERROR: Mode must be export or import.") - for mode in args: - if mode == 'export': - sftp(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[SFTP_EXPORT]) - s3(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[BUCKET]) - elif mode == 'import': - sftp(settings.PEARSON[SFTP_IMPORT], settings.PEARSON[LOCAL_IMPORT]) - s3(settings.PEARSON[LOCAL_IMPORT], settings.PEARSON[BUCKET]) - else: - print("ERROR: Mode must be export or import.") def sftp(files_from, files_to): with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'):