Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Various changes thanks to feedback from Brian to make the existing

Browse Source 
export commands handle --dest-from-settings and --destination and fail
unless one is provided as well as rename pearson.py to pearson_transfer
and allow is to call the import/export commands directly.

I've set it to die in pearson_transfer.py if the django PEARSON settings
aren't available.  I don't want to try and provide defaults, these must
exist or it simply fails.
This commit is contained in:
Ashley Penney
2013-01-16 11:26:39 -05:00
parent ced29a25ec
commit 378b1ff0c3
3 changed files with 95 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
common/djangoapps/student/management/commands/pearson_export_cdd.py
View File

@@ -37,7 +37,20 @@ class Command(BaseCommand):
("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
])
def handle(self, **kwargs):
option_list = BaseCommand.option_list + (
make_option('--dest-from-settings',
action='store_true',
dest='dest-from-settings',
default=False,
help='Retrieve the destination to export to from django? True/False'),
make_option('--destination',
action='store_true',
dest='destination',
default=None,
help='Where to store the exported files')
)
def handle(self, **options):
# update time should use UTC in order to be comparable to the user_updated_at
# field
uploaded_at = datetime.utcnow()
@@ -48,13 +61,23 @@ class Command(BaseCommand):
# Name will use timestamp -- this is UTC, so it will look funny,
# but it should at least be consistent with the other timestamps
# used in the system.
if not os.path.isdir(settings.PEARSON[LOCAL_EXPORT]):
os.makedirs(settings.PEARSON[LOCAL_EXPORT])
destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT],
uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
if options['dest-from-settings'] is True:
if settings.PEARSON[LOCAL_EXPORT]:
dest = settings.PEARSON[LOCAL_EXPORT]
else:
raise CommandError('--dest-from-settings was enabled but the'
'PEARSON[LOCAL_EXPORT] setting was not set.')
elif options['destination']:
dest = options['destination']
else:
destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT],
uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
raise ComamndError('--destination or --dest-from-settings must be used')
if not os.path.isdir(dest):
os.makedirs(dest)
destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
else:
destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
# strings must be in latin-1 format. CSV parser will

52
common/djangoapps/student/management/commands/pearson_export_ead.py
View File

@@ -23,24 +23,29 @@ class Command(BaseCommand):
("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
])
option_list = BaseCommand.option_list + (
make_option(
'--dump_all',
action='store_true',
dest='dump_all',
make_option('--dest-from-settings',
action='store_true',
dest='dest-from-settings',
default=False,
help='Retrieve the destination to export to from django? True/False'),
make_option('--destination',
action='store_true',
dest='destination',
default=None,
help='Where to store the exported files'),
make_option('--dump_all',
action='store_true',
dest='dump_all',
),
make_option(
'--force_add',
action='store_true',
dest='force_add',
make_option('--force_add',
action='store_true',
dest='force_add',
),
)
def handle(self, **kwargs):
# update time should use UTC in order to be comparable to the user_updated_at
def handle(self, **options):
# update time should use UTC in order to be comparable to the user_updated_at
# field
uploaded_at = datetime.utcnow()
@@ -50,13 +55,22 @@ class Command(BaseCommand):
# Name will use timestamp -- this is UTC, so it will look funny,
# but it should at least be consistent with the other timestamps
# used in the system.
if not os.path.isdir(settings.PEARSON[LOCAL_EXPORT]):
os.makedirs(settings.PEARSON[LOCAL_EXPORT])
destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT],
uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
if options['dest-from-settings'] is True:
if settings.PEARSON[LOCAL_EXPORT]:
dest = settings.PEARSON[LOCAL_EXPORT]
else:
raise CommandError('--dest-from-settings was enabled but the'
'PEARSON[LOCAL_EXPORT] setting was not set.')
elif options['destination']:
dest = options['destination']
else:
destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT],
uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
raise ComamndError('--destination or --dest-from-settings must be used')
if not os.path.isdir(dest):
os.makedirs(dest)
destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
else:
destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
dump_all = kwargs['dump_all']

50
common/djangoapps/student/management/commands/pearson.py → common/djangoapps/student/management/commands/pearson_transfer.py
View File

@@ -12,26 +12,40 @@ dog_http_api.api_key = settings.DATADOG_API
class Command(BaseCommand):
option_list = BaseCommand.option_list
args = '<mode>'
help = """
Mode should be import or export depending on if you're fetching from pearson or
sending to them.
"""
option_list = BaseCommand.option_list + (
make_option('--mode',
action='store_true',
dest='mode',
default='both',
help='mode is import, export, or both'),
)
def handle(self, *args):
if len(args) < 1:
raise CommandError('Usage is pearson {0}'.format(self.args))
def handle(self, **options):
if not settings.PEARSON:
raise CommandError('No PEARSON entries in auth/env.json.')
def import_pearson():
sftp(settings.PEARSON[SFTP_IMPORT], settings.PEARSON[LOCAL_IMPORT])
s3(settings.PEARSON[LOCAL_IMPORT], settings.PEARSON[BUCKET])
call_command('pearson_import', 'dest_from_settings=True')
def export_pearson():
call_command('pearson_export_ccd', 'dest_from_settings=True')
call_command('pearson_export_ead', 'dest_from_settings=True')
sftp(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[SFTP_EXPORT])
s3(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[BUCKET])
if options['mode'] == 'both':
export_pearson()
import_pearson()
elif options['mode'] == 'export':
export_pearson()
elif options['mode'] == 'import':
import_pearson()
else:
print("ERROR: Mode must be export or import.")
for mode in args:
if mode == 'export':
sftp(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[SFTP_EXPORT])
s3(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[BUCKET])
elif mode == 'import':
sftp(settings.PEARSON[SFTP_IMPORT], settings.PEARSON[LOCAL_IMPORT])
s3(settings.PEARSON[LOCAL_IMPORT], settings.PEARSON[BUCKET])
else:
print("ERROR: Mode must be export or import.")
def sftp(files_from, files_to):
with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'):