diff --git a/common/djangoapps/student/management/commands/pearson_export_cdd.py b/common/djangoapps/student/management/commands/pearson_export_cdd.py index 67230c7f74..463eec6b70 100644 --- a/common/djangoapps/student/management/commands/pearson_export_cdd.py +++ b/common/djangoapps/student/management/commands/pearson_export_cdd.py @@ -1,15 +1,17 @@ import csv +import os from collections import OrderedDict from datetime import datetime -from os.path import isdir from optparse import make_option -from django.core.management.base import BaseCommand +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError from student.models import TestCenterUser + class Command(BaseCommand): - + CSV_TO_MODEL_FIELDS = OrderedDict([ # Skipping optional field CandidateID ("ClientCandidateID", "client_candidate_id"), @@ -34,43 +36,52 @@ class Command(BaseCommand): ("FAXCountryCode", "fax_country_code"), ("CompanyName", "company_name"), # Skipping optional field CustomQuestion - ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store + ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store ]) + # define defaults, even thought 'store_true' shouldn't need them. + # (call_command will set None as default value for all options that don't have one, + # so one cannot rely on presence/absence of flags in that world.) option_list = BaseCommand.option_list + ( - make_option( - '--dump_all', - action='store_true', - dest='dump_all', - ), + make_option('--dest-from-settings', + action='store_true', + dest='dest-from-settings', + default=False, + help='Retrieve the destination to export to from django.'), + make_option('--destination', + action='store', + dest='destination', + default=None, + help='Where to store the exported files') ) - - args = '' - help = """ - Export user demographic information from TestCenterUser model into a tab delimited - text file with a format that Pearson expects. - """ - def handle(self, *args, **kwargs): - if len(args) < 1: - print Command.help - return + def handle(self, **options): # update time should use UTC in order to be comparable to the user_updated_at # field uploaded_at = datetime.utcnow() - # if specified destination is an existing directory, then + # if specified destination is an existing directory, then # create a filename for it automatically. If it doesn't exist, - # or exists as a file, then we will just write to it. + # then we will create the directory. # Name will use timestamp -- this is UTC, so it will look funny, - # but it should at least be consistent with the other timestamps + # but it should at least be consistent with the other timestamps # used in the system. - dest = args[0] - if isdir(dest): - destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat")) + if 'dest-from-settings' in options and options['dest-from-settings']: + if 'LOCAL_EXPORT' in settings.PEARSON: + dest = settings.PEARSON['LOCAL_EXPORT'] + else: + raise CommandError('--dest-from-settings was enabled but the' + 'PEARSON[LOCAL_EXPORT] setting was not set.') + elif 'destination' in options and options['destination']: + dest = options['destination'] else: - destfile = dest - + raise CommandError('--destination or --dest-from-settings must be used') + + if not os.path.isdir(dest): + os.makedirs(dest) + + destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat")) + # strings must be in latin-1 format. CSV parser will # otherwise convert unicode objects to ascii. def ensure_encoding(value): @@ -78,8 +89,8 @@ class Command(BaseCommand): return value.encode('iso-8859-1') else: return value - - dump_all = kwargs['dump_all'] + +# dump_all = options['dump_all'] with open(destfile, "wb") as outfile: writer = csv.DictWriter(outfile, @@ -89,7 +100,7 @@ class Command(BaseCommand): extrasaction='ignore') writer.writeheader() for tcu in TestCenterUser.objects.order_by('id'): - if dump_all or tcu.needs_uploading: + if tcu.needs_uploading: # or dump_all record = dict((csv_field, ensure_encoding(getattr(tcu, model_field))) for csv_field, model_field in Command.CSV_TO_MODEL_FIELDS.items()) @@ -97,6 +108,3 @@ class Command(BaseCommand): writer.writerow(record) tcu.uploaded_at = uploaded_at tcu.save() - - - diff --git a/common/djangoapps/student/management/commands/pearson_export_ead.py b/common/djangoapps/student/management/commands/pearson_export_ead.py index de3bfc04ee..49cdc9957a 100644 --- a/common/djangoapps/student/management/commands/pearson_export_ead.py +++ b/common/djangoapps/student/management/commands/pearson_export_ead.py @@ -1,15 +1,17 @@ import csv +import os from collections import OrderedDict from datetime import datetime -from os.path import isdir, join from optparse import make_option -from django.core.management.base import BaseCommand +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError from student.models import TestCenterRegistration + class Command(BaseCommand): - + CSV_TO_MODEL_FIELDS = OrderedDict([ ('AuthorizationTransactionType', 'authorization_transaction_type'), ('AuthorizationID', 'authorization_id'), @@ -20,51 +22,60 @@ class Command(BaseCommand): ('Accommodations', 'accommodation_code'), ('EligibilityApptDateFirst', 'eligibility_appointment_date_first'), ('EligibilityApptDateLast', 'eligibility_appointment_date_last'), - ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store + ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store ]) - args = '' - help = """ - Export user registration information from TestCenterRegistration model into a tab delimited - text file with a format that Pearson expects. - """ - option_list = BaseCommand.option_list + ( - make_option( - '--dump_all', - action='store_true', - dest='dump_all', - ), - make_option( - '--force_add', - action='store_true', - dest='force_add', - ), + make_option('--dest-from-settings', + action='store_true', + dest='dest-from-settings', + default=False, + help='Retrieve the destination to export to from django.'), + make_option('--destination', + action='store', + dest='destination', + default=None, + help='Where to store the exported files'), + make_option('--dump_all', + action='store_true', + dest='dump_all', + default=False, + ), + make_option('--force_add', + action='store_true', + dest='force_add', + default=False, + ), ) - - - def handle(self, *args, **kwargs): - if len(args) < 1: - print Command.help - return - # update time should use UTC in order to be comparable to the user_updated_at + def handle(self, **options): + # update time should use UTC in order to be comparable to the user_updated_at # field uploaded_at = datetime.utcnow() - # if specified destination is an existing directory, then + # if specified destination is an existing directory, then # create a filename for it automatically. If it doesn't exist, - # or exists as a file, then we will just write to it. + # then we will create the directory. # Name will use timestamp -- this is UTC, so it will look funny, - # but it should at least be consistent with the other timestamps + # but it should at least be consistent with the other timestamps # used in the system. - dest = args[0] - if isdir(dest): - destfile = join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat")) + if 'dest-from-settings' in options and options['dest-from-settings']: + if 'LOCAL_EXPORT' in settings.PEARSON: + dest = settings.PEARSON['LOCAL_EXPORT'] + else: + raise CommandError('--dest-from-settings was enabled but the' + 'PEARSON[LOCAL_EXPORT] setting was not set.') + elif 'destination' in options and options['destination']: + dest = options['destination'] else: - destfile = dest + raise CommandError('--destination or --dest-from-settings must be used') - dump_all = kwargs['dump_all'] + if not os.path.isdir(dest): + os.makedirs(dest) + + destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat")) + + dump_all = options['dump_all'] with open(destfile, "wb") as outfile: writer = csv.DictWriter(outfile, @@ -81,13 +92,9 @@ class Command(BaseCommand): record["LastUpdate"] = record["LastUpdate"].strftime("%Y/%m/%d %H:%M:%S") record["EligibilityApptDateFirst"] = record["EligibilityApptDateFirst"].strftime("%Y/%m/%d") record["EligibilityApptDateLast"] = record["EligibilityApptDateLast"].strftime("%Y/%m/%d") - if kwargs['force_add']: + if options['force_add']: record['AuthorizationTransactionType'] = 'Add' writer.writerow(record) tcr.uploaded_at = uploaded_at tcr.save() - - - - diff --git a/common/djangoapps/student/management/commands/pearson_import_conf_zip.py b/common/djangoapps/student/management/commands/pearson_import_conf_zip.py new file mode 100644 index 0000000000..bf7c4481fd --- /dev/null +++ b/common/djangoapps/student/management/commands/pearson_import_conf_zip.py @@ -0,0 +1,119 @@ +import csv + +from zipfile import ZipFile, is_zipfile +from time import strptime, strftime + +from collections import OrderedDict +from datetime import datetime +from os.path import isdir +from optparse import make_option +from dogapi import dog_http_api, dog_stats_api + +from django.core.management.base import BaseCommand, CommandError +from django.conf import settings + +from student.models import TestCenterUser, TestCenterRegistration + + +class Command(BaseCommand): + + dog_http_api.api_key = settings.DATADOG_API + args = '' + help = """ + Import Pearson confirmation files and update TestCenterUser + and TestCenterRegistration tables with status. + """ + + @staticmethod + def datadog_error(string, tags): + dog_http_api.event("Pearson Import", string, alert_type='error', tags=tags) + + def handle(self, *args, **kwargs): + if len(args) < 1: + print Command.help + return + + source_zip = args[0] + if not is_zipfile(source_zip): + error = "Input file is not a zipfile: \"{}\"".format(source_zip) + Command.datadog_error(error, source_zip) + raise CommandError(error) + + # loop through all files in zip, and process them based on filename prefix: + with ZipFile(source_zip, 'r') as zipfile: + for fileinfo in zipfile.infolist(): + with zipfile.open(fileinfo) as zipentry: + if fileinfo.filename.startswith("eac-"): + self.process_eac(zipentry) + elif fileinfo.filename.startswith("vcdc-"): + self.process_vcdc(zipentry) + else: + error = "Unrecognized confirmation file type\"{}\" in confirmation zip file \"{}\"".format(fileinfo.filename, zipfile) + Command.datadog_error(error, source_zip) + raise CommandError(error) + + def process_eac(self, eacfile): + print "processing eac" + reader = csv.DictReader(eacfile, delimiter="\t") + for row in reader: + client_authorization_id = row['ClientAuthorizationID'] + if not client_authorization_id: + if row['Status'] == 'Error': + Command.datadog_error("Error in EAD file processing ({}): {}".format(row['Date'], row['Message']), eacfile.name) + else: + Command.datadog_error("Encountered bad record: {}".format(row), eacfile.name) + else: + try: + registration = TestCenterRegistration.objects.get(client_authorization_id=client_authorization_id) + Command.datadog_error("Found authorization record for user {}".format(registration.testcenter_user.user.username), eacfile) + # now update the record: + registration.upload_status = row['Status'] + registration.upload_error_message = row['Message'] + try: + registration.processed_at = strftime('%Y-%m-%d %H:%M:%S', strptime(row['Date'], '%Y/%m/%d %H:%M:%S')) + except ValueError as ve: + Command.datadog_error("Bad Date value found for {}: message {}".format(client_authorization_id, ve), eacfile.name) + # store the authorization Id if one is provided. (For debugging) + if row['AuthorizationID']: + try: + registration.authorization_id = int(row['AuthorizationID']) + except ValueError as ve: + Command.datadog_error("Bad AuthorizationID value found for {}: message {}".format(client_authorization_id, ve), eacfile.name) + + registration.confirmed_at = datetime.utcnow() + registration.save() + except TestCenterRegistration.DoesNotExist: + Command.datadog_error("Failed to find record for client_auth_id {}".format(client_authorization_id), eacfile.name) + + def process_vcdc(self, vcdcfile): + print "processing vcdc" + reader = csv.DictReader(vcdcfile, delimiter="\t") + for row in reader: + client_candidate_id = row['ClientCandidateID'] + if not client_candidate_id: + if row['Status'] == 'Error': + Command.datadog_error("Error in CDD file processing ({}): {}".format(row['Date'], row['Message']), vcdcfile.name) + else: + Command.datadog_error("Encountered bad record: {}".format(row), vcdcfile.name) + else: + try: + tcuser = TestCenterUser.objects.get(client_candidate_id=client_candidate_id) + Command.datadog_error("Found demographics record for user {}".format(tcuser.user.username), vcdcfile.name) + # now update the record: + tcuser.upload_status = row['Status'] + tcuser.upload_error_message = row['Message'] + try: + tcuser.processed_at = strftime('%Y-%m-%d %H:%M:%S', strptime(row['Date'], '%Y/%m/%d %H:%M:%S')) + except ValueError as ve: + Command.datadog_error("Bad Date value found for {}: message {}".format(client_candidate_id, ve), vcdcfile.name) + # store the candidate Id if one is provided. (For debugging) + if row['CandidateID']: + try: + tcuser.candidate_id = int(row['CandidateID']) + except ValueError as ve: + Command.datadog_error("Bad CandidateID value found for {}: message {}".format(client_candidate_id, ve), vcdcfile.name) + tcuser.confirmed_at = datetime.utcnow() + tcuser.save() + except TestCenterUser.DoesNotExist: + Command.datadog_error(" Failed to find record for client_candidate_id {}".format(client_candidate_id), vcdcfile.name) + diff --git a/common/djangoapps/student/management/commands/pearson_make_tc_user.py b/common/djangoapps/student/management/commands/pearson_make_tc_user.py index da9bfc3bd0..87e0b4dadd 100644 --- a/common/djangoapps/student/management/commands/pearson_make_tc_user.py +++ b/common/djangoapps/student/management/commands/pearson_make_tc_user.py @@ -1,7 +1,7 @@ from optparse import make_option from django.contrib.auth.models import User -from django.core.management.base import BaseCommand +from django.core.management.base import BaseCommand, CommandError from student.models import TestCenterUser, TestCenterUserForm @@ -161,15 +161,16 @@ class Command(BaseCommand): if form.is_valid(): form.update_and_save() else: + errorlist = [] if (len(form.errors) > 0): - print "Field Form errors encountered:" - for fielderror in form.errors: - print "Field Form Error: %s" % fielderror - if (len(form.non_field_errors()) > 0): - print "Non-field Form errors encountered:" - for nonfielderror in form.non_field_errors: - print "Non-field Form Error: %s" % nonfielderror - + errorlist.append("Field Form errors encountered:") + for fielderror in form.errors: + errorlist.append("Field Form Error: {}".format(fielderror)) + if (len(form.non_field_errors()) > 0): + errorlist.append("Non-field Form errors encountered:") + for nonfielderror in form.non_field_errors: + errorlist.append("Non-field Form Error: {}".format(nonfielderror)) + raise CommandError("\n".join(errorlist)) else: print "No changes necessary to make to existing user's demographics." diff --git a/common/djangoapps/student/management/commands/pearson_transfer.py b/common/djangoapps/student/management/commands/pearson_transfer.py new file mode 100644 index 0000000000..2124bdceb6 --- /dev/null +++ b/common/djangoapps/student/management/commands/pearson_transfer.py @@ -0,0 +1,160 @@ +import os +from optparse import make_option +from stat import S_ISDIR + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from django.core.management import call_command +from dogapi import dog_http_api, dog_stats_api +import paramiko +import boto + +dog_http_api.api_key = settings.DATADOG_API + + +class Command(BaseCommand): + help = """ + This command handles the importing and exporting of student records for + Pearson. It uses some other Django commands to export and import the + files and then uploads over SFTP to Pearson and stuffs the entry in an + S3 bucket for archive purposes. + + Usage: django-admin.py pearson-transfer --mode [import|export|both] + """ + + option_list = BaseCommand.option_list + ( + make_option('--mode', + action='store', + dest='mode', + default='both', + choices=('import', 'export', 'both'), + help='mode is import, export, or both'), + ) + + def handle(self, **options): + + if not hasattr(settings, 'PEARSON'): + raise CommandError('No PEARSON entries in auth/env.json.') + + # check settings needed for either import or export: + for value in ['SFTP_HOSTNAME', 'SFTP_USERNAME', 'SFTP_PASSWORD', 'S3_BUCKET']: + if value not in settings.PEARSON: + raise CommandError('No entry in the PEARSON settings' + '(env/auth.json) for {0}'.format(value)) + + for value in ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']: + if not hasattr(settings, value): + raise CommandError('No entry in the AWS settings' + '(env/auth.json) for {0}'.format(value)) + + # check additional required settings for import and export: + if options['mode'] in ('export', 'both'): + for value in ['LOCAL_EXPORT','SFTP_EXPORT']: + if value not in settings.PEARSON: + raise CommandError('No entry in the PEARSON settings' + '(env/auth.json) for {0}'.format(value)) + # make sure that the import directory exists or can be created: + source_dir = settings.PEARSON['LOCAL_EXPORT'] + if not os.path.isdir(source_dir): + os.makedirs(source_dir) + + if options['mode'] in ('import', 'both'): + for value in ['LOCAL_IMPORT','SFTP_IMPORT']: + if value not in settings.PEARSON: + raise CommandError('No entry in the PEARSON settings' + '(env/auth.json) for {0}'.format(value)) + # make sure that the import directory exists or can be created: + dest_dir = settings.PEARSON['LOCAL_IMPORT'] + if not os.path.isdir(dest_dir): + os.makedirs(dest_dir) + + + def sftp(files_from, files_to, mode, deleteAfterCopy=False): + with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'): + try: + t = paramiko.Transport((settings.PEARSON['SFTP_HOSTNAME'], 22)) + t.connect(username=settings.PEARSON['SFTP_USERNAME'], + password=settings.PEARSON['SFTP_PASSWORD']) + sftp = paramiko.SFTPClient.from_transport(t) + + if mode == 'export': + try: + sftp.chdir(files_to) + except IOError: + raise CommandError('SFTP destination path does not exist: {}'.format(files_to)) + for filename in os.listdir(files_from): + sftp.put(files_from + '/' + filename, filename) + if deleteAfterCopy: + os.remove(os.path.join(files_from, filename)) + else: + try: + sftp.chdir(files_from) + except IOError: + raise CommandError('SFTP source path does not exist: {}'.format(files_from)) + for filename in sftp.listdir('.'): + # skip subdirectories + if not S_ISDIR(sftp.stat(filename).st_mode): + sftp.get(filename, files_to + '/' + filename) + # delete files from sftp server once they are successfully pulled off: + if deleteAfterCopy: + sftp.remove(filename) + except: + dog_http_api.event('pearson {0}'.format(mode), + 'sftp uploading failed', + alert_type='error') + raise + finally: + sftp.close() + t.close() + + def s3(files_from, bucket, mode, deleteAfterCopy=False): + with dog_stats_api.timer('pearson.{0}'.format(mode), tags='s3'): + try: + for filename in os.listdir(files_from): + upload_file_to_s3(bucket, files_from, filename) + if deleteAfterCopy: + os.remove(files_from + '/' + filename) + except: + dog_http_api.event('pearson {0}'.format(mode), + 's3 archiving failed') + raise + + def upload_file_to_s3(bucket, source_dir, filename): + """ + Upload file to S3 + """ + s3 = boto.connect_s3(settings.AWS_ACCESS_KEY_ID, + settings.AWS_SECRET_ACCESS_KEY) + from boto.s3.key import Key + b = s3.get_bucket(bucket) + k = Key(b) + k.key = "{filename}".format(filename=filename) + k.set_contents_from_filename(os.path.join(source_dir, filename)) + + def export_pearson(): + options = { 'dest-from-settings' : True } + call_command('pearson_export_cdd', **options) + call_command('pearson_export_ead', **options) + mode = 'export' + sftp(settings.PEARSON['LOCAL_EXPORT'], settings.PEARSON['SFTP_EXPORT'], mode, deleteAfterCopy = False) + s3(settings.PEARSON['LOCAL_EXPORT'], settings.PEARSON['S3_BUCKET'], mode, deleteAfterCopy=True) + + def import_pearson(): + mode = 'import' + try: + sftp(settings.PEARSON['SFTP_IMPORT'], settings.PEARSON['LOCAL_IMPORT'], mode, deleteAfterCopy = True) + s3(settings.PEARSON['LOCAL_IMPORT'], settings.PEARSON['S3_BUCKET'], mode, deleteAfterCopy=False) + except Exception as e: + dog_http_api.event('Pearson Import failure', str(e)) + raise e + else: + for filename in os.listdir(settings.PEARSON['LOCAL_IMPORT']): + filepath = os.path.join(settings.PEARSON['LOCAL_IMPORT'], filename) + call_command('pearson_import_conf_zip', filepath) + os.remove(filepath) + + # actually do the work! + if options['mode'] in ('export', 'both'): + export_pearson() + if options['mode'] in ('import', 'both'): + import_pearson() diff --git a/common/djangoapps/student/management/commands/tests/__init__.py b/common/djangoapps/student/management/commands/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/common/djangoapps/student/management/commands/tests/test_pearson.py b/common/djangoapps/student/management/commands/tests/test_pearson.py new file mode 100644 index 0000000000..199557bf87 --- /dev/null +++ b/common/djangoapps/student/management/commands/tests/test_pearson.py @@ -0,0 +1,382 @@ +''' +Created on Jan 17, 2013 + +@author: brian +''' +import logging +import os +from tempfile import mkdtemp +import cStringIO +import sys + +from django.test import TestCase +from django.core.management import call_command +from nose.plugins.skip import SkipTest + +from student.models import User, TestCenterRegistration, TestCenterUser, get_testcenter_registration + +log = logging.getLogger(__name__) + +def create_tc_user(username): + user = User.objects.create_user(username, '{}@edx.org'.format(username), 'fakepass') + options = { + 'first_name' : 'TestFirst', + 'last_name' : 'TestLast', + 'address_1' : 'Test Address', + 'city' : 'TestCity', + 'state' : 'Alberta', + 'postal_code' : 'A0B 1C2', + 'country' : 'CAN', + 'phone' : '252-1866', + 'phone_country_code' : '1', + } + call_command('pearson_make_tc_user', username, **options) + return TestCenterUser.objects.get(user=user) + + +def create_tc_registration(username, course_id = 'org1/course1/term1', exam_code = 'exam1', accommodation_code = None): + + options = { 'exam_series_code' : exam_code, + 'eligibility_appointment_date_first' : '2013-01-01T00:00', + 'eligibility_appointment_date_last' : '2013-12-31T23:59', + 'accommodation_code' : accommodation_code, + } + + call_command('pearson_make_tc_registration', username, course_id, **options) + user = User.objects.get(username=username) + registrations = get_testcenter_registration(user, course_id, exam_code) + return registrations[0] + +def create_multiple_registrations(prefix='test'): + username1 = '{}_multiple1'.format(prefix) + create_tc_user(username1) + create_tc_registration(username1) + create_tc_registration(username1, course_id = 'org1/course2/term1') + create_tc_registration(username1, exam_code = 'exam2') + username2 = '{}_multiple2'.format(prefix) + create_tc_user(username2) + create_tc_registration(username2) + username3 = '{}_multiple3'.format(prefix) + create_tc_user(username3) + create_tc_registration(username3, course_id = 'org1/course2/term1') + username4 = '{}_multiple4'.format(prefix) + create_tc_user(username4) + create_tc_registration(username4, exam_code = 'exam2') + +def get_command_error_text(*args, **options): + stderr_string = None + old_stderr = sys.stderr + sys.stderr = cStringIO.StringIO() + try: + call_command(*args, **options) + except SystemExit, why1: + # The goal here is to catch CommandError calls. + # But these are actually translated into nice messages, + # and sys.exit(1) is then called. For testing, we + # want to catch what sys.exit throws, and get the + # relevant text either from stdout or stderr. + if (why1.message > 0): + stderr_string = sys.stderr.getvalue() + else: + raise why1 + except Exception, why: + raise why + + finally: + sys.stderr = old_stderr + + if stderr_string is None: + raise Exception("Expected call to {} to fail, but it succeeded!".format(args[0])) + return stderr_string + +def get_error_string_for_management_call(*args, **options): + stdout_string = None + old_stdout = sys.stdout + old_stderr = sys.stderr + sys.stdout = cStringIO.StringIO() + sys.stderr = cStringIO.StringIO() + try: + call_command(*args, **options) + except SystemExit, why1: + # The goal here is to catch CommandError calls. + # But these are actually translated into nice messages, + # and sys.exit(1) is then called. For testing, we + # want to catch what sys.exit throws, and get the + # relevant text either from stdout or stderr. + if (why1.message == 1): + stdout_string = sys.stdout.getvalue() + stderr_string = sys.stderr.getvalue() + else: + raise why1 + except Exception, why: + raise why + + finally: + sys.stdout = old_stdout + sys.stderr = old_stderr + + if stdout_string is None: + raise Exception("Expected call to {} to fail, but it succeeded!".format(args[0])) + return stdout_string, stderr_string + + +def get_file_info(dirpath): + filelist = os.listdir(dirpath) + print 'Files found: {}'.format(filelist) + numfiles = len(filelist) + if numfiles == 1: + filepath = os.path.join(dirpath, filelist[0]) + with open(filepath, 'r') as cddfile: + filecontents = cddfile.readlines() + numlines = len(filecontents) + return filepath, numlines + else: + raise Exception("Expected to find a single file in {}, but found {}".format(dirpath,filelist)) + +class PearsonTestCase(TestCase): + ''' + Base class for tests running Pearson-related commands + ''' + import_dir = mkdtemp(prefix="import") + export_dir = mkdtemp(prefix="export") + + def assertErrorContains(self, error_message, expected): + self.assertTrue(error_message.find(expected) >= 0, 'error message "{}" did not contain "{}"'.format(error_message, expected)) + + def tearDown(self): + def delete_temp_dir(dirname): + if os.path.exists(dirname): + for filename in os.listdir(dirname): + os.remove(os.path.join(dirname, filename)) + os.rmdir(dirname) + + # clean up after any test data was dumped to temp directory + delete_temp_dir(self.import_dir) + delete_temp_dir(self.export_dir) + + # and clean up the database: +# TestCenterUser.objects.all().delete() +# TestCenterRegistration.objects.all().delete() + +class PearsonCommandTestCase(PearsonTestCase): + + def test_missing_demographic_fields(self): + # We won't bother to test all details of form validation here. + # It is enough to show that it works here, but deal with test cases for the form + # validation in the student tests, not these management tests. + username = 'baduser' + User.objects.create_user(username, '{}@edx.org'.format(username), 'fakepass') + options = {} + error_string = get_command_error_text('pearson_make_tc_user', username, **options) + self.assertTrue(error_string.find('Field Form errors encountered:') >= 0) + self.assertTrue(error_string.find('Field Form Error: city') >= 0) + self.assertTrue(error_string.find('Field Form Error: first_name') >= 0) + self.assertTrue(error_string.find('Field Form Error: last_name') >= 0) + self.assertTrue(error_string.find('Field Form Error: country') >= 0) + self.assertTrue(error_string.find('Field Form Error: phone_country_code') >= 0) + self.assertTrue(error_string.find('Field Form Error: phone') >= 0) + self.assertTrue(error_string.find('Field Form Error: address_1') >= 0) + self.assertErrorContains(error_string, 'Field Form Error: address_1') + + def test_create_good_testcenter_user(self): + testcenter_user = create_tc_user("test1") + self.assertIsNotNone(testcenter_user) + + def test_create_good_testcenter_registration(self): + username = 'test1' + create_tc_user(username) + registration = create_tc_registration(username) + self.assertIsNotNone(registration) + + def test_cdd_missing_option(self): + error_string = get_command_error_text('pearson_export_cdd', **{}) + self.assertErrorContains(error_string, 'Error: --destination or --dest-from-settings must be used') + + def test_ead_missing_option(self): + error_string = get_command_error_text('pearson_export_ead', **{}) + self.assertErrorContains(error_string, 'Error: --destination or --dest-from-settings must be used') + + def test_export_single_cdd(self): + # before we generate any tc_users, we expect there to be nothing to output: + options = { 'dest-from-settings' : True } + with self.settings(PEARSON={ 'LOCAL_EXPORT' : self.export_dir }): + call_command('pearson_export_cdd', **options) + (filepath, numlines) = get_file_info(self.export_dir) + self.assertEquals(numlines, 1, "Expect cdd file to have no non-header lines") + os.remove(filepath) + + # generating a tc_user should result in a line in the output + username = 'test_single_cdd' + create_tc_user(username) + call_command('pearson_export_cdd', **options) + (filepath, numlines) = get_file_info(self.export_dir) + self.assertEquals(numlines, 2, "Expect cdd file to have one non-header line") + os.remove(filepath) + + # output after registration should not have any entries again. + call_command('pearson_export_cdd', **options) + (filepath, numlines) = get_file_info(self.export_dir) + self.assertEquals(numlines, 1, "Expect cdd file to have no non-header lines") + os.remove(filepath) + + # if we modify the record, then it should be output again: + user_options = { 'first_name' : 'NewTestFirst', } + call_command('pearson_make_tc_user', username, **user_options) + call_command('pearson_export_cdd', **options) + (filepath, numlines) = get_file_info(self.export_dir) + self.assertEquals(numlines, 2, "Expect cdd file to have one non-header line") + os.remove(filepath) + + def test_export_single_ead(self): + # before we generate any registrations, we expect there to be nothing to output: + options = { 'dest-from-settings' : True } + with self.settings(PEARSON={ 'LOCAL_EXPORT' : self.export_dir }): + call_command('pearson_export_ead', **options) + (filepath, numlines) = get_file_info(self.export_dir) + self.assertEquals(numlines, 1, "Expect ead file to have no non-header lines") + os.remove(filepath) + + # generating a registration should result in a line in the output + username = 'test_single_ead' + create_tc_user(username) + create_tc_registration(username) + call_command('pearson_export_ead', **options) + (filepath, numlines) = get_file_info(self.export_dir) + self.assertEquals(numlines, 2, "Expect ead file to have one non-header line") + os.remove(filepath) + + # output after registration should not have any entries again. + call_command('pearson_export_ead', **options) + (filepath, numlines) = get_file_info(self.export_dir) + self.assertEquals(numlines, 1, "Expect ead file to have no non-header lines") + os.remove(filepath) + + # if we modify the record, then it should be output again: + create_tc_registration(username, accommodation_code='EQPMNT') + call_command('pearson_export_ead', **options) + (filepath, numlines) = get_file_info(self.export_dir) + self.assertEquals(numlines, 2, "Expect ead file to have one non-header line") + os.remove(filepath) + + def test_export_multiple(self): + create_multiple_registrations("export") + with self.settings(PEARSON={ 'LOCAL_EXPORT' : self.export_dir }): + options = { 'dest-from-settings' : True } + call_command('pearson_export_cdd', **options) + (filepath, numlines) = get_file_info(self.export_dir) + self.assertEquals(numlines, 5, "Expect cdd file to have four non-header lines: total was {}".format(numlines)) + os.remove(filepath) + + call_command('pearson_export_ead', **options) + (filepath, numlines) = get_file_info(self.export_dir) + self.assertEquals(numlines, 7, "Expect ead file to have six non-header lines: total was {}".format(numlines)) + os.remove(filepath) + + +# def test_bad_demographic_option(self): +# username = 'nonuser' +# output_string, stderrmsg = get_error_string_for_management_call('pearson_make_tc_user', username, **{'--garbage' : None }) +# print stderrmsg +# self.assertErrorContains(stderrmsg, 'Unexpected option') +# +# def test_missing_demographic_user(self): +# username = 'nonuser' +# output_string, error_string = get_error_string_for_management_call('pearson_make_tc_user', username, **{}) +# self.assertErrorContains(error_string, 'User matching query does not exist') + +# credentials for a test SFTP site: +SFTP_HOSTNAME = 'ec2-23-20-150-101.compute-1.amazonaws.com' +SFTP_USERNAME = 'pearsontest' +SFTP_PASSWORD = 'password goes here' + +S3_BUCKET = 'edx-pearson-archive' +AWS_ACCESS_KEY_ID = 'put yours here' +AWS_SECRET_ACCESS_KEY = 'put yours here' + +class PearsonTransferTestCase(PearsonTestCase): + ''' + Class for tests running Pearson transfers + ''' + + def test_transfer_config(self): + with self.settings(DATADOG_API='FAKE_KEY'): + # TODO: why is this failing with the wrong error message?! + stderrmsg = get_command_error_text('pearson_transfer', **{'mode' : 'garbage'}) + self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries') + with self.settings(DATADOG_API='FAKE_KEY'): + stderrmsg = get_command_error_text('pearson_transfer') + self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries') + with self.settings(DATADOG_API='FAKE_KEY', + PEARSON={'LOCAL_EXPORT' : self.export_dir, + 'LOCAL_IMPORT' : self.import_dir }): + stderrmsg = get_command_error_text('pearson_transfer') + self.assertErrorContains(stderrmsg, 'Error: No entry in the PEARSON settings') + + def test_transfer_export_missing_dest_dir(self): + raise SkipTest() + create_multiple_registrations('export_missing_dest') + with self.settings(DATADOG_API='FAKE_KEY', + PEARSON={'LOCAL_EXPORT' : self.export_dir, + 'SFTP_EXPORT' : 'this/does/not/exist', + 'SFTP_HOSTNAME' : SFTP_HOSTNAME, + 'SFTP_USERNAME' : SFTP_USERNAME, + 'SFTP_PASSWORD' : SFTP_PASSWORD, + 'S3_BUCKET' : S3_BUCKET, + }, + AWS_ACCESS_KEY_ID = AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY = AWS_SECRET_ACCESS_KEY): + options = { 'mode' : 'export'} + stderrmsg = get_command_error_text('pearson_transfer', **options) + self.assertErrorContains(stderrmsg, 'Error: SFTP destination path does not exist') + + def test_transfer_export(self): + raise SkipTest() + create_multiple_registrations("transfer_export") + with self.settings(DATADOG_API='FAKE_KEY', + PEARSON={'LOCAL_EXPORT' : self.export_dir, + 'SFTP_EXPORT' : 'results/topvue', + 'SFTP_HOSTNAME' : SFTP_HOSTNAME, + 'SFTP_USERNAME' : SFTP_USERNAME, + 'SFTP_PASSWORD' : SFTP_PASSWORD, + 'S3_BUCKET' : S3_BUCKET, + }, + AWS_ACCESS_KEY_ID = AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY = AWS_SECRET_ACCESS_KEY): + options = { 'mode' : 'export'} +# call_command('pearson_transfer', **options) +# # confirm that the export directory is still empty: +# self.assertEqual(len(os.listdir(self.export_dir)), 0, "expected export directory to be empty") + + def test_transfer_import_missing_source_dir(self): + raise SkipTest() + create_multiple_registrations('import_missing_src') + with self.settings(DATADOG_API='FAKE_KEY', + PEARSON={'LOCAL_IMPORT' : self.import_dir, + 'SFTP_IMPORT' : 'this/does/not/exist', + 'SFTP_HOSTNAME' : SFTP_HOSTNAME, + 'SFTP_USERNAME' : SFTP_USERNAME, + 'SFTP_PASSWORD' : SFTP_PASSWORD, + 'S3_BUCKET' : S3_BUCKET, + }, + AWS_ACCESS_KEY_ID = AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY = AWS_SECRET_ACCESS_KEY): + options = { 'mode' : 'import'} + stderrmsg = get_command_error_text('pearson_transfer', **options) + self.assertErrorContains(stderrmsg, 'Error: SFTP source path does not exist') + + def test_transfer_import(self): + raise SkipTest() + create_multiple_registrations('import_missing_src') + with self.settings(DATADOG_API='FAKE_KEY', + PEARSON={'LOCAL_IMPORT' : self.import_dir, + 'SFTP_IMPORT' : 'results', + 'SFTP_HOSTNAME' : SFTP_HOSTNAME, + 'SFTP_USERNAME' : SFTP_USERNAME, + 'SFTP_PASSWORD' : SFTP_PASSWORD, + 'S3_BUCKET' : S3_BUCKET, + }, + AWS_ACCESS_KEY_ID = AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY = AWS_SECRET_ACCESS_KEY): + options = { 'mode' : 'import'} + call_command('pearson_transfer', **options) + self.assertEqual(len(os.listdir(self.import_dir)), 0, "expected import directory to be empty") diff --git a/common/djangoapps/student/models.py b/common/djangoapps/student/models.py index f13a691215..0d8a643ecb 100644 --- a/common/djangoapps/student/models.py +++ b/common/djangoapps/student/models.py @@ -428,6 +428,10 @@ class TestCenterRegistration(models.Model): # TODO: figure out if this should really go in the database (with a default value). return 1 + @property + def needs_uploading(self): + return self.uploaded_at is None or self.uploaded_at < self.user_updated_at + @classmethod def create(cls, testcenter_user, exam, accommodation_request): registration = cls(testcenter_user = testcenter_user) @@ -549,7 +553,11 @@ def get_testcenter_registration(user, course_id, exam_series_code): except TestCenterUser.DoesNotExist: return [] return TestCenterRegistration.objects.filter(testcenter_user=tcu, course_id=course_id, exam_series_code=exam_series_code) - + +# nosetests thinks that anything with _test_ in the name is a test. +# Correct this (https://nose.readthedocs.org/en/latest/finding_tests.html) +get_testcenter_registration.__test__ = False + def unique_id_for_user(user): """ Return a unique id for a user, suitable for inserting into diff --git a/github-requirements.txt b/github-requirements.txt index 468d55ce65..62e47a328f 100644 --- a/github-requirements.txt +++ b/github-requirements.txt @@ -3,3 +3,4 @@ -e git://github.com/MITx/django-pipeline.git#egg=django-pipeline -e git://github.com/MITx/django-wiki.git@e2e84558#egg=django-wiki -e git://github.com/dementrock/pystache_custom.git@776973740bdaad83a3b029f96e415a7d1e8bec2f#egg=pystache_custom-dev +-e git://github.com/MITx/dogapi.git@003a4fc9#egg=dogapi diff --git a/requirements.txt b/requirements.txt index 996388a51d..fa4688b711 100644 --- a/requirements.txt +++ b/requirements.txt @@ -58,4 +58,4 @@ factory_boy Shapely==1.2.16 ipython==0.13.1 xmltodict==0.4.1 - +paramiko==1.9.0