From 1d13b1a9bd6ec595b35c727468c484b2a36f3047 Mon Sep 17 00:00:00 2001
From: Ashley Penney <apenney@edx.org>
Date: Wed, 16 Jan 2013 09:45:22 -0500
Subject: [PATCH 01/29] Make various changes to handle the s3/sftp part of the
 pearson process.

---
 .../student/management/commands/pearson.py    | 74 +++++++++++++++++++
 .../management/commands/pearson_export_cdd.py | 32 +++-----
 .../management/commands/pearson_export_ead.py | 26 +++----
 3 files changed, 93 insertions(+), 39 deletions(-)
 create mode 100644 common/djangoapps/student/management/commands/pearson.py

diff --git a/common/djangoapps/student/management/commands/pearson.py b/common/djangoapps/student/management/commands/pearson.py
new file mode 100644
index 0000000000..2a8b02928c
--- /dev/null
+++ b/common/djangoapps/student/management/commands/pearson.py
@@ -0,0 +1,74 @@
+from optparse import make_option
+
+from django.contrib.auth.models import User
+from django.core.management.base import BaseCommand, CommandError
+import re
+from dogapi import dog_http_api, dog_stats_api
+import paramiko
+
+dog_http_api.api_key = settings.DATADOG_API
+
+
+class Command(BaseCommand):
+
+    option_list = BaseCommand.option_list
+    args = '<mode>'
+    help = """
+    Mode should be import or export depending on if you're fetching from pearson or
+    sending to them.
+    """
+
+    def handle(self, *args):
+        if len(args) < 1:
+            raise CommandError('Usage is pearson {0}'.format(self.args))
+
+        for mode in args:
+            if mode == 'export':
+                sftp(settings.PEARSON_LOCAL_IMPORT, settings.PEARSON_SFTP_IMPORT)
+                s3(settings.PEARSON_LOCAL, settings.PEARSON_BUCKET)
+            elif mode == 'import':
+                sftp(settings.PEARSON_SFTP_EXPORT, settings.PEARSON_LOCAL_EXPORT)
+                s3(settings.PEARSON_LOCAL_EXPORT, settings.PEARSON_BUCKET)
+            else:
+                print("ERROR:  Mode must be export or import.")
+
+        def sftp(files_from, files_to):
+            with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'):
+                try:
+                    t = paramiko.Transport((hostname, 22))
+                    t.connect(username=settings.PEARSON_SFTP_USERNAME,
+                              password=settings.PEARSON_SFTP_PASSWORD)
+                    sftp = paramiko.SFTPClient.from_transport(t)
+                    if os.path.isdir(files_from):
+                        for file in os.listdir(files_from):
+                            sftp.put(files_from+'/'+filename,
+                                     files_to+'/'+filename)
+                    else:
+                        for file in sftp.listdir(files_from):
+                            sftp.get(files_from+'/'+filename,
+                                     files_to+'/'+filename)
+                except:
+                    dog_http_api.event('pearson {0}'.format(mode),
+                                       'sftp uploading failed', alert_type='error')
+                    raise
+
+        def s3(files_from, bucket):
+            with dog_stats_api.timer('pearson.{0}'.format(mode), tags='s3'):
+                try:
+                    for filename in os.listdir(files):
+                        upload_file_to_s3(bucket, files_from+'/'+filename)
+                except:
+                    dog_http_api.event('pearson {0}'.format(mode), 's3 archiving failed')
+                    raise
+
+
+        def upload_file_to_s3(bucket, filename):
+            """
+            Upload file to S3
+            """
+            s3 = boto.connect_s3()
+            from boto.s3.key import Key
+            b = s3.get_bucket(bucket)
+            k = Key(b)
+            k.key = "{filename}".format(filename=filename)
+            k.set_contents_from_filename(filename)
diff --git a/common/djangoapps/student/management/commands/pearson_export_cdd.py b/common/djangoapps/student/management/commands/pearson_export_cdd.py
index 67230c7f74..cebc098080 100644
--- a/common/djangoapps/student/management/commands/pearson_export_cdd.py
+++ b/common/djangoapps/student/management/commands/pearson_export_cdd.py
@@ -37,39 +37,25 @@ class Command(BaseCommand):
         ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
     ])
 
-    option_list = BaseCommand.option_list + (
-        make_option(
-            '--dump_all',
-            action='store_true',
-            dest='dump_all',
-        ),
-    )
-    
-    args = '<output_file_or_dir>'
-    help = """
-    Export user demographic information from TestCenterUser model into a tab delimited
-    text file with a format that Pearson expects.
-    """
-    def handle(self, *args, **kwargs):
-        if len(args) < 1:
-            print Command.help
-            return
-
+    def handle(self, **kwargs):
         # update time should use UTC in order to be comparable to the user_updated_at 
         # field
         uploaded_at = datetime.utcnow()
 
         # if specified destination is an existing directory, then 
         # create a filename for it automatically.  If it doesn't exist,
-        # or exists as a file, then we will just write to it.
+        # then we will create the directory.
         # Name will use timestamp -- this is UTC, so it will look funny,
         # but it should at least be consistent with the other timestamps 
         # used in the system.
-        dest = args[0]
-        if isdir(dest):
-            destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
+        if not os.path.isdir(settings.PEARSON_LOCAL_EXPORT):
+            os.makedirs(settings.PEARSON_LOCAL_EXPORT)
+            destfile = os.path.join(settings.PEARSON_LOCAL_EXPORT,
+                    uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
         else:
-            destfile = dest
+            destfile = os.path.join(settings.PEARSON_LOCAL_EXPORT,
+                    uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
+
         
         # strings must be in latin-1 format.  CSV parser will
         # otherwise convert unicode objects to ascii.
diff --git a/common/djangoapps/student/management/commands/pearson_export_ead.py b/common/djangoapps/student/management/commands/pearson_export_ead.py
index de3bfc04ee..5423282346 100644
--- a/common/djangoapps/student/management/commands/pearson_export_ead.py
+++ b/common/djangoapps/student/management/commands/pearson_export_ead.py
@@ -23,11 +23,6 @@ class Command(BaseCommand):
         ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
     ])
 
-    args = '<output_file_or_dir>'
-    help = """
-    Export user registration information from TestCenterRegistration model into a tab delimited
-    text file with a format that Pearson expects.
-    """
 
     option_list = BaseCommand.option_list + (
         make_option(
@@ -43,26 +38,25 @@ class Command(BaseCommand):
     )
     
     
-    def handle(self, *args, **kwargs):
-        if len(args) < 1:
-            print Command.help
-            return
+    def handle(self, **kwargs):
 
         # update time should use UTC in order to be comparable to the user_updated_at 
         # field
         uploaded_at = datetime.utcnow()
 
-        # if specified destination is an existing directory, then 
+        # if specified destination is an existing directory, then
         # create a filename for it automatically.  If it doesn't exist,
-        # or exists as a file, then we will just write to it.
+        # then we will create the directory.
         # Name will use timestamp -- this is UTC, so it will look funny,
-        # but it should at least be consistent with the other timestamps 
+        # but it should at least be consistent with the other timestamps
         # used in the system.
-        dest = args[0]
-        if isdir(dest):
-            destfile = join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
+        if not os.path.isdir(settings.PEARSON_LOCAL_EXPORT):
+            os.makedirs(settings.PEARSON_LOCAL_EXPORT)
+            destfile = os.path.join(settings.PEARSON_LOCAL_EXPORT,
+                    uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
         else:
-            destfile = dest
+            destfile = os.path.join(settings.PEARSON_LOCAL_EXPORT,
+                    uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
 
         dump_all = kwargs['dump_all']
 

From 1acf2dbba350272b06df29fe25d71f248060f1c4 Mon Sep 17 00:00:00 2001
From: Ashley Penney <apenney@edx.org>
Date: Wed, 16 Jan 2013 09:52:04 -0500
Subject: [PATCH 02/29] Use a dictionary for all the pearson stuff to keep the
 auth/env stuff clean.

---
 .../student/management/commands/pearson.py           | 12 +++++++-----
 .../management/commands/pearson_export_cdd.py        |  8 ++++----
 .../management/commands/pearson_export_ead.py        |  8 ++++----
 3 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/common/djangoapps/student/management/commands/pearson.py b/common/djangoapps/student/management/commands/pearson.py
index 2a8b02928c..a317b08d62 100644
--- a/common/djangoapps/student/management/commands/pearson.py
+++ b/common/djangoapps/student/management/commands/pearson.py
@@ -5,6 +5,7 @@ from django.core.management.base import BaseCommand, CommandError
 import re
 from dogapi import dog_http_api, dog_stats_api
 import paramiko
+import boto
 
 dog_http_api.api_key = settings.DATADOG_API
 
@@ -24,11 +25,11 @@ class Command(BaseCommand):
 
         for mode in args:
             if mode == 'export':
-                sftp(settings.PEARSON_LOCAL_IMPORT, settings.PEARSON_SFTP_IMPORT)
-                s3(settings.PEARSON_LOCAL, settings.PEARSON_BUCKET)
+                sftp(settings.PEARSON[LOCAL_IMPORT], settings.PEARSON[SFTP_IMPORT])
+                s3(settings.PEARSON_LOCAL, settings.PEARSON[BUCKET])
             elif mode == 'import':
-                sftp(settings.PEARSON_SFTP_EXPORT, settings.PEARSON_LOCAL_EXPORT)
-                s3(settings.PEARSON_LOCAL_EXPORT, settings.PEARSON_BUCKET)
+                sftp(settings.PEARSON[SFTP_EXPORT], settings.PEARSON[LOCAL_EXPORT])
+                s3(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[BUCKET])
             else:
                 print("ERROR:  Mode must be export or import.")
 
@@ -66,7 +67,8 @@ class Command(BaseCommand):
             """
             Upload file to S3
             """
-            s3 = boto.connect_s3()
+            s3 = boto.connect_s3(settings.AWS_ACCESS_KEY_ID,
+                    settings.AWS_SECRET_ACCESS_KEY)
             from boto.s3.key import Key
             b = s3.get_bucket(bucket)
             k = Key(b)
diff --git a/common/djangoapps/student/management/commands/pearson_export_cdd.py b/common/djangoapps/student/management/commands/pearson_export_cdd.py
index cebc098080..dcb9f5cd97 100644
--- a/common/djangoapps/student/management/commands/pearson_export_cdd.py
+++ b/common/djangoapps/student/management/commands/pearson_export_cdd.py
@@ -48,12 +48,12 @@ class Command(BaseCommand):
         # Name will use timestamp -- this is UTC, so it will look funny,
         # but it should at least be consistent with the other timestamps 
         # used in the system.
-        if not os.path.isdir(settings.PEARSON_LOCAL_EXPORT):
-            os.makedirs(settings.PEARSON_LOCAL_EXPORT)
-            destfile = os.path.join(settings.PEARSON_LOCAL_EXPORT,
+        if not os.path.isdir(settings.PEARSON[LOCAL_EXPORT]):
+            os.makedirs(settings.PEARSON[LOCAL_EXPORT])
+            destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT],
                     uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
         else:
-            destfile = os.path.join(settings.PEARSON_LOCAL_EXPORT,
+            destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT],
                     uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
 
         
diff --git a/common/djangoapps/student/management/commands/pearson_export_ead.py b/common/djangoapps/student/management/commands/pearson_export_ead.py
index 5423282346..9520e9d013 100644
--- a/common/djangoapps/student/management/commands/pearson_export_ead.py
+++ b/common/djangoapps/student/management/commands/pearson_export_ead.py
@@ -50,12 +50,12 @@ class Command(BaseCommand):
         # Name will use timestamp -- this is UTC, so it will look funny,
         # but it should at least be consistent with the other timestamps
         # used in the system.
-        if not os.path.isdir(settings.PEARSON_LOCAL_EXPORT):
-            os.makedirs(settings.PEARSON_LOCAL_EXPORT)
-            destfile = os.path.join(settings.PEARSON_LOCAL_EXPORT,
+        if not os.path.isdir(settings.PEARSON[LOCAL_EXPORT]):
+            os.makedirs(settings.PEARSON[LOCAL_EXPORT])
+            destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT],
                     uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
         else:
-            destfile = os.path.join(settings.PEARSON_LOCAL_EXPORT,
+            destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT],
                     uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
 
         dump_all = kwargs['dump_all']

From 61ea2d7b5da004053073c575c6bb4a8a066296dc Mon Sep 17 00:00:00 2001
From: Ashley Penney <apenney@edx.org>
Date: Wed, 16 Jan 2013 10:02:10 -0500
Subject: [PATCH 03/29] Couple of fixes to the settings data.

---
 .../student/management/commands/pearson.py           | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/common/djangoapps/student/management/commands/pearson.py b/common/djangoapps/student/management/commands/pearson.py
index a317b08d62..0752aea8be 100644
--- a/common/djangoapps/student/management/commands/pearson.py
+++ b/common/djangoapps/student/management/commands/pearson.py
@@ -25,11 +25,11 @@ class Command(BaseCommand):
 
         for mode in args:
             if mode == 'export':
-                sftp(settings.PEARSON[LOCAL_IMPORT], settings.PEARSON[SFTP_IMPORT])
-                s3(settings.PEARSON_LOCAL, settings.PEARSON[BUCKET])
-            elif mode == 'import':
-                sftp(settings.PEARSON[SFTP_EXPORT], settings.PEARSON[LOCAL_EXPORT])
+                sftp(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[SFTP_EXPORT])
                 s3(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[BUCKET])
+            elif mode == 'import':
+                sftp(settings.PEARSON[SFTP_IMPORT], settings.PEARSON[LOCAL_IMPORT])
+                s3(settings.PEARSON[LOCAL_IMPORT], settings.PEARSON[BUCKET])
             else:
                 print("ERROR:  Mode must be export or import.")
 
@@ -37,8 +37,8 @@ class Command(BaseCommand):
             with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'):
                 try:
                     t = paramiko.Transport((hostname, 22))
-                    t.connect(username=settings.PEARSON_SFTP_USERNAME,
-                              password=settings.PEARSON_SFTP_PASSWORD)
+                    t.connect(username=settings.PEARSON[SFTP_USERNAME],
+                              password=settings.PEARSON[SFTP_PASSWORD])
                     sftp = paramiko.SFTPClient.from_transport(t)
                     if os.path.isdir(files_from):
                         for file in os.listdir(files_from):

From ced29a25ece11ea50799515f22f307a1e19d74e7 Mon Sep 17 00:00:00 2001
From: Ashley Penney <apenney@edx.org>
Date: Wed, 16 Jan 2013 10:17:30 -0500
Subject: [PATCH 04/29] Add paramiko to requirements.

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 996388a51d..fa4688b711 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -58,4 +58,4 @@ factory_boy
 Shapely==1.2.16
 ipython==0.13.1
 xmltodict==0.4.1
-
+paramiko==1.9.0

From 378b1ff0c35c809558bb081f5acc09be9a516db4 Mon Sep 17 00:00:00 2001
From: Ashley Penney <apenney@edx.org>
Date: Wed, 16 Jan 2013 11:26:39 -0500
Subject: [PATCH 05/29] Various changes thanks to feedback from Brian to make
 the existing export commands handle --dest-from-settings and --destination
 and fail unless one is provided as well as rename pearson.py to
 pearson_transfer and allow is to call the import/export commands directly.

I've set it to die in pearson_transfer.py if the django PEARSON settings
aren't available.  I don't want to try and provide defaults, these must
exist or it simply fails.
---
 .../management/commands/pearson_export_cdd.py | 37 ++++++++++---
 .../management/commands/pearson_export_ead.py | 52 ++++++++++++-------
 .../{pearson.py => pearson_transfer.py}       | 50 +++++++++++-------
 3 files changed, 95 insertions(+), 44 deletions(-)
 rename common/djangoapps/student/management/commands/{pearson.py => pearson_transfer.py} (63%)

diff --git a/common/djangoapps/student/management/commands/pearson_export_cdd.py b/common/djangoapps/student/management/commands/pearson_export_cdd.py
index dcb9f5cd97..c2c13916ab 100644
--- a/common/djangoapps/student/management/commands/pearson_export_cdd.py
+++ b/common/djangoapps/student/management/commands/pearson_export_cdd.py
@@ -37,7 +37,20 @@ class Command(BaseCommand):
         ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
     ])
 
-    def handle(self, **kwargs):
+    option_list = BaseCommand.option_list + (
+        make_option('--dest-from-settings',
+                    action='store_true',
+                    dest='dest-from-settings',
+                    default=False,
+                    help='Retrieve the destination to export to from django? True/False'),
+        make_option('--destination',
+                    action='store_true',
+                    dest='destination',
+                    default=None,
+                    help='Where to store the exported files')
+    )
+
+    def handle(self, **options):
         # update time should use UTC in order to be comparable to the user_updated_at 
         # field
         uploaded_at = datetime.utcnow()
@@ -48,13 +61,23 @@ class Command(BaseCommand):
         # Name will use timestamp -- this is UTC, so it will look funny,
         # but it should at least be consistent with the other timestamps 
         # used in the system.
-        if not os.path.isdir(settings.PEARSON[LOCAL_EXPORT]):
-            os.makedirs(settings.PEARSON[LOCAL_EXPORT])
-            destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT],
-                    uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
+        if options['dest-from-settings'] is True:
+            if settings.PEARSON[LOCAL_EXPORT]:
+                dest = settings.PEARSON[LOCAL_EXPORT]
+            else:
+                raise CommandError('--dest-from-settings was enabled but the'
+                        'PEARSON[LOCAL_EXPORT] setting was not set.')
+        elif options['destination']:
+            dest = options['destination']
         else:
-            destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT],
-                    uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
+            raise ComamndError('--destination or --dest-from-settings must be used')
+                
+
+        if not os.path.isdir(dest):
+            os.makedirs(dest)
+            destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
+        else:
+            destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
 
         
         # strings must be in latin-1 format.  CSV parser will
diff --git a/common/djangoapps/student/management/commands/pearson_export_ead.py b/common/djangoapps/student/management/commands/pearson_export_ead.py
index 9520e9d013..6e3149778d 100644
--- a/common/djangoapps/student/management/commands/pearson_export_ead.py
+++ b/common/djangoapps/student/management/commands/pearson_export_ead.py
@@ -23,24 +23,29 @@ class Command(BaseCommand):
         ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
     ])
 
-
     option_list = BaseCommand.option_list + (
-        make_option(
-            '--dump_all',
-            action='store_true',
-            dest='dump_all',
+        make_option('--dest-from-settings',
+                    action='store_true',
+                    dest='dest-from-settings',
+                    default=False,
+                    help='Retrieve the destination to export to from django? True/False'),
+        make_option('--destination',
+                    action='store_true',
+                    dest='destination',
+                    default=None,
+                    help='Where to store the exported files'),
+        make_option('--dump_all',
+                    action='store_true',
+                    dest='dump_all',
         ),
-        make_option(
-            '--force_add',
-            action='store_true',
-            dest='force_add',
+        make_option('--force_add',
+                    action='store_true',
+                    dest='force_add',
         ),
     )
-    
-    
-    def handle(self, **kwargs):
 
-        # update time should use UTC in order to be comparable to the user_updated_at 
+    def handle(self, **options):
+        # update time should use UTC in order to be comparable to the user_updated_at
         # field
         uploaded_at = datetime.utcnow()
 
@@ -50,13 +55,22 @@ class Command(BaseCommand):
         # Name will use timestamp -- this is UTC, so it will look funny,
         # but it should at least be consistent with the other timestamps
         # used in the system.
-        if not os.path.isdir(settings.PEARSON[LOCAL_EXPORT]):
-            os.makedirs(settings.PEARSON[LOCAL_EXPORT])
-            destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT],
-                    uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
+        if options['dest-from-settings'] is True:
+            if settings.PEARSON[LOCAL_EXPORT]:
+                dest = settings.PEARSON[LOCAL_EXPORT]
+            else:
+                raise CommandError('--dest-from-settings was enabled but the'
+                        'PEARSON[LOCAL_EXPORT] setting was not set.')
+        elif options['destination']:
+            dest = options['destination']
         else:
-            destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT],
-                    uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
+            raise ComamndError('--destination or --dest-from-settings must be used')
+
+        if not os.path.isdir(dest):
+            os.makedirs(dest)
+            destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
+        else:
+            destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
 
         dump_all = kwargs['dump_all']
 
diff --git a/common/djangoapps/student/management/commands/pearson.py b/common/djangoapps/student/management/commands/pearson_transfer.py
similarity index 63%
rename from common/djangoapps/student/management/commands/pearson.py
rename to common/djangoapps/student/management/commands/pearson_transfer.py
index 0752aea8be..1d04936216 100644
--- a/common/djangoapps/student/management/commands/pearson.py
+++ b/common/djangoapps/student/management/commands/pearson_transfer.py
@@ -12,26 +12,40 @@ dog_http_api.api_key = settings.DATADOG_API
 
 class Command(BaseCommand):
 
-    option_list = BaseCommand.option_list
-    args = '<mode>'
-    help = """
-    Mode should be import or export depending on if you're fetching from pearson or
-    sending to them.
-    """
+    option_list = BaseCommand.option_list + (
+        make_option('--mode',
+                    action='store_true',
+                    dest='mode',
+                    default='both',
+                    help='mode is import, export, or both'),
+    )
 
-    def handle(self, *args):
-        if len(args) < 1:
-            raise CommandError('Usage is pearson {0}'.format(self.args))
+    def handle(self, **options):
+
+        if not settings.PEARSON:
+            raise CommandError('No PEARSON entries in auth/env.json.')
+
+        def import_pearson():
+            sftp(settings.PEARSON[SFTP_IMPORT], settings.PEARSON[LOCAL_IMPORT])
+            s3(settings.PEARSON[LOCAL_IMPORT], settings.PEARSON[BUCKET])
+            call_command('pearson_import', 'dest_from_settings=True')
+
+        def export_pearson():
+            call_command('pearson_export_ccd', 'dest_from_settings=True')
+            call_command('pearson_export_ead', 'dest_from_settings=True')
+            sftp(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[SFTP_EXPORT])
+            s3(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[BUCKET])
+
+        if options['mode'] == 'both':
+            export_pearson()
+            import_pearson()
+        elif options['mode'] == 'export':
+            export_pearson()
+        elif options['mode'] == 'import':
+            import_pearson()
+        else:
+            print("ERROR:  Mode must be export or import.")
 
-        for mode in args:
-            if mode == 'export':
-                sftp(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[SFTP_EXPORT])
-                s3(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[BUCKET])
-            elif mode == 'import':
-                sftp(settings.PEARSON[SFTP_IMPORT], settings.PEARSON[LOCAL_IMPORT])
-                s3(settings.PEARSON[LOCAL_IMPORT], settings.PEARSON[BUCKET])
-            else:
-                print("ERROR:  Mode must be export or import.")
 
         def sftp(files_from, files_to):
             with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'):

From 0555ebc4347c9eaa0943067386f55b3d0bf6d139 Mon Sep 17 00:00:00 2001
From: Ashley Penney <apenney@edx.org>
Date: Wed, 16 Jan 2013 12:04:53 -0500
Subject: [PATCH 06/29] Bunch of fixes to how I do if/else checks, fix a typo
 in Command and repair the for filename part of sftp.

---
 .../management/commands/pearson_export_cdd.py     | 14 ++++++--------
 .../management/commands/pearson_export_ead.py     | 15 +++++++--------
 .../management/commands/pearson_transfer.py       |  6 +++---
 3 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/common/djangoapps/student/management/commands/pearson_export_cdd.py b/common/djangoapps/student/management/commands/pearson_export_cdd.py
index c2c13916ab..2655960ff5 100644
--- a/common/djangoapps/student/management/commands/pearson_export_cdd.py
+++ b/common/djangoapps/student/management/commands/pearson_export_cdd.py
@@ -44,7 +44,7 @@ class Command(BaseCommand):
                     default=False,
                     help='Retrieve the destination to export to from django? True/False'),
         make_option('--destination',
-                    action='store_true',
+                    action='store',
                     dest='destination',
                     default=None,
                     help='Where to store the exported files')
@@ -61,24 +61,22 @@ class Command(BaseCommand):
         # Name will use timestamp -- this is UTC, so it will look funny,
         # but it should at least be consistent with the other timestamps 
         # used in the system.
-        if options['dest-from-settings'] is True:
-            if settings.PEARSON[LOCAL_EXPORT]:
+        if 'dest-from-settings' in options:
+            if LOCAL_EXPORT in settings.PEARSON:
                 dest = settings.PEARSON[LOCAL_EXPORT]
             else:
                 raise CommandError('--dest-from-settings was enabled but the'
                         'PEARSON[LOCAL_EXPORT] setting was not set.')
-        elif options['destination']:
+        elif 'destination' in options:
             dest = options['destination']
         else:
-            raise ComamndError('--destination or --dest-from-settings must be used')
+            raise CommandError('--destination or --dest-from-settings must be used')
                 
 
         if not os.path.isdir(dest):
             os.makedirs(dest)
-            destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
-        else:
-            destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
 
+        destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
         
         # strings must be in latin-1 format.  CSV parser will
         # otherwise convert unicode objects to ascii.
diff --git a/common/djangoapps/student/management/commands/pearson_export_ead.py b/common/djangoapps/student/management/commands/pearson_export_ead.py
index 6e3149778d..7697c99e4a 100644
--- a/common/djangoapps/student/management/commands/pearson_export_ead.py
+++ b/common/djangoapps/student/management/commands/pearson_export_ead.py
@@ -30,7 +30,7 @@ class Command(BaseCommand):
                     default=False,
                     help='Retrieve the destination to export to from django? True/False'),
         make_option('--destination',
-                    action='store_true',
+                    action='store',
                     dest='destination',
                     default=None,
                     help='Where to store the exported files'),
@@ -55,22 +55,21 @@ class Command(BaseCommand):
         # Name will use timestamp -- this is UTC, so it will look funny,
         # but it should at least be consistent with the other timestamps
         # used in the system.
-        if options['dest-from-settings'] is True:
-            if settings.PEARSON[LOCAL_EXPORT]:
+        if 'dest-from-settings' in options:
+            if LOCAL_EXPORT in settings.PEARSON:
                 dest = settings.PEARSON[LOCAL_EXPORT]
             else:
                 raise CommandError('--dest-from-settings was enabled but the'
                         'PEARSON[LOCAL_EXPORT] setting was not set.')
-        elif options['destination']:
+        elif destinations in options:
             dest = options['destination']
         else:
-            raise ComamndError('--destination or --dest-from-settings must be used')
+            raise CommandError('--destination or --dest-from-settings must be used')
 
         if not os.path.isdir(dest):
             os.makedirs(dest)
-            destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
-        else:
-            destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
+
+        destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
 
         dump_all = kwargs['dump_all']
 
diff --git a/common/djangoapps/student/management/commands/pearson_transfer.py b/common/djangoapps/student/management/commands/pearson_transfer.py
index 1d04936216..7d4a2ead3f 100644
--- a/common/djangoapps/student/management/commands/pearson_transfer.py
+++ b/common/djangoapps/student/management/commands/pearson_transfer.py
@@ -14,7 +14,7 @@ class Command(BaseCommand):
 
     option_list = BaseCommand.option_list + (
         make_option('--mode',
-                    action='store_true',
+                    action='store',
                     dest='mode',
                     default='both',
                     help='mode is import, export, or both'),
@@ -55,11 +55,11 @@ class Command(BaseCommand):
                               password=settings.PEARSON[SFTP_PASSWORD])
                     sftp = paramiko.SFTPClient.from_transport(t)
                     if os.path.isdir(files_from):
-                        for file in os.listdir(files_from):
+                        for filename in os.listdir(files_from):
                             sftp.put(files_from+'/'+filename,
                                      files_to+'/'+filename)
                     else:
-                        for file in sftp.listdir(files_from):
+                        for filename in sftp.listdir(files_from):
                             sftp.get(files_from+'/'+filename,
                                      files_to+'/'+filename)
                 except:

From 3d5599c829fb8d7d06e89ce331b67fa6c419997d Mon Sep 17 00:00:00 2001
From: Ashley Penney <apenney@edx.org>
Date: Wed, 16 Jan 2013 12:12:22 -0500
Subject: [PATCH 07/29] Further fixes to close the ssh connection explictly,
 make both the default option if nothing is provided, and not bother passing
 true when calling the subcommands.

---
 .../management/commands/pearson_transfer.py       | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/common/djangoapps/student/management/commands/pearson_transfer.py b/common/djangoapps/student/management/commands/pearson_transfer.py
index 7d4a2ead3f..d07f75d011 100644
--- a/common/djangoapps/student/management/commands/pearson_transfer.py
+++ b/common/djangoapps/student/management/commands/pearson_transfer.py
@@ -28,23 +28,21 @@ class Command(BaseCommand):
         def import_pearson():
             sftp(settings.PEARSON[SFTP_IMPORT], settings.PEARSON[LOCAL_IMPORT])
             s3(settings.PEARSON[LOCAL_IMPORT], settings.PEARSON[BUCKET])
-            call_command('pearson_import', 'dest_from_settings=True')
+            call_command('pearson_import', 'dest_from_settings')
 
         def export_pearson():
-            call_command('pearson_export_ccd', 'dest_from_settings=True')
-            call_command('pearson_export_ead', 'dest_from_settings=True')
+            call_command('pearson_export_ccd', 'dest_from_settings')
+            call_command('pearson_export_ead', 'dest_from_settings')
             sftp(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[SFTP_EXPORT])
             s3(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[BUCKET])
 
-        if options['mode'] == 'both':
-            export_pearson()
-            import_pearson()
-        elif options['mode'] == 'export':
+        if options['mode'] == 'export':
             export_pearson()
         elif options['mode'] == 'import':
             import_pearson()
         else:
-            print("ERROR:  Mode must be export or import.")
+            export_pearson()
+            import_pearson()
 
 
         def sftp(files_from, files_to):
@@ -62,6 +60,7 @@ class Command(BaseCommand):
                         for filename in sftp.listdir(files_from):
                             sftp.get(files_from+'/'+filename,
                                      files_to+'/'+filename)
+                    t.close()
                 except:
                     dog_http_api.event('pearson {0}'.format(mode),
                                        'sftp uploading failed', alert_type='error')

From 5431332cecf5773f33b56856df2f41c0b0b6156f Mon Sep 17 00:00:00 2001
From: Ashley Penney <apenney@edx.org>
Date: Wed, 16 Jan 2013 12:14:43 -0500
Subject: [PATCH 08/29] Fix up help.

---
 .../student/management/commands/pearson_export_cdd.py           | 2 +-
 .../student/management/commands/pearson_export_ead.py           | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/common/djangoapps/student/management/commands/pearson_export_cdd.py b/common/djangoapps/student/management/commands/pearson_export_cdd.py
index 2655960ff5..2f68ccc7d1 100644
--- a/common/djangoapps/student/management/commands/pearson_export_cdd.py
+++ b/common/djangoapps/student/management/commands/pearson_export_cdd.py
@@ -42,7 +42,7 @@ class Command(BaseCommand):
                     action='store_true',
                     dest='dest-from-settings',
                     default=False,
-                    help='Retrieve the destination to export to from django? True/False'),
+                    help='Retrieve the destination to export to from django.'),
         make_option('--destination',
                     action='store',
                     dest='destination',
diff --git a/common/djangoapps/student/management/commands/pearson_export_ead.py b/common/djangoapps/student/management/commands/pearson_export_ead.py
index 7697c99e4a..0f81cb1df9 100644
--- a/common/djangoapps/student/management/commands/pearson_export_ead.py
+++ b/common/djangoapps/student/management/commands/pearson_export_ead.py
@@ -28,7 +28,7 @@ class Command(BaseCommand):
                     action='store_true',
                     dest='dest-from-settings',
                     default=False,
-                    help='Retrieve the destination to export to from django? True/False'),
+                    help='Retrieve the destination to export to from django.'),
         make_option('--destination',
                     action='store',
                     dest='destination',

From 482cefd246539169d3c0de8704af29ca6e3c5bf2 Mon Sep 17 00:00:00 2001
From: Ashley Penney <apenney@edx.org>
Date: Wed, 16 Jan 2013 13:37:33 -0500
Subject: [PATCH 09/29] Bunch of fixes to pep8 formatting, missing imports,
 change of kwargs to options, quoting variables and a few other fixups.

---
 .../management/commands/pearson_export_cdd.py | 27 +++++-----
 .../management/commands/pearson_export_ead.py | 27 +++++-----
 .../management/commands/pearson_transfer.py   | 53 +++++++++++--------
 3 files changed, 55 insertions(+), 52 deletions(-)

diff --git a/common/djangoapps/student/management/commands/pearson_export_cdd.py b/common/djangoapps/student/management/commands/pearson_export_cdd.py
index 2f68ccc7d1..14c652f11e 100644
--- a/common/djangoapps/student/management/commands/pearson_export_cdd.py
+++ b/common/djangoapps/student/management/commands/pearson_export_cdd.py
@@ -1,15 +1,16 @@
 import csv
+import os
 from collections import OrderedDict
 from datetime import datetime
-from os.path import isdir
 from optparse import make_option
 
 from django.core.management.base import BaseCommand
 
 from student.models import TestCenterUser
 
+
 class Command(BaseCommand):
-    
+
     CSV_TO_MODEL_FIELDS = OrderedDict([
         # Skipping optional field CandidateID
         ("ClientCandidateID", "client_candidate_id"),
@@ -34,7 +35,7 @@ class Command(BaseCommand):
         ("FAXCountryCode", "fax_country_code"),
         ("CompanyName", "company_name"),
         # Skipping optional field CustomQuestion
-        ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
+        ("LastUpdate", "user_updated_at"),  # in UTC, so same as what we store
     ])
 
     option_list = BaseCommand.option_list + (
@@ -55,29 +56,28 @@ class Command(BaseCommand):
         # field
         uploaded_at = datetime.utcnow()
 
-        # if specified destination is an existing directory, then 
+        # if specified destination is an existing directory, then
         # create a filename for it automatically.  If it doesn't exist,
         # then we will create the directory.
         # Name will use timestamp -- this is UTC, so it will look funny,
-        # but it should at least be consistent with the other timestamps 
+        # but it should at least be consistent with the other timestamps
         # used in the system.
         if 'dest-from-settings' in options:
-            if LOCAL_EXPORT in settings.PEARSON:
-                dest = settings.PEARSON[LOCAL_EXPORT]
+            if 'LOCAL_EXPORT' in settings.PEARSON:
+                dest = settings.PEARSON['LOCAL_EXPORT']
             else:
                 raise CommandError('--dest-from-settings was enabled but the'
-                        'PEARSON[LOCAL_EXPORT] setting was not set.')
+                                   'PEARSON[LOCAL_EXPORT] setting was not set.')
         elif 'destination' in options:
             dest = options['destination']
         else:
             raise CommandError('--destination or --dest-from-settings must be used')
-                
 
         if not os.path.isdir(dest):
             os.makedirs(dest)
 
         destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
-        
+
         # strings must be in latin-1 format.  CSV parser will
         # otherwise convert unicode objects to ascii.
         def ensure_encoding(value):
@@ -85,8 +85,8 @@ class Command(BaseCommand):
                 return value.encode('iso-8859-1')
             else:
                 return value
-            
-        dump_all = kwargs['dump_all']
+
+        dump_all = options['dump_all']
 
         with open(destfile, "wb") as outfile:
             writer = csv.DictWriter(outfile,
@@ -104,6 +104,3 @@ class Command(BaseCommand):
                     writer.writerow(record)
                     tcu.uploaded_at = uploaded_at
                     tcu.save()
-
-
-        
diff --git a/common/djangoapps/student/management/commands/pearson_export_ead.py b/common/djangoapps/student/management/commands/pearson_export_ead.py
index 0f81cb1df9..9368ac5ddf 100644
--- a/common/djangoapps/student/management/commands/pearson_export_ead.py
+++ b/common/djangoapps/student/management/commands/pearson_export_ead.py
@@ -1,15 +1,16 @@
 import csv
+import os
 from collections import OrderedDict
 from datetime import datetime
-from os.path import isdir, join
 from optparse import make_option
 
 from django.core.management.base import BaseCommand
 
 from student.models import TestCenterRegistration
 
+
 class Command(BaseCommand):
-    
+
     CSV_TO_MODEL_FIELDS = OrderedDict([
         ('AuthorizationTransactionType', 'authorization_transaction_type'),
         ('AuthorizationID', 'authorization_id'),
@@ -20,7 +21,7 @@ class Command(BaseCommand):
         ('Accommodations', 'accommodation_code'),
         ('EligibilityApptDateFirst', 'eligibility_appointment_date_first'),
         ('EligibilityApptDateLast', 'eligibility_appointment_date_last'),
-        ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
+        ("LastUpdate", "user_updated_at"),  # in UTC, so same as what we store
     ])
 
     option_list = BaseCommand.option_list + (
@@ -37,11 +38,11 @@ class Command(BaseCommand):
         make_option('--dump_all',
                     action='store_true',
                     dest='dump_all',
-        ),
+                    ),
         make_option('--force_add',
                     action='store_true',
                     dest='force_add',
-        ),
+                    ),
     )
 
     def handle(self, **options):
@@ -56,12 +57,12 @@ class Command(BaseCommand):
         # but it should at least be consistent with the other timestamps
         # used in the system.
         if 'dest-from-settings' in options:
-            if LOCAL_EXPORT in settings.PEARSON:
-                dest = settings.PEARSON[LOCAL_EXPORT]
+            if 'LOCAL_EXPORT' in settings.PEARSON:
+                dest = settings.PEARSON['LOCAL_EXPORT']
             else:
                 raise CommandError('--dest-from-settings was enabled but the'
-                        'PEARSON[LOCAL_EXPORT] setting was not set.')
-        elif destinations in options:
+                                   'PEARSON[LOCAL_EXPORT] setting was not set.')
+        elif 'destinations' in options:
             dest = options['destination']
         else:
             raise CommandError('--destination or --dest-from-settings must be used')
@@ -71,7 +72,7 @@ class Command(BaseCommand):
 
         destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
 
-        dump_all = kwargs['dump_all']
+        dump_all = options['dump_all']
 
         with open(destfile, "wb") as outfile:
             writer = csv.DictWriter(outfile,
@@ -88,13 +89,9 @@ class Command(BaseCommand):
                     record["LastUpdate"] = record["LastUpdate"].strftime("%Y/%m/%d %H:%M:%S")
                     record["EligibilityApptDateFirst"] = record["EligibilityApptDateFirst"].strftime("%Y/%m/%d")
                     record["EligibilityApptDateLast"] = record["EligibilityApptDateLast"].strftime("%Y/%m/%d")
-                    if kwargs['force_add']:
+                    if options['force_add']:
                         record['AuthorizationTransactionType'] = 'Add'
 
                     writer.writerow(record)
                     tcr.uploaded_at = uploaded_at
                     tcr.save()
-
-
-
-        
diff --git a/common/djangoapps/student/management/commands/pearson_transfer.py b/common/djangoapps/student/management/commands/pearson_transfer.py
index d07f75d011..8183bf3c05 100644
--- a/common/djangoapps/student/management/commands/pearson_transfer.py
+++ b/common/djangoapps/student/management/commands/pearson_transfer.py
@@ -1,11 +1,10 @@
 from optparse import make_option
 
-from django.contrib.auth.models import User
 from django.core.management.base import BaseCommand, CommandError
-import re
 from dogapi import dog_http_api, dog_stats_api
 import paramiko
 import boto
+import os
 
 dog_http_api.api_key = settings.DATADOG_API
 
@@ -25,16 +24,26 @@ class Command(BaseCommand):
         if not settings.PEARSON:
             raise CommandError('No PEARSON entries in auth/env.json.')
 
+        for value in ['LOCAL_IMPORT', 'SFTP_IMPORT', 'BUCKET', 'LOCAL_EXPORT',
+                      'SFTP_EXPORT']:
+            if value not in settings.PEARSON:
+                raise CommandError('No entry in the PEARSON settings'
+                                   '(env/auth.json) for {0}'.format(value))
+
         def import_pearson():
-            sftp(settings.PEARSON[SFTP_IMPORT], settings.PEARSON[LOCAL_IMPORT])
-            s3(settings.PEARSON[LOCAL_IMPORT], settings.PEARSON[BUCKET])
+            sftp(settings.PEARSON['SFTP_IMPORT'],
+                 settings.PEARSON['LOCAL_IMPORT'], options['mode'])
+            s3(settings.PEARSON['LOCAL_IMPORT'],
+               settings.PEARSON['BUCKET'], options['mode'])
             call_command('pearson_import', 'dest_from_settings')
 
         def export_pearson():
             call_command('pearson_export_ccd', 'dest_from_settings')
             call_command('pearson_export_ead', 'dest_from_settings')
-            sftp(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[SFTP_EXPORT])
-            s3(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[BUCKET])
+            sftp(settings.PEARSON['LOCAL_EXPORT'],
+                 settings.PEARSON['SFTP_EXPORT'], options['mode'])
+            s3(settings.PEARSON['LOCAL_EXPORT'],
+               settings.PEARSON['BUCKET'], options['mode'])
 
         if options['mode'] == 'export':
             export_pearson()
@@ -44,44 +53,44 @@ class Command(BaseCommand):
             export_pearson()
             import_pearson()
 
-
-        def sftp(files_from, files_to):
+        def sftp(files_from, files_to, mode):
             with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'):
                 try:
-                    t = paramiko.Transport((hostname, 22))
-                    t.connect(username=settings.PEARSON[SFTP_USERNAME],
-                              password=settings.PEARSON[SFTP_PASSWORD])
+                    t = paramiko.Transport((settings.PEARSON['SFTP_HOSTNAME'], 22))
+                    t.connect(username=settings.PEARSON['SFTP_USERNAME'],
+                              password=settings.PEARSON['SFTP_PASSWORD'])
                     sftp = paramiko.SFTPClient.from_transport(t)
                     if os.path.isdir(files_from):
                         for filename in os.listdir(files_from):
-                            sftp.put(files_from+'/'+filename,
-                                     files_to+'/'+filename)
+                            sftp.put(files_from + '/' + filename,
+                                     files_to + '/' + filename)
                     else:
                         for filename in sftp.listdir(files_from):
-                            sftp.get(files_from+'/'+filename,
-                                     files_to+'/'+filename)
+                            sftp.get(files_from + '/' + filename,
+                                     files_to + '/' + filename)
                     t.close()
                 except:
                     dog_http_api.event('pearson {0}'.format(mode),
-                                       'sftp uploading failed', alert_type='error')
+                                       'sftp uploading failed',
+                                       alert_type='error')
                     raise
 
-        def s3(files_from, bucket):
+        def s3(files_from, bucket, mode):
             with dog_stats_api.timer('pearson.{0}'.format(mode), tags='s3'):
                 try:
-                    for filename in os.listdir(files):
-                        upload_file_to_s3(bucket, files_from+'/'+filename)
+                    for filename in os.listdir(files_from):
+                        upload_file_to_s3(bucket, files_from + '/' + filename)
                 except:
-                    dog_http_api.event('pearson {0}'.format(mode), 's3 archiving failed')
+                    dog_http_api.event('pearson {0}'.format(mode),
+                                       's3 archiving failed')
                     raise
 
-
         def upload_file_to_s3(bucket, filename):
             """
             Upload file to S3
             """
             s3 = boto.connect_s3(settings.AWS_ACCESS_KEY_ID,
-                    settings.AWS_SECRET_ACCESS_KEY)
+                                 settings.AWS_SECRET_ACCESS_KEY)
             from boto.s3.key import Key
             b = s3.get_bucket(bucket)
             k = Key(b)

From 46e9a8f6ac58c1a79ee0097cd6fbfd829eae9b1d Mon Sep 17 00:00:00 2001
From: Ashley Penney <apenney@edx.org>
Date: Thu, 17 Jan 2013 10:09:11 -0500
Subject: [PATCH 10/29] Add help string.

---
 .../student/management/commands/pearson_transfer.py       | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/common/djangoapps/student/management/commands/pearson_transfer.py b/common/djangoapps/student/management/commands/pearson_transfer.py
index 8183bf3c05..4888806c73 100644
--- a/common/djangoapps/student/management/commands/pearson_transfer.py
+++ b/common/djangoapps/student/management/commands/pearson_transfer.py
@@ -10,6 +10,14 @@ dog_http_api.api_key = settings.DATADOG_API
 
 
 class Command(BaseCommand):
+    help = """
+    This command handles the importing and exporting of student records for
+    Pearson.  It uses some other Django commands to export and import the
+    files and then uploads over SFTP to pearson and stuffs the entry in an
+    S3 bucket for archive purposes.
+
+    Usage: django-admin.py pearson-transfer --mode [import|export|both]
+    """
 
     option_list = BaseCommand.option_list + (
         make_option('--mode',

From 5e76c9d033960221bb167d8e32caeae5898d93f5 Mon Sep 17 00:00:00 2001
From: Brian Wilson <brian@edx.org>
Date: Thu, 17 Jan 2013 10:43:20 -0500
Subject: [PATCH 11/29] add pearson_import_conf_zip command

---
 .../commands/pearson_import_conf_zip.py       | 108 ++++++++++++++++++
 1 file changed, 108 insertions(+)
 create mode 100644 common/djangoapps/student/management/commands/pearson_import_conf_zip.py

diff --git a/common/djangoapps/student/management/commands/pearson_import_conf_zip.py b/common/djangoapps/student/management/commands/pearson_import_conf_zip.py
new file mode 100644
index 0000000000..7b69a29dc2
--- /dev/null
+++ b/common/djangoapps/student/management/commands/pearson_import_conf_zip.py
@@ -0,0 +1,108 @@
+import csv
+
+from zipfile import ZipFile, is_zipfile
+from time import strptime, strftime
+
+from collections import OrderedDict
+from datetime import datetime
+from os.path import isdir
+from optparse import make_option
+
+from django.core.management.base import BaseCommand, CommandError
+
+from student.models import TestCenterUser, TestCenterRegistration
+
+class Command(BaseCommand):
+    
+    
+    args = '<input zip file>'
+    help = """
+    Import Pearson confirmation files and update TestCenterUser and TestCenterRegistration tables
+    with status.
+    """
+    def handle(self, *args, **kwargs):
+        if len(args) < 1:
+            print Command.help
+            return
+
+        source_zip = args[0]
+        if not is_zipfile(source_zip):
+            raise CommandError("Input file is not a zipfile: \"{}\"".format(source_zip))        
+        
+        # loop through all files in zip, and process them based on filename prefix:
+        with ZipFile(source_zip, 'r') as zipfile:
+            for fileinfo in zipfile.infolist():
+                with zipfile.open(fileinfo) as zipentry:
+                    if fileinfo.filename.startswith("eac-"):
+                        self.process_eac(zipentry)
+                    elif fileinfo.filename.startswith("vcdc-"):
+                        self.process_vcdc(zipentry)
+                    else:
+                        raise CommandError("Unrecognized confirmation file type \"{}\" in confirmation zip file \"{}\"".format(fileinfo.filename, zipfile))
+        
+    def process_eac(self, eacfile):       
+        print "processing eac"
+        reader = csv.DictReader(eacfile, delimiter="\t")
+        for row in reader:
+            client_authorization_id = row['ClientAuthorizationID']
+            if not client_authorization_id:
+                if row['Status'] == 'Error':
+                    print "Error in EAD file processing ({}): {}".format(row['Date'], row['Message'])
+                else:
+                    print "Encountered bad record: {}".format(row)
+            else:
+                try:
+                    registration = TestCenterRegistration.objects.get(client_authorization_id=client_authorization_id)
+                    print "Found authorization record for user {}".format(registration.testcenter_user.user.username)
+                    # now update the record:
+                    registration.upload_status = row['Status']
+                    registration.upload_error_message =  row['Message']
+                    try:
+                        registration.processed_at = strftime('%Y-%m-%d %H:%M:%S', strptime(row['Date'], '%Y/%m/%d %H:%M:%S'))
+                    except ValueError as ve:
+                        print "Bad Date value found for {}: message {}".format(client_authorization_id, ve)
+                    # store the authorization Id if one is provided.  (For debugging)
+                    if row['AuthorizationID']:
+                        try:
+                            registration.authorization_id = int(row['AuthorizationID'])
+                        except ValueError as ve:
+                            print "Bad AuthorizationID value found for {}: message {}".format(client_authorization_id, ve)
+                                
+                    registration.confirmed_at = datetime.utcnow()
+                    registration.save()
+                except TestCenterRegistration.DoesNotExist:
+                    print " Failed to find record for client_auth_id {}".format(client_authorization_id)
+
+        
+    def process_vcdc(self, vcdcfile):       
+        print "processing vcdc"
+        reader = csv.DictReader(vcdcfile, delimiter="\t")
+        for row in reader:
+            client_candidate_id = row['ClientCandidateID']
+            if not client_candidate_id:
+                if row['Status'] == 'Error':
+                    print "Error in CDD file processing ({}): {}".format(row['Date'], row['Message'])
+                else:
+                    print "Encountered bad record: {}".format(row)
+            else:
+                try:
+                    tcuser = TestCenterUser.objects.get(client_candidate_id=client_candidate_id)
+                    print "Found demographics record for user {}".format(tcuser.user.username)
+                    # now update the record:
+                    tcuser.upload_status = row['Status']
+                    tcuser.upload_error_message =  row['Message']
+                    try:
+                        tcuser.processed_at = strftime('%Y-%m-%d %H:%M:%S', strptime(row['Date'], '%Y/%m/%d %H:%M:%S'))
+                    except ValueError as ve:
+                        print "Bad Date value found for {}: message {}".format(client_candidate_id, ve)
+                    # store the candidate Id if one is provided.  (For debugging)
+                    if row['CandidateID']:
+                        try:
+                            tcuser.candidate_id = int(row['CandidateID'])
+                        except ValueError as ve:
+                            print "Bad CandidateID value found for {}: message {}".format(client_candidate_id, ve)
+                    tcuser.confirmed_at = datetime.utcnow()
+                    tcuser.save()
+                except TestCenterUser.DoesNotExist:
+                    print " Failed to find record for client_candidate_id {}".format(client_candidate_id)
+        

From 333f2e5167556043e2f09de06aa297ab08ee50de Mon Sep 17 00:00:00 2001
From: Ashley Penney <apenney@edx.org>
Date: Thu, 17 Jan 2013 10:48:16 -0500
Subject: [PATCH 12/29] Fix ccd->cdd, I typed ccd so many times while working
 on this code.  I am bad at typing!

---
 .../djangoapps/student/management/commands/pearson_transfer.py  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/djangoapps/student/management/commands/pearson_transfer.py b/common/djangoapps/student/management/commands/pearson_transfer.py
index 4888806c73..57401897dc 100644
--- a/common/djangoapps/student/management/commands/pearson_transfer.py
+++ b/common/djangoapps/student/management/commands/pearson_transfer.py
@@ -46,7 +46,7 @@ class Command(BaseCommand):
             call_command('pearson_import', 'dest_from_settings')
 
         def export_pearson():
-            call_command('pearson_export_ccd', 'dest_from_settings')
+            call_command('pearson_export_cdd', 'dest_from_settings')
             call_command('pearson_export_ead', 'dest_from_settings')
             sftp(settings.PEARSON['LOCAL_EXPORT'],
                  settings.PEARSON['SFTP_EXPORT'], options['mode'])

From 3a4091b798df7d4c1f7c9f1f3686eac77235cba3 Mon Sep 17 00:00:00 2001
From: Ashley Penney <apenney@edx.org>
Date: Thu, 17 Jan 2013 13:54:43 -0500
Subject: [PATCH 13/29] Tweaks to enable datadog error events as well as some
 pep8 tidyup vim was shouting at me about.

---
 .../commands/pearson_import_conf_zip.py       | 55 ++++++++++---------
 1 file changed, 30 insertions(+), 25 deletions(-)

diff --git a/common/djangoapps/student/management/commands/pearson_import_conf_zip.py b/common/djangoapps/student/management/commands/pearson_import_conf_zip.py
index 7b69a29dc2..74037b708f 100644
--- a/common/djangoapps/student/management/commands/pearson_import_conf_zip.py
+++ b/common/djangoapps/student/management/commands/pearson_import_conf_zip.py
@@ -7,19 +7,25 @@ from collections import OrderedDict
 from datetime import datetime
 from os.path import isdir
 from optparse import make_option
+from dogapi import dog_http_api, dog_stats_api
 
 from django.core.management.base import BaseCommand, CommandError
 
 from student.models import TestCenterUser, TestCenterRegistration
 
+
 class Command(BaseCommand):
-    
-    
+
+    dog_http_api.api_key = settings.DATADOG_API
     args = '<input zip file>'
     help = """
-    Import Pearson confirmation files and update TestCenterUser and TestCenterRegistration tables
-    with status.
+    Import Pearson confirmation files and update TestCenterUser
+    and TestCenterRegistration tables with status.
     """
+
+    def datadog_error(string):
+        dog_http_api.event("Pearson Import", string, alert_type='error')
+
     def handle(self, *args, **kwargs):
         if len(args) < 1:
             print Command.help
@@ -27,8 +33,8 @@ class Command(BaseCommand):
 
         source_zip = args[0]
         if not is_zipfile(source_zip):
-            raise CommandError("Input file is not a zipfile: \"{}\"".format(source_zip))        
-        
+            raise CommandError("Input file is not a zipfile: \"{}\"".format(source_zip))
+
         # loop through all files in zip, and process them based on filename prefix:
         with ZipFile(source_zip, 'r') as zipfile:
             for fileinfo in zipfile.infolist():
@@ -39,70 +45,69 @@ class Command(BaseCommand):
                         self.process_vcdc(zipentry)
                     else:
                         raise CommandError("Unrecognized confirmation file type \"{}\" in confirmation zip file \"{}\"".format(fileinfo.filename, zipfile))
-        
-    def process_eac(self, eacfile):       
+
+    def process_eac(self, eacfile):
         print "processing eac"
         reader = csv.DictReader(eacfile, delimiter="\t")
         for row in reader:
             client_authorization_id = row['ClientAuthorizationID']
             if not client_authorization_id:
                 if row['Status'] == 'Error':
-                    print "Error in EAD file processing ({}): {}".format(row['Date'], row['Message'])
+                    Command.datadog_error("Error in EAD file processing ({}): {}".format(row['Date'], row['Message']))
                 else:
-                    print "Encountered bad record: {}".format(row)
+                    Command.datadog_error("Encountered bad record: {}".format(row))
             else:
                 try:
                     registration = TestCenterRegistration.objects.get(client_authorization_id=client_authorization_id)
-                    print "Found authorization record for user {}".format(registration.testcenter_user.user.username)
+                    Command.datadog_error("Found authorization record for user {}".format(registration.testcenter_user.user.username))
                     # now update the record:
                     registration.upload_status = row['Status']
                     registration.upload_error_message =  row['Message']
                     try:
                         registration.processed_at = strftime('%Y-%m-%d %H:%M:%S', strptime(row['Date'], '%Y/%m/%d %H:%M:%S'))
                     except ValueError as ve:
-                        print "Bad Date value found for {}: message {}".format(client_authorization_id, ve)
+                        Command.datadog_error("Bad Date value found for {}: message {}".format(client_authorization_id, ve))
                     # store the authorization Id if one is provided.  (For debugging)
                     if row['AuthorizationID']:
                         try:
                             registration.authorization_id = int(row['AuthorizationID'])
                         except ValueError as ve:
-                            print "Bad AuthorizationID value found for {}: message {}".format(client_authorization_id, ve)
-                                
+                            Command.datadog_error("Bad AuthorizationID value found for {}: message {}".format(client_authorization_id, ve))
+
                     registration.confirmed_at = datetime.utcnow()
                     registration.save()
                 except TestCenterRegistration.DoesNotExist:
-                    print " Failed to find record for client_auth_id {}".format(client_authorization_id)
+                    Command.datadog_error("Failed to find record for client_auth_id {}".format(client_authorization_id))
 
-        
-    def process_vcdc(self, vcdcfile):       
+    def process_vcdc(self, vcdcfile):
         print "processing vcdc"
         reader = csv.DictReader(vcdcfile, delimiter="\t")
         for row in reader:
             client_candidate_id = row['ClientCandidateID']
             if not client_candidate_id:
                 if row['Status'] == 'Error':
-                    print "Error in CDD file processing ({}): {}".format(row['Date'], row['Message'])
+                    Command.datadog_error("Error in CDD file processing ({}): {}".format(row['Date'], row['Message']))
                 else:
-                    print "Encountered bad record: {}".format(row)
+                    Command.datadog_error("Encountered bad record: {}".format(row))
             else:
                 try:
                     tcuser = TestCenterUser.objects.get(client_candidate_id=client_candidate_id)
-                    print "Found demographics record for user {}".format(tcuser.user.username)
+                    Command.datadog_error("Found demographics record for user {}".format(tcuser.user.username))
                     # now update the record:
                     tcuser.upload_status = row['Status']
-                    tcuser.upload_error_message =  row['Message']
+                    tcuser.upload_error_message = row['Message']
                     try:
                         tcuser.processed_at = strftime('%Y-%m-%d %H:%M:%S', strptime(row['Date'], '%Y/%m/%d %H:%M:%S'))
                     except ValueError as ve:
-                        print "Bad Date value found for {}: message {}".format(client_candidate_id, ve)
+                        Command.datadog_error("Bad Date value found for {}: message {}".format(client_candidate_id, ve))
                     # store the candidate Id if one is provided.  (For debugging)
                     if row['CandidateID']:
                         try:
                             tcuser.candidate_id = int(row['CandidateID'])
                         except ValueError as ve:
-                            print "Bad CandidateID value found for {}: message {}".format(client_candidate_id, ve)
+                            Command.datadog_error("Bad CandidateID value found for {}: message {}".format(client_candidate_id, ve))
                     tcuser.confirmed_at = datetime.utcnow()
                     tcuser.save()
                 except TestCenterUser.DoesNotExist:
-                    print " Failed to find record for client_candidate_id {}".format(client_candidate_id)
-        
+                    Command.datadog_error(" Failed to find record for client_candidate_id {}".format(client_candidate_id))
+

From 05b36bdd092de2294381fb2fcbbc2fbb03b63ed6 Mon Sep 17 00:00:00 2001
From: Ashley Penney <apenney@edx.org>
Date: Thu, 17 Jan 2013 15:43:37 -0500
Subject: [PATCH 14/29] Add tags to the datadog event which will be set to the
 appropriate filename so we can see the cause of failures.

---
 .../commands/pearson_import_conf_zip.py       | 32 ++++++++++---------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/common/djangoapps/student/management/commands/pearson_import_conf_zip.py b/common/djangoapps/student/management/commands/pearson_import_conf_zip.py
index 74037b708f..a491a73868 100644
--- a/common/djangoapps/student/management/commands/pearson_import_conf_zip.py
+++ b/common/djangoapps/student/management/commands/pearson_import_conf_zip.py
@@ -23,8 +23,8 @@ class Command(BaseCommand):
     and TestCenterRegistration tables with status.
     """
 
-    def datadog_error(string):
-        dog_http_api.event("Pearson Import", string, alert_type='error')
+    def datadog_error(string, tags):
+        dog_http_api.event("Pearson Import", string, alert_type='error', tags=tags)
 
     def handle(self, *args, **kwargs):
         if len(args) < 1:
@@ -44,7 +44,9 @@ class Command(BaseCommand):
                     elif fileinfo.filename.startswith("vcdc-"):
                         self.process_vcdc(zipentry)
                     else:
-                        raise CommandError("Unrecognized confirmation file type \"{}\" in confirmation zip file \"{}\"".format(fileinfo.filename, zipfile))
+                        error = "Unrecognized confirmation file type\"{}\" in confirmation zip file \"{}\"".format(fileinfo.filename, zipfile)
+                        Command.datadog_error(error, source_zip)
+                        raise CommandError(error)
 
     def process_eac(self, eacfile):
         print "processing eac"
@@ -53,31 +55,31 @@ class Command(BaseCommand):
             client_authorization_id = row['ClientAuthorizationID']
             if not client_authorization_id:
                 if row['Status'] == 'Error':
-                    Command.datadog_error("Error in EAD file processing ({}): {}".format(row['Date'], row['Message']))
+                    Command.datadog_error("Error in EAD file processing ({}): {}".format(row['Date'], row['Message']), eacfile)
                 else:
-                    Command.datadog_error("Encountered bad record: {}".format(row))
+                    Command.datadog_error("Encountered bad record: {}".format(row), eacfile)
             else:
                 try:
                     registration = TestCenterRegistration.objects.get(client_authorization_id=client_authorization_id)
-                    Command.datadog_error("Found authorization record for user {}".format(registration.testcenter_user.user.username))
+                    Command.datadog_error("Found authorization record for user {}".format(registration.testcenter_user.user.username), eacfile)
                     # now update the record:
                     registration.upload_status = row['Status']
                     registration.upload_error_message =  row['Message']
                     try:
                         registration.processed_at = strftime('%Y-%m-%d %H:%M:%S', strptime(row['Date'], '%Y/%m/%d %H:%M:%S'))
                     except ValueError as ve:
-                        Command.datadog_error("Bad Date value found for {}: message {}".format(client_authorization_id, ve))
+                        Command.datadog_error("Bad Date value found for {}: message {}".format(client_authorization_id, ve), eacfile)
                     # store the authorization Id if one is provided.  (For debugging)
                     if row['AuthorizationID']:
                         try:
                             registration.authorization_id = int(row['AuthorizationID'])
                         except ValueError as ve:
-                            Command.datadog_error("Bad AuthorizationID value found for {}: message {}".format(client_authorization_id, ve))
+                            Command.datadog_error("Bad AuthorizationID value found for {}: message {}".format(client_authorization_id, ve), eacfile)
 
                     registration.confirmed_at = datetime.utcnow()
                     registration.save()
                 except TestCenterRegistration.DoesNotExist:
-                    Command.datadog_error("Failed to find record for client_auth_id {}".format(client_authorization_id))
+                    Command.datadog_error("Failed to find record for client_auth_id {}".format(client_authorization_id), eacfile)
 
     def process_vcdc(self, vcdcfile):
         print "processing vcdc"
@@ -86,28 +88,28 @@ class Command(BaseCommand):
             client_candidate_id = row['ClientCandidateID']
             if not client_candidate_id:
                 if row['Status'] == 'Error':
-                    Command.datadog_error("Error in CDD file processing ({}): {}".format(row['Date'], row['Message']))
+                    Command.datadog_error("Error in CDD file processing ({}): {}".format(row['Date'], row['Message']), vcdcfile)
                 else:
-                    Command.datadog_error("Encountered bad record: {}".format(row))
+                    Command.datadog_error("Encountered bad record: {}".format(row), vcdcfile)
             else:
                 try:
                     tcuser = TestCenterUser.objects.get(client_candidate_id=client_candidate_id)
-                    Command.datadog_error("Found demographics record for user {}".format(tcuser.user.username))
+                    Command.datadog_error("Found demographics record for user {}".format(tcuser.user.username), vcdcfile)
                     # now update the record:
                     tcuser.upload_status = row['Status']
                     tcuser.upload_error_message = row['Message']
                     try:
                         tcuser.processed_at = strftime('%Y-%m-%d %H:%M:%S', strptime(row['Date'], '%Y/%m/%d %H:%M:%S'))
                     except ValueError as ve:
-                        Command.datadog_error("Bad Date value found for {}: message {}".format(client_candidate_id, ve))
+                        Command.datadog_error("Bad Date value found for {}: message {}".format(client_candidate_id, ve), vcdcfile)
                     # store the candidate Id if one is provided.  (For debugging)
                     if row['CandidateID']:
                         try:
                             tcuser.candidate_id = int(row['CandidateID'])
                         except ValueError as ve:
-                            Command.datadog_error("Bad CandidateID value found for {}: message {}".format(client_candidate_id, ve))
+                            Command.datadog_error("Bad CandidateID value found for {}: message {}".format(client_candidate_id, ve), vcdcfile)
                     tcuser.confirmed_at = datetime.utcnow()
                     tcuser.save()
                 except TestCenterUser.DoesNotExist:
-                    Command.datadog_error(" Failed to find record for client_candidate_id {}".format(client_candidate_id))
+                    Command.datadog_error(" Failed to find record for client_candidate_id {}".format(client_candidate_id), vcdcfile)
 

From 3aacba1f63b48b6d72883c2fd0a8b56d26573cc1 Mon Sep 17 00:00:00 2001
From: Ashley Penney <apenney@edx.org>
Date: Thu, 17 Jan 2013 15:44:08 -0500
Subject: [PATCH 15/29] Run over each file and run the import.  We could
 probably do this as a try/except and not delete if the output of the import
 failed but it may be simply easier to refetch those files from the S3 backup
 and try again.

---
 .../student/management/commands/pearson_transfer.py           | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/common/djangoapps/student/management/commands/pearson_transfer.py b/common/djangoapps/student/management/commands/pearson_transfer.py
index 57401897dc..0037231a38 100644
--- a/common/djangoapps/student/management/commands/pearson_transfer.py
+++ b/common/djangoapps/student/management/commands/pearson_transfer.py
@@ -43,7 +43,9 @@ class Command(BaseCommand):
                  settings.PEARSON['LOCAL_IMPORT'], options['mode'])
             s3(settings.PEARSON['LOCAL_IMPORT'],
                settings.PEARSON['BUCKET'], options['mode'])
-            call_command('pearson_import', 'dest_from_settings')
+            for file in os.listdir(settings.PEARSON['LOCAL_IMPORT']):
+                call_command('pearson_import_conf_zip', 'dest_from_settings')
+                os.remove(file)
 
         def export_pearson():
             call_command('pearson_export_cdd', 'dest_from_settings')

From 274cb8d865f00734e383a17ae3bdc29883acae45 Mon Sep 17 00:00:00 2001
From: Ashley Penney <apenney@edx.org>
Date: Thu, 17 Jan 2013 15:50:37 -0500
Subject: [PATCH 16/29] Actually call the appropriate file.

---
 .../djangoapps/student/management/commands/pearson_transfer.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/common/djangoapps/student/management/commands/pearson_transfer.py b/common/djangoapps/student/management/commands/pearson_transfer.py
index 0037231a38..aede8fe92d 100644
--- a/common/djangoapps/student/management/commands/pearson_transfer.py
+++ b/common/djangoapps/student/management/commands/pearson_transfer.py
@@ -44,7 +44,8 @@ class Command(BaseCommand):
             s3(settings.PEARSON['LOCAL_IMPORT'],
                settings.PEARSON['BUCKET'], options['mode'])
             for file in os.listdir(settings.PEARSON['LOCAL_IMPORT']):
-                call_command('pearson_import_conf_zip', 'dest_from_settings')
+                call_command('pearson_import_conf_zip',
+                             settings.PEARSON['LOCAL_IMPORT'] + '/' + file)
                 os.remove(file)
 
         def export_pearson():

From d0ecae30d7edaa64ae24b383c532a34faebd03d3 Mon Sep 17 00:00:00 2001
From: Ashley Penney <apenney@edx.org>
Date: Thu, 17 Jan 2013 16:24:44 -0500
Subject: [PATCH 17/29] Fix datadog logging to use .name on the file objects
 and add an additional logging line.

---
 .../commands/pearson_import_conf_zip.py       | 26 ++++++++++---------
 .../management/commands/pearson_transfer.py   | 21 +++++++++------
 2 files changed, 27 insertions(+), 20 deletions(-)

diff --git a/common/djangoapps/student/management/commands/pearson_import_conf_zip.py b/common/djangoapps/student/management/commands/pearson_import_conf_zip.py
index a491a73868..fa9741dc68 100644
--- a/common/djangoapps/student/management/commands/pearson_import_conf_zip.py
+++ b/common/djangoapps/student/management/commands/pearson_import_conf_zip.py
@@ -33,7 +33,9 @@ class Command(BaseCommand):
 
         source_zip = args[0]
         if not is_zipfile(source_zip):
-            raise CommandError("Input file is not a zipfile: \"{}\"".format(source_zip))
+            error = "Input file is not a zipfile: \"{}\"".format(source_zip)
+            Command.datadog_error(error, source_zip)
+            raise CommandError(error)
 
         # loop through all files in zip, and process them based on filename prefix:
         with ZipFile(source_zip, 'r') as zipfile:
@@ -55,9 +57,9 @@ class Command(BaseCommand):
             client_authorization_id = row['ClientAuthorizationID']
             if not client_authorization_id:
                 if row['Status'] == 'Error':
-                    Command.datadog_error("Error in EAD file processing ({}): {}".format(row['Date'], row['Message']), eacfile)
+                    Command.datadog_error("Error in EAD file processing ({}): {}".format(row['Date'], row['Message']), eacfile.name)
                 else:
-                    Command.datadog_error("Encountered bad record: {}".format(row), eacfile)
+                    Command.datadog_error("Encountered bad record: {}".format(row), eacfile.name)
             else:
                 try:
                     registration = TestCenterRegistration.objects.get(client_authorization_id=client_authorization_id)
@@ -68,18 +70,18 @@ class Command(BaseCommand):
                     try:
                         registration.processed_at = strftime('%Y-%m-%d %H:%M:%S', strptime(row['Date'], '%Y/%m/%d %H:%M:%S'))
                     except ValueError as ve:
-                        Command.datadog_error("Bad Date value found for {}: message {}".format(client_authorization_id, ve), eacfile)
+                        Command.datadog_error("Bad Date value found for {}: message {}".format(client_authorization_id, ve), eacfile.name)
                     # store the authorization Id if one is provided.  (For debugging)
                     if row['AuthorizationID']:
                         try:
                             registration.authorization_id = int(row['AuthorizationID'])
                         except ValueError as ve:
-                            Command.datadog_error("Bad AuthorizationID value found for {}: message {}".format(client_authorization_id, ve), eacfile)
+                            Command.datadog_error("Bad AuthorizationID value found for {}: message {}".format(client_authorization_id, ve), eacfile.name)
 
                     registration.confirmed_at = datetime.utcnow()
                     registration.save()
                 except TestCenterRegistration.DoesNotExist:
-                    Command.datadog_error("Failed to find record for client_auth_id {}".format(client_authorization_id), eacfile)
+                    Command.datadog_error("Failed to find record for client_auth_id {}".format(client_authorization_id), eacfile.name)
 
     def process_vcdc(self, vcdcfile):
         print "processing vcdc"
@@ -88,28 +90,28 @@ class Command(BaseCommand):
             client_candidate_id = row['ClientCandidateID']
             if not client_candidate_id:
                 if row['Status'] == 'Error':
-                    Command.datadog_error("Error in CDD file processing ({}): {}".format(row['Date'], row['Message']), vcdcfile)
+                    Command.datadog_error("Error in CDD file processing ({}): {}".format(row['Date'], row['Message']), vcdcfile.name)
                 else:
-                    Command.datadog_error("Encountered bad record: {}".format(row), vcdcfile)
+                    Command.datadog_error("Encountered bad record: {}".format(row), vcdcfile.name)
             else:
                 try:
                     tcuser = TestCenterUser.objects.get(client_candidate_id=client_candidate_id)
-                    Command.datadog_error("Found demographics record for user {}".format(tcuser.user.username), vcdcfile)
+                    Command.datadog_error("Found demographics record for user {}".format(tcuser.user.username), vcdcfile.name)
                     # now update the record:
                     tcuser.upload_status = row['Status']
                     tcuser.upload_error_message = row['Message']
                     try:
                         tcuser.processed_at = strftime('%Y-%m-%d %H:%M:%S', strptime(row['Date'], '%Y/%m/%d %H:%M:%S'))
                     except ValueError as ve:
-                        Command.datadog_error("Bad Date value found for {}: message {}".format(client_candidate_id, ve), vcdcfile)
+                        Command.datadog_error("Bad Date value found for {}: message {}".format(client_candidate_id, ve), vcdcfile.name)
                     # store the candidate Id if one is provided.  (For debugging)
                     if row['CandidateID']:
                         try:
                             tcuser.candidate_id = int(row['CandidateID'])
                         except ValueError as ve:
-                            Command.datadog_error("Bad CandidateID value found for {}: message {}".format(client_candidate_id, ve), vcdcfile)
+                            Command.datadog_error("Bad CandidateID value found for {}: message {}".format(client_candidate_id, ve), vcdcfile.name)
                     tcuser.confirmed_at = datetime.utcnow()
                     tcuser.save()
                 except TestCenterUser.DoesNotExist:
-                    Command.datadog_error(" Failed to find record for client_candidate_id {}".format(client_candidate_id), vcdcfile)
+                    Command.datadog_error(" Failed to find record for client_candidate_id {}".format(client_candidate_id), vcdcfile.name)
 
diff --git a/common/djangoapps/student/management/commands/pearson_transfer.py b/common/djangoapps/student/management/commands/pearson_transfer.py
index aede8fe92d..5f126a24f0 100644
--- a/common/djangoapps/student/management/commands/pearson_transfer.py
+++ b/common/djangoapps/student/management/commands/pearson_transfer.py
@@ -39,14 +39,18 @@ class Command(BaseCommand):
                                    '(env/auth.json) for {0}'.format(value))
 
         def import_pearson():
-            sftp(settings.PEARSON['SFTP_IMPORT'],
-                 settings.PEARSON['LOCAL_IMPORT'], options['mode'])
-            s3(settings.PEARSON['LOCAL_IMPORT'],
-               settings.PEARSON['BUCKET'], options['mode'])
-            for file in os.listdir(settings.PEARSON['LOCAL_IMPORT']):
-                call_command('pearson_import_conf_zip',
-                             settings.PEARSON['LOCAL_IMPORT'] + '/' + file)
-                os.remove(file)
+            try:
+                sftp(settings.PEARSON['SFTP_IMPORT'],
+                     settings.PEARSON['LOCAL_IMPORT'], options['mode'])
+                s3(settings.PEARSON['LOCAL_IMPORT'],
+                   settings.PEARSON['BUCKET'], options['mode'])
+            except Exception as e:
+                dog_http_api.event('Pearson Import failure', str(e))
+            else:
+                for file in os.listdir(settings.PEARSON['LOCAL_IMPORT']):
+                    call_command('pearson_import_conf_zip',
+                                 settings.PEARSON['LOCAL_IMPORT'] + '/' + file)
+                    os.remove(file)
 
         def export_pearson():
             call_command('pearson_export_cdd', 'dest_from_settings')
@@ -79,6 +83,7 @@ class Command(BaseCommand):
                         for filename in sftp.listdir(files_from):
                             sftp.get(files_from + '/' + filename,
                                      files_to + '/' + filename)
+                            sftp.remove(files_from + '/' + filename)
                     t.close()
                 except:
                     dog_http_api.event('pearson {0}'.format(mode),

From 491dd408aa7193553e8d5f38e7386e150d7f3e7c Mon Sep 17 00:00:00 2001
From: Ashley Penney <apenney@edx.org>
Date: Thu, 17 Jan 2013 16:31:21 -0500
Subject: [PATCH 18/29] Add dogapi for datadog.

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index fa4688b711..7e4d913c30 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -59,3 +59,4 @@ Shapely==1.2.16
 ipython==0.13.1
 xmltodict==0.4.1
 paramiko==1.9.0
+dogapi==1.1.2

From 60ae54b2e15a14095af8c5c2bef5111ef57d9e06 Mon Sep 17 00:00:00 2001
From: Ashley Penney <apenney@edx.org>
Date: Thu, 17 Jan 2013 16:32:34 -0500
Subject: [PATCH 19/29] Actually, lets use Cale's fixes.

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 7e4d913c30..b1d769e9c2 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -59,4 +59,4 @@ Shapely==1.2.16
 ipython==0.13.1
 xmltodict==0.4.1
 paramiko==1.9.0
-dogapi==1.1.2
+git+ssh://git@github.com/MITx/dogapi.git@003a4fc9#egg=dogapi%

From c5979f8efa749376b32f03b4c112c1a39fd514ca Mon Sep 17 00:00:00 2001
From: Ashley Penney <apenney@edx.org>
Date: Thu, 17 Jan 2013 16:38:35 -0500
Subject: [PATCH 20/29] Stray %

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index b1d769e9c2..1b1384912b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -59,4 +59,4 @@ Shapely==1.2.16
 ipython==0.13.1
 xmltodict==0.4.1
 paramiko==1.9.0
-git+ssh://git@github.com/MITx/dogapi.git@003a4fc9#egg=dogapi%
+git+ssh://git@github.com/MITx/dogapi.git@003a4fc9#egg=dogapi

From 7bcfc44b7141d0171dc3a372647d36a81689f4ae Mon Sep 17 00:00:00 2001
From: Brian Wilson <brian@edx.org>
Date: Thu, 17 Jan 2013 17:50:26 -0500
Subject: [PATCH 21/29] add first test shell

---
 .../management/commands/test/__init__.py      |  0
 .../management/commands/test/test_pearson.py  | 53 +++++++++++++++++++
 2 files changed, 53 insertions(+)
 create mode 100644 common/djangoapps/student/management/commands/test/__init__.py
 create mode 100644 common/djangoapps/student/management/commands/test/test_pearson.py

diff --git a/common/djangoapps/student/management/commands/test/__init__.py b/common/djangoapps/student/management/commands/test/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/common/djangoapps/student/management/commands/test/test_pearson.py b/common/djangoapps/student/management/commands/test/test_pearson.py
new file mode 100644
index 0000000000..b7f24ce232
--- /dev/null
+++ b/common/djangoapps/student/management/commands/test/test_pearson.py
@@ -0,0 +1,53 @@
+'''
+Created on Jan 17, 2013
+
+@author: brian
+'''
+import logging
+
+from django.test import TestCase
+from student.models import User, TestCenterRegistration, TestCenterUser, unique_id_for_user
+from mock import Mock
+from datetime import datetime
+from django.core import management
+
+COURSE_1 = 'edX/toy/2012_Fall'
+COURSE_2 = 'edx/full/6.002_Spring_2012'
+
+log = logging.getLogger(__name__)
+
+class PearsonTestCase(TestCase):
+    '''
+    Base class for tests running Pearson-related commands
+    '''
+
+    def test_create_good_testcenter_user(self):
+        username = "rusty"
+#        user = Mock(username=username)
+#        # id = unique_id_for_user(user)
+#        course = Mock(end_of_course_survey_url=survey_url)
+
+
+        newuser = User.objects.create_user(username, 'rusty@edx.org', 'fakepass')
+#        newuser.first_name='Rusty'
+#        newuser.last_name='Skids'
+#        newuser.is_staff=True
+#        newuser.is_active=True
+#        newuser.is_superuser=True
+#        newuser.last_login=datetime(2012, 1, 1)
+#        newuser.date_joined=datetime(2011, 1, 1)
+
+#        newuser.save(using='default')
+        options = {
+                   'first_name' : 'TestFirst',
+                   'last_name' : 'TestLast',
+                   'address_1' : 'Test Address',
+                   'city' : 'TestCity',
+                   'state' : 'Alberta',
+                   'postal_code' : 'A0B 1C2',
+                   'country' : 'CAN',
+                   'phone' : '252-1866',
+                   'phone_country_code' : '1',
+                    }
+        management.call_command('pearson_make_tc_user', username, options)
+        
\ No newline at end of file

From d30974b560e23e0a0f8e9c1ecd495c689f69c21d Mon Sep 17 00:00:00 2001
From: Brian Wilson <brian@edx.org>
Date: Fri, 18 Jan 2013 04:00:32 -0500
Subject: [PATCH 22/29] Get pearson export working in a unit test

---
 .../management/commands/pearson_export_cdd.py |  5 +-
 .../management/commands/pearson_export_ead.py |  1 +
 .../management/commands/test/test_pearson.py  | 84 ++++++++++++-------
 common/djangoapps/student/models.py           |  4 +
 4 files changed, 62 insertions(+), 32 deletions(-)

diff --git a/common/djangoapps/student/management/commands/pearson_export_cdd.py b/common/djangoapps/student/management/commands/pearson_export_cdd.py
index 14c652f11e..ba43dd3bc0 100644
--- a/common/djangoapps/student/management/commands/pearson_export_cdd.py
+++ b/common/djangoapps/student/management/commands/pearson_export_cdd.py
@@ -4,6 +4,7 @@ from collections import OrderedDict
 from datetime import datetime
 from optparse import make_option
 
+from django.conf import settings
 from django.core.management.base import BaseCommand
 
 from student.models import TestCenterUser
@@ -86,7 +87,7 @@ class Command(BaseCommand):
             else:
                 return value
 
-        dump_all = options['dump_all']
+#        dump_all = options['dump_all']
 
         with open(destfile, "wb") as outfile:
             writer = csv.DictWriter(outfile,
@@ -96,7 +97,7 @@ class Command(BaseCommand):
                                     extrasaction='ignore')
             writer.writeheader()
             for tcu in TestCenterUser.objects.order_by('id'):
-                if dump_all or tcu.needs_uploading:
+                if tcu.needs_uploading: # or dump_all
                     record = dict((csv_field, ensure_encoding(getattr(tcu, model_field)))
                                   for csv_field, model_field
                                   in Command.CSV_TO_MODEL_FIELDS.items())
diff --git a/common/djangoapps/student/management/commands/pearson_export_ead.py b/common/djangoapps/student/management/commands/pearson_export_ead.py
index 9368ac5ddf..492cba154b 100644
--- a/common/djangoapps/student/management/commands/pearson_export_ead.py
+++ b/common/djangoapps/student/management/commands/pearson_export_ead.py
@@ -4,6 +4,7 @@ from collections import OrderedDict
 from datetime import datetime
 from optparse import make_option
 
+from django.conf import settings
 from django.core.management.base import BaseCommand
 
 from student.models import TestCenterRegistration
diff --git a/common/djangoapps/student/management/commands/test/test_pearson.py b/common/djangoapps/student/management/commands/test/test_pearson.py
index b7f24ce232..e0ded22a1d 100644
--- a/common/djangoapps/student/management/commands/test/test_pearson.py
+++ b/common/djangoapps/student/management/commands/test/test_pearson.py
@@ -6,39 +6,18 @@ Created on Jan 17, 2013
 import logging
 
 from django.test import TestCase
-from student.models import User, TestCenterRegistration, TestCenterUser, unique_id_for_user
-from mock import Mock
-from datetime import datetime
+from student.models import User, TestCenterRegistration, TestCenterUser
+# This is stupid!  Because I import a function with the word "test" in the name,
+# the unittest framework tries to run *it* as a test?!  Crazy!
+from student.models import get_testcenter_registration as get_tc_registration
 from django.core import management
 
-COURSE_1 = 'edX/toy/2012_Fall'
-COURSE_2 = 'edx/full/6.002_Spring_2012'
-
 log = logging.getLogger(__name__)
 
-class PearsonTestCase(TestCase):
-    '''
-    Base class for tests running Pearson-related commands
-    '''
 
-    def test_create_good_testcenter_user(self):
-        username = "rusty"
-#        user = Mock(username=username)
-#        # id = unique_id_for_user(user)
-#        course = Mock(end_of_course_survey_url=survey_url)
-
-
-        newuser = User.objects.create_user(username, 'rusty@edx.org', 'fakepass')
-#        newuser.first_name='Rusty'
-#        newuser.last_name='Skids'
-#        newuser.is_staff=True
-#        newuser.is_active=True
-#        newuser.is_superuser=True
-#        newuser.last_login=datetime(2012, 1, 1)
-#        newuser.date_joined=datetime(2011, 1, 1)
-
-#        newuser.save(using='default')
-        options = {
+def create_tc_user(username):
+    user = User.objects.create_user(username, '{}@edx.org'.format(username), 'fakepass')
+    options = {
                    'first_name' : 'TestFirst',
                    'last_name' : 'TestLast',
                    'address_1' : 'Test Address',
@@ -49,5 +28,50 @@ class PearsonTestCase(TestCase):
                    'phone' : '252-1866',
                    'phone_country_code' : '1',
                     }
-        management.call_command('pearson_make_tc_user', username, options)
-        
\ No newline at end of file
+    management.call_command('pearson_make_tc_user', username, **options)
+    return TestCenterUser.objects.get(user=user)
+    
+    
+def create_tc_registration(username, course_id, exam_code, accommodation_code):
+    
+    options = { 'exam_series_code' : exam_code,
+               'eligibility_appointment_date_first' : '2013-01-01T00:00',
+               'eligibility_appointment_date_last' : '2013-12-31T23:59',
+               'accommodation_code' : accommodation_code, 
+               }
+
+    management.call_command('pearson_make_tc_registration', username, course_id, **options)
+    user = User.objects.get(username=username)
+    registrations = get_tc_registration(user, course_id, exam_code)
+    return registrations[0]
+    
+class PearsonTestCase(TestCase):
+    '''
+    Base class for tests running Pearson-related commands
+    '''
+
+    def test_create_good_testcenter_user(self):
+        testcenter_user = create_tc_user("test1")
+        
+    def test_create_good_testcenter_registration(self):
+        username = 'test1'
+        course_id = 'org1/course1/term1'
+        exam_code = 'exam1'
+        accommodation_code = 'NONE'
+        testcenter_user = create_tc_user(username)
+        registration = create_tc_registration(username, course_id, exam_code, accommodation_code)
+        
+    def test_export(self):
+        username = 'test1'
+        course_id = 'org1/course1/term1'
+        exam_code = 'exam1'
+        accommodation_code = 'NONE'
+        testcenter_user = create_tc_user(username)
+        registration = create_tc_registration(username, course_id, exam_code, accommodation_code)
+        output_dir = "./tmpOutput"
+        options = { 'destination' : output_dir }
+        with self.settings(PEARSON={ 'LOCAL_EXPORT' : output_dir }):
+            management.call_command('pearson_export_cdd', **options)
+            management.call_command('pearson_export_ead', **options)
+        # TODO: check that files were output....
+        
diff --git a/common/djangoapps/student/models.py b/common/djangoapps/student/models.py
index f13a691215..c9cb94d81a 100644
--- a/common/djangoapps/student/models.py
+++ b/common/djangoapps/student/models.py
@@ -428,6 +428,10 @@ class TestCenterRegistration(models.Model):
         # TODO: figure out if this should really go in the database (with a default value).
         return 1
     
+    @property
+    def needs_uploading(self):
+        return self.uploaded_at is None or self.uploaded_at < self.user_updated_at
+    
     @classmethod
     def create(cls, testcenter_user, exam, accommodation_request):
         registration = cls(testcenter_user = testcenter_user)

From 553528cd1c900e654e0c1556c0afe26dbbe57267 Mon Sep 17 00:00:00 2001
From: Brian Wilson <brian@edx.org>
Date: Fri, 18 Jan 2013 14:52:36 -0500
Subject: [PATCH 23/29] change get_testcenter_registration to
 get_tc_registration, so it's not treated as a test.

---
 .../management/commands/pearson_make_tc_registration.py     | 6 +++---
 .../student/management/commands/{test => tests}/__init__.py | 0
 .../management/commands/{test => tests}/test_pearson.py     | 2 +-
 common/djangoapps/student/models.py                         | 2 +-
 common/djangoapps/student/views.py                          | 6 +++---
 5 files changed, 8 insertions(+), 8 deletions(-)
 rename common/djangoapps/student/management/commands/{test => tests}/__init__.py (100%)
 rename common/djangoapps/student/management/commands/{test => tests}/test_pearson.py (97%)

diff --git a/common/djangoapps/student/management/commands/pearson_make_tc_registration.py b/common/djangoapps/student/management/commands/pearson_make_tc_registration.py
index 81a478d19d..545c8cd8a8 100644
--- a/common/djangoapps/student/management/commands/pearson_make_tc_registration.py
+++ b/common/djangoapps/student/management/commands/pearson_make_tc_registration.py
@@ -4,7 +4,7 @@ from time import strftime
 from django.contrib.auth.models import User
 from django.core.management.base import BaseCommand, CommandError
 
-from student.models import TestCenterUser, TestCenterRegistration, TestCenterRegistrationForm, get_testcenter_registration
+from student.models import TestCenterUser, TestCenterRegistration, TestCenterRegistrationForm, get_tc_registration
 from student.views import course_from_id
 from xmodule.course_module import CourseDescriptor
 from xmodule.modulestore.exceptions import ItemNotFoundError
@@ -134,7 +134,7 @@ class Command(BaseCommand):
         
         # create and save the registration:
         needs_updating = False
-        registrations = get_testcenter_registration(student, course_id, exam_code)
+        registrations = get_tc_registration(student, course_id, exam_code)
         if len(registrations) > 0:
             registration = registrations[0]
             for fieldname in UPDATE_FIELDS:
@@ -181,7 +181,7 @@ class Command(BaseCommand):
         change_internal = False
         if 'exam_series_code' in our_options:
             exam_code = our_options['exam_series_code']
-        registration = get_testcenter_registration(student, course_id, exam_code)[0]
+        registration = get_tc_registration(student, course_id, exam_code)[0]
         for internal_field in [ 'upload_error_message', 'upload_status', 'authorization_id']:
             if internal_field in our_options:
                 registration.__setattr__(internal_field, our_options[internal_field])
diff --git a/common/djangoapps/student/management/commands/test/__init__.py b/common/djangoapps/student/management/commands/tests/__init__.py
similarity index 100%
rename from common/djangoapps/student/management/commands/test/__init__.py
rename to common/djangoapps/student/management/commands/tests/__init__.py
diff --git a/common/djangoapps/student/management/commands/test/test_pearson.py b/common/djangoapps/student/management/commands/tests/test_pearson.py
similarity index 97%
rename from common/djangoapps/student/management/commands/test/test_pearson.py
rename to common/djangoapps/student/management/commands/tests/test_pearson.py
index e0ded22a1d..29f9120e98 100644
--- a/common/djangoapps/student/management/commands/test/test_pearson.py
+++ b/common/djangoapps/student/management/commands/tests/test_pearson.py
@@ -9,7 +9,7 @@ from django.test import TestCase
 from student.models import User, TestCenterRegistration, TestCenterUser
 # This is stupid!  Because I import a function with the word "test" in the name,
 # the unittest framework tries to run *it* as a test?!  Crazy!
-from student.models import get_testcenter_registration as get_tc_registration
+from student.models import get_tc_registration
 from django.core import management
 
 log = logging.getLogger(__name__)
diff --git a/common/djangoapps/student/models.py b/common/djangoapps/student/models.py
index c9cb94d81a..84bbc76a80 100644
--- a/common/djangoapps/student/models.py
+++ b/common/djangoapps/student/models.py
@@ -547,7 +547,7 @@ class TestCenterRegistrationForm(ModelForm):
     
     
     
-def get_testcenter_registration(user, course_id, exam_series_code):
+def get_tc_registration(user, course_id, exam_series_code):
     try:
         tcu = TestCenterUser.objects.get(user=user)
     except TestCenterUser.DoesNotExist:
diff --git a/common/djangoapps/student/views.py b/common/djangoapps/student/views.py
index 61b49e6022..650f0a0280 100644
--- a/common/djangoapps/student/views.py
+++ b/common/djangoapps/student/views.py
@@ -31,7 +31,7 @@ from student.models import (Registration, UserProfile, TestCenterUser, TestCente
                             TestCenterRegistration, TestCenterRegistrationForm,
                             PendingNameChange, PendingEmailChange,
                             CourseEnrollment, unique_id_for_user,
-                            get_testcenter_registration)
+                            get_tc_registration)
 
 from certificates.models import CertificateStatuses, certificate_status_for_student
 
@@ -612,7 +612,7 @@ def exam_registration_info(user, course):
         return None
 
     exam_code = exam_info.exam_series_code
-    registrations = get_testcenter_registration(user, course.id, exam_code)
+    registrations = get_tc_registration(user, course.id, exam_code)
     if registrations:
         registration = registrations[0]
     else:
@@ -712,7 +712,7 @@ def create_exam_registration(request, post_override=None):
     needs_saving = False
     exam = course.current_test_center_exam
     exam_code = exam.exam_series_code
-    registrations = get_testcenter_registration(user, course_id, exam_code)
+    registrations = get_tc_registration(user, course_id, exam_code)
     if registrations:
         registration = registrations[0]
         # NOTE: we do not bother to check here to see if the registration has changed,

From f4703b40cb7098dbe44b91f050d23ead8359464f Mon Sep 17 00:00:00 2001
From: Brian Wilson <brian@edx.org>
Date: Fri, 18 Jan 2013 18:34:51 -0500
Subject: [PATCH 24/29] add test-with-settings

---
 .../management/commands/tests/test_pearson.py | 54 ++++++++++++++++---
 1 file changed, 47 insertions(+), 7 deletions(-)

diff --git a/common/djangoapps/student/management/commands/tests/test_pearson.py b/common/djangoapps/student/management/commands/tests/test_pearson.py
index 29f9120e98..fd9eb31631 100644
--- a/common/djangoapps/student/management/commands/tests/test_pearson.py
+++ b/common/djangoapps/student/management/commands/tests/test_pearson.py
@@ -4,14 +4,16 @@ Created on Jan 17, 2013
 @author: brian
 '''
 import logging
+import os
+from tempfile import mkdtemp
+import cStringIO
+import sys
 
 from django.test import TestCase
-from student.models import User, TestCenterRegistration, TestCenterUser
-# This is stupid!  Because I import a function with the word "test" in the name,
-# the unittest framework tries to run *it* as a test?!  Crazy!
-from student.models import get_tc_registration
 from django.core import management
 
+from student.models import User, TestCenterRegistration, TestCenterUser, get_tc_registration
+
 log = logging.getLogger(__name__)
 
 
@@ -49,7 +51,40 @@ class PearsonTestCase(TestCase):
     '''
     Base class for tests running Pearson-related commands
     '''
+    import_dir = mkdtemp(prefix="import")
+    export_dir = mkdtemp(prefix="export")
+    
+    
+    def tearDown(self):
+        def delete_temp_dir(dirname):
+            if os.path.exists(dirname):
+                for filename in os.listdir(dirname):
+                    os.remove(os.path.join(dirname, filename))
+                os.rmdir(dirname)
+            
+        # clean up after any test data was dumped to temp directory
+        delete_temp_dir(self.import_dir)
+        delete_temp_dir(self.export_dir)
 
+    def test_missing_demographic_fields(self):
+        old_stdout = sys.stdout
+        sys.stdout = cStringIO.StringIO()
+        username = 'baduser'
+        User.objects.create_user(username, '{}@edx.org'.format(username), 'fakepass')
+        options = {}
+               
+        self.assertRaises(BaseException, management.call_command, 'pearson_make_tc_user', username, **options)
+        output_string = sys.stdout.getvalue()
+        self.assertTrue(output_string.find('Field Form errors encountered:') >= 0)
+        self.assertTrue(output_string.find('Field Form Error:  city') >= 0)
+        self.assertTrue(output_string.find('Field Form Error:  first_name') >= 0)
+        self.assertTrue(output_string.find('Field Form Error:  last_name') >= 0)
+        self.assertTrue(output_string.find('Field Form Error:  country') >= 0)
+        self.assertTrue(output_string.find('Field Form Error:  phone_country_code') >= 0)
+        self.assertTrue(output_string.find('Field Form Error:  phone') >= 0)
+        self.assertTrue(output_string.find('Field Form Error:  address_1') >= 0)
+        sys.stdout = old_stdout
+        
     def test_create_good_testcenter_user(self):
         testcenter_user = create_tc_user("test1")
         
@@ -68,10 +103,15 @@ class PearsonTestCase(TestCase):
         accommodation_code = 'NONE'
         testcenter_user = create_tc_user(username)
         registration = create_tc_registration(username, course_id, exam_code, accommodation_code)
-        output_dir = "./tmpOutput"
-        options = { 'destination' : output_dir }
-        with self.settings(PEARSON={ 'LOCAL_EXPORT' : output_dir }):
+        #options = { 'destination' : self.export_dir }
+        options = { '--dest-from-settings' : None }
+        with self.settings(PEARSON={ 'LOCAL_EXPORT' : self.export_dir }):
             management.call_command('pearson_export_cdd', **options)
+            print 'Files found: {}'.format(os.listdir(self.export_dir))
+            self.assertEquals(len(os.listdir(self.export_dir)), 1, "Expect cdd file to be created")
             management.call_command('pearson_export_ead', **options)
+            print 'Files found: {}'.format(os.listdir(self.export_dir))
+            self.assertEquals(len(os.listdir(self.export_dir)), 2, "Expect ead file to also be created")
+
         # TODO: check that files were output....
         

From 740d0403e908d32d656b1b8c8631db759d4b4c7b Mon Sep 17 00:00:00 2001
From: Brian Wilson <brian@edx.org>
Date: Tue, 22 Jan 2013 11:32:39 -0500
Subject: [PATCH 25/29] change name of function back to
 get_testcenter_registration, and disable its use as a test

---
 .../management/commands/pearson_make_tc_registration.py   | 6 +++---
 .../student/management/commands/tests/test_pearson.py     | 4 ++--
 common/djangoapps/student/models.py                       | 8 ++++++--
 common/djangoapps/student/views.py                        | 6 +++---
 4 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/common/djangoapps/student/management/commands/pearson_make_tc_registration.py b/common/djangoapps/student/management/commands/pearson_make_tc_registration.py
index 545c8cd8a8..81a478d19d 100644
--- a/common/djangoapps/student/management/commands/pearson_make_tc_registration.py
+++ b/common/djangoapps/student/management/commands/pearson_make_tc_registration.py
@@ -4,7 +4,7 @@ from time import strftime
 from django.contrib.auth.models import User
 from django.core.management.base import BaseCommand, CommandError
 
-from student.models import TestCenterUser, TestCenterRegistration, TestCenterRegistrationForm, get_tc_registration
+from student.models import TestCenterUser, TestCenterRegistration, TestCenterRegistrationForm, get_testcenter_registration
 from student.views import course_from_id
 from xmodule.course_module import CourseDescriptor
 from xmodule.modulestore.exceptions import ItemNotFoundError
@@ -134,7 +134,7 @@ class Command(BaseCommand):
         
         # create and save the registration:
         needs_updating = False
-        registrations = get_tc_registration(student, course_id, exam_code)
+        registrations = get_testcenter_registration(student, course_id, exam_code)
         if len(registrations) > 0:
             registration = registrations[0]
             for fieldname in UPDATE_FIELDS:
@@ -181,7 +181,7 @@ class Command(BaseCommand):
         change_internal = False
         if 'exam_series_code' in our_options:
             exam_code = our_options['exam_series_code']
-        registration = get_tc_registration(student, course_id, exam_code)[0]
+        registration = get_testcenter_registration(student, course_id, exam_code)[0]
         for internal_field in [ 'upload_error_message', 'upload_status', 'authorization_id']:
             if internal_field in our_options:
                 registration.__setattr__(internal_field, our_options[internal_field])
diff --git a/common/djangoapps/student/management/commands/tests/test_pearson.py b/common/djangoapps/student/management/commands/tests/test_pearson.py
index fd9eb31631..538cd2812a 100644
--- a/common/djangoapps/student/management/commands/tests/test_pearson.py
+++ b/common/djangoapps/student/management/commands/tests/test_pearson.py
@@ -12,7 +12,7 @@ import sys
 from django.test import TestCase
 from django.core import management
 
-from student.models import User, TestCenterRegistration, TestCenterUser, get_tc_registration
+from student.models import User, TestCenterRegistration, TestCenterUser, get_testcenter_registration
 
 log = logging.getLogger(__name__)
 
@@ -44,7 +44,7 @@ def create_tc_registration(username, course_id, exam_code, accommodation_code):
 
     management.call_command('pearson_make_tc_registration', username, course_id, **options)
     user = User.objects.get(username=username)
-    registrations = get_tc_registration(user, course_id, exam_code)
+    registrations = get_testcenter_registration(user, course_id, exam_code)
     return registrations[0]
     
 class PearsonTestCase(TestCase):
diff --git a/common/djangoapps/student/models.py b/common/djangoapps/student/models.py
index 84bbc76a80..0d8a643ecb 100644
--- a/common/djangoapps/student/models.py
+++ b/common/djangoapps/student/models.py
@@ -547,13 +547,17 @@ class TestCenterRegistrationForm(ModelForm):
     
     
     
-def get_tc_registration(user, course_id, exam_series_code):
+def get_testcenter_registration(user, course_id, exam_series_code):
     try:
         tcu = TestCenterUser.objects.get(user=user)
     except TestCenterUser.DoesNotExist:
         return []
     return TestCenterRegistration.objects.filter(testcenter_user=tcu, course_id=course_id, exam_series_code=exam_series_code)
-        
+
+# nosetests thinks that anything with _test_ in the name is a test.
+# Correct this (https://nose.readthedocs.org/en/latest/finding_tests.html)
+get_testcenter_registration.__test__ = False
+   
 def unique_id_for_user(user):
     """
     Return a unique id for a user, suitable for inserting into
diff --git a/common/djangoapps/student/views.py b/common/djangoapps/student/views.py
index 650f0a0280..61b49e6022 100644
--- a/common/djangoapps/student/views.py
+++ b/common/djangoapps/student/views.py
@@ -31,7 +31,7 @@ from student.models import (Registration, UserProfile, TestCenterUser, TestCente
                             TestCenterRegistration, TestCenterRegistrationForm,
                             PendingNameChange, PendingEmailChange,
                             CourseEnrollment, unique_id_for_user,
-                            get_tc_registration)
+                            get_testcenter_registration)
 
 from certificates.models import CertificateStatuses, certificate_status_for_student
 
@@ -612,7 +612,7 @@ def exam_registration_info(user, course):
         return None
 
     exam_code = exam_info.exam_series_code
-    registrations = get_tc_registration(user, course.id, exam_code)
+    registrations = get_testcenter_registration(user, course.id, exam_code)
     if registrations:
         registration = registrations[0]
     else:
@@ -712,7 +712,7 @@ def create_exam_registration(request, post_override=None):
     needs_saving = False
     exam = course.current_test_center_exam
     exam_code = exam.exam_series_code
-    registrations = get_tc_registration(user, course_id, exam_code)
+    registrations = get_testcenter_registration(user, course_id, exam_code)
     if registrations:
         registration = registrations[0]
         # NOTE: we do not bother to check here to see if the registration has changed,

From d395c4448d09e77446ac83533c6342b4e0b738fa Mon Sep 17 00:00:00 2001
From: Brian Wilson <brian@edx.org>
Date: Tue, 22 Jan 2013 17:58:40 -0500
Subject: [PATCH 26/29] add more pearson tests, and update commands in response

---
 .../management/commands/pearson_export_cdd.py |   9 +-
 .../management/commands/pearson_export_ead.py |   8 +-
 .../management/commands/pearson_transfer.py   |  11 +-
 .../management/commands/tests/test_pearson.py | 192 ++++++++++++++----
 4 files changed, 176 insertions(+), 44 deletions(-)

diff --git a/common/djangoapps/student/management/commands/pearson_export_cdd.py b/common/djangoapps/student/management/commands/pearson_export_cdd.py
index ba43dd3bc0..463eec6b70 100644
--- a/common/djangoapps/student/management/commands/pearson_export_cdd.py
+++ b/common/djangoapps/student/management/commands/pearson_export_cdd.py
@@ -5,7 +5,7 @@ from datetime import datetime
 from optparse import make_option
 
 from django.conf import settings
-from django.core.management.base import BaseCommand
+from django.core.management.base import BaseCommand, CommandError
 
 from student.models import TestCenterUser
 
@@ -39,6 +39,9 @@ class Command(BaseCommand):
         ("LastUpdate", "user_updated_at"),  # in UTC, so same as what we store
     ])
 
+    # define defaults, even thought 'store_true' shouldn't need them.  
+    # (call_command will set None as default value for all options that don't have one,
+    # so one cannot rely on presence/absence of flags in that world.)
     option_list = BaseCommand.option_list + (
         make_option('--dest-from-settings',
                     action='store_true',
@@ -63,13 +66,13 @@ class Command(BaseCommand):
         # Name will use timestamp -- this is UTC, so it will look funny,
         # but it should at least be consistent with the other timestamps
         # used in the system.
-        if 'dest-from-settings' in options:
+        if 'dest-from-settings' in options and options['dest-from-settings']:
             if 'LOCAL_EXPORT' in settings.PEARSON:
                 dest = settings.PEARSON['LOCAL_EXPORT']
             else:
                 raise CommandError('--dest-from-settings was enabled but the'
                                    'PEARSON[LOCAL_EXPORT] setting was not set.')
-        elif 'destination' in options:
+        elif 'destination' in options and options['destination']:
             dest = options['destination']
         else:
             raise CommandError('--destination or --dest-from-settings must be used')
diff --git a/common/djangoapps/student/management/commands/pearson_export_ead.py b/common/djangoapps/student/management/commands/pearson_export_ead.py
index 492cba154b..49cdc9957a 100644
--- a/common/djangoapps/student/management/commands/pearson_export_ead.py
+++ b/common/djangoapps/student/management/commands/pearson_export_ead.py
@@ -5,7 +5,7 @@ from datetime import datetime
 from optparse import make_option
 
 from django.conf import settings
-from django.core.management.base import BaseCommand
+from django.core.management.base import BaseCommand, CommandError
 
 from student.models import TestCenterRegistration
 
@@ -39,10 +39,12 @@ class Command(BaseCommand):
         make_option('--dump_all',
                     action='store_true',
                     dest='dump_all',
+                    default=False,
                     ),
         make_option('--force_add',
                     action='store_true',
                     dest='force_add',
+                    default=False,
                     ),
     )
 
@@ -57,13 +59,13 @@ class Command(BaseCommand):
         # Name will use timestamp -- this is UTC, so it will look funny,
         # but it should at least be consistent with the other timestamps
         # used in the system.
-        if 'dest-from-settings' in options:
+        if 'dest-from-settings' in options and options['dest-from-settings']:
             if 'LOCAL_EXPORT' in settings.PEARSON:
                 dest = settings.PEARSON['LOCAL_EXPORT']
             else:
                 raise CommandError('--dest-from-settings was enabled but the'
                                    'PEARSON[LOCAL_EXPORT] setting was not set.')
-        elif 'destinations' in options:
+        elif 'destination' in options and options['destination']:
             dest = options['destination']
         else:
             raise CommandError('--destination or --dest-from-settings must be used')
diff --git a/common/djangoapps/student/management/commands/pearson_transfer.py b/common/djangoapps/student/management/commands/pearson_transfer.py
index 5f126a24f0..c216d2ceac 100644
--- a/common/djangoapps/student/management/commands/pearson_transfer.py
+++ b/common/djangoapps/student/management/commands/pearson_transfer.py
@@ -1,10 +1,12 @@
+import os
 from optparse import make_option
 
+from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
+from django.core.management import call_command
 from dogapi import dog_http_api, dog_stats_api
 import paramiko
 import boto
-import os
 
 dog_http_api.api_key = settings.DATADOG_API
 
@@ -13,7 +15,7 @@ class Command(BaseCommand):
     help = """
     This command handles the importing and exporting of student records for
     Pearson.  It uses some other Django commands to export and import the
-    files and then uploads over SFTP to pearson and stuffs the entry in an
+    files and then uploads over SFTP to Pearson and stuffs the entry in an
     S3 bucket for archive purposes.
 
     Usage: django-admin.py pearson-transfer --mode [import|export|both]
@@ -29,11 +31,12 @@ class Command(BaseCommand):
 
     def handle(self, **options):
 
+        # TODO: this doesn't work. Need to check if it's a property.
         if not settings.PEARSON:
             raise CommandError('No PEARSON entries in auth/env.json.')
 
-        for value in ['LOCAL_IMPORT', 'SFTP_IMPORT', 'BUCKET', 'LOCAL_EXPORT',
-                      'SFTP_EXPORT']:
+        for value in ['LOCAL_IMPORT', 'SFTP_IMPORT', 'LOCAL_EXPORT',
+                      'SFTP_EXPORT', 'SFTP_HOSTNAME', 'SFTP_USERNAME', 'SFTP_PASSWORD']:
             if value not in settings.PEARSON:
                 raise CommandError('No entry in the PEARSON settings'
                                    '(env/auth.json) for {0}'.format(value))
diff --git a/common/djangoapps/student/management/commands/tests/test_pearson.py b/common/djangoapps/student/management/commands/tests/test_pearson.py
index 538cd2812a..d5594926d2 100644
--- a/common/djangoapps/student/management/commands/tests/test_pearson.py
+++ b/common/djangoapps/student/management/commands/tests/test_pearson.py
@@ -10,7 +10,7 @@ import cStringIO
 import sys
 
 from django.test import TestCase
-from django.core import management
+from django.core.management import call_command
 
 from student.models import User, TestCenterRegistration, TestCenterUser, get_testcenter_registration
 
@@ -30,11 +30,11 @@ def create_tc_user(username):
                    'phone' : '252-1866',
                    'phone_country_code' : '1',
                     }
-    management.call_command('pearson_make_tc_user', username, **options)
+    call_command('pearson_make_tc_user', username, **options)
     return TestCenterUser.objects.get(user=user)
     
     
-def create_tc_registration(username, course_id, exam_code, accommodation_code):
+def create_tc_registration(username, course_id = 'org1/course1/term1', exam_code = 'exam1', accommodation_code = None):
     
     options = { 'exam_series_code' : exam_code,
                'eligibility_appointment_date_first' : '2013-01-01T00:00',
@@ -42,11 +42,55 @@ def create_tc_registration(username, course_id, exam_code, accommodation_code):
                'accommodation_code' : accommodation_code, 
                }
 
-    management.call_command('pearson_make_tc_registration', username, course_id, **options)
+    call_command('pearson_make_tc_registration', username, course_id, **options)
     user = User.objects.get(username=username)
     registrations = get_testcenter_registration(user, course_id, exam_code)
     return registrations[0]
+
+def get_error_string_for_management_call(*args, **options):
+    stdout_string = None
+    old_stdout = sys.stdout
+    old_stderr = sys.stderr
+    sys.stdout = cStringIO.StringIO()
+    sys.stderr = cStringIO.StringIO()
+    try:
+        call_command(*args, **options)
+    except BaseException, why1:
+        # The goal here is to catch CommandError calls.
+        # But these are actually translated into nice messages,
+        # and sys.exit(1) is then called.  For testing, we
+        # want to catch what sys.exit throws, and get the
+        # relevant text either from stdout or stderr. 
+        # TODO: this should really check to see that we
+        # arrived here because of a sys.exit(1).  Otherwise
+        # we should just raise the exception.
+        stdout_string = sys.stdout.getvalue()
+        stderr_string = sys.stderr.getvalue()
+    except Exception, why:
+        raise why
     
+    finally:
+        sys.stdout = old_stdout
+        sys.stderr = old_stderr
+            
+    if stdout_string is None:
+        raise Exception("Expected call to {} to fail, but it succeeded!".format(args[0]))            
+    return stdout_string, stderr_string
+        
+
+def get_file_info(dirpath):
+    filelist = os.listdir(dirpath)
+    print 'Files found: {}'.format(filelist)
+    numfiles = len(filelist)
+    if numfiles == 1:
+        filepath = os.path.join(dirpath, filelist[0])
+        with open(filepath, 'r') as cddfile:
+            filecontents = cddfile.readlines()
+            numlines = len(filecontents)
+            return filepath, numlines
+    else:
+        raise Exception("Expected to find a single file in {}, but found {}".format(dirpath,filelist))
+            
 class PearsonTestCase(TestCase):
     '''
     Base class for tests running Pearson-related commands
@@ -54,7 +98,9 @@ class PearsonTestCase(TestCase):
     import_dir = mkdtemp(prefix="import")
     export_dir = mkdtemp(prefix="export")
     
-    
+    def assertErrorContains(self, error_message, expected):
+        self.assertTrue(error_message.find(expected) >= 0, 'error message "{}" did not contain "{}"'.format(error_message, expected))
+        
     def tearDown(self):
         def delete_temp_dir(dirname):
             if os.path.exists(dirname):
@@ -67,14 +113,13 @@ class PearsonTestCase(TestCase):
         delete_temp_dir(self.export_dir)
 
     def test_missing_demographic_fields(self):
-        old_stdout = sys.stdout
-        sys.stdout = cStringIO.StringIO()
+        # We won't bother to test all details of form validation here.  
+        # It is enough to show that it works here, but deal with test cases for the form
+        # validation in the student tests, not these management tests.
         username = 'baduser'
         User.objects.create_user(username, '{}@edx.org'.format(username), 'fakepass')
         options = {}
-               
-        self.assertRaises(BaseException, management.call_command, 'pearson_make_tc_user', username, **options)
-        output_string = sys.stdout.getvalue()
+        output_string, _ = get_error_string_for_management_call('pearson_make_tc_user', username, **options) 
         self.assertTrue(output_string.find('Field Form errors encountered:') >= 0)
         self.assertTrue(output_string.find('Field Form Error:  city') >= 0)
         self.assertTrue(output_string.find('Field Form Error:  first_name') >= 0)
@@ -83,35 +128,114 @@ class PearsonTestCase(TestCase):
         self.assertTrue(output_string.find('Field Form Error:  phone_country_code') >= 0)
         self.assertTrue(output_string.find('Field Form Error:  phone') >= 0)
         self.assertTrue(output_string.find('Field Form Error:  address_1') >= 0)
-        sys.stdout = old_stdout
+        self.assertErrorContains(output_string, 'Field Form Error:  address_1')
         
     def test_create_good_testcenter_user(self):
         testcenter_user = create_tc_user("test1")
+        self.assertIsNotNone(testcenter_user)
         
     def test_create_good_testcenter_registration(self):
         username = 'test1'
-        course_id = 'org1/course1/term1'
-        exam_code = 'exam1'
-        accommodation_code = 'NONE'
-        testcenter_user = create_tc_user(username)
-        registration = create_tc_registration(username, course_id, exam_code, accommodation_code)
-        
-    def test_export(self):
-        username = 'test1'
-        course_id = 'org1/course1/term1'
-        exam_code = 'exam1'
-        accommodation_code = 'NONE'
-        testcenter_user = create_tc_user(username)
-        registration = create_tc_registration(username, course_id, exam_code, accommodation_code)
-        #options = { 'destination' : self.export_dir }
-        options = { '--dest-from-settings' : None }
-        with self.settings(PEARSON={ 'LOCAL_EXPORT' : self.export_dir }):
-            management.call_command('pearson_export_cdd', **options)
-            print 'Files found: {}'.format(os.listdir(self.export_dir))
-            self.assertEquals(len(os.listdir(self.export_dir)), 1, "Expect cdd file to be created")
-            management.call_command('pearson_export_ead', **options)
-            print 'Files found: {}'.format(os.listdir(self.export_dir))
-            self.assertEquals(len(os.listdir(self.export_dir)), 2, "Expect ead file to also be created")
+        create_tc_user(username)
+        registration = create_tc_registration(username)
+        self.assertIsNotNone(registration)
 
-        # TODO: check that files were output....
+    def test_cdd_missing_option(self):
+        _, error_string = get_error_string_for_management_call('pearson_export_cdd', **{})
+        self.assertErrorContains(error_string, 'Error: --destination or --dest-from-settings must be used')
+   
+    def test_ead_missing_option(self):
+        _, error_string = get_error_string_for_management_call('pearson_export_ead', **{})
+        self.assertErrorContains(error_string, 'Error: --destination or --dest-from-settings must be used')
+
+    def test_export_single_cdd(self):
+        # before we generate any tc_users, we expect there to be nothing to output:
+        options = { 'dest-from-settings' : True }
+        with self.settings(PEARSON={ 'LOCAL_EXPORT' : self.export_dir }):
+            call_command('pearson_export_cdd', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 1, "Expect cdd file to have no non-header lines")
+            os.remove(filepath)
+
+            # generating a tc_user should result in a line in the output    
+            username = 'test_single_cdd'
+            create_tc_user(username)
+            call_command('pearson_export_cdd', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 2, "Expect cdd file to have one non-header line")
+            os.remove(filepath)
+
+            # output after registration should not have any entries again.
+            call_command('pearson_export_cdd', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 1, "Expect cdd file to have no non-header lines")
+            os.remove(filepath)
+
+            # if we modify the record, then it should be output again:
+            user_options = { 'first_name' : 'NewTestFirst', }
+            call_command('pearson_make_tc_user', username, **user_options)
+            call_command('pearson_export_cdd', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 2, "Expect cdd file to have one non-header line")
+            os.remove(filepath)
+            
+    def test_export_single_ead(self):
+        # before we generate any registrations, we expect there to be nothing to output:
+        options = { 'dest-from-settings' : True }
+        with self.settings(PEARSON={ 'LOCAL_EXPORT' : self.export_dir }):
+            call_command('pearson_export_ead', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 1, "Expect ead file to have no non-header lines")
+            os.remove(filepath)
+
+            # generating a registration should result in a line in the output    
+            username = 'test_single_ead'
+            create_tc_user(username)
+            create_tc_registration(username)
+            call_command('pearson_export_ead', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 2, "Expect ead file to have one non-header line")
+            os.remove(filepath)
+
+            # output after registration should not have any entries again.
+            call_command('pearson_export_ead', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 1, "Expect ead file to have no non-header lines")
+            os.remove(filepath)
         
+            # if we modify the record, then it should be output again:
+            create_tc_registration(username, accommodation_code='EQPMNT')
+            call_command('pearson_export_ead', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 2, "Expect ead file to have one non-header line")
+            os.remove(filepath)
+
+    def test_export_multiple(self):
+        username1 = 'test_multiple1'
+        create_tc_user(username1)
+        create_tc_registration(username1)
+        create_tc_registration(username1, course_id = 'org1/course2/term1')
+        create_tc_registration(username1, exam_code = 'exam2')
+        username2 = 'test_multiple2'
+        create_tc_user(username2)
+        create_tc_registration(username2)
+        username3 = 'test_multiple3'
+        create_tc_user(username3)
+        create_tc_registration(username3, course_id = 'org1/course2/term1')
+        username4 = 'test_multiple4'
+        create_tc_user(username4)
+        create_tc_registration(username4, exam_code = 'exam2')
+
+        with self.settings(PEARSON={ 'LOCAL_EXPORT' : self.export_dir }):
+            options = { 'dest-from-settings' : True }
+            call_command('pearson_export_cdd', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 5, "Expect cdd file to have four non-header lines: total was {}".format(numlines))
+            os.remove(filepath)
+
+            call_command('pearson_export_ead', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 7, "Expect ead file to have six non-header lines: total was {}".format(numlines))
+            os.remove(filepath)
+
+

From 1199b1ecfa97cf623d1275ea0220af99119ba8e3 Mon Sep 17 00:00:00 2001
From: Brian Wilson <brian@edx.org>
Date: Wed, 23 Jan 2013 18:22:18 -0500
Subject: [PATCH 27/29] debug pearson import and update unit tests

---
 .../commands/pearson_import_conf_zip.py       |   2 +
 .../commands/pearson_make_tc_user.py          |  19 +-
 .../management/commands/pearson_transfer.py   | 133 +++++++----
 .../management/commands/tests/test_pearson.py | 209 +++++++++++++++---
 4 files changed, 275 insertions(+), 88 deletions(-)

diff --git a/common/djangoapps/student/management/commands/pearson_import_conf_zip.py b/common/djangoapps/student/management/commands/pearson_import_conf_zip.py
index fa9741dc68..bf7c4481fd 100644
--- a/common/djangoapps/student/management/commands/pearson_import_conf_zip.py
+++ b/common/djangoapps/student/management/commands/pearson_import_conf_zip.py
@@ -10,6 +10,7 @@ from optparse import make_option
 from dogapi import dog_http_api, dog_stats_api
 
 from django.core.management.base import BaseCommand, CommandError
+from django.conf import settings
 
 from student.models import TestCenterUser, TestCenterRegistration
 
@@ -23,6 +24,7 @@ class Command(BaseCommand):
     and TestCenterRegistration tables with status.
     """
 
+    @staticmethod
     def datadog_error(string, tags):
         dog_http_api.event("Pearson Import", string, alert_type='error', tags=tags)
 
diff --git a/common/djangoapps/student/management/commands/pearson_make_tc_user.py b/common/djangoapps/student/management/commands/pearson_make_tc_user.py
index da9bfc3bd0..87e0b4dadd 100644
--- a/common/djangoapps/student/management/commands/pearson_make_tc_user.py
+++ b/common/djangoapps/student/management/commands/pearson_make_tc_user.py
@@ -1,7 +1,7 @@
 from optparse import make_option
 
 from django.contrib.auth.models import User
-from django.core.management.base import BaseCommand
+from django.core.management.base import BaseCommand, CommandError
 
 from student.models import TestCenterUser, TestCenterUserForm
 
@@ -161,15 +161,16 @@ class Command(BaseCommand):
             if form.is_valid():
                 form.update_and_save()
             else:
+                errorlist = []
                 if (len(form.errors) > 0):
-                    print "Field Form errors encountered:"
-                for fielderror in form.errors:
-                    print "Field Form Error:  %s" % fielderror
-                    if (len(form.non_field_errors()) > 0):
-                        print "Non-field Form errors encountered:"
-                        for nonfielderror in form.non_field_errors:
-                            print "Non-field Form Error:  %s" % nonfielderror
-                    
+                    errorlist.append("Field Form errors encountered:")
+                    for fielderror in form.errors:
+                        errorlist.append("Field Form Error:  {}".format(fielderror))
+                if (len(form.non_field_errors()) > 0):
+                    errorlist.append("Non-field Form errors encountered:")
+                    for nonfielderror in form.non_field_errors:
+                        errorlist.append("Non-field Form Error:  {}".format(nonfielderror))
+                raise CommandError("\n".join(errorlist))    
         else:
             print "No changes necessary to make to existing user's demographics."
             
diff --git a/common/djangoapps/student/management/commands/pearson_transfer.py b/common/djangoapps/student/management/commands/pearson_transfer.py
index c216d2ceac..2124bdceb6 100644
--- a/common/djangoapps/student/management/commands/pearson_transfer.py
+++ b/common/djangoapps/student/management/commands/pearson_transfer.py
@@ -1,5 +1,6 @@
 import os
 from optparse import make_option
+from stat import S_ISDIR
 
 from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
@@ -26,85 +27,99 @@ class Command(BaseCommand):
                     action='store',
                     dest='mode',
                     default='both',
+                    choices=('import', 'export', 'both'),
                     help='mode is import, export, or both'),
     )
 
     def handle(self, **options):
 
-        # TODO: this doesn't work. Need to check if it's a property.
-        if not settings.PEARSON:
+        if not hasattr(settings, 'PEARSON'):
             raise CommandError('No PEARSON entries in auth/env.json.')
 
-        for value in ['LOCAL_IMPORT', 'SFTP_IMPORT', 'LOCAL_EXPORT',
-                      'SFTP_EXPORT', 'SFTP_HOSTNAME', 'SFTP_USERNAME', 'SFTP_PASSWORD']:
+        # check settings needed for either import or export:
+        for value in ['SFTP_HOSTNAME', 'SFTP_USERNAME', 'SFTP_PASSWORD', 'S3_BUCKET']:
             if value not in settings.PEARSON:
                 raise CommandError('No entry in the PEARSON settings'
                                    '(env/auth.json) for {0}'.format(value))
 
-        def import_pearson():
-            try:
-                sftp(settings.PEARSON['SFTP_IMPORT'],
-                     settings.PEARSON['LOCAL_IMPORT'], options['mode'])
-                s3(settings.PEARSON['LOCAL_IMPORT'],
-                   settings.PEARSON['BUCKET'], options['mode'])
-            except Exception as e:
-                dog_http_api.event('Pearson Import failure', str(e))
-            else:
-                for file in os.listdir(settings.PEARSON['LOCAL_IMPORT']):
-                    call_command('pearson_import_conf_zip',
-                                 settings.PEARSON['LOCAL_IMPORT'] + '/' + file)
-                    os.remove(file)
+        for value in ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']:
+            if not hasattr(settings, value):
+                raise CommandError('No entry in the AWS settings'
+                                   '(env/auth.json) for {0}'.format(value))
+                
+        # check additional required settings for import and export:
+        if options['mode'] in ('export', 'both'):
+            for value in ['LOCAL_EXPORT','SFTP_EXPORT']: 
+                if value not in settings.PEARSON:
+                    raise CommandError('No entry in the PEARSON settings'
+                                       '(env/auth.json) for {0}'.format(value))
+            # make sure that the import directory exists or can be created:
+            source_dir = settings.PEARSON['LOCAL_EXPORT']
+            if not os.path.isdir(source_dir):
+                os.makedirs(source_dir)
+                    
+        if options['mode'] in ('import', 'both'):
+            for value in ['LOCAL_IMPORT','SFTP_IMPORT']: 
+                if value not in settings.PEARSON:
+                    raise CommandError('No entry in the PEARSON settings'
+                                       '(env/auth.json) for {0}'.format(value))
+            # make sure that the import directory exists or can be created:
+            dest_dir = settings.PEARSON['LOCAL_IMPORT']
+            if not os.path.isdir(dest_dir):
+                os.makedirs(dest_dir)
 
-        def export_pearson():
-            call_command('pearson_export_cdd', 'dest_from_settings')
-            call_command('pearson_export_ead', 'dest_from_settings')
-            sftp(settings.PEARSON['LOCAL_EXPORT'],
-                 settings.PEARSON['SFTP_EXPORT'], options['mode'])
-            s3(settings.PEARSON['LOCAL_EXPORT'],
-               settings.PEARSON['BUCKET'], options['mode'])
 
-        if options['mode'] == 'export':
-            export_pearson()
-        elif options['mode'] == 'import':
-            import_pearson()
-        else:
-            export_pearson()
-            import_pearson()
-
-        def sftp(files_from, files_to, mode):
+        def sftp(files_from, files_to, mode, deleteAfterCopy=False):
             with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'):
                 try:
                     t = paramiko.Transport((settings.PEARSON['SFTP_HOSTNAME'], 22))
                     t.connect(username=settings.PEARSON['SFTP_USERNAME'],
                               password=settings.PEARSON['SFTP_PASSWORD'])
                     sftp = paramiko.SFTPClient.from_transport(t)
-                    if os.path.isdir(files_from):
+                    
+                    if mode == 'export':
+                        try:
+                            sftp.chdir(files_to)
+                        except IOError:
+                            raise CommandError('SFTP destination path does not exist: {}'.format(files_to))
                         for filename in os.listdir(files_from):
-                            sftp.put(files_from + '/' + filename,
-                                     files_to + '/' + filename)
+                            sftp.put(files_from + '/' + filename, filename)
+                            if deleteAfterCopy:
+                                os.remove(os.path.join(files_from, filename))
                     else:
-                        for filename in sftp.listdir(files_from):
-                            sftp.get(files_from + '/' + filename,
-                                     files_to + '/' + filename)
-                            sftp.remove(files_from + '/' + filename)
-                    t.close()
+                        try:
+                            sftp.chdir(files_from)
+                        except IOError:
+                            raise CommandError('SFTP source path does not exist: {}'.format(files_from))
+                        for filename in sftp.listdir('.'):
+                            # skip subdirectories 
+                            if not S_ISDIR(sftp.stat(filename).st_mode):
+                                sftp.get(filename, files_to + '/' + filename)
+                                # delete files from sftp server once they are successfully pulled off:
+                                if deleteAfterCopy:
+                                    sftp.remove(filename)
                 except:
                     dog_http_api.event('pearson {0}'.format(mode),
                                        'sftp uploading failed',
                                        alert_type='error')
                     raise
+                finally:
+                    sftp.close()
+                    t.close()
 
-        def s3(files_from, bucket, mode):
+        def s3(files_from, bucket, mode, deleteAfterCopy=False):
             with dog_stats_api.timer('pearson.{0}'.format(mode), tags='s3'):
                 try:
                     for filename in os.listdir(files_from):
-                        upload_file_to_s3(bucket, files_from + '/' + filename)
+                        upload_file_to_s3(bucket, files_from, filename)
+                        if deleteAfterCopy:
+                            os.remove(files_from + '/' + filename)
                 except:
                     dog_http_api.event('pearson {0}'.format(mode),
                                        's3 archiving failed')
                     raise
 
-        def upload_file_to_s3(bucket, filename):
+        def upload_file_to_s3(bucket, source_dir, filename):
             """
             Upload file to S3
             """
@@ -114,4 +129,32 @@ class Command(BaseCommand):
             b = s3.get_bucket(bucket)
             k = Key(b)
             k.key = "{filename}".format(filename=filename)
-            k.set_contents_from_filename(filename)
+            k.set_contents_from_filename(os.path.join(source_dir, filename))
+
+        def export_pearson():
+            options = { 'dest-from-settings' : True }
+            call_command('pearson_export_cdd', **options)
+            call_command('pearson_export_ead', **options)
+            mode = 'export'
+            sftp(settings.PEARSON['LOCAL_EXPORT'], settings.PEARSON['SFTP_EXPORT'], mode, deleteAfterCopy = False)
+            s3(settings.PEARSON['LOCAL_EXPORT'], settings.PEARSON['S3_BUCKET'], mode, deleteAfterCopy=True)
+
+        def import_pearson():
+            mode = 'import'
+            try:
+                sftp(settings.PEARSON['SFTP_IMPORT'], settings.PEARSON['LOCAL_IMPORT'], mode, deleteAfterCopy = True)
+                s3(settings.PEARSON['LOCAL_IMPORT'], settings.PEARSON['S3_BUCKET'], mode, deleteAfterCopy=False)
+            except Exception as e:
+                dog_http_api.event('Pearson Import failure', str(e))
+                raise e
+            else:
+                for filename in os.listdir(settings.PEARSON['LOCAL_IMPORT']):
+                    filepath = os.path.join(settings.PEARSON['LOCAL_IMPORT'], filename)
+                    call_command('pearson_import_conf_zip', filepath)
+                    os.remove(filepath)
+
+        # actually do the work!
+        if options['mode'] in ('export', 'both'):
+            export_pearson()
+        if options['mode'] in ('import', 'both'):
+            import_pearson()
diff --git a/common/djangoapps/student/management/commands/tests/test_pearson.py b/common/djangoapps/student/management/commands/tests/test_pearson.py
index d5594926d2..199557bf87 100644
--- a/common/djangoapps/student/management/commands/tests/test_pearson.py
+++ b/common/djangoapps/student/management/commands/tests/test_pearson.py
@@ -11,12 +11,12 @@ import sys
 
 from django.test import TestCase
 from django.core.management import call_command
+from nose.plugins.skip import SkipTest
 
 from student.models import User, TestCenterRegistration, TestCenterUser, get_testcenter_registration
 
 log = logging.getLogger(__name__)
 
-
 def create_tc_user(username):
     user = User.objects.create_user(username, '{}@edx.org'.format(username), 'fakepass')
     options = {
@@ -47,6 +47,48 @@ def create_tc_registration(username, course_id = 'org1/course1/term1', exam_code
     registrations = get_testcenter_registration(user, course_id, exam_code)
     return registrations[0]
 
+def create_multiple_registrations(prefix='test'):
+    username1 = '{}_multiple1'.format(prefix)
+    create_tc_user(username1)
+    create_tc_registration(username1)
+    create_tc_registration(username1, course_id = 'org1/course2/term1')
+    create_tc_registration(username1, exam_code = 'exam2')
+    username2 = '{}_multiple2'.format(prefix)
+    create_tc_user(username2)
+    create_tc_registration(username2)
+    username3 = '{}_multiple3'.format(prefix)
+    create_tc_user(username3)
+    create_tc_registration(username3, course_id = 'org1/course2/term1')
+    username4 = '{}_multiple4'.format(prefix)
+    create_tc_user(username4)
+    create_tc_registration(username4, exam_code = 'exam2')
+
+def get_command_error_text(*args, **options):
+    stderr_string = None
+    old_stderr = sys.stderr
+    sys.stderr = cStringIO.StringIO()
+    try:
+        call_command(*args, **options)
+    except SystemExit, why1:
+        # The goal here is to catch CommandError calls.
+        # But these are actually translated into nice messages,
+        # and sys.exit(1) is then called.  For testing, we
+        # want to catch what sys.exit throws, and get the
+        # relevant text either from stdout or stderr. 
+        if (why1.message > 0):
+            stderr_string = sys.stderr.getvalue()
+        else:
+            raise why1
+    except Exception, why:
+        raise why
+    
+    finally:
+        sys.stderr = old_stderr
+            
+    if stderr_string is None:
+        raise Exception("Expected call to {} to fail, but it succeeded!".format(args[0]))            
+    return stderr_string
+  
 def get_error_string_for_management_call(*args, **options):
     stdout_string = None
     old_stdout = sys.stdout
@@ -55,17 +97,17 @@ def get_error_string_for_management_call(*args, **options):
     sys.stderr = cStringIO.StringIO()
     try:
         call_command(*args, **options)
-    except BaseException, why1:
+    except SystemExit, why1:
         # The goal here is to catch CommandError calls.
         # But these are actually translated into nice messages,
         # and sys.exit(1) is then called.  For testing, we
         # want to catch what sys.exit throws, and get the
         # relevant text either from stdout or stderr. 
-        # TODO: this should really check to see that we
-        # arrived here because of a sys.exit(1).  Otherwise
-        # we should just raise the exception.
-        stdout_string = sys.stdout.getvalue()
-        stderr_string = sys.stderr.getvalue()
+        if (why1.message == 1):
+            stdout_string = sys.stdout.getvalue()
+            stderr_string = sys.stderr.getvalue()
+        else:
+            raise why1
     except Exception, why:
         raise why
     
@@ -111,6 +153,12 @@ class PearsonTestCase(TestCase):
         # clean up after any test data was dumped to temp directory
         delete_temp_dir(self.import_dir)
         delete_temp_dir(self.export_dir)
+        
+        # and clean up the database:
+#        TestCenterUser.objects.all().delete()
+#        TestCenterRegistration.objects.all().delete()
+
+class PearsonCommandTestCase(PearsonTestCase):
 
     def test_missing_demographic_fields(self):
         # We won't bother to test all details of form validation here.  
@@ -119,16 +167,16 @@ class PearsonTestCase(TestCase):
         username = 'baduser'
         User.objects.create_user(username, '{}@edx.org'.format(username), 'fakepass')
         options = {}
-        output_string, _ = get_error_string_for_management_call('pearson_make_tc_user', username, **options) 
-        self.assertTrue(output_string.find('Field Form errors encountered:') >= 0)
-        self.assertTrue(output_string.find('Field Form Error:  city') >= 0)
-        self.assertTrue(output_string.find('Field Form Error:  first_name') >= 0)
-        self.assertTrue(output_string.find('Field Form Error:  last_name') >= 0)
-        self.assertTrue(output_string.find('Field Form Error:  country') >= 0)
-        self.assertTrue(output_string.find('Field Form Error:  phone_country_code') >= 0)
-        self.assertTrue(output_string.find('Field Form Error:  phone') >= 0)
-        self.assertTrue(output_string.find('Field Form Error:  address_1') >= 0)
-        self.assertErrorContains(output_string, 'Field Form Error:  address_1')
+        error_string = get_command_error_text('pearson_make_tc_user', username, **options) 
+        self.assertTrue(error_string.find('Field Form errors encountered:') >= 0)
+        self.assertTrue(error_string.find('Field Form Error:  city') >= 0)
+        self.assertTrue(error_string.find('Field Form Error:  first_name') >= 0)
+        self.assertTrue(error_string.find('Field Form Error:  last_name') >= 0)
+        self.assertTrue(error_string.find('Field Form Error:  country') >= 0)
+        self.assertTrue(error_string.find('Field Form Error:  phone_country_code') >= 0)
+        self.assertTrue(error_string.find('Field Form Error:  phone') >= 0)
+        self.assertTrue(error_string.find('Field Form Error:  address_1') >= 0)
+        self.assertErrorContains(error_string, 'Field Form Error:  address_1')
         
     def test_create_good_testcenter_user(self):
         testcenter_user = create_tc_user("test1")
@@ -141,11 +189,11 @@ class PearsonTestCase(TestCase):
         self.assertIsNotNone(registration)
 
     def test_cdd_missing_option(self):
-        _, error_string = get_error_string_for_management_call('pearson_export_cdd', **{})
+        error_string = get_command_error_text('pearson_export_cdd', **{})
         self.assertErrorContains(error_string, 'Error: --destination or --dest-from-settings must be used')
    
     def test_ead_missing_option(self):
-        _, error_string = get_error_string_for_management_call('pearson_export_ead', **{})
+        error_string = get_command_error_text('pearson_export_ead', **{})
         self.assertErrorContains(error_string, 'Error: --destination or --dest-from-settings must be used')
 
     def test_export_single_cdd(self):
@@ -211,21 +259,7 @@ class PearsonTestCase(TestCase):
             os.remove(filepath)
 
     def test_export_multiple(self):
-        username1 = 'test_multiple1'
-        create_tc_user(username1)
-        create_tc_registration(username1)
-        create_tc_registration(username1, course_id = 'org1/course2/term1')
-        create_tc_registration(username1, exam_code = 'exam2')
-        username2 = 'test_multiple2'
-        create_tc_user(username2)
-        create_tc_registration(username2)
-        username3 = 'test_multiple3'
-        create_tc_user(username3)
-        create_tc_registration(username3, course_id = 'org1/course2/term1')
-        username4 = 'test_multiple4'
-        create_tc_user(username4)
-        create_tc_registration(username4, exam_code = 'exam2')
-
+        create_multiple_registrations("export")
         with self.settings(PEARSON={ 'LOCAL_EXPORT' : self.export_dir }):
             options = { 'dest-from-settings' : True }
             call_command('pearson_export_cdd', **options)
@@ -239,3 +273,110 @@ class PearsonTestCase(TestCase):
             os.remove(filepath)
 
 
+#    def test_bad_demographic_option(self):
+#        username = 'nonuser'
+#        output_string, stderrmsg = get_error_string_for_management_call('pearson_make_tc_user', username, **{'--garbage' : None })
+#        print stderrmsg
+#        self.assertErrorContains(stderrmsg, 'Unexpected option')
+#
+#    def test_missing_demographic_user(self):
+#        username = 'nonuser'
+#        output_string, error_string = get_error_string_for_management_call('pearson_make_tc_user', username, **{})
+#        self.assertErrorContains(error_string, 'User matching query does not exist')
+
+# credentials for a test SFTP site:
+SFTP_HOSTNAME = 'ec2-23-20-150-101.compute-1.amazonaws.com'
+SFTP_USERNAME = 'pearsontest'
+SFTP_PASSWORD = 'password goes here'
+
+S3_BUCKET = 'edx-pearson-archive'
+AWS_ACCESS_KEY_ID = 'put yours here'
+AWS_SECRET_ACCESS_KEY = 'put yours here'
+
+class PearsonTransferTestCase(PearsonTestCase):
+    '''
+    Class for tests running Pearson transfers
+    '''
+
+    def test_transfer_config(self):
+        with self.settings(DATADOG_API='FAKE_KEY'):
+            # TODO: why is this failing with the wrong error message?!
+            stderrmsg = get_command_error_text('pearson_transfer', **{'mode' : 'garbage'})
+            self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries')
+        with self.settings(DATADOG_API='FAKE_KEY'):
+            stderrmsg = get_command_error_text('pearson_transfer')
+            self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries')
+        with self.settings(DATADOG_API='FAKE_KEY',
+                           PEARSON={'LOCAL_EXPORT' : self.export_dir,
+                                    'LOCAL_IMPORT' : self.import_dir }):
+            stderrmsg = get_command_error_text('pearson_transfer')
+            self.assertErrorContains(stderrmsg, 'Error: No entry in the PEARSON settings')
+
+    def test_transfer_export_missing_dest_dir(self):
+        raise SkipTest()
+        create_multiple_registrations('export_missing_dest')
+        with self.settings(DATADOG_API='FAKE_KEY',
+                           PEARSON={'LOCAL_EXPORT' : self.export_dir,
+                                    'SFTP_EXPORT' : 'this/does/not/exist',
+                                    'SFTP_HOSTNAME' : SFTP_HOSTNAME,
+                                    'SFTP_USERNAME' : SFTP_USERNAME,
+                                    'SFTP_PASSWORD' : SFTP_PASSWORD, 
+                                    'S3_BUCKET' : S3_BUCKET,
+                                    },
+                           AWS_ACCESS_KEY_ID = AWS_ACCESS_KEY_ID,
+                           AWS_SECRET_ACCESS_KEY = AWS_SECRET_ACCESS_KEY):
+            options = { 'mode' : 'export'}
+            stderrmsg = get_command_error_text('pearson_transfer', **options)
+            self.assertErrorContains(stderrmsg, 'Error: SFTP destination path does not exist')
+
+    def test_transfer_export(self):
+        raise SkipTest()
+        create_multiple_registrations("transfer_export")
+        with self.settings(DATADOG_API='FAKE_KEY',
+                           PEARSON={'LOCAL_EXPORT' : self.export_dir,
+                                    'SFTP_EXPORT' : 'results/topvue',
+                                    'SFTP_HOSTNAME' : SFTP_HOSTNAME,
+                                    'SFTP_USERNAME' : SFTP_USERNAME,
+                                    'SFTP_PASSWORD' : SFTP_PASSWORD, 
+                                    'S3_BUCKET' : S3_BUCKET,
+                                    },
+                           AWS_ACCESS_KEY_ID = AWS_ACCESS_KEY_ID,
+                           AWS_SECRET_ACCESS_KEY = AWS_SECRET_ACCESS_KEY):
+            options = { 'mode' : 'export'}
+#            call_command('pearson_transfer', **options)
+#            # confirm that the export directory is still empty:
+#            self.assertEqual(len(os.listdir(self.export_dir)), 0, "expected export directory to be empty")
+
+    def test_transfer_import_missing_source_dir(self):
+        raise SkipTest()
+        create_multiple_registrations('import_missing_src')
+        with self.settings(DATADOG_API='FAKE_KEY',
+                           PEARSON={'LOCAL_IMPORT' : self.import_dir,
+                                    'SFTP_IMPORT' : 'this/does/not/exist',
+                                    'SFTP_HOSTNAME' : SFTP_HOSTNAME,
+                                    'SFTP_USERNAME' : SFTP_USERNAME,
+                                    'SFTP_PASSWORD' : SFTP_PASSWORD, 
+                                    'S3_BUCKET' : S3_BUCKET,
+                                    },
+                           AWS_ACCESS_KEY_ID = AWS_ACCESS_KEY_ID,
+                           AWS_SECRET_ACCESS_KEY = AWS_SECRET_ACCESS_KEY):
+            options = { 'mode' : 'import'}
+            stderrmsg = get_command_error_text('pearson_transfer', **options)
+            self.assertErrorContains(stderrmsg, 'Error: SFTP source path does not exist')
+
+    def test_transfer_import(self):
+        raise SkipTest()
+        create_multiple_registrations('import_missing_src')
+        with self.settings(DATADOG_API='FAKE_KEY',
+                           PEARSON={'LOCAL_IMPORT' : self.import_dir,
+                                    'SFTP_IMPORT' : 'results',
+                                    'SFTP_HOSTNAME' : SFTP_HOSTNAME,
+                                    'SFTP_USERNAME' : SFTP_USERNAME,
+                                    'SFTP_PASSWORD' : SFTP_PASSWORD, 
+                                    'S3_BUCKET' : S3_BUCKET,
+                                    },
+                           AWS_ACCESS_KEY_ID = AWS_ACCESS_KEY_ID,
+                           AWS_SECRET_ACCESS_KEY = AWS_SECRET_ACCESS_KEY):
+            options = { 'mode' : 'import'}
+            call_command('pearson_transfer', **options)
+            self.assertEqual(len(os.listdir(self.import_dir)), 0, "expected import directory to be empty")

From 645cfbce21fef12f5793764f5ff9b6bc5111a920 Mon Sep 17 00:00:00 2001
From: Ashley Penney <apenney@edx.org>
Date: Thu, 24 Jan 2013 10:00:13 -0500
Subject: [PATCH 28/29] Move this into the github requirements where it
 belongs.

---
 github-requirements.txt | 1 +
 requirements.txt        | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/github-requirements.txt b/github-requirements.txt
index 468d55ce65..32193ec853 100644
--- a/github-requirements.txt
+++ b/github-requirements.txt
@@ -3,3 +3,4 @@
 -e git://github.com/MITx/django-pipeline.git#egg=django-pipeline
 -e git://github.com/MITx/django-wiki.git@e2e84558#egg=django-wiki
 -e git://github.com/dementrock/pystache_custom.git@776973740bdaad83a3b029f96e415a7d1e8bec2f#egg=pystache_custom-dev
+-e git+ssh://git@github.com/MITx/dogapi.git@003a4fc9#egg=dogapi
diff --git a/requirements.txt b/requirements.txt
index 1b1384912b..fa4688b711 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -59,4 +59,3 @@ Shapely==1.2.16
 ipython==0.13.1
 xmltodict==0.4.1
 paramiko==1.9.0
-git+ssh://git@github.com/MITx/dogapi.git@003a4fc9#egg=dogapi

From 126ff55cd530d15cd7c7b07fcf62029512da8767 Mon Sep 17 00:00:00 2001
From: Ashley Penney <apenney@edx.org>
Date: Thu, 24 Jan 2013 10:06:09 -0500
Subject: [PATCH 29/29] Not sure why this was set to use SSH.

---
 github-requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/github-requirements.txt b/github-requirements.txt
index 32193ec853..62e47a328f 100644
--- a/github-requirements.txt
+++ b/github-requirements.txt
@@ -3,4 +3,4 @@
 -e git://github.com/MITx/django-pipeline.git#egg=django-pipeline
 -e git://github.com/MITx/django-wiki.git@e2e84558#egg=django-wiki
 -e git://github.com/dementrock/pystache_custom.git@776973740bdaad83a3b029f96e415a7d1e8bec2f#egg=pystache_custom-dev
--e git+ssh://git@github.com/MITx/dogapi.git@003a4fc9#egg=dogapi
+-e git://github.com/MITx/dogapi.git@003a4fc9#egg=dogapi