fix mega merge conflicts

common/djangoapps/cache_toolbox/core.py

@@ -12,6 +12,7 @@ from django.core.cache import cache
 from django.db import DEFAULT_DB_ALIAS
 
 from . import app_settings
+from xmodule.contentstore.content import StaticContent
 
 
 def get_instance(model, instance_or_pk, timeout=None, using=None):
@@ -108,14 +109,14 @@ def instance_key(model, instance_or_pk):
         getattr(instance_or_pk, 'pk', instance_or_pk),
     )
 
-def content_key(filename):
-    return 'content:%s' % (filename)
 
 def set_cached_content(content):
-    cache.set(content_key(content.filename), content)
+    cache.set(str(content.location), content)
 
-def get_cached_content(filename):
-    return cache.get(content_key(filename))
 
-def del_cached_content(filename):
-    cache.delete(content_key(filename))
+def get_cached_content(location):
+    return cache.get(str(location))
+
+
+def del_cached_content(location):
+    cache.delete(str(location))

common/djangoapps/contentserver/middleware.py

@@ -5,6 +5,7 @@ from django.http import HttpResponse, Http404, HttpResponseNotModified
 
 from xmodule.contentstore.django import contentstore
 from xmodule.contentstore.content import StaticContent, XASSET_LOCATION_TAG
+from xmodule.modulestore import InvalidLocationError
 from cache_toolbox.core import get_cached_content, set_cached_content
 from xmodule.exceptions import NotFoundError
 
@@ -12,16 +13,25 @@ from xmodule.exceptions import NotFoundError
 class StaticContentServer(object):
     def process_request(self, request):
         # look to see if the request is prefixed with 'c4x' tag
-        if request.path.startswith('/' + XASSET_LOCATION_TAG):
+        if request.path.startswith('/' + XASSET_LOCATION_TAG + '/'):
+            try:
+                loc = StaticContent.get_location_from_path(request.path)
+            except InvalidLocationError:
+                # return a 'Bad Request' to browser as we have a malformed Location
+                response = HttpResponse()
+                response.status_code = 400
+                return response     
 
             # first look in our cache so we don't have to round-trip to the DB
-            content = get_cached_content(request.path)
+            content = get_cached_content(loc)
             if content is None:
                 # nope, not in cache, let's fetch from DB
                 try:
-                    content = contentstore().find(request.path)
+                    content = contentstore().find(loc)
                 except NotFoundError:
-                    raise Http404
+                    response = HttpResponse()
+                    response.status_code = 404
+                    return response
 
                 # since we fetched it from DB, let's cache it going forward
                 set_cached_content(content)

common/djangoapps/course_groups/cohorts.py

@@ -0,0 +1,277 @@
+"""
+This file contains the logic for cohort groups, as exposed internally to the
+forums, and to the cohort admin views.
+"""
+
+from django.contrib.auth.models import User
+from django.http import Http404
+import logging
+import random
+
+from courseware import courses
+from student.models import get_user_by_username_or_email
+from .models import CourseUserGroup
+
+log = logging.getLogger(__name__)
+
+
+# tl;dr: global state is bad.  capa reseeds random every time a problem is loaded.  Even
+# if and when that's fixed, it's a good idea to have a local generator to avoid any other
+# code that messes with the global random module.
+_local_random = None
+
+def local_random():
+    """
+    Get the local random number generator.  In a function so that we don't run
+    random.Random() at import time.
+    """
+    # ironic, isn't it?
+    global _local_random
+    
+    if _local_random is None:
+        _local_random = random.Random()
+
+    return _local_random
+
+def is_course_cohorted(course_id):
+    """
+    Given a course id, return a boolean for whether or not the course is
+    cohorted.
+
+    Raises:
+       Http404 if the course doesn't exist.
+    """
+    return courses.get_course_by_id(course_id).is_cohorted
+
+
+def get_cohort_id(user, course_id):
+    """
+    Given a course id and a user, return the id of the cohort that user is
+    assigned to in that course.  If they don't have a cohort, return None.
+    """
+    cohort = get_cohort(user, course_id)
+    return None if cohort is None else cohort.id
+
+
+def is_commentable_cohorted(course_id, commentable_id):
+    """
+    Args:
+        course_id: string
+        commentable_id: string
+
+    Returns:
+        Bool: is this commentable cohorted?
+
+    Raises:
+        Http404 if the course doesn't exist.
+    """
+    course = courses.get_course_by_id(course_id)
+
+    if not course.is_cohorted:
+        # this is the easy case :)
+        ans = False
+    elif commentable_id in course.top_level_discussion_topic_ids:
+        # top level discussions have to be manually configured as cohorted
+        # (default is not)
+        ans = commentable_id in course.cohorted_discussions
+    else:
+        # inline discussions are cohorted by default
+        ans = True
+
+    log.debug("is_commentable_cohorted({0}, {1}) = {2}".format(course_id,
+                                                               commentable_id,
+                                                               ans))
+    return ans
+
+
+def get_cohorted_commentables(course_id):
+    """
+    Given a course_id return a list of strings representing cohorted commentables
+    """
+
+    course = courses.get_course_by_id(course_id)
+
+    if not course.is_cohorted:
+        # this is the easy case :)
+        ans = []
+    else:
+        ans = course.cohorted_discussions
+
+    return ans
+
+
+def get_cohort(user, course_id):
+    """
+    Given a django User and a course_id, return the user's cohort in that
+    cohort.
+
+    Arguments:
+        user: a Django User object.
+        course_id: string in the format 'org/course/run'
+
+    Returns:
+        A CourseUserGroup object if the course is cohorted and the User has a
+        cohort, else None.
+
+    Raises:
+       ValueError if the course_id doesn't exist.
+    """
+    # First check whether the course is cohorted (users shouldn't be in a cohort
+    # in non-cohorted courses, but settings can change after course starts)
+    try:
+        course = courses.get_course_by_id(course_id)
+    except Http404:
+        raise ValueError("Invalid course_id")
+
+    if not course.is_cohorted:
+        return None
+
+    try:
+        return CourseUserGroup.objects.get(course_id=course_id,
+                                            group_type=CourseUserGroup.COHORT,
+                                            users__id=user.id)
+    except CourseUserGroup.DoesNotExist:
+        # Didn't find the group.  We'll go on to create one if needed.
+        pass
+
+    if not course.auto_cohort:
+        return None
+
+    choices = course.auto_cohort_groups
+    n = len(choices)
+    if n == 0:
+        # Nowhere to put user
+        log.warning("Course %s is auto-cohorted, but there are no"
+                    " auto_cohort_groups specified",
+                    course_id)
+        return None
+
+    # Put user in a random group, creating it if needed
+    group_name = local_random().choice(choices)
+
+    group, created = CourseUserGroup.objects.get_or_create(
+        course_id=course_id,
+        group_type=CourseUserGroup.COHORT,
+        name=group_name)
+
+    user.course_groups.add(group)
+    return group
+
+
+def get_course_cohorts(course_id):
+    """
+    Get a list of all the cohorts in the given course.
+
+    Arguments:
+        course_id: string in the format 'org/course/run'
+
+    Returns:
+        A list of CourseUserGroup objects.  Empty if there are no cohorts. Does
+        not check whether the course is cohorted.
+    """
+    return list(CourseUserGroup.objects.filter(course_id=course_id,
+                                               group_type=CourseUserGroup.COHORT))
+
+### Helpers for cohort management views
+
+
+def get_cohort_by_name(course_id, name):
+    """
+    Return the CourseUserGroup object for the given cohort.  Raises DoesNotExist
+    it isn't present.
+    """
+    return CourseUserGroup.objects.get(course_id=course_id,
+                                       group_type=CourseUserGroup.COHORT,
+                                       name=name)
+
+
+def get_cohort_by_id(course_id, cohort_id):
+    """
+    Return the CourseUserGroup object for the given cohort.  Raises DoesNotExist
+    it isn't present.  Uses the course_id for extra validation...
+    """
+    return CourseUserGroup.objects.get(course_id=course_id,
+                                       group_type=CourseUserGroup.COHORT,
+                                       id=cohort_id)
+
+
+def add_cohort(course_id, name):
+    """
+    Add a cohort to a course.  Raises ValueError if a cohort of the same name already
+    exists.
+    """
+    log.debug("Adding cohort %s to %s", name, course_id)
+    if CourseUserGroup.objects.filter(course_id=course_id,
+                                      group_type=CourseUserGroup.COHORT,
+                                      name=name).exists():
+        raise ValueError("Can't create two cohorts with the same name")
+
+    return CourseUserGroup.objects.create(course_id=course_id,
+                                          group_type=CourseUserGroup.COHORT,
+                                          name=name)
+
+
+class CohortConflict(Exception):
+    """
+    Raised when user to be added is already in another cohort in same course.
+    """
+    pass
+
+
+def add_user_to_cohort(cohort, username_or_email):
+    """
+    Look up the given user, and if successful, add them to the specified cohort.
+
+    Arguments:
+        cohort: CourseUserGroup
+        username_or_email: string.  Treated as email if has '@'
+
+    Returns:
+        User object.
+
+    Raises:
+        User.DoesNotExist if can't find user.
+
+        ValueError if user already present in this cohort.
+
+        CohortConflict if user already in another cohort.
+    """
+    user = get_user_by_username_or_email(username_or_email)
+
+    # If user in any cohorts in this course already, complain
+    course_cohorts = CourseUserGroup.objects.filter(
+        course_id=cohort.course_id,
+        users__id=user.id,
+        group_type=CourseUserGroup.COHORT)
+    if course_cohorts.exists():
+        if course_cohorts[0] == cohort:
+            raise ValueError("User {0} already present in cohort {1}".format(
+                user.username,
+                cohort.name))
+        else:
+            raise CohortConflict("User {0} is in another cohort {1} in course"
+                                 .format(user.username,
+                                         course_cohorts[0].name))
+
+    cohort.users.add(user)
+    return user
+
+
+def get_course_cohort_names(course_id):
+    """
+    Return a list of the cohort names in a course.
+    """
+    return [c.name for c in get_course_cohorts(course_id)]
+
+
+def delete_empty_cohort(course_id, name):
+    """
+    Remove an empty cohort.  Raise ValueError if cohort is not empty.
+    """
+    cohort = get_cohort_by_name(course_id, name)
+    if cohort.users.exists():
+        raise ValueError(
+            "Can't delete non-empty cohort {0} in course {1}".format(
+                name, course_id))
+
+    cohort.delete()

common/djangoapps/course_groups/models.py

@@ -0,0 +1,33 @@
+import logging
+
+from django.contrib.auth.models import User
+from django.db import models
+
+log = logging.getLogger(__name__)
+
+
+class CourseUserGroup(models.Model):
+    """
+    This model represents groups of users in a course.  Groups may have different types,
+    which may be treated specially.  For example, a user can be in at most one cohort per
+    course, and cohorts are used to split up the forums by group.
+    """
+    class Meta:
+        unique_together = (('name', 'course_id'), )
+
+    name = models.CharField(max_length=255,
+                            help_text=("What is the name of this group?  "
+                                       "Must be unique within a course."))
+    users = models.ManyToManyField(User, db_index=True, related_name='course_groups',
+                                   help_text="Who is in this group?")
+
+    # Note: groups associated with particular runs of a course.  E.g. Fall 2012 and Spring
+    # 2013 versions of 6.00x will have separate groups.
+    course_id = models.CharField(max_length=255, db_index=True,
+                                 help_text="Which course is this group associated with?")
+
+    # For now, only have group type 'cohort', but adding a type field to support
+    # things like 'question_discussion', 'friends', 'off-line-class', etc
+    COHORT = 'cohort'
+    GROUP_TYPE_CHOICES = ((COHORT, 'Cohort'),)
+    group_type = models.CharField(max_length=20, choices=GROUP_TYPE_CHOICES)

common/djangoapps/course_groups/tests/tests.py

@@ -0,0 +1,276 @@
+import django.test
+from django.contrib.auth.models import User
+from django.conf import settings
+
+from django.test.utils import override_settings
+
+from course_groups.models import CourseUserGroup
+from course_groups.cohorts import (get_cohort, get_course_cohorts,
+                                   is_commentable_cohorted, get_cohort_by_name)
+
+from xmodule.modulestore.django import modulestore, _MODULESTORES
+
+# NOTE: running this with the lms.envs.test config works without
+# manually overriding the modulestore.  However, running with
+# cms.envs.test doesn't.
+
+
+def xml_store_config(data_dir):
+    return {
+    'default': {
+        'ENGINE': 'xmodule.modulestore.xml.XMLModuleStore',
+        'OPTIONS': {
+            'data_dir': data_dir,
+            'default_class': 'xmodule.hidden_module.HiddenDescriptor',
+        }
+    }
+}
+
+TEST_DATA_DIR = settings.COMMON_TEST_DATA_ROOT
+TEST_DATA_XML_MODULESTORE = xml_store_config(TEST_DATA_DIR)
+
+
+@override_settings(MODULESTORE=TEST_DATA_XML_MODULESTORE)
+class TestCohorts(django.test.TestCase):
+
+
+    @staticmethod
+    def topic_name_to_id(course, name):
+        """
+        Given a discussion topic name, return an id for that name (includes
+        course and url_name).
+        """
+        return "{course}_{run}_{name}".format(course=course.location.course,
+                                              run=course.url_name,
+                                              name=name)
+
+
+    @staticmethod
+    def config_course_cohorts(course, discussions,
+                              cohorted,
+                              cohorted_discussions=None,
+                              auto_cohort=None,
+                              auto_cohort_groups=None):
+        """
+        Given a course with no discussion set up, add the discussions and set
+        the cohort config appropriately.
+
+        Arguments:
+            course: CourseDescriptor
+            discussions: list of topic names strings.  Picks ids and sort_keys
+                automatically.
+            cohorted: bool.
+            cohorted_discussions: optional list of topic names.  If specified,
+                converts them to use the same ids as topic names.
+            auto_cohort: optional bool.
+            auto_cohort_groups: optional list of strings
+                      (names of groups to put students into).
+
+        Returns:
+            Nothing -- modifies course in place.
+        """
+        def to_id(name):
+            return TestCohorts.topic_name_to_id(course, name)
+
+        topics = dict((name, {"sort_key": "A",
+                              "id": to_id(name)})
+                      for name in discussions)
+
+        course.discussion_topics = topics
+
+        d = {"cohorted": cohorted}
+        if cohorted_discussions is not None:
+            d["cohorted_discussions"] = [to_id(name)
+                                         for name in cohorted_discussions]
+
+        if auto_cohort is not None:
+            d["auto_cohort"] = auto_cohort
+        if auto_cohort_groups is not None:
+            d["auto_cohort_groups"] = auto_cohort_groups
+
+        course.cohort_config = d
+
+
+    def setUp(self):
+        """
+        Make sure that course is reloaded every time--clear out the modulestore.
+        """
+        # don't like this, but don't know a better way to undo all changes made
+        # to course.  We don't have a course.clone() method.
+        _MODULESTORES.clear()
+
+
+    def test_get_cohort(self):
+        """
+        Make sure get_cohort() does the right thing when the course is cohorted
+        """
+        course = modulestore().get_course("edX/toy/2012_Fall")
+        self.assertEqual(course.id, "edX/toy/2012_Fall")
+        self.assertFalse(course.is_cohorted)
+
+        user = User.objects.create(username="test", email="a@b.com")
+        other_user = User.objects.create(username="test2", email="a2@b.com")
+
+        self.assertIsNone(get_cohort(user, course.id), "No cohort created yet")
+
+        cohort = CourseUserGroup.objects.create(name="TestCohort",
+                                                course_id=course.id,
+                               group_type=CourseUserGroup.COHORT)
+
+        cohort.users.add(user)
+
+        self.assertIsNone(get_cohort(user, course.id),
+                          "Course isn't cohorted, so shouldn't have a cohort")
+
+        # Make the course cohorted...
+        self.config_course_cohorts(course, [], cohorted=True)
+
+        self.assertEquals(get_cohort(user, course.id).id, cohort.id,
+                          "Should find the right cohort")
+
+        self.assertEquals(get_cohort(other_user, course.id), None,
+                          "other_user shouldn't have a cohort")
+
+    def test_auto_cohorting(self):
+        """
+        Make sure get_cohort() does the right thing when the course is auto_cohorted
+        """
+        course = modulestore().get_course("edX/toy/2012_Fall")
+        self.assertEqual(course.id, "edX/toy/2012_Fall")
+        self.assertFalse(course.is_cohorted)
+
+        user1 = User.objects.create(username="test", email="a@b.com")
+        user2 = User.objects.create(username="test2", email="a2@b.com")
+        user3 = User.objects.create(username="test3", email="a3@b.com")
+
+        cohort = CourseUserGroup.objects.create(name="TestCohort",
+                                                course_id=course.id,
+                               group_type=CourseUserGroup.COHORT)
+
+        # user1 manually added to a cohort
+        cohort.users.add(user1)
+
+        # Make the course auto cohorted...
+        self.config_course_cohorts(course, [], cohorted=True,
+                                   auto_cohort=True,
+                                   auto_cohort_groups=["AutoGroup"])
+
+        self.assertEquals(get_cohort(user1, course.id).id, cohort.id,
+                          "user1 should stay put")
+
+        self.assertEquals(get_cohort(user2, course.id).name, "AutoGroup",
+                          "user2 should be auto-cohorted")
+
+        # Now make the group list empty
+        self.config_course_cohorts(course, [], cohorted=True,
+                                   auto_cohort=True,
+                                   auto_cohort_groups=[])
+
+        self.assertEquals(get_cohort(user3, course.id), None,
+                          "No groups->no auto-cohorting")
+
+        # Now make it different
+        self.config_course_cohorts(course, [], cohorted=True,
+                                   auto_cohort=True,
+                                   auto_cohort_groups=["OtherGroup"])
+
+        self.assertEquals(get_cohort(user3, course.id).name, "OtherGroup",
+                          "New list->new group")
+        self.assertEquals(get_cohort(user2, course.id).name, "AutoGroup",
+                          "user2 should still be in originally placed cohort")
+
+
+    def test_auto_cohorting_randomization(self):
+        """
+        Make sure get_cohort() randomizes properly.
+        """
+        course = modulestore().get_course("edX/toy/2012_Fall")
+        self.assertEqual(course.id, "edX/toy/2012_Fall")
+        self.assertFalse(course.is_cohorted)
+
+        groups = ["group_{0}".format(n) for n in range(5)]
+        self.config_course_cohorts(course, [], cohorted=True,
+                                   auto_cohort=True,
+                                   auto_cohort_groups=groups)
+
+        # Assign 100 users to cohorts
+        for i in range(100):
+            user = User.objects.create(username="test_{0}".format(i),
+                                       email="a@b{0}.com".format(i))
+            get_cohort(user, course.id)
+
+        # Now make sure that the assignment was at least vaguely random:
+        # each cohort should have at least 1, and fewer than 50 students.
+        # (with 5 groups, probability of 0 users in any group is about
+        # .8**100= 2.0e-10)
+        for cohort_name in groups:
+            cohort = get_cohort_by_name(course.id, cohort_name)
+            num_users = cohort.users.count()
+            self.assertGreater(num_users, 1)
+            self.assertLess(num_users, 50)
+
+
+
+    def test_get_course_cohorts(self):
+        course1_id = 'a/b/c'
+        course2_id = 'e/f/g'
+
+        # add some cohorts to course 1
+        cohort = CourseUserGroup.objects.create(name="TestCohort",
+                                                course_id=course1_id,
+                                                group_type=CourseUserGroup.COHORT)
+
+        cohort = CourseUserGroup.objects.create(name="TestCohort2",
+                                                course_id=course1_id,
+                                                group_type=CourseUserGroup.COHORT)
+
+
+        # second course should have no cohorts
+        self.assertEqual(get_course_cohorts(course2_id), [])
+
+        cohorts = sorted([c.name for c in get_course_cohorts(course1_id)])
+        self.assertEqual(cohorts, ['TestCohort', 'TestCohort2'])
+
+
+    def test_is_commentable_cohorted(self):
+        course = modulestore().get_course("edX/toy/2012_Fall")
+        self.assertFalse(course.is_cohorted)
+
+        def to_id(name):
+            return self.topic_name_to_id(course, name)
+
+        # no topics
+        self.assertFalse(is_commentable_cohorted(course.id, to_id("General")),
+                         "Course doesn't even have a 'General' topic")
+
+        # not cohorted
+        self.config_course_cohorts(course, ["General", "Feedback"],
+                                   cohorted=False)
+
+        self.assertFalse(is_commentable_cohorted(course.id, to_id("General")),
+                         "Course isn't cohorted")
+
+        # cohorted, but top level topics aren't
+        self.config_course_cohorts(course, ["General", "Feedback"],
+                                   cohorted=True)
+
+        self.assertTrue(course.is_cohorted)
+        self.assertFalse(is_commentable_cohorted(course.id, to_id("General")),
+                         "Course is cohorted, but 'General' isn't.")
+
+        self.assertTrue(
+            is_commentable_cohorted(course.id, to_id("random")),
+            "Non-top-level discussion is always cohorted in cohorted courses.")
+
+        # cohorted, including "Feedback" top-level topics aren't
+        self.config_course_cohorts(course, ["General", "Feedback"],
+                                   cohorted=True,
+                                   cohorted_discussions=["Feedback"])
+
+        self.assertTrue(course.is_cohorted)
+        self.assertFalse(is_commentable_cohorted(course.id, to_id("General")),
+                         "Course is cohorted, but 'General' isn't.")
+
+        self.assertTrue(
+            is_commentable_cohorted(course.id, to_id("Feedback")),
+            "Feedback was listed as cohorted.  Should be.")

common/djangoapps/course_groups/views.py

@@ -0,0 +1,222 @@
+from django_future.csrf import ensure_csrf_cookie
+from django.contrib.auth.decorators import login_required
+from django.views.decorators.http import require_POST
+from django.contrib.auth.models import User
+from django.core.context_processors import csrf
+from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
+from django.core.urlresolvers import reverse
+from django.http import HttpResponse, HttpResponseForbidden, Http404
+from django.shortcuts import redirect
+import json
+import logging
+import re
+
+from courseware.courses import get_course_with_access
+from mitxmako.shortcuts import render_to_response, render_to_string
+
+from .models import CourseUserGroup
+from . import cohorts
+
+import track.views
+
+
+log = logging.getLogger(__name__)
+
+
+def json_http_response(data):
+    """
+    Return an HttpResponse with the data json-serialized and the right content
+    type header.
+    """
+    return HttpResponse(json.dumps(data), content_type="application/json")
+
+
+def split_by_comma_and_whitespace(s):
+    """
+    Split a string both by commas and whitespice.  Returns a list.
+    """
+    return re.split(r'[\s,]+', s)
+
+
+@ensure_csrf_cookie
+def list_cohorts(request, course_id):
+    """
+    Return json dump of dict:
+
+    {'success': True,
+     'cohorts': [{'name': name, 'id': id}, ...]}
+    """
+    get_course_with_access(request.user, course_id, 'staff')
+
+    all_cohorts = [{'name': c.name, 'id': c.id}
+               for c in cohorts.get_course_cohorts(course_id)]
+
+    return json_http_response({'success': True,
+                            'cohorts': all_cohorts})
+
+
+@ensure_csrf_cookie
+@require_POST
+def add_cohort(request, course_id):
+    """
+    Return json of dict:
+    {'success': True,
+     'cohort': {'id': id,
+                'name': name}}
+
+                or
+
+    {'success': False,
+     'msg': error_msg} if there's an error
+    """
+    get_course_with_access(request.user, course_id, 'staff')
+
+    name = request.POST.get("name")
+    if not name:
+        return json_http_response({'success': False,
+                                'msg': "No name specified"})
+
+    try:
+        cohort = cohorts.add_cohort(course_id, name)
+    except ValueError as err:
+        return json_http_response({'success': False,
+                                'msg': str(err)})
+
+    return json_http_response({'success': 'True',
+                            'cohort': {
+                                'id': cohort.id,
+                                'name': cohort.name
+                                }})
+
+
+@ensure_csrf_cookie
+def users_in_cohort(request, course_id, cohort_id):
+    """
+    Return users in the cohort.  Show up to 100 per page, and page
+    using the 'page' GET attribute in the call.  Format:
+
+    Returns:
+        Json dump of dictionary in the following format:
+        {'success': True,
+         'page': page,
+         'num_pages': paginator.num_pages,
+         'users': [{'username': ..., 'email': ..., 'name': ...}]
+    }
+    """
+    get_course_with_access(request.user, course_id, 'staff')
+
+    # this will error if called with a non-int cohort_id.  That's ok--it
+    # shoudn't happen for valid clients.
+    cohort = cohorts.get_cohort_by_id(course_id, int(cohort_id))
+
+    paginator = Paginator(cohort.users.all(), 100)
+    page = request.GET.get('page')
+    try:
+        users = paginator.page(page)
+    except PageNotAnInteger:
+        # return the first page
+        page = 1
+        users = paginator.page(page)
+    except EmptyPage:
+        # Page is out of range.  Return last page
+        page = paginator.num_pages
+        contacts = paginator.page(page)
+
+    user_info = [{'username': u.username,
+                  'email': u.email,
+                  'name': '{0} {1}'.format(u.first_name, u.last_name)}
+                  for u in users]
+
+    return json_http_response({'success': True,
+                            'page': page,
+                            'num_pages': paginator.num_pages,
+                            'users': user_info})
+
+
+@ensure_csrf_cookie
+@require_POST
+def add_users_to_cohort(request, course_id, cohort_id):
+    """
+    Return json dict of:
+
+    {'success': True,
+     'added': [{'username': username,
+                'name': name,
+                'email': email}, ...],
+     'conflict': [{'username_or_email': ...,
+                    'msg': ...}],  # in another cohort
+     'present': [str1, str2, ...],    # already there
+     'unknown': [str1, str2, ...]}
+    """
+    get_course_with_access(request.user, course_id, 'staff')
+
+    cohort = cohorts.get_cohort_by_id(course_id, cohort_id)
+
+    users = request.POST.get('users', '')
+    added = []
+    present = []
+    conflict = []
+    unknown = []
+    for username_or_email in split_by_comma_and_whitespace(users):
+        try:
+            user = cohorts.add_user_to_cohort(cohort, username_or_email)
+            added.append({'username': user.username,
+                          'name': "{0} {1}".format(user.first_name, user.last_name),
+                          'email': user.email,
+                          })
+        except ValueError:
+            present.append(username_or_email)
+        except User.DoesNotExist:
+            unknown.append(username_or_email)
+        except cohorts.CohortConflict as err:
+            conflict.append({'username_or_email': username_or_email,
+                              'msg': str(err)})
+
+
+    return json_http_response({'success': True,
+                            'added': added,
+                            'present': present,
+                            'conflict': conflict,
+                            'unknown': unknown})
+
+
+@ensure_csrf_cookie
+@require_POST
+def remove_user_from_cohort(request, course_id, cohort_id):
+    """
+    Expects 'username': username in POST data.
+
+    Return json dict of:
+
+    {'success': True} or
+    {'success': False,
+     'msg': error_msg}
+    """
+    get_course_with_access(request.user, course_id, 'staff')
+
+    username = request.POST.get('username')
+    if username is None:
+        return json_http_response({'success': False,
+                                'msg': 'No username specified'})
+
+    cohort = cohorts.get_cohort_by_id(course_id, cohort_id)
+    try:
+        user = User.objects.get(username=username)
+        cohort.users.remove(user)
+        return json_http_response({'success': True})
+    except User.DoesNotExist:
+        log.debug('no user')
+        return json_http_response({'success': False,
+                                'msg': "No user '{0}'".format(username)})
+
+
+def debug_cohort_mgmt(request, course_id):
+    """
+    Debugging view for dev.
+    """
+    # add staff check to make sure it's safe if it's accidentally deployed.
+    get_course_with_access(request.user, course_id, 'staff')
+
+    context = {'cohorts_ajax_url': reverse('cohorts',
+                                           kwargs={'course_id': course_id})}
+    return render_to_response('/course_groups/debug.html', context)

common/djangoapps/external_auth/admin.py

@@ -5,8 +5,9 @@ django admin pages for courseware model
 from external_auth.models import *
 from django.contrib import admin
 
+
 class ExternalAuthMapAdmin(admin.ModelAdmin):
-    search_fields = ['external_id','user__username']
+    search_fields = ['external_id', 'user__username']
     date_hierarchy = 'dtcreated'
 
 admin.site.register(ExternalAuthMap, ExternalAuthMapAdmin)

common/djangoapps/external_auth/models.py

@@ -12,20 +12,20 @@ file and check it in at the same time as your model changes. To do that,
 from django.db import models
 from django.contrib.auth.models import User
 
+
 class ExternalAuthMap(models.Model):
     class Meta:
         unique_together = (('external_id', 'external_domain'), )
     external_id = models.CharField(max_length=255, db_index=True)
     external_domain = models.CharField(max_length=255, db_index=True)
-    external_credentials = models.TextField(blank=True)  # JSON dictionary 
+    external_credentials = models.TextField(blank=True)  # JSON dictionary
     external_email = models.CharField(max_length=255, db_index=True)
-    external_name = models.CharField(blank=True,max_length=255, db_index=True)
+    external_name = models.CharField(blank=True, max_length=255, db_index=True)
     user = models.OneToOneField(User, unique=True, db_index=True, null=True)
-    internal_password = models.CharField(blank=True, max_length=31)	# randomly generated
-    dtcreated = models.DateTimeField('creation date',auto_now_add=True)
-    dtsignup = models.DateTimeField('signup date',null=True)		# set after signup
-    
+    internal_password = models.CharField(blank=True, max_length=31)  	# randomly generated
+    dtcreated = models.DateTimeField('creation date', auto_now_add=True)
+    dtsignup = models.DateTimeField('signup date', null=True)		# set after signup
+
     def __unicode__(self):
         s = "[%s] = (%s / %s)" % (self.external_id, self.external_name, self.external_email)
         return s
-    

common/djangoapps/external_auth/tests/test_openid_provider.py

@@ -0,0 +1,211 @@
+'''
+Created on Jan 18, 2013
+
+@author: brian
+'''
+import openid
+from openid.fetchers import HTTPFetcher, HTTPResponse
+from urlparse import parse_qs
+
+from django.conf import settings
+from django.test import TestCase, LiveServerTestCase
+# from django.contrib.auth.models import User
+from django.core.urlresolvers import reverse
+from django.test.client import RequestFactory
+
+
+class MyFetcher(HTTPFetcher):
+    """A fetcher that uses server-internal calls for performing HTTP
+    requests.
+    """
+
+    def __init__(self, client):
+        """@param client: A test client object"""
+
+        super(MyFetcher, self).__init__()
+        self.client = client
+
+    def fetch(self, url, body=None, headers=None):
+        """Perform an HTTP request
+
+        @raises Exception: Any exception that can be raised by Django
+
+        @see: C{L{HTTPFetcher.fetch}}
+        """
+        if body:
+            # method = 'POST'
+            # undo the URL encoding of the POST arguments
+            data = parse_qs(body)
+            response = self.client.post(url, data)
+        else:
+            # method = 'GET'
+            data = {}
+            if headers and 'Accept' in headers:
+                data['CONTENT_TYPE'] = headers['Accept']
+            response = self.client.get(url, data)
+
+        # Translate the test client response to the fetcher's HTTP response abstraction
+        content = response.content
+        final_url = url
+        response_headers = {}
+        if 'Content-Type' in response:
+            response_headers['content-type'] = response['Content-Type']
+        if 'X-XRDS-Location' in response:
+            response_headers['x-xrds-location'] = response['X-XRDS-Location']
+        status = response.status_code
+
+        return HTTPResponse(
+            body=content,
+            final_url=final_url,
+            headers=response_headers,
+            status=status,
+            )
+
+
+class OpenIdProviderTest(TestCase):
+
+#    def setUp(self):
+#        username = 'viewtest'
+#        email = 'view@test.com'
+#        password = 'foo'
+#        user = User.objects.create_user(username, email, password)
+
+    def testBeginLoginWithXrdsUrl(self):
+        # skip the test if openid is not enabled (as in cms.envs.test):
+        if not settings.MITX_FEATURES.get('AUTH_USE_OPENID') or not settings.MITX_FEATURES.get('AUTH_USE_OPENID_PROVIDER'):
+            return
+
+        # the provider URL must be converted to an absolute URL in order to be
+        # used as an openid provider.
+        provider_url = reverse('openid-provider-xrds')
+        factory = RequestFactory()
+        request = factory.request()
+        abs_provider_url = request.build_absolute_uri(location=provider_url)
+
+        # In order for this absolute URL to work (i.e. to get xrds, then authentication)
+        # in the test environment, we either need a live server that works with the default
+        # fetcher (i.e. urlopen2), or a test server that is reached through a custom fetcher.
+        # Here we do the latter:
+        fetcher = MyFetcher(self.client)
+        openid.fetchers.setDefaultFetcher(fetcher, wrap_exceptions=False)
+
+        # now we can begin the login process by invoking a local openid client,
+        # with a pointer to the (also-local) openid provider:
+        with self.settings(OPENID_SSO_SERVER_URL=abs_provider_url):
+            url = reverse('openid-login')
+            resp = self.client.post(url)
+            code = 200
+            self.assertEqual(resp.status_code, code,
+                             "got code {0} for url '{1}'. Expected code {2}"
+                             .format(resp.status_code, url, code))
+
+    def testBeginLoginWithLoginUrl(self):
+        # skip the test if openid is not enabled (as in cms.envs.test):
+        if not settings.MITX_FEATURES.get('AUTH_USE_OPENID') or not settings.MITX_FEATURES.get('AUTH_USE_OPENID_PROVIDER'):
+            return
+
+        # the provider URL must be converted to an absolute URL in order to be
+        # used as an openid provider.
+        provider_url = reverse('openid-provider-login')
+        factory = RequestFactory()
+        request = factory.request()
+        abs_provider_url = request.build_absolute_uri(location=provider_url)
+
+        # In order for this absolute URL to work (i.e. to get xrds, then authentication)
+        # in the test environment, we either need a live server that works with the default
+        # fetcher (i.e. urlopen2), or a test server that is reached through a custom fetcher.
+        # Here we do the latter:
+        fetcher = MyFetcher(self.client)
+        openid.fetchers.setDefaultFetcher(fetcher, wrap_exceptions=False)
+
+        # now we can begin the login process by invoking a local openid client,
+        # with a pointer to the (also-local) openid provider:
+        with self.settings(OPENID_SSO_SERVER_URL=abs_provider_url):
+            url = reverse('openid-login')
+            resp = self.client.post(url)
+            code = 200
+            self.assertEqual(resp.status_code, code,
+                             "got code {0} for url '{1}'. Expected code {2}"
+                             .format(resp.status_code, url, code))
+            self.assertContains(resp, '<input name="openid.mode" type="hidden" value="checkid_setup" />', html=True)
+            self.assertContains(resp, '<input name="openid.ns" type="hidden" value="http://specs.openid.net/auth/2.0" />', html=True)
+            self.assertContains(resp, '<input name="openid.identity" type="hidden" value="http://specs.openid.net/auth/2.0/identifier_select" />', html=True)
+            self.assertContains(resp, '<input name="openid.claimed_id" type="hidden" value="http://specs.openid.net/auth/2.0/identifier_select" />', html=True)
+            self.assertContains(resp, '<input name="openid.ns.ax" type="hidden" value="http://openid.net/srv/ax/1.0" />', html=True)
+            self.assertContains(resp, '<input name="openid.ax.mode" type="hidden" value="fetch_request" />', html=True)
+            self.assertContains(resp, '<input name="openid.ax.required" type="hidden" value="email,fullname,old_email,firstname,old_nickname,lastname,old_fullname,nickname" />', html=True)
+            self.assertContains(resp, '<input name="openid.ax.type.fullname" type="hidden" value="http://axschema.org/namePerson" />', html=True)
+            self.assertContains(resp, '<input name="openid.ax.type.lastname" type="hidden" value="http://axschema.org/namePerson/last" />', html=True)
+            self.assertContains(resp, '<input name="openid.ax.type.firstname" type="hidden" value="http://axschema.org/namePerson/first" />', html=True)
+            self.assertContains(resp, '<input name="openid.ax.type.nickname" type="hidden" value="http://axschema.org/namePerson/friendly" />', html=True)
+            self.assertContains(resp, '<input name="openid.ax.type.email" type="hidden" value="http://axschema.org/contact/email" />', html=True)
+            self.assertContains(resp, '<input name="openid.ax.type.old_email" type="hidden" value="http://schema.openid.net/contact/email" />', html=True)
+            self.assertContains(resp, '<input name="openid.ax.type.old_nickname" type="hidden" value="http://schema.openid.net/namePerson/friendly" />', html=True)
+            self.assertContains(resp, '<input name="openid.ax.type.old_fullname" type="hidden" value="http://schema.openid.net/namePerson" />', html=True)
+            self.assertContains(resp, '<input type="submit" value="Continue" />', html=True)
+            # this should work on the server:
+            self.assertContains(resp, '<input name="openid.realm" type="hidden" value="http://testserver/" />', html=True)
+
+            # not included here are elements that will vary from run to run:
+            # <input name="openid.return_to" type="hidden" value="http://testserver/openid/complete/?janrain_nonce=2013-01-23T06%3A20%3A17ZaN7j6H" />
+            # <input name="openid.assoc_handle" type="hidden" value="{HMAC-SHA1}{50ff8120}{rh87+Q==}" />
+
+
+    def testOpenIdSetup(self):
+        if not settings.MITX_FEATURES.get('AUTH_USE_OPENID_PROVIDER'):
+            return
+        url = reverse('openid-provider-login')
+        post_args = {
+                     "openid.mode": "checkid_setup",
+                     "openid.return_to": "http://testserver/openid/complete/?janrain_nonce=2013-01-23T06%3A20%3A17ZaN7j6H",
+                     "openid.assoc_handle": "{HMAC-SHA1}{50ff8120}{rh87+Q==}",
+                     "openid.claimed_id": "http://specs.openid.net/auth/2.0/identifier_select",
+                     "openid.ns": "http://specs.openid.net/auth/2.0",
+                     "openid.realm": "http://testserver/",
+                     "openid.identity": "http://specs.openid.net/auth/2.0/identifier_select",
+                     "openid.ns.ax": "http://openid.net/srv/ax/1.0",
+                     "openid.ax.mode": "fetch_request",
+                     "openid.ax.required": "email,fullname,old_email,firstname,old_nickname,lastname,old_fullname,nickname",
+                     "openid.ax.type.fullname": "http://axschema.org/namePerson",
+                     "openid.ax.type.lastname": "http://axschema.org/namePerson/last",
+                     "openid.ax.type.firstname": "http://axschema.org/namePerson/first",
+                     "openid.ax.type.nickname": "http://axschema.org/namePerson/friendly",
+                     "openid.ax.type.email": "http://axschema.org/contact/email",
+                     "openid.ax.type.old_email": "http://schema.openid.net/contact/email",
+                     "openid.ax.type.old_nickname": "http://schema.openid.net/namePerson/friendly",
+                     "openid.ax.type.old_fullname": "http://schema.openid.net/namePerson",
+                     }
+        resp = self.client.post(url, post_args)
+        code = 200
+        self.assertEqual(resp.status_code, code,
+                         "got code {0} for url '{1}'. Expected code {2}"
+                         .format(resp.status_code, url, code))
+
+
+# In order for this absolute URL to work (i.e. to get xrds, then authentication)
+# in the test environment, we either need a live server that works with the default
+# fetcher (i.e. urlopen2), or a test server that is reached through a custom fetcher.
+# Here we do the former.
+class OpenIdProviderLiveServerTest(LiveServerTestCase):
+
+    def testBeginLogin(self):
+        # skip the test if openid is not enabled (as in cms.envs.test):
+        if not settings.MITX_FEATURES.get('AUTH_USE_OPENID') or not settings.MITX_FEATURES.get('AUTH_USE_OPENID_PROVIDER'):
+            return
+
+        # the provider URL must be converted to an absolute URL in order to be
+        # used as an openid provider.
+        provider_url = reverse('openid-provider-xrds')
+        factory = RequestFactory()
+        request = factory.request()
+        abs_provider_url = request.build_absolute_uri(location=provider_url)
+
+        # now we can begin the login process by invoking a local openid client,
+        # with a pointer to the (also-local) openid provider:
+        with self.settings(OPENID_SSO_SERVER_URL=abs_provider_url):
+            url = reverse('openid-login')
+            resp = self.client.post(url)
+            code = 200
+            self.assertEqual(resp.status_code, code,
+                             "got code {0} for url '{1}'. Expected code {2}"
+                             .format(resp.status_code, url, code))

common/djangoapps/external_auth/views.py

@@ -217,6 +217,52 @@ def ssl_dn_extract_info(dn):
     return (user, email, fullname)
 
 
+def ssl_get_cert_from_request(request):
+    """
+    Extract user information from certificate, if it exists, returning (user, email, fullname).
+    Else return None.
+    """
+    certkey = "SSL_CLIENT_S_DN"  # specify the request.META field to use
+
+    cert = request.META.get(certkey, '')
+    if not cert:
+        cert = request.META.get('HTTP_' + certkey, '')
+    if not cert:
+        try:
+            # try the direct apache2 SSL key
+            cert = request._req.subprocess_env.get(certkey, '')
+        except Exception:
+            return ''
+
+    return cert
+
+    (user, email, fullname) = ssl_dn_extract_info(cert)
+    return (user, email, fullname)
+
+
+def ssl_login_shortcut(fn):
+    """
+    Python function decorator for login procedures, to allow direct login
+    based on existing ExternalAuth record and MIT ssl certificate.
+    """
+    def wrapped(*args, **kwargs):
+        if not settings.MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES']:
+            return fn(*args, **kwargs)
+        request = args[0]
+        cert = ssl_get_cert_from_request(request)
+        if not cert:		# no certificate information - show normal login window
+            return fn(*args, **kwargs)
+
+        (user, email, fullname) = ssl_dn_extract_info(cert)
+        return external_login_or_signup(request,
+                                        external_id=email,
+                                        external_domain="ssl:MIT",
+                                        credentials=cert,
+                                        email=email,
+                                        fullname=fullname)
+    return wrapped
+
+
 @csrf_exempt
 def ssl_login(request):
     """
@@ -234,17 +280,7 @@ def ssl_login(request):
 
     Else continues on with student.views.index, and no authentication.
     """
-    certkey = "SSL_CLIENT_S_DN"  # specify the request.META field to use
-
-    cert = request.META.get(certkey, '')
-    if not cert:
-        cert = request.META.get('HTTP_' + certkey, '')
-    if not cert:
-        try:
-            # try the direct apache2 SSL key
-            cert = request._req.subprocess_env.get(certkey, '')
-        except Exception:
-            cert = None
+    cert = ssl_get_cert_from_request(request)
 
     if not cert:
         # no certificate information - go onward to main index
@@ -402,7 +438,9 @@ def provider_login(request):
     store = DjangoOpenIDStore()
     server = Server(store, endpoint)
 
-    # handle OpenID request
+    # first check to see if the request is an OpenID request.
+    # If so, the client will have specified an 'openid.mode' as part
+    # of the request.
     querydict = dict(request.REQUEST.items())
     error = False
     if 'openid.mode' in request.GET or 'openid.mode' in request.POST:
@@ -422,6 +460,8 @@ def provider_login(request):
                                     openid_request.answer(False), {})
 
         # checkid_setup, so display login page
+        # (by falling through to the provider_login at the
+        # bottom of this method).
         elif openid_request.mode == 'checkid_setup':
             if openid_request.idSelect():
                 # remember request and original path
@@ -440,8 +480,10 @@ def provider_login(request):
             return provider_respond(server, openid_request,
                                     server.handleRequest(openid_request), {})
 
-    # handle login
-    if request.method == 'POST' and 'openid_setup' in request.session:
+    # handle login redirection:  these are also sent to this view function,
+    # but are distinguished by lacking the openid mode.  We also know that
+    # they are posts, because they come from the popup
+    elif request.method == 'POST' and 'openid_setup' in request.session:
         # get OpenID request from session
         openid_setup = request.session['openid_setup']
         openid_request = openid_setup['request']
@@ -453,6 +495,8 @@ def provider_login(request):
             return default_render_failure(request, "Invalid OpenID trust root")
 
         # check if user with given email exists
+        # Failure is redirected to this method (by using the original URL),
+        # which will bring up the login dialog.
         email = request.POST.get('email', None)
         try:
             user = User.objects.get(email=email)
@@ -462,7 +506,8 @@ def provider_login(request):
             log.warning(msg)
             return HttpResponseRedirect(openid_request_url)
 
-        # attempt to authenticate user
+        # attempt to authenticate user (but not actually log them in...)
+        # Failure is again redirected to the login dialog.
         username = user.username
         password = request.POST.get('password', None)
         user = authenticate(username=username, password=password)
@@ -473,7 +518,8 @@ def provider_login(request):
             log.warning(msg)
             return HttpResponseRedirect(openid_request_url)
 
-        # authentication succeeded, so log user in
+        # authentication succeeded, so fetch user information
+        # that was requested
         if user is not None and user.is_active:
             # remove error from session since login succeeded
             if 'openid_error' in request.session:
@@ -496,15 +542,21 @@ def provider_login(request):
             # missing fields is up to the Consumer. The proper change
             # should only return the username, however this will likely
             # break the CS50 client. Temporarily we will be returning
-            # username filling in for fullname in addition to username 
+            # username filling in for fullname in addition to username
             # as sreg nickname.
+
+            # Note too that this is hardcoded, and not really responding to
+            # the extensions that were registered in the first place.
             results = {
                 'nickname': user.username,
                 'email': user.email,
                 'fullname': user.username
                 }
+
+            # the request succeeded:
             return provider_respond(server, openid_request, response, results)
 
+        # the account is not active, so redirect back to the login page:
         request.session['openid_error'] = True
         msg = "Login failed - Account not active for user {0}".format(username)
         log.warning(msg)
@@ -523,7 +575,7 @@ def provider_login(request):
         'return_to': return_to
     })
 
-    # custom XRDS header necessary for discovery process
+    # add custom XRDS header necessary for discovery process
     response['X-XRDS-Location'] = get_xrds_url('xrds', request)
     return response
 

common/djangoapps/heartbeat/urls.py

@@ -0,0 +1,5 @@
+from django.conf.urls import *
+
+urlpatterns = patterns('',
+    url(r'^$', 'heartbeat.views.heartbeat', name='heartbeat'),
+)

common/djangoapps/heartbeat/views.py

@@ -0,0 +1,16 @@
+import json
+from datetime import datetime
+from django.http import HttpResponse
+from xmodule.modulestore.django import modulestore
+from dogapi import dog_stats_api
+
+@dog_stats_api.timed('edxapp.heartbeat')
+def heartbeat(request):
+    """
+    Simple view that a loadbalancer can check to verify that the app is up
+    """
+    output = {
+        'date': datetime.now().isoformat(),
+        'courses': [course.location.url() for course in modulestore().get_courses()],
+    }
+    return HttpResponse(json.dumps(output, indent=4))

common/djangoapps/mitxmako/makoloader.py

@@ -9,37 +9,39 @@ from django.template.loaders.app_directories import Loader as AppDirectoriesLoad
 from mitxmako.template import Template
 import mitxmako.middleware
 
+import tempdir
 
 log = logging.getLogger(__name__)
 
+
 class MakoLoader(object):
     """
     This is a Django loader object which will load the template as a
     Mako template if the first line is "## mako". It is based off BaseLoader
     in django.template.loader.
     """
-    
+
     is_usable = False
 
     def __init__(self, base_loader):
         # base_loader is an instance of a BaseLoader subclass
         self.base_loader = base_loader
-        
+
         module_directory = getattr(settings, 'MAKO_MODULE_DIR', None)
-        
+
         if module_directory is None:
             log.warning("For more caching of mako templates, set the MAKO_MODULE_DIR in settings!")
-            module_directory = tempfile.mkdtemp()
-        
+            module_directory = tempdir.mkdtemp_clean()
+
         self.module_directory = module_directory
-        
-        
+
+
     def __call__(self, template_name, template_dirs=None):
         return self.load_template(template_name, template_dirs)
 
     def load_template(self, template_name, template_dirs=None):
         source, file_path = self.load_template_source(template_name, template_dirs)
-        
+
         if source.startswith("## mako\n"):
             # This is a mako template
             template = Template(filename=file_path, module_directory=self.module_directory, uri=template_name)
@@ -56,23 +58,24 @@ class MakoLoader(object):
                 # This allows for correct identification (later) of the actual template that does
                 # not exist.
                 return source, file_path
-    
+
     def load_template_source(self, template_name, template_dirs=None):
         # Just having this makes the template load as an instance, instead of a class.
         return self.base_loader.load_template_source(template_name, template_dirs)
 
     def reset(self):
         self.base_loader.reset()
-    
+
 
 class MakoFilesystemLoader(MakoLoader):
     is_usable = True
-    
+
     def __init__(self):
         MakoLoader.__init__(self, FilesystemLoader())
-        
+
+
 class MakoAppDirectoriesLoader(MakoLoader):
     is_usable = True
-    
+
     def __init__(self):
         MakoLoader.__init__(self, AppDirectoriesLoader())

common/djangoapps/mitxmako/middleware.py

@@ -13,7 +13,7 @@
 #   limitations under the License.
 
 from mako.lookup import TemplateLookup
-import tempfile
+import tempdir
 from django.template import RequestContext
 from django.conf import settings
 
@@ -29,7 +29,7 @@ class MakoMiddleware(object):
         module_directory = getattr(settings, 'MAKO_MODULE_DIR', None)
 
         if module_directory is None:
-            module_directory = tempfile.mkdtemp()
+            module_directory = tempdir.mkdtemp_clean()
 
         for location in template_locations:
             lookup[location] = TemplateLookup(directories=template_locations[location],

common/djangoapps/mitxmako/template.py

@@ -20,13 +20,15 @@ from mitxmako import middleware
 django_variables = ['lookup', 'output_encoding', 'encoding_errors']
 
 # TODO: We should make this a Django Template subclass that simply has the MakoTemplate inside of it? (Intead of inheriting from MakoTemplate)
+
+
 class Template(MakoTemplate):
     """
     This bridges the gap between a Mako template and a djano template. It can
     be rendered like it is a django template because the arguments are transformed
     in a way that MakoTemplate can understand.
     """
-    
+
     def __init__(self, *args, **kwargs):
         """Overrides base __init__ to provide django variable overrides"""
         if not kwargs.get('no_django', False):
@@ -34,8 +36,8 @@ class Template(MakoTemplate):
             overrides['lookup'] = overrides['lookup']['main']
             kwargs.update(overrides)
         super(Template, self).__init__(*args, **kwargs)
-    
-    
+
+
     def render(self, context_instance):
         """
         This takes a render call with a context (from Django) and translates
@@ -43,7 +45,7 @@ class Template(MakoTemplate):
         """
         # collapse context_instance to a single dictionary for mako
         context_dictionary = {}
-        
+
         # In various testing contexts, there might not be a current request context.
         if middleware.requestcontext is not None:
             for d in middleware.requestcontext:
@@ -53,5 +55,5 @@ class Template(MakoTemplate):
         context_dictionary['settings'] = settings
         context_dictionary['MITX_ROOT_URL'] = settings.MITX_ROOT_URL
         context_dictionary['django_context'] = context_instance
-                
+
         return super(Template, self).render_unicode(**context_dictionary)

common/djangoapps/mitxmako/templatetag_helpers.py

@@ -2,14 +2,15 @@ from django.template import loader
 from django.template.base import Template, Context
 from django.template.loader import get_template, select_template
 
+
 def django_template_include(file_name, mako_context):
     """
     This can be used within a mako template to include a django template
     in the way that a django-style {% include %} does. Pass it context
     which can be the mako context ('context') or a dictionary.
     """
-    
-    dictionary = dict( mako_context )
+
+    dictionary = dict(mako_context)
     return loader.render_to_string(file_name, dictionary=dictionary)
 
 
@@ -18,7 +19,7 @@ def render_inclusion(func, file_name, takes_context, django_context, *args, **kw
     This allows a mako template to call a template tag function (written
     for django templates) that is an "inclusion tag". These functions are
     decorated with @register.inclusion_tag.
-    
+
     -func: This is the function that is registered as an inclusion tag.
     You must import it directly using a python import statement.
     -file_name: This is the filename of the template, passed into the
@@ -29,10 +30,10 @@ def render_inclusion(func, file_name, takes_context, django_context, *args, **kw
     a copy of the django context is available as 'django_context'.
     -*args and **kwargs are the arguments to func.
     """
-    
+
     if takes_context:
         args = [django_context] + list(args)
-    
+
     _dict = func(*args, **kwargs)
     if isinstance(file_name, Template):
         t = file_name
@@ -40,14 +41,12 @@ def render_inclusion(func, file_name, takes_context, django_context, *args, **kw
         t = select_template(file_name)
     else:
         t = get_template(file_name)
-        
+
     nodelist = t.nodelist
-    
+
     new_context = Context(_dict)
     csrf_token = django_context.get('csrf_token', None)
     if csrf_token is not None:
         new_context['csrf_token'] = csrf_token
-    
-    return nodelist.render(new_context)
-    
 
+    return nodelist.render(new_context)

common/djangoapps/static_replace.py

@@ -1,58 +0,0 @@
-import logging
-import re
-
-from staticfiles.storage import staticfiles_storage
-from staticfiles import finders
-from django.conf import settings
-
-log = logging.getLogger(__name__)
-
-def try_staticfiles_lookup(path):
-    """
-    Try to lookup a path in staticfiles_storage.  If it fails, return
-    a dead link instead of raising an exception.
-    """
-    try:
-        url = staticfiles_storage.url(path)
-    except Exception as err:
-        log.warning("staticfiles_storage couldn't find path {0}: {1}".format(
-            path, str(err)))
-        # Just return the original path; don't kill everything.
-        url = path
-    return url
-
-
-def replace(static_url, prefix=None):
-    if prefix is None:
-        prefix = ''
-    else:
-        prefix = prefix + '/'
-
-    quote = static_url.group('quote')
-
-    servable = (
-        # If in debug mode, we'll serve up anything that the finders can find
-        (settings.DEBUG and finders.find(static_url.group('rest'), True)) or
-        # Otherwise, we'll only serve up stuff that the storages can find
-        staticfiles_storage.exists(static_url.group('rest'))
-    )
-
-    if servable:
-        return static_url.group(0)
-    else:
-        # don't error if file can't be found
-        url = try_staticfiles_lookup(prefix + static_url.group('rest'))
-        return "".join([quote, url, quote])
-
-
-def replace_urls(text, staticfiles_prefix=None, replace_prefix='/static/'):
-    def replace_url(static_url):
-        return replace(static_url, staticfiles_prefix)
-
-    return re.sub(r"""
-        (?x)                 # flags=re.VERBOSE
-        (?P<quote>\\?['"])   # the opening quotes
-        (?P<prefix>{prefix}) # the prefix
-        (?P<rest>.*?)        # everything else in the url
-        (?P=quote)           # the first matching closing quote
-        """.format(prefix=replace_prefix), replace_url, text)

common/djangoapps/static_replace/__init__.py

@@ -0,0 +1,121 @@
+import logging
+import re
+
+from staticfiles.storage import staticfiles_storage
+from staticfiles import finders
+from django.conf import settings
+
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.xml import XMLModuleStore
+from xmodule.contentstore.content import StaticContent
+
+log = logging.getLogger(__name__)
+
+
+def _url_replace_regex(prefix):
+    """
+    Match static urls in quotes that don't end in '?raw'.
+
+    To anyone contemplating making this more complicated:
+    http://xkcd.com/1171/
+    """
+    return r"""
+        (?x)                      # flags=re.VERBOSE
+        (?P<quote>\\?['"])        # the opening quotes
+        (?P<prefix>{prefix})      # the prefix
+        (?P<rest>.*?)             # everything else in the url
+        (?P=quote)                # the first matching closing quote
+        """.format(prefix=prefix)
+
+
+def try_staticfiles_lookup(path):
+    """
+    Try to lookup a path in staticfiles_storage.  If it fails, return
+    a dead link instead of raising an exception.
+    """
+    try:
+        url = staticfiles_storage.url(path)
+    except Exception as err:
+        log.warning("staticfiles_storage couldn't find path {0}: {1}".format(
+            path, str(err)))
+        # Just return the original path; don't kill everything.
+        url = path
+    return url
+
+
+def replace_course_urls(text, course_id):
+    """
+    Replace /course/$stuff urls with /courses/$course_id/$stuff urls
+
+    text: The text to replace
+    course_module: A CourseDescriptor
+
+    returns: text with the links replaced
+    """
+
+
+    def replace_course_url(match):
+        quote = match.group('quote')
+        rest = match.group('rest')
+        return "".join([quote, '/courses/' + course_id + '/', rest, quote])
+
+    return re.sub(_url_replace_regex('/course/'), replace_course_url, text)
+
+
+def replace_static_urls(text, data_directory, course_namespace=None):
+    """
+    Replace /static/$stuff urls either with their correct url as generated by collectstatic,
+    (/static/$md5_hashed_stuff) or by the course-specific content static url
+    /static/$course_data_dir/$stuff, or, if course_namespace is not None, by the
+    correct url in the contentstore (c4x://)
+
+    text: The source text to do the substitution in
+    data_directory: The directory in which course data is stored
+    course_namespace: The course identifier used to distinguish static content for this course in studio
+    """
+
+    def replace_static_url(match):
+        original = match.group(0)
+        prefix = match.group('prefix')
+        quote = match.group('quote')
+        rest = match.group('rest')
+
+        # Don't mess with things that end in '?raw'
+        if rest.endswith('?raw'):
+            return original
+
+        # In debug mode, if we can find the url as is,
+        if settings.DEBUG and finders.find(rest, True):
+            return original
+        # if we're running with a MongoBacked store course_namespace is not None, then use studio style urls
+        elif  course_namespace is not None and not isinstance(modulestore(), XMLModuleStore):
+            # first look in the static file pipeline and see if we are trying to reference
+            # a piece of static content which is in the mitx repo (e.g. JS associated with an xmodule)
+            if staticfiles_storage.exists(rest):
+                url = staticfiles_storage.url(rest)
+            else:
+                # if not, then assume it's courseware specific content and then look in the
+                # Mongo-backed database
+                url = StaticContent.convert_legacy_static_url(rest, course_namespace)
+        # Otherwise, look the file up in staticfiles_storage, and append the data directory if needed
+        else:
+            course_path = "/".join((data_directory, rest))
+
+            try:
+                if staticfiles_storage.exists(rest):
+                    url = staticfiles_storage.url(rest)
+                else:
+                    url = staticfiles_storage.url(course_path)
+            # And if that fails, assume that it's course content, and add manually data directory
+            except Exception as err:
+                log.warning("staticfiles_storage couldn't find path {0}: {1}".format(
+                    rest, str(err)))
+                url = "".join([prefix, course_path])
+
+        return "".join([quote, url, quote])
+
+    return re.sub(
+        _url_replace_regex('/static/(?!{data_dir})'.format(data_dir=data_directory)),
+        replace_static_url,
+        text
+    )

common/djangoapps/static_replace/management/commands/clear_collectstatic_cache.py

@@ -0,0 +1,15 @@
+###
+### Script for importing courseware from XML format
+###
+
+from django.core.management.base import NoArgsCommand
+from django.core.cache import get_cache
+
+
+class Command(NoArgsCommand):
+    help = \
+'''Import the specified data directory into the default ModuleStore'''
+
+    def handle_noargs(self, **options):
+        staticfiles_cache = get_cache('staticfiles')
+        staticfiles_cache.clear()

common/djangoapps/static_replace/test/test_static_replace.py

@@ -0,0 +1,111 @@
+import re
+
+from nose.tools import assert_equals, assert_true, assert_false
+from static_replace import (replace_static_urls, replace_course_urls,
+                            _url_replace_regex)
+from mock import patch, Mock
+from xmodule.modulestore import Location
+from xmodule.modulestore.mongo import MongoModuleStore
+from xmodule.modulestore.xml import XMLModuleStore
+
+DATA_DIRECTORY = 'data_dir'
+COURSE_ID = 'org/course/run'
+NAMESPACE = Location('org', 'course', 'run', None, None)
+STATIC_SOURCE = '"/static/file.png"'
+
+
+def test_multi_replace():
+    course_source = '"/course/file.png"'
+
+    assert_equals(
+        replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY),
+        replace_static_urls(replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY), DATA_DIRECTORY)
+    )
+    assert_equals(
+        replace_course_urls(course_source, COURSE_ID),
+        replace_course_urls(replace_course_urls(course_source, COURSE_ID), COURSE_ID)
+    )
+
+
+@patch('static_replace.staticfiles_storage')
+def test_storage_url_exists(mock_storage):
+    mock_storage.exists.return_value = True
+    mock_storage.url.return_value = '/static/file.png'
+
+    assert_equals('"/static/file.png"', replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY))
+    mock_storage.exists.called_once_with('file.png')
+    mock_storage.url.called_once_with('data_dir/file.png')
+
+
+@patch('static_replace.staticfiles_storage')
+def test_storage_url_not_exists(mock_storage):
+    mock_storage.exists.return_value = False
+    mock_storage.url.return_value = '/static/data_dir/file.png'
+
+    assert_equals('"/static/data_dir/file.png"', replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY))
+    mock_storage.exists.called_once_with('file.png')
+    mock_storage.url.called_once_with('file.png')
+
+
+@patch('static_replace.StaticContent')
+@patch('static_replace.modulestore')
+def test_mongo_filestore(mock_modulestore, mock_static_content):
+
+    mock_modulestore.return_value = Mock(MongoModuleStore)
+    mock_static_content.convert_legacy_static_url.return_value = "c4x://mock_url"
+
+    # No namespace => no change to path
+    assert_equals('"/static/data_dir/file.png"', replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY))
+
+    # Namespace => content url
+    assert_equals(
+        '"' + mock_static_content.convert_legacy_static_url.return_value + '"',
+        replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY, NAMESPACE)
+    )
+
+    mock_static_content.convert_legacy_static_url.assert_called_once_with('file.png', NAMESPACE)
+
+
+@patch('static_replace.settings')
+@patch('static_replace.modulestore')
+@patch('static_replace.staticfiles_storage')
+def test_data_dir_fallback(mock_storage, mock_modulestore, mock_settings):
+    mock_modulestore.return_value = Mock(XMLModuleStore)
+    mock_storage.url.side_effect = Exception
+
+    mock_storage.exists.return_value = True
+    assert_equals('"/static/data_dir/file.png"', replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY))
+
+    mock_storage.exists.return_value = False
+    assert_equals('"/static/data_dir/file.png"', replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY))
+
+
+def test_raw_static_check():
+    """
+    Make sure replace_static_urls leaves alone things that end in '.raw'
+    """
+    path = '"/static/foo.png?raw"'
+    assert_equals(path, replace_static_urls(path, DATA_DIRECTORY))
+
+    text = 'text <tag a="/static/js/capa/protex/protex.nocache.js?raw"/><div class="'
+    assert_equals(path, replace_static_urls(path, text))
+
+
+def test_regex():
+    yes = ('"/static/foo.png"',
+           '"/static/foo.png"',
+           "'/static/foo.png'")
+
+    no = ('"/not-static/foo.png"',
+          '"/static/foo',  # no matching quote
+          )
+
+    regex = _url_replace_regex('/static/')
+
+    for s in yes:
+        print 'Should match: {0!r}'.format(s)
+        assert_true(re.match(regex, s))
+
+    for s in no:
+        print 'Should not match: {0!r}'.format(s)
+        assert_false(re.match(regex, s))

common/djangoapps/status/__init__.py

@@ -1 +0,0 @@
-

common/djangoapps/status/status.py

@@ -10,6 +10,7 @@ import sys
 
 log = logging.getLogger(__name__)
 
+
 def get_site_status_msg(course_id):
     """
     Look for a file settings.STATUS_MESSAGE_PATH.  If found, read it,

common/djangoapps/status/tests.py

@@ -1,10 +1,10 @@
 from django.conf import settings
 from django.test import TestCase
 import os
-from override_settings import override_settings
+from django.test.utils import override_settings
 from tempfile import NamedTemporaryFile
 
-from status import get_site_status_msg
+from .status import get_site_status_msg
 
 # Get a name where we can put test files
 TMP_FILE = NamedTemporaryFile(delete=False)

common/djangoapps/student/admin.py

@@ -12,6 +12,8 @@ admin.site.register(UserTestGroup)
 
 admin.site.register(CourseEnrollment)
 
+admin.site.register(CourseEnrollmentAllowed)
+
 admin.site.register(Registration)
 
 admin.site.register(PendingNameChange)

common/djangoapps/student/management/commands/6002exportusers.py

@@ -57,7 +57,7 @@ from student.userprofile. '''
                 d[key] = item
             return d
 
-        extracted = [{'up':extract_dict(up_keys, t[0]), 'u':extract_dict(user_keys, t[1])} for t in user_tuples]
+        extracted = [{'up': extract_dict(up_keys, t[0]), 'u':extract_dict(user_keys, t[1])} for t in user_tuples]
         fp = open('transfer_users.txt', 'w')
         json.dump(extracted, fp)
         fp.close()

common/djangoapps/student/management/commands/add_to_group.py

@@ -3,6 +3,7 @@ from optparse import make_option
 from django.core.management.base import BaseCommand, CommandError
 from django.contrib.auth.models import User, Group
 
+
 class Command(BaseCommand):
     option_list = BaseCommand.option_list + (
         make_option('--list',

common/djangoapps/student/management/commands/cert_restriction.py

@@ -0,0 +1,109 @@
+from django.core.management.base import BaseCommand, CommandError
+import os
+from optparse import make_option
+from student.models import UserProfile
+import csv
+
+
+class Command(BaseCommand):
+
+    help = """
+    Sets or gets certificate restrictions for users
+    from embargoed countries. (allow_certificate in
+    userprofile)
+
+    CSV should be comma delimited with double quoted entries.
+
+        $ ... cert_restriction --import path/to/userlist.csv
+
+    Export a list of students who have "allow_certificate" in
+    userprofile set to True
+
+        $ ... cert_restriction --output path/to/export.csv
+
+    Enable a single user so she is not on the restricted list
+
+        $ ... cert_restriction -e user
+
+    Disable a single user so she is on the restricted list
+
+        $ ... cert_restriction -d user
+
+    """
+
+    option_list = BaseCommand.option_list + (
+        make_option('-i', '--import',
+                    metavar='IMPORT_FILE',
+                    dest='import',
+                    default=False,
+                    help='csv file to import, comma delimitted file with '
+                         'double-quoted entries'),
+        make_option('-o', '--output',
+                    metavar='EXPORT_FILE',
+                    dest='output',
+                    default=False,
+                    help='csv file to export'),
+        make_option('-e', '--enable',
+                    metavar='STUDENT',
+                    dest='enable',
+                    default=False,
+                    help="enable a single student's certificate"),
+        make_option('-d', '--disable',
+                    metavar='STUDENT',
+                    dest='disable',
+                    default=False,
+                    help="disable a single student's certificate")
+    )
+
+    def handle(self, *args, **options):
+        if options['output']:
+
+            if os.path.exists(options['output']):
+                raise CommandError("File {0} already exists".format(
+                    options['output']))
+            disabled_users = UserProfile.objects.filter(
+                allow_certificate=False)
+
+            with open(options['output'], 'w') as csvfile:
+                csvwriter = csv.writer(csvfile, delimiter=',', quotechar='"',
+                                       quoting=csv.QUOTE_MINIMAL)
+                for user in disabled_users:
+                    csvwriter.writerow([user.user.username])
+
+        elif options['import']:
+
+            if not os.path.exists(options['import']):
+                raise CommandError("File {0} does not exist".format(
+                    options['import']))
+
+            print "Importing students from {0}".format(options['import'])
+
+            students = None
+            with open(options['import']) as csvfile:
+                student_list = csv.reader(csvfile, delimiter=',',
+                                          quotechar='"')
+                students = [student[0] for student in student_list]
+            if not students:
+                raise CommandError(
+                    "Unable to read student data from {0}".format(
+                        options['import']))
+            UserProfile.objects.filter(user__username__in=students).update(
+                allow_certificate=False)
+
+        elif options['enable']:
+
+            print "Enabling {0} for certificate download".format(
+                options['enable'])
+            cert_allow = UserProfile.objects.get(
+                user__username=options['enable'])
+            cert_allow.allow_certificate = True
+            cert_allow.save()
+
+        elif options['disable']:
+
+            print "Disabling {0} for certificate download".format(
+                options['disable'])
+            cert_allow = UserProfile.objects.get(
+                user__username=options['disable'])
+            cert_allow.allow_certificate = False
+            cert_allow.save()

common/djangoapps/student/management/commands/create_random_users.py

@@ -8,6 +8,7 @@ from student.models import UserProfile, CourseEnrollment
 
 from student.views import _do_create_account, get_random_post_override
 
+
 def create(n, course_id):
     """Create n users, enrolling them in course_id if it's not None"""
     for i in range(n):
@@ -15,6 +16,7 @@ def create(n, course_id):
         if course_id is not None:
             CourseEnrollment.objects.create(user=user, course_id=course_id)
 
+
 class Command(BaseCommand):
     help = """Create N new users, with random parameters.
 

common/djangoapps/student/management/commands/pearson_dump.py

@@ -0,0 +1,77 @@
+from optparse import make_option
+from json import dump
+from datetime import datetime
+
+from django.core.management.base import BaseCommand, CommandError
+
+from student.models import TestCenterRegistration
+
+
+class Command(BaseCommand):
+
+    args = '<output JSON file>'
+    help = """
+    Dump information as JSON from TestCenterRegistration tables, including username and status.
+    """
+
+    option_list = BaseCommand.option_list + (
+        make_option('--course_id',
+                    action='store',
+                    dest='course_id',
+                    help='Specify a particular course.'),
+        make_option('--exam_series_code',
+                    action='store',
+                    dest='exam_series_code',
+                    default=None,
+                    help='Specify a particular exam, using the Pearson code'),
+        make_option('--accommodation_pending',
+                    action='store_true',
+                    dest='accommodation_pending',
+                    default=False,
+                    ),
+    )
+
+    def handle(self, *args, **options):
+        if len(args) < 1:
+            outputfile = datetime.utcnow().strftime("pearson-dump-%Y%m%d-%H%M%S.json")
+        else:
+            outputfile = args[0]
+
+        # construct the query object to dump:
+        registrations = TestCenterRegistration.objects.all()
+        if 'course_id' in options and options['course_id']:
+            registrations = registrations.filter(course_id=options['course_id'])
+        if 'exam_series_code' in options and options['exam_series_code']:
+            registrations = registrations.filter(exam_series_code=options['exam_series_code'])
+
+        # collect output:
+        output = []
+        for registration in registrations:
+            if 'accommodation_pending' in options and options['accommodation_pending'] and not registration.accommodation_is_pending:
+                continue
+            record = {'username': registration.testcenter_user.user.username,
+                      'email': registration.testcenter_user.email,
+                      'first_name': registration.testcenter_user.first_name,
+                      'last_name': registration.testcenter_user.last_name,
+                      'client_candidate_id': registration.client_candidate_id,
+                      'client_authorization_id': registration.client_authorization_id,
+                      'course_id': registration.course_id,
+                      'exam_series_code': registration.exam_series_code,
+                      'accommodation_request': registration.accommodation_request,
+                      'accommodation_code': registration.accommodation_code,
+                      'registration_status': registration.registration_status(),
+                      'demographics_status': registration.demographics_status(),
+                      'accommodation_status': registration.accommodation_status(),
+                      }
+            if len(registration.upload_error_message) > 0:
+                record['registration_error'] = registration.upload_error_message
+            if len(registration.testcenter_user.upload_error_message) > 0:
+                record['demographics_error'] = registration.testcenter_user.upload_error_message
+            if registration.needs_uploading:
+                record['needs_uploading'] = True
+
+            output.append(record)
+
+        # dump output:
+        with open(outputfile, 'w') as outfile:
+            dump(output, outfile, indent=2)

common/djangoapps/student/management/commands/pearson_export_cdd.py

@@ -1,14 +1,19 @@
 import csv
-import uuid
-from collections import defaultdict, OrderedDict
+import os
+from collections import OrderedDict
 from datetime import datetime
+from optparse import make_option
 
+from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
 
 from student.models import TestCenterUser
 
+
 class Command(BaseCommand):
+
     CSV_TO_MODEL_FIELDS = OrderedDict([
+        # Skipping optional field CandidateID
         ("ClientCandidateID", "client_candidate_id"),
         ("FirstName", "first_name"),
         ("LastName", "last_name"),
@@ -31,22 +36,63 @@ class Command(BaseCommand):
         ("FAXCountryCode", "fax_country_code"),
         ("CompanyName", "company_name"),
         # Skipping optional field CustomQuestion
-        ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
+        ("LastUpdate", "user_updated_at"),  # in UTC, so same as what we store
     ])
 
-    args = '<output_file>'
-    help = """
-    Export user information from TestCenterUser model into a tab delimited
-    text file with a format that Pearson expects.
-    """
-    def handle(self, *args, **kwargs):
-        if len(args) < 1:
-            print Command.help
-            return
+    # define defaults, even thought 'store_true' shouldn't need them.
+    # (call_command will set None as default value for all options that don't have one,
+    # so one cannot rely on presence/absence of flags in that world.)
+    option_list = BaseCommand.option_list + (
+        make_option('--dest-from-settings',
+                    action='store_true',
+                    dest='dest-from-settings',
+                    default=False,
+                    help='Retrieve the destination to export to from django.'),
+        make_option('--destination',
+                    action='store',
+                    dest='destination',
+                    default=None,
+                    help='Where to store the exported files')
+    )
 
-        self.reset_sample_data()
+    def handle(self, **options):
+        # update time should use UTC in order to be comparable to the user_updated_at
+        # field
+        uploaded_at = datetime.utcnow()
 
-        with open(args[0], "wb") as outfile:
+        # if specified destination is an existing directory, then
+        # create a filename for it automatically.  If it doesn't exist,
+        # then we will create the directory.
+        # Name will use timestamp -- this is UTC, so it will look funny,
+        # but it should at least be consistent with the other timestamps
+        # used in the system.
+        if 'dest-from-settings' in options and options['dest-from-settings']:
+            if 'LOCAL_EXPORT' in settings.PEARSON:
+                dest = settings.PEARSON['LOCAL_EXPORT']
+            else:
+                raise CommandError('--dest-from-settings was enabled but the'
+                                   'PEARSON[LOCAL_EXPORT] setting was not set.')
+        elif 'destination' in options and options['destination']:
+            dest = options['destination']
+        else:
+            raise CommandError('--destination or --dest-from-settings must be used')
+
+        if not os.path.isdir(dest):
+            os.makedirs(dest)
+
+        destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
+
+        # strings must be in latin-1 format.  CSV parser will
+        # otherwise convert unicode objects to ascii.
+        def ensure_encoding(value):
+            if isinstance(value, unicode):
+                return value.encode('iso-8859-1')
+            else:
+                return value
+
+#        dump_all = options['dump_all']
+
+        with open(destfile, "wb") as outfile:
             writer = csv.DictWriter(outfile,
                                     Command.CSV_TO_MODEL_FIELDS,
                                     delimiter="\t",
@@ -54,103 +100,11 @@ class Command(BaseCommand):
                                     extrasaction='ignore')
             writer.writeheader()
             for tcu in TestCenterUser.objects.order_by('id'):
-                record = dict((csv_field, getattr(tcu, model_field))
-                              for csv_field, model_field
-                              in Command.CSV_TO_MODEL_FIELDS.items())
-                record["LastUpdate"] = record["LastUpdate"].strftime("%Y/%m/%d %H:%M:%S")
-                writer.writerow(record)
-
-    def reset_sample_data(self):
-        def make_sample(**kwargs):
-            data = dict((model_field, kwargs.get(model_field, ""))
-                        for model_field in Command.CSV_TO_MODEL_FIELDS.values())
-            return TestCenterUser(**data)
-        
-        def generate_id():
-            return "edX{:012}".format(uuid.uuid4().int % (10**12))
-        
-        # TestCenterUser.objects.all().delete()
-        
-        samples = [
-            make_sample(
-                client_candidate_id=generate_id(),
-                first_name="Jack",
-                last_name="Doe",
-                middle_name="C",
-                address_1="11 Cambridge Center",
-                address_2="Suite 101",
-                city="Cambridge",
-                state="MA",
-                postal_code="02140",
-                country="USA",
-                phone="(617)555-5555",
-                phone_country_code="1",
-                user_updated_at=datetime.utcnow()
-            ),
-            make_sample(
-                client_candidate_id=generate_id(),
-                first_name="Clyde",
-                last_name="Smith",
-                middle_name="J",
-                suffix="Jr.",
-                salutation="Mr.",
-                address_1="1 Penny Lane",
-                city="Honolulu",
-                state="HI",
-                postal_code="96792",
-                country="USA",
-                phone="555-555-5555",
-                phone_country_code="1",
-                user_updated_at=datetime.utcnow()
-            ),
-            make_sample(
-                client_candidate_id=generate_id(),
-                first_name="Patty",
-                last_name="Lee",
-                salutation="Dr.",
-                address_1="P.O. Box 555",
-                city="Honolulu",
-                state="HI",
-                postal_code="96792",
-                country="USA",
-                phone="808-555-5555",
-                phone_country_code="1",
-                user_updated_at=datetime.utcnow()
-            ),
-            make_sample(
-                client_candidate_id=generate_id(),
-                first_name="Jimmy",
-                last_name="James",
-                address_1="2020 Palmer Blvd.",
-                city="Springfield",
-                state="MA",
-                postal_code="96792",
-                country="USA",
-                phone="917-555-5555",
-                phone_country_code="1",
-                extension="2039",
-                fax="917-555-5556",
-                fax_country_code="1",
-                company_name="ACME Traps",
-                user_updated_at=datetime.utcnow()
-            ),
-            make_sample(
-                client_candidate_id=generate_id(),
-                first_name="Yeong-Un",
-                last_name="Seo",
-                address_1="Duryu, Lotte 101",
-                address_2="Apt 55",
-                city="Daegu",
-                country="KOR",
-                phone="917-555-5555",
-                phone_country_code="011",
-                user_updated_at=datetime.utcnow()
-            ),
-
-        ]
-        
-        for tcu in samples:
-            tcu.save()
-        
-
-        
+                if tcu.needs_uploading:   # or dump_all
+                    record = dict((csv_field, ensure_encoding(getattr(tcu, model_field)))
+                                  for csv_field, model_field
+                                  in Command.CSV_TO_MODEL_FIELDS.items())
+                    record["LastUpdate"] = record["LastUpdate"].strftime("%Y/%m/%d %H:%M:%S")
+                    writer.writerow(record)
+                    tcu.uploaded_at = uploaded_at
+                    tcu.save()

common/djangoapps/student/management/commands/pearson_export_ead.py

@@ -1,150 +1,102 @@
 import csv
-import uuid
-from collections import defaultdict, OrderedDict
+import os
+from collections import OrderedDict
 from datetime import datetime
+from optparse import make_option
 
+from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
 
-from student.models import TestCenterUser
+from student.models import TestCenterRegistration, ACCOMMODATION_REJECTED_CODE
 
-def generate_id():
-    return "{:012}".format(uuid.uuid4().int % (10**12))
 
 class Command(BaseCommand):
-    args = '<output_file>'
-    help = """
-    Export user information from TestCenterUser model into a tab delimited
-    text file with a format that Pearson expects.
-    """
-    FIELDS = [
-        'AuthorizationTransactionType',
-        'AuthorizationID',
-        'ClientAuthorizationID',
-        'ClientCandidateID',
-        'ExamAuthorizationCount',
-        'ExamSeriesCode',
-        'EligibilityApptDateFirst',
-        'EligibilityApptDateLast',
-        'LastUpdate',
-    ]
-    
-    def handle(self, *args, **kwargs):
-        if len(args) < 1:
-            print Command.help
-            return
 
-        # self.reset_sample_data()
+    CSV_TO_MODEL_FIELDS = OrderedDict([
+        ('AuthorizationTransactionType', 'authorization_transaction_type'),
+        ('AuthorizationID', 'authorization_id'),
+        ('ClientAuthorizationID', 'client_authorization_id'),
+        ('ClientCandidateID', 'client_candidate_id'),
+        ('ExamAuthorizationCount', 'exam_authorization_count'),
+        ('ExamSeriesCode', 'exam_series_code'),
+        ('Accommodations', 'accommodation_code'),
+        ('EligibilityApptDateFirst', 'eligibility_appointment_date_first'),
+        ('EligibilityApptDateLast', 'eligibility_appointment_date_last'),
+        ("LastUpdate", "user_updated_at"),  # in UTC, so same as what we store
+    ])
 
-        with open(args[0], "wb") as outfile:
+    option_list = BaseCommand.option_list + (
+        make_option('--dest-from-settings',
+                    action='store_true',
+                    dest='dest-from-settings',
+                    default=False,
+                    help='Retrieve the destination to export to from django.'),
+        make_option('--destination',
+                    action='store',
+                    dest='destination',
+                    default=None,
+                    help='Where to store the exported files'),
+        make_option('--dump_all',
+                    action='store_true',
+                    dest='dump_all',
+                    default=False,
+                    ),
+        make_option('--force_add',
+                    action='store_true',
+                    dest='force_add',
+                    default=False,
+                    ),
+    )
+
+    def handle(self, **options):
+        # update time should use UTC in order to be comparable to the user_updated_at
+        # field
+        uploaded_at = datetime.utcnow()
+
+        # if specified destination is an existing directory, then
+        # create a filename for it automatically.  If it doesn't exist,
+        # then we will create the directory.
+        # Name will use timestamp -- this is UTC, so it will look funny,
+        # but it should at least be consistent with the other timestamps
+        # used in the system.
+        if 'dest-from-settings' in options and options['dest-from-settings']:
+            if 'LOCAL_EXPORT' in settings.PEARSON:
+                dest = settings.PEARSON['LOCAL_EXPORT']
+            else:
+                raise CommandError('--dest-from-settings was enabled but the'
+                                   'PEARSON[LOCAL_EXPORT] setting was not set.')
+        elif 'destination' in options and options['destination']:
+            dest = options['destination']
+        else:
+            raise CommandError('--destination or --dest-from-settings must be used')
+
+        if not os.path.isdir(dest):
+            os.makedirs(dest)
+
+        destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
+
+        dump_all = options['dump_all']
+
+        with open(destfile, "wb") as outfile:
             writer = csv.DictWriter(outfile,
-                                    Command.FIELDS,
+                                    Command.CSV_TO_MODEL_FIELDS,
                                     delimiter="\t",
                                     quoting=csv.QUOTE_MINIMAL,
                                     extrasaction='ignore')
             writer.writeheader()
-            for tcu in TestCenterUser.objects.order_by('id')[:5]:
-                record = defaultdict(
-                    lambda: "",
-                    AuthorizationTransactionType="Add",
-                    ClientAuthorizationID=generate_id(),
-                    ClientCandidateID=tcu.client_candidate_id,
-                    ExamAuthorizationCount="1",
-                    ExamSeriesCode="6002x001",
-                    EligibilityApptDateFirst="2012/12/15",
-                    EligibilityApptDateLast="2012/12/30",
-                    LastUpdate=datetime.utcnow().strftime("%Y/%m/%d %H:%M:%S")
-                )
-                writer.writerow(record)
+            for tcr in TestCenterRegistration.objects.order_by('id'):
+                if dump_all or tcr.needs_uploading:
+                    record = dict((csv_field, getattr(tcr, model_field))
+                                  for csv_field, model_field
+                                  in Command.CSV_TO_MODEL_FIELDS.items())
+                    record["LastUpdate"] = record["LastUpdate"].strftime("%Y/%m/%d %H:%M:%S")
+                    record["EligibilityApptDateFirst"] = record["EligibilityApptDateFirst"].strftime("%Y/%m/%d")
+                    record["EligibilityApptDateLast"] = record["EligibilityApptDateLast"].strftime("%Y/%m/%d")
+                    if record["Accommodations"] == ACCOMMODATION_REJECTED_CODE:
+                        record["Accommodations"] = ""
+                    if options['force_add']:
+                        record['AuthorizationTransactionType'] = 'Add'
 
-
-    def reset_sample_data(self):
-        def make_sample(**kwargs):
-            data = dict((model_field, kwargs.get(model_field, ""))
-                        for model_field in Command.CSV_TO_MODEL_FIELDS.values())
-            return TestCenterUser(**data)
-        
-        # TestCenterUser.objects.all().delete()
-        
-        samples = [
-            make_sample(
-                client_candidate_id=generate_id(),
-                first_name="Jack",
-                last_name="Doe",
-                middle_name="C",
-                address_1="11 Cambridge Center",
-                address_2="Suite 101",
-                city="Cambridge",
-                state="MA",
-                postal_code="02140",
-                country="USA",
-                phone="(617)555-5555",
-                phone_country_code="1",
-                user_updated_at=datetime.utcnow()
-            ),
-            make_sample(
-                client_candidate_id=generate_id(),
-                first_name="Clyde",
-                last_name="Smith",
-                middle_name="J",
-                suffix="Jr.",
-                salutation="Mr.",
-                address_1="1 Penny Lane",
-                city="Honolulu",
-                state="HI",
-                postal_code="96792",
-                country="USA",
-                phone="555-555-5555",
-                phone_country_code="1",
-                user_updated_at=datetime.utcnow()
-            ),
-            make_sample(
-                client_candidate_id=generate_id(),
-                first_name="Patty",
-                last_name="Lee",
-                salutation="Dr.",
-                address_1="P.O. Box 555",
-                city="Honolulu",
-                state="HI",
-                postal_code="96792",
-                country="USA",
-                phone="808-555-5555",
-                phone_country_code="1",
-                user_updated_at=datetime.utcnow()
-            ),
-            make_sample(
-                client_candidate_id=generate_id(),
-                first_name="Jimmy",
-                last_name="James",
-                address_1="2020 Palmer Blvd.",
-                city="Springfield",
-                state="MA",
-                postal_code="96792",
-                country="USA",
-                phone="917-555-5555",
-                phone_country_code="1",
-                extension="2039",
-                fax="917-555-5556",
-                fax_country_code="1",
-                company_name="ACME Traps",
-                user_updated_at=datetime.utcnow()
-            ),
-            make_sample(
-                client_candidate_id=generate_id(),
-                first_name="Yeong-Un",
-                last_name="Seo",
-                address_1="Duryu, Lotte 101",
-                address_2="Apt 55",
-                city="Daegu",
-                country="KOR",
-                phone="917-555-5555",
-                phone_country_code="011",
-                user_updated_at=datetime.utcnow()
-            ),
-
-        ]
-        
-        for tcu in samples:
-            tcu.save()
-        
-
-        
+                    writer.writerow(record)
+                    tcr.uploaded_at = uploaded_at
+                    tcr.save()

common/djangoapps/student/management/commands/pearson_import_conf_zip.py

@@ -0,0 +1,118 @@
+import csv
+
+from zipfile import ZipFile, is_zipfile
+from time import strptime, strftime
+
+from collections import OrderedDict
+from datetime import datetime
+from os.path import isdir
+from optparse import make_option
+from dogapi import dog_http_api, dog_stats_api
+
+from django.core.management.base import BaseCommand, CommandError
+from django.conf import settings
+
+from student.models import TestCenterUser, TestCenterRegistration
+
+
+class Command(BaseCommand):
+
+    dog_http_api.api_key = settings.DATADOG_API
+    args = '<input zip file>'
+    help = """
+    Import Pearson confirmation files and update TestCenterUser
+    and TestCenterRegistration tables with status.
+    """
+
+    @staticmethod
+    def datadog_error(string, tags):
+        dog_http_api.event("Pearson Import", string, alert_type='error', tags=[tags])
+
+    def handle(self, *args, **kwargs):
+        if len(args) < 1:
+            print Command.help
+            return
+
+        source_zip = args[0]
+        if not is_zipfile(source_zip):
+            error = "Input file is not a zipfile: \"{}\"".format(source_zip)
+            Command.datadog_error(error, source_zip)
+            raise CommandError(error)
+
+        # loop through all files in zip, and process them based on filename prefix:
+        with ZipFile(source_zip, 'r') as zipfile:
+            for fileinfo in zipfile.infolist():
+                with zipfile.open(fileinfo) as zipentry:
+                    if fileinfo.filename.startswith("eac-"):
+                        self.process_eac(zipentry)
+                    elif fileinfo.filename.startswith("vcdc-"):
+                        self.process_vcdc(zipentry)
+                    else:
+                        error = "Unrecognized confirmation file type\"{}\" in confirmation zip file \"{}\"".format(fileinfo.filename, zipfile)
+                        Command.datadog_error(error, source_zip)
+                        raise CommandError(error)
+
+    def process_eac(self, eacfile):
+        print "processing eac"
+        reader = csv.DictReader(eacfile, delimiter="\t")
+        for row in reader:
+            client_authorization_id = row['ClientAuthorizationID']
+            if not client_authorization_id:
+                if row['Status'] == 'Error':
+                    Command.datadog_error("Error in EAD file processing ({}): {}".format(row['Date'], row['Message']), eacfile.name)
+                else:
+                    Command.datadog_error("Encountered bad record: {}".format(row), eacfile.name)
+            else:
+                try:
+                    registration = TestCenterRegistration.objects.get(client_authorization_id=client_authorization_id)
+                    Command.datadog_error("Found authorization record for user {}".format(registration.testcenter_user.user.username), eacfile.name)
+                    # now update the record:
+                    registration.upload_status = row['Status']
+                    registration.upload_error_message =  row['Message']
+                    try:
+                        registration.processed_at = strftime('%Y-%m-%d %H:%M:%S', strptime(row['Date'], '%Y/%m/%d %H:%M:%S'))
+                    except ValueError as ve:
+                        Command.datadog_error("Bad Date value found for {}: message {}".format(client_authorization_id, ve), eacfile.name)
+                    # store the authorization Id if one is provided.  (For debugging)
+                    if row['AuthorizationID']:
+                        try:
+                            registration.authorization_id = int(row['AuthorizationID'])
+                        except ValueError as ve:
+                            Command.datadog_error("Bad AuthorizationID value found for {}: message {}".format(client_authorization_id, ve), eacfile.name)
+
+                    registration.confirmed_at = datetime.utcnow()
+                    registration.save()
+                except TestCenterRegistration.DoesNotExist:
+                    Command.datadog_error("Failed to find record for client_auth_id {}".format(client_authorization_id), eacfile.name)
+
+    def process_vcdc(self, vcdcfile):
+        print "processing vcdc"
+        reader = csv.DictReader(vcdcfile, delimiter="\t")
+        for row in reader:
+            client_candidate_id = row['ClientCandidateID']
+            if not client_candidate_id:
+                if row['Status'] == 'Error':
+                    Command.datadog_error("Error in CDD file processing ({}): {}".format(row['Date'], row['Message']), vcdcfile.name)
+                else:
+                    Command.datadog_error("Encountered bad record: {}".format(row), vcdcfile.name)
+            else:
+                try:
+                    tcuser = TestCenterUser.objects.get(client_candidate_id=client_candidate_id)
+                    Command.datadog_error("Found demographics record for user {}".format(tcuser.user.username), vcdcfile.name)
+                    # now update the record:
+                    tcuser.upload_status = row['Status']
+                    tcuser.upload_error_message = row['Message']
+                    try:
+                        tcuser.processed_at = strftime('%Y-%m-%d %H:%M:%S', strptime(row['Date'], '%Y/%m/%d %H:%M:%S'))
+                    except ValueError as ve:
+                        Command.datadog_error("Bad Date value found for {}: message {}".format(client_candidate_id, ve), vcdcfile.name)
+                    # store the candidate Id if one is provided.  (For debugging)
+                    if row['CandidateID']:
+                        try:
+                            tcuser.candidate_id = int(row['CandidateID'])
+                        except ValueError as ve:
+                            Command.datadog_error("Bad CandidateID value found for {}: message {}".format(client_candidate_id, ve), vcdcfile.name)
+                    tcuser.confirmed_at = datetime.utcnow()
+                    tcuser.save()
+                except TestCenterUser.DoesNotExist:
+                    Command.datadog_error(" Failed to find record for client_candidate_id {}".format(client_candidate_id), vcdcfile.name)

common/djangoapps/student/management/commands/pearson_make_tc_registration.py

@@ -0,0 +1,207 @@
+from optparse import make_option
+from time import strftime
+
+from django.contrib.auth.models import User
+from django.core.management.base import BaseCommand, CommandError
+
+from student.models import TestCenterUser, TestCenterRegistration, TestCenterRegistrationForm, get_testcenter_registration
+from student.views import course_from_id
+from xmodule.course_module import CourseDescriptor
+from xmodule.modulestore.exceptions import ItemNotFoundError
+
+
+class Command(BaseCommand):
+    option_list = BaseCommand.option_list + (
+        # registration info:
+        make_option(
+            '--accommodation_request',
+            action='store',
+            dest='accommodation_request',
+        ),
+        make_option(
+            '--accommodation_code',
+            action='store',
+            dest='accommodation_code',
+        ),
+        make_option(
+            '--client_authorization_id',
+            action='store',
+            dest='client_authorization_id',
+        ),
+        # exam info:
+        make_option(
+            '--exam_series_code',
+            action='store',
+            dest='exam_series_code',
+        ),
+        make_option(
+            '--eligibility_appointment_date_first',
+            action='store',
+            dest='eligibility_appointment_date_first',
+            help='use YYYY-MM-DD format if overriding existing course values, or YYYY-MM-DDTHH:MM if not using an existing course.'
+        ),
+        make_option(
+            '--eligibility_appointment_date_last',
+            action='store',
+            dest='eligibility_appointment_date_last',
+            help='use YYYY-MM-DD format if overriding existing course values, or YYYY-MM-DDTHH:MM if not using an existing course.'
+        ),
+        # internal values:
+        make_option(
+            '--authorization_id',
+            action='store',
+            dest='authorization_id',
+            help='ID we receive from Pearson for a particular authorization'
+        ),
+        make_option(
+            '--upload_status',
+            action='store',
+            dest='upload_status',
+            help='status value assigned by Pearson'
+        ),
+        make_option(
+            '--upload_error_message',
+            action='store',
+            dest='upload_error_message',
+            help='error message provided by Pearson on a failure.'
+        ),
+        # control values:
+        make_option(
+            '--ignore_registration_dates',
+            action='store_true',
+            dest='ignore_registration_dates',
+            help='find exam info for course based on exam_series_code, even if the exam is not active.'
+        ),
+        make_option(
+            '--create_dummy_exam',
+            action='store_true',
+            dest='create_dummy_exam',
+            help='create dummy exam info for course, even if course exists'
+        ),
+    )
+    args = "<student_username course_id>"
+    help = "Create or modify a TestCenterRegistration entry for a given Student"
+
+    @staticmethod
+    def is_valid_option(option_name):
+        base_options = set(option.dest for option in BaseCommand.option_list)
+        return option_name not in base_options
+
+
+    def handle(self, *args, **options):
+        username = args[0]
+        course_id = args[1]
+        print username, course_id
+
+        our_options = dict((k, v) for k, v in options.items()
+                           if Command.is_valid_option(k) and v is not None)
+        try:
+            student = User.objects.get(username=username)
+        except User.DoesNotExist:
+            raise CommandError("User \"{}\" does not exist".format(username))
+
+        try:
+            testcenter_user = TestCenterUser.objects.get(user=student)
+        except TestCenterUser.DoesNotExist:
+            raise CommandError("User \"{}\" does not have an existing demographics record".format(username))
+
+        # get an "exam" object.  Check to see if a course_id was specified, and use information from that:
+        exam = None
+        create_dummy_exam = 'create_dummy_exam' in our_options and our_options['create_dummy_exam']
+        if not create_dummy_exam:
+            try:
+                course = course_from_id(course_id)
+                if 'ignore_registration_dates' in our_options:
+                    examlist = [exam for exam in course.test_center_exams if exam.exam_series_code == our_options.get('exam_series_code')]
+                    exam = examlist[0] if len(examlist) > 0 else None
+                else:
+                    exam = course.current_test_center_exam
+            except ItemNotFoundError:
+                pass
+        else:
+            # otherwise use explicit values (so we don't have to define a course):
+            exam_name = "Dummy Placeholder Name"
+            exam_info = {'Exam_Series_Code': our_options['exam_series_code'],
+                          'First_Eligible_Appointment_Date': our_options['eligibility_appointment_date_first'],
+                          'Last_Eligible_Appointment_Date': our_options['eligibility_appointment_date_last'],
+                          }
+            exam = CourseDescriptor.TestCenterExam(course_id, exam_name, exam_info)
+            # update option values for date_first and date_last to use YYYY-MM-DD format
+            # instead of YYYY-MM-DDTHH:MM
+            our_options['eligibility_appointment_date_first'] = strftime("%Y-%m-%d", exam.first_eligible_appointment_date)
+            our_options['eligibility_appointment_date_last'] = strftime("%Y-%m-%d", exam.last_eligible_appointment_date)
+
+        if exam is None:
+            raise CommandError("Exam for course_id {} does not exist".format(course_id))
+
+        exam_code = exam.exam_series_code
+
+        UPDATE_FIELDS = ('accommodation_request',
+                          'accommodation_code',
+                          'client_authorization_id',
+                          'exam_series_code',
+                          'eligibility_appointment_date_first',
+                          'eligibility_appointment_date_last',
+                          )
+
+        # create and save the registration:
+        needs_updating = False
+        registrations = get_testcenter_registration(student, course_id, exam_code)
+        if len(registrations) > 0:
+            registration = registrations[0]
+            for fieldname in UPDATE_FIELDS:
+                if fieldname in our_options and registration.__getattribute__(fieldname) != our_options[fieldname]:
+                    needs_updating = True;
+        else:
+            accommodation_request = our_options.get('accommodation_request', '')
+            registration = TestCenterRegistration.create(testcenter_user, exam, accommodation_request)
+            needs_updating = True
+
+
+        if needs_updating:
+            # first update the record with the new values, if any:
+            for fieldname in UPDATE_FIELDS:
+                if fieldname in our_options and fieldname not in TestCenterRegistrationForm.Meta.fields:
+                    registration.__setattr__(fieldname, our_options[fieldname])
+
+            # the registration form normally populates the data dict with
+            # the accommodation request (if any).  But here we want to
+            # specify only those values that might change, so update the dict with existing
+            # values.
+            form_options = dict(our_options)
+            for propname in TestCenterRegistrationForm.Meta.fields:
+                if propname not in form_options:
+                    form_options[propname] = registration.__getattribute__(propname)
+            form = TestCenterRegistrationForm(instance=registration, data=form_options)
+            if form.is_valid():
+                form.update_and_save()
+                print "Updated registration information for user's registration: username \"{}\" course \"{}\", examcode \"{}\"".format(student.username, course_id, exam_code)
+            else:
+                if (len(form.errors) > 0):
+                    print "Field Form errors encountered:"
+                for fielderror in form.errors:
+                    for msg in form.errors[fielderror]:
+                        print "Field Form Error:  {} -- {}".format(fielderror, msg)
+                    if (len(form.non_field_errors()) > 0):
+                        print "Non-field Form errors encountered:"
+                        for nonfielderror in form.non_field_errors:
+                            print "Non-field Form Error:  %s" % nonfielderror
+
+        else:
+            print "No changes necessary to make to existing user's registration."
+
+        # override internal values:
+        change_internal = False
+        if 'exam_series_code' in our_options:
+            exam_code = our_options['exam_series_code']
+        registration = get_testcenter_registration(student, course_id, exam_code)[0]
+        for internal_field in ['upload_error_message', 'upload_status', 'authorization_id']:
+            if internal_field in our_options:
+                registration.__setattr__(internal_field, our_options[internal_field])
+                change_internal = True
+
+        if change_internal:
+            print "Updated  confirmation information in existing user's registration."
+            registration.save()
+        else:
+            print "No changes necessary to make to confirmation information in existing user's registration."

common/djangoapps/student/management/commands/pearson_make_tc_user.py

@@ -1,46 +1,65 @@
-import uuid
-from datetime import datetime
 from optparse import make_option
 
 from django.contrib.auth.models import User
 from django.core.management.base import BaseCommand, CommandError
 
-from student.models import TestCenterUser
+from student.models import TestCenterUser, TestCenterUserForm
+
 
 class Command(BaseCommand):
     option_list = BaseCommand.option_list + (
-        make_option(
-            '--client_candidate_id',
-            action='store',
-            dest='client_candidate_id',
-            help='ID we assign a user to identify them to Pearson'
-        ),        
+        # demographics:
         make_option(
             '--first_name',
             action='store',
             dest='first_name',
-        ),        
+        ),
+        make_option(
+            '--middle_name',
+            action='store',
+            dest='middle_name',
+        ),
         make_option(
             '--last_name',
             action='store',
             dest='last_name',
-        ),        
+        ),
+        make_option(
+            '--suffix',
+            action='store',
+            dest='suffix',
+        ),
+        make_option(
+            '--salutation',
+            action='store',
+            dest='salutation',
+        ),
         make_option(
             '--address_1',
             action='store',
             dest='address_1',
-        ),        
+        ),
+        make_option(
+            '--address_2',
+            action='store',
+            dest='address_2',
+        ),
+        make_option(
+            '--address_3',
+            action='store',
+            dest='address_3',
+        ),
         make_option(
             '--city',
             action='store',
             dest='city',
-        ),        
+        ),
         make_option(
             '--state',
             action='store',
             dest='state',
             help='Two letter code (e.g. MA)'
-        ),        
+        ),
         make_option(
             '--postal_code',
             action='store',
@@ -57,16 +76,57 @@ class Command(BaseCommand):
             action='store',
             dest='phone',
             help='Pretty free-form (parens, spaces, dashes), but no country code'
-        ),        
+        ),
+        make_option(
+            '--extension',
+            action='store',
+            dest='extension',
+        ),
         make_option(
             '--phone_country_code',
             action='store',
             dest='phone_country_code',
             help='Phone country code, just "1" for the USA'
         ),
+        make_option(
+            '--fax',
+            action='store',
+            dest='fax',
+            help='Pretty free-form (parens, spaces, dashes), but no country code'
+        ),
+        make_option(
+            '--fax_country_code',
+            action='store',
+            dest='fax_country_code',
+            help='Fax country code, just "1" for the USA'
+        ),
+        make_option(
+            '--company_name',
+            action='store',
+            dest='company_name',
+        ),
+        # internal values:
+        make_option(
+            '--client_candidate_id',
+            action='store',
+            dest='client_candidate_id',
+            help='ID we assign a user to identify them to Pearson'
+        ),
+        make_option(
+            '--upload_status',
+            action='store',
+            dest='upload_status',
+            help='status value assigned by Pearson'
+        ),
+        make_option(
+            '--upload_error_message',
+            action='store',
+            dest='upload_error_message',
+            help='error message provided by Pearson on a failure.'
+        ),
     )
     args = "<student_username>"
-    help = "Create a TestCenterUser entry for a given Student"
+    help = "Create or modify a TestCenterUser entry for a given Student"
 
     @staticmethod
     def is_valid_option(option_name):
@@ -79,7 +139,52 @@ class Command(BaseCommand):
         print username
 
         our_options = dict((k, v) for k, v in options.items()
-                           if Command.is_valid_option(k))
+                           if Command.is_valid_option(k) and v is not None)
         student = User.objects.get(username=username)
-        student.test_center_user = TestCenterUser(**our_options)
-        student.test_center_user.save()
+        try:
+            testcenter_user = TestCenterUser.objects.get(user=student)
+            needs_updating = testcenter_user.needs_update(our_options)
+        except TestCenterUser.DoesNotExist:
+            # do additional initialization here:
+            testcenter_user = TestCenterUser.create(student)
+            needs_updating = True
+
+        if needs_updating:
+            # the registration form normally populates the data dict with
+            # all values from the testcenter_user.  But here we only want to
+            # specify those values that change, so update the dict with existing
+            # values.
+            form_options = dict(our_options)
+            for propname in TestCenterUser.user_provided_fields():
+                if propname not in form_options:
+                    form_options[propname] = testcenter_user.__getattribute__(propname)
+            form = TestCenterUserForm(instance=testcenter_user, data=form_options)
+            if form.is_valid():
+                form.update_and_save()
+            else:
+                errorlist = []
+                if (len(form.errors) > 0):
+                    errorlist.append("Field Form errors encountered:")
+                    for fielderror in form.errors:
+                        errorlist.append("Field Form Error:  {}".format(fielderror))
+                if (len(form.non_field_errors()) > 0):
+                    errorlist.append("Non-field Form errors encountered:")
+                    for nonfielderror in form.non_field_errors:
+                        errorlist.append("Non-field Form Error:  {}".format(nonfielderror))
+                raise CommandError("\n".join(errorlist))
+        else:
+            print "No changes necessary to make to existing user's demographics."
+
+        # override internal values:
+        change_internal = False
+        testcenter_user = TestCenterUser.objects.get(user=student)
+        for internal_field in ['upload_error_message', 'upload_status', 'client_candidate_id']:
+            if internal_field in our_options:
+                testcenter_user.__setattr__(internal_field, our_options[internal_field])
+                change_internal = True
+
+        if change_internal:
+            testcenter_user.save()
+            print "Updated  confirmation information in existing user's demographics."
+        else:
+            print "No changes necessary to make to confirmation information in existing user's demographics."

common/djangoapps/student/management/commands/pearson_transfer.py

@@ -0,0 +1,164 @@
+import os
+from optparse import make_option
+from stat import S_ISDIR
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+from django.core.management import call_command
+from dogapi import dog_http_api, dog_stats_api
+import paramiko
+import boto
+
+dog_http_api.api_key = settings.DATADOG_API
+dog_stats_api.start(api_key=settings.DATADOG_API, statsd=True)
+
+
+class Command(BaseCommand):
+    help = """
+    This command handles the importing and exporting of student records for
+    Pearson.  It uses some other Django commands to export and import the
+    files and then uploads over SFTP to Pearson and stuffs the entry in an
+    S3 bucket for archive purposes.
+
+    Usage: django-admin.py pearson-transfer --mode [import|export|both]
+    """
+
+    option_list = BaseCommand.option_list + (
+        make_option('--mode',
+                    action='store',
+                    dest='mode',
+                    default='both',
+                    choices=('import', 'export', 'both'),
+                    help='mode is import, export, or both'),
+    )
+
+    def handle(self, **options):
+
+        if not hasattr(settings, 'PEARSON'):
+            raise CommandError('No PEARSON entries in auth/env.json.')
+
+        # check settings needed for either import or export:
+        for value in ['SFTP_HOSTNAME', 'SFTP_USERNAME', 'SFTP_PASSWORD', 'S3_BUCKET']:
+            if value not in settings.PEARSON:
+                raise CommandError('No entry in the PEARSON settings'
+                                   '(env/auth.json) for {0}'.format(value))
+
+        for value in ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']:
+            if not hasattr(settings, value):
+                raise CommandError('No entry in the AWS settings'
+                                   '(env/auth.json) for {0}'.format(value))
+
+        # check additional required settings for import and export:
+        if options['mode'] in ('export', 'both'):
+            for value in ['LOCAL_EXPORT', 'SFTP_EXPORT']:
+                if value not in settings.PEARSON:
+                    raise CommandError('No entry in the PEARSON settings'
+                                       '(env/auth.json) for {0}'.format(value))
+            # make sure that the import directory exists or can be created:
+            source_dir = settings.PEARSON['LOCAL_EXPORT']
+            if not os.path.isdir(source_dir):
+                os.makedirs(source_dir)
+
+        if options['mode'] in ('import', 'both'):
+            for value in ['LOCAL_IMPORT', 'SFTP_IMPORT']:
+                if value not in settings.PEARSON:
+                    raise CommandError('No entry in the PEARSON settings'
+                                       '(env/auth.json) for {0}'.format(value))
+            # make sure that the import directory exists or can be created:
+            dest_dir = settings.PEARSON['LOCAL_IMPORT']
+            if not os.path.isdir(dest_dir):
+                os.makedirs(dest_dir)
+
+
+        def sftp(files_from, files_to, mode, deleteAfterCopy=False):
+            with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'):
+                try:
+                    t = paramiko.Transport((settings.PEARSON['SFTP_HOSTNAME'], 22))
+                    t.connect(username=settings.PEARSON['SFTP_USERNAME'],
+                              password=settings.PEARSON['SFTP_PASSWORD'])
+                    sftp = paramiko.SFTPClient.from_transport(t)
+
+                    if mode == 'export':
+                        try:
+                            sftp.chdir(files_to)
+                        except IOError:
+                            raise CommandError('SFTP destination path does not exist: {}'.format(files_to))
+                        for filename in os.listdir(files_from):
+                            sftp.put(files_from + '/' + filename, filename)
+                            if deleteAfterCopy:
+                                os.remove(os.path.join(files_from, filename))
+                    else:
+                        try:
+                            sftp.chdir(files_from)
+                        except IOError:
+                            raise CommandError('SFTP source path does not exist: {}'.format(files_from))
+                        for filename in sftp.listdir('.'):
+                            # skip subdirectories
+                            if not S_ISDIR(sftp.stat(filename).st_mode):
+                                sftp.get(filename, files_to + '/' + filename)
+                                # delete files from sftp server once they are successfully pulled off:
+                                if deleteAfterCopy:
+                                    sftp.remove(filename)
+                except:
+                    dog_http_api.event('pearson {0}'.format(mode),
+                                       'sftp uploading failed',
+                                       alert_type='error')
+                    raise
+                finally:
+                    sftp.close()
+                    t.close()
+
+        def s3(files_from, bucket, mode, deleteAfterCopy=False):
+            with dog_stats_api.timer('pearson.{0}'.format(mode), tags='s3'):
+                try:
+                    for filename in os.listdir(files_from):
+                        source_file = os.path.join(files_from, filename)
+                        # use mode as name of directory into which to write files
+                        dest_file = os.path.join(mode, filename)
+                        upload_file_to_s3(bucket, source_file, dest_file)
+                        if deleteAfterCopy:
+                            os.remove(files_from + '/' + filename)
+                except:
+                    dog_http_api.event('pearson {0}'.format(mode),
+                                       's3 archiving failed')
+                    raise
+
+        def upload_file_to_s3(bucket, source_file, dest_file):
+            """
+            Upload file to S3
+            """
+            s3 = boto.connect_s3(settings.AWS_ACCESS_KEY_ID,
+                                 settings.AWS_SECRET_ACCESS_KEY)
+            from boto.s3.key import Key
+            b = s3.get_bucket(bucket)
+            k = Key(b)
+            k.key = "{filename}".format(filename=dest_file)
+            k.set_contents_from_filename(source_file)
+
+        def export_pearson():
+            options = {'dest-from-settings': True}
+            call_command('pearson_export_cdd', **options)
+            call_command('pearson_export_ead', **options)
+            mode = 'export'
+            sftp(settings.PEARSON['LOCAL_EXPORT'], settings.PEARSON['SFTP_EXPORT'], mode, deleteAfterCopy=False)
+            s3(settings.PEARSON['LOCAL_EXPORT'], settings.PEARSON['S3_BUCKET'], mode, deleteAfterCopy=True)
+
+        def import_pearson():
+            mode = 'import'
+            try:
+                sftp(settings.PEARSON['SFTP_IMPORT'], settings.PEARSON['LOCAL_IMPORT'], mode, deleteAfterCopy=True)
+                s3(settings.PEARSON['LOCAL_IMPORT'], settings.PEARSON['S3_BUCKET'], mode, deleteAfterCopy=False)
+            except Exception as e:
+                dog_http_api.event('Pearson Import failure', str(e))
+                raise e
+            else:
+                for filename in os.listdir(settings.PEARSON['LOCAL_IMPORT']):
+                    filepath = os.path.join(settings.PEARSON['LOCAL_IMPORT'], filename)
+                    call_command('pearson_import_conf_zip', filepath)
+                    os.remove(filepath)
+
+        # actually do the work!
+        if options['mode'] in ('export', 'both'):
+            export_pearson()
+        if options['mode'] in ('import', 'both'):
+            import_pearson()

common/djangoapps/student/management/commands/tests/test_pearson.py

@@ -0,0 +1,386 @@
+'''
+Created on Jan 17, 2013
+
+@author: brian
+'''
+import logging
+import os
+from tempfile import mkdtemp
+import cStringIO
+import shutil
+import sys
+
+from django.test import TestCase
+from django.core.management import call_command
+from nose.plugins.skip import SkipTest
+
+from student.models import User, TestCenterRegistration, TestCenterUser, get_testcenter_registration
+
+log = logging.getLogger(__name__)
+
+
+def create_tc_user(username):
+    user = User.objects.create_user(username, '{}@edx.org'.format(username), 'fakepass')
+    options = {
+                   'first_name': 'TestFirst',
+                   'last_name': 'TestLast',
+                   'address_1': 'Test Address',
+                   'city': 'TestCity',
+                   'state': 'Alberta',
+                   'postal_code': 'A0B 1C2',
+                   'country': 'CAN',
+                   'phone': '252-1866',
+                   'phone_country_code': '1',
+                    }
+    call_command('pearson_make_tc_user', username, **options)
+    return TestCenterUser.objects.get(user=user)
+
+
+def create_tc_registration(username, course_id='org1/course1/term1', exam_code='exam1', accommodation_code=None):
+
+    options = {'exam_series_code': exam_code,
+               'eligibility_appointment_date_first': '2013-01-01T00:00',
+               'eligibility_appointment_date_last': '2013-12-31T23:59',
+               'accommodation_code': accommodation_code,
+               'create_dummy_exam': True,
+               }
+
+    call_command('pearson_make_tc_registration', username, course_id, **options)
+    user = User.objects.get(username=username)
+    registrations = get_testcenter_registration(user, course_id, exam_code)
+    return registrations[0]
+
+
+def create_multiple_registrations(prefix='test'):
+    username1 = '{}_multiple1'.format(prefix)
+    create_tc_user(username1)
+    create_tc_registration(username1)
+    create_tc_registration(username1, course_id='org1/course2/term1')
+    create_tc_registration(username1, exam_code='exam2')
+    username2 = '{}_multiple2'.format(prefix)
+    create_tc_user(username2)
+    create_tc_registration(username2)
+    username3 = '{}_multiple3'.format(prefix)
+    create_tc_user(username3)
+    create_tc_registration(username3, course_id='org1/course2/term1')
+    username4 = '{}_multiple4'.format(prefix)
+    create_tc_user(username4)
+    create_tc_registration(username4, exam_code='exam2')
+
+
+def get_command_error_text(*args, **options):
+    stderr_string = None
+    old_stderr = sys.stderr
+    sys.stderr = cStringIO.StringIO()
+    try:
+        call_command(*args, **options)
+    except SystemExit, why1:
+        # The goal here is to catch CommandError calls.
+        # But these are actually translated into nice messages,
+        # and sys.exit(1) is then called.  For testing, we
+        # want to catch what sys.exit throws, and get the
+        # relevant text either from stdout or stderr.
+        if (why1.message > 0):
+            stderr_string = sys.stderr.getvalue()
+        else:
+            raise why1
+    except Exception, why:
+        raise why
+
+    finally:
+        sys.stderr = old_stderr
+
+    if stderr_string is None:
+        raise Exception("Expected call to {} to fail, but it succeeded!".format(args[0]))
+    return stderr_string
+
+
+def get_error_string_for_management_call(*args, **options):
+    stdout_string = None
+    old_stdout = sys.stdout
+    old_stderr = sys.stderr
+    sys.stdout = cStringIO.StringIO()
+    sys.stderr = cStringIO.StringIO()
+    try:
+        call_command(*args, **options)
+    except SystemExit, why1:
+        # The goal here is to catch CommandError calls.
+        # But these are actually translated into nice messages,
+        # and sys.exit(1) is then called.  For testing, we
+        # want to catch what sys.exit throws, and get the
+        # relevant text either from stdout or stderr.
+        if (why1.message == 1):
+            stdout_string = sys.stdout.getvalue()
+            stderr_string = sys.stderr.getvalue()
+        else:
+            raise why1
+    except Exception, why:
+        raise why
+
+    finally:
+        sys.stdout = old_stdout
+        sys.stderr = old_stderr
+
+    if stdout_string is None:
+        raise Exception("Expected call to {} to fail, but it succeeded!".format(args[0]))
+    return stdout_string, stderr_string
+
+
+def get_file_info(dirpath):
+    filelist = os.listdir(dirpath)
+    print 'Files found: {}'.format(filelist)
+    numfiles = len(filelist)
+    if numfiles == 1:
+        filepath = os.path.join(dirpath, filelist[0])
+        with open(filepath, 'r') as cddfile:
+            filecontents = cddfile.readlines()
+            numlines = len(filecontents)
+            return filepath, numlines
+    else:
+        raise Exception("Expected to find a single file in {}, but found {}".format(dirpath, filelist))
+
+
+class PearsonTestCase(TestCase):
+    '''
+    Base class for tests running Pearson-related commands
+    '''
+
+    def assertErrorContains(self, error_message, expected):
+        self.assertTrue(error_message.find(expected) >= 0, 'error message "{}" did not contain "{}"'.format(error_message, expected))
+
+    def setUp(self):
+        self.import_dir = mkdtemp(prefix="import")
+        self.addCleanup(shutil.rmtree, self.import_dir)
+        self.export_dir = mkdtemp(prefix="export")
+        self.addCleanup(shutil.rmtree, self.export_dir)
+
+    def tearDown(self):
+        pass
+        # and clean up the database:
+#        TestCenterUser.objects.all().delete()
+#        TestCenterRegistration.objects.all().delete()
+
+
+class PearsonCommandTestCase(PearsonTestCase):
+
+    def test_missing_demographic_fields(self):
+        # We won't bother to test all details of form validation here.
+        # It is enough to show that it works here, but deal with test cases for the form
+        # validation in the student tests, not these management tests.
+        username = 'baduser'
+        User.objects.create_user(username, '{}@edx.org'.format(username), 'fakepass')
+        options = {}
+        error_string = get_command_error_text('pearson_make_tc_user', username, **options)
+        self.assertTrue(error_string.find('Field Form errors encountered:') >= 0)
+        self.assertTrue(error_string.find('Field Form Error:  city') >= 0)
+        self.assertTrue(error_string.find('Field Form Error:  first_name') >= 0)
+        self.assertTrue(error_string.find('Field Form Error:  last_name') >= 0)
+        self.assertTrue(error_string.find('Field Form Error:  country') >= 0)
+        self.assertTrue(error_string.find('Field Form Error:  phone_country_code') >= 0)
+        self.assertTrue(error_string.find('Field Form Error:  phone') >= 0)
+        self.assertTrue(error_string.find('Field Form Error:  address_1') >= 0)
+        self.assertErrorContains(error_string, 'Field Form Error:  address_1')
+
+    def test_create_good_testcenter_user(self):
+        testcenter_user = create_tc_user("test_good_user")
+        self.assertIsNotNone(testcenter_user)
+
+    def test_create_good_testcenter_registration(self):
+        username = 'test_good_registration'
+        create_tc_user(username)
+        registration = create_tc_registration(username)
+        self.assertIsNotNone(registration)
+
+    def test_cdd_missing_option(self):
+        error_string = get_command_error_text('pearson_export_cdd', **{})
+        self.assertErrorContains(error_string, 'Error: --destination or --dest-from-settings must be used')
+
+    def test_ead_missing_option(self):
+        error_string = get_command_error_text('pearson_export_ead', **{})
+        self.assertErrorContains(error_string, 'Error: --destination or --dest-from-settings must be used')
+
+    def test_export_single_cdd(self):
+        # before we generate any tc_users, we expect there to be nothing to output:
+        options = {'dest-from-settings': True}
+        with self.settings(PEARSON={'LOCAL_EXPORT': self.export_dir}):
+            call_command('pearson_export_cdd', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 1, "Expect cdd file to have no non-header lines")
+            os.remove(filepath)
+
+            # generating a tc_user should result in a line in the output
+            username = 'test_single_cdd'
+            create_tc_user(username)
+            call_command('pearson_export_cdd', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 2, "Expect cdd file to have one non-header line")
+            os.remove(filepath)
+
+            # output after registration should not have any entries again.
+            call_command('pearson_export_cdd', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 1, "Expect cdd file to have no non-header lines")
+            os.remove(filepath)
+
+            # if we modify the record, then it should be output again:
+            user_options = {'first_name': 'NewTestFirst', }
+            call_command('pearson_make_tc_user', username, **user_options)
+            call_command('pearson_export_cdd', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 2, "Expect cdd file to have one non-header line")
+            os.remove(filepath)
+
+    def test_export_single_ead(self):
+        # before we generate any registrations, we expect there to be nothing to output:
+        options = {'dest-from-settings': True}
+        with self.settings(PEARSON={'LOCAL_EXPORT': self.export_dir}):
+            call_command('pearson_export_ead', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 1, "Expect ead file to have no non-header lines")
+            os.remove(filepath)
+
+            # generating a registration should result in a line in the output
+            username = 'test_single_ead'
+            create_tc_user(username)
+            create_tc_registration(username)
+            call_command('pearson_export_ead', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 2, "Expect ead file to have one non-header line")
+            os.remove(filepath)
+
+            # output after registration should not have any entries again.
+            call_command('pearson_export_ead', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 1, "Expect ead file to have no non-header lines")
+            os.remove(filepath)
+
+            # if we modify the record, then it should be output again:
+            create_tc_registration(username, accommodation_code='EQPMNT')
+            call_command('pearson_export_ead', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 2, "Expect ead file to have one non-header line")
+            os.remove(filepath)
+
+    def test_export_multiple(self):
+        create_multiple_registrations("export")
+        with self.settings(PEARSON={'LOCAL_EXPORT': self.export_dir}):
+            options = {'dest-from-settings': True}
+            call_command('pearson_export_cdd', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 5, "Expect cdd file to have four non-header lines: total was {}".format(numlines))
+            os.remove(filepath)
+
+            call_command('pearson_export_ead', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 7, "Expect ead file to have six non-header lines: total was {}".format(numlines))
+            os.remove(filepath)
+
+
+#    def test_bad_demographic_option(self):
+#        username = 'nonuser'
+#        output_string, stderrmsg = get_error_string_for_management_call('pearson_make_tc_user', username, **{'--garbage' : None })
+#        print stderrmsg
+#        self.assertErrorContains(stderrmsg, 'Unexpected option')
+#
+#    def test_missing_demographic_user(self):
+#        username = 'nonuser'
+#        output_string, error_string = get_error_string_for_management_call('pearson_make_tc_user', username, **{})
+#        self.assertErrorContains(error_string, 'User matching query does not exist')
+
+# credentials for a test SFTP site:
+SFTP_HOSTNAME = 'ec2-23-20-150-101.compute-1.amazonaws.com'
+SFTP_USERNAME = 'pearsontest'
+SFTP_PASSWORD = 'password goes here'
+
+S3_BUCKET = 'edx-pearson-archive'
+AWS_ACCESS_KEY_ID = 'put yours here'
+AWS_SECRET_ACCESS_KEY = 'put yours here'
+
+
+class PearsonTransferTestCase(PearsonTestCase):
+    '''
+    Class for tests running Pearson transfers
+    '''
+
+    def test_transfer_config(self):
+        with self.settings(DATADOG_API='FAKE_KEY'):
+            # TODO: why is this failing with the wrong error message?!
+            stderrmsg = get_command_error_text('pearson_transfer', **{'mode': 'garbage'})
+            self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries')
+        with self.settings(DATADOG_API='FAKE_KEY'):
+            stderrmsg = get_command_error_text('pearson_transfer')
+            self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries')
+        with self.settings(DATADOG_API='FAKE_KEY',
+                           PEARSON={'LOCAL_EXPORT': self.export_dir,
+                                    'LOCAL_IMPORT': self.import_dir}):
+            stderrmsg = get_command_error_text('pearson_transfer')
+            self.assertErrorContains(stderrmsg, 'Error: No entry in the PEARSON settings')
+
+    def test_transfer_export_missing_dest_dir(self):
+        raise SkipTest()
+        create_multiple_registrations('export_missing_dest')
+        with self.settings(DATADOG_API='FAKE_KEY',
+                           PEARSON={'LOCAL_EXPORT': self.export_dir,
+                                    'SFTP_EXPORT': 'this/does/not/exist',
+                                    'SFTP_HOSTNAME': SFTP_HOSTNAME,
+                                    'SFTP_USERNAME': SFTP_USERNAME,
+                                    'SFTP_PASSWORD': SFTP_PASSWORD,
+                                    'S3_BUCKET': S3_BUCKET,
+                                    },
+                           AWS_ACCESS_KEY_ID=AWS_ACCESS_KEY_ID,
+                           AWS_SECRET_ACCESS_KEY=AWS_SECRET_ACCESS_KEY):
+            options = {'mode': 'export'}
+            stderrmsg = get_command_error_text('pearson_transfer', **options)
+            self.assertErrorContains(stderrmsg, 'Error: SFTP destination path does not exist')
+
+    def test_transfer_export(self):
+        raise SkipTest()
+        create_multiple_registrations("transfer_export")
+        with self.settings(DATADOG_API='FAKE_KEY',
+                           PEARSON={'LOCAL_EXPORT': self.export_dir,
+                                    'SFTP_EXPORT': 'results/topvue',
+                                    'SFTP_HOSTNAME': SFTP_HOSTNAME,
+                                    'SFTP_USERNAME': SFTP_USERNAME,
+                                    'SFTP_PASSWORD': SFTP_PASSWORD,
+                                    'S3_BUCKET': S3_BUCKET,
+                                    },
+                           AWS_ACCESS_KEY_ID=AWS_ACCESS_KEY_ID,
+                           AWS_SECRET_ACCESS_KEY=AWS_SECRET_ACCESS_KEY):
+            options = {'mode': 'export'}
+#            call_command('pearson_transfer', **options)
+#            # confirm that the export directory is still empty:
+#            self.assertEqual(len(os.listdir(self.export_dir)), 0, "expected export directory to be empty")
+
+    def test_transfer_import_missing_source_dir(self):
+        raise SkipTest()
+        create_multiple_registrations('import_missing_src')
+        with self.settings(DATADOG_API='FAKE_KEY',
+                           PEARSON={'LOCAL_IMPORT': self.import_dir,
+                                    'SFTP_IMPORT': 'this/does/not/exist',
+                                    'SFTP_HOSTNAME': SFTP_HOSTNAME,
+                                    'SFTP_USERNAME': SFTP_USERNAME,
+                                    'SFTP_PASSWORD': SFTP_PASSWORD,
+                                    'S3_BUCKET': S3_BUCKET,
+                                    },
+                           AWS_ACCESS_KEY_ID=AWS_ACCESS_KEY_ID,
+                           AWS_SECRET_ACCESS_KEY=AWS_SECRET_ACCESS_KEY):
+            options = {'mode': 'import'}
+            stderrmsg = get_command_error_text('pearson_transfer', **options)
+            self.assertErrorContains(stderrmsg, 'Error: SFTP source path does not exist')
+
+    def test_transfer_import(self):
+        raise SkipTest()
+        create_multiple_registrations('import_missing_src')
+        with self.settings(DATADOG_API='FAKE_KEY',
+                           PEARSON={'LOCAL_IMPORT': self.import_dir,
+                                    'SFTP_IMPORT': 'results',
+                                    'SFTP_HOSTNAME': SFTP_HOSTNAME,
+                                    'SFTP_USERNAME': SFTP_USERNAME,
+                                    'SFTP_PASSWORD': SFTP_PASSWORD,
+                                    'S3_BUCKET': S3_BUCKET,
+                                    },
+                           AWS_ACCESS_KEY_ID=AWS_ACCESS_KEY_ID,
+                           AWS_SECRET_ACCESS_KEY=AWS_SECRET_ACCESS_KEY):
+            options = {'mode': 'import'}
+            call_command('pearson_transfer', **options)
+            self.assertEqual(len(os.listdir(self.import_dir)), 0, "expected import directory to be empty")

common/djangoapps/student/migrations/0020_add_test_center_user.py

@@ -185,4 +185,4 @@ class Migration(SchemaMigration):
         }
     }
 
-    complete_apps = ['student']
+    complete_apps = ['student']

common/djangoapps/student/migrations/0021_remove_askbot.py

@@ -26,14 +26,17 @@ class Migration(SchemaMigration):
 
     def forwards(self, orm):
         "Kill the askbot"
-        # For MySQL, we're batching the alters together for performance reasons
-        if db.backend_name == 'mysql':
-            drops = ["drop `{0}`".format(col) for col in ASKBOT_AUTH_USER_COLUMNS]
-            statement = "alter table `auth_user` {0};".format(", ".join(drops))
-            db.execute(statement)
-        else:
-            for column in ASKBOT_AUTH_USER_COLUMNS:
-                db.delete_column('auth_user', column)            
+        try:
+            # For MySQL, we're batching the alters together for performance reasons
+            if db.backend_name == 'mysql':
+                drops = ["drop `{0}`".format(col) for col in ASKBOT_AUTH_USER_COLUMNS]
+                statement = "alter table `auth_user` {0};".format(", ".join(drops))
+                db.execute(statement)
+            else:
+                for column in ASKBOT_AUTH_USER_COLUMNS:
+                    db.delete_column('auth_user', column)
+        except Exception as ex:
+            print "Couldn't remove askbot because of {0} -- it was probably never here to begin with.".format(ex)
 
     def backwards(self, orm):
         raise RuntimeError("Cannot reverse this migration: there's no going back to Askbot.")

common/djangoapps/student/migrations/0022_auto__add_courseenrollmentallowed__add_unique_courseenrollmentallowed_.py

@@ -0,0 +1,155 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'CourseEnrollmentAllowed'
+        db.create_table('student_courseenrollmentallowed', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('email', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)),
+            ('course_id', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)),
+            ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, null=True, db_index=True, blank=True)),
+        ))
+        db.send_create_signal('student', ['CourseEnrollmentAllowed'])
+
+        # Adding unique constraint on 'CourseEnrollmentAllowed', fields ['email', 'course_id']
+        db.create_unique('student_courseenrollmentallowed', ['email', 'course_id'])
+
+
+    def backwards(self, orm):
+        # Removing unique constraint on 'CourseEnrollmentAllowed', fields ['email', 'course_id']
+        db.delete_unique('student_courseenrollmentallowed', ['email', 'course_id'])
+
+        # Deleting model 'CourseEnrollmentAllowed'
+        db.delete_table('student_courseenrollmentallowed')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'student.courseenrollment': {
+            'Meta': {'unique_together': "(('user', 'course_id'),)", 'object_name': 'CourseEnrollment'},
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'null': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'student.courseenrollmentallowed': {
+            'Meta': {'unique_together': "(('email', 'course_id'),)", 'object_name': 'CourseEnrollmentAllowed'},
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'null': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'email': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'student.pendingemailchange': {
+            'Meta': {'object_name': 'PendingEmailChange'},
+            'activation_key': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '32', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'new_email': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True'})
+        },
+        'student.pendingnamechange': {
+            'Meta': {'object_name': 'PendingNameChange'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'new_name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'rationale': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True'})
+        },
+        'student.registration': {
+            'Meta': {'object_name': 'Registration', 'db_table': "'auth_registration'"},
+            'activation_key': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '32', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'unique': 'True'})
+        },
+        'student.testcenteruser': {
+            'Meta': {'object_name': 'TestCenterUser'},
+            'address_1': ('django.db.models.fields.CharField', [], {'max_length': '40'}),
+            'address_2': ('django.db.models.fields.CharField', [], {'max_length': '40', 'blank': 'True'}),
+            'address_3': ('django.db.models.fields.CharField', [], {'max_length': '40', 'blank': 'True'}),
+            'candidate_id': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'db_index': 'True'}),
+            'city': ('django.db.models.fields.CharField', [], {'max_length': '32', 'db_index': 'True'}),
+            'client_candidate_id': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'company_name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
+            'country': ('django.db.models.fields.CharField', [], {'max_length': '3', 'db_index': 'True'}),
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'extension': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '8', 'blank': 'True'}),
+            'fax': ('django.db.models.fields.CharField', [], {'max_length': '35', 'blank': 'True'}),
+            'fax_country_code': ('django.db.models.fields.CharField', [], {'max_length': '3', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'middle_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'phone': ('django.db.models.fields.CharField', [], {'max_length': '35'}),
+            'phone_country_code': ('django.db.models.fields.CharField', [], {'max_length': '3', 'db_index': 'True'}),
+            'postal_code': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '16', 'blank': 'True'}),
+            'salutation': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
+            'state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'blank': 'True'}),
+            'suffix': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'updated_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'to': "orm['auth.User']", 'unique': 'True'}),
+            'user_updated_at': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'})
+        },
+        'student.userprofile': {
+            'Meta': {'object_name': 'UserProfile', 'db_table': "'auth_userprofile'"},
+            'courseware': ('django.db.models.fields.CharField', [], {'default': "'course.xml'", 'max_length': '255', 'blank': 'True'}),
+            'gender': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '6', 'null': 'True', 'blank': 'True'}),
+            'goals': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'language': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'blank': 'True'}),
+            'level_of_education': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '6', 'null': 'True', 'blank': 'True'}),
+            'location': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'blank': 'True'}),
+            'mailing_address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'meta': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'profile'", 'unique': 'True', 'to': "orm['auth.User']"}),
+            'year_of_birth': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'})
+        },
+        'student.usertestgroup': {
+            'Meta': {'object_name': 'UserTestGroup'},
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '32', 'db_index': 'True'}),
+            'users': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.User']", 'db_index': 'True', 'symmetrical': 'False'})
+        }
+    }
+
+    complete_apps = ['student']

common/djangoapps/student/migrations/0023_add_test_center_registration.py

@@ -0,0 +1,241 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'TestCenterRegistration'
+        db.create_table('student_testcenterregistration', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('testcenter_user', self.gf('django.db.models.fields.related.ForeignKey')(default=None, to=orm['student.TestCenterUser'])),
+            ('course_id', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)),
+            ('created_at', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, db_index=True, blank=True)),
+            ('updated_at', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, db_index=True, blank=True)),
+            ('user_updated_at', self.gf('django.db.models.fields.DateTimeField')(db_index=True)),
+            ('client_authorization_id', self.gf('django.db.models.fields.CharField')(unique=True, max_length=20, db_index=True)),
+            ('exam_series_code', self.gf('django.db.models.fields.CharField')(max_length=15, db_index=True)),
+            ('eligibility_appointment_date_first', self.gf('django.db.models.fields.DateField')(db_index=True)),
+            ('eligibility_appointment_date_last', self.gf('django.db.models.fields.DateField')(db_index=True)),
+            ('accommodation_code', self.gf('django.db.models.fields.CharField')(max_length=64, blank=True)),
+            ('accommodation_request', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=1024, blank=True)),
+            ('uploaded_at', self.gf('django.db.models.fields.DateTimeField')(null=True, db_index=True)),
+            ('processed_at', self.gf('django.db.models.fields.DateTimeField')(null=True, db_index=True)),
+            ('upload_status', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=20, blank=True)),
+            ('upload_error_message', self.gf('django.db.models.fields.CharField')(max_length=512, blank=True)),
+            ('authorization_id', self.gf('django.db.models.fields.IntegerField')(null=True, db_index=True)),
+            ('confirmed_at', self.gf('django.db.models.fields.DateTimeField')(null=True, db_index=True)),
+        ))
+        db.send_create_signal('student', ['TestCenterRegistration'])
+
+        # Adding field 'TestCenterUser.uploaded_at'
+        db.add_column('student_testcenteruser', 'uploaded_at',
+                      self.gf('django.db.models.fields.DateTimeField')(db_index=True, null=True, blank=True),
+                      keep_default=False)
+
+        # Adding field 'TestCenterUser.processed_at'
+        db.add_column('student_testcenteruser', 'processed_at',
+                      self.gf('django.db.models.fields.DateTimeField')(null=True, db_index=True),
+                      keep_default=False)
+
+        # Adding field 'TestCenterUser.upload_status'
+        db.add_column('student_testcenteruser', 'upload_status',
+                      self.gf('django.db.models.fields.CharField')(db_index=True, default='', max_length=20, blank=True),
+                      keep_default=False)
+
+        # Adding field 'TestCenterUser.upload_error_message'
+        db.add_column('student_testcenteruser', 'upload_error_message',
+                      self.gf('django.db.models.fields.CharField')(default='', max_length=512, blank=True),
+                      keep_default=False)
+
+        # Adding field 'TestCenterUser.confirmed_at'
+        db.add_column('student_testcenteruser', 'confirmed_at',
+                      self.gf('django.db.models.fields.DateTimeField')(null=True, db_index=True),
+                      keep_default=False)
+
+        # Adding index on 'TestCenterUser', fields ['company_name']
+        db.create_index('student_testcenteruser', ['company_name'])
+
+        # Adding unique constraint on 'TestCenterUser', fields ['client_candidate_id']
+        db.create_unique('student_testcenteruser', ['client_candidate_id'])
+
+
+    def backwards(self, orm):
+        # Removing unique constraint on 'TestCenterUser', fields ['client_candidate_id']
+        db.delete_unique('student_testcenteruser', ['client_candidate_id'])
+
+        # Removing index on 'TestCenterUser', fields ['company_name']
+        db.delete_index('student_testcenteruser', ['company_name'])
+
+        # Deleting model 'TestCenterRegistration'
+        db.delete_table('student_testcenterregistration')
+
+        # Deleting field 'TestCenterUser.uploaded_at'
+        db.delete_column('student_testcenteruser', 'uploaded_at')
+
+        # Deleting field 'TestCenterUser.processed_at'
+        db.delete_column('student_testcenteruser', 'processed_at')
+
+        # Deleting field 'TestCenterUser.upload_status'
+        db.delete_column('student_testcenteruser', 'upload_status')
+
+        # Deleting field 'TestCenterUser.upload_error_message'
+        db.delete_column('student_testcenteruser', 'upload_error_message')
+
+        # Deleting field 'TestCenterUser.confirmed_at'
+        db.delete_column('student_testcenteruser', 'confirmed_at')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'student.courseenrollment': {
+            'Meta': {'unique_together': "(('user', 'course_id'),)", 'object_name': 'CourseEnrollment'},
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'null': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'student.courseenrollmentallowed': {
+            'Meta': {'unique_together': "(('email', 'course_id'),)", 'object_name': 'CourseEnrollmentAllowed'},
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'null': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'email': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'student.pendingemailchange': {
+            'Meta': {'object_name': 'PendingEmailChange'},
+            'activation_key': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '32', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'new_email': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True'})
+        },
+        'student.pendingnamechange': {
+            'Meta': {'object_name': 'PendingNameChange'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'new_name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'rationale': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True'})
+        },
+        'student.registration': {
+            'Meta': {'object_name': 'Registration', 'db_table': "'auth_registration'"},
+            'activation_key': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '32', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'unique': 'True'})
+        },
+        'student.testcenterregistration': {
+            'Meta': {'object_name': 'TestCenterRegistration'},
+            'accommodation_code': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}),
+            'accommodation_request': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '1024', 'blank': 'True'}),
+            'authorization_id': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'db_index': 'True'}),
+            'client_authorization_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '20', 'db_index': 'True'}),
+            'confirmed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'eligibility_appointment_date_first': ('django.db.models.fields.DateField', [], {'db_index': 'True'}),
+            'eligibility_appointment_date_last': ('django.db.models.fields.DateField', [], {'db_index': 'True'}),
+            'exam_series_code': ('django.db.models.fields.CharField', [], {'max_length': '15', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'processed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
+            'testcenter_user': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'to': "orm['student.TestCenterUser']"}),
+            'updated_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'upload_error_message': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
+            'upload_status': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'blank': 'True'}),
+            'uploaded_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
+            'user_updated_at': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'})
+        },
+        'student.testcenteruser': {
+            'Meta': {'object_name': 'TestCenterUser'},
+            'address_1': ('django.db.models.fields.CharField', [], {'max_length': '40'}),
+            'address_2': ('django.db.models.fields.CharField', [], {'max_length': '40', 'blank': 'True'}),
+            'address_3': ('django.db.models.fields.CharField', [], {'max_length': '40', 'blank': 'True'}),
+            'candidate_id': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'db_index': 'True'}),
+            'city': ('django.db.models.fields.CharField', [], {'max_length': '32', 'db_index': 'True'}),
+            'client_candidate_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '50', 'db_index': 'True'}),
+            'company_name': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'blank': 'True'}),
+            'confirmed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
+            'country': ('django.db.models.fields.CharField', [], {'max_length': '3', 'db_index': 'True'}),
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'extension': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '8', 'blank': 'True'}),
+            'fax': ('django.db.models.fields.CharField', [], {'max_length': '35', 'blank': 'True'}),
+            'fax_country_code': ('django.db.models.fields.CharField', [], {'max_length': '3', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'middle_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'phone': ('django.db.models.fields.CharField', [], {'max_length': '35'}),
+            'phone_country_code': ('django.db.models.fields.CharField', [], {'max_length': '3', 'db_index': 'True'}),
+            'postal_code': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '16', 'blank': 'True'}),
+            'processed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
+            'salutation': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
+            'state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'blank': 'True'}),
+            'suffix': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'updated_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'upload_error_message': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
+            'upload_status': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'blank': 'True'}),
+            'uploaded_at': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'to': "orm['auth.User']", 'unique': 'True'}),
+            'user_updated_at': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'})
+        },
+        'student.userprofile': {
+            'Meta': {'object_name': 'UserProfile', 'db_table': "'auth_userprofile'"},
+            'courseware': ('django.db.models.fields.CharField', [], {'default': "'course.xml'", 'max_length': '255', 'blank': 'True'}),
+            'gender': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '6', 'null': 'True', 'blank': 'True'}),
+            'goals': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'language': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'blank': 'True'}),
+            'level_of_education': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '6', 'null': 'True', 'blank': 'True'}),
+            'location': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'blank': 'True'}),
+            'mailing_address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'meta': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'profile'", 'unique': 'True', 'to': "orm['auth.User']"}),
+            'year_of_birth': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'})
+        },
+        'student.usertestgroup': {
+            'Meta': {'object_name': 'UserTestGroup'},
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '32', 'db_index': 'True'}),
+            'users': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.User']", 'db_index': 'True', 'symmetrical': 'False'})
+        }
+    }
+
+    complete_apps = ['student']

common/djangoapps/student/migrations/0024_add_allow_certificate.py

@@ -0,0 +1,172 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'UserProfile.allow_certificate'
+        db.add_column('auth_userprofile', 'allow_certificate',
+                      self.gf('django.db.models.fields.BooleanField')(default=True),
+                      keep_default=False)
+
+
+    def backwards(self, orm):
+        # Deleting field 'UserProfile.allow_certificate'
+        db.delete_column('auth_userprofile', 'allow_certificate')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'student.courseenrollment': {
+            'Meta': {'unique_together': "(('user', 'course_id'),)", 'object_name': 'CourseEnrollment'},
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'null': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'student.courseenrollmentallowed': {
+            'Meta': {'unique_together': "(('email', 'course_id'),)", 'object_name': 'CourseEnrollmentAllowed'},
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'null': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'email': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'student.pendingemailchange': {
+            'Meta': {'object_name': 'PendingEmailChange'},
+            'activation_key': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '32', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'new_email': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True'})
+        },
+        'student.pendingnamechange': {
+            'Meta': {'object_name': 'PendingNameChange'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'new_name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'rationale': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True'})
+        },
+        'student.registration': {
+            'Meta': {'object_name': 'Registration', 'db_table': "'auth_registration'"},
+            'activation_key': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '32', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'unique': 'True'})
+        },
+        'student.testcenterregistration': {
+            'Meta': {'object_name': 'TestCenterRegistration'},
+            'accommodation_code': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}),
+            'accommodation_request': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '1024', 'blank': 'True'}),
+            'authorization_id': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'db_index': 'True'}),
+            'client_authorization_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '20', 'db_index': 'True'}),
+            'confirmed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'eligibility_appointment_date_first': ('django.db.models.fields.DateField', [], {'db_index': 'True'}),
+            'eligibility_appointment_date_last': ('django.db.models.fields.DateField', [], {'db_index': 'True'}),
+            'exam_series_code': ('django.db.models.fields.CharField', [], {'max_length': '15', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'processed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
+            'testcenter_user': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'to': "orm['student.TestCenterUser']"}),
+            'updated_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'upload_error_message': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
+            'upload_status': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'blank': 'True'}),
+            'uploaded_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
+            'user_updated_at': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'})
+        },
+        'student.testcenteruser': {
+            'Meta': {'object_name': 'TestCenterUser'},
+            'address_1': ('django.db.models.fields.CharField', [], {'max_length': '40'}),
+            'address_2': ('django.db.models.fields.CharField', [], {'max_length': '40', 'blank': 'True'}),
+            'address_3': ('django.db.models.fields.CharField', [], {'max_length': '40', 'blank': 'True'}),
+            'candidate_id': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'db_index': 'True'}),
+            'city': ('django.db.models.fields.CharField', [], {'max_length': '32', 'db_index': 'True'}),
+            'client_candidate_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '50', 'db_index': 'True'}),
+            'company_name': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'blank': 'True'}),
+            'confirmed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
+            'country': ('django.db.models.fields.CharField', [], {'max_length': '3', 'db_index': 'True'}),
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'extension': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '8', 'blank': 'True'}),
+            'fax': ('django.db.models.fields.CharField', [], {'max_length': '35', 'blank': 'True'}),
+            'fax_country_code': ('django.db.models.fields.CharField', [], {'max_length': '3', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'middle_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'phone': ('django.db.models.fields.CharField', [], {'max_length': '35'}),
+            'phone_country_code': ('django.db.models.fields.CharField', [], {'max_length': '3', 'db_index': 'True'}),
+            'postal_code': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '16', 'blank': 'True'}),
+            'processed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
+            'salutation': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
+            'state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'blank': 'True'}),
+            'suffix': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'updated_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'upload_error_message': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
+            'upload_status': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'blank': 'True'}),
+            'uploaded_at': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'to': "orm['auth.User']", 'unique': 'True'}),
+            'user_updated_at': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'})
+        },
+        'student.userprofile': {
+            'Meta': {'object_name': 'UserProfile', 'db_table': "'auth_userprofile'"},
+            'allow_certificate': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'courseware': ('django.db.models.fields.CharField', [], {'default': "'course.xml'", 'max_length': '255', 'blank': 'True'}),
+            'gender': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '6', 'null': 'True', 'blank': 'True'}),
+            'goals': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'language': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'blank': 'True'}),
+            'level_of_education': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '6', 'null': 'True', 'blank': 'True'}),
+            'location': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'blank': 'True'}),
+            'mailing_address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'meta': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'profile'", 'unique': 'True', 'to': "orm['auth.User']"}),
+            'year_of_birth': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'})
+        },
+        'student.usertestgroup': {
+            'Meta': {'object_name': 'UserTestGroup'},
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '32', 'db_index': 'True'}),
+            'users': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.User']", 'db_index': 'True', 'symmetrical': 'False'})
+        }
+    }
+
+    complete_apps = ['student']

common/djangoapps/student/models.py

@@ -1,30 +1,5 @@
 """
-Models for Student Information
-
-Replication Notes
-
-TODO: Update this to be consistent with reality  (no portal servers, no more askbot)
-
-In our live deployment, we intend to run in a scenario where there is a pool of
-Portal servers that hold the canoncial user information and that user
-information is replicated to slave Course server pools. Each Course has a set of
-servers that serves only its content and has users that are relevant only to it.
-
-We replicate the following tables into the Course DBs where the user is
-enrolled. Only the Portal servers should ever write to these models.
-* UserProfile
-* CourseEnrollment
-
-We do a partial replication of:
-* User -- Askbot extends this and uses the extra fields, so we replicate only
-          the stuff that comes with basic django_auth and ignore the rest.)
-
-There are a couple different scenarios:
-
-1. There's an update of User or UserProfile -- replicate it to all Course DBs
-   that the user is enrolled in (found via CourseEnrollment).
-2. There's a change in CourseEnrollment. We need to push copies of UserProfile,
-   CourseEnrollment, and the base fields in User
+Models for User Information (students, staff, etc)
 
 Migration Notes
 
@@ -40,6 +15,8 @@ import hashlib
 import json
 import logging
 import uuid
+from random import randint
+from time import strftime
 
 
 from django.conf import settings
@@ -47,9 +24,9 @@ from django.contrib.auth.models import User
 from django.db import models
 from django.db.models.signals import post_save
 from django.dispatch import receiver
+from django.forms import ModelForm, forms
 
 import comment_client as cc
-from django_comment_client.models import Role
 
 
 log = logging.getLogger(__name__)
@@ -98,10 +75,15 @@ class UserProfile(models.Model):
     GENDER_CHOICES = (('m', 'Male'), ('f', 'Female'), ('o', 'Other'))
     gender = models.CharField(blank=True, null=True, max_length=6, db_index=True,
                               choices=GENDER_CHOICES)
-    LEVEL_OF_EDUCATION_CHOICES = (('p_se', 'Doctorate in science or engineering'),
-                                  ('p_oth', 'Doctorate in another field'),
+
+    # [03/21/2013] removed these, but leaving comment since there'll still be
+    # p_se and p_oth in the existing data in db.
+    # ('p_se', 'Doctorate in science or engineering'),
+    # ('p_oth', 'Doctorate in another field'),
+    LEVEL_OF_EDUCATION_CHOICES = (('p', 'Doctorate'),
                                   ('m', "Master's or professional degree"),
                                   ('b', "Bachelor's degree"),
+                                  ('a', "Associate's degree"),
                                   ('hs', "Secondary/high school"),
                                   ('jhs', "Junior secondary/junior high/middle school"),
                                   ('el', "Elementary/primary school"),
@@ -113,6 +95,7 @@ class UserProfile(models.Model):
                          )
     mailing_address = models.TextField(blank=True, null=True)
     goals = models.TextField(blank=True, null=True)
+    allow_certificate = models.BooleanField(default=1)
 
     def get_meta(self):
         js_str = self.meta
@@ -126,6 +109,10 @@ class UserProfile(models.Model):
     def set_meta(self, js):
         self.meta = json.dumps(js)
 
+TEST_CENTER_STATUS_ACCEPTED = "Accepted"
+TEST_CENTER_STATUS_ERROR = "Error"
+
+
 class TestCenterUser(models.Model):
     """This is our representation of the User for in-person testing, and
     specifically for Pearson at this point. A few things to note:
@@ -141,6 +128,9 @@ class TestCenterUser(models.Model):
     The field names and lengths are modeled on the conventions and constraints
     of Pearson's data import system, including oddities such as suffix having
     a limit of 255 while last_name only gets 50.
+
+    Also storing here the confirmation information received from Pearson (if any)
+    as to the success or failure of the upload.  (VCDC file)
     """
     # Our own record keeping...
     user = models.ForeignKey(User, unique=True, default=None)
@@ -151,12 +141,8 @@ class TestCenterUser(models.Model):
     # updated_at, this will not get incremented when we do a batch data import.
     user_updated_at = models.DateTimeField(db_index=True)
 
-    # Unique ID given to us for this User by the Testing Center. It's null when
-    # we first create the User entry, and is assigned by Pearson later.
-    candidate_id = models.IntegerField(null=True, db_index=True)
-
-    # Unique ID we assign our user for a the Test Center.
-    client_candidate_id = models.CharField(max_length=50, db_index=True)
+    # Unique ID we assign our user for the Test Center.
+    client_candidate_id = models.CharField(unique=True, max_length=50, db_index=True)
 
     # Name
     first_name = models.CharField(max_length=30, db_index=True)
@@ -187,19 +173,425 @@ class TestCenterUser(models.Model):
     fax_country_code = models.CharField(max_length=3, blank=True)
 
     # Company
-    company_name = models.CharField(max_length=50, blank=True)
+    company_name = models.CharField(max_length=50, blank=True, db_index=True)
+
+    # time at which edX sent the registration to the test center
+    uploaded_at = models.DateTimeField(null=True, blank=True, db_index=True)
+
+    # confirmation back from the test center, as well as timestamps
+    # on when they processed the request, and when we received
+    # confirmation back.
+    processed_at = models.DateTimeField(null=True, db_index=True)
+    upload_status = models.CharField(max_length=20, blank=True, db_index=True)  # 'Error' or 'Accepted'
+    upload_error_message = models.CharField(max_length=512, blank=True)
+    # Unique ID given to us for this User by the Testing Center. It's null when
+    # we first create the User entry, and may be assigned by Pearson later.
+    # (However, it may never be set if we are always initiating such candidate creation.)
+    candidate_id = models.IntegerField(null=True, db_index=True)
+    confirmed_at = models.DateTimeField(null=True, db_index=True)
+
+    @property
+    def needs_uploading(self):
+        return self.uploaded_at is None or self.uploaded_at < self.user_updated_at
+
+    @staticmethod
+    def user_provided_fields():
+        return ['first_name', 'middle_name', 'last_name', 'suffix', 'salutation',
+                'address_1', 'address_2', 'address_3', 'city', 'state', 'postal_code', 'country',
+                'phone', 'extension', 'phone_country_code', 'fax', 'fax_country_code', 'company_name']
 
     @property
     def email(self):
         return self.user.email
 
+    def needs_update(self, fields):
+        for fieldname in TestCenterUser.user_provided_fields():
+            if fieldname in fields and getattr(self, fieldname) != fields[fieldname]:
+                return True
+
+        return False
+
+    @staticmethod
+    def _generate_edx_id(prefix):
+        NUM_DIGITS = 12
+        return u"{}{:012}".format(prefix, randint(1, 10 ** NUM_DIGITS - 1))
+
+    @staticmethod
+    def _generate_candidate_id():
+        return TestCenterUser._generate_edx_id("edX")
+
+    @classmethod
+    def create(cls, user):
+        testcenter_user = cls(user=user)
+        # testcenter_user.candidate_id remains unset
+        # assign an ID of our own:
+        cand_id = cls._generate_candidate_id()
+        while TestCenterUser.objects.filter(client_candidate_id=cand_id).exists():
+            cand_id = cls._generate_candidate_id()
+        testcenter_user.client_candidate_id = cand_id
+        return testcenter_user
+
+    @property
+    def is_accepted(self):
+        return self.upload_status == TEST_CENTER_STATUS_ACCEPTED
+
+    @property
+    def is_rejected(self):
+        return self.upload_status == TEST_CENTER_STATUS_ERROR
+
+    @property
+    def is_pending(self):
+        return not self.is_accepted and not self.is_rejected
+
+
+class TestCenterUserForm(ModelForm):
+    class Meta:
+        model = TestCenterUser
+        fields = ('first_name', 'middle_name', 'last_name', 'suffix', 'salutation',
+                'address_1', 'address_2', 'address_3', 'city', 'state', 'postal_code', 'country',
+                'phone', 'extension', 'phone_country_code', 'fax', 'fax_country_code', 'company_name')
+
+    def update_and_save(self):
+        new_user = self.save(commit=False)
+        # create additional values here:
+        new_user.user_updated_at = datetime.utcnow()
+        new_user.upload_status = ''
+        new_user.save()
+        log.info("Updated demographic information for user's test center exam registration: username \"{}\" ".format(new_user.user.username))
+
+    # add validation:
+
+    def clean_country(self):
+        code = self.cleaned_data['country']
+        if code and (len(code) != 3 or not code.isalpha()):
+            raise forms.ValidationError(u'Must be three characters (ISO 3166-1):  e.g. USA, CAN, MNG')
+        return code.upper()
+
+    def clean(self):
+        def _can_encode_as_latin(fieldvalue):
+            try:
+                fieldvalue.encode('iso-8859-1')
+            except UnicodeEncodeError:
+                return False
+            return True
+
+        cleaned_data = super(TestCenterUserForm, self).clean()
+
+        # check for interactions between fields:
+        if 'country' in cleaned_data:
+            country = cleaned_data.get('country')
+            if country == 'USA' or country == 'CAN':
+                if 'state' in cleaned_data and len(cleaned_data['state']) == 0:
+                    self._errors['state'] = self.error_class([u'Required if country is USA or CAN.'])
+                    del cleaned_data['state']
+
+                if 'postal_code' in cleaned_data and len(cleaned_data['postal_code']) == 0:
+                    self._errors['postal_code'] = self.error_class([u'Required if country is USA or CAN.'])
+                    del cleaned_data['postal_code']
+
+        if 'fax' in cleaned_data and len(cleaned_data['fax']) > 0 and 'fax_country_code' in cleaned_data and len(cleaned_data['fax_country_code']) == 0:
+            self._errors['fax_country_code'] = self.error_class([u'Required if fax is specified.'])
+            del cleaned_data['fax_country_code']
+
+        # check encoding for all fields:
+        cleaned_data_fields = [fieldname for fieldname in cleaned_data]
+        for fieldname in cleaned_data_fields:
+            if not _can_encode_as_latin(cleaned_data[fieldname]):
+                self._errors[fieldname] = self.error_class([u'Must only use characters in Latin-1 (iso-8859-1) encoding'])
+                del cleaned_data[fieldname]
+
+        # Always return the full collection of cleaned data.
+        return cleaned_data
+
+# our own code to indicate that a request has been rejected.
+ACCOMMODATION_REJECTED_CODE = 'NONE'
+
+ACCOMMODATION_CODES = (
+                      (ACCOMMODATION_REJECTED_CODE, 'No Accommodation Granted'),
+                      ('EQPMNT', 'Equipment'),
+                      ('ET12ET', 'Extra Time - 1/2 Exam Time'),
+                      ('ET30MN', 'Extra Time - 30 Minutes'),
+                      ('ETDBTM', 'Extra Time - Double Time'),
+                      ('SEPRMM', 'Separate Room'),
+                      ('SRREAD', 'Separate Room and Reader'),
+                      ('SRRERC', 'Separate Room and Reader/Recorder'),
+                      ('SRRECR', 'Separate Room and Recorder'),
+                      ('SRSEAN', 'Separate Room and Service Animal'),
+                      ('SRSGNR', 'Separate Room and Sign Language Interpreter'),
+                      )
+
+ACCOMMODATION_CODE_DICT = {code: name for (code, name) in ACCOMMODATION_CODES}
+
+
+class TestCenterRegistration(models.Model):
+    """
+    This is our representation of a user's registration for in-person testing,
+    and specifically for Pearson at this point. A few things to note:
+
+    * Pearson only supports Latin-1, so we have to make sure that the data we
+      capture here will work with that encoding.  This is less of an issue
+      than for the TestCenterUser.
+    * Registrations are only created here when a user registers to take an exam in person.
+
+    The field names and lengths are modeled on the conventions and constraints
+    of Pearson's data import system.
+    """
+    # to find an exam registration, we key off of the user and course_id.
+    # If multiple exams per course are possible, we would also need to add the
+    # exam_series_code.
+    testcenter_user = models.ForeignKey(TestCenterUser, default=None)
+    course_id = models.CharField(max_length=128, db_index=True)
+
+    created_at = models.DateTimeField(auto_now_add=True, db_index=True)
+    updated_at = models.DateTimeField(auto_now=True, db_index=True)
+    # user_updated_at happens only when the user makes a change to their data,
+    # and is something Pearson needs to know to manage updates. Unlike
+    # updated_at, this will not get incremented when we do a batch data import.
+    # The appointment dates, the exam count, and the accommodation codes can be updated,
+    # but hopefully this won't happen often.
+    user_updated_at = models.DateTimeField(db_index=True)
+    # "client_authorization_id" is our unique identifier for the authorization.
+    # This must be present for an update or delete to be sent to Pearson.
+    client_authorization_id = models.CharField(max_length=20, unique=True, db_index=True)
+
+    # information about the test, from the course policy:
+    exam_series_code = models.CharField(max_length=15, db_index=True)
+    eligibility_appointment_date_first = models.DateField(db_index=True)
+    eligibility_appointment_date_last = models.DateField(db_index=True)
+
+    # this is really a list of codes, using an '*' as a delimiter.
+    # So it's not a choice list.  We use the special value of ACCOMMODATION_REJECTED_CODE
+    # to indicate the rejection of an accommodation request.
+    accommodation_code = models.CharField(max_length=64, blank=True)
+
+    # store the original text of the accommodation request.
+    accommodation_request = models.CharField(max_length=1024, blank=True, db_index=True)
+
+    # time at which edX sent the registration to the test center
+    uploaded_at = models.DateTimeField(null=True, db_index=True)
+
+    # confirmation back from the test center, as well as timestamps
+    # on when they processed the request, and when we received
+    # confirmation back.
+    processed_at = models.DateTimeField(null=True, db_index=True)
+    upload_status = models.CharField(max_length=20, blank=True, db_index=True)  # 'Error' or 'Accepted'
+    upload_error_message = models.CharField(max_length=512, blank=True)
+    # Unique ID given to us for this registration by the Testing Center. It's null when
+    # we first create the registration entry, and may be assigned by Pearson later.
+    # (However, it may never be set if we are always initiating such candidate creation.)
+    authorization_id = models.IntegerField(null=True, db_index=True)
+    confirmed_at = models.DateTimeField(null=True, db_index=True)
+
+    @property
+    def candidate_id(self):
+        return self.testcenter_user.candidate_id
+
+    @property
+    def client_candidate_id(self):
+        return self.testcenter_user.client_candidate_id
+
+    @property
+    def authorization_transaction_type(self):
+        if self.authorization_id is not None:
+            return 'Update'
+        elif self.uploaded_at is None:
+            return 'Add'
+        elif self.registration_is_rejected:
+            # Assume that if the registration was rejected before,
+            # it is more likely this is the (first) correction
+            # than a second correction in flight before the first was
+            # processed.
+            return 'Add'
+        else:
+            # TODO: decide what to send when we have uploaded an initial version,
+            # but have not received confirmation back from that upload.  If the
+            # registration here has been changed, then we don't know if this changed
+            # registration should be submitted as an 'add' or an 'update'.
+            #
+            # If the first registration were lost or in error (e.g. bad code),
+            # the second should be an "Add".  If the first were processed successfully,
+            # then the second should be an "Update".  We just don't know....
+            return 'Update'
+
+    @property
+    def exam_authorization_count(self):
+        # Someday this could go in the database (with a default value).  But at present,
+        # we do not expect anyone to be authorized to take an exam more than once.
+        return 1
+
+    @property
+    def needs_uploading(self):
+        return self.uploaded_at is None or self.uploaded_at < self.user_updated_at
+
+    @classmethod
+    def create(cls, testcenter_user, exam, accommodation_request):
+        registration = cls(testcenter_user=testcenter_user)
+        registration.course_id = exam.course_id
+        registration.accommodation_request = accommodation_request.strip()
+        registration.exam_series_code = exam.exam_series_code
+        registration.eligibility_appointment_date_first = strftime("%Y-%m-%d", exam.first_eligible_appointment_date)
+        registration.eligibility_appointment_date_last = strftime("%Y-%m-%d", exam.last_eligible_appointment_date)
+        registration.client_authorization_id = cls._create_client_authorization_id()
+        # accommodation_code remains blank for now, along with Pearson confirmation information
+        return registration
+
+    @staticmethod
+    def _generate_authorization_id():
+        return TestCenterUser._generate_edx_id("edXexam")
+
+    @staticmethod
+    def _create_client_authorization_id():
+        """
+        Return a unique id for a registration, suitable for using as an authorization code
+        for Pearson.  It must fit within 20 characters.
+        """
+        # generate a random value, and check to see if it already is in use here
+        auth_id = TestCenterRegistration._generate_authorization_id()
+        while TestCenterRegistration.objects.filter(client_authorization_id=auth_id).exists():
+            auth_id = TestCenterRegistration._generate_authorization_id()
+        return auth_id
+
+    # methods for providing registration status details on registration page:
+    @property
+    def demographics_is_accepted(self):
+        return self.testcenter_user.is_accepted
+
+    @property
+    def demographics_is_rejected(self):
+        return self.testcenter_user.is_rejected
+
+    @property
+    def demographics_is_pending(self):
+        return self.testcenter_user.is_pending
+
+    @property
+    def accommodation_is_accepted(self):
+        return len(self.accommodation_request) > 0 and len(self.accommodation_code) > 0 and self.accommodation_code != ACCOMMODATION_REJECTED_CODE
+
+    @property
+    def accommodation_is_rejected(self):
+        return len(self.accommodation_request) > 0 and self.accommodation_code == ACCOMMODATION_REJECTED_CODE
+
+    @property
+    def accommodation_is_pending(self):
+        return len(self.accommodation_request) > 0 and len(self.accommodation_code) == 0
+
+    @property
+    def accommodation_is_skipped(self):
+        return len(self.accommodation_request) == 0
+
+    @property
+    def registration_is_accepted(self):
+        return self.upload_status == TEST_CENTER_STATUS_ACCEPTED
+
+    @property
+    def registration_is_rejected(self):
+        return self.upload_status == TEST_CENTER_STATUS_ERROR
+
+    @property
+    def registration_is_pending(self):
+        return not self.registration_is_accepted and not self.registration_is_rejected
+
+    # methods for providing registration status summary on dashboard page:
+    @property
+    def is_accepted(self):
+        return self.registration_is_accepted and self.demographics_is_accepted
+
+    @property
+    def is_rejected(self):
+        return self.registration_is_rejected or self.demographics_is_rejected
+
+    @property
+    def is_pending(self):
+        return not self.is_accepted and not self.is_rejected
+
+    def get_accommodation_codes(self):
+        return self.accommodation_code.split('*')
+
+    def get_accommodation_names(self):
+        return [ACCOMMODATION_CODE_DICT.get(code, "Unknown code " + code) for code in self.get_accommodation_codes()]
+
+    @property
+    def registration_signup_url(self):
+        return settings.PEARSONVUE_SIGNINPAGE_URL
+
+    def demographics_status(self):
+        if self.demographics_is_accepted:
+            return "Accepted"
+        elif self.demographics_is_rejected:
+            return "Rejected"
+        else:
+            return "Pending"
+
+    def accommodation_status(self):
+        if self.accommodation_is_skipped:
+            return "Skipped"
+        elif self.accommodation_is_accepted:
+            return "Accepted"
+        elif self.accommodation_is_rejected:
+            return "Rejected"
+        else:
+            return "Pending"
+
+    def registration_status(self):
+        if self.registration_is_accepted:
+            return "Accepted"
+        elif self.registration_is_rejected:
+            return "Rejected"
+        else:
+            return "Pending"
+
+
+class TestCenterRegistrationForm(ModelForm):
+    class Meta:
+        model = TestCenterRegistration
+        fields = ('accommodation_request', 'accommodation_code')
+
+    def clean_accommodation_request(self):
+        code = self.cleaned_data['accommodation_request']
+        if code and len(code) > 0:
+            return code.strip()
+        return code
+
+    def update_and_save(self):
+        registration = self.save(commit=False)
+        # create additional values here:
+        registration.user_updated_at = datetime.utcnow()
+        registration.upload_status = ''
+        registration.save()
+        log.info("Updated registration information for user's test center exam registration: username \"{}\" course \"{}\", examcode \"{}\"".format(registration.testcenter_user.user.username, registration.course_id, registration.exam_series_code))
+
+    def clean_accommodation_code(self):
+        code = self.cleaned_data['accommodation_code']
+        if code:
+            code = code.upper()
+            codes = code.split('*')
+            for codeval in codes:
+                if codeval not in ACCOMMODATION_CODE_DICT:
+                    raise forms.ValidationError(u'Invalid accommodation code specified: "{}"'.format(codeval))
+        return code
+
+
+
+def get_testcenter_registration(user, course_id, exam_series_code):
+    try:
+        tcu = TestCenterUser.objects.get(user=user)
+    except TestCenterUser.DoesNotExist:
+        return []
+    return TestCenterRegistration.objects.filter(testcenter_user=tcu, course_id=course_id, exam_series_code=exam_series_code)
+
+# nosetests thinks that anything with _test_ in the name is a test.
+# Correct this (https://nose.readthedocs.org/en/latest/finding_tests.html)
+get_testcenter_registration.__test__ = False
+
+
 def unique_id_for_user(user):
     """
     Return a unique id for a user, suitable for inserting into
     e.g. personalized survey links.
     """
-    # include the secret key as a salt, and to make the ids unique accross
-    # different LMS installs.    
+    # include the secret key as a salt, and to make the ids unique across
+    # different LMS installs.
     h = hashlib.md5()
     h.update(settings.SECRET_KEY)
     h.update(str(user.id))
@@ -262,19 +654,40 @@ class CourseEnrollment(models.Model):
         return "[CourseEnrollment] %s: %s (%s)" % (self.user, self.course_id, self.created)
 
 
-@receiver(post_save, sender=CourseEnrollment)
-def assign_default_role(sender, instance, **kwargs):
-    if instance.user.is_staff:
-        role = Role.objects.get_or_create(course_id=instance.course_id, name="Moderator")[0]
-    else:
-        role = Role.objects.get_or_create(course_id=instance.course_id, name="Student")[0]
+class CourseEnrollmentAllowed(models.Model):
+    """
+    Table of users (specified by email address strings) who are allowed to enroll in a specified course.
+    The user may or may not (yet) exist.  Enrollment by users listed in this table is allowed
+    even if the enrollment time window is past.
+    """
+    email = models.CharField(max_length=255, db_index=True)
+    course_id = models.CharField(max_length=255, db_index=True)
 
-    logging.info("assign_default_role: adding %s as %s" % (instance.user, role))
-    instance.user.roles.add(role)
+    created = models.DateTimeField(auto_now_add=True, null=True, db_index=True)
+
+    class Meta:
+        unique_together = (('email', 'course_id'), )
+
+    def __unicode__(self):
+        return "[CourseEnrollmentAllowed] %s: %s (%s)" % (self.email, self.course_id, self.created)
 
 #cache_relation(User.profile)
 
-#### Helper methods for use from python manage.py shell.
+#### Helper methods for use from python manage.py shell and other classes.
+
+
+def get_user_by_username_or_email(username_or_email):
+    """
+    Return a User object, looking up by email if username_or_email contains a
+    '@', otherwise by username.
+
+    Raises:
+        User.DoesNotExist is lookup fails.
+    """
+    if '@' in username_or_email:
+        return User.objects.get(email=username_or_email)
+    else:
+        return User.objects.get(username=username_or_email)
 
 
 def get_user(email):
@@ -364,168 +777,3 @@ def update_user_information(sender, instance, created, **kwargs):
         log = logging.getLogger("mitx.discussion")
         log.error(unicode(e))
         log.error("update user info to discussion failed for user with id: " + str(instance.id))
-
-
-########################## REPLICATION SIGNALS #################################
-# @receiver(post_save, sender=User)
-def replicate_user_save(sender, **kwargs):
-    user_obj = kwargs['instance']
-    if not should_replicate(user_obj):
-        return
-    for course_db_name in db_names_to_replicate_to(user_obj.id):
-        replicate_user(user_obj, course_db_name)
-
-
-# @receiver(post_save, sender=CourseEnrollment)
-def replicate_enrollment_save(sender, **kwargs):
-    """This is called when a Student enrolls in a course. It has to do the
-    following:
-
-    1. Make sure the User is copied into the Course DB. It may already exist
-       (someone deleting and re-adding a course). This has to happen first or
-       the foreign key constraint breaks.
-    2. Replicate the CourseEnrollment.
-    3. Replicate the UserProfile.
-    """
-    if not is_portal():
-        return
-
-    enrollment_obj = kwargs['instance']
-    log.debug("Replicating user because of new enrollment")
-    for course_db_name in db_names_to_replicate_to(enrollment_obj.user.id):
-        replicate_user(enrollment_obj.user, course_db_name)
-
-    log.debug("Replicating enrollment because of new enrollment")
-    replicate_model(CourseEnrollment.save, enrollment_obj, enrollment_obj.user_id)
-
-    log.debug("Replicating user profile because of new enrollment")
-    user_profile = UserProfile.objects.get(user_id=enrollment_obj.user_id)
-    replicate_model(UserProfile.save, user_profile, enrollment_obj.user_id)
-
-
-# @receiver(post_delete, sender=CourseEnrollment)
-def replicate_enrollment_delete(sender, **kwargs):
-    enrollment_obj = kwargs['instance']
-    return replicate_model(CourseEnrollment.delete, enrollment_obj, enrollment_obj.user_id)
-
-
-# @receiver(post_save, sender=UserProfile)
-def replicate_userprofile_save(sender, **kwargs):
-    """We just updated the UserProfile (say an update to the name), so push that
-    change to all Course DBs that we're enrolled in."""
-    user_profile_obj = kwargs['instance']
-    return replicate_model(UserProfile.save, user_profile_obj, user_profile_obj.user_id)
-
-
-######### Replication functions #########
-USER_FIELDS_TO_COPY = ["id", "username", "first_name", "last_name", "email",
-                       "password", "is_staff", "is_active", "is_superuser",
-                       "last_login", "date_joined"]
-
-
-def replicate_user(portal_user, course_db_name):
-    """Replicate a User to the correct Course DB. This is more complicated than
-    it should be because Askbot extends the auth_user table and adds its own
-    fields. So we need to only push changes to the standard fields and leave
-    the rest alone so that Askbot changes at the Course DB level don't get
-    overridden.
-    """
-    try:
-        course_user = User.objects.using(course_db_name).get(id=portal_user.id)
-        log.debug("User {0} found in Course DB, replicating fields to {1}"
-                  .format(course_user, course_db_name))
-    except User.DoesNotExist:
-        log.debug("User {0} not found in Course DB, creating copy in {1}"
-                  .format(portal_user, course_db_name))
-        course_user = User()
-
-    for field in USER_FIELDS_TO_COPY:
-        setattr(course_user, field, getattr(portal_user, field))
-
-    mark_handled(course_user)
-    course_user.save(using=course_db_name)
-    unmark(course_user)
-
-
-def replicate_model(model_method, instance, user_id):
-    """
-    model_method is the model action that we want replicated. For instance,
-                 UserProfile.save
-    """
-    if not should_replicate(instance):
-        return
-
-    course_db_names = db_names_to_replicate_to(user_id)
-    log.debug("Replicating {0} for user {1} to DBs: {2}"
-              .format(model_method, user_id, course_db_names))
-
-    mark_handled(instance)
-    for db_name in course_db_names:
-        model_method(instance, using=db_name)
-    unmark(instance)
-
-
-######### Replication Helpers #########
-
-
-def is_valid_course_id(course_id):
-    """Right now, the only database that's not a course database is 'default'.
-    I had nicer checking in here originally -- it would scan the courses that
-    were in the system and only let you choose that. But it was annoying to run
-    tests with, since we don't have course data for some for our course test
-    databases. Hence the lazy version.
-    """
-    return course_id != 'default'
-
-
-def is_portal():
-    """Are we in the portal pool? Only Portal servers are allowed to replicate
-    their changes. For now, only Portal servers see multiple DBs, so we use
-    that to decide."""
-    return len(settings.DATABASES) > 1
-
-
-def db_names_to_replicate_to(user_id):
-    """Return a list of DB names that this user_id is enrolled in."""
-    return [c.course_id
-            for c in CourseEnrollment.objects.filter(user_id=user_id)
-            if is_valid_course_id(c.course_id)]
-
-
-def marked_handled(instance):
-    """Have we marked this instance as being handled to avoid infinite loops
-    caused by saving models in post_save hooks for the same models?"""
-    return hasattr(instance, '_do_not_copy_to_course_db') and instance._do_not_copy_to_course_db
-
-
-def mark_handled(instance):
-    """You have to mark your instance with this function or else we'll go into
-    an infinite loop since we're putting listeners on Model saves/deletes and
-    the act of replication requires us to call the same model method.
-
-    We create a _replicated attribute to differentiate the first save of this
-    model vs. the duplicate save we force on to the course database. Kind of
-    a hack -- suggestions welcome.
-    """
-    instance._do_not_copy_to_course_db = True
-
-
-def unmark(instance):
-    """If we don't unmark a model after we do replication, then consecutive
-    save() calls won't be properly replicated."""
-    instance._do_not_copy_to_course_db = False
-
-
-def should_replicate(instance):
-    """Should this instance be replicated? We need to be a Portal server and
-    the instance has to not have been marked_handled."""
-    if marked_handled(instance):
-        # Basically, avoid an infinite loop. You should
-        log.debug("{0} should not be replicated because it's been marked"
-                  .format(instance))
-        return False
-    if not is_portal():
-        log.debug("{0} should not be replicated because we're not a portal."
-                  .format(instance))
-        return False
-    return True

common/djangoapps/student/tests.py

@@ -1,290 +0,0 @@
-"""
-This file demonstrates writing tests using the unittest module. These will pass
-when you run "manage.py test".
-
-Replace this with more appropriate tests for your application.
-"""
-import logging
-from datetime import datetime
-from hashlib import sha1
-
-from django.test import TestCase
-from mock import patch, Mock
-from nose.plugins.skip import SkipTest
-
-from .models import (User, UserProfile, CourseEnrollment,
-                     replicate_user, USER_FIELDS_TO_COPY,
-                     unique_id_for_user)
-from .views import process_survey_link, _cert_info
-
-COURSE_1 = 'edX/toy/2012_Fall'
-COURSE_2 = 'edx/full/6.002_Spring_2012'
-
-log = logging.getLogger(__name__)
-
-class ReplicationTest(TestCase):
-
-    multi_db = True
-
-    def test_user_replication(self):
-        """Test basic user replication."""
-        raise SkipTest()
-        portal_user = User.objects.create_user('rusty', 'rusty@edx.org', 'fakepass')
-        portal_user.first_name='Rusty'
-        portal_user.last_name='Skids'
-        portal_user.is_staff=True
-        portal_user.is_active=True
-        portal_user.is_superuser=True
-        portal_user.last_login=datetime(2012, 1, 1)
-        portal_user.date_joined=datetime(2011, 1, 1)
-        # This is an Askbot field and will break if askbot is not included
-
-        if hasattr(portal_user, 'seen_response_count'):
-            portal_user.seen_response_count = 10
-
-        portal_user.save(using='default')
-
-        # We replicate this user to Course 1, then pull the same user and verify
-        # that the fields copied over properly.
-        replicate_user(portal_user, COURSE_1)
-        course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
-
-        # Make sure the fields we care about got copied over for this user.
-        for field in USER_FIELDS_TO_COPY:
-            self.assertEqual(getattr(portal_user, field),
-                             getattr(course_user, field),
-                             "{0} not copied from {1} to {2}".format(
-                                 field, portal_user, course_user
-                             ))
-
-        # This hasattr lameness is here because we don't want this test to be
-        # triggered when we're being run by CMS tests (Askbot doesn't exist
-        # there, so the test will fail).
-        #
-        # seen_response_count isn't a field we care about, so it shouldn't have
-        # been copied over.
-        if hasattr(portal_user, 'seen_response_count'):
-            portal_user.seen_response_count = 20
-            replicate_user(portal_user, COURSE_1)
-            course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
-            self.assertEqual(portal_user.seen_response_count, 20)
-            self.assertEqual(course_user.seen_response_count, 0)
-
-        # Another replication should work for an email change however, since
-        # it's a field we care about.
-        portal_user.email = "clyde@edx.org"
-        replicate_user(portal_user, COURSE_1)
-        course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
-        self.assertEqual(portal_user.email, course_user.email)
-
-        # During this entire time, the user data should never have made it over
-        # to COURSE_2
-        self.assertRaises(User.DoesNotExist,
-                          User.objects.using(COURSE_2).get,
-                          id=portal_user.id)
-
-
-    def test_enrollment_for_existing_user_info(self):
-        """Test the effect of Enrolling in a class if you've already got user
-        data to be copied over."""
-        raise SkipTest()
-        # Create our User
-        portal_user = User.objects.create_user('jack', 'jack@edx.org', 'fakepass')
-        portal_user.first_name = "Jack"
-        portal_user.save()
-
-        # Set up our UserProfile info
-        portal_user_profile = UserProfile.objects.create(
-                                  user=portal_user,
-                                  name="Jack Foo",
-                                  level_of_education=None,
-                                  gender='m',
-                                  mailing_address=None,
-                                  goals="World domination",
-                              )
-        portal_user_profile.save()
-
-        # Now let's see if creating a CourseEnrollment copies all the relevant
-        # data.
-        portal_enrollment = CourseEnrollment.objects.create(user=portal_user,
-                                                            course_id=COURSE_1)
-        portal_enrollment.save()
-
-        # Grab all the copies we expect
-        course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
-        self.assertEquals(portal_user, course_user)
-        self.assertRaises(User.DoesNotExist,
-                          User.objects.using(COURSE_2).get,
-                          id=portal_user.id)
-
-        course_enrollment = CourseEnrollment.objects.using(COURSE_1).get(id=portal_enrollment.id)
-        self.assertEquals(portal_enrollment, course_enrollment)
-        self.assertRaises(CourseEnrollment.DoesNotExist,
-                          CourseEnrollment.objects.using(COURSE_2).get,
-                          id=portal_enrollment.id)
-
-        course_user_profile = UserProfile.objects.using(COURSE_1).get(id=portal_user_profile.id)
-        self.assertEquals(portal_user_profile, course_user_profile)
-        self.assertRaises(UserProfile.DoesNotExist,
-                          UserProfile.objects.using(COURSE_2).get,
-                          id=portal_user_profile.id)
-
-        log.debug("Make sure our seen_response_count is not replicated.")
-        if hasattr(portal_user, 'seen_response_count'):
-            portal_user.seen_response_count = 200
-            course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
-            self.assertEqual(portal_user.seen_response_count, 200)
-            self.assertEqual(course_user.seen_response_count, 0)
-            portal_user.save()
-
-            course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
-            self.assertEqual(portal_user.seen_response_count, 200)
-            self.assertEqual(course_user.seen_response_count, 0)
-
-            portal_user.email = 'jim@edx.org'
-            portal_user.save()
-            course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
-            self.assertEqual(portal_user.email, 'jim@edx.org')
-            self.assertEqual(course_user.email, 'jim@edx.org')
-
-
-
-    def test_enrollment_for_user_info_after_enrollment(self):
-        """Test the effect of modifying User data after you've enrolled."""
-        raise SkipTest()
-
-        # Create our User
-        portal_user = User.objects.create_user('patty', 'patty@edx.org', 'fakepass')
-        portal_user.first_name = "Patty"
-        portal_user.save()
-
-        # Set up our UserProfile info
-        portal_user_profile = UserProfile.objects.create(
-                                  user=portal_user,
-                                  name="Patty Foo",
-                                  level_of_education=None,
-                                  gender='f',
-                                  mailing_address=None,
-                                  goals="World peace",
-                              )
-        portal_user_profile.save()
-
-        # Now let's see if creating a CourseEnrollment copies all the relevant
-        # data when things are saved.
-        portal_enrollment = CourseEnrollment.objects.create(user=portal_user,
-                                                            course_id=COURSE_1)
-        portal_enrollment.save()
-
-        portal_user.last_name = "Bar"
-        portal_user.save()
-        portal_user_profile.gender = 'm'
-        portal_user_profile.save()
-
-        # Grab all the copies we expect, and make sure it doesn't end up in
-        # places we don't expect.
-        course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
-        self.assertEquals(portal_user, course_user)
-        self.assertRaises(User.DoesNotExist,
-                          User.objects.using(COURSE_2).get,
-                          id=portal_user.id)
-
-        course_enrollment = CourseEnrollment.objects.using(COURSE_1).get(id=portal_enrollment.id)
-        self.assertEquals(portal_enrollment, course_enrollment)
-        self.assertRaises(CourseEnrollment.DoesNotExist,
-                          CourseEnrollment.objects.using(COURSE_2).get,
-                          id=portal_enrollment.id)
-
-        course_user_profile = UserProfile.objects.using(COURSE_1).get(id=portal_user_profile.id)
-        self.assertEquals(portal_user_profile, course_user_profile)
-        self.assertRaises(UserProfile.DoesNotExist,
-                          UserProfile.objects.using(COURSE_2).get,
-                          id=portal_user_profile.id)
-
-
-class CourseEndingTest(TestCase):
-    """Test things related to course endings: certificates, surveys, etc"""
-
-    def test_process_survey_link(self):
-        username = "fred"
-        user = Mock(username=username)
-        id = unique_id_for_user(user)
-        link1 = "http://www.mysurvey.com"
-        self.assertEqual(process_survey_link(link1, user), link1)
-
-        link2 = "http://www.mysurvey.com?unique={UNIQUE_ID}"
-        link2_expected = "http://www.mysurvey.com?unique={UNIQUE_ID}".format(UNIQUE_ID=id)
-        self.assertEqual(process_survey_link(link2, user), link2_expected)
-
-    def test_cert_info(self):
-        user = Mock(username="fred")
-        survey_url = "http://a_survey.com"
-        course = Mock(end_of_course_survey_url=survey_url)
-
-        self.assertEqual(_cert_info(user, course, None),
-                         {'status': 'processing',
-                          'show_disabled_download_button': False,
-                          'show_download_url': False,
-                          'show_survey_button': False,})
-
-        cert_status = {'status': 'unavailable'}
-        self.assertEqual(_cert_info(user, course, cert_status),
-                         {'status': 'processing',
-                          'show_disabled_download_button': False,
-                          'show_download_url': False,
-                          'show_survey_button': False})
-
-        cert_status = {'status': 'generating', 'grade': '67'}
-        self.assertEqual(_cert_info(user, course, cert_status),
-                         {'status': 'generating',
-                          'show_disabled_download_button': True,
-                          'show_download_url': False,
-                          'show_survey_button': True,
-                          'survey_url': survey_url,
-                          'grade': '67'
-                          })
-
-        cert_status = {'status': 'regenerating', 'grade': '67'}
-        self.assertEqual(_cert_info(user, course, cert_status),
-                         {'status': 'generating',
-                          'show_disabled_download_button': True,
-                          'show_download_url': False,
-                          'show_survey_button': True,
-                          'survey_url': survey_url,
-                          'grade': '67'
-                          })
-
-        download_url = 'http://s3.edx/cert'
-        cert_status = {'status': 'downloadable', 'grade': '67',
-                       'download_url': download_url}
-        self.assertEqual(_cert_info(user, course, cert_status),
-                         {'status': 'ready',
-                          'show_disabled_download_button': False,
-                          'show_download_url': True,
-                          'download_url': download_url,
-                          'show_survey_button': True,
-                          'survey_url': survey_url,
-                          'grade': '67'
-                          })
-
-        cert_status = {'status': 'notpassing', 'grade': '67',
-                       'download_url': download_url}
-        self.assertEqual(_cert_info(user, course, cert_status),
-                         {'status': 'notpassing',
-                          'show_disabled_download_button': False,
-                          'show_download_url': False,
-                          'show_survey_button': True,
-                          'survey_url': survey_url,
-                          'grade': '67'
-                          })
-
-        # Test a course that doesn't have a survey specified
-        course2 = Mock(end_of_course_survey_url=None)
-        cert_status = {'status': 'notpassing', 'grade': '67',
-                       'download_url': download_url}
-        self.assertEqual(_cert_info(user, course2, cert_status),
-                         {'status': 'notpassing',
-                          'show_disabled_download_button': False,
-                          'show_download_url': False,
-                          'show_survey_button': False,
-                          'grade': '67'
-                          })

common/djangoapps/student/tests/factories.py

@@ -0,0 +1,59 @@
+from student.models import (User, UserProfile, Registration,
+                            CourseEnrollmentAllowed, CourseEnrollment)
+from django.contrib.auth.models import Group
+from datetime import datetime
+from factory import Factory, SubFactory
+from uuid import uuid4
+
+
+class GroupFactory(Factory):
+    FACTORY_FOR = Group
+
+    name = 'staff_MITx/999/Robot_Super_Course'
+
+
+class UserProfileFactory(Factory):
+    FACTORY_FOR = UserProfile
+
+    user = None
+    name = 'Robot Test'
+    level_of_education = None
+    gender = 'm'
+    mailing_address = None
+    goals = 'World domination'
+
+
+class RegistrationFactory(Factory):
+    FACTORY_FOR = Registration
+
+    user = None
+    activation_key = uuid4().hex
+
+
+class UserFactory(Factory):
+    FACTORY_FOR = User
+
+    username = 'robot'
+    email = 'robot+test@edx.org'
+    password = 'test'
+    first_name = 'Robot'
+    last_name = 'Test'
+    is_staff = False
+    is_active = True
+    is_superuser = False
+    last_login = datetime(2012, 1, 1)
+    date_joined = datetime(2011, 1, 1)
+
+
+class CourseEnrollmentFactory(Factory):
+    FACTORY_FOR = CourseEnrollment
+
+    user = SubFactory(UserFactory)
+    course_id = 'edX/toy/2012_Fall'
+
+
+class CourseEnrollmentAllowedFactory(Factory):
+    FACTORY_FOR = CourseEnrollmentAllowed
+
+    email = 'test@edx.org'
+    course_id = 'edX/test/2012_Fall'

common/djangoapps/student/tests/tests.py

@@ -0,0 +1,107 @@
+"""
+This file demonstrates writing tests using the unittest module. These will pass
+when you run "manage.py test".
+
+Replace this with more appropriate tests for your application.
+"""
+import logging
+
+from django.test import TestCase
+from mock import Mock
+
+from student.models import unique_id_for_user
+from student.views import process_survey_link, _cert_info
+
+COURSE_1 = 'edX/toy/2012_Fall'
+COURSE_2 = 'edx/full/6.002_Spring_2012'
+
+log = logging.getLogger(__name__)
+
+
+class CourseEndingTest(TestCase):
+    """Test things related to course endings: certificates, surveys, etc"""
+
+    def test_process_survey_link(self):
+        username = "fred"
+        user = Mock(username=username)
+        id = unique_id_for_user(user)
+        link1 = "http://www.mysurvey.com"
+        self.assertEqual(process_survey_link(link1, user), link1)
+
+        link2 = "http://www.mysurvey.com?unique={UNIQUE_ID}"
+        link2_expected = "http://www.mysurvey.com?unique={UNIQUE_ID}".format(UNIQUE_ID=id)
+        self.assertEqual(process_survey_link(link2, user), link2_expected)
+
+    def test_cert_info(self):
+        user = Mock(username="fred")
+        survey_url = "http://a_survey.com"
+        course = Mock(end_of_course_survey_url=survey_url)
+
+        self.assertEqual(_cert_info(user, course, None),
+                         {'status': 'processing',
+                          'show_disabled_download_button': False,
+                          'show_download_url': False,
+                          'show_survey_button': False, })
+
+        cert_status = {'status': 'unavailable'}
+        self.assertEqual(_cert_info(user, course, cert_status),
+                         {'status': 'processing',
+                          'show_disabled_download_button': False,
+                          'show_download_url': False,
+                          'show_survey_button': False})
+
+        cert_status = {'status': 'generating', 'grade': '67'}
+        self.assertEqual(_cert_info(user, course, cert_status),
+                         {'status': 'generating',
+                          'show_disabled_download_button': True,
+                          'show_download_url': False,
+                          'show_survey_button': True,
+                          'survey_url': survey_url,
+                          'grade': '67'
+                          })
+
+        cert_status = {'status': 'regenerating', 'grade': '67'}
+        self.assertEqual(_cert_info(user, course, cert_status),
+                         {'status': 'generating',
+                          'show_disabled_download_button': True,
+                          'show_download_url': False,
+                          'show_survey_button': True,
+                          'survey_url': survey_url,
+                          'grade': '67'
+                          })
+
+        download_url = 'http://s3.edx/cert'
+        cert_status = {'status': 'downloadable', 'grade': '67',
+                       'download_url': download_url}
+        self.assertEqual(_cert_info(user, course, cert_status),
+                         {'status': 'ready',
+                          'show_disabled_download_button': False,
+                          'show_download_url': True,
+                          'download_url': download_url,
+                          'show_survey_button': True,
+                          'survey_url': survey_url,
+                          'grade': '67'
+                          })
+
+        cert_status = {'status': 'notpassing', 'grade': '67',
+                       'download_url': download_url}
+        self.assertEqual(_cert_info(user, course, cert_status),
+                         {'status': 'notpassing',
+                          'show_disabled_download_button': False,
+                          'show_download_url': False,
+                          'show_survey_button': True,
+                          'survey_url': survey_url,
+                          'grade': '67'
+                          })
+
+        # Test a course that doesn't have a survey specified
+        course2 = Mock(end_of_course_survey_url=None)
+        cert_status = {'status': 'notpassing', 'grade': '67',
+                       'download_url': download_url}
+        self.assertEqual(_cert_info(user, course2, cert_status),
+                         {'status': 'notpassing',
+                          'show_disabled_download_button': False,
+                          'show_download_url': False,
+                          'show_survey_button': False,
+                          'grade': '67'
+                          })

common/djangoapps/student/views.py

@@ -1,53 +1,58 @@
 import datetime
 import feedparser
-import itertools
 import json
 import logging
 import random
 import string
 import sys
-import time
 import urllib
 import uuid
 
+
 from django.conf import settings
 from django.contrib.auth import logout, authenticate, login
 from django.contrib.auth.forms import PasswordResetForm
 from django.contrib.auth.models import User
 from django.contrib.auth.decorators import login_required
+from django.core.cache import cache
 from django.core.context_processors import csrf
 from django.core.mail import send_mail
+from django.core.urlresolvers import reverse
 from django.core.validators import validate_email, validate_slug, ValidationError
 from django.db import IntegrityError
-from django.http import HttpResponse, HttpResponseForbidden, Http404
+from django.http import HttpResponse, HttpResponseRedirect, Http404
 from django.shortcuts import redirect
+from django_future.csrf import ensure_csrf_cookie, csrf_exempt
+
 from mitxmako.shortcuts import render_to_response, render_to_string
 from bs4 import BeautifulSoup
-from django.core.cache import cache
 
-from django_future.csrf import ensure_csrf_cookie, csrf_exempt
-from student.models import (Registration, UserProfile,
+from student.models import (Registration, UserProfile, TestCenterUser, TestCenterUserForm,
+                            TestCenterRegistration, TestCenterRegistrationForm,
                             PendingNameChange, PendingEmailChange,
-                            CourseEnrollment, unique_id_for_user)
+                            CourseEnrollment, unique_id_for_user,
+                            get_testcenter_registration)
 
 from certificates.models import CertificateStatuses, certificate_status_for_student
 
 from xmodule.course_module import CourseDescriptor
 from xmodule.modulestore.exceptions import ItemNotFoundError
 from xmodule.modulestore.django import modulestore
-from xmodule.modulestore.exceptions import ItemNotFoundError
+from xmodule.modulestore import Location
 
-from datetime import date
 from collections import namedtuple
 
-from courseware.courses import get_courses_by_university
+from courseware.courses import get_courses, sort_by_announcement
 from courseware.access import has_access
+from courseware.views import get_module_for_descriptor, jump_to
+from courseware.model_data import ModelDataCache
 
 from statsd import statsd
 
 log = logging.getLogger("mitx.student")
 Article = namedtuple('Article', 'title url author image deck publication publish_date')
 
+
 def csrf_token(context):
     ''' A csrf token that can be included in a form.
     '''
@@ -71,19 +76,21 @@ def index(request, extra_context={}, user=None):
     '''
 
     # The course selection work is done in courseware.courses.
-    domain = settings.MITX_FEATURES.get('FORCE_UNIVERSITY_DOMAIN')	# normally False
-    if domain==False:				# do explicit check, because domain=None is valid
+    domain = settings.MITX_FEATURES.get('FORCE_UNIVERSITY_DOMAIN')  	# normally False
+    if domain == False:				# do explicit check, because domain=None is valid
         domain = request.META.get('HTTP_HOST')
-    universities = get_courses_by_university(None,
-                                             domain=domain)
+
+    courses = get_courses(None, domain=domain)
+    courses = sort_by_announcement(courses)
 
     # Get the 3 most recent news
     top_news = _get_news(top=3)
 
-    context = {'universities': universities, 'news': top_news}
+    context = {'courses': courses, 'news': top_news}
     context.update(extra_context)
     return render_to_response('index.html', context)
 
+
 def course_from_id(course_id):
     """Return the CourseDescriptor corresponding to this course_id"""
     course_loc = CourseDescriptor.id_to_location(course_id)
@@ -93,6 +100,7 @@ import re
 day_pattern = re.compile('\s\d+,\s')
 multimonth_pattern = re.compile('\s?\-\s?\S+\s')
 
+
 def get_date_for_press(publish_date):
     import datetime
     # strip off extra months, and just use the first:
@@ -103,9 +111,10 @@ def get_date_for_press(publish_date):
         date = datetime.datetime.strptime(date, "%B, %Y")
     return date
 
+
 def press(request):
     json_articles = cache.get("student_press_json_articles")
-    if json_articles == None:
+    if json_articles is None:
         if hasattr(settings, 'RSS_URL'):
             content = urllib.urlopen(settings.PRESS_URL).read()
             json_articles = json.loads(content)
@@ -131,7 +140,7 @@ def cert_info(user, course):
     Get the certificate info needed to render the dashboard section for the given
     student and course.  Returns a dictionary with keys:
 
-    'status': one of 'generating', 'ready', 'notpassing', 'processing'
+    'status': one of 'generating', 'ready', 'notpassing', 'processing', 'restricted'
     'show_download_url': bool
     'download_url': url, only present if show_download_url is True
     'show_disabled_download_button': bool -- true if state is 'generating'
@@ -144,6 +153,7 @@ def cert_info(user, course):
 
     return _cert_info(user, course, certificate_status_for_student(user, course.id))
 
+
 def _cert_info(user, course, cert_status):
     """
     Implements the logic for cert_info -- split out for testing.
@@ -164,15 +174,16 @@ def _cert_info(user, course, cert_status):
         CertificateStatuses.regenerating: 'generating',
         CertificateStatuses.downloadable: 'ready',
         CertificateStatuses.notpassing: 'notpassing',
+        CertificateStatuses.restricted: 'restricted',
         }
 
     status = template_state.get(cert_status['status'], default_status)
 
     d = {'status': status,
          'show_download_url': status == 'ready',
-         'show_disabled_download_button': status == 'generating',}
+         'show_disabled_download_button': status == 'generating', }
 
-    if (status in ('generating', 'ready', 'notpassing') and
+    if (status in ('generating', 'ready', 'notpassing', 'restricted') and
         course.end_of_course_survey_url is not None):
         d.update({
          'show_survey_button': True,
@@ -188,7 +199,7 @@ def _cert_info(user, course, cert_status):
         else:
             d['download_url'] = cert_status['download_url']
 
-    if status in ('generating', 'ready', 'notpassing'):
+    if status in ('generating', 'ready', 'notpassing', 'restricted'):
         if 'grade' not in cert_status:
             # Note: as of 11/20/2012, we know there are students in this state-- cs169.1x,
             # who need to be regraded (we weren't tracking 'notpassing' at first).
@@ -199,6 +210,7 @@ def _cert_info(user, course, cert_status):
 
     return d
 
+
 @login_required
 @ensure_csrf_cookie
 def dashboard(request):
@@ -232,7 +244,9 @@ def dashboard(request):
     show_courseware_links_for = frozenset(course.id for course in courses
                                           if has_access(request.user, course, 'load'))
 
-    cert_statuses = { course.id: cert_info(request.user, course) for course in courses}
+    cert_statuses = {course.id: cert_info(request.user, course) for course in courses}
+
+    exam_registrations = {course.id: exam_registration_info(request.user, course) for course in courses}
 
     # Get the 3 most recent news
     top_news = _get_news(top=3)
@@ -241,9 +255,10 @@ def dashboard(request):
                'message': message,
                'staff_access': staff_access,
                'errored_courses': errored_courses,
-               'show_courseware_links_for' : show_courseware_links_for,
+               'show_courseware_links_for': show_courseware_links_for,
                'cert_statuses': cert_statuses,
                'news': top_news,
+               'exam_registrations': exam_registrations,
                }
 
     return render_to_response('dashboard.html', context)
@@ -285,7 +300,7 @@ def change_enrollment(request):
     action = request.POST.get("enrollment_action", "")
 
     course_id = request.POST.get("course_id", None)
-    if course_id == None:
+    if course_id is None:
         return HttpResponse(json.dumps({'success': False,
                                         'error': 'There was an error receiving the course id.'}))
 
@@ -295,22 +310,27 @@ def change_enrollment(request):
         try:
             course = course_from_id(course_id)
         except ItemNotFoundError:
-            log.warning("User {0} tried to enroll in non-existant course {1}"
-                      .format(user.username, enrollment.course_id))
+            log.warning("User {0} tried to enroll in non-existent course {1}"
+                      .format(user.username, course_id))
             return {'success': False, 'error': 'The course requested does not exist.'}
 
         if not has_access(user, course, 'enroll'):
             return {'success': False,
                     'error': 'enrollment in {} not allowed at this time'
-                    .format(course.display_name)}
+                    .format(course.display_name_with_default)}
 
-        org, course_num, run=course_id.split("/")
+        org, course_num, run = course_id.split("/")
         statsd.increment("common.student.enrollment",
                         tags=["org:{0}".format(org),
                               "course:{0}".format(course_num),
                               "run:{0}".format(run)])
 
-        enrollment, created = CourseEnrollment.objects.get_or_create(user=user, course_id=course.id)
+        try:
+            enrollment, created = CourseEnrollment.objects.get_or_create(user=user, course_id=course.id)
+        except IntegrityError:
+            # If we've already created this enrollment in a separate transaction,
+            # then just continue
+            pass
         return {'success': True}
 
     elif action == "unenroll":
@@ -318,7 +338,7 @@ def change_enrollment(request):
             enrollment = CourseEnrollment.objects.get(user=user, course_id=course_id)
             enrollment.delete()
 
-            org, course_num, run=course_id.split("/")
+            org, course_num, run = course_id.split("/")
             statsd.increment("common.student.unenrollment",
                             tags=["org:{0}".format(org),
                                   "course:{0}".format(course_num),
@@ -333,6 +353,14 @@ def change_enrollment(request):
     return {'success': False, 'error': 'We weren\'t able to unenroll you. Please try again.'}
 
 
+@ensure_csrf_cookie
+def accounts_login(request, error=""):
+
+
+    return render_to_response('accounts_login.html', {'error': error})
+
+
+
 # Need different levels of logging
 @ensure_csrf_cookie
 def login_user(request, error=""):
@@ -346,14 +374,14 @@ def login_user(request, error=""):
     try:
         user = User.objects.get(email=email)
     except User.DoesNotExist:
-        log.warning("Login failed - Unknown user email: {0}".format(email))
+        log.warning(u"Login failed - Unknown user email: {0}".format(email))
         return HttpResponse(json.dumps({'success': False,
                                         'value': 'Email or password is incorrect.'}))  # TODO: User error message
 
     username = user.username
     user = authenticate(username=username, password=password)
     if user is None:
-        log.warning("Login failed - password for {0} is invalid".format(email))
+        log.warning(u"Login failed - password for {0} is invalid".format(email))
         return HttpResponse(json.dumps({'success': False,
                                         'value': 'Email or password is incorrect.'}))
 
@@ -361,7 +389,7 @@ def login_user(request, error=""):
         try:
             login(request, user)
             if request.POST.get('remember') == 'true':
-                request.session.set_expiry(None)  # or change to 604800 for 7 days
+                request.session.set_expiry(604800)
                 log.debug("Setting user session to never expire")
             else:
                 request.session.set_expiry(0)
@@ -369,7 +397,7 @@ def login_user(request, error=""):
             log.critical("Login failed - Could not create session. Is memcached running?")
             log.exception(e)
 
-        log.info("Login success - {0} ({1})".format(username, email))
+        log.info(u"Login success - {0} ({1})".format(username, email))
 
         try_change_enrollment(request)
 
@@ -377,7 +405,7 @@ def login_user(request, error=""):
 
         return HttpResponse(json.dumps({'success': True}))
 
-    log.warning("Login failed - Account not active for user {0}, resending activation".format(username))
+    log.warning(u"Login failed - Account not active for user {0}, resending activation".format(username))
 
     reactivation_email_for_user(user)
     not_activated_msg = "This account has not been activated. We have " + \
@@ -408,6 +436,7 @@ def change_setting(request):
     return HttpResponse(json.dumps({'success': True,
                                     'location': up.location, }))
 
+
 def _do_create_account(post_vars):
     """
     Given cleaned post variables, create the User and UserProfile objects, as well as the
@@ -453,8 +482,9 @@ def _do_create_account(post_vars):
     try:
         profile.year_of_birth = int(post_vars['year_of_birth'])
     except (ValueError, KeyError):
-        profile.year_of_birth = None  # If they give us garbage, just ignore it instead
-                                # of asking them to put an integer.
+        # If they give us garbage, just ignore it instead
+        # of asking them to put an integer.
+        profile.year_of_birth = None
     try:
         profile.save()
     except Exception:
@@ -528,13 +558,13 @@ def create_account(request, post_override=None):
     try:
         validate_slug(post_vars['username'])
     except ValidationError:
-        js['value'] = "Username should only consist of A-Z and 0-9.".format(field=a)
+        js['value'] = "Username should only consist of A-Z and 0-9, with no spaces.".format(field=a)
         js['field'] = 'username'
         return HttpResponse(json.dumps(js))
 
     # Ok, looks like everything is legit.  Create the account.
     ret = _do_create_account(post_vars)
-    if isinstance(ret,HttpResponse):		# if there was an error then return that
+    if isinstance(ret, HttpResponse):		# if there was an error then return that
         return ret
     (user, profile, registration) = ret
 
@@ -574,7 +604,7 @@ def create_account(request, post_override=None):
         eamap.user = login_user
         eamap.dtsignup = datetime.datetime.now()
         eamap.save()
-        log.debug('Updated ExternalAuthMap for %s to be %s' % (post_vars['username'],eamap))
+        log.debug('Updated ExternalAuthMap for %s to be %s' % (post_vars['username'], eamap))
 
         if settings.MITX_FEATURES.get('BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'):
             log.debug('bypassing activation email')
@@ -587,6 +617,175 @@ def create_account(request, post_override=None):
     return HttpResponse(json.dumps(js), mimetype="application/json")
 
 
+def exam_registration_info(user, course):
+    """ Returns a Registration object if the user is currently registered for a current
+    exam of the course.  Returns None if the user is not registered, or if there is no
+    current exam for the course.
+    """
+    exam_info = course.current_test_center_exam
+    if exam_info is None:
+        return None
+
+    exam_code = exam_info.exam_series_code
+    registrations = get_testcenter_registration(user, course.id, exam_code)
+    if registrations:
+        registration = registrations[0]
+    else:
+        registration = None
+    return registration
+
+
+@login_required
+@ensure_csrf_cookie
+def begin_exam_registration(request, course_id):
+    """ Handles request to register the user for the current
+    test center exam of the specified course.  Called by form
+    in dashboard.html.
+    """
+    user = request.user
+
+    try:
+        course = course_from_id(course_id)
+    except ItemNotFoundError:
+        log.error("User {0} enrolled in non-existent course {1}".format(user.username, course_id))
+        raise Http404
+
+    # get the exam to be registered for:
+    # (For now, we just assume there is one at most.)
+    # if there is no exam now (because someone bookmarked this stupid page),
+    # then return a 404:
+    exam_info = course.current_test_center_exam
+    if exam_info is None:
+        raise Http404
+
+    # determine if the user is registered for this course:
+    registration = exam_registration_info(user, course)
+
+    # we want to populate the registration page with the relevant information,
+    # if it already exists.  Create an empty object otherwise.
+    try:
+        testcenteruser = TestCenterUser.objects.get(user=user)
+    except TestCenterUser.DoesNotExist:
+        testcenteruser = TestCenterUser()
+        testcenteruser.user = user
+
+    context = {'course': course,
+               'user': user,
+               'testcenteruser': testcenteruser,
+               'registration': registration,
+               'exam_info': exam_info,
+               }
+
+    return render_to_response('test_center_register.html', context)
+
+
+@ensure_csrf_cookie
+def create_exam_registration(request, post_override=None):
+    '''
+    JSON call to create a test center exam registration.
+    Called by form in test_center_register.html
+    '''
+    post_vars = post_override if post_override else request.POST
+
+    # first determine if we need to create a new TestCenterUser, or if we are making any update
+    # to an existing TestCenterUser.
+    username = post_vars['username']
+    user = User.objects.get(username=username)
+    course_id = post_vars['course_id']
+    course = course_from_id(course_id)  # assume it will be found....
+
+    # make sure that any demographic data values received from the page have been stripped.
+    # Whitespace is not an acceptable response for any of these values
+    demographic_data = {}
+    for fieldname in TestCenterUser.user_provided_fields():
+        if fieldname in post_vars:
+            demographic_data[fieldname] = (post_vars[fieldname]).strip()
+
+    try:
+        testcenter_user = TestCenterUser.objects.get(user=user)
+        needs_updating = testcenter_user.needs_update(demographic_data)
+        log.info("User {0} enrolled in course {1} {2}updating demographic info for exam registration".format(user.username, course_id, "" if needs_updating else "not "))
+    except TestCenterUser.DoesNotExist:
+        # do additional initialization here:
+        testcenter_user = TestCenterUser.create(user)
+        needs_updating = True
+        log.info("User {0} enrolled in course {1} creating demographic info for exam registration".format(user.username, course_id))
+
+    # perform validation:
+    if needs_updating:
+        # first perform validation on the user information
+        # using a Django Form.
+        form = TestCenterUserForm(instance=testcenter_user, data=demographic_data)
+        if form.is_valid():
+            form.update_and_save()
+        else:
+            response_data = {'success': False}
+            # return a list of errors...
+            response_data['field_errors'] = form.errors
+            response_data['non_field_errors'] = form.non_field_errors()
+            return HttpResponse(json.dumps(response_data), mimetype="application/json")
+
+    # create and save the registration:
+    needs_saving = False
+    exam = course.current_test_center_exam
+    exam_code = exam.exam_series_code
+    registrations = get_testcenter_registration(user, course_id, exam_code)
+    if registrations:
+        registration = registrations[0]
+        # NOTE: we do not bother to check here to see if the registration has changed,
+        # because at the moment there is no way for a user to change anything about their
+        # registration.  They only provide an optional accommodation request once, and
+        # cannot make changes to it thereafter.
+        # It is possible that the exam_info content has been changed, such as the
+        # scheduled exam dates, but those kinds of changes should not be handled through
+        # this registration screen.
+
+    else:
+        accommodation_request = post_vars.get('accommodation_request', '')
+        registration = TestCenterRegistration.create(testcenter_user, exam, accommodation_request)
+        needs_saving = True
+        log.info("User {0} enrolled in course {1} creating new exam registration".format(user.username, course_id))
+
+    if needs_saving:
+        # do validation of registration.  (Mainly whether an accommodation request is too long.)
+        form = TestCenterRegistrationForm(instance=registration, data=post_vars)
+        if form.is_valid():
+            form.update_and_save()
+        else:
+            response_data = {'success': False}
+            # return a list of errors...
+            response_data['field_errors'] = form.errors
+            response_data['non_field_errors'] = form.non_field_errors()
+            return HttpResponse(json.dumps(response_data), mimetype="application/json")
+
+
+    # only do the following if there is accommodation text to send,
+    # and a destination to which to send it.
+    # TODO: still need to create the accommodation email templates
+#    if 'accommodation_request' in post_vars and 'TESTCENTER_ACCOMMODATION_REQUEST_EMAIL' in settings:
+#        d = {'accommodation_request': post_vars['accommodation_request'] }
+#
+#        # composes accommodation email
+#        subject = render_to_string('emails/accommodation_email_subject.txt', d)
+#        # Email subject *must not* contain newlines
+#        subject = ''.join(subject.splitlines())
+#        message = render_to_string('emails/accommodation_email.txt', d)
+#
+#        try:
+#            dest_addr = settings['TESTCENTER_ACCOMMODATION_REQUEST_EMAIL']
+#            from_addr = user.email
+#            send_mail(subject, message, from_addr, [dest_addr], fail_silently=False)
+#        except:
+#            log.exception(sys.exc_info())
+#            response_data = {'success': False}
+#            response_data['non_field_errors'] =  [ 'Could not send accommodation e-mail.', ]
+#            return HttpResponse(json.dumps(response_data), mimetype="application/json")
+
+
+    js = {'success': True}
+    return HttpResponse(json.dumps(js), mimetype="application/json")
+
+
 def get_random_post_override():
     """
     Return a dictionary suitable for passing to post_vars of _do_create_account or post_override
@@ -641,7 +840,7 @@ def password_reset(request):
 
     # By default, Django doesn't allow Users with is_active = False to reset their passwords,
     # but this bites people who signed up a long time ago, never activated, and forgot their
-    # password. So for their sake, we'll auto-activate a user for whome password_reset is called.
+    # password. So for their sake, we'll auto-activate a user for whom password_reset is called.
     try:
         user = User.objects.get(email=request.POST['email'])
         user.is_active = True
@@ -651,16 +850,17 @@ def password_reset(request):
 
     form = PasswordResetForm(request.POST)
     if form.is_valid():
-        form.save(use_https = request.is_secure(),
-                  from_email = settings.DEFAULT_FROM_EMAIL,
-                  request = request,
-                  domain_override = request.get_host())
-        return HttpResponse(json.dumps({'success':True,
+        form.save(use_https=request.is_secure(),
+                  from_email=settings.DEFAULT_FROM_EMAIL,
+                  request=request,
+                  domain_override=request.get_host())
+        return HttpResponse(json.dumps({'success': True,
                                         'value': render_to_string('registration/password_reset_done.html', {})}))
     else:
         return HttpResponse(json.dumps({'success': False,
                                         'error': 'Invalid e-mail'}))
 
+
 @ensure_csrf_cookie
 def reactivation_email(request):
     ''' Send an e-mail to reactivate a deactivated account, or to
@@ -673,6 +873,7 @@ def reactivation_email(request):
                                         'error': 'No inactive user with this e-mail exists'}))
     return reactivation_email_for_user(user)
 
+
 def reactivation_email_for_user(user):
     reg = Registration.objects.get(user=user)
 
@@ -813,11 +1014,11 @@ def pending_name_changes(request):
 
     changes = list(PendingNameChange.objects.all())
     js = {'students': [{'new_name': c.new_name,
-                        'rationale':c.rationale,
-                        'old_name':UserProfile.objects.get(user=c.user).name,
-                        'email':c.user.email,
-                        'uid':c.user.id,
-                        'cid':c.id} for c in changes]}
+                        'rationale': c.rationale,
+                        'old_name': UserProfile.objects.get(user=c.user).name,
+                        'email': c.user.email,
+                        'uid': c.user.id,
+                        'cid': c.id} for c in changes]}
     return render_to_response('name_changes.html', js)
 
 
@@ -872,32 +1073,138 @@ def accept_name_change(request):
 
     return accept_name_change_by_id(int(request.POST['id']))
 
-# TODO: This is a giant kludge to give Pearson something to test against ASAP.
-#       Will need to get replaced by something that actually ties into TestCenterUser
 @csrf_exempt
 def test_center_login(request):
-    if not settings.MITX_FEATURES.get('ENABLE_PEARSON_HACK_TEST'):
-        raise Http404
+    # errors are returned by navigating to the error_url, adding a query parameter named "code"
+    # which contains the error code describing the exceptional condition.
+    def makeErrorURL(error_url, error_code):
+        log.error("generating error URL with error code {}".format(error_code))
+        return "{}?code={}".format(error_url, error_code);
 
-    client_candidate_id = request.POST.get("clientCandidateID")
-    # registration_id = request.POST.get("registrationID")
-    exit_url = request.POST.get("exitURL")
+    # get provided error URL, which will be used as a known prefix for returning error messages to the
+    # Pearson shell.
     error_url = request.POST.get("errorURL")
 
+    # TODO: check that the parameters have not been tampered with, by comparing the code provided by Pearson
+    # with the code we calculate for the same parameters.
+    if 'code' not in request.POST:
+        return HttpResponseRedirect(makeErrorURL(error_url, "missingSecurityCode"));
+    code = request.POST.get("code")
+
+    # calculate SHA for query string
+    # TODO: figure out how to get the original query string, so we can hash it and compare.
+
+
+    if 'clientCandidateID' not in request.POST:
+        return HttpResponseRedirect(makeErrorURL(error_url, "missingClientCandidateID"));
+    client_candidate_id = request.POST.get("clientCandidateID")
+
+    # TODO: check remaining parameters, and maybe at least log if they're not matching
+    # expected values....
+    # registration_id = request.POST.get("registrationID")
+    # exit_url = request.POST.get("exitURL")
+
+    # find testcenter_user that matches the provided ID:
+    try:
+        testcenteruser = TestCenterUser.objects.get(client_candidate_id=client_candidate_id)
+    except TestCenterUser.DoesNotExist:
+        log.error("not able to find demographics for cand ID {}".format(client_candidate_id))
+        return HttpResponseRedirect(makeErrorURL(error_url, "invalidClientCandidateID"));
+
+    # find testcenter_registration that matches the provided exam code:
+    # Note that we could rely in future on either the registrationId or the exam code,
+    # or possibly both.  But for now we know what to do with an ExamSeriesCode,
+    # while we currently have no record of RegistrationID values at all.
+    if 'vueExamSeriesCode' not in request.POST:
+        # we are not allowed to make up a new error code, according to Pearson,
+        # so instead of "missingExamSeriesCode", we use a valid one that is
+        # inaccurate but at least distinct.  (Sigh.)
+        log.error("missing exam series code for cand ID {}".format(client_candidate_id))
+        return HttpResponseRedirect(makeErrorURL(error_url, "missingPartnerID"));
+    exam_series_code = request.POST.get('vueExamSeriesCode')
+    # special case for supporting test user:
+    if client_candidate_id == "edX003671291147" and exam_series_code != '6002x001':
+        log.warning("test user {} using unexpected exam code {}, coercing to 6002x001".format(client_candidate_id, exam_series_code))
+        exam_series_code = '6002x001'
+
+    registrations = TestCenterRegistration.objects.filter(testcenter_user=testcenteruser, exam_series_code=exam_series_code)
+    if not registrations:
+        log.error("not able to find exam registration for exam {} and cand ID {}".format(exam_series_code, client_candidate_id))
+        return HttpResponseRedirect(makeErrorURL(error_url, "noTestsAssigned"));
+
+    # TODO: figure out what to do if there are more than one registrations....
+    # for now, just take the first...
+    registration = registrations[0]
+
+    course_id = registration.course_id
+    course = course_from_id(course_id)  # assume it will be found....
+    if not course:
+        log.error("not able to find course from ID {} for cand ID {}".format(course_id, client_candidate_id))
+        return HttpResponseRedirect(makeErrorURL(error_url, "incorrectCandidateTests"));
+    exam = course.get_test_center_exam(exam_series_code)
+    if not exam:
+        log.error("not able to find exam {} for course ID {} and cand ID {}".format(exam_series_code, course_id, client_candidate_id))
+        return HttpResponseRedirect(makeErrorURL(error_url, "incorrectCandidateTests"));
+    location = exam.exam_url
+    log.info("proceeding with test of cand {} on exam {} for course {}: URL = {}".format(client_candidate_id, exam_series_code, course_id, location))
+
+    # check if the test has already been taken
+    timelimit_descriptor = modulestore().get_instance(course_id, Location(location))
+    if not timelimit_descriptor:
+        log.error("cand {} on exam {} for course {}: descriptor not found for location {}".format(client_candidate_id, exam_series_code, course_id, location))
+        return HttpResponseRedirect(makeErrorURL(error_url, "missingClientProgram"));
+
+    timelimit_module_cache = ModelDataCache.cache_for_descriptor_descendents(course_id, testcenteruser.user,
+                                                                             timelimit_descriptor, depth=None)
+    timelimit_module = get_module_for_descriptor(request.user, request, timelimit_descriptor,
+                                                 timelimit_module_cache, course_id, position=None)
+    if not timelimit_module.category == 'timelimit':
+        log.error("cand {} on exam {} for course {}: non-timelimit module at location {}".format(client_candidate_id, exam_series_code, course_id, location))
+        return HttpResponseRedirect(makeErrorURL(error_url, "missingClientProgram"));
+
+    if timelimit_module and timelimit_module.has_ended:
+        log.warning("cand {} on exam {} for course {}: test already over at {}".format(client_candidate_id, exam_series_code, course_id, timelimit_module.ending_at))
+        return HttpResponseRedirect(makeErrorURL(error_url, "allTestsTaken"));
+
+    # check if we need to provide an accommodation:
+    time_accommodation_mapping = {'ET12ET' : 'ADDHALFTIME',
+                                  'ET30MN' : 'ADD30MIN',
+                                  'ETDBTM' : 'ADDDOUBLE', }
+
+    time_accommodation_code = None
+    for code in registration.get_accommodation_codes():
+        if code in time_accommodation_mapping:
+            time_accommodation_code = time_accommodation_mapping[code]
+    # special, hard-coded client ID used by Pearson shell for testing:
     if client_candidate_id == "edX003671291147":
-        user = authenticate(username=settings.PEARSON_TEST_USER,
-                            password=settings.PEARSON_TEST_PASSWORD)
-        login(request, user)
-        return redirect('/courses/MITx/6.002x/2012_Fall/courseware/Final_Exam/Final_Exam_Fall_2012/')
-    else:
-        return HttpResponseForbidden()
+        time_accommodation_code = 'TESTING'
+
+    if time_accommodation_code:
+        timelimit_module.accommodation_code = time_accommodation_code
+        log.info("cand {} on exam {} for course {}: receiving accommodation {}".format(client_candidate_id, exam_series_code, course_id, time_accommodation_code))
+
+    # UGLY HACK!!!
+    # Login assumes that authentication has occurred, and that there is a
+    # backend annotation on the user object, indicating which backend
+    # against which the user was authenticated.  We're authenticating here
+    # against the registration entry, and assuming that the request given
+    # this information is correct, we allow the user to be logged in
+    # without a password.  This could all be formalized in a backend object
+    # that does the above checking.
+    # TODO: (brian) create a backend class to do this.
+    # testcenteruser.user.backend = "%s.%s" % (backend.__module__, backend.__class__.__name__)
+    testcenteruser.user.backend = "%s.%s" % ("TestcenterAuthenticationModule", "TestcenterAuthenticationClass")
+    login(request, testcenteruser.user)
+
+    # And start the test:
+    return jump_to(request, course_id, location)
 
 
 def _get_news(top=None):
     "Return the n top news items on settings.RSS_URL"
 
     feed_data = cache.get("students_index_rss_feed_data")
-    if feed_data == None:
+    if feed_data is None:
         if hasattr(settings, 'RSS_URL'):
             feed_data = urllib.urlopen(settings.RSS_URL).read()
         else:

common/djangoapps/terrain/__init__.py

@@ -0,0 +1,6 @@
+# Use this as your lettuce terrain file so that the common steps
+# across all lms apps can be put in terrain/common
+# See https://groups.google.com/forum/?fromgroups=#!msg/lettuce-users/5VyU9B4HcX8/USgbGIJdS5QJ
+from terrain.browser import *
+from terrain.steps import *
+from terrain.factories import *

common/djangoapps/terrain/browser.py

@@ -0,0 +1,43 @@
+from lettuce import before, after, world
+from splinter.browser import Browser
+from logging import getLogger
+
+# Let the LMS and CMS do their one-time setup
+# For example, setting up mongo caches
+from lms import one_time_startup
+from cms import one_time_startup
+
+logger = getLogger(__name__)
+logger.info("Loading the lettuce acceptance testing terrain file...")
+
+from django.core.management import call_command
+
+
+@before.harvest
+def initial_setup(server):
+    '''
+    Launch the browser once before executing the tests
+    '''
+    # Launch the browser app (choose one of these below)
+    world.browser = Browser('chrome')
+    # world.browser = Browser('phantomjs')
+    # world.browser = Browser('firefox')
+
+
+@before.each_scenario
+def reset_data(scenario):
+    '''
+    Clean out the django test database defined in the
+    envs/acceptance.py file: mitx_all/db/test_mitx.db
+    '''
+    logger.debug("Flushing the test database...")
+    call_command('flush', interactive=False)
+
+
+@after.all
+def teardown_browser(total):
+    '''
+    Quit the browser after executing the tests
+    '''
+    world.browser.quit()
+    pass

common/djangoapps/terrain/factories.py

@@ -0,0 +1,64 @@
+'''
+Factories are defined in other modules and absorbed here into the
+lettuce world so that they can be used by both unit tests
+and integration / BDD tests.
+'''
+import student.tests.factories as sf
+import xmodule.modulestore.tests.factories as xf
+from lettuce import world
+
+
+@world.absorb
+class UserFactory(sf.UserFactory):
+    """
+    User account for lms / cms
+    """
+    pass
+
+
+@world.absorb
+class UserProfileFactory(sf.UserProfileFactory):
+    """
+    Demographics etc for the User
+    """
+    pass
+
+
+@world.absorb
+class RegistrationFactory(sf.RegistrationFactory):
+    """
+    Activation key for registering the user account
+    """
+    pass
+
+
+@world.absorb
+class GroupFactory(sf.GroupFactory):
+    """
+    Groups for user permissions for courses
+    """
+    pass
+
+
+@world.absorb
+class CourseEnrollmentAllowedFactory(sf.CourseEnrollmentAllowed):
+    """
+    Users allowed to enroll in the course outside of the usual window
+    """
+    pass
+
+
+@world.absorb
+class CourseFactory(xf.CourseFactory):
+    """
+    Courseware courses
+    """
+    pass
+
+
+@world.absorb
+class ItemFactory(xf.ItemFactory):
+    """
+    Everything included inside a course
+    """
+    pass

common/djangoapps/terrain/steps.py

@@ -0,0 +1,248 @@
+from lettuce import world, step
+from .factories import *
+from lettuce.django import django_url
+from django.conf import settings
+from django.http import HttpRequest
+from django.contrib.auth.models import User
+from django.contrib.auth import authenticate, login
+from django.contrib.auth.middleware import AuthenticationMiddleware
+from django.contrib.sessions.middleware import SessionMiddleware
+from student.models import CourseEnrollment
+from urllib import quote_plus
+from nose.tools import assert_equals
+from bs4 import BeautifulSoup
+import time
+import re
+import os.path
+from selenium.common.exceptions import WebDriverException
+
+from logging import getLogger
+logger = getLogger(__name__)
+
+
+@step(u'I wait (?:for )?"(\d+)" seconds?$')
+def wait(step, seconds):
+    time.sleep(float(seconds))
+
+
+@step('I reload the page$')
+def reload_the_page(step):
+    world.browser.reload()
+
+
+@step('I press the browser back button$')
+def browser_back(step):
+    world.browser.driver.back()
+
+
+@step('I (?:visit|access|open) the homepage$')
+def i_visit_the_homepage(step):
+    world.browser.visit(django_url('/'))
+    assert world.browser.is_element_present_by_css('header.global', 10)
+
+
+@step(u'I (?:visit|access|open) the dashboard$')
+def i_visit_the_dashboard(step):
+    world.browser.visit(django_url('/dashboard'))
+    assert world.browser.is_element_present_by_css('section.container.dashboard', 5)
+
+
+@step('I should be on the dashboard page$')
+def i_should_be_on_the_dashboard(step):
+    assert world.browser.is_element_present_by_css('section.container.dashboard', 5)
+    assert world.browser.title == 'Dashboard'
+
+
+@step(u'I (?:visit|access|open) the courses page$')
+def i_am_on_the_courses_page(step):
+    world.browser.visit(django_url('/courses'))
+    assert world.browser.is_element_present_by_css('section.courses')
+
+
+@step(u'I press the "([^"]*)" button$')
+def and_i_press_the_button(step, value):
+    button_css = 'input[value="%s"]' % value
+    world.browser.find_by_css(button_css).first.click()
+
+
+@step(u'I click the link with the text "([^"]*)"$')
+def click_the_link_with_the_text_group1(step, linktext):
+    world.browser.find_link_by_text(linktext).first.click()
+
+
+@step('I should see that the path is "([^"]*)"$')
+def i_should_see_that_the_path_is(step, path):
+    assert world.browser.url == django_url(path)
+
+
+@step(u'the page title should be "([^"]*)"$')
+def the_page_title_should_be(step, title):
+    assert_equals(world.browser.title, title)
+
+
+@step(u'the page title should contain "([^"]*)"$')
+def the_page_title_should_contain(step, title):
+    assert(title in world.browser.title)
+
+
+@step('I am a logged in user$')
+def i_am_logged_in_user(step):
+    create_user('robot')
+    log_in('robot', 'test')
+
+
+@step('I am not logged in$')
+def i_am_not_logged_in(step):
+    world.browser.cookies.delete()
+
+
+@step('I am staff for course "([^"]*)"$')
+def i_am_staff_for_course_by_id(step, course_id):
+    register_by_course_id(course_id, True)
+
+
+@step('I log in$')
+def i_log_in(step):
+    log_in('robot', 'test')
+
+
+@step(u'I am an edX user$')
+def i_am_an_edx_user(step):
+    create_user('robot')
+
+#### helper functions
+
+
+@world.absorb
+def scroll_to_bottom():
+    # Maximize the browser
+    world.browser.execute_script("window.scrollTo(0, screen.height);")
+
+
+@world.absorb
+def create_user(uname):
+
+    # If the user already exists, don't try to create it again
+    if len(User.objects.filter(username=uname)) > 0:
+        return
+
+    portal_user = UserFactory.build(username=uname, email=uname + '@edx.org')
+    portal_user.set_password('test')
+    portal_user.save()
+
+    registration = world.RegistrationFactory(user=portal_user)
+    registration.register(portal_user)
+    registration.activate()
+
+    user_profile = world.UserProfileFactory(user=portal_user)
+
+
+@world.absorb
+def log_in(username, password):
+    '''
+    Log the user in programatically
+    '''
+
+    # Authenticate the user
+    user = authenticate(username=username, password=password)
+    assert(user is not None and user.is_active)
+
+    # Send a fake HttpRequest to log the user in
+    # We need to process the request using
+    # Session middleware and Authentication middleware
+    # to ensure that session state can be stored
+    request = HttpRequest()
+    SessionMiddleware().process_request(request)
+    AuthenticationMiddleware().process_request(request)
+    login(request, user)
+
+    # Save the session
+    request.session.save()
+
+    # Retrieve the sessionid and add it to the browser's cookies
+    cookie_dict = {settings.SESSION_COOKIE_NAME: request.session.session_key}
+    try:
+        world.browser.cookies.add(cookie_dict)
+
+    # WebDriver has an issue where we cannot set cookies
+    # before we make a GET request, so if we get an error,
+    # we load the '/' page and try again
+    except:
+        world.browser.visit(django_url('/'))
+        world.browser.cookies.add(cookie_dict)
+
+
+@world.absorb
+def register_by_course_id(course_id, is_staff=False):
+    create_user('robot')
+    u = User.objects.get(username='robot')
+    if is_staff:
+        u.is_staff = True
+        u.save()
+    CourseEnrollment.objects.get_or_create(user=u, course_id=course_id)
+
+
+@world.absorb
+def save_the_html(path='/tmp'):
+    u = world.browser.url
+    html = world.browser.html.encode('ascii', 'ignore')
+    filename = '%s.html' % quote_plus(u)
+    f = open('%s/%s' % (path, filename), 'w')
+    f.write(html)
+    f.close
+
+
+@world.absorb
+def save_the_course_content(path='/tmp'):
+    html = world.browser.html.encode('ascii', 'ignore')
+    soup = BeautifulSoup(html)
+
+    # get rid of the header, we only want to compare the body
+    soup.head.decompose()
+
+    # for now, remove the data-id attributes, because they are
+    # causing mismatches between cms-master and master
+    for item in soup.find_all(attrs={'data-id': re.compile('.*')}):
+        del item['data-id']
+
+    # we also need to remove them from unrendered problems,
+    # where they are contained in the text of divs instead of
+    # in attributes of tags
+    # Be careful of whether or not it was the last attribute
+    # and needs a trailing space
+    for item in soup.find_all(text=re.compile(' data-id=".*?" ')):
+        s = unicode(item.string)
+        item.string.replace_with(re.sub(' data-id=".*?" ', ' ', s))
+
+    for item in soup.find_all(text=re.compile(' data-id=".*?"')):
+        s = unicode(item.string)
+        item.string.replace_with(re.sub(' data-id=".*?"', ' ', s))
+
+    # prettify the html so it will compare better, with
+    # each HTML tag on its own line
+    output = soup.prettify()
+
+    # use string slicing to grab everything after 'courseware/' in the URL
+    u = world.browser.url
+    section_url = u[u.find('courseware/') + 11:]
+
+
+    if not os.path.exists(path):
+        os.makedirs(path)
+
+    filename = '%s.html' % (quote_plus(section_url))
+    f = open('%s/%s' % (path, filename), 'w')
+    f.write(output)
+    f.close
+
+@world.absorb
+def css_click(css_selector):
+    try:
+        world.browser.find_by_css(css_selector).click()
+
+    except WebDriverException:
+        # Occassionally, MathJax or other JavaScript can cover up
+        # an element  temporarily.
+        # If this happens, wait a second, then try again
+        time.sleep(1)
+        world.browser.find_by_css(css_selector).click()

common/djangoapps/track/migrations/0001_initial.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'TrackingLog'
+        db.create_table('track_trackinglog', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('dtcreated', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
+            ('username', self.gf('django.db.models.fields.CharField')(max_length=32, blank=True)),
+            ('ip', self.gf('django.db.models.fields.CharField')(max_length=32, blank=True)),
+            ('event_source', self.gf('django.db.models.fields.CharField')(max_length=32)),
+            ('event_type', self.gf('django.db.models.fields.CharField')(max_length=32, blank=True)),
+            ('event', self.gf('django.db.models.fields.TextField')(blank=True)),
+            ('agent', self.gf('django.db.models.fields.CharField')(max_length=256, blank=True)),
+            ('page', self.gf('django.db.models.fields.CharField')(max_length=32, null=True, blank=True)),
+            ('time', self.gf('django.db.models.fields.DateTimeField')()),
+        ))
+        db.send_create_signal('track', ['TrackingLog'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'TrackingLog'
+        db.delete_table('track_trackinglog')
+
+
+    models = {
+        'track.trackinglog': {
+            'Meta': {'object_name': 'TrackingLog'},
+            'agent': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
+            'dtcreated': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'event': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'event_source': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
+            'event_type': ('django.db.models.fields.CharField', [], {'max_length': '32', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'ip': ('django.db.models.fields.CharField', [], {'max_length': '32', 'blank': 'True'}),
+            'page': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True', 'blank': 'True'}),
+            'time': ('django.db.models.fields.DateTimeField', [], {}),
+            'username': ('django.db.models.fields.CharField', [], {'max_length': '32', 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['track']

common/djangoapps/track/migrations/0002_auto__add_field_trackinglog_host__chg_field_trackinglog_event_type__ch.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'TrackingLog.host'
+        db.add_column('track_trackinglog', 'host',
+                      self.gf('django.db.models.fields.CharField')(default='', max_length=64, blank=True),
+                      keep_default=False)
+
+
+        # Changing field 'TrackingLog.event_type'
+        db.alter_column('track_trackinglog', 'event_type', self.gf('django.db.models.fields.CharField')(max_length=512))
+
+        # Changing field 'TrackingLog.page'
+        db.alter_column('track_trackinglog', 'page', self.gf('django.db.models.fields.CharField')(max_length=512, null=True))
+
+    def backwards(self, orm):
+        # Deleting field 'TrackingLog.host'
+        db.delete_column('track_trackinglog', 'host')
+
+
+        # Changing field 'TrackingLog.event_type'
+        db.alter_column('track_trackinglog', 'event_type', self.gf('django.db.models.fields.CharField')(max_length=32))
+
+        # Changing field 'TrackingLog.page'
+        db.alter_column('track_trackinglog', 'page', self.gf('django.db.models.fields.CharField')(max_length=32, null=True))
+
+    models = {
+        'track.trackinglog': {
+            'Meta': {'object_name': 'TrackingLog'},
+            'agent': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
+            'dtcreated': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'event': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'event_source': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
+            'event_type': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
+            'host': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'ip': ('django.db.models.fields.CharField', [], {'max_length': '32', 'blank': 'True'}),
+            'page': ('django.db.models.fields.CharField', [], {'max_length': '512', 'null': 'True', 'blank': 'True'}),
+            'time': ('django.db.models.fields.DateTimeField', [], {}),
+            'username': ('django.db.models.fields.CharField', [], {'max_length': '32', 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['track']

common/djangoapps/track/models.py

@@ -2,20 +2,20 @@ from django.db import models
 
 from django.db import models
 
+
 class TrackingLog(models.Model):
-    dtcreated = models.DateTimeField('creation date',auto_now_add=True)
-    username = models.CharField(max_length=32,blank=True)
-    ip = models.CharField(max_length=32,blank=True)
+    dtcreated = models.DateTimeField('creation date', auto_now_add=True)
+    username = models.CharField(max_length=32, blank=True)
+    ip = models.CharField(max_length=32, blank=True)
     event_source = models.CharField(max_length=32)
-    event_type = models.CharField(max_length=32,blank=True)
+    event_type = models.CharField(max_length=512, blank=True)
     event = models.TextField(blank=True)
-    agent = models.CharField(max_length=256,blank=True)
-    page = models.CharField(max_length=32,blank=True,null=True)
+    agent = models.CharField(max_length=256, blank=True)
+    page = models.CharField(max_length=512, blank=True, null=True)
     time = models.DateTimeField('event time')
+    host = models.CharField(max_length=64, blank=True)
 
     def __unicode__(self):
         s = "[%s] %s@%s: %s | %s | %s | %s" % (self.time, self.username, self.ip, self.event_source,
                                                self.event_type, self.page, self.event)
         return s
-
-

common/djangoapps/track/views.py

@@ -17,19 +17,21 @@ from track.models import TrackingLog
 
 log = logging.getLogger("tracking")
 
-LOGFIELDS = ['username','ip','event_source','event_type','event','agent','page','time']
+LOGFIELDS = ['username', 'ip', 'event_source', 'event_type', 'event', 'agent', 'page', 'time', 'host']
+
 
 def log_event(event):
     event_str = json.dumps(event)
     log.info(event_str[:settings.TRACK_MAX_EVENT])
     if settings.MITX_FEATURES.get('ENABLE_SQL_TRACKING_LOGS'):
         event['time'] = dateutil.parser.parse(event['time'])
-        tldat = TrackingLog(**dict( (x,event[x]) for x in LOGFIELDS ))
+        tldat = TrackingLog(**dict((x, event[x]) for x in LOGFIELDS))
         try:
             tldat.save()
         except Exception as err:
             log.exception(err)
 
+
 def user_track(request):
     try:  # TODO: Do the same for many of the optional META parameters
         username = request.user.username
@@ -58,6 +60,7 @@ def user_track(request):
         "agent": agent,
         "page": request.GET['page'],
         "time": datetime.datetime.utcnow().isoformat(),
+        "host": request.META['SERVER_NAME'],
         }
     log_event(event)
     return HttpResponse('success')
@@ -83,15 +86,17 @@ def server_track(request, event_type, event, page=None):
         "agent": agent,
         "page": page,
         "time": datetime.datetime.utcnow().isoformat(),
+        "host": request.META['SERVER_NAME'],
         }
 
-    if event_type.startswith("/event_logs") and request.user.is_staff:	# don't log
+    if event_type.startswith("/event_logs") and request.user.is_staff:  	# don't log
         return
     log_event(event)
 
+
 @login_required
 @ensure_csrf_cookie
-def view_tracking_log(request,args=''):
+def view_tracking_log(request, args=''):
     if not request.user.is_staff:
         return redirect('/')
     nlen = 100
@@ -102,16 +107,15 @@ def view_tracking_log(request,args=''):
                 nlen = int(arg)
             if arg.startswith('username='):
                 username = arg[9:]
-            
+
     record_instances = TrackingLog.objects.all().order_by('-time')
     if username:
         record_instances = record_instances.filter(username=username)
     record_instances = record_instances[0:nlen]
-    
+
     # fix dtstamp
     fmt = '%a %d-%b-%y %H:%M:%S'  # "%Y-%m-%d %H:%M:%S %Z%z"
     for rinst in record_instances:
         rinst.dtstr = rinst.time.replace(tzinfo=pytz.utc).astimezone(pytz.timezone('US/Eastern')).strftime(fmt)
 
-    return render_to_response('tracking_log.html',{'records':record_instances})
-
+    return render_to_response('tracking_log.html', {'records': record_instances})

common/djangoapps/util/cache.py

@@ -58,4 +58,3 @@ def cache_if_anonymous(view_func):
             return view_func(request, *args, **kwargs)
 
     return _decorated
-

common/djangoapps/util/json_request.py

@@ -13,7 +13,7 @@ def expect_json(view_function):
     def expect_json_with_cloned_request(request, *args, **kwargs):
         # cdodge: fix postback errors in CMS. The POST 'content-type' header can include additional information
         # e.g. 'charset', so we can't do a direct string compare
-        if request.META['CONTENT_TYPE'].lower().startswith("application/json"):
+        if request.META.get('CONTENT_TYPE', '').lower().startswith("application/json"):
             cloned_request = copy.copy(request)
             cloned_request.POST = cloned_request.POST.copy()
             cloned_request.POST.update(json.loads(request.body))

common/djangoapps/util/views.py

@@ -93,6 +93,7 @@ def accepts(request, media_type):
     accept = parse_accept_header(request.META.get("HTTP_ACCEPT", ""))
     return media_type in [t for (t, p, q) in accept]
 
+
 def debug_request(request):
     """Return a pretty printed version of the request"""
 

common/djangoapps/xmodule_modifiers.py

@@ -2,17 +2,18 @@ import re
 import json
 import logging
 import time
+import static_replace
 
 from django.conf import settings
 from functools import wraps
-from static_replace import replace_urls
 from mitxmako.shortcuts import render_to_string
 from xmodule.seq_module import SequenceModule
 from xmodule.vertical_module import VerticalModule
 
 log = logging.getLogger("mitx.xmodule_modifiers")
 
-def wrap_xmodule(get_html, module, template):
+
+def wrap_xmodule(get_html, module, template, context=None):
     """
     Wraps the results of get_html in a standard <section> with identifying
     data so that the appropriate javascript module can be loaded onto it.
@@ -21,17 +22,23 @@ def wrap_xmodule(get_html, module, template):
     module: An XModule
     template: A template that takes the variables:
         content: the results of get_html,
+        display_name: the display name of the xmodule, if available (None otherwise)
         class_: the module class name
         module_name: the js_module_name of the module
     """
+    if context is None:
+        context = {}
 
     @wraps(get_html)
     def _get_html():
-        return render_to_string(template, {
+        context.update({
             'content': get_html(),
+            'display_name': module.display_name,
             'class_': module.__class__.__name__,
             'module_name': module.js_module_name
         })
+
+        return render_to_string(template, context)
     return _get_html
 
 
@@ -43,10 +50,11 @@ def replace_course_urls(get_html, course_id):
     """
     @wraps(get_html)
     def _get_html():
-        return replace_urls(get_html(), staticfiles_prefix='/courses/'+course_id, replace_prefix='/course/')
+        return static_replace.replace_course_urls(get_html(), course_id)
     return _get_html
 
-def replace_static_urls(get_html, prefix):
+
+def replace_static_urls(get_html, data_dir, course_namespace=None):
     """
     Updates the supplied module with a new get_html function that wraps
     the old get_html function and substitutes urls of the form /static/...
@@ -55,7 +63,7 @@ def replace_static_urls(get_html, prefix):
 
     @wraps(get_html)
     def _get_html():
-        return replace_urls(get_html(), staticfiles_prefix=prefix)
+        return static_replace.replace_static_urls(get_html(), data_dir, course_namespace)
     return _get_html
 
 
@@ -93,51 +101,34 @@ def add_histogram(get_html, module, user):
     @wraps(get_html)
     def _get_html():
 
-        if type(module) in [SequenceModule, VerticalModule]:	# TODO: make this more general, eg use an XModule attribute instead
+        if type(module) in [SequenceModule, VerticalModule]:  	# TODO: make this more general, eg use an XModule attribute instead
             return get_html()
 
         module_id = module.id
         histogram = grade_histogram(module_id)
         render_histogram = len(histogram) > 0
 
-        # TODO (ichuang): Remove after fall 2012 LMS migration done
-        if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'):
-            [filepath, filename] = module.definition.get('filename', ['', None])
-            osfs = module.system.filestore
-            if filename is not None and osfs.exists(filename):
-                # if original, unmangled filename exists then use it (github
-                # doesn't like symlinks)
-                filepath = filename
-            data_dir = osfs.root_path.rsplit('/')[-1]
-            giturl = module.metadata.get('giturl','https://github.com/MITx')
-            edit_link = "%s/%s/tree/master/%s" % (giturl,data_dir,filepath)
-        else:
-            edit_link = False
-            # Need to define all the variables that are about to be used
-            giturl = ""
-            data_dir = ""
-        source_file = module.metadata.get('source_file','')	# source used to generate the problem XML, eg latex or word
+        source_file = module.lms.source_file  # source used to generate the problem XML, eg latex or word
 
         # useful to indicate to staff if problem has been released or not
         # TODO (ichuang): use _has_access_descriptor.can_load in lms.courseware.access, instead of now>mstart comparison here
         now = time.gmtime()
         is_released = "unknown"
-        mstart = getattr(module.descriptor,'start')
+        mstart = module.descriptor.lms.start
+
         if mstart is not None:
             is_released = "<font color='red'>Yes!</font>" if (now > mstart) else "<font color='green'>Not yet</font>"
 
-        staff_context = {'definition': module.definition.get('data'),
-                         'metadata': json.dumps(module.metadata, indent=4),
+        staff_context = {'fields': [(field.name, getattr(module, field.name)) for field in module.fields],
+                         'lms_fields': [(field.name, getattr(module.lms, field.name)) for field in module.lms.fields],
                          'location': module.location,
-                         'xqa_key': module.metadata.get('xqa_key',''),
-                         'source_file' : source_file,
-                         'source_url': '%s/%s/tree/master/%s' % (giturl,data_dir,source_file),
+                         'xqa_key': module.lms.xqa_key,
+                         'source_file': source_file,
                          'category': str(module.__class__.__name__),
                          # Template uses element_id in js function names, so can't allow dashes
-                         'element_id': module.location.html_id().replace('-','_'),
-                         'edit_link': edit_link,
+                         'element_id': module.location.html_id().replace('-', '_'),
                          'user': user,
-                         'xqa_server' : settings.MITX_FEATURES.get('USE_XQA_SERVER','http://xqa:server@content-qa.mitx.mit.edu/xqa'),
+                         'xqa_server': settings.MITX_FEATURES.get('USE_XQA_SERVER', 'http://xqa:server@content-qa.mitx.mit.edu/xqa'),
                          'histogram': json.dumps(histogram),
                          'render_histogram': render_histogram,
                          'module_content': get_html(),
@@ -146,4 +137,3 @@ def add_histogram(get_html, module, user):
         return render_to_string("staff_problem_info.html", staff_context)
 
     return _get_html
-

common/lib/.gitignore

@@ -0,0 +1 @@
+*/jasmine_test_runner.html

common/lib/capa/.coveragerc

@@ -7,6 +7,7 @@ source = common/lib/capa
 ignore_errors = True
 
 [html]
+title = Capa Python Test Coverage Report
 directory = reports/common/lib/capa/cover
 
 [xml]

common/lib/capa/capa/calc.py

@@ -121,9 +121,9 @@ def evaluator(variables, functions, string, cs=False):
     # confusing. They may also conflict with variables if we ever allow e.g.
     # 5R instead of 5*R
     suffixes = {'%': 0.01, 'k': 1e3, 'M': 1e6, 'G': 1e9,
-              'T': 1e12,#  'P':1e15,'E':1e18,'Z':1e21,'Y':1e24,
+              'T': 1e12,  # 'P':1e15,'E':1e18,'Z':1e21,'Y':1e24,
               'c': 1e-2, 'm': 1e-3, 'u': 1e-6,
-              'n': 1e-9, 'p': 1e-12}#  ,'f':1e-15,'a':1e-18,'z':1e-21,'y':1e-24}
+              'n': 1e-9, 'p': 1e-12}  # ,'f':1e-15,'a':1e-18,'z':1e-21,'y':1e-24}
 
     def super_float(text):
         ''' Like float, but with si extensions. 1k goes to 1000'''
@@ -183,7 +183,7 @@ def evaluator(variables, functions, string, cs=False):
 
     # 0.33k or -17
     number = (Optional(minus | plus) + inner_number
-              + Optional(CaselessLiteral("E") + Optional("-") + number_part)
+              + Optional(CaselessLiteral("E") + Optional((plus | minus)) + number_part)
               + Optional(number_suffix))
     number = number.setParseAction(number_parse_action)  # Convert to number
 

common/lib/capa/capa/capa_problem.py

@@ -16,7 +16,6 @@ This is used by capa_module.
 from __future__ import division
 
 from datetime import datetime
-import json
 import logging
 import math
 import numpy
@@ -29,18 +28,19 @@ import sys
 
 from lxml import etree
 from xml.sax.saxutils import unescape
+from copy import deepcopy
 
 import chem
-import chem.chemcalc
-import chem.chemtools
 import chem.miller
+import verifiers
+import verifiers.draganddrop
 
 import calc
-from correctmap import CorrectMap
+from .correctmap import CorrectMap
 import eia
 import inputtypes
 import customrender
-from util import contextualize_text, convert_files_to_filenames
+from .util import contextualize_text, convert_files_to_filenames
 import xqueue_interface
 
 # to be replaced with auto-registering
@@ -67,14 +67,12 @@ global_context = {'random': random,
                   'scipy': scipy,
                   'calc': calc,
                   'eia': eia,
-                  'chemcalc': chem.chemcalc,
-                  'chemtools': chem.chemtools,
-                  'miller': chem.miller}
+                  'draganddrop': verifiers.draganddrop}
 
 # These should be removed from HTML output, including all subelements
-html_problem_semantics = ["codeparam", "responseparam", "answer", "script", "hintgroup", "openendedparam","openendedrubric"]
+html_problem_semantics = ["codeparam", "responseparam", "answer", "script", "hintgroup", "openendedparam", "openendedrubric"]
 
-log = logging.getLogger('mitx.' + __name__)
+log = logging.getLogger(__name__)
 
 #-----------------------------------------------------------------------------
 # main class for this module
@@ -93,8 +91,13 @@ class LoncapaProblem(object):
 
          - problem_text (string): xml defining the problem
          - id           (string): identifier for this problem; often a filename (no spaces)
-         - state        (dict): student state
-         - seed         (int): random number generator seed (int)
+         - seed         (int): random number generator seed (int) 
+         - state        (dict): containing the following keys:
+                                - 'seed' - (int) random number generator seed
+                                - 'student_answers' - (dict) maps input id to the stored answer for that input
+                                - 'correct_map' (CorrectMap) a map of each input to their 'correctness'
+                                - 'done' - (bool) indicates whether or not this problem is considered done
+                                - 'input_state' - (dict) maps input_id to a dictionary that holds the state for that input
          - system       (ModuleSystem): ModuleSystem instance which provides OS,
                                         rendering, and user context
 
@@ -104,21 +107,25 @@ class LoncapaProblem(object):
         self.do_reset()
         self.problem_id = id
         self.system = system
-        self.seed = seed
+        if self.system is None:
+            raise Exception()
+
+        state = state if state else {}
+
+        # Set seed according to the following priority:
+        #       1. Contained in problem's state
+        #       2. Passed into capa_problem via constructor
+        #       3. Assign from the OS's random number generator
+        self.seed = state.get('seed', seed)
+        if self.seed is None:
+            self.seed = struct.unpack('i', os.urandom(4))
+        self.student_answers = state.get('student_answers', {})
+        if 'correct_map' in state:
+            self.correct_map.set_dict(state['correct_map'])
+        self.done = state.get('done', False)
+        self.input_state = state.get('input_state', {})
 
-        if state:
-            if 'seed' in state:
-                self.seed = state['seed']
-            if 'student_answers' in state:
-                self.student_answers = state['student_answers']
-            if 'correct_map' in state:
-                self.correct_map.set_dict(state['correct_map'])
-            if 'done' in state:
-                self.done = state['done']
 
-        # TODO: Does this deplete the Linux entropy pool? Is this fast enough?
-        if not self.seed:
-            self.seed = struct.unpack('i', os.urandom(4))[0]
 
         # Convert startouttext and endouttext to proper <text></text>
         problem_text = re.sub("startouttext\s*/", "text", problem_text)
@@ -143,6 +150,13 @@ class LoncapaProblem(object):
         if not self.student_answers:  # True when student_answers is an empty dict
             self.set_initial_display()
 
+        # dictionary of InputType objects associated with this problem
+        #   input_id string -> InputType object
+        self.inputs = {}
+
+        self.extracted_tree = self._extract_html(self.tree)
+
+
     def do_reset(self):
         '''
         Reset internal state to unfinished, with no answers
@@ -175,6 +189,7 @@ class LoncapaProblem(object):
         return {'seed': self.seed,
                 'student_answers': self.student_answers,
                 'correct_map': self.correct_map.get_dict(),
+                'input_state': self.input_state,
                 'done': self.done}
 
     def get_max_score(self):
@@ -224,6 +239,20 @@ class LoncapaProblem(object):
         self.correct_map.set_dict(cmap.get_dict())
         return cmap
 
+    def ungraded_response(self, xqueue_msg, queuekey):
+        '''
+        Handle any responses from the xqueue that do not contain grades
+        Will try to pass the queue message to all inputtypes that can handle ungraded responses 
+
+        Does not return any value
+        '''
+        # check against each inputtype
+        for the_input in self.inputs.values():
+            # if the input type has an ungraded function, pass in the values
+            if hasattr(the_input, 'ungraded_response'):
+                the_input.ungraded_response(xqueue_msg, queuekey)
+
+
     def is_queued(self):
         '''
         Returns True if any part of the problem has been submitted to an external queue
@@ -321,7 +350,27 @@ class LoncapaProblem(object):
         '''
         Main method called externally to get the HTML to be rendered for this capa Problem.
         '''
-        return contextualize_text(etree.tostring(self._extract_html(self.tree)), self.context)
+        html = contextualize_text(etree.tostring(self._extract_html(self.tree)), self.context)
+        return html
+
+
+    def handle_input_ajax(self, get):
+        '''
+        InputTypes can support specialized AJAX calls. Find the correct input and pass along the correct data
+
+        Also, parse out the dispatch from the get so that it can be passed onto the input type nicely
+        '''
+
+        # pull out the id
+        input_id = get['input_id']
+        if self.inputs[input_id]:
+            dispatch = get['dispatch']
+            return self.inputs[input_id].handle_ajax(dispatch, get)
+        else:
+            log.warning("Could not find matching input for id: %s" % input_id)
+            return {}
+
+
 
     # ======= Private Methods Below ========
 
@@ -450,11 +499,13 @@ class LoncapaProblem(object):
                 exec code in context, context
             except Exception as err:
                 log.exception("Error while execing script code: " + code)
-                msg = "Error while executing script code: %s" % str(err).replace('<','&lt;')
+                msg = "Error while executing script code: %s" % str(err).replace('<', '&lt;')
                 raise responsetypes.LoncapaProblemError(msg)
             finally:
                 sys.path = original_path
 
+
+
     def _extract_html(self, problemtree):  # private
         '''
         Main (private) function which converts Problem XML tree to HTML.
@@ -468,7 +519,7 @@ class LoncapaProblem(object):
         if (problemtree.tag == 'script' and problemtree.get('type')
             and 'javascript' in problemtree.get('type')):
             # leave javascript intact.
-            return problemtree
+            return deepcopy(problemtree)
 
         if problemtree.tag in html_problem_semantics:
             return
@@ -481,8 +532,9 @@ class LoncapaProblem(object):
             msg = ''
             hint = ''
             hintmode = None
+            input_id = problemtree.get('id')
             if problemid in self.correct_map:
-                pid = problemtree.get('id')
+                pid = input_id
                 status = self.correct_map.get_correctness(pid)
                 msg = self.correct_map.get_msg(pid)
                 hint = self.correct_map.get_hint(pid)
@@ -491,23 +543,29 @@ class LoncapaProblem(object):
             value = ""
             if self.student_answers and problemid in self.student_answers:
                 value = self.student_answers[problemid]
-
+            
+            if input_id not in self.input_state:
+                self.input_state[input_id] = {}
+                
             # do the rendering
-
             state = {'value': value,
                    'status': status,
-                   'id': problemtree.get('id'),
+                   'id': input_id,
+                   'input_state': self.input_state[input_id],
                    'feedback': {'message': msg,
                                 'hint': hint,
-                                'hintmode': hintmode,}}
+                                'hintmode': hintmode, }}
 
             input_type_cls = inputtypes.registry.get_class_for_tag(problemtree.tag)
-            the_input = input_type_cls(self.system, problemtree, state)
-            return the_input.get_html()
+            # save the input type so that we can make ajax calls on it if we need to
+            self.inputs[input_id] = input_type_cls(self.system, problemtree, state)
+            return self.inputs[input_id].get_html()
 
         # let each Response render itself
         if problemtree in self.responders:
-            return self.responders[problemtree].render_html(self._extract_html)
+            overall_msg = self.correct_map.get_overall_message()
+            return self.responders[problemtree].render_html(self._extract_html,
+                                                response_msg=overall_msg)
 
         # let each custom renderer render itself:
         if problemtree.tag in customrender.registry.registered_tags():

common/lib/capa/capa/checker.py

@@ -12,8 +12,8 @@ from path import path
 from cStringIO import StringIO
 from collections import defaultdict
 
-from calc import UndefinedVariable
-from capa_problem import LoncapaProblem
+from .calc import UndefinedVariable
+from .capa_problem import LoncapaProblem
 from mako.lookup import TemplateLookup
 
 logging.basicConfig(format="%(levelname)s %(message)s")

common/lib/capa/capa/chem/__init__.py

@@ -1 +0,0 @@
-

common/lib/capa/capa/chem/chemcalc.py

@@ -17,17 +17,17 @@ from nltk.tree import Tree
 ARROWS = ('<->', '->')
 
 ## Defines a simple pyparsing tokenizer for chemical equations
-elements = ['Ac','Ag','Al','Am','Ar','As','At','Au','B','Ba','Be',
-            'Bh','Bi','Bk','Br','C','Ca','Cd','Ce','Cf','Cl','Cm',
-            'Cn','Co','Cr','Cs','Cu','Db','Ds','Dy','Er','Es','Eu',
-            'F','Fe','Fl','Fm','Fr','Ga','Gd','Ge','H','He','Hf',
-            'Hg','Ho','Hs','I','In','Ir','K','Kr','La','Li','Lr',
-            'Lu','Lv','Md','Mg','Mn','Mo','Mt','N','Na','Nb','Nd',
-            'Ne','Ni','No','Np','O','Os','P','Pa','Pb','Pd','Pm',
-            'Po','Pr','Pt','Pu','Ra','Rb','Re','Rf','Rg','Rh','Rn',
-            'Ru','S','Sb','Sc','Se','Sg','Si','Sm','Sn','Sr','Ta',
-            'Tb','Tc','Te','Th','Ti','Tl','Tm','U','Uuo','Uup',
-            'Uus','Uut','V','W','Xe','Y','Yb','Zn','Zr']
+elements = ['Ac', 'Ag', 'Al', 'Am', 'Ar', 'As', 'At', 'Au', 'B', 'Ba', 'Be',
+            'Bh', 'Bi', 'Bk', 'Br', 'C', 'Ca', 'Cd', 'Ce', 'Cf', 'Cl', 'Cm',
+            'Cn', 'Co', 'Cr', 'Cs', 'Cu', 'Db', 'Ds', 'Dy', 'Er', 'Es', 'Eu',
+            'F', 'Fe', 'Fl', 'Fm', 'Fr', 'Ga', 'Gd', 'Ge', 'H', 'He', 'Hf',
+            'Hg', 'Ho', 'Hs', 'I', 'In', 'Ir', 'K', 'Kr', 'La', 'Li', 'Lr',
+            'Lu', 'Lv', 'Md', 'Mg', 'Mn', 'Mo', 'Mt', 'N', 'Na', 'Nb', 'Nd',
+            'Ne', 'Ni', 'No', 'Np', 'O', 'Os', 'P', 'Pa', 'Pb', 'Pd', 'Pm',
+            'Po', 'Pr', 'Pt', 'Pu', 'Ra', 'Rb', 'Re', 'Rf', 'Rg', 'Rh', 'Rn',
+            'Ru', 'S', 'Sb', 'Sc', 'Se', 'Sg', 'Si', 'Sm', 'Sn', 'Sr', 'Ta',
+            'Tb', 'Tc', 'Te', 'Th', 'Ti', 'Tl', 'Tm', 'U', 'Uuo', 'Uup',
+            'Uus', 'Uut', 'V', 'W', 'Xe', 'Y', 'Yb', 'Zn', 'Zr']
 digits = map(str, range(10))
 symbols = list("[](){}^+-/")
 phases = ["(s)", "(l)", "(g)", "(aq)"]
@@ -252,7 +252,7 @@ def _get_final_tree(s):
     '''
     tokenized = tokenizer.parseString(s)
     parsed = parser.parse(tokenized)
-    merged = _merge_children(parsed, {'S','group'})
+    merged = _merge_children(parsed, {'S', 'group'})
     final = _clean_parse_tree(merged)
     return final
 

common/lib/capa/capa/chem/tests.py

@@ -2,7 +2,7 @@ import codecs
 from fractions import Fraction
 import unittest
 
-from chemcalc import (compare_chemical_expression, divide_chemical_expression,
+from .chemcalc import (compare_chemical_expression, divide_chemical_expression,
                       render_to_html, chemical_equations_equal)
 
 import miller
@@ -277,7 +277,6 @@ class Test_Render_Equations(unittest.TestCase):
 
     def test_render9(self):
         s = "5[Ni(NH3)4]^2+ + 5/2SO4^2-"
-        #import ipdb; ipdb.set_trace()
         out = render_to_html(s)
         correct = u'<span class="math">5[Ni(NH<sub>3</sub>)<sub>4</sub>]<sup>2+</sup>+<sup>5</sup>&frasl;<sub>2</sub>SO<sub>4</sub><sup>2-</sup></span>'
         log(out + ' ------- ' + correct, 'html')

common/lib/capa/capa/correctmap.py

@@ -3,6 +3,7 @@
 #
 # Used by responsetypes and capa_problem
 
+
 class CorrectMap(object):
     """
     Stores map between answer_id and response evaluation result for each question
@@ -26,6 +27,7 @@ class CorrectMap(object):
         self.cmap = dict()
         self.items = self.cmap.items
         self.keys = self.cmap.keys
+        self.overall_message = ""
         self.set(*args, **kwargs)
 
     def __getitem__(self, *args, **kwargs):
@@ -45,7 +47,7 @@ class CorrectMap(object):
             queuestate=None, **kwargs):
 
         if answer_id is not None:
-            self.cmap[answer_id] = {'correctness': correctness,
+            self.cmap[str(answer_id)] = {'correctness': correctness,
                                     'npoints': npoints,
                                     'msg': msg,
                                     'hint': hint,
@@ -93,7 +95,7 @@ class CorrectMap(object):
 
     def is_correct(self, answer_id):
         if answer_id in self.cmap:
-            return self.cmap[answer_id]['correctness'] == 'correct'
+            return self.cmap[answer_id]['correctness'] in ['correct', 'partially-correct']
         return None
 
     def is_queued(self, answer_id):
@@ -103,9 +105,13 @@ class CorrectMap(object):
         return self.is_queued(answer_id) and self.cmap[answer_id]['queuestate']['key'] == test_key
 
     def get_queuetime_str(self, answer_id):
-        return self.cmap[answer_id]['queuestate']['time']
+        if self.cmap[answer_id]['queuestate']:
+            return self.cmap[answer_id]['queuestate']['time']
+        else:
+            return None
 
     def get_npoints(self, answer_id):
+        """Return the number of points for an answer, used for partial credit."""
         npoints = self.get_property(answer_id, 'npoints')
         if npoints is not None:
             return npoints
@@ -152,6 +158,15 @@ class CorrectMap(object):
         if not isinstance(other_cmap, CorrectMap):
             raise Exception('CorrectMap.update called with invalid argument %s' % other_cmap)
         self.cmap.update(other_cmap.get_dict())
+        self.set_overall_message(other_cmap.get_overall_message())
 
 
+    def set_overall_message(self, message_str):
+        """ Set a message that applies to the question as a whole,
+            rather than to individual inputs. """
+        self.overall_message = str(message_str) if message_str else ""
 
+    def get_overall_message(self):
+        """ Retrieve a message that applies to the question as a whole.
+        If no message is available, returns the empty string """
+        return self.overall_message

common/lib/capa/capa/customrender.py

@@ -6,7 +6,7 @@ These tags do not have state, so they just get passed the system (for access to
 and the xml element.
 """
 
-from registry import TagRegistry
+from .registry import TagRegistry
 
 import logging
 import re
@@ -15,13 +15,15 @@ import json
 
 from lxml import etree
 import xml.sax.saxutils as saxutils
-from registry import TagRegistry
+from .registry import TagRegistry
 
-log = logging.getLogger('mitx.' + __name__)
+log = logging.getLogger(__name__)
 
 registry = TagRegistry()
 
 #-----------------------------------------------------------------------------
+
+
 class MathRenderer(object):
     tags = ['math']
 
@@ -77,6 +79,7 @@ registry.register(MathRenderer)
 
 #-----------------------------------------------------------------------------
 
+
 class SolutionRenderer(object):
     '''
     A solution is just a <span>...</span> which is given an ID, that is used for displaying an
@@ -97,4 +100,3 @@ class SolutionRenderer(object):
         return etree.XML(html)
 
 registry.register(SolutionRenderer)
-

common/lib/capa/capa/inputtypes.py

@@ -13,6 +13,9 @@ Module containing the problem elements which render into input objects
 - imageinput  (for clickable image)
 - optioninput (for option list)
 - filesubmission (upload a file)
+- crystallography
+- vsepr_input
+- drag_and_drop
 
 These are matched by *.html files templates/*.html which are mako templates with the
 actual html.
@@ -34,22 +37,26 @@ graded status as'status'
 # makes sense, but a bunch of problems have markup that assumes block.  Bigger TODO: figure out a
 # general css and layout strategy for capa, document it, then implement it.
 
-from collections import namedtuple
 import json
 import logging
 from lxml import etree
 import re
 import shlex  # for splitting quoted strings
 import sys
+import pyparsing
 
-from registry import TagRegistry
+from .registry import TagRegistry
+from capa.chem import chemcalc
+import xqueue_interface
+from datetime import datetime
 
-log = logging.getLogger('mitx.' + __name__)
+log = logging.getLogger(__name__)
 
 #########################################################################
 
 registry = TagRegistry()
 
+
 class Attribute(object):
     """
     Allows specifying required and optional attributes for input types.
@@ -90,7 +97,8 @@ class Attribute(object):
         """
         val = element.get(self.name)
         if self.default == self._sentinel and val is None:
-            raise ValueError('Missing required attribute {0}.'.format(self.name))
+            raise ValueError(
+                'Missing required attribute {0}.'.format(self.name))
 
         if val is None:
             # not required, so return default
@@ -125,6 +133,8 @@ class InputTypeBase(object):
                       * 'id' -- the id of this input, typically
                                 "{problem-location}_{response-num}_{input-num}"
                       * 'status' (answered, unanswered, unsubmitted)
+                      * 'input_state' -- dictionary containing any inputtype-specific state
+                                        that has been preserved
                       * 'feedback' (dictionary containing keys for hints, errors, or other
                          feedback from previous attempt.  Specifically 'message', 'hint',
                          'hintmode'.  If 'hintmode' is 'always', the hint is always displayed.)
@@ -142,7 +152,8 @@ class InputTypeBase(object):
 
         self.id = state.get('id', xml.get('id'))
         if self.id is None:
-            raise ValueError("input id state is None. xml is {0}".format(etree.tostring(xml)))
+            raise ValueError("input id state is None. xml is {0}".format(
+                etree.tostring(xml)))
 
         self.value = state.get('value', '')
 
@@ -150,6 +161,7 @@ class InputTypeBase(object):
         self.msg = feedback.get('message', '')
         self.hint = feedback.get('hint', '')
         self.hintmode = feedback.get('hintmode', None)
+        self.input_state = state.get('input_state', {})
 
         # put hint above msg if it should be displayed
         if self.hintmode == 'always':
@@ -162,14 +174,15 @@ class InputTypeBase(object):
             self.process_requirements()
 
             # Call subclass "constructor" -- means they don't have to worry about calling
-            # super().__init__, and are isolated from changes to the input constructor interface.
+            # super().__init__, and are isolated from changes to the input
+            # constructor interface.
             self.setup()
         except Exception as err:
             # Something went wrong: add xml to message, but keep the traceback
-            msg = "Error in xml '{x}': {err} ".format(x=etree.tostring(xml), err=str(err))
+            msg = "Error in xml '{x}': {err} ".format(
+                x=etree.tostring(xml), err=str(err))
             raise Exception, msg, sys.exc_info()[2]
 
-
     @classmethod
     def get_attributes(cls):
         """
@@ -179,7 +192,6 @@ class InputTypeBase(object):
         """
         return []
 
-
     def process_requirements(self):
         """
         Subclasses can declare lists of required and optional attributes.  This
@@ -189,7 +201,8 @@ class InputTypeBase(object):
         Processes attributes, putting the results in the self.loaded_attributes dictionary.  Also creates a set
         self.to_render, containing the names of attributes that should be included in the context by default.
         """
-        # Use local dicts and sets so that if there are exceptions, we don't end up in a partially-initialized state.
+        # Use local dicts and sets so that if there are exceptions, we don't
+        # end up in a partially-initialized state.
         loaded = {}
         to_render = set()
         for a in self.get_attributes():
@@ -210,6 +223,18 @@ class InputTypeBase(object):
         """
         pass
 
+    def handle_ajax(self, dispatch, get):
+        """
+        InputTypes that need to handle specialized AJAX should override this.
+
+        Input:
+            dispatch: a string that can be used to determine how to handle the data passed in
+            get: a dictionary containing the data that was sent with the ajax call
+
+        Output:
+            a dictionary object that can be serialized into JSON. This will be sent back to the Javascript.
+        """
+        pass
 
     def _get_render_context(self):
         """
@@ -228,8 +253,9 @@ class InputTypeBase(object):
             'value': self.value,
             'status': self.status,
             'msg': self.msg,
-            }
-        context.update((a, v) for (a, v) in self.loaded_attributes.iteritems() if a in self.to_render)
+        }
+        context.update((a, v) for (
+            a, v) in self.loaded_attributes.iteritems() if a in self.to_render)
         context.update(self._extra_context())
         return context
 
@@ -347,6 +373,11 @@ class ChoiceGroup(InputTypeBase):
 
         self.choices = self.extract_choices(self.xml)
 
+    @classmethod
+    def get_attributes(cls):
+        return [Attribute("show_correctness", "always"),
+                Attribute("submitted_message", "Answer received.")]
+
     def _extra_context(self):
         return {'input_type': self.html_input_type,
                 'choices': self.choices,
@@ -409,8 +440,7 @@ class JavascriptInput(InputTypeBase):
         return [Attribute('params', None),
                 Attribute('problem_state', None),
                 Attribute('display_class', None),
-                Attribute('display_file', None),]
-
+                Attribute('display_file', None), ]
 
     def setup(self):
         # Need to provide a value that JSON can parse if there is no
@@ -434,7 +464,6 @@ class TextLine(InputTypeBase):
     template = "textline.html"
     tags = ['textline']
 
-
     @classmethod
     def get_attributes(cls):
         """
@@ -449,12 +478,12 @@ class TextLine(InputTypeBase):
 
             # Attributes below used in setup(), not rendered directly.
             Attribute('math', None, render=False),
-            # TODO: 'dojs' flag is temporary, for backwards compatibility with 8.02x
+            # TODO: 'dojs' flag is temporary, for backwards compatibility with
+            # 8.02x
             Attribute('dojs', None, render=False),
             Attribute('preprocessorClassName', None, render=False),
             Attribute('preprocessorSrc', None, render=False),
-            ]
-
+        ]
 
     def setup(self):
         self.do_math = bool(self.loaded_attributes['math'] or
@@ -465,20 +494,21 @@ class TextLine(InputTypeBase):
         self.preprocessor = None
         if self.do_math:
             # Preprocessor to insert between raw input and Mathjax
-            self.preprocessor = {'class_name': self.loaded_attributes['preprocessorClassName'],
-                                 'script_src': self.loaded_attributes['preprocessorSrc']}
+            self.preprocessor = {
+                'class_name': self.loaded_attributes['preprocessorClassName'],
+                'script_src': self.loaded_attributes['preprocessorSrc']}
             if None in self.preprocessor.values():
                 self.preprocessor = None
 
-
     def _extra_context(self):
         return {'do_math': self.do_math,
-                'preprocessor': self.preprocessor,}
+                'preprocessor': self.preprocessor, }
 
 registry.register(TextLine)
 
 #-----------------------------------------------------------------------------
 
+
 class FileSubmission(InputTypeBase):
     """
     Upload some files (e.g. for programming assignments)
@@ -504,7 +534,7 @@ class FileSubmission(InputTypeBase):
         Convert the list of allowed files to a convenient format.
         """
         return [Attribute('allowed_files', '[]', transform=cls.parse_files),
-                Attribute('required_files', '[]', transform=cls.parse_files),]
+                Attribute('required_files', '[]', transform=cls.parse_files), ]
 
     def setup(self):
         """
@@ -513,15 +543,15 @@ class FileSubmission(InputTypeBase):
         """
         # Check if problem has been queued
         self.queue_len = 0
-        # Flag indicating that the problem has been queued, 'msg' is length of queue
+        # Flag indicating that the problem has been queued, 'msg' is length of
+        # queue
         if self.status == 'incomplete':
             self.status = 'queued'
             self.queue_len = self.msg
             self.msg = FileSubmission.submitted_msg
 
     def _extra_context(self):
-        return {'queue_len': self.queue_len,}
-        return context
+        return {'queue_len': self.queue_len, }
 
 registry.register(FileSubmission)
 
@@ -536,8 +566,9 @@ class CodeInput(InputTypeBase):
 
     template = "codeinput.html"
     tags = ['codeinput',
-            'textbox',        # Another (older) name--at some point we may want to make it use a
-                              # non-codemirror editor.
+            'textbox',
+            # Another (older) name--at some point we may want to make it use a
+            # non-codemirror editor.
             ]
 
     # pulled out for testing
@@ -560,32 +591,195 @@ class CodeInput(InputTypeBase):
                 Attribute('tabsize', 4, transform=int),
                 ]
 
-    def setup(self):
+    def setup_code_response_rendering(self):
         """
         Implement special logic: handle queueing state, and default input.
         """
-        # if no student input yet, then use the default input given by the problem
-        if not self.value:
-            self.value = self.xml.text
+        # if no student input yet, then use the default input given by the
+        # problem
+        if not self.value and self.xml.text:
+            self.value = self.xml.text.strip()
 
         # Check if problem has been queued
         self.queue_len = 0
-        # Flag indicating that the problem has been queued, 'msg' is length of queue
+        # Flag indicating that the problem has been queued, 'msg' is length of
+        # queue
         if self.status == 'incomplete':
             self.status = 'queued'
             self.queue_len = self.msg
             self.msg = self.submitted_msg
 
+
+    def setup(self):
+        ''' setup this input type '''
+        self.setup_code_response_rendering()
+
     def _extra_context(self):
         """Defined queue_len, add it """
-        return {'queue_len': self.queue_len,}
+        return {'queue_len': self.queue_len, }
 
 registry.register(CodeInput)
 
 
 #-----------------------------------------------------------------------------
+
+
+class MatlabInput(CodeInput):
+    '''
+    InputType for handling Matlab code input
+
+    TODO: API_KEY will go away once we have a way to specify it per-course
+    Example:
+     <matlabinput rows="10" cols="80" tabsize="4">
+        Initial Text
+        <plot_payload>
+          %api_key=API_KEY
+        </plot_payload>
+    </matlabinput>
+    '''
+    template = "matlabinput.html"
+    tags = ['matlabinput']
+
+    plot_submitted_msg = ("Submitted. As soon as a response is returned, "
+                    "this message will be replaced by that feedback.")
+
+    def setup(self):
+        '''
+        Handle matlab-specific parsing
+        '''
+        self.setup_code_response_rendering()
+
+        xml = self.xml
+        self.plot_payload = xml.findtext('./plot_payload')
+
+        # Check if problem has been queued
+        self.queuename = 'matlab'
+        self.queue_msg = ''
+        if 'queue_msg' in self.input_state and self.status in ['queued','incomplete', 'unsubmitted']:
+            self.queue_msg = self.input_state['queue_msg']
+        if 'queued' in self.input_state and self.input_state['queuestate'] is not None:
+            self.status = 'queued'
+            self.queue_len = 1
+            self.msg = self.plot_submitted_msg
+
+
+    def handle_ajax(self, dispatch, get):
+        ''' 
+        Handle AJAX calls directed to this input
+
+        Args:
+            - dispatch (str) - indicates how we want this ajax call to be handled
+            - get (dict) - dictionary of key-value pairs that contain useful data
+        Returns:
+            
+        '''
+
+        if dispatch == 'plot':
+            return self._plot_data(get)
+        return {}
+
+    def ungraded_response(self, queue_msg, queuekey):
+        ''' 
+        Handle the response from the XQueue
+        Stores the response in the input_state so it can be rendered later
+
+        Args:
+            - queue_msg (str) - message returned from the queue. The message to be rendered
+            - queuekey (str) - a key passed to the queue. Will be matched up to verify that this is the response we're waiting for
+
+        Returns:
+            nothing
+        '''
+        # check the queuekey against the saved queuekey
+        if('queuestate' in self.input_state and self.input_state['queuestate'] == 'queued' 
+                and self.input_state['queuekey'] == queuekey):
+            msg = self._parse_data(queue_msg)
+            # save the queue message so that it can be rendered later
+            self.input_state['queue_msg'] = msg
+            self.input_state['queuestate'] = None
+            self.input_state['queuekey'] = None
+
+    def _extra_context(self):
+        ''' Set up additional context variables'''
+        extra_context = {
+                'queue_len': self.queue_len,
+                'queue_msg': self.queue_msg
+                }
+        return extra_context
+
+    def _parse_data(self, queue_msg):
+        '''
+        Parses the message out of the queue message
+        Args:
+            queue_msg (str) - a JSON encoded string
+        Returns:
+            returns the value for the the key 'msg' in queue_msg
+        '''
+        try:
+            result = json.loads(queue_msg)
+        except (TypeError, ValueError):
+            log.error("External message should be a JSON serialized dict."
+                    " Received queue_msg = %s" % queue_msg)
+            raise
+        msg = result['msg']
+        return msg
+
+
+    def _plot_data(self, get):
+        ''' 
+        AJAX handler for the plot button
+        Args:
+            get (dict) - should have key 'submission' which contains the student submission
+        Returns:
+            dict - 'success' - whether or not we successfully queued this submission
+                 - 'message' - message to be rendered in case of error 
+        '''
+        # only send data if xqueue exists
+        if self.system.xqueue is None:
+            return {'success': False, 'message': 'Cannot connect to the queue'}
+
+        # pull relevant info out of get
+        response = get['submission']
+
+        # construct xqueue headers
+        qinterface = self.system.xqueue['interface']
+        qtime = datetime.strftime(datetime.utcnow(), xqueue_interface.dateformat)
+        callback_url = self.system.xqueue['construct_callback']('ungraded_response')
+        anonymous_student_id = self.system.anonymous_student_id
+        queuekey = xqueue_interface.make_hashkey(str(self.system.seed) + qtime +
+                                                 anonymous_student_id +
+                                                 self.id)
+        xheader = xqueue_interface.make_xheader(
+                lms_callback_url = callback_url,
+                lms_key = queuekey,
+                queue_name = self.queuename)
+
+        # save the input state
+        self.input_state['queuekey'] = queuekey
+        self.input_state['queuestate'] = 'queued'
+
+
+        # construct xqueue body
+        student_info = {'anonymous_student_id': anonymous_student_id,
+                'submission_time': qtime}
+        contents = {'grader_payload': self.plot_payload,
+                    'student_info': json.dumps(student_info),
+                    'student_response': response}
+
+        (error, msg) = qinterface.send_to_queue(header=xheader,
+                                                body = json.dumps(contents))
+
+        return {'success': error == 0, 'message': msg}
+
+
+registry.register(MatlabInput)
+
+
+#-----------------------------------------------------------------------------
+
 class Schematic(InputTypeBase):
     """
+    InputType for the schematic editor
     """
 
     template = "schematicinput.html"
@@ -602,14 +796,14 @@ class Schematic(InputTypeBase):
             Attribute('parts', None),
             Attribute('analyses', None),
             Attribute('initial_value', None),
-            Attribute('submit_analyses', None),]
+            Attribute('submit_analyses', None), ]
 
-        return context
 
 registry.register(Schematic)
 
 #-----------------------------------------------------------------------------
 
+
 class ImageInput(InputTypeBase):
     """
     Clickable image as an input field.  Element should specify the image source, height,
@@ -631,14 +825,14 @@ class ImageInput(InputTypeBase):
         """
         return [Attribute('src'),
                 Attribute('height'),
-                Attribute('width'),]
-
+                Attribute('width'), ]
 
     def setup(self):
         """
         if value is of the form [x,y] then parse it and send along coordinates of previous answer
         """
-        m = re.match('\[([0-9]+),([0-9]+)]', self.value.strip().replace(' ', ''))
+        m = re.match('\[([0-9]+),([0-9]+)]',
+                     self.value.strip().replace(' ', ''))
         if m:
             # Note: we subtract 15 to compensate for the size of the dot on the screen.
             # (is a 30x30 image--lms/static/green-pointer.png).
@@ -646,7 +840,6 @@ class ImageInput(InputTypeBase):
         else:
             (self.gx, self.gy) = (0, 0)
 
-
     def _extra_context(self):
 
         return {'gx': self.gx,
@@ -656,6 +849,7 @@ registry.register(ImageInput)
 
 #-----------------------------------------------------------------------------
 
+
 class Crystallography(InputTypeBase):
     """
     An input for crystallography -- user selects 3 points on the axes, and we get a plane.
@@ -692,7 +886,7 @@ class VseprInput(InputTypeBase):
     @classmethod
     def get_attributes(cls):
         """
-        Note: height, width are required.
+        Note: height, width, molecules and geometries are required.
         """
         return [Attribute('height'),
                 Attribute('width'),
@@ -702,7 +896,7 @@ class VseprInput(InputTypeBase):
 
 registry.register(VseprInput)
 
-#--------------------------------------------------------------------------------
+#-------------------------------------------------------------------------
 
 
 class ChemicalEquationInput(InputTypeBase):
@@ -724,62 +918,367 @@ class ChemicalEquationInput(InputTypeBase):
         """
         Can set size of text field.
         """
-        return [Attribute('size', '20'),]
+        return [Attribute('size', '20'), ]
 
     def _extra_context(self):
         """
         TODO (vshnayder): Get rid of this once we have a standard way of requiring js to be loaded.
         """
-        return {'previewer': '/static/js/capa/chemical_equation_preview.js',}
+        return {'previewer': '/static/js/capa/chemical_equation_preview.js', }
+
+    def handle_ajax(self, dispatch, get):
+        '''
+        Since we only have chemcalc preview this input, check to see if it
+        matches the corresponding dispatch and send it through if it does
+        '''
+        if dispatch == 'preview_chemcalc':
+            return self.preview_chemcalc(get)
+        return {}
+
+    def preview_chemcalc(self, get):
+        """
+        Render an html preview of a chemical formula or equation.  get should
+        contain a key 'formula' and value 'some formula string'.
+
+        Returns a json dictionary:
+        {
+           'preview' : 'the-preview-html' or ''
+           'error' : 'the-error' or ''
+        }
+        """
+
+        result = {'preview': '',
+                  'error': ''}
+        formula = get['formula']
+        if formula is None:
+            result['error'] = "No formula specified."
+            return result
+
+        try:
+            result['preview'] = chemcalc.render_to_html(formula)
+        except pyparsing.ParseException as p:
+            result['error'] = "Couldn't parse formula: {0}".format(p)
+        except Exception:
+            # this is unexpected, so log
+            log.warning(
+                "Error while previewing chemical formula", exc_info=True)
+            result['error'] = "Error while rendering preview"
+
+        return result
 
 registry.register(ChemicalEquationInput)
 
 #-----------------------------------------------------------------------------
 
-class OpenEndedInput(InputTypeBase):
+
+class DragAndDropInput(InputTypeBase):
     """
-    A text area input for code--uses codemirror, does syntax highlighting, special tab handling,
-    etc.
+    Input for drag and drop problems. Allows student to drag and drop images and
+    labels to base image.
     """
 
-    template = "openendedinput.html"
-    tags = ['openendedinput']
+    template = 'drag_and_drop_input.html'
+    tags = ['drag_and_drop_input']
 
-    # pulled out for testing
-    submitted_msg = ("Feedback not yet available.  Reload to check again. "
-                     "Once the problem is graded, this message will be "
-                     "replaced with the grader's feedback")
+    def setup(self):
+
+        def parse(tag, tag_type):
+            """Parses <tag ... /> xml element to dictionary. Stores
+                'draggable' and 'target' tags with attributes to dictionary and
+                returns last.
+
+                Args:
+                    tag: xml etree element <tag...> with attributes
+
+                    tag_type: 'draggable' or 'target'.
+
+                    If tag_type is 'draggable' : all attributes except id
+                    (name or label or icon or can_reuse) are optional
+
+                    If tag_type is 'target' all attributes (name, x, y, w, h)
+                    are required. (x, y) - coordinates of center of target,
+                    w, h - weight and height of target.
+
+                Returns:
+                    Dictionary of vaues of attributes:
+                    dict{'name': smth, 'label': smth, 'icon': smth,
+                    'can_reuse': smth}.
+            """
+            tag_attrs = dict()
+            tag_attrs['draggable'] = {'id': Attribute._sentinel,
+                                      'label': "", 'icon': "",
+                                      'can_reuse': ""}
+
+            tag_attrs['target'] = {'id': Attribute._sentinel,
+                                   'x': Attribute._sentinel,
+                                   'y': Attribute._sentinel,
+                                   'w': Attribute._sentinel,
+                                   'h': Attribute._sentinel}
+
+            dic = dict()
+
+            for attr_name in tag_attrs[tag_type].keys():
+                dic[attr_name] = Attribute(attr_name,
+                                           default=tag_attrs[tag_type][attr_name]).parse_from_xml(tag)
+
+            if tag_type == 'draggable' and not self.no_labels:
+                dic['label'] = dic['label'] or dic['id']
+
+            if tag_type == 'draggable':
+                dic['target_fields'] = [parse(target, 'target') for target in
+                                                tag.iterchildren('target')]
+
+            return dic
+
+        # add labels to images?:
+        self.no_labels = Attribute('no_labels',
+                                   default="False").parse_from_xml(self.xml)
+
+        to_js = dict()
+
+        # image drag and drop onto
+        to_js['base_image'] = Attribute('img').parse_from_xml(self.xml)
+
+        # outline places on image where to drag adn drop
+        to_js['target_outline'] = Attribute('target_outline',
+                                            default="False").parse_from_xml(self.xml)
+        # one draggable per target?
+        to_js['one_per_target'] = Attribute('one_per_target',
+                                            default="True").parse_from_xml(self.xml)
+        # list of draggables
+        to_js['draggables'] = [parse(draggable, 'draggable') for draggable in
+                               self.xml.iterchildren('draggable')]
+        # list of targets
+        to_js['targets'] = [parse(target, 'target') for target in
+                            self.xml.iterchildren('target')]
+
+        # custom background color for labels:
+        label_bg_color = Attribute('label_bg_color',
+                                   default=None).parse_from_xml(self.xml)
+        if label_bg_color:
+            to_js['label_bg_color'] = label_bg_color
+
+        self.loaded_attributes['drag_and_drop_json'] = json.dumps(to_js)
+        self.to_render.add('drag_and_drop_json')
+
+registry.register(DragAndDropInput)
+
+#-------------------------------------------------------------------------
+
+
+class EditAMoleculeInput(InputTypeBase):
+    """
+    An input type for edit-a-molecule.  Integrates with the molecule editor java applet.
+
+    Example:
+
+    <editamolecule size="50"/>
+
+    options: size -- width of the textbox.
+    """
+
+    template = "editamolecule.html"
+    tags = ['editamoleculeinput']
 
     @classmethod
     def get_attributes(cls):
         """
-        Convert options to a convenient format.
+        Can set size of text field.
         """
-        return [Attribute('rows', '30'),
-                Attribute('cols', '80'),
-                Attribute('hidden', ''),
-                ]
-
-    def setup(self):
-        """
-        Implement special logic: handle queueing state, and default input.
-        """
-        # if no student input yet, then use the default input given by the problem
-        if not self.value:
-            self.value = self.xml.text
-
-        # Check if problem has been queued
-        self.queue_len = 0
-        # Flag indicating that the problem has been queued, 'msg' is length of queue
-        if self.status == 'incomplete':
-            self.status = 'queued'
-            self.queue_len = self.msg
-            self.msg = self.submitted_msg
+        return [Attribute('file'),
+                Attribute('missing', None)]
 
     def _extra_context(self):
-        """Defined queue_len, add it """
-        return {'queue_len': self.queue_len,}
+        """
+        """
+        context = {
+            'applet_loader': '/static/js/capa/editamolecule.js',
+        }
 
-registry.register(OpenEndedInput)
+        return context
+
+registry.register(EditAMoleculeInput)
 
 #-----------------------------------------------------------------------------
+
+
+class DesignProtein2dInput(InputTypeBase):
+    """
+    An input type for design of a protein in 2D. Integrates with the Protex java applet.
+
+    Example:
+
+    <designprotein2d width="800" hight="500" target_shape="E;NE;NW;W;SW;E;none" />
+    """
+
+    template = "designprotein2dinput.html"
+    tags = ['designprotein2dinput']
+
+    @classmethod
+    def get_attributes(cls):
+        """
+        Note: width, hight, and target_shape are required.
+        """
+        return [Attribute('width'),
+                Attribute('height'),
+                Attribute('target_shape')
+                ]
+
+    def _extra_context(self):
+        """
+        """
+        context = {
+            'applet_loader': '/static/js/capa/design-protein-2d.js',
+        }
+
+        return context
+
+registry.register(DesignProtein2dInput)
+
+#-----------------------------------------------------------------------------
+
+
+class EditAGeneInput(InputTypeBase):
+    """
+        An input type for editing a gene. Integrates with the genex java applet.
+
+        Example:
+
+        <editagene width="800" hight="500" dna_sequence="ETAAGGCTATAACCGA" />
+        """
+
+    template = "editageneinput.html"
+    tags = ['editageneinput']
+
+    @classmethod
+    def get_attributes(cls):
+        """
+        Note: width, height, and dna_sequencee are required.
+        """
+        return [Attribute('width'),
+                Attribute('height'),
+                Attribute('dna_sequence'),
+                Attribute('genex_problem_number')
+                ]
+
+    def _extra_context(self):
+        """
+            """
+        context = {
+            'applet_loader': '/static/js/capa/edit-a-gene.js',
+        }
+
+        return context
+
+registry.register(EditAGeneInput)
+
+#---------------------------------------------------------------------
+
+
+class AnnotationInput(InputTypeBase):
+    """
+    Input type for annotations: students can enter some notes or other text
+    (currently ungraded), and then choose from a set of tags/optoins, which are graded.
+
+    Example:
+
+        <annotationinput>
+            <title>Annotation Exercise</title>
+            <text>
+                They are the ones who, at the public assembly, had put savage derangement [ate] into my thinking
+                [phrenes] |89 on that day when I myself deprived Achilles of his honorific portion [geras]
+            </text>
+            <comment>Agamemnon says that ate or 'derangement' was the cause of his actions: why could Zeus say the same thing?</comment>
+            <comment_prompt>Type a commentary below:</comment_prompt>
+            <tag_prompt>Select one tag:</tag_prompt>
+            <options>
+                <option choice="correct">ate - both a cause and an effect</option>
+                <option choice="incorrect">ate - a cause</option>
+                <option choice="partially-correct">ate - an effect</option>
+            </options>
+        </annotationinput>
+
+    # TODO: allow ordering to be randomized
+    """
+
+    template = "annotationinput.html"
+    tags = ['annotationinput']
+
+    def setup(self):
+        xml = self.xml
+
+        self.debug = False  # set to True to display extra debug info with input
+        self.return_to_annotation = True  # return only works in conjunction with annotatable xmodule
+
+        self.title = xml.findtext('./title', 'Annotation Exercise')
+        self.text = xml.findtext('./text')
+        self.comment = xml.findtext('./comment')
+        self.comment_prompt = xml.findtext(
+            './comment_prompt', 'Type a commentary below:')
+        self.tag_prompt = xml.findtext('./tag_prompt', 'Select one tag:')
+        self.options = self._find_options()
+
+        # Need to provide a value that JSON can parse if there is no
+        # student-supplied value yet.
+        if self.value == '':
+            self.value = 'null'
+
+        self._validate_options()
+
+    def _find_options(self):
+        ''' Returns an array of dicts where each dict represents an option. '''
+        elements = self.xml.findall('./options/option')
+        return [{
+                'id': index,
+                'description': option.text,
+                'choice': option.get('choice')
+                } for (index, option) in enumerate(elements)]
+
+    def _validate_options(self):
+        ''' Raises a ValueError if the choice attribute is missing or invalid. '''
+        valid_choices = ('correct', 'partially-correct', 'incorrect')
+        for option in self.options:
+            choice = option['choice']
+            if choice is None:
+                raise ValueError('Missing required choice attribute.')
+            elif choice not in valid_choices:
+                raise ValueError('Invalid choice attribute: {0}. Must be one of: {1}'.format(
+                    choice, ', '.join(valid_choices)))
+
+    def _unpack(self, json_value):
+        ''' Unpacks the json input state into a dict. '''
+        d = json.loads(json_value)
+        if type(d) != dict:
+            d = {}
+
+        comment_value = d.get('comment', '')
+        if not isinstance(comment_value, basestring):
+            comment_value = ''
+
+        options_value = d.get('options', [])
+        if not isinstance(options_value, list):
+            options_value = []
+
+        return {
+            'options_value': options_value,
+            'has_options_value': len(options_value) > 0,  # for convenience
+            'comment_value': comment_value,
+        }
+
+    def _extra_context(self):
+        extra_context = {
+            'title': self.title,
+            'text': self.text,
+            'comment': self.comment,
+            'comment_prompt': self.comment_prompt,
+            'tag_prompt': self.tag_prompt,
+            'options': self.options,
+            'return_to_annotation': self.return_to_annotation,
+            'debug': self.debug
+        }
+
+        extra_context.update(self._unpack(self.value))
+
+        return extra_context
+
+registry.register(AnnotationInput)

common/lib/capa/capa/responsetypes.py

@@ -28,15 +28,15 @@ from collections import namedtuple
 from shapely.geometry import Point, MultiPoint
 
 # specific library imports
-from calc import evaluator, UndefinedVariable
-from correctmap import CorrectMap
+from .calc import evaluator, UndefinedVariable
+from .correctmap import CorrectMap
 from datetime import datetime
-from util import *
+from .util import *
 from lxml import etree
-from lxml.html.soupparser import fromstring as fromstring_bs	 # uses Beautiful Soup!!! FIXME?
+from lxml.html.soupparser import fromstring as fromstring_bs     # uses Beautiful Soup!!! FIXME?
 import xqueue_interface
 
-log = logging.getLogger('mitx.' + __name__)
+log = logging.getLogger(__name__)
 
 
 #-----------------------------------------------------------------------------
@@ -101,7 +101,6 @@ class LoncapaResponse(object):
 
       - hint_tag             : xhtml tag identifying hint associated with this response inside
                                hintgroup
-
     """
     __metaclass__ = abc.ABCMeta  # abc = Abstract Base Class
 
@@ -129,21 +128,25 @@ class LoncapaResponse(object):
 
         for abox in inputfields:
             if abox.tag not in self.allowed_inputfields:
-                msg = "%s: cannot have input field %s" % (unicode(self), abox.tag)
-                msg += "\nSee XML source line %s" % getattr(xml, 'sourceline', '<unavailable>')
+                msg = "%s: cannot have input field %s" % (
+                    unicode(self), abox.tag)
+                msg += "\nSee XML source line %s" % getattr(
+                    xml, 'sourceline', '<unavailable>')
                 raise LoncapaProblemError(msg)
 
         if self.max_inputfields and len(inputfields) > self.max_inputfields:
             msg = "%s: cannot have more than %s input fields" % (
                 unicode(self), self.max_inputfields)
-            msg += "\nSee XML source line %s" % getattr(xml, 'sourceline', '<unavailable>')
+            msg += "\nSee XML source line %s" % getattr(
+                xml, 'sourceline', '<unavailable>')
             raise LoncapaProblemError(msg)
 
         for prop in self.required_attributes:
             if not xml.get(prop):
                 msg = "Error in problem specification: %s missing required attribute %s" % (
                     unicode(self), prop)
-                msg += "\nSee XML source line %s" % getattr(xml, 'sourceline', '<unavailable>')
+                msg += "\nSee XML source line %s" % getattr(
+                    xml, 'sourceline', '<unavailable>')
                 raise LoncapaProblemError(msg)
 
         # ordered list of answer_id values for this response
@@ -164,7 +167,8 @@ class LoncapaResponse(object):
         for entry in self.inputfields:
             answer = entry.get('correct_answer')
             if answer:
-                self.default_answer_map[entry.get('id')] = contextualize_text(answer, self.context)
+                self.default_answer_map[entry.get(
+                    'id')] = contextualize_text(answer, self.context)
 
         if hasattr(self, 'setup_response'):
             self.setup_response()
@@ -175,22 +179,33 @@ class LoncapaResponse(object):
         '''
         return sum(self.maxpoints.values())
 
-    def render_html(self, renderer):
+    def render_html(self, renderer, response_msg=''):
         '''
         Return XHTML Element tree representation of this Response.
 
         Arguments:
 
           - renderer : procedure which produces HTML given an ElementTree
+          - response_msg: a message displayed at the end of the Response
         '''
         # render ourself as a <span> + our content
         tree = etree.Element('span')
+
+        # problem author can make this span display:inline
+        if self.xml.get('inline', ''):
+            tree.set('class', 'inline')
+
         for item in self.xml:
             # call provided procedure to do the rendering
             item_xhtml = renderer(item)
             if item_xhtml is not None:
                 tree.append(item_xhtml)
         tree.tail = self.xml.tail
+
+        # Add a <div> for the message at the end of the response
+        if response_msg:
+            tree.append(self._render_response_msg_html(response_msg))
+
         return tree
 
     def evaluate_answers(self, student_answers, old_cmap):
@@ -201,7 +216,8 @@ class LoncapaResponse(object):
         Returns the new CorrectMap, with (correctness,msg,hint,hintmode) for each answer_id.
         '''
         new_cmap = self.get_score(student_answers)
-        self.get_hints(convert_files_to_filenames(student_answers), new_cmap, old_cmap)
+        self.get_hints(convert_files_to_filenames(
+            student_answers), new_cmap, old_cmap)
         # log.debug('new_cmap = %s' % new_cmap)
         return new_cmap
 
@@ -221,26 +237,27 @@ class LoncapaResponse(object):
         # hint specified by function?
         hintfn = hintgroup.get('hintfn')
         if hintfn:
-            '''
-            Hint is determined by a function defined in the <script> context; evaluate
-            that function to obtain list of hint, hintmode for each answer_id.
+            # Hint is determined by a function defined in the <script> context; evaluate
+            # that function to obtain list of hint, hintmode for each answer_id.
 
-            The function should take arguments (answer_ids, student_answers, new_cmap, old_cmap)
-            and it should modify new_cmap as appropriate.
+            # The function should take arguments (answer_ids, student_answers, new_cmap, old_cmap)
+            # and it should modify new_cmap as appropriate.
 
-            We may extend this in the future to add another argument which provides a
-            callback procedure to a social hint generation system.
-            '''
+            # We may extend this in the future to add another argument which provides a
+            # callback procedure to a social hint generation system.
             if not hintfn in self.context:
                 msg = 'missing specified hint function %s in script context' % hintfn
-                msg += "\nSee XML source line %s" % getattr(self.xml, 'sourceline', '<unavailable>')
+                msg += "\nSee XML source line %s" % getattr(
+                    self.xml, 'sourceline', '<unavailable>')
                 raise LoncapaProblemError(msg)
 
             try:
-                self.context[hintfn](self.answer_ids, student_answers, new_cmap, old_cmap)
+                self.context[hintfn](
+                    self.answer_ids, student_answers, new_cmap, old_cmap)
             except Exception as err:
                 msg = 'Error %s in evaluating hint function %s' % (err, hintfn)
-                msg += "\nSee XML source line %s" % getattr(self.xml, 'sourceline', '<unavailable>')
+                msg += "\nSee XML source line %s" % getattr(
+                    self.xml, 'sourceline', '<unavailable>')
                 raise ResponseError(msg)
             return
 
@@ -262,17 +279,19 @@ class LoncapaResponse(object):
 
         if (self.hint_tag is not None
             and hintgroup.find(self.hint_tag) is not None
-            and hasattr(self, 'check_hint_condition')):
+                and hasattr(self, 'check_hint_condition')):
 
             rephints = hintgroup.findall(self.hint_tag)
-            hints_to_show = self.check_hint_condition(rephints, student_answers)
+            hints_to_show = self.check_hint_condition(
+                rephints, student_answers)
 
             # can be 'on_request' or 'always' (default)
             hintmode = hintgroup.get('mode', 'always')
             for hintpart in hintgroup.findall('hintpart'):
                 if hintpart.get('on') in hints_to_show:
                     hint_text = hintpart.find('text').text
-                    # make the hint appear after the last answer box in this response
+                    # make the hint appear after the last answer box in this
+                    # response
                     aid = self.answer_ids[-1]
                     new_cmap.set_hint_and_mode(aid, hint_text, hintmode)
             log.debug('after hint: new_cmap = %s' % new_cmap)
@@ -315,6 +334,28 @@ class LoncapaResponse(object):
     def __unicode__(self):
         return u'LoncapaProblem Response %s' % self.xml.tag
 
+    def _render_response_msg_html(self, response_msg):
+        """ Render a <div> for a message that applies to the entire response.
+
+        *response_msg* is a string, which may contain XHTML markup
+
+        Returns an etree element representing the response message <div> """
+        # First try wrapping the text in a <div> and parsing
+        # it as an XHTML tree
+        try:
+            response_msg_div = etree.XML('<div>%s</div>' % str(response_msg))
+
+        # If we can't do that, create the <div> and set the message
+        # as the text of the <div>
+        except:
+            response_msg_div = etree.Element('div')
+            response_msg_div.text = str(response_msg)
+
+        # Set the css class of the message <div>
+        response_msg_div.set("class", "response_message")
+
+        return response_msg_div
+
 
 #-----------------------------------------------------------------------------
 
@@ -353,20 +394,20 @@ class JavascriptResponse(LoncapaResponse):
         # until we decide on exactly how to solve this issue. For now, files are
         # manually being compiled to DATA_DIR/js/compiled.
 
-        #latestTimestamp = 0
-        #basepath = self.system.filestore.root_path + '/js/'
-        #for filename in (self.display_dependencies + [self.display]):
+        # latestTimestamp = 0
+        # basepath = self.system.filestore.root_path + '/js/'
+        # for filename in (self.display_dependencies + [self.display]):
         #    filepath = basepath + filename
         #    timestamp = os.stat(filepath).st_mtime
         #    if timestamp > latestTimestamp:
         #        latestTimestamp = timestamp
         #
-        #h = hashlib.md5()
-        #h.update(self.answer_id + str(self.display_dependencies))
-        #compiled_filename = 'compiled/' + h.hexdigest() + '.js'
-        #compiled_filepath = basepath + compiled_filename
+        # h = hashlib.md5()
+        # h.update(self.answer_id + str(self.display_dependencies))
+        # compiled_filename = 'compiled/' + h.hexdigest() + '.js'
+        # compiled_filepath = basepath + compiled_filename
 
-        #if not os.path.exists(compiled_filepath) or os.stat(compiled_filepath).st_mtime < latestTimestamp:
+        # if not os.path.exists(compiled_filepath) or os.stat(compiled_filepath).st_mtime < latestTimestamp:
         #    outfile = open(compiled_filepath, 'w')
         #    for filename in (self.display_dependencies + [self.display]):
         #        filepath = basepath + filename
@@ -388,7 +429,7 @@ class JavascriptResponse(LoncapaResponse):
                                          id=self.xml.get('id'))[0]
 
         self.display_xml = self.xml.xpath('//*[@id=$id]//display',
-                                         id=self.xml.get('id'))[0]
+                                          id=self.xml.get('id'))[0]
 
         self.xml.remove(self.generator_xml)
         self.xml.remove(self.grader_xml)
@@ -399,17 +440,20 @@ class JavascriptResponse(LoncapaResponse):
         self.display = self.display_xml.get("src")
 
         if self.generator_xml.get("dependencies"):
-            self.generator_dependencies = self.generator_xml.get("dependencies").split()
+            self.generator_dependencies = self.generator_xml.get(
+                "dependencies").split()
         else:
             self.generator_dependencies = []
 
         if self.grader_xml.get("dependencies"):
-            self.grader_dependencies = self.grader_xml.get("dependencies").split()
+            self.grader_dependencies = self.grader_xml.get(
+                "dependencies").split()
         else:
             self.grader_dependencies = []
 
         if self.display_xml.get("dependencies"):
-            self.display_dependencies = self.display_xml.get("dependencies").split()
+            self.display_dependencies = self.display_xml.get(
+                "dependencies").split()
         else:
             self.display_dependencies = []
 
@@ -430,10 +474,10 @@ class JavascriptResponse(LoncapaResponse):
 
         return subprocess.check_output(subprocess_args, env=self.get_node_env())
 
-
     def generate_problem_state(self):
 
-        generator_file = os.path.dirname(os.path.normpath(__file__)) + '/javascript_problem_generator.js'
+        generator_file = os.path.dirname(os.path.normpath(
+            __file__)) + '/javascript_problem_generator.js'
         output = self.call_node([generator_file,
                                  self.generator,
                                  json.dumps(self.generator_dependencies),
@@ -447,17 +491,18 @@ class JavascriptResponse(LoncapaResponse):
         params = {}
 
         for param in self.xml.xpath('//*[@id=$id]//responseparam',
-                                        id=self.xml.get('id')):
+                                    id=self.xml.get('id')):
 
             raw_param = param.get("value")
-            params[param.get("name")] = json.loads(contextualize_text(raw_param, self.context))
+            params[param.get("name")] = json.loads(
+                contextualize_text(raw_param, self.context))
 
         return params
 
     def prepare_inputfield(self):
 
         for inputfield in self.xml.xpath('//*[@id=$id]//javascriptinput',
-                                        id=self.xml.get('id')):
+                                         id=self.xml.get('id')):
 
             escapedict = {'"': '&quot;'}
 
@@ -470,7 +515,7 @@ class JavascriptResponse(LoncapaResponse):
                                                     escapedict)
             inputfield.set("problem_state", encoded_problem_state)
 
-            inputfield.set("display_file",  self.display_filename)
+            inputfield.set("display_file", self.display_filename)
             inputfield.set("display_class", self.display_class)
 
     def get_score(self, student_answers):
@@ -488,7 +533,8 @@ class JavascriptResponse(LoncapaResponse):
         if submission is None or submission == '':
             submission = json.dumps(None)
 
-        grader_file = os.path.dirname(os.path.normpath(__file__)) + '/javascript_problem_grader.js'
+        grader_file = os.path.dirname(os.path.normpath(
+            __file__)) + '/javascript_problem_grader.js'
         outputs = self.call_node([grader_file,
                                   self.grader,
                                   json.dumps(self.grader_dependencies),
@@ -497,8 +543,8 @@ class JavascriptResponse(LoncapaResponse):
                                   json.dumps(self.params)]).split('\n')
 
         all_correct = json.loads(outputs[0].strip())
-        evaluation  = outputs[1].strip()
-        solution    = outputs[2].strip()
+        evaluation = outputs[1].strip()
+        solution = outputs[2].strip()
         return (all_correct, evaluation, solution)
 
     def get_answers(self):
@@ -508,9 +554,7 @@ class JavascriptResponse(LoncapaResponse):
         return {self.answer_id: self.solution}
 
 
-
 #-----------------------------------------------------------------------------
-
 class ChoiceResponse(LoncapaResponse):
     """
     This response type is used when the student chooses from a discrete set of
@@ -568,9 +612,10 @@ class ChoiceResponse(LoncapaResponse):
         self.assign_choice_names()
 
         correct_xml = self.xml.xpath('//*[@id=$id]//choice[@correct="true"]',
-                                         id=self.xml.get('id'))
+                                     id=self.xml.get('id'))
 
-        self.correct_choices = set([choice.get('name') for choice in correct_xml])
+        self.correct_choices = set([choice.get(
+            'name') for choice in correct_xml])
 
     def assign_choice_names(self):
         '''
@@ -623,13 +668,20 @@ class MultipleChoiceResponse(LoncapaResponse):
     allowed_inputfields = ['choicegroup']
 
     def setup_response(self):
-        # call secondary setup for MultipleChoice questions, to set name attributes
+        # call secondary setup for MultipleChoice questions, to set name
+        # attributes
         self.mc_setup_response()
 
         # define correct choices (after calling secondary setup)
         xml = self.xml
-        cxml = xml.xpath('//*[@id=$id]//choice[@correct="true"]', id=xml.get('id'))
-        self.correct_choices = [choice.get('name') for choice in cxml]
+        cxml = xml.xpath('//*[@id=$id]//choice', id=xml.get('id'))
+
+        # contextualize correct attribute and then select ones for which
+        # correct = "true"
+        self.correct_choices = [
+            contextualize_text(choice.get('name'), self.context)
+            for choice in cxml
+            if contextualize_text(choice.get('correct'), self.context) == "true"]
 
     def mc_setup_response(self):
         '''
@@ -655,7 +707,7 @@ class MultipleChoiceResponse(LoncapaResponse):
         # log.debug('%s: student_answers=%s, correct_choices=%s' % (
         #   unicode(self), student_answers, self.correct_choices))
         if (self.answer_id in student_answers
-            and student_answers[self.answer_id] in self.correct_choices):
+                and student_answers[self.answer_id] in self.correct_choices):
             return CorrectMap(self.answer_id, 'correct')
         else:
             return CorrectMap(self.answer_id, 'incorrect')
@@ -723,7 +775,8 @@ class OptionResponse(LoncapaResponse):
         return cmap
 
     def get_answers(self):
-        amap = dict([(af.get('id'), af.get('correct')) for af in self.answer_fields])
+        amap = dict([(af.get('id'), contextualize_text(af.get(
+            'correct'), self.context)) for af in self.answer_fields])
         # log.debug('%s: expected answers=%s' % (unicode(self),amap))
         return amap
 
@@ -743,8 +796,9 @@ class NumericalResponse(LoncapaResponse):
         context = self.context
         self.correct_answer = contextualize_text(xml.get('answer'), context)
         try:
-            self.tolerance_xml = xml.xpath('//*[@id=$id]//responseparam[@type="tolerance"]/@default',
-                                           id=xml.get('id'))[0]
+            self.tolerance_xml = xml.xpath(
+                '//*[@id=$id]//responseparam[@type="tolerance"]/@default',
+                id=xml.get('id'))[0]
             self.tolerance = contextualize_text(self.tolerance_xml, context)
         except Exception:
             self.tolerance = '0'
@@ -761,17 +815,21 @@ class NumericalResponse(LoncapaResponse):
         try:
             correct_ans = complex(self.correct_answer)
         except ValueError:
-            log.debug("Content error--answer '{0}' is not a valid complex number".format(self.correct_answer))
-            raise StudentInputError("There was a problem with the staff answer to this problem")
+            log.debug("Content error--answer '{0}' is not a valid complex number".format(
+                self.correct_answer))
+            raise StudentInputError(
+                "There was a problem with the staff answer to this problem")
 
         try:
-            correct = compare_with_tolerance(evaluator(dict(), dict(), student_answer),
-                                             correct_ans, self.tolerance)
+            correct = compare_with_tolerance(
+                evaluator(dict(), dict(), student_answer),
+                correct_ans, self.tolerance)
         # We should catch this explicitly.
         # I think this is just pyparsing.ParseException, calc.UndefinedVariable:
         # But we'd need to confirm
         except:
-            # Use the traceback-preserving version of re-raising with a different type
+            # Use the traceback-preserving version of re-raising with a
+            # different type
             import sys
             type, value, traceback = sys.exc_info()
 
@@ -800,7 +858,8 @@ class StringResponse(LoncapaResponse):
     max_inputfields = 1
 
     def setup_response(self):
-        self.correct_answer = contextualize_text(self.xml.get('answer'), self.context).strip()
+        self.correct_answer = contextualize_text(
+            self.xml.get('answer'), self.context).strip()
 
     def get_score(self, student_answers):
         '''Grade a string response '''
@@ -809,7 +868,8 @@ class StringResponse(LoncapaResponse):
         return CorrectMap(self.answer_id, 'correct' if correct else 'incorrect')
 
     def check_string(self, expected, given):
-        if self.xml.get('type') == 'ci': return given.lower() == expected.lower()
+        if self.xml.get('type') == 'ci':
+            return given.lower() == expected.lower()
         return given == expected
 
     def check_hint_condition(self, hxml_set, student_answers):
@@ -817,8 +877,10 @@ class StringResponse(LoncapaResponse):
         hints_to_show = []
         for hxml in hxml_set:
             name = hxml.get('name')
-            correct_answer = contextualize_text(hxml.get('answer'), self.context).strip()
-            if self.check_string(correct_answer, given): hints_to_show.append(name)
+            correct_answer = contextualize_text(
+                hxml.get('answer'), self.context).strip()
+            if self.check_string(correct_answer, given):
+                hints_to_show.append(name)
         log.debug('hints_to_show = %s' % hints_to_show)
         return hints_to_show
 
@@ -833,7 +895,7 @@ class CustomResponse(LoncapaResponse):
     Custom response.  The python code to be run should be in <answer>...</answer>
     or in a <script>...</script>
     '''
-    snippets = [{'snippet': """<customresponse>
+    snippets = [{'snippet': r"""<customresponse>
     <text>
     <br/>
     Suppose that \(I(t)\) rises from \(0\) to \(I_S\) at a time \(t_0 \neq 0\)
@@ -852,7 +914,7 @@ class CustomResponse(LoncapaResponse):
         correct[0] ='incorrect'
     </answer>
     </customresponse>"""},
-    {'snippet': """<script type="loncapa/python"><![CDATA[
+                {'snippet': """<script type="loncapa/python"><![CDATA[
 
 def sympy_check2():
   messages[0] = '%s:%s' % (submission[0],fromjs[0].replace('<','&lt;'))
@@ -869,12 +931,17 @@ def sympy_check2():
 
     response_tag = 'customresponse'
 
-    allowed_inputfields = ['textline', 'textbox', 'crystallography', 'chemicalequationinput', 'vsepr_input']
+    allowed_inputfields = ['textline', 'textbox', 'crystallography',
+                           'chemicalequationinput', 'vsepr_input',
+                           'drag_and_drop_input', 'editamoleculeinput',
+                           'designprotein2dinput', 'editageneinput',
+                           'annotationinput']
 
     def setup_response(self):
         xml = self.xml
 
-        # if <customresponse> has an "expect" (or "answer") attribute then save that
+        # if <customresponse> has an "expect" (or "answer") attribute then save
+        # that
         self.expect = xml.get('expect') or xml.get('answer')
         self.myid = xml.get('id')
 
@@ -898,7 +965,8 @@ def sympy_check2():
                 if cfn in self.context:
                     self.code = self.context[cfn]
                 else:
-                    msg = "%s: can't find cfn %s in context" % (unicode(self), cfn)
+                    msg = "%s: can't find cfn %s in context" % (
+                        unicode(self), cfn)
                     msg += "\nSee XML source line %s" % getattr(self.xml, 'sourceline',
                                                                 '<unavailable>')
                     raise LoncapaProblemError(msg)
@@ -911,7 +979,8 @@ def sympy_check2():
             else:
                 answer_src = answer.get('src')
                 if answer_src is not None:
-                    self.code = self.system.filesystem.open('src/' + answer_src).read()
+                    self.code = self.system.filesystem.open(
+                        'src/' + answer_src).read()
                 else:
                     self.code = answer.text
 
@@ -952,6 +1021,7 @@ def sympy_check2():
         # not expecting 'unknown's
         correct = ['unknown'] * len(idset)
         messages = [''] * len(idset)
+        overall_message = ""
 
         # put these in the context of the check function evaluator
         # note that this doesn't help the "cfn" version - only the exec version
@@ -983,26 +1053,32 @@ def sympy_check2():
             # the list of messages to be filled in by the check function
             'messages': messages,
 
+            # a message that applies to the entire response
+            # instead of a particular input
+            'overall_message': overall_message,
+
             # any options to be passed to the cfn
             'options': self.xml.get('options'),
             'testdat': 'hello world',
-            })
+        })
 
         # pass self.system.debug to cfn
         self.context['debug'] = self.system.DEBUG
 
         # exec the check function
-        if type(self.code) == str:
+        if isinstance(self.code, basestring):
             try:
                 exec self.code in self.context['global_context'], self.context
                 correct = self.context['correct']
                 messages = self.context['messages']
+                overall_message = self.context['overall_message']
             except Exception as err:
                 print "oops in customresponse (code) error %s" % err
                 print "context = ", self.context
                 print traceback.format_exc()
                 # Notify student
-                raise StudentInputError("Error: Problem could not be evaluated with your input")
+                raise StudentInputError(
+                    "Error: Problem could not be evaluated with your input")
         else:
             # self.code is not a string; assume its a function
 
@@ -1011,18 +1087,22 @@ def sympy_check2():
             ret = None
             log.debug(" submission = %s" % submission)
             try:
-                answer_given = submission[0] if (len(idset) == 1) else submission
+                answer_given = submission[0] if (
+                    len(idset) == 1) else submission
                 # handle variable number of arguments in check function, for backwards compatibility
                 # with various Tutor2 check functions
-                args = [self.expect, answer_given, student_answers, self.answer_ids[0]]
+                args = [self.expect, answer_given,
+                        student_answers, self.answer_ids[0]]
                 argspec = inspect.getargspec(fn)
                 nargs = len(argspec.args) - len(argspec.defaults or [])
                 kwargs = {}
                 for argname in argspec.args[nargs:]:
-                    kwargs[argname] = self.context[argname] if argname in self.context else None
+                    kwargs[argname] = self.context[
+                        argname] if argname in self.context else None
 
                 log.debug('[customresponse] answer_given=%s' % answer_given)
-                log.debug('nargs=%d, args=%s, kwargs=%s' % (nargs, args, kwargs))
+                log.debug('nargs=%d, args=%s, kwargs=%s' % (
+                    nargs, args, kwargs))
 
                 ret = fn(*args[:nargs], **kwargs)
             except Exception as err:
@@ -1030,35 +1110,107 @@ def sympy_check2():
                 # print "context = ",self.context
                 log.error(traceback.format_exc())
                 raise Exception("oops in customresponse (cfn) error %s" % err)
-            log.debug("[courseware.capa.responsetypes.customresponse.get_score] ret = %s" % ret)
+            log.debug(
+                "[courseware.capa.responsetypes.customresponse.get_score] ret = %s" % ret)
+
             if type(ret) == dict:
-                correct = ['correct'] * len(idset) if ret['ok'] else ['incorrect'] * len(idset)
-                msg = ret['msg']
 
-                if 1:
-                    # try to clean up message html
-                    msg = '<html>' + msg + '</html>'
-                    msg = msg.replace('&#60;', '&lt;')
-                    #msg = msg.replace('&lt;','<')
-                    msg = etree.tostring(fromstring_bs(msg, convertEntities=None),
-                                         pretty_print=True)
-                    #msg = etree.tostring(fromstring_bs(msg),pretty_print=True)
-                    msg = msg.replace('&#13;', '')
-                    #msg = re.sub('<html>(.*)</html>','\\1',msg,flags=re.M|re.DOTALL)	# python 2.7
-                    msg = re.sub('(?ms)<html>(.*)</html>', '\\1', msg)
+                # One kind of dictionary the check function can return has the
+                # form {'ok': BOOLEAN, 'msg': STRING}
+                # If there are multiple inputs, they all get marked
+                # to the same correct/incorrect value
+                if 'ok' in ret:
+                    correct = ['correct'] * len(idset) if ret[
+                        'ok'] else ['incorrect'] * len(idset)
+                    msg = ret.get('msg', None)
+                    msg = self.clean_message_html(msg)
 
-                messages[0] = msg
+                    # If there is only one input, apply the message to that input
+                    # Otherwise, apply the message to the whole problem
+                    if len(idset) > 1:
+                        overall_message = msg
+                    else:
+                        messages[0] = msg
+
+                # Another kind of dictionary the check function can return has
+                # the form:
+                # {'overall_message': STRING,
+                #  'input_list': [{ 'ok': BOOLEAN, 'msg': STRING }, ...] }
+                #
+                # This allows the function to return an 'overall message'
+                # that applies to the entire problem, as well as correct/incorrect
+                # status and messages for individual inputs
+                elif 'input_list' in ret:
+                    overall_message = ret.get('overall_message', '')
+                    input_list = ret['input_list']
+
+                    correct = []
+                    messages = []
+                    for input_dict in input_list:
+                        correct.append('correct' 
+                                if input_dict['ok'] else 'incorrect')
+                        msg = (self.clean_message_html(input_dict['msg']) 
+                                if 'msg' in input_dict else None)
+                        messages.append(msg)
+
+                # Otherwise, we do not recognize the dictionary
+                # Raise an exception
+                else:
+                    log.error(traceback.format_exc())
+                    raise Exception(
+                        "CustomResponse: check function returned an invalid dict")
+
+            # The check function can return a boolean value,
+            # indicating whether all inputs should be marked
+            # correct or incorrect
             else:
-                correct = ['correct'] * len(idset) if ret else ['incorrect'] * len(idset)
+                n = len(idset)
+                correct = ['correct'] * n if ret else ['incorrect'] * n
 
         # build map giving "correct"ness of the answer(s)
         correct_map = CorrectMap()
+
+        overall_message = self.clean_message_html(overall_message)
+        correct_map.set_overall_message(overall_message)
+
         for k in range(len(idset)):
-            npoints = self.maxpoints[idset[k]] if correct[k] == 'correct' else 0
+            npoints = (self.maxpoints[idset[k]] 
+                    if correct[k] == 'correct' else 0)
             correct_map.set(idset[k], correct[k], msg=messages[k],
                             npoints=npoints)
         return correct_map
 
+    def clean_message_html(self, msg):
+
+        # If *msg* is an empty string, then the code below
+        # will return "</html>".  To avoid this, we first check
+        # that *msg* is a non-empty string.
+        if msg:
+
+            # When we parse *msg* using etree, there needs to be a root
+            # element, so we wrap the *msg* text in <html> tags
+            msg = '<html>' + msg + '</html>'
+
+            # Replace < characters
+            msg = msg.replace('&#60;', '&lt;')
+
+            # Use etree to prettify the HTML
+            msg = etree.tostring(fromstring_bs(msg, convertEntities=None),
+                                 pretty_print=True)
+
+            msg = msg.replace('&#13;', '')
+
+            # Remove the <html> tags we introduced earlier, so we're
+            # left with just the prettified message markup
+            msg = re.sub('(?ms)<html>(.*)</html>', '\\1', msg)
+
+            # Strip leading and trailing whitespace
+            return msg.strip()
+
+        # If we start with an empty string, then return an empty string
+        else:
+            return ""
+
     def get_answers(self):
         '''
         Give correct answer expected for this response.
@@ -1082,7 +1234,7 @@ class SymbolicResponse(CustomResponse):
     """
     Symbolic math response checking, using symmath library.
     """
-    snippets = [{'snippet': '''<problem>
+    snippets = [{'snippet': r'''<problem>
       <text>Compute \[ \exp\left(-i \frac{\theta}{2} \left[ \begin{matrix} 0 & 1 \\ 1 & 0 \end{matrix} \right] \right) \]
       and give the resulting \(2\times 2\) matrix: <br/>
         <symbolicresponse answer="">
@@ -1119,8 +1271,9 @@ class CodeResponse(LoncapaResponse):
 
     Expects 'xqueue' dict in ModuleSystem with the following keys that are needed by CodeResponse:
         system.xqueue = { 'interface': XqueueInterface object,
-                          'callback_url': Per-StudentModule callback URL
-                                          where results are posted (string),
+                          'construct_callback': Per-StudentModule callback URL
+                                          constructor, defaults to using 'score_update'
+                                          as the correct dispatch (function),
                           'default_queuename': Default queuename to submit request (string)
                         }
 
@@ -1129,7 +1282,7 @@ class CodeResponse(LoncapaResponse):
     """
 
     response_tag = 'coderesponse'
-    allowed_inputfields = ['textbox', 'filesubmission']
+    allowed_inputfields = ['textbox', 'filesubmission', 'matlabinput']
     max_inputfields = 1
 
     def setup_response(self):
@@ -1141,10 +1294,17 @@ class CodeResponse(LoncapaResponse):
         xml = self.xml
         # TODO: XML can override external resource (grader/queue) URL
         self.url = xml.get('url', None)
-        self.queue_name = xml.get('queuename', self.system.xqueue['default_queuename'])
+
+        # We do not support xqueue within Studio.
+        if self.system.xqueue is not None:
+            default_queuename = self.system.xqueue['default_queuename']
+        else:
+            default_queuename = None
+        self.queue_name = xml.get('queuename', default_queuename)
 
         # VS[compat]:
-        #   Check if XML uses the ExternalResponse format or the generic CodeResponse format
+        # Check if XML uses the ExternalResponse format or the generic
+        # CodeResponse format
         codeparam = self.xml.find('codeparam')
         if codeparam is None:
             self._parse_externalresponse_xml()
@@ -1158,12 +1318,14 @@ class CodeResponse(LoncapaResponse):
             self.answer (an answer to display to the student in the LMS)
             self.payload
         '''
-        # Note that CodeResponse is agnostic to the specific contents of grader_payload
+        # Note that CodeResponse is agnostic to the specific contents of
+        # grader_payload
         grader_payload = codeparam.find('grader_payload')
         grader_payload = grader_payload.text if grader_payload is not None else ''
         self.payload = {'grader_payload': grader_payload}
 
-        self.initial_display = find_with_default(codeparam, 'initial_display', '')
+        self.initial_display = find_with_default(
+            codeparam, 'initial_display', '')
         self.answer = find_with_default(codeparam, 'answer_display',
                                         'No answer provided.')
 
@@ -1185,8 +1347,10 @@ class CodeResponse(LoncapaResponse):
         else:  # no <answer> stanza; get code from <script>
             code = self.context['script_code']
             if not code:
-                msg = '%s: Missing answer script code for coderesponse' % unicode(self)
-                msg += "\nSee XML source line %s" % getattr(self.xml, 'sourceline', '<unavailable>')
+                msg = '%s: Missing answer script code for coderesponse' % unicode(
+                    self)
+                msg += "\nSee XML source line %s" % getattr(
+                    self.xml, 'sourceline', '<unavailable>')
                 raise LoncapaProblemError(msg)
 
         tests = self.xml.get('tests')
@@ -1201,7 +1365,8 @@ class CodeResponse(LoncapaResponse):
         try:
             exec(code, penv, penv)
         except Exception as err:
-            log.error('Error in CodeResponse %s: Error in problem reference code' % err)
+            log.error(
+                'Error in CodeResponse %s: Error in problem reference code' % err)
             raise Exception(err)
         try:
             self.answer = penv['answer']
@@ -1214,7 +1379,7 @@ class CodeResponse(LoncapaResponse):
         # Finally, make the ExternalResponse input XML format conform to the generic
         # exteral grader interface
         #   The XML tagging of grader_payload is pyxserver-specific
-        grader_payload  = '<pyxserver>'
+        grader_payload = '<pyxserver>'
         grader_payload += '<tests>' + tests + '</tests>\n'
         grader_payload += '<processor>' + code + '</processor>'
         grader_payload += '</pyxserver>'
@@ -1227,9 +1392,16 @@ class CodeResponse(LoncapaResponse):
         except Exception as err:
             log.error('Error in CodeResponse %s: cannot get student answer for %s;'
                       ' student_answers=%s' %
-                (err, self.answer_id, convert_files_to_filenames(student_answers)))
+                     (err, self.answer_id, convert_files_to_filenames(student_answers)))
             raise Exception(err)
 
+        # We do not support xqueue within Studio.
+        if self.system.xqueue is None:
+            cmap = CorrectMap()
+            cmap.set(self.answer_id, queuestate=None,
+                     msg='Error checking problem: no external queueing server is configured.')
+            return cmap
+
         # Prepare xqueue request
         #------------------------------------------------------------
 
@@ -1242,9 +1414,11 @@ class CodeResponse(LoncapaResponse):
         queuekey = xqueue_interface.make_hashkey(str(self.system.seed) + qtime +
                                                  anonymous_student_id +
                                                  self.answer_id)
-        xheader = xqueue_interface.make_xheader(lms_callback_url=self.system.xqueue['callback_url'],
-                                                lms_key=queuekey,
-                                                queue_name=self.queue_name)
+        callback_url = self.system.xqueue['construct_callback']()
+        xheader = xqueue_interface.make_xheader(
+            lms_callback_url=callback_url,
+            lms_key=queuekey,
+            queue_name=self.queue_name)
 
         # Generate body
         if is_list_of_files(submission):
@@ -1255,13 +1429,16 @@ class CodeResponse(LoncapaResponse):
 
         contents = self.payload.copy()
 
-        # Metadata related to the student submission revealed to the external grader
+        # Metadata related to the student submission revealed to the external
+        # grader
         student_info = {'anonymous_student_id': anonymous_student_id,
                         'submission_time': qtime,
-                       }
+                        }
         contents.update({'student_info': json.dumps(student_info)})
 
-        # Submit request. When successful, 'msg' is the prior length of the queue
+        # Submit request. When successful, 'msg' is the prior length of the
+        # queue
+
         if is_list_of_files(submission):
             # TODO: Is there any information we want to send here?
             contents.update({'student_response': ''})
@@ -1275,7 +1452,7 @@ class CodeResponse(LoncapaResponse):
 
         # State associated with the queueing request
         queuestate = {'key': queuekey,
-                      'time': qtime,}
+                      'time': qtime, }
 
         cmap = CorrectMap()
         if error:
@@ -1289,13 +1466,15 @@ class CodeResponse(LoncapaResponse):
             #   2) Frontend: correctness='incomplete' eventually trickles down
             #      through inputtypes.textbox and .filesubmission to inform the
             #      browser to poll the LMS
-            cmap.set(self.answer_id, queuestate=queuestate, correctness='incomplete', msg=msg)
+            cmap.set(self.answer_id, queuestate=queuestate,
+                     correctness='incomplete', msg=msg)
 
         return cmap
 
     def update_score(self, score_msg, oldcmap, queuekey):
 
-        (valid_score_msg, correct, points, msg) = self._parse_score_msg(score_msg)
+        (valid_score_msg, correct, points,
+         msg) = self._parse_score_msg(score_msg)
         if not valid_score_msg:
             oldcmap.set(self.answer_id,
                         msg='Invalid grader reply. Please contact the course staff.')
@@ -1307,14 +1486,16 @@ class CodeResponse(LoncapaResponse):
         self.context['correct'] = correctness
 
         # Replace 'oldcmap' with new grading results if queuekey matches.  If queuekey
-        # does not match, we keep waiting for the score_msg whose key actually matches
+        # does not match, we keep waiting for the score_msg whose key actually
+        # matches
         if oldcmap.is_right_queuekey(self.answer_id, queuekey):
             # Sanity check on returned points
             if points < 0:
                 points = 0
             # Queuestate is consumed
-            oldcmap.set(self.answer_id, npoints=points, correctness=correctness,
-                        msg=msg.replace('&nbsp;', '&#160;'), queuestate=None)
+            oldcmap.set(
+                self.answer_id, npoints=points, correctness=correctness,
+                msg=msg.replace('&nbsp;', '&#160;'), queuestate=None)
         else:
             log.debug('CodeResponse: queuekey %s does not match for answer_id=%s.' %
                       (queuekey, self.answer_id))
@@ -1434,15 +1615,18 @@ main()
         if answer is not None:
             answer_src = answer.get('src')
             if answer_src is not None:
-                self.code = self.system.filesystem.open('src/' + answer_src).read()
+                self.code = self.system.filesystem.open(
+                    'src/' + answer_src).read()
             else:
                 self.code = answer.text
         else:
             # no <answer> stanza; get code from <script>
             self.code = self.context['script_code']
             if not self.code:
-                msg = '%s: Missing answer script code for externalresponse' % unicode(self)
-                msg += "\nSee XML source line %s" % getattr(self.xml, 'sourceline', '<unavailable>')
+                msg = '%s: Missing answer script code for externalresponse' % unicode(
+                    self)
+                msg += "\nSee XML source line %s" % getattr(
+                    self.xml, 'sourceline', '<unavailable>')
                 raise LoncapaProblemError(msg)
 
         self.tests = xml.get('tests')
@@ -1465,10 +1649,12 @@ main()
         payload.update(extra_payload)
 
         try:
-            # call external server. TODO: synchronous call, can block for a long time
+            # call external server. TODO: synchronous call, can block for a
+            # long time
             r = requests.post(self.url, data=payload)
         except Exception as err:
-            msg = 'Error %s - cannot connect to external server url=%s' % (err, self.url)
+            msg = 'Error %s - cannot connect to external server url=%s' % (
+                err, self.url)
             log.error(msg)
             raise Exception(msg)
 
@@ -1476,13 +1662,15 @@ main()
             log.info('response = %s' % r.text)
 
         if (not r.text) or (not r.text.strip()):
-            raise Exception('Error: no response from external server url=%s' % self.url)
+            raise Exception(
+                'Error: no response from external server url=%s' % self.url)
 
         try:
             # response is XML; parse it
             rxml = etree.fromstring(r.text)
         except Exception as err:
-            msg = 'Error %s - cannot parse response from external server r.text=%s' % (err, r.text)
+            msg = 'Error %s - cannot parse response from external server r.text=%s' % (
+                err, r.text)
             log.error(msg)
             raise Exception(msg)
 
@@ -1507,7 +1695,8 @@ main()
         except Exception as err:
             log.error('Error %s' % err)
             if self.system.DEBUG:
-                cmap.set_dict(dict(zip(sorted(self.answer_ids), ['incorrect'] * len(idset))))
+                cmap.set_dict(dict(zip(sorted(
+                    self.answer_ids), ['incorrect'] * len(idset))))
                 cmap.set_property(
                     self.answer_ids[0], 'msg',
                     '<span class="inline-error">%s</span>' % str(err).replace('<', '&lt;'))
@@ -1524,7 +1713,8 @@ main()
         # create CorrectMap
         for key in idset:
             idx = idset.index(key)
-            msg = rxml.find('message').text.replace('&nbsp;', '&#160;') if idx == 0 else None
+            msg = rxml.find('message').text.replace(
+                '&nbsp;', '&#160;') if idx == 0 else None
             cmap.set(key, self.context['correct'][idx], msg=msg)
 
         return cmap
@@ -1539,7 +1729,8 @@ main()
         except Exception as err:
             log.error('Error %s' % err)
             if self.system.DEBUG:
-                msg = '<span class="inline-error">%s</span>' % str(err).replace('<', '&lt;')
+                msg = '<span class="inline-error">%s</span>' % str(
+                    err).replace('<', '&lt;')
                 exans = [''] * len(self.answer_ids)
                 exans[0] = msg
 
@@ -1586,8 +1777,9 @@ class FormulaResponse(LoncapaResponse):
         self.correct_answer = contextualize_text(xml.get('answer'), context)
         self.samples = contextualize_text(xml.get('samples'), context)
         try:
-            self.tolerance_xml = xml.xpath('//*[@id=$id]//responseparam[@type="tolerance"]/@default',
-                                           id=xml.get('id'))[0]
+            self.tolerance_xml = xml.xpath(
+                '//*[@id=$id]//responseparam[@type="tolerance"]/@default',
+                id=xml.get('id'))[0]
             self.tolerance = contextualize_text(self.tolerance_xml, context)
         except Exception:
             self.tolerance = '0.00001'
@@ -1609,14 +1801,15 @@ class FormulaResponse(LoncapaResponse):
 
     def get_score(self, student_answers):
         given = student_answers[self.answer_id]
-        correctness = self.check_formula(self.correct_answer, given, self.samples)
+        correctness = self.check_formula(
+            self.correct_answer, given, self.samples)
         return CorrectMap(self.answer_id, correctness)
 
     def check_formula(self, expected, given, samples):
         variables = samples.split('@')[0].split(',')
         numsamples = int(samples.split('@')[1].split('#')[1])
         sranges = zip(*map(lambda x: map(float, x.split(",")),
-                         samples.split('@')[1].split('#')[0].split(':')))
+                           samples.split('@')[1].split('#')[0].split(':')))
 
         ranges = dict(zip(variables, sranges))
         for i in range(numsamples):
@@ -1627,22 +1820,26 @@ class FormulaResponse(LoncapaResponse):
                 value = random.uniform(*ranges[var])
                 instructor_variables[str(var)] = value
                 student_variables[str(var)] = value
-            #log.debug('formula: instructor_vars=%s, expected=%s' % (instructor_variables,expected))
+            # log.debug('formula: instructor_vars=%s, expected=%s' %
+            # (instructor_variables,expected))
             instructor_result = evaluator(instructor_variables, dict(),
                                           expected, cs=self.case_sensitive)
             try:
-                #log.debug('formula: student_vars=%s, given=%s' % (student_variables,given))
+                # log.debug('formula: student_vars=%s, given=%s' %
+                # (student_variables,given))
                 student_result = evaluator(student_variables,
                                            dict(),
                                            given,
                                            cs=self.case_sensitive)
             except UndefinedVariable as uv:
-                log.debug('formularesponse: undefined variable in given=%s' % given)
-                raise StudentInputError("Invalid input: " + uv.message + " not permitted in answer")
+                log.debug(
+                    'formularesponse: undefined variable in given=%s' % given)
+                raise StudentInputError(
+                    "Invalid input: " + uv.message + " not permitted in answer")
             except Exception as err:
-                #traceback.print_exc()
+                # traceback.print_exc()
                 log.debug('formularesponse: error %s in formula' % err)
-                raise StudentInputError("Invalid input: Could not parse '%s' as a formula" %\
+                raise StudentInputError("Invalid input: Could not parse '%s' as a formula" %
                                         cgi.escape(given))
             if numpy.isnan(student_result) or numpy.isinf(student_result):
                 return "incorrect"
@@ -1666,9 +1863,11 @@ class FormulaResponse(LoncapaResponse):
         for hxml in hxml_set:
             samples = hxml.get('samples')
             name = hxml.get('name')
-            correct_answer = contextualize_text(hxml.get('answer'), self.context)
+            correct_answer = contextualize_text(
+                hxml.get('answer'), self.context)
             try:
-                correctness = self.check_formula(correct_answer, given, samples)
+                correctness = self.check_formula(
+                    correct_answer, given, samples)
             except Exception:
                 correctness = 'incorrect'
             if correctness == 'correct':
@@ -1699,11 +1898,13 @@ class SchematicResponse(LoncapaResponse):
 
     def get_score(self, student_answers):
         from capa_problem import global_context
-        submission = [json.loads(student_answers[k]) for k in sorted(self.answer_ids)]
+        submission = [json.loads(student_answers[
+                                 k]) for k in sorted(self.answer_ids)]
         self.context.update({'submission': submission})
         exec self.code in global_context, self.context
         cmap = CorrectMap()
-        cmap.set_dict(dict(zip(sorted(self.answer_ids), self.context['correct'])))
+        cmap.set_dict(dict(zip(sorted(
+            self.answer_ids), self.context['correct'])))
         return cmap
 
     def get_answers(self):
@@ -1760,17 +1961,20 @@ class ImageResponse(LoncapaResponse):
         self.ielements = self.inputfields
         self.answer_ids = [ie.get('id') for ie in self.ielements]
 
+
     def get_score(self, student_answers):
         correct_map = CorrectMap()
-        expectedset = self.get_answers()
-        for aid in self.answer_ids:	 # loop through IDs of <imageinput>
+        expectedset = self.get_mapped_answers()
+        for aid in self.answer_ids:  # loop through IDs of <imageinput>
         #  fields in our stanza
-            given = student_answers[aid]  # this should be a string of the form '[x,y]'
+            given = student_answers[
+                aid]  # this should be a string of the form '[x,y]'
             correct_map.set(aid, 'incorrect')
             if not given:  # No answer to parse. Mark as incorrect and move on
                 continue
             # parse given answer
-            m = re.match('\[([0-9]+),([0-9]+)]', given.strip().replace(' ', ''))
+            m = re.match(
+                '\[([0-9]+),([0-9]+)]', given.strip().replace(' ', ''))
             if not m:
                 raise Exception('[capamodule.capa.responsetypes.imageinput] '
                                 'error grading %s (input=%s)' % (aid, given))
@@ -1778,20 +1982,24 @@ class ImageResponse(LoncapaResponse):
 
             rectangles, regions = expectedset
             if rectangles[aid]:  # rectangles part - for backward compatibility
-                # Check whether given point lies in any of the solution rectangles
+                # Check whether given point lies in any of the solution
+                # rectangles
                 solution_rectangles = rectangles[aid].split(';')
                 for solution_rectangle in solution_rectangles:
                     # parse expected answer
                     # TODO: Compile regexp on file load
-                    m = re.match('[\(\[]([0-9]+),([0-9]+)[\)\]]-[\(\[]([0-9]+),([0-9]+)[\)\]]',
-                                 solution_rectangle.strip().replace(' ', ''))
+                    m = re.match(
+                        '[\(\[]([0-9]+),([0-9]+)[\)\]]-[\(\[]([0-9]+),([0-9]+)[\)\]]',
+                        solution_rectangle.strip().replace(' ', ''))
                     if not m:
                         msg = 'Error in problem specification! cannot parse rectangle in %s' % (
                             etree.tostring(self.ielements[aid], pretty_print=True))
-                        raise Exception('[capamodule.capa.responsetypes.imageinput] ' + msg)
+                        raise Exception(
+                            '[capamodule.capa.responsetypes.imageinput] ' + msg)
                     (llx, lly, urx, ury) = [int(x) for x in m.groups()]
 
-                    # answer is correct if (x,y) is within the specified rectangle
+                    # answer is correct if (x,y) is within the specified
+                    # rectangle
                     if (llx <= gx <= urx) and (lly <= gy <= ury):
                         correct_map.set(aid, 'correct')
                         break
@@ -1811,351 +2019,160 @@ class ImageResponse(LoncapaResponse):
                             break
         return correct_map
 
+    def get_mapped_answers(self):
+        '''
+        Returns the internal representation of the answers
+
+        Input:
+            None
+        Returns:
+            tuple (dict, dict) -
+                rectangles (dict) - a map of inputs to the defined rectangle for that input
+                regions (dict) - a map of inputs to the defined region for that input
+        '''
+        answers =  (
+            dict([(ie.get('id'), ie.get(
+                'rectangle')) for ie in self.ielements]),
+            dict([(ie.get('id'), ie.get('regions')) for ie in self.ielements]))
+        return answers
+
     def get_answers(self):
-        return (dict([(ie.get('id'), ie.get('rectangle')) for ie in self.ielements]),
-                dict([(ie.get('id'), ie.get('regions')) for ie in self.ielements]))
+        '''
+        Returns the external representation of the answers
+
+        Input:
+            None
+        Returns:
+            dict (str, (str, str)) - a map of inputs to a tuple of their rectange
+                and their regions
+        '''
+        answers = {}
+        for ie in self.ielements:
+            ie_id = ie.get('id')
+            answers[ie_id] = (ie.get('rectangle'), ie.get('regions'))
+
+        return answers
+            
+
+
 #-----------------------------------------------------------------------------
 
-class OpenEndedResponse(LoncapaResponse):
-    """
-    Grade student open ended responses using an external grading system,
-    accessed through the xqueue system.
 
-    Expects 'xqueue' dict in ModuleSystem with the following keys that are
-    needed by OpenEndedResponse:
+class AnnotationResponse(LoncapaResponse):
+    '''
+    Checking of annotation responses.
 
-        system.xqueue = { 'interface': XqueueInterface object,
-                          'callback_url': Per-StudentModule callback URL
-                                          where results are posted (string),
-                        }
-
-    External requests are only submitted for student submission grading
-        (i.e. and not for getting reference answers)
-
-    By default, uses the OpenEndedResponse.DEFAULT_QUEUE queue.
-    """
-
-    DEFAULT_QUEUE = 'open-ended'
-    response_tag = 'openendedresponse'
-    allowed_inputfields = ['openendedinput']
+    The response contains both a comment (student commentary) and an option (student tag).
+    Only the tag is currently graded. Answers may be incorrect, partially correct, or correct.
+    '''
+    response_tag = 'annotationresponse'
+    allowed_inputfields = ['annotationinput']
     max_inputfields = 1
+    default_scoring = {'incorrect': 0, 'partially-correct': 1, 'correct': 2}
 
     def setup_response(self):
-        '''
-        Configure OpenEndedResponse from XML.
-        '''
         xml = self.xml
-        self.url = xml.get('url', None)
-        self.queue_name = xml.get('queuename', self.DEFAULT_QUEUE)
-
-        # The openendedparam tag encapsulates all grader settings
-        oeparam = self.xml.find('openendedparam')
-        prompt = self.xml.find('prompt')
-        rubric = self.xml.find('openendedrubric')
-
-        if oeparam is None:
-            raise ValueError("No oeparam found in problem xml.")
-        if prompt is None:
-            raise ValueError("No prompt found in problem xml.")
-        if rubric is None:
-            raise ValueError("No rubric found in problem xml.")
-
-        self._parse(oeparam, prompt, rubric)
-
-    @staticmethod
-    def stringify_children(node):
-        """
-        Modify code from stringify_children in xmodule.  Didn't import directly
-        in order to avoid capa depending on xmodule (seems to be avoided in
-        code)
-        """
-        parts=[node.text if node.text is not None else '']
-        for p in node.getchildren():
-            parts.append(etree.tostring(p, with_tail=True, encoding='unicode'))
-
-        return ' '.join(parts)
-
-    def _parse(self, oeparam, prompt, rubric):
-        '''
-        Parse OpenEndedResponse XML:
-            self.initial_display
-            self.payload - dict containing keys --
-            'grader' : path to grader settings file, 'problem_id' : id of the problem
-
-            self.answer - What to display when show answer is clicked
-        '''
-        # Note that OpenEndedResponse is agnostic to the specific contents of grader_payload
-        prompt_string = self.stringify_children(prompt)
-        rubric_string = self.stringify_children(rubric)
-
-        grader_payload = oeparam.find('grader_payload')
-        grader_payload = grader_payload.text if grader_payload is not None else ''
-
-        #Update grader payload with student id.  If grader payload not json, error.
-        try:
-            parsed_grader_payload = json.loads(grader_payload)
-            # NOTE: self.system.location is valid because the capa_module
-            # __init__ adds it (easiest way to get problem location into
-            # response types)
-        except TypeError, ValueError:
-            log.exception("Grader payload %r is not a json object!", grader_payload)
-        parsed_grader_payload.update({
-            'location' : self.system.location,
-            'course_id' : self.system.course_id,
-            'prompt' : prompt_string,
-            'rubric' : rubric_string,
-        })
-        updated_grader_payload = json.dumps(parsed_grader_payload)
-
-        self.payload = {'grader_payload': updated_grader_payload}
-
-        self.initial_display = find_with_default(oeparam, 'initial_display', '')
-        self.answer = find_with_default(oeparam, 'answer_display', 'No answer given.')
-        try:
-            self.max_score = int(find_with_default(oeparam, 'max_score', 1))
-        except ValueError:
-            self.max_score = 1
+        self.scoring_map = self._get_scoring_map()
+        self.answer_map = self._get_answer_map()
+        self.maxpoints = self._get_max_points()
 
     def get_score(self, student_answers):
+        ''' Returns a CorrectMap for the student answer, which may include
+            partially correct answers.'''
+        student_answer = student_answers[self.answer_id]
+        student_option = self._get_submitted_option_id(student_answer)
 
-        try:
-            submission = student_answers[self.answer_id]
-        except KeyError:
-            msg = ('Cannot get student answer for answer_id: {0}. student_answers {1}'
-                   .format(self.answer_id, student_answers))
-            log.exception(msg)
-            raise LoncapaProblemError(msg)
+        scoring = self.scoring_map[self.answer_id]
+        is_valid = student_option is not None and student_option in scoring.keys(
+        )
 
-        # Prepare xqueue request
-        #------------------------------------------------------------
+        (correctness, points) = ('incorrect', None)
+        if is_valid:
+            correctness = scoring[student_option]['correctness']
+            points = scoring[student_option]['points']
 
-        qinterface = self.system.xqueue['interface']
-        qtime = datetime.strftime(datetime.now(), xqueue_interface.dateformat)
-
-        anonymous_student_id = self.system.anonymous_student_id
-
-        # Generate header
-        queuekey = xqueue_interface.make_hashkey(str(self.system.seed) + qtime +
-                                                 anonymous_student_id +
-                                                 self.answer_id)
-
-        xheader = xqueue_interface.make_xheader(lms_callback_url=self.system.xqueue['callback_url'],
-            lms_key=queuekey,
-            queue_name=self.queue_name)
-
-        self.context.update({'submission': submission})
-
-        contents = self.payload.copy()
-
-        # Metadata related to the student submission revealed to the external grader
-        student_info = {'anonymous_student_id': anonymous_student_id,
-                        'submission_time': qtime,
-                        }
-
-        #Update contents with student response and student info
-        contents.update({
-            'student_info': json.dumps(student_info),
-            'student_response': submission,
-            'max_score' : self.max_score
-            })
-
-        # Submit request. When successful, 'msg' is the prior length of the queue
-        (error, msg) = qinterface.send_to_queue(header=xheader,
-            body=json.dumps(contents))
-
-        # State associated with the queueing request
-        queuestate = {'key': queuekey,
-                      'time': qtime,}
-
-        cmap = CorrectMap()
-        if error:
-            cmap.set(self.answer_id, queuestate=None,
-                msg='Unable to deliver your submission to grader. (Reason: {0}.)'
-                    ' Please try again later.'.format(msg))
-        else:
-            # Queueing mechanism flags:
-            #   1) Backend: Non-null CorrectMap['queuestate'] indicates that
-            #      the problem has been queued
-            #   2) Frontend: correctness='incomplete' eventually trickles down
-            #      through inputtypes.textbox and .filesubmission to inform the
-            #      browser that the submission is queued (and it could e.g. poll)
-            cmap.set(self.answer_id, queuestate=queuestate,
-                     correctness='incomplete', msg=msg)
-
-        return cmap
-
-    def update_score(self, score_msg, oldcmap, queuekey):
-        log.debug(score_msg)
-        score_msg = self._parse_score_msg(score_msg)
-        if not score_msg.valid:
-            oldcmap.set(self.answer_id,
-                msg = 'Invalid grader reply. Please contact the course staff.')
-            return oldcmap
-
-        correctness = 'correct' if score_msg.correct else 'incorrect'
-
-        # TODO: Find out how this is used elsewhere, if any
-        self.context['correct'] = correctness
-
-        # Replace 'oldcmap' with new grading results if queuekey matches.  If queuekey
-        # does not match, we keep waiting for the score_msg whose key actually matches
-        if oldcmap.is_right_queuekey(self.answer_id, queuekey):
-            # Sanity check on returned points
-            points = score_msg.points
-            if points < 0:
-                points = 0
-
-            # Queuestate is consumed, so reset it to None
-            oldcmap.set(self.answer_id, npoints=points, correctness=correctness,
-                msg = score_msg.msg.replace('&nbsp;', '&#160;'), queuestate=None)
-        else:
-            log.debug('OpenEndedResponse: queuekey {0} does not match for answer_id={1}.'.format(
-                queuekey, self.answer_id))
-
-        return oldcmap
+        return CorrectMap(self.answer_id, correctness=correctness, npoints=points)
 
     def get_answers(self):
-        anshtml = '<span class="openended-answer"><pre><code>{0}</code></pre></span>'.format(self.answer)
-        return {self.answer_id: anshtml}
+        return self.answer_map
 
-    def get_initial_display(self):
-        return {self.answer_id: self.initial_display}
+    def _get_scoring_map(self):
+        ''' Returns a dict of option->scoring for each input. '''
+        scoring = self.default_scoring
+        choices = dict([(choice, choice) for choice in scoring])
+        scoring_map = {}
 
-    def _convert_longform_feedback_to_html(self, response_items):
-        """
-        Take in a dictionary, and return html strings for display to student.
-        Input:
-            response_items: Dictionary with keys success, feedback.
-                if success is True, feedback should be a dictionary, with keys for
-                   types of feedback, and the corresponding feedback values.
-                if success is False, feedback is actually an error string.
+        for inputfield in self.inputfields:
+            option_scoring = dict([(option['id'], {
+                    'correctness': choices.get(option['choice']),
+                    'points': scoring.get(option['choice'])
+                }) for option in self._find_options(inputfield) ])
 
-                NOTE: this will need to change when we integrate peer grading, because
-                that will have more complex feedback.
+            scoring_map[inputfield.get('id')] = option_scoring
 
-        Output:
-            String -- html that can be displayed to the student.
-        """
+        return scoring_map
 
-        # We want to display available feedback in a particular order.
-        # This dictionary specifies which goes first--lower first.
-        priorities = {# These go at the start of the feedback
-                      'spelling': 0,
-                      'grammar': 1,
-                      # needs to be after all the other feedback
-                      'markup_text': 3}
+    def _get_answer_map(self):
+        ''' Returns a dict of answers for each input.'''
+        answer_map = {}
+        for inputfield in self.inputfields:
+            correct_option = self._find_option_with_choice(
+                inputfield, 'correct')
+            if correct_option is not None:
+                input_id = inputfield.get('id')
+                answer_map[input_id] = correct_option.get('description')
+        return answer_map
 
-        default_priority = 2
+    def _get_max_points(self):
+        ''' Returns a dict of the max points for each input: input id -> maxpoints. '''
+        scoring = self.default_scoring
+        correct_points = scoring.get('correct')
+        return dict([(inputfield.get('id'), correct_points) for inputfield in self.inputfields])
 
-        def get_priority(elt):
-            """
-            Args:
-                elt: a tuple of feedback-type, feedback
-            Returns:
-                the priority for this feedback type
-            """
-            return priorities.get(elt[0], default_priority)
+    def _find_options(self, inputfield):
+        ''' Returns an array of dicts where each dict represents an option. '''
+        elements = inputfield.findall('./options/option')
+        return [{
+                'id': index,
+                'description': option.text,
+                'choice': option.get('choice')
+                } for (index, option) in enumerate(elements)]
 
-        def format_feedback(feedback_type, value):
-            return """
-            <div class="{feedback_type}">
-            {value}
-            </div>
-            """.format(feedback_type, value)
+    def _find_option_with_choice(self, inputfield, choice):
+        ''' Returns the option with the given choice value, otherwise None. '''
+        for option in self._find_options(inputfield):
+            if option['choice'] == choice:
+                return option
 
-        # TODO (vshnayder): design and document the details of this format so
-        # that we can do proper escaping here (e.g. are the graders allowed to
-        # include HTML?)
+    def _unpack(self, json_value):
+        ''' Unpacks a student response value submitted as JSON.'''
+        d = json.loads(json_value)
+        if type(d) != dict:
+            d = {}
 
-        for tag in ['success', 'feedback']:
-            if tag not in response_items:
-                return format_feedback('errors', 'Error getting feedback')
+        comment_value = d.get('comment', '')
+        if not isinstance(d, basestring):
+            comment_value = ''
 
-        feedback_items = response_items['feedback']
-        try:
-            feedback = json.loads(feedback_items)
-        except (TypeError, ValueError):
-            log.exception("feedback_items have invalid json %r", feedback_items)
-            return format_feedback('errors', 'Could not parse feedback')
+        options_value = d.get('options', [])
+        if not isinstance(options_value, list):
+            options_value = []
 
-        if response_items['success']:
-            if len(feedback) == 0:
-                return format_feedback('errors', 'No feedback available')
+        return {
+            'options_value': options_value,
+            'comment_value': comment_value
+        }
 
-            feedback_lst = sorted(feedback.items(), key=get_priority)
-            return u"\n".join(format_feedback(k, v) for k, v in feedback_lst)
-        else:
-            return format_feedback('errors', response_items['feedback'])
-
-
-    def _format_feedback(self, response_items):
-        """
-        Input:
-            Dictionary called feedback.  Must contain keys seen below.
-        Output:
-            Return error message or feedback template
-        """
-
-        feedback = self._convert_longform_feedback_to_html(response_items)
-
-        if not response_items['success']:
-            return self.system.render_template("open_ended_error.html",
-                                               {'errors' : feedback})
-
-        feedback_template = self.system.render_template("open_ended_feedback.html", {
-            'grader_type': response_items['grader_type'],
-            'score': response_items['score'],
-            'feedback': feedback,
-        })
-
-        return feedback_template
-
-
-    def _parse_score_msg(self, score_msg):
-        """
-         Grader reply is a JSON-dump of the following dict
-           { 'correct': True/False,
-             'score': Numeric value (floating point is okay) to assign to answer
-             'msg': grader_msg
-             'feedback' : feedback from grader
-             }
-
-        Returns (valid_score_msg, correct, score, msg):
-            valid_score_msg: Flag indicating valid score_msg format (Boolean)
-            correct:         Correctness of submission (Boolean)
-            score:           Points to be assigned (numeric, can be float)
-        """
-        fail = ScoreMessage(valid=False, correct=False, points=0, msg='')
-        try:
-            score_result = json.loads(score_msg)
-        except (TypeError, ValueError):
-            log.error("External grader message should be a JSON-serialized dict."
-                      " Received score_msg = {0}".format(score_msg))
-            return fail
-
-        if not isinstance(score_result, dict):
-            log.error("External grader message should be a JSON-serialized dict."
-                      " Received score_result = {0}".format(score_result))
-            return fail
-
-        for tag in ['score', 'feedback', 'grader_type', 'success']:
-            if tag not in score_result:
-                log.error("External grader message is missing required tag: {0}"
-                          .format(tag))
-                return fail
-
-        feedback = self._format_feedback(score_result)
-
-        # HACK: for now, just assume it's correct if you got more than 2/3.
-        # Also assumes that score_result['score'] is an integer.
-        score_ratio = int(score_result['score']) / self.max_score
-        correct = (score_ratio >= 0.66)
-
-        #Currently ignore msg and only return feedback (which takes the place of msg)
-        return ScoreMessage(valid=True, correct=correct,
-                            score=score_result['score'], msg=feedback)
+    def _get_submitted_option_id(self, student_answer):
+        ''' Return the single option that was selected, otherwise None.'''
+        submitted = self._unpack(student_answer)
+        option_ids = submitted['options_value']
+        if len(option_ids) == 1:
+            return option_ids[0]
+        return None
 
 #-----------------------------------------------------------------------------
+
 # TEMPORARY: List of all response subclasses
 # FIXME: To be replaced by auto-registration
 
@@ -2173,4 +2190,4 @@ __all__ = [CodeResponse,
            MultipleChoiceResponse,
            TrueFalseResponse,
            JavascriptResponse,
-           OpenEndedResponse]
+           AnnotationResponse]

common/lib/capa/capa/templates/annotationinput.html

@@ -0,0 +1,70 @@
+<form class="annotation-input">
+<div class="script_placeholder" data-src="/static/js/capa/annotationinput.js"/>
+
+<div class="annotation-header">
+    ${title}
+
+    % if return_to_annotation:
+        <a class="annotation-return" href="javascript:void(0)">Return to Annotation</a><br/>
+    % endif
+</div>
+<div class="annotation-body">
+
+    <div class="block block-highlight">${text}</div>
+    <div class="block block-comment">${comment}</div>
+
+    <div class="block">${comment_prompt}</div>
+    <textarea class="comment" id="input_${id}_comment" name="input_${id}_comment">${comment_value|h}</textarea>
+
+    <div class="block">${tag_prompt}</div>
+    <ul class="tags">
+    % for option in options:
+        <li>
+            % if has_options_value:
+                % if all([c == 'correct' for c in option['choice'], status]):
+                <span class="tag-status correct" id="status_${id}"></span>
+                % elif all([c == 'partially-correct' for c in option['choice'], status]):
+                <span class="tag-status partially-correct" id="status_${id}"></span>
+                % elif all([c == 'incorrect' for c in option['choice'], status]):
+                <span class="tag-status incorrect" id="status_${id}"></span>
+                % endif
+            % endif
+
+            <span class="tag
+            % if option['id'] in options_value:
+                selected
+            % endif
+            " data-id="${option['id']}">
+                ${option['description']}
+            </span>
+        </li>
+    % endfor
+    </ul>
+
+    % if debug:
+    <div class="debug-value">
+        Rendered with value:<br/>
+        <pre>${value|h}</pre>
+        Current input value:<br/>
+        <input type="text" class="value" name="input_${id}" id="input_${id}" value="${value|h}" />
+    </div>
+    % else:
+        <input type="hidden" class="value" name="input_${id}" id="input_${id}" value="${value|h}" />
+    % endif
+
+    % if status == 'unsubmitted':
+    <span class="unanswered" style="display:inline-block;" id="status_${id}"></span>
+    % elif status == 'incomplete':
+    <span class="incorrect" id="status_${id}"></span>
+    % elif status == 'incorrect' and not has_options_value:
+    <span class="incorrect" id="status_${id}"></span>
+    % endif
+
+    <p id="answer_${id}" class="answer answer-annotation"></p>
+</div>
+</form>
+
+% if msg:
+<span class="message">${msg|n}</span>
+% endif
+

common/lib/capa/capa/templates/chemicalequationinput.html

@@ -11,7 +11,7 @@
     <div class="incorrect" id="status_${id}">
     % endif
 
-  <input type="text" name="input_${id}" id="input_${id}" value="${value|h}"
+    <input type="text" name="input_${id}" id="input_${id}" data-input-id="${id}" value="${value|h}"
    % if size:
    size="${size}"
    % endif
@@ -31,7 +31,6 @@
 
       <div id="input_${id}_preview" class="equation">
       </div>
-      
   <p id="answer_${id}" class="answer"></p>
 
 % if status in ['unsubmitted', 'correct', 'incorrect', 'incomplete']:

common/lib/capa/capa/templates/choicegroup.html

@@ -1,25 +1,49 @@
 <form class="choicegroup capa_inputtype" id="inputtype_${id}">
-  <div class="indicator_container">
-    % if status == 'unsubmitted':
-    <span class="unanswered" style="display:inline-block;" id="status_${id}"></span>
-    % elif status == 'correct': 
-    <span class="correct" id="status_${id}"></span>
-    % elif status == 'incorrect':
-    <span class="incorrect" id="status_${id}"></span>
-    % elif status == 'incomplete':
-    <span class="incorrect" id="status_${id}"></span>
+    <div class="indicator_container">
+    % if input_type == 'checkbox' or not value:
+        % if status == 'unsubmitted' or show_correctness == 'never':
+        <span class="unanswered" style="display:inline-block;" id="status_${id}"></span>
+        % elif status == 'correct': 
+        <span class="correct" id="status_${id}"></span>
+        % elif status == 'incorrect':
+        <span class="incorrect" id="status_${id}"></span>
+        % elif status == 'incomplete':
+        <span class="incorrect" id="status_${id}"></span>
+        % endif
     % endif
-  </div>
+    </div>
 
-  <fieldset>
-    % for choice_id, choice_description in choices:
-    <label for="input_${id}_${choice_id}">  <input type="${input_type}" name="input_${id}${name_array_suffix}" id="input_${id}_${choice_id}" value="${choice_id}"
-    % if choice_id in value:
-    checked="true"
-    % endif
-    /> ${choice_description} </label>
-    % endfor
-    <span id="answer_${id}"></span>
-  </fieldset>
+    <fieldset>
 
+        % for choice_id, choice_description in choices:
+        <label for="input_${id}_${choice_id}"
+            % if input_type == 'radio' and choice_id == value:
+            <% 
+                if status == 'correct':
+                    correctness = 'correct'
+                elif status == 'incorrect':
+                    correctness = 'incorrect'
+                else:
+                    correctness = None
+            %>
+            % if correctness and not show_correctness=='never':
+            class="choicegroup_${correctness}"
+            % endif
+            % endif
+            >  
+            <input type="${input_type}" name="input_${id}${name_array_suffix}" id="input_${id}_${choice_id}" value="${choice_id}"
+            % if input_type == 'radio' and choice_id == value:
+            checked="true" 
+            % elif input_type != 'radio' and choice_id in value:
+            checked="true"
+            % endif
+
+            /> ${choice_description} </label>
+        % endfor
+        <span id="answer_${id}"></span>
+    </fieldset>
+
+    % if show_correctness == "never" and (value or status not in ['unsubmitted']):
+    <div class="capa_alert">${submitted_message}</div>
+    %endif
 </form>

common/lib/capa/capa/templates/codeinput.html

@@ -50,6 +50,7 @@
         },
         smartIndent: false
       });
+      $("#textbox_${id}").find('.CodeMirror-scroll').height(${int(13.5*eval(rows))});
     });
   </script>
 </section>

common/lib/capa/capa/templates/designprotein2dinput.html

@@ -0,0 +1,35 @@
+<section id="designprotein2dinput_${id}" class="designprotein2dinput">
+    <div class="script_placeholder" data-src="/static/js/capa/protex/protex.nocache.js?raw"/>
+    <div class="script_placeholder" data-src="${applet_loader}"/>
+
+    % if status == 'unsubmitted':
+    <div class="unanswered" id="status_${id}">
+    % elif status == 'correct':
+    <div class="correct" id="status_${id}">
+    % elif status == 'incorrect':
+    <div class="incorrect" id="status_${id}">
+    % elif status == 'incomplete':
+    <div class="incomplete" id="status_${id}">
+    % endif
+    
+    <div id="protex_container"></div>
+    <input type="hidden" name="target_shape" id="target_shape" value ="${target_shape}"></input>
+    <input type="hidden" name="input_${id}" id="input_${id}" value="${value|h}"/>
+
+    <p class="status">
+      % if status == 'unsubmitted':
+      unanswered
+      % elif status == 'correct':
+      correct
+      % elif status == 'incorrect':
+      incorrect
+      % elif status == 'incomplete':
+      incomplete
+      % endif
+    </p>
+
+    <p id="answer_${id}" class="answer"></p>
+  % if status in ['unsubmitted', 'correct', 'incorrect', 'incomplete']:
+  </div>
+% endif
+</section>

common/lib/capa/capa/templates/drag_and_drop_input.html

@@ -0,0 +1,46 @@
+<section id="inputtype_${id}" class="capa_inputtype" >
+    <div class="drag_and_drop_problem_div" id="drag_and_drop_div_${id}"
+     data-plain-id="${id}">
+    </div>
+
+    <div class="drag_and_drop_problem_json" id="drag_and_drop_json_${id}"
+        style="display:none;">${drag_and_drop_json}</div>
+
+    <div class="script_placeholder" data-src="/static/js/capa/drag_and_drop.js"></div>
+
+    % if status == 'unsubmitted':
+        <div class="unanswered" id="status_${id}">
+    % elif status == 'correct':
+        <div class="correct" id="status_${id}">
+    % elif status == 'incorrect':
+        <div class="incorrect" id="status_${id}">
+    % elif status == 'incomplete':
+        <div class="incorrect" id="status_${id}">
+    % endif
+
+
+    <input type="text" name="input_${id}" id="input_${id}" value="${value|h}"
+    style="display:none;"/>
+
+    <p class="status">
+        % if status == 'unsubmitted':
+        unanswered
+        % elif status == 'correct':
+        correct
+        % elif status == 'incorrect':
+        incorrect
+        % elif status == 'incomplete':
+        incomplete
+        % endif
+    </p>
+
+    <p id="answer_${id}" class="answer"></p>
+
+    % if msg:
+        <span class="message">${msg|n}</span>
+    % endif
+
+    % if status in ['unsubmitted', 'correct', 'incorrect', 'incomplete']:
+        </div>
+    % endif
+</section>

common/lib/capa/capa/templates/editageneinput.html

@@ -0,0 +1,37 @@
+ <section id="editageneinput_${id}" class="editageneinput">
+    <div class="script_placeholder" data-src="/static/js/capa/genex/genex.nocache.js?raw"/>
+    <div class="script_placeholder" data-src="${applet_loader}"/>
+
+    % if status == 'unsubmitted':
+    <div class="unanswered" id="status_${id}">
+    % elif status == 'correct':
+    <div class="correct" id="status_${id}">
+    % elif status == 'incorrect':
+    <div class="incorrect" id="status_${id}">
+    % elif status == 'incomplete':
+    <div class="incomplete" id="status_${id}">
+    % endif
+    
+    <div id="genex_container"></div>
+    <input type="hidden" name="dna_sequence" id="dna_sequence" value ="${dna_sequence}"></input>
+    <input type="hidden" name="genex_problem_number" id="genex_problem_number" value ="${genex_problem_number}"></input>
+    <input type="hidden" name="input_${id}" id="input_${id}" value="${value|h}"/>
+
+    <p class="status">
+      % if status == 'unsubmitted':
+      unanswered
+      % elif status == 'correct':
+      correct
+      % elif status == 'incorrect':
+      incorrect
+      % elif status == 'incomplete':
+      incomplete
+      % endif
+    </p>
+
+    <p id="answer_${id}" class="answer"></p>
+  % if status in ['unsubmitted', 'correct', 'incorrect', 'incomplete']:
+  </div>
+% endif
+</section>
+

common/lib/capa/capa/templates/editamolecule.html

@@ -0,0 +1,43 @@
+<section id="editamoleculeinput_${id}" class="editamoleculeinput">
+  <div class="script_placeholder" data-src="${applet_loader}"/>
+
+  % if status == 'unsubmitted':
+  <div class="unanswered" id="status_${id}">
+  % elif status == 'correct':
+  <div class="correct" id="status_${id}">
+  % elif status == 'incorrect':
+  <div class="incorrect" id="status_${id}">
+  % elif status == 'incomplete':
+  <div class="incorrect" id="status_${id}">
+  % endif
+
+    <div id="applet_${id}" class="applet" data-molfile-src="${file}" style="display:block;width:500px;height:400px">
+    </div>
+
+    <br/>
+
+    <input type="hidden" name="input_${id}" id="input_${id}" value="${value|h}"/>
+
+    <button id="reset_${id}" class="reset">Reset</button>
+
+    <p id="answer_${id}" class="answer"></p>
+
+    <p class="status">
+      % if status == 'unsubmitted':
+      unanswered
+      % elif status == 'correct':
+      correct
+      % elif status == 'incorrect':
+      incorrect
+      % elif status == 'incomplete':
+      incomplete
+      % endif
+    </p>
+    <br/> <br/>
+
+    <div class="error_message" style="padding: 5px 5px 5px 5px; background-color:#FA6666; height:60px;width:400px; display: none"></div>
+
+  % if status in ['unsubmitted', 'correct', 'incorrect', 'incomplete']:
+  </div>
+  % endif
+</section>

common/lib/capa/capa/templates/matlabinput.html

@@ -0,0 +1,117 @@
+<section id="textbox_${id}" class="textbox">
+  <textarea rows="${rows}" cols="${cols}" name="input_${id}" id="input_${id}"
+       % if hidden:
+      	    style="display:none;"
+       % endif
+  >${value|h}</textarea>
+
+  <div class="grader-status">
+    % if status == 'unsubmitted':
+      <span class="unanswered" style="display:inline-block;" id="status_${id}">Unanswered</span>
+    % elif status == 'correct':
+      <span class="correct" id="status_${id}">Correct</span>
+    % elif status == 'incorrect':
+      <span class="incorrect" id="status_${id}">Incorrect</span>
+    % elif status == 'queued':
+      <span class="processing" id="status_${id}">Queued</span>
+      <span style="display:none;" class="xqueue" id="${id}" >${queue_len}</span>
+    % endif
+
+    % if hidden:
+      <div style="display:none;" name="${hidden}" inputid="input_${id}" />
+    % endif
+
+    <p class="debug">${status}</p>
+  </div>
+
+  <span id="answer_${id}"></span>
+
+  <div class="external-grader-message">
+    ${msg|n}
+  </div>
+  <div class="external-grader-message">
+    ${queue_msg|n}
+  </div>
+
+  <div class="plot-button">
+      <input type="button" class="save" name="plot-button" id="plot_${id}" value="Plot" />
+  </div>
+
+  <script>
+    // Note: We need to make the area follow the CodeMirror for this to work.
+    $(function(){
+      var cm = CodeMirror.fromTextArea(document.getElementById("input_${id}"), {
+        % if linenumbers == 'true':
+        lineNumbers: true,
+        % endif
+        mode: "matlab",
+        matchBrackets: true,
+        lineWrapping: true,
+        indentUnit: "${tabsize}",
+        tabSize: "${tabsize}",
+        indentWithTabs: false,
+        extraKeys: {
+            "Tab": function(cm) {
+                cm.replaceSelection("${' '*tabsize}", "end");
+            }
+        },
+        smartIndent: false
+      });
+
+      $("#textbox_${id}").find('.CodeMirror-scroll').height(${int(13.5*eval(rows))});
+
+      var gentle_alert = function (parent_elt, msg) {
+          if($(parent_elt).find('.capa_alert').length) {
+              $(parent_elt).find('.capa_alert').remove();
+          }
+          var alert_elem = "<div>" + msg + "</div>";
+          alert_elem = $(alert_elem).addClass('capa_alert');
+          $(parent_elt).find('.action').after(alert_elem);
+          $(parent_elt).find('.capa_alert').css({opacity: 0}).animate({opacity: 1}, 700);
+      }
+      
+
+      // hook up the plot button
+      var plot = function(event) {
+          var problem_elt = $(event.target).closest('.problems-wrapper');
+          url = $(event.target).closest('.problems-wrapper').data('url');
+          input_id = "${id}";
+
+          // save the codemirror text to the textarea
+          cm.save();
+          var input = $("#input_${id}");
+          // pull out the coded text
+          submission = input.val();
+
+          answer = input.serialize();
+
+          // setup callback for after we send information to plot
+          var plot_callback = function(response) {
+              if(response.success) {
+                  window.location.reload();
+              }
+              else {
+                  gentle_alert(problem_elt, msg);
+              }
+          }
+
+          var save_callback = function(response) { 
+                  if(response.success) {
+                      // send information to the problem's plot functionality
+                      Problem.inputAjax(url, input_id, 'plot', 
+                        {'submission': submission}, plot_callback);
+                  }
+                  else {
+                      gentle_alert(problem_elt, msg);
+                  }
+          }
+
+          // save the answer
+          $.postWithPrefix(url + '/problem_save', answer, save_callback);
+
+      }
+      $('#plot_${id}').click(plot);
+
+    });
+  </script>
+</section>

common/lib/capa/capa/templates/openendedinput.html

@@ -1,32 +0,0 @@
-<section id="openended_${id}" class="openended">
-  <textarea rows="${rows}" cols="${cols}" name="input_${id}" class="short-form-response" id="input_${id}"
-       % if hidden:
-      	    style="display:none;"
-       % endif
-  >${value|h}</textarea>
-
-  <div class="grader-status">
-    % if status == 'unsubmitted':
-      <span class="unanswered" style="display:inline-block;" id="status_${id}">Unanswered</span>
-    % elif status == 'correct':
-      <span class="correct" id="status_${id}">Correct</span>
-    % elif status == 'incorrect':
-      <span class="incorrect" id="status_${id}">Incorrect</span>
-    % elif status == 'queued':
-      <span class="grading" id="status_${id}">Submitted for grading</span>
-    % endif
-
-    % if hidden:
-      <div style="display:none;" name="${hidden}" inputid="input_${id}" />
-    % endif
-  </div>
-
-  <span id="answer_${id}"></span>
-
-    % if status == 'queued':
-    <input name="reload" class="reload" type="button" value="Recheck for Feedback" onclick="document.location.reload(true);" />
-    % endif
-  <div class="external-grader-message">
-    ${msg|n}
-  </div>
-</section>

common/lib/capa/capa/tests/__init__.py

@@ -2,12 +2,13 @@ import fs
 import fs.osfs
 import os
 
-from mock import Mock
+from mock import Mock, MagicMock
 
 import xml.sax.saxutils as saxutils
 
 TEST_DIR = os.path.dirname(os.path.realpath(__file__))
 
+
 def tst_render_template(template, context):
     """
     A test version of render to template.  Renders to the repr of the context, completely ignoring
@@ -15,6 +16,11 @@ def tst_render_template(template, context):
     """
     return '<div>{0}</div>'.format(saxutils.escape(repr(context)))
 
+def calledback_url(dispatch = 'score_update'):
+    return dispatch
+
+xqueue_interface = MagicMock()
+xqueue_interface.send_to_queue.return_value = (0, 'Success!')
 
 test_system = Mock(
     ajax_url='courses/course_id/modx/a_location',
@@ -25,7 +31,7 @@ test_system = Mock(
     user=Mock(),
     filestore=fs.osfs.OSFS(os.path.join(TEST_DIR, "test_files")),
     debug=True,
-    xqueue={'interface':None, 'callback_url':'/', 'default_queuename': 'testqueue', 'waittime': 10},
+    xqueue={'interface': xqueue_interface, 'construct_callback': calledback_url, 'default_queuename': 'testqueue', 'waittime': 10},
     node_path=os.environ.get("NODE_PATH", "/usr/local/lib/node_modules"),
-    anonymous_student_id = 'student'
+    anonymous_student_id='student'
 )

common/lib/capa/capa/tests/response_xml_factory.py

@@ -0,0 +1,707 @@
+from lxml import etree
+from abc import ABCMeta, abstractmethod
+
+
+class ResponseXMLFactory(object):
+    """ Abstract base class for capa response XML factories.
+    Subclasses override create_response_element and
+    create_input_element to produce XML of particular response types"""
+
+    __metaclass__ = ABCMeta
+
+    @abstractmethod
+    def create_response_element(self, **kwargs):
+        """ Subclasses override to return an etree element
+        representing the capa response XML
+        (e.g. <numericalresponse>).
+
+        The tree should NOT contain any input elements
+        (such as <textline />) as these will be added later."""
+        return None
+
+    @abstractmethod
+    def create_input_element(self, **kwargs):
+        """ Subclasses override this to return an etree element
+        representing the capa input XML (such as <textline />)"""
+        return None
+
+    def build_xml(self, **kwargs):
+        """ Construct an XML string for a capa response
+        based on **kwargs.
+
+        **kwargs is a dictionary that will be passed
+        to create_response_element() and create_input_element().
+        See the subclasses below for other keyword arguments
+        you can specify.
+
+        For all response types, **kwargs can contain:
+
+        *question_text*: The text of the question to display,
+            wrapped in <p> tags.
+
+        *explanation_text*: The detailed explanation that will
+            be shown if the user answers incorrectly.
+
+        *script*: The embedded Python script (a string)
+
+        *num_responses*: The number of responses to create [DEFAULT: 1]
+
+        *num_inputs*: The number of input elements
+            to create [DEFAULT: 1]
+
+        Returns a string representation of the XML tree.
+        """
+
+        # Retrieve keyward arguments
+        question_text = kwargs.get('question_text', '')
+        explanation_text = kwargs.get('explanation_text', '')
+        script = kwargs.get('script', None)
+        num_responses = kwargs.get('num_responses', 1)
+        num_inputs = kwargs.get('num_inputs', 1)
+
+        # The root is <problem>
+        root = etree.Element("problem")
+
+        # Add a script if there is one
+        if script:
+            script_element = etree.SubElement(root, "script")
+            script_element.set("type", "loncapa/python")
+            script_element.text = str(script)
+
+        # The problem has a child <p> with question text
+        question = etree.SubElement(root, "p")
+        question.text = question_text
+
+        # Add the response(s)
+        for i in range(0, int(num_responses)):
+            response_element = self.create_response_element(**kwargs)
+            root.append(response_element)
+
+            # Add input elements
+            for j in range(0, int(num_inputs)):
+                input_element = self.create_input_element(**kwargs)
+                if not (None == input_element):
+                    response_element.append(input_element)
+
+            # The problem has an explanation of the solution
+            if explanation_text:
+                explanation = etree.SubElement(root, "solution")
+                explanation_div = etree.SubElement(explanation, "div")
+                explanation_div.set("class", "detailed-solution")
+                explanation_div.text = explanation_text
+
+        return etree.tostring(root)
+
+    @staticmethod
+    def textline_input_xml(**kwargs):
+        """ Create a <textline/> XML element
+
+        Uses **kwargs:
+
+        *math_display*: If True, then includes a MathJax display of user input
+
+        *size*: An integer representing the width of the text line
+        """
+        math_display = kwargs.get('math_display', False)
+        size = kwargs.get('size', None)
+
+        input_element = etree.Element('textline')
+
+        if math_display:
+            input_element.set('math', '1')
+
+        if size:
+            input_element.set('size', str(size))
+
+        return input_element
+
+    @staticmethod
+    def choicegroup_input_xml(**kwargs):
+        """ Create a <choicegroup> XML element
+
+        Uses **kwargs:
+
+        *choice_type*: Can be "checkbox", "radio", or "multiple"
+
+        *choices*: List of True/False values indicating whether
+                            a particular choice is correct or not.
+                            Users must choose *all* correct options in order
+                            to be marked correct.
+                            DEFAULT: [True]
+
+        *choice_names": List of strings identifying the choices.
+                        If specified, you must ensure that
+                        len(choice_names) == len(choices)
+        """
+        # Names of group elements
+        group_element_names = {'checkbox': 'checkboxgroup',
+                                'radio': 'radiogroup',
+                                'multiple': 'choicegroup'}
+
+        # Retrieve **kwargs
+        choices = kwargs.get('choices', [True])
+        choice_type = kwargs.get('choice_type', 'multiple')
+        choice_names = kwargs.get('choice_names', [None] * len(choices))
+
+        # Create the <choicegroup>, <checkboxgroup>, or <radiogroup> element
+        assert(choice_type in group_element_names)
+        group_element = etree.Element(group_element_names[choice_type])
+
+        # Create the <choice> elements
+        for (correct_val, name) in zip(choices, choice_names):
+            choice_element = etree.SubElement(group_element, "choice")
+            choice_element.set("correct", "true" if correct_val else "false")
+
+            # Add a name identifying the choice, if one exists
+            # For simplicity, we use the same string as both the
+            # name attribute and the text of the element
+            if name:
+                choice_element.text = str(name)
+                choice_element.set("name", str(name))
+
+        return group_element
+
+
+class NumericalResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <numericalresponse> XML trees """
+
+    def create_response_element(self, **kwargs):
+        """ Create a <numericalresponse> XML element.
+        Uses **kwarg keys:
+
+        *answer*: The correct answer (e.g. "5")
+
+        *tolerance*: The tolerance within which a response
+        is considered correct.  Can be a decimal (e.g. "0.01")
+        or percentage (e.g. "2%")
+        """
+
+        answer = kwargs.get('answer', None)
+        tolerance = kwargs.get('tolerance', None)
+
+        response_element = etree.Element('numericalresponse')
+
+        if answer:
+            response_element.set('answer', str(answer))
+
+        if tolerance:
+            responseparam_element = etree.SubElement(response_element, 'responseparam')
+            responseparam_element.set('type', 'tolerance')
+            responseparam_element.set('default', str(tolerance))
+
+        return response_element
+
+    def create_input_element(self, **kwargs):
+        return ResponseXMLFactory.textline_input_xml(**kwargs)
+
+
+class CustomResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <customresponse> XML trees """
+
+    def create_response_element(self, **kwargs):
+        """ Create a <customresponse> XML element.
+
+        Uses **kwargs:
+
+        *cfn*: the Python code to run.  Can be inline code,
+        or the name of a function defined in earlier <script> tags.
+
+        Should have the form: cfn(expect, answer_given, student_answers)
+        where expect is a value (see below),
+        answer_given is a single value (for 1 input)
+        or a list of values (for multiple inputs),
+        and student_answers is a dict of answers by input ID.
+
+        *expect*: The value passed to the function cfn
+
+        *answer*: Inline script that calculates the answer
+        """
+
+        # Retrieve **kwargs
+        cfn = kwargs.get('cfn', None)
+        expect = kwargs.get('expect', None)
+        answer = kwargs.get('answer', None)
+
+        # Create the response element
+        response_element = etree.Element("customresponse")
+
+        if cfn:
+            response_element.set('cfn', str(cfn))
+
+        if expect:
+            response_element.set('expect', str(expect))
+
+        if answer:
+            answer_element = etree.SubElement(response_element, "answer")
+            answer_element.text = str(answer)
+
+        return response_element
+
+    def create_input_element(self, **kwargs):
+        return ResponseXMLFactory.textline_input_xml(**kwargs)
+
+
+class SchematicResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for creating <schematicresponse> XML trees """
+
+    def create_response_element(self, **kwargs):
+        """ Create the <schematicresponse> XML element.
+
+        Uses *kwargs*:
+
+        *answer*: The Python script used to evaluate the answer.
+        """
+        answer_script = kwargs.get('answer', None)
+
+        # Create the <schematicresponse> element
+        response_element = etree.Element("schematicresponse")
+
+        # Insert the <answer> script if one is provided
+        if answer_script:
+            answer_element = etree.SubElement(response_element, "answer")
+            answer_element.set("type", "loncapa/python")
+            answer_element.text = str(answer_script)
+
+        return response_element
+
+    def create_input_element(self, **kwargs):
+        """ Create the <schematic> XML element.
+
+        Although <schematic> can have several attributes,
+        (*height*, *width*, *parts*, *analyses*, *submit_analysis*, and *initial_value*),
+        none of them are used in the capa module.
+        For testing, we create a bare-bones version of <schematic>."""
+        return etree.Element("schematic")
+
+
+class CodeResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for creating <coderesponse> XML trees """
+
+    def build_xml(self, **kwargs):
+        # Since we are providing an <answer> tag,
+        # we should override the default behavior
+        # of including a <solution> tag as well
+        kwargs['explanation_text'] = None
+        return super(CodeResponseXMLFactory, self).build_xml(**kwargs)
+
+    def create_response_element(self, **kwargs):
+        """ Create a <coderesponse> XML element:
+
+            Uses **kwargs:
+
+            *initial_display*: The code that initially appears in the textbox
+                                [DEFAULT: "Enter code here"]
+            *answer_display*: The answer to display to the student
+                                [DEFAULT: "This is the correct answer!"]
+            *grader_payload*: A JSON-encoded string sent to the grader
+                                [DEFAULT: empty dict string]
+        """
+        # Get **kwargs
+        initial_display = kwargs.get("initial_display", "Enter code here")
+        answer_display = kwargs.get("answer_display", "This is the correct answer!")
+        grader_payload = kwargs.get("grader_payload", '{}')
+
+        # Create the <coderesponse> element
+        response_element = etree.Element("coderesponse")
+        codeparam_element = etree.SubElement(response_element, "codeparam")
+
+        # Set the initial display text
+        initial_element = etree.SubElement(codeparam_element, "initial_display")
+        initial_element.text = str(initial_display)
+
+        # Set the answer display text
+        answer_element = etree.SubElement(codeparam_element, "answer_display")
+        answer_element.text = str(answer_display)
+
+        # Set the grader payload string
+        grader_element = etree.SubElement(codeparam_element, "grader_payload")
+        grader_element.text = str(grader_payload)
+
+        # Create the input within the response
+        input_element = etree.SubElement(response_element, "textbox")
+        input_element.set("mode", "python")
+
+        return response_element
+
+    def create_input_element(self, **kwargs):
+        # Since we create this in create_response_element(),
+        # return None here
+        return None
+
+
+class ChoiceResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for creating <choiceresponse> XML trees """
+
+    def create_response_element(self, **kwargs):
+        """ Create a <choiceresponse> element """
+        return etree.Element("choiceresponse")
+
+    def create_input_element(self, **kwargs):
+        """ Create a <checkboxgroup> element."""
+        return ResponseXMLFactory.choicegroup_input_xml(**kwargs)
+
+
+class FormulaResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for creating <formularesponse> XML trees """
+
+    def create_response_element(self, **kwargs):
+        """ Create a <formularesponse> element.
+
+        *sample_dict*: A dictionary of the form:
+                        { VARIABLE_NAME: (MIN, MAX), ....}
+
+                        This specifies the range within which
+                        to numerically sample each variable to check
+                        student answers.
+                        [REQUIRED]
+
+        *num_samples*: The number of times to sample the student's answer
+                        to numerically compare it to the correct answer.
+
+        *tolerance*: The tolerance within which answers will be accepted
+                        [DEFAULT: 0.01]
+
+        *answer*: The answer to the problem.  Can be a formula string
+                    or a Python variable defined in a script
+                    (e.g. "$calculated_answer" for a Python variable
+                    called calculated_answer)
+                    [REQUIRED]
+
+        *hints*: List of (hint_prompt, hint_name, hint_text) tuples
+                Where *hint_prompt* is the formula for which we show the hint,
+                *hint_name* is an internal identifier for the hint,
+                and *hint_text* is the text we show for the hint.
+        """
+        # Retrieve kwargs
+        sample_dict = kwargs.get("sample_dict", None)
+        num_samples = kwargs.get("num_samples", None)
+        tolerance = kwargs.get("tolerance", 0.01)
+        answer = kwargs.get("answer", None)
+        hint_list = kwargs.get("hints", None)
+
+        assert(answer)
+        assert(sample_dict and num_samples)
+
+        # Create the <formularesponse> element
+        response_element = etree.Element("formularesponse")
+
+        # Set the sample information
+        sample_str = self._sample_str(sample_dict, num_samples, tolerance)
+        response_element.set("samples", sample_str)
+
+
+        # Set the tolerance
+        responseparam_element = etree.SubElement(response_element, "responseparam")
+        responseparam_element.set("type", "tolerance")
+        responseparam_element.set("default", str(tolerance))
+
+        # Set the answer
+        response_element.set("answer", str(answer))
+
+        # Include hints, if specified
+        if hint_list:
+            hintgroup_element = etree.SubElement(response_element, "hintgroup")
+
+            for (hint_prompt, hint_name, hint_text) in hint_list:
+
+                # For each hint, create a <formulahint> element
+                formulahint_element = etree.SubElement(hintgroup_element, "formulahint")
+
+                # We could sample a different range, but for simplicity,
+                # we use the same sample string for the hints
+                # that we used previously.
+                formulahint_element.set("samples", sample_str)
+
+                formulahint_element.set("answer", str(hint_prompt))
+                formulahint_element.set("name", str(hint_name))
+
+                # For each hint, create a <hintpart> element
+                # corresponding to the <formulahint>
+                hintpart_element = etree.SubElement(hintgroup_element, "hintpart")
+                hintpart_element.set("on", str(hint_name))
+                text_element = etree.SubElement(hintpart_element, "text")
+                text_element.text = str(hint_text)
+
+        return response_element
+
+    def create_input_element(self, **kwargs):
+        return ResponseXMLFactory.textline_input_xml(**kwargs)
+
+    def _sample_str(self, sample_dict, num_samples, tolerance):
+        # Loncapa uses a special format for sample strings:
+        # "x,y,z@4,5,3:10,12,8#4" means plug in values for (x,y,z)
+        # from within the box defined by points (4,5,3) and (10,12,8)
+        # The "#4" means to repeat 4 times.
+        variables = [str(v) for v in sample_dict.keys()]
+        low_range_vals = [str(f[0]) for f in sample_dict.values()]
+        high_range_vals = [str(f[1]) for f in sample_dict.values()]
+        sample_str = (",".join(sample_dict.keys()) + "@" +
+                        ",".join(low_range_vals) + ":" +
+                        ",".join(high_range_vals) +
+                        "#" + str(num_samples))
+        return sample_str
+
+
+class ImageResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <imageresponse> XML """
+
+    def create_response_element(self, **kwargs):
+        """ Create the <imageresponse> element."""
+        return etree.Element("imageresponse")
+
+
+    def create_input_element(self, **kwargs):
+        """ Create the <imageinput> element.
+
+        Uses **kwargs:
+
+        *src*: URL for the image file [DEFAULT: "/static/image.jpg"]
+
+        *width*: Width of the image [DEFAULT: 100]
+
+        *height*: Height of the image [DEFAULT: 100]
+
+        *rectangle*: String representing the rectangles the user should select.
+
+                    Take the form "(x1,y1)-(x2,y2)", where the two (x,y)
+                    tuples define the corners of the rectangle.
+
+                    Can include multiple rectangles separated by a semicolon, e.g.
+                    "(490,11)-(556,98);(242,202)-(296,276)"
+
+        *regions*: String representing the regions a user can select
+
+                    Take the form "[ [[x1,y1], [x2,y2], [x3,y3]],
+                                    [[x1,y1], [x2,y2], [x3,y3]] ]"
+                    (Defines two regions, each with 3 points)
+
+        REQUIRED: Either *rectangle* or *region* (or both)
+        """
+
+        # Get the **kwargs
+        src = kwargs.get("src", "/static/image.jpg")
+        width = kwargs.get("width", 100)
+        height = kwargs.get("height", 100)
+        rectangle = kwargs.get('rectangle', None)
+        regions = kwargs.get('regions', None)
+
+        assert(rectangle or regions)
+
+        # Create the <imageinput> element
+        input_element = etree.Element("imageinput")
+        input_element.set("src", str(src))
+        input_element.set("width", str(width))
+        input_element.set("height", str(height))
+
+        if rectangle:
+            input_element.set("rectangle", rectangle)
+
+        if regions:
+            input_element.set("regions", regions)
+
+        return input_element
+
+
+class JavascriptResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <javascriptresponse> XML """
+
+    def create_response_element(self, **kwargs):
+        """ Create the <javascriptresponse> element.
+
+        Uses **kwargs:
+
+        *generator_src*: Name of the JS file to generate the problem.
+        *grader_src*: Name of the JS file to grade the problem.
+        *display_class*: Name of the class used to display the problem
+        *display_src*: Name of the JS file used to display the problem
+        *param_dict*: Dictionary of parameters to pass to the JS
+        """
+        # Get **kwargs
+        generator_src = kwargs.get("generator_src", None)
+        grader_src = kwargs.get("grader_src", None)
+        display_class = kwargs.get("display_class", None)
+        display_src = kwargs.get("display_src", None)
+        param_dict = kwargs.get("param_dict", {})
+
+        # Both display_src and display_class given,
+        # or neither given
+        assert((display_src and display_class) or
+                (not display_src and not display_class))
+
+        # Create the <javascriptresponse> element
+        response_element = etree.Element("javascriptresponse")
+
+        if generator_src:
+            generator_element = etree.SubElement(response_element, "generator")
+            generator_element.set("src", str(generator_src))
+
+        if grader_src:
+            grader_element = etree.SubElement(response_element, "grader")
+            grader_element.set("src", str(grader_src))
+
+        if display_class and display_src:
+            display_element = etree.SubElement(response_element, "display")
+            display_element.set("class", str(display_class))
+            display_element.set("src", str(display_src))
+
+        for (param_name, param_val) in param_dict.items():
+            responseparam_element = etree.SubElement(response_element, "responseparam")
+            responseparam_element.set("name", str(param_name))
+            responseparam_element.set("value", str(param_val))
+
+        return response_element
+
+    def create_input_element(self, **kwargs):
+        """ Create the <javascriptinput> element """
+        return etree.Element("javascriptinput")
+
+
+class MultipleChoiceResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <multiplechoiceresponse> XML """
+
+    def create_response_element(self, **kwargs):
+        """ Create the <multiplechoiceresponse> element"""
+        return etree.Element('multiplechoiceresponse')
+
+    def create_input_element(self, **kwargs):
+        """ Create the <choicegroup> element"""
+        kwargs['choice_type'] = 'multiple'
+        return ResponseXMLFactory.choicegroup_input_xml(**kwargs)
+
+
+class TrueFalseResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <truefalseresponse> XML """
+
+    def create_response_element(self, **kwargs):
+        """ Create the <truefalseresponse> element"""
+        return etree.Element('truefalseresponse')
+
+    def create_input_element(self, **kwargs):
+        """ Create the <choicegroup> element"""
+        kwargs['choice_type'] = 'multiple'
+        return ResponseXMLFactory.choicegroup_input_xml(**kwargs)
+
+
+class OptionResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <optionresponse> XML"""
+
+    def create_response_element(self, **kwargs):
+        """ Create the <optionresponse> element"""
+        return etree.Element("optionresponse")
+
+    def create_input_element(self, **kwargs):
+        """ Create the <optioninput> element.
+
+        Uses **kwargs:
+
+        *options*: a list of possible options the user can choose from [REQUIRED]
+                    You must specify at least 2 options.
+        *correct_option*: the correct choice from the list of options [REQUIRED]
+        """
+
+        options_list = kwargs.get('options', None)
+        correct_option = kwargs.get('correct_option', None)
+
+        assert(options_list and correct_option)
+        assert(len(options_list) > 1)
+        assert(correct_option in options_list)
+
+        # Create the <optioninput> element
+        optioninput_element = etree.Element("optioninput")
+
+        # Set the "options" attribute
+        # Format: "('first', 'second', 'third')"
+        options_attr_string = ",".join(["'%s'" % str(o) for o in options_list])
+        options_attr_string = "(%s)" % options_attr_string
+        optioninput_element.set('options', options_attr_string)
+
+        # Set the "correct" attribute
+        optioninput_element.set('correct', str(correct_option))
+
+        return optioninput_element
+
+
+class StringResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <stringresponse> XML """
+
+    def create_response_element(self, **kwargs):
+        """ Create a <stringresponse> XML element.
+
+            Uses **kwargs:
+
+            *answer*: The correct answer (a string) [REQUIRED]
+
+            *case_sensitive*: Whether the response is case-sensitive (True/False)
+                            [DEFAULT: True]
+
+            *hints*: List of (hint_prompt, hint_name, hint_text) tuples
+                Where *hint_prompt* is the string for which we show the hint,
+                *hint_name* is an internal identifier for the hint,
+                and *hint_text* is the text we show for the hint.
+        """
+        # Retrieve the **kwargs
+        answer = kwargs.get("answer", None)
+        case_sensitive = kwargs.get("case_sensitive", True)
+        hint_list = kwargs.get('hints', None)
+        assert(answer)
+
+        # Create the <stringresponse> element
+        response_element = etree.Element("stringresponse")
+
+        # Set the answer attribute
+        response_element.set("answer", str(answer))
+
+        # Set the case sensitivity
+        response_element.set("type", "cs" if case_sensitive else "ci")
+
+        # Add the hints if specified
+        if hint_list:
+            hintgroup_element = etree.SubElement(response_element, "hintgroup")
+            for (hint_prompt, hint_name, hint_text) in hint_list:
+                stringhint_element = etree.SubElement(hintgroup_element, "stringhint")
+                stringhint_element.set("answer", str(hint_prompt))
+                stringhint_element.set("name", str(hint_name))
+
+                hintpart_element = etree.SubElement(hintgroup_element, "hintpart")
+                hintpart_element.set("on", str(hint_name))
+
+                hint_text_element = etree.SubElement(hintpart_element, "text")
+                hint_text_element.text = str(hint_text)
+
+        return response_element
+
+    def create_input_element(self, **kwargs):
+        return ResponseXMLFactory.textline_input_xml(**kwargs)
+
+
+class AnnotationResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for creating <annotationresponse> XML trees """
+    def create_response_element(self, **kwargs):
+        """ Create a <annotationresponse> element """
+        return etree.Element("annotationresponse")
+
+    def create_input_element(self, **kwargs):
+        """ Create a <annotationinput> element."""
+
+        input_element = etree.Element("annotationinput")
+
+        text_children = [
+            {'tag': 'title', 'text': kwargs.get('title', 'super cool annotation')},
+            {'tag': 'text', 'text': kwargs.get('text', 'texty text')},
+            {'tag': 'comment', 'text':kwargs.get('comment', 'blah blah erudite comment blah blah')},
+            {'tag': 'comment_prompt', 'text': kwargs.get('comment_prompt', 'type a commentary below')},
+            {'tag': 'tag_prompt', 'text': kwargs.get('tag_prompt', 'select one tag')}
+        ]
+
+        for child in text_children:
+            etree.SubElement(input_element, child['tag']).text = child['text']
+
+        default_options = [('green', 'correct'),('eggs', 'incorrect'), ('ham', 'partially-correct')]
+        options = kwargs.get('options', default_options)
+        options_element = etree.SubElement(input_element, 'options')
+
+        for (description, correctness) in options:
+            option_element = etree.SubElement(options_element, 'option', {'choice': correctness})
+            option_element.text = description
+
+        return input_element

common/lib/capa/capa/tests/test_correctmap.py

@@ -0,0 +1,152 @@
+import unittest
+from capa.correctmap import CorrectMap
+import datetime
+
+class CorrectMapTest(unittest.TestCase):
+
+    def setUp(self):
+        self.cmap = CorrectMap()
+
+    def test_set_input_properties(self):
+
+        # Set the correctmap properties for two inputs
+        self.cmap.set(answer_id='1_2_1',
+                    correctness='correct',
+                    npoints=5,
+                    msg='Test message',
+                    hint='Test hint',
+                    hintmode='always',
+                    queuestate={'key':'secretstring',
+                                'time':'20130228100026'})
+
+        self.cmap.set(answer_id='2_2_1',
+                    correctness='incorrect',
+                    npoints=None,
+                    msg=None,
+                    hint=None,
+                    hintmode=None,
+                    queuestate=None)
+
+        # Assert that each input has the expected properties
+        self.assertTrue(self.cmap.is_correct('1_2_1'))
+        self.assertFalse(self.cmap.is_correct('2_2_1'))
+
+        self.assertEqual(self.cmap.get_correctness('1_2_1'), 'correct')
+        self.assertEqual(self.cmap.get_correctness('2_2_1'), 'incorrect')
+
+        self.assertEqual(self.cmap.get_npoints('1_2_1'), 5)
+        self.assertEqual(self.cmap.get_npoints('2_2_1'), 0)
+
+        self.assertEqual(self.cmap.get_msg('1_2_1'), 'Test message')
+        self.assertEqual(self.cmap.get_msg('2_2_1'), None)
+
+        self.assertEqual(self.cmap.get_hint('1_2_1'), 'Test hint')
+        self.assertEqual(self.cmap.get_hint('2_2_1'), None)
+
+        self.assertEqual(self.cmap.get_hintmode('1_2_1'), 'always')
+        self.assertEqual(self.cmap.get_hintmode('2_2_1'), None)
+
+        self.assertTrue(self.cmap.is_queued('1_2_1'))
+        self.assertFalse(self.cmap.is_queued('2_2_1'))
+
+        self.assertEqual(self.cmap.get_queuetime_str('1_2_1'), '20130228100026')
+        self.assertEqual(self.cmap.get_queuetime_str('2_2_1'), None)
+
+        self.assertTrue(self.cmap.is_right_queuekey('1_2_1', 'secretstring'))
+        self.assertFalse(self.cmap.is_right_queuekey('1_2_1', 'invalidstr'))
+        self.assertFalse(self.cmap.is_right_queuekey('1_2_1', ''))
+        self.assertFalse(self.cmap.is_right_queuekey('1_2_1', None))
+
+        self.assertFalse(self.cmap.is_right_queuekey('2_2_1', 'secretstring'))
+        self.assertFalse(self.cmap.is_right_queuekey('2_2_1', 'invalidstr'))
+        self.assertFalse(self.cmap.is_right_queuekey('2_2_1', ''))
+        self.assertFalse(self.cmap.is_right_queuekey('2_2_1', None))
+
+
+    def test_get_npoints(self):
+        # Set the correctmap properties for 4 inputs
+        # 1) correct, 5 points
+        # 2) correct, None points
+        # 3) incorrect, 5 points
+        # 4) incorrect, None points
+        # 5) correct, 0 points
+        self.cmap.set(answer_id='1_2_1',
+                    correctness='correct',
+                    npoints=5)
+
+        self.cmap.set(answer_id='2_2_1',
+                    correctness='correct',
+                    npoints=None)
+
+        self.cmap.set(answer_id='3_2_1',
+                    correctness='incorrect',
+                    npoints=5)
+
+        self.cmap.set(answer_id='4_2_1',
+                    correctness='incorrect',
+                    npoints=None)
+
+        self.cmap.set(answer_id='5_2_1',
+                    correctness='correct',
+                    npoints=0)
+
+        # Assert that we get the expected points
+        # If points assigned --> npoints
+        # If no points assigned and correct --> 1 point
+        # If no points assigned and incorrect --> 0 points
+        self.assertEqual(self.cmap.get_npoints('1_2_1'), 5)
+        self.assertEqual(self.cmap.get_npoints('2_2_1'), 1)
+        self.assertEqual(self.cmap.get_npoints('3_2_1'), 5)
+        self.assertEqual(self.cmap.get_npoints('4_2_1'), 0)
+        self.assertEqual(self.cmap.get_npoints('5_2_1'), 0)
+
+
+    def test_set_overall_message(self):
+
+        # Default is an empty string string
+        self.assertEqual(self.cmap.get_overall_message(), "")
+
+        # Set a message that applies to the whole question
+        self.cmap.set_overall_message("Test message")
+
+        # Retrieve the message
+        self.assertEqual(self.cmap.get_overall_message(), "Test message")
+
+        # Setting the message to None --> empty string
+        self.cmap.set_overall_message(None)
+        self.assertEqual(self.cmap.get_overall_message(), "")
+
+    def test_update_from_correctmap(self):
+        # Initialize a CorrectMap with some properties
+        self.cmap.set(answer_id='1_2_1',
+                    correctness='correct',
+                    npoints=5,
+                    msg='Test message',
+                    hint='Test hint',
+                    hintmode='always',
+                    queuestate={'key':'secretstring',
+                                'time':'20130228100026'})
+
+        self.cmap.set_overall_message("Test message")
+
+        # Create a second cmap, then update it to have the same properties
+        # as the first cmap
+        other_cmap = CorrectMap()
+        other_cmap.update(self.cmap)
+        
+        # Assert that it has all the same properties
+        self.assertEqual(other_cmap.get_overall_message(),
+                        self.cmap.get_overall_message())
+
+        self.assertEqual(other_cmap.get_dict(),
+                        self.cmap.get_dict())
+
+
+    def test_update_from_invalid(self):
+        # Should get an exception if we try to update() a CorrectMap
+        # with a non-CorrectMap value
+        invalid_list = [None, "string", 5, datetime.datetime.today()]
+
+        for invalid in invalid_list:
+            with self.assertRaises(Exception):
+                self.cmap.update(invalid)

common/lib/capa/capa/tests/test_customrender.py

@@ -8,6 +8,7 @@ from capa import customrender
 # just a handy shortcut
 lookup_tag = customrender.registry.get_class_for_tag
 
+
 def extract_context(xml):
     """
     Given an xml element corresponding to the output of test_system.render_template, get back the
@@ -15,9 +16,11 @@ def extract_context(xml):
     """
     return eval(xml.text)
 
+
 def quote_attr(s):
     return saxutils.quoteattr(s)[1:-1]  # don't want the outer quotes
 
+
 class HelperTest(unittest.TestCase):
     '''
     Make sure that our helper function works!
@@ -50,7 +53,7 @@ class SolutionRenderTest(unittest.TestCase):
         # our test_system "renders" templates to a div with the repr of the context
         xml = renderer.get_html()
         context = extract_context(xml)
-        self.assertEqual(context, {'id' : 'solution_12'})
+        self.assertEqual(context, {'id': 'solution_12'})
 
 
 class MathRenderTest(unittest.TestCase):
@@ -65,12 +68,11 @@ class MathRenderTest(unittest.TestCase):
         renderer = lookup_tag('math')(test_system, element)
 
         self.assertEqual(renderer.mathstr, mathjax_out)
-    
+
     def test_parsing(self):
         self.check_parse('$abc$', '[mathjaxinline]abc[/mathjaxinline]')
         self.check_parse('$abc', '$abc')
         self.check_parse(r'$\displaystyle 2+2$', '[mathjax] 2+2[/mathjax]')
-        
+
 
     # NOTE: not testing get_html yet because I don't understand why it's doing what it's doing.
-        

common/lib/capa/capa/tests/test_files/choiceresponse_checkbox.xml

@@ -1,59 +0,0 @@
-<problem>
-<choiceresponse>
-  <checkboxgroup>
-    <choice correct="false">
-      <startouttext />This is foil One.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Two.<endouttext />
-    </choice>
-    <choice correct="true">
-      <startouttext />This is foil Three.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Four.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Five.<endouttext />
-    </choice>
-  </checkboxgroup>
-</choiceresponse>
-<choiceresponse>
-  <checkboxgroup>
-    <choice correct="false">
-      <startouttext />This is foil One.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Two.<endouttext />
-    </choice>
-    <choice correct="true">
-      <startouttext />This is foil Three.<endouttext />
-    </choice>
-    <choice correct="true">
-      <startouttext />This is foil Four.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Five.<endouttext />
-    </choice>
-  </checkboxgroup>
-</choiceresponse>
-<choiceresponse>
-  <checkboxgroup>
-    <choice correct="false">
-      <startouttext />This is foil One.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Two.<endouttext />
-    </choice>
-    <choice correct="true">
-      <startouttext />This is foil Three.<endouttext />
-    </choice>
-    <choice correct="true">
-      <startouttext />This is foil Four.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Five.<endouttext />
-    </choice>
-  </checkboxgroup>
-</choiceresponse>
-</problem>

common/lib/capa/capa/tests/test_files/choiceresponse_radio.xml

@@ -1,40 +0,0 @@
-<problem>
-<choiceresponse>
-  <radiogroup>
-    <choice correct="false">
-      <startouttext />This is foil One.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Two.<endouttext />
-    </choice>
-    <choice correct="true">
-      <startouttext />This is foil Three.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Four.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Five.<endouttext />
-    </choice>
-  </radiogroup>
-</choiceresponse>
-<choiceresponse>
-  <radiogroup>
-    <choice correct="false">
-      <startouttext />This is foil One.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Two.<endouttext />
-    </choice>
-    <choice correct="true">
-      <startouttext />This is foil Three.<endouttext />
-    </choice>
-    <choice correct="true">
-      <startouttext />This is foil Four.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Five.<endouttext />
-    </choice>
-  </radiogroup>
-</choiceresponse>
-</problem>

common/lib/capa/capa/tests/test_files/coderesponse.xml

@@ -1,33 +0,0 @@
-<problem>
-<text>
-<h2>Code response</h2>
-
-<p>
-</p>
-
-<text>
-Write a program to compute the square of a number
-  <coderesponse tests="repeat:2,generate">
-    <textbox rows="10" cols="70"  mode="python"/>
-    <codeparam>
-      <initial_display>def square(x):</initial_display>
-      <answer_display>answer</answer_display>
-      <grader_payload>grader stuff</grader_payload>
-    </codeparam>
-  </coderesponse>
-</text>
-
-<text>
-Write a program to compute the square of a number
-  <coderesponse tests="repeat:2,generate">
-    <textbox rows="10" cols="70"  mode="python"/>
-    <codeparam>
-      <initial_display>def square(x):</initial_display>
-      <answer_display>answer</answer_display>
-      <grader_payload>grader stuff</grader_payload>
-    </codeparam>
-  </coderesponse>
-</text>
-
-</text>
-</problem>

common/lib/capa/capa/tests/test_files/coderesponse_externalresponseformat.xml

@@ -1,101 +0,0 @@
-<problem>
-<text>
-<h2>Code response</h2>
-
-<p>
-</p>
-
-<text>
-Write a program to compute the square of a number
-  <coderesponse tests="repeat:2,generate">
-    <textbox rows="10" cols="70"  mode="python"/>
-    <answer><![CDATA[
-initial_display = """
-def square(n):
-"""
-
-answer = """
-def square(n):
-    return n**2    
-"""
-
-preamble = """ 
-import sys, time
-"""
-
-test_program = """
-import random
-import operator
-
-def testSquare(n = None):
-    if n is None:
-       n = random.randint(2, 20)
-    print 'Test is: square(%d)'%n
-    return str(square(n))
-
-def main():
-   f = os.fdopen(3,'w')
-   test = int(sys.argv[1])
-   rndlist = map(int,os.getenv('rndlist').split(','))
-   random.seed(rndlist[0])
-   if test == 1: f.write(testSquare(0))
-   elif test == 2: f.write(testSquare(1))
-   else:  f.write(testSquare())
-   f.close()
-
-main()
-sys.exit(0)
-"""
-]]>
-    </answer>
-  </coderesponse>
-</text>
-
-<text>
-Write a program to compute the cube of a number
-  <coderesponse tests="repeat:2,generate">
-    <textbox rows="10" cols="70"  mode="python"/>
-    <answer><![CDATA[
-initial_display = """
-def cube(n):
-"""
-
-answer = """
-def cube(n):
-    return n**3    
-"""
-
-preamble = """ 
-import sys, time
-"""
-
-test_program = """
-import random
-import operator
-
-def testCube(n = None):
-    if n is None:
-       n = random.randint(2, 20)
-    print 'Test is: cube(%d)'%n
-    return str(cube(n))
-
-def main():
-   f = os.fdopen(3,'w')
-   test = int(sys.argv[1])
-   rndlist = map(int,os.getenv('rndlist').split(','))
-   random.seed(rndlist[0])
-   if test == 1: f.write(testCube(0))
-   elif test == 2: f.write(testCube(1))
-   else:  f.write(testCube())
-   f.close()
-
-main()
-sys.exit(0)
-"""
-]]>
-    </answer>
-  </coderesponse>
-</text>
-
-</text>
-</problem>

common/lib/capa/capa/tests/test_files/filename_convert_test.txt

@@ -0,0 +1 @@
+This file is used to test converting file handles to filenames in the capa utility module

common/lib/capa/capa/tests/test_files/formularesponse_with_hint.xml

@@ -1,45 +0,0 @@
-<problem>
-<script type="loncapa/python">
-# from loncapa import *
-x1 = 4 # lc_random(2,4,1)
-y1 = 5 # lc_random(3,7,1)
-
-x2 = 10 # lc_random(x1+1,9,1)
-y2 = 20 # lc_random(y1+1,15,1)
-
-m = (y2-y1)/(x2-x1)
-b = y1 - m*x1
-answer = "%s*x+%s" % (m,b)
-answer = answer.replace('+-','-')
-
-inverted_m = (x2-x1)/(y2-y1)
-inverted_b = b
-wrongans = "%s*x+%s" % (inverted_m,inverted_b)
-wrongans = wrongans.replace('+-','-')
-</script>
-
-<text>
-<p>Hints can be provided to students, based on the last response given, as well as the history of responses given.  Here is an example of a hint produced by a Formula Response problem.</p>
-
-<p>
-What is the equation of the line which passess through ($x1,$y1) and
-($x2,$y2)?</p>
-
-<p>The correct answer is <tt>$answer</tt>. A common error is to invert the equation for the slope.  Enter <tt>
-$wrongans</tt> to see a hint.</p>
-
-</text>
-
-<formularesponse samples="x@-5:5#11" id="11" answer="$answer">
- <responseparam description="Numerical Tolerance" type="tolerance" default="0.001" name="tol" /> 
- <text>y = <textline size="25" /></text>
-  <hintgroup>
-   <formulahint samples="x@-5:5#11" answer="$wrongans" name="inversegrad">
-   </formulahint>
-   <hintpart on="inversegrad">
-    <text>You have inverted the slope in the question.</text>
-   </hintpart>
-  </hintgroup>     
-</formularesponse>
-</problem>
-

common/lib/capa/capa/tests/test_files/imageresponse.xml

@@ -1,40 +0,0 @@
-<problem>
-<text><p>
-Two skiers are on frictionless black diamond ski slopes.
-Hello</p></text>
-
-<imageresponse max="1" loncapaid="11">
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" rectangle="(490,11)-(556,98)"/>
-<text>Click on the image where the top skier will stop momentarily if the top skier starts from rest.</text>
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" rectangle="(242,202)-(296,276)"/>
-<text>Click on the image where the lower skier will stop momentarily if the lower skier starts from rest.</text>
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" rectangle="(490,11)-(556,98);(242,202)-(296,276)"/>
-<text>Click on either of the two positions as discussed previously.</text>
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" rectangle="(490,11)-(556,98);(242,202)-(296,276)"/>
-<text>Click on either of the two positions as discussed previously.</text>
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" rectangle="(490,11)-(556,98);(242,202)-(296,276)"/>
-<text>Click on either of the two positions as discussed previously.</text>
-<hintgroup showoncorrect="no">
-<text><p>Use conservation of energy.</p></text>
-</hintgroup>
-</imageresponse>
-
-
-<imageresponse max="1" loncapaid="12">
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" rectangle="(490,11)-(556,98)" regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"/>
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" rectangle="(490,11)-(556,98)" regions='[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]'/>
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"/>
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"/>
-<text>Click on either of the two positions as discussed previously.</text>
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" regions="[[[10,10], [20,10], [20, 30]]]"/>
-<text>Click on either of the two positions as discussed previously.</text>
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" regions="[[10,10], [30,30], [15, 15]]"/>
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" regions="[[10,10], [30,30], [10, 30], [30, 10]]"/>
-<text>Click on either of the two positions as discussed previously.</text>
-<hintgroup showoncorrect="no">
-<text><p>Use conservation of energy.</p></text>
-</hintgroup>
-</imageresponse>
-
-
-</problem>

common/lib/capa/capa/tests/test_files/javascriptresponse.xml

@@ -1,13 +0,0 @@
-<problem>
-
-<javascriptresponse>
-  <generator src="test_problem_generator.js"/>
-  <grader src="test_problem_grader.js"/>
-  <display class="TestProblemDisplay" src="test_problem_display.js"/>
-  <responseparam name="value" value="4"/>
-  <javascriptinput>
-  </javascriptinput>
-</javascriptresponse>
-
-</problem>
-