Merge branch 'master' into tests/diana/update-oe-unit-tests

.ruby-version

@@ -0,0 +1 @@
+1.8.7-p371

Gemfile

@@ -1,7 +1,7 @@
 source :rubygems
-ruby "1.9.3"
-gem 'rake'
+ruby "1.8.7"
+gem 'rake', '~> 10.0.3'
 gem 'sass', '3.1.15'
 gem 'bourbon', '~> 1.3.6'
-gem 'colorize'
-gem 'launchy'
+gem 'colorize', '~> 0.5.8'
+gem 'launchy', '~> 2.1.2'

cms/envs/common.py

@@ -34,7 +34,7 @@ MITX_FEATURES = {
     'GITHUB_PUSH': False,
     'ENABLE_DISCUSSION_SERVICE': False,
     'AUTH_USE_MIT_CERTIFICATES' : False,
-    'STUB_VIDEO_FOR_TESTING': False, # do not display video when running automated acceptance tests    
+    'STUB_VIDEO_FOR_TESTING': False, # do not display video when running automated acceptance tests
 }
 ENABLE_JASMINE = False
 
@@ -281,7 +281,7 @@ INSTALLED_APPS = (
     'contentstore',
     'auth',
     'student',  # misleading name due to sharing with lms
-
+    'course_groups',  # not used in cms (yet), but tests run
     # For asset pipelining
     'pipeline',
     'staticfiles',

common/djangoapps/course_groups/cohorts.py

@@ -0,0 +1,214 @@
+"""
+This file contains the logic for cohort groups, as exposed internally to the
+forums, and to the cohort admin views.
+"""
+
+from django.contrib.auth.models import User
+from django.http import Http404
+import logging
+
+from courseware import courses
+from student.models import get_user_by_username_or_email
+from .models import CourseUserGroup
+
+log = logging.getLogger(__name__)
+
+def is_course_cohorted(course_id):
+    """
+    Given a course id, return a boolean for whether or not the course is
+    cohorted.
+
+    Raises:
+       Http404 if the course doesn't exist.
+    """
+    return courses.get_course_by_id(course_id).is_cohorted
+
+
+def get_cohort_id(user, course_id):
+    """
+    Given a course id and a user, return the id of the cohort that user is
+    assigned to in that course.  If they don't have a cohort, return None.
+    """
+    cohort = get_cohort(user, course_id)
+    return None if cohort is None else cohort.id
+
+
+def is_commentable_cohorted(course_id, commentable_id):
+    """
+    Args:
+        course_id: string
+        commentable_id: string
+
+    Returns:
+        Bool: is this commentable cohorted?
+
+    Raises:
+        Http404 if the course doesn't exist.
+    """
+    course = courses.get_course_by_id(course_id)
+
+    if not course.is_cohorted:
+        # this is the easy case :)
+        ans = False
+    elif commentable_id in course.top_level_discussion_topic_ids:
+        # top level discussions have to be manually configured as cohorted
+        # (default is not)
+        ans = commentable_id in course.cohorted_discussions
+    else:
+        # inline discussions are cohorted by default
+        ans = True
+
+    log.debug("is_commentable_cohorted({0}, {1}) = {2}".format(course_id,
+                                                               commentable_id,
+                                                               ans))
+    return ans
+
+
+def get_cohort(user, course_id):
+    """
+    Given a django User and a course_id, return the user's cohort in that
+    cohort.
+
+    Arguments:
+        user: a Django User object.
+        course_id: string in the format 'org/course/run'
+
+    Returns:
+        A CourseUserGroup object if the course is cohorted and the User has a
+        cohort, else None.
+
+    Raises:
+       ValueError if the course_id doesn't exist.
+    """
+    # First check whether the course is cohorted (users shouldn't be in a cohort
+    # in non-cohorted courses, but settings can change after course starts)
+    try:
+        course = courses.get_course_by_id(course_id)
+    except Http404:
+        raise ValueError("Invalid course_id")
+
+    if not course.is_cohorted:
+        return None
+
+    try:
+        return CourseUserGroup.objects.get(course_id=course_id,
+                                            group_type=CourseUserGroup.COHORT,
+                                            users__id=user.id)
+    except CourseUserGroup.DoesNotExist:
+        # TODO: add auto-cohorting logic here once we know what that will be.
+        return None
+
+
+def get_course_cohorts(course_id):
+    """
+    Get a list of all the cohorts in the given course.
+
+    Arguments:
+        course_id: string in the format 'org/course/run'
+
+    Returns:
+        A list of CourseUserGroup objects.  Empty if there are no cohorts. Does
+        not check whether the course is cohorted.
+    """
+    return list(CourseUserGroup.objects.filter(course_id=course_id,
+                                               group_type=CourseUserGroup.COHORT))
+
+### Helpers for cohort management views
+
+def get_cohort_by_name(course_id, name):
+    """
+    Return the CourseUserGroup object for the given cohort.  Raises DoesNotExist
+    it isn't present.
+    """
+    return CourseUserGroup.objects.get(course_id=course_id,
+                                       group_type=CourseUserGroup.COHORT,
+                                       name=name)
+
+def get_cohort_by_id(course_id, cohort_id):
+    """
+    Return the CourseUserGroup object for the given cohort.  Raises DoesNotExist
+    it isn't present.  Uses the course_id for extra validation...
+    """
+    return CourseUserGroup.objects.get(course_id=course_id,
+                                       group_type=CourseUserGroup.COHORT,
+                                       id=cohort_id)
+
+def add_cohort(course_id, name):
+    """
+    Add a cohort to a course.  Raises ValueError if a cohort of the same name already
+    exists.
+    """
+    log.debug("Adding cohort %s to %s", name, course_id)
+    if CourseUserGroup.objects.filter(course_id=course_id,
+                                      group_type=CourseUserGroup.COHORT,
+                                      name=name).exists():
+        raise ValueError("Can't create two cohorts with the same name")
+
+    return CourseUserGroup.objects.create(course_id=course_id,
+                                          group_type=CourseUserGroup.COHORT,
+                                          name=name)
+
+class CohortConflict(Exception):
+    """
+    Raised when user to be added is already in another cohort in same course.
+    """
+    pass
+
+def add_user_to_cohort(cohort, username_or_email):
+    """
+    Look up the given user, and if successful, add them to the specified cohort.
+
+    Arguments:
+        cohort: CourseUserGroup
+        username_or_email: string.  Treated as email if has '@'
+
+    Returns:
+        User object.
+
+    Raises:
+        User.DoesNotExist if can't find user.
+
+        ValueError if user already present in this cohort.
+
+        CohortConflict if user already in another cohort.
+    """
+    user = get_user_by_username_or_email(username_or_email)
+
+    # If user in any cohorts in this course already, complain
+    course_cohorts = CourseUserGroup.objects.filter(
+        course_id=cohort.course_id,
+        users__id=user.id,
+        group_type=CourseUserGroup.COHORT)
+    if course_cohorts.exists():
+        if course_cohorts[0] == cohort:
+            raise ValueError("User {0} already present in cohort {1}".format(
+                user.username,
+                cohort.name))
+        else:
+            raise CohortConflict("User {0} is in another cohort {1} in course"
+                                 .format(user.username,
+                                         course_cohorts[0].name))
+
+    cohort.users.add(user)
+    return user
+
+
+def get_course_cohort_names(course_id):
+    """
+    Return a list of the cohort names in a course.
+    """
+    return [c.name for c in get_course_cohorts(course_id)]
+
+
+def delete_empty_cohort(course_id, name):
+    """
+    Remove an empty cohort.  Raise ValueError if cohort is not empty.
+    """
+    cohort = get_cohort_by_name(course_id, name)
+    if cohort.users.exists():
+        raise ValueError(
+            "Can't delete non-empty cohort {0} in course {1}".format(
+                name, course_id))
+
+    cohort.delete()
+

common/djangoapps/course_groups/models.py

@@ -0,0 +1,34 @@
+import logging
+
+from django.contrib.auth.models import User
+from django.db import models
+
+log = logging.getLogger(__name__)
+
+class CourseUserGroup(models.Model):
+    """
+    This model represents groups of users in a course.  Groups may have different types,
+    which may be treated specially.  For example, a user can be in at most one cohort per
+    course, and cohorts are used to split up the forums by group.
+    """
+    class Meta:
+        unique_together = (('name', 'course_id'), )
+
+    name = models.CharField(max_length=255,
+                            help_text=("What is the name of this group?  "
+                                       "Must be unique within a course."))
+    users = models.ManyToManyField(User, db_index=True, related_name='course_groups',
+                                   help_text="Who is in this group?")
+
+    # Note: groups associated with particular runs of a course.  E.g. Fall 2012 and Spring
+    # 2013 versions of 6.00x will have separate groups.
+    course_id = models.CharField(max_length=255, db_index=True,
+                                 help_text="Which course is this group associated with?")
+
+    # For now, only have group type 'cohort', but adding a type field to support
+    # things like 'question_discussion', 'friends', 'off-line-class', etc
+    COHORT = 'cohort'
+    GROUP_TYPE_CHOICES = ((COHORT, 'Cohort'),)
+    group_type = models.CharField(max_length=20, choices=GROUP_TYPE_CHOICES)
+
+

common/djangoapps/course_groups/tests/tests.py

@@ -0,0 +1,170 @@
+import django.test
+from django.contrib.auth.models import User
+from django.conf import settings
+
+from override_settings import override_settings
+
+from course_groups.models import CourseUserGroup
+from course_groups.cohorts import (get_cohort, get_course_cohorts,
+                                   is_commentable_cohorted)
+
+from xmodule.modulestore.django import modulestore, _MODULESTORES
+
+class TestCohorts(django.test.TestCase):
+
+
+    @staticmethod
+    def topic_name_to_id(course, name):
+        """
+        Given a discussion topic name, return an id for that name (includes
+        course and url_name).
+        """
+        return "{course}_{run}_{name}".format(course=course.location.course,
+                                              run=course.url_name,
+                                              name=name)
+
+
+    @staticmethod
+    def config_course_cohorts(course, discussions,
+                              cohorted, cohorted_discussions=None):
+        """
+        Given a course with no discussion set up, add the discussions and set
+        the cohort config appropriately.
+
+        Arguments:
+            course: CourseDescriptor
+            discussions: list of topic names strings.  Picks ids and sort_keys
+                automatically.
+            cohorted: bool.
+            cohorted_discussions: optional list of topic names.  If specified,
+                converts them to use the same ids as topic names.
+
+        Returns:
+            Nothing -- modifies course in place.
+        """
+        def to_id(name):
+            return TestCohorts.topic_name_to_id(course, name)
+
+        topics = dict((name, {"sort_key": "A",
+                              "id": to_id(name)})
+                      for name in discussions)
+
+        course.metadata["discussion_topics"] = topics
+
+        d = {"cohorted": cohorted}
+        if cohorted_discussions is not None:
+            d["cohorted_discussions"] = [to_id(name)
+                                         for name in cohorted_discussions]
+        course.metadata["cohort_config"] = d
+
+
+    def setUp(self):
+        """
+        Make sure that course is reloaded every time--clear out the modulestore.
+        """
+        # don't like this, but don't know a better way to undo all changes made
+        # to course.  We don't have a course.clone() method.
+        _MODULESTORES.clear()
+
+
+    def test_get_cohort(self):
+        # Need to fix this, but after we're testing on staging.  (Looks like
+        # problem is that when get_cohort internally tries to look up the
+        # course.id, it fails, even though we loaded it through the modulestore.
+
+        # Proper fix: give all tests a standard modulestore that uses the test
+        # dir.
+        course = modulestore().get_course("edX/toy/2012_Fall")
+        self.assertEqual(course.id, "edX/toy/2012_Fall")
+        self.assertFalse(course.is_cohorted)
+        
+        user = User.objects.create(username="test", email="a@b.com")
+        other_user = User.objects.create(username="test2", email="a2@b.com")
+
+        self.assertIsNone(get_cohort(user, course.id), "No cohort created yet")
+
+        cohort = CourseUserGroup.objects.create(name="TestCohort",
+                                                course_id=course.id,
+                               group_type=CourseUserGroup.COHORT)
+
+        cohort.users.add(user)
+
+        self.assertIsNone(get_cohort(user, course.id),
+                          "Course isn't cohorted, so shouldn't have a cohort")
+
+        # Make the course cohorted...
+        self.config_course_cohorts(course, [], cohorted=True)
+
+        self.assertEquals(get_cohort(user, course.id).id, cohort.id,
+                          "Should find the right cohort")
+
+        self.assertEquals(get_cohort(other_user, course.id), None,
+                          "other_user shouldn't have a cohort")
+
+
+    def test_get_course_cohorts(self):
+        course1_id = 'a/b/c'
+        course2_id = 'e/f/g'
+
+        # add some cohorts to course 1
+        cohort = CourseUserGroup.objects.create(name="TestCohort",
+                                                course_id=course1_id,
+                                                group_type=CourseUserGroup.COHORT)
+
+        cohort = CourseUserGroup.objects.create(name="TestCohort2",
+                                                course_id=course1_id,
+                                                group_type=CourseUserGroup.COHORT)
+
+
+        # second course should have no cohorts
+        self.assertEqual(get_course_cohorts(course2_id), [])
+
+        cohorts = sorted([c.name for c in get_course_cohorts(course1_id)])
+        self.assertEqual(cohorts, ['TestCohort', 'TestCohort2'])
+
+
+    def test_is_commentable_cohorted(self):
+        course = modulestore().get_course("edX/toy/2012_Fall")
+        self.assertFalse(course.is_cohorted)
+
+        def to_id(name):
+            return self.topic_name_to_id(course, name)
+
+        # no topics
+        self.assertFalse(is_commentable_cohorted(course.id, to_id("General")),
+                         "Course doesn't even have a 'General' topic")
+
+        # not cohorted
+        self.config_course_cohorts(course, ["General", "Feedback"],
+                                   cohorted=False)
+
+        self.assertFalse(is_commentable_cohorted(course.id, to_id("General")),
+                         "Course isn't cohorted")
+
+        # cohorted, but top level topics aren't
+        self.config_course_cohorts(course, ["General", "Feedback"],
+                                   cohorted=True)
+
+        self.assertTrue(course.is_cohorted)
+        self.assertFalse(is_commentable_cohorted(course.id, to_id("General")),
+                         "Course is cohorted, but 'General' isn't.")
+
+        self.assertTrue(
+            is_commentable_cohorted(course.id, to_id("random")),
+            "Non-top-level discussion is always cohorted in cohorted courses.")
+
+        # cohorted, including "Feedback" top-level topics aren't
+        self.config_course_cohorts(course, ["General", "Feedback"],
+                                   cohorted=True,
+                                   cohorted_discussions=["Feedback"])
+
+        self.assertTrue(course.is_cohorted)
+        self.assertFalse(is_commentable_cohorted(course.id, to_id("General")),
+                         "Course is cohorted, but 'General' isn't.")
+
+        self.assertTrue(
+            is_commentable_cohorted(course.id, to_id("Feedback")),
+            "Feedback was listed as cohorted.  Should be.")
+
+
+

common/djangoapps/course_groups/views.py

@@ -0,0 +1,219 @@
+from django_future.csrf import ensure_csrf_cookie
+from django.contrib.auth.decorators import login_required
+from django.views.decorators.http import require_POST
+from django.contrib.auth.models import User
+from django.core.context_processors import csrf
+from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
+from django.core.urlresolvers import reverse
+from django.http import HttpResponse, HttpResponseForbidden, Http404
+from django.shortcuts import redirect
+import json
+import logging
+import re
+
+from courseware.courses import get_course_with_access
+from mitxmako.shortcuts import render_to_response, render_to_string
+
+from .models import CourseUserGroup
+from . import cohorts
+
+import track.views
+
+
+log = logging.getLogger(__name__)
+
+def json_http_response(data):
+    """
+    Return an HttpResponse with the data json-serialized and the right content
+    type header.
+    """
+    return HttpResponse(json.dumps(data), content_type="application/json")
+
+def split_by_comma_and_whitespace(s):
+    """
+    Split a string both by commas and whitespice.  Returns a list.
+    """
+    return re.split(r'[\s,]+', s)
+
+
+@ensure_csrf_cookie
+def list_cohorts(request, course_id):
+    """
+    Return json dump of dict:
+
+    {'success': True,
+     'cohorts': [{'name': name, 'id': id}, ...]}
+    """
+    get_course_with_access(request.user, course_id, 'staff')
+
+    all_cohorts = [{'name': c.name, 'id': c.id}
+               for c in cohorts.get_course_cohorts(course_id)]
+
+    return json_http_response({'success': True,
+                            'cohorts': all_cohorts})
+
+
+@ensure_csrf_cookie
+@require_POST
+def add_cohort(request, course_id):
+    """
+    Return json of dict:
+    {'success': True,
+     'cohort': {'id': id,
+                'name': name}}
+
+                or
+
+    {'success': False,
+     'msg': error_msg} if there's an error
+    """
+    get_course_with_access(request.user, course_id, 'staff')
+
+    name = request.POST.get("name")
+    if not name:
+        return json_http_response({'success': False,
+                                'msg': "No name specified"})
+
+    try:
+        cohort = cohorts.add_cohort(course_id, name)
+    except ValueError as err:
+        return json_http_response({'success': False,
+                                'msg': str(err)})
+
+    return json_http_response({'success': 'True',
+                            'cohort': {
+                                'id': cohort.id,
+                                'name': cohort.name
+                                }})
+
+
+@ensure_csrf_cookie
+def users_in_cohort(request, course_id, cohort_id):
+    """
+    Return users in the cohort.  Show up to 100 per page, and page
+    using the 'page' GET attribute in the call.  Format:
+
+    Returns:
+        Json dump of dictionary in the following format:
+        {'success': True,
+         'page': page,
+         'num_pages': paginator.num_pages,
+         'users': [{'username': ..., 'email': ..., 'name': ...}]
+    }
+    """
+    get_course_with_access(request.user, course_id, 'staff')
+
+    # this will error if called with a non-int cohort_id.  That's ok--it
+    # shoudn't happen for valid clients.
+    cohort = cohorts.get_cohort_by_id(course_id, int(cohort_id))
+
+    paginator = Paginator(cohort.users.all(), 100)
+    page = request.GET.get('page')
+    try:
+        users = paginator.page(page)
+    except PageNotAnInteger:
+        # return the first page
+        page = 1
+        users = paginator.page(page)
+    except EmptyPage:
+        # Page is out of range.  Return last page
+        page = paginator.num_pages
+        contacts = paginator.page(page)
+
+    user_info = [{'username': u.username,
+                  'email': u.email,
+                  'name': '{0} {1}'.format(u.first_name, u.last_name)}
+                  for u in users]
+
+    return json_http_response({'success': True,
+                            'page': page,
+                            'num_pages': paginator.num_pages,
+                            'users': user_info})
+
+
+@ensure_csrf_cookie
+@require_POST
+def add_users_to_cohort(request, course_id, cohort_id):
+    """
+    Return json dict of:
+
+    {'success': True,
+     'added': [{'username': username,
+                'name': name,
+                'email': email}, ...],
+     'conflict': [{'username_or_email': ...,
+                    'msg': ...}],  # in another cohort
+     'present': [str1, str2, ...],    # already there
+     'unknown': [str1, str2, ...]}
+    """
+    get_course_with_access(request.user, course_id, 'staff')
+
+    cohort = cohorts.get_cohort_by_id(course_id, cohort_id)
+
+    users = request.POST.get('users', '')
+    added = []
+    present = []
+    conflict = []
+    unknown = []
+    for username_or_email in split_by_comma_and_whitespace(users):
+        try:
+            user = cohorts.add_user_to_cohort(cohort, username_or_email)
+            added.append({'username': user.username,
+                          'name': "{0} {1}".format(user.first_name, user.last_name),
+                          'email': user.email,
+                          })
+        except ValueError:
+            present.append(username_or_email)
+        except User.DoesNotExist:
+            unknown.append(username_or_email)
+        except cohorts.CohortConflict as err:
+            conflict.append({'username_or_email': username_or_email,
+                              'msg': str(err)})
+
+
+    return json_http_response({'success': True,
+                            'added': added,
+                            'present': present,
+                            'conflict': conflict,
+                            'unknown': unknown})
+
+@ensure_csrf_cookie
+@require_POST
+def remove_user_from_cohort(request, course_id, cohort_id):
+    """
+    Expects 'username': username in POST data.
+
+    Return json dict of:
+
+    {'success': True} or
+    {'success': False,
+     'msg': error_msg}
+    """
+    get_course_with_access(request.user, course_id, 'staff')
+
+    username = request.POST.get('username')
+    if username is None:
+        return json_http_response({'success': False,
+                                'msg': 'No username specified'})
+
+    cohort = cohorts.get_cohort_by_id(course_id, cohort_id)
+    try:
+        user = User.objects.get(username=username)
+        cohort.users.remove(user)
+        return json_http_response({'success': True})
+    except User.DoesNotExist:
+        log.debug('no user')
+        return json_http_response({'success': False,
+                                'msg': "No user '{0}'".format(username)})
+
+
+def debug_cohort_mgmt(request, course_id):
+    """
+    Debugging view for dev.
+    """
+    # add staff check to make sure it's safe if it's accidentally deployed.
+    get_course_with_access(request.user, course_id, 'staff')
+
+    context = {'cohorts_ajax_url': reverse('cohorts',
+                                           kwargs={'course_id': course_id})}
+    return render_to_response('/course_groups/debug.html', context)

common/djangoapps/student/management/commands/pearson_export_cdd.py

@@ -1,15 +1,17 @@
 import csv
+import os
 from collections import OrderedDict
 from datetime import datetime
-from os.path import isdir
 from optparse import make_option
 
-from django.core.management.base import BaseCommand
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
 
 from student.models import TestCenterUser
 
+
 class Command(BaseCommand):
-    
+
     CSV_TO_MODEL_FIELDS = OrderedDict([
         # Skipping optional field CandidateID
         ("ClientCandidateID", "client_candidate_id"),
@@ -34,43 +36,52 @@ class Command(BaseCommand):
         ("FAXCountryCode", "fax_country_code"),
         ("CompanyName", "company_name"),
         # Skipping optional field CustomQuestion
-        ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
+        ("LastUpdate", "user_updated_at"),  # in UTC, so same as what we store
     ])
 
+    # define defaults, even thought 'store_true' shouldn't need them.  
+    # (call_command will set None as default value for all options that don't have one,
+    # so one cannot rely on presence/absence of flags in that world.)
     option_list = BaseCommand.option_list + (
-        make_option(
-            '--dump_all',
-            action='store_true',
-            dest='dump_all',
-        ),
+        make_option('--dest-from-settings',
+                    action='store_true',
+                    dest='dest-from-settings',
+                    default=False,
+                    help='Retrieve the destination to export to from django.'),
+        make_option('--destination',
+                    action='store',
+                    dest='destination',
+                    default=None,
+                    help='Where to store the exported files')
     )
-    
-    args = '<output_file_or_dir>'
-    help = """
-    Export user demographic information from TestCenterUser model into a tab delimited
-    text file with a format that Pearson expects.
-    """
-    def handle(self, *args, **kwargs):
-        if len(args) < 1:
-            print Command.help
-            return
 
+    def handle(self, **options):
         # update time should use UTC in order to be comparable to the user_updated_at 
         # field
         uploaded_at = datetime.utcnow()
 
-        # if specified destination is an existing directory, then 
+        # if specified destination is an existing directory, then
         # create a filename for it automatically.  If it doesn't exist,
-        # or exists as a file, then we will just write to it.
+        # then we will create the directory.
         # Name will use timestamp -- this is UTC, so it will look funny,
-        # but it should at least be consistent with the other timestamps 
+        # but it should at least be consistent with the other timestamps
         # used in the system.
-        dest = args[0]
-        if isdir(dest):
-            destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
+        if 'dest-from-settings' in options and options['dest-from-settings']:
+            if 'LOCAL_EXPORT' in settings.PEARSON:
+                dest = settings.PEARSON['LOCAL_EXPORT']
+            else:
+                raise CommandError('--dest-from-settings was enabled but the'
+                                   'PEARSON[LOCAL_EXPORT] setting was not set.')
+        elif 'destination' in options and options['destination']:
+            dest = options['destination']
         else:
-            destfile = dest
-        
+            raise CommandError('--destination or --dest-from-settings must be used')
+
+        if not os.path.isdir(dest):
+            os.makedirs(dest)
+
+        destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
+
         # strings must be in latin-1 format.  CSV parser will
         # otherwise convert unicode objects to ascii.
         def ensure_encoding(value):
@@ -78,8 +89,8 @@ class Command(BaseCommand):
                 return value.encode('iso-8859-1')
             else:
                 return value
-            
-        dump_all = kwargs['dump_all']
+
+#        dump_all = options['dump_all']
 
         with open(destfile, "wb") as outfile:
             writer = csv.DictWriter(outfile,
@@ -89,7 +100,7 @@ class Command(BaseCommand):
                                     extrasaction='ignore')
             writer.writeheader()
             for tcu in TestCenterUser.objects.order_by('id'):
-                if dump_all or tcu.needs_uploading:
+                if tcu.needs_uploading: # or dump_all
                     record = dict((csv_field, ensure_encoding(getattr(tcu, model_field)))
                                   for csv_field, model_field
                                   in Command.CSV_TO_MODEL_FIELDS.items())
@@ -97,6 +108,3 @@ class Command(BaseCommand):
                     writer.writerow(record)
                     tcu.uploaded_at = uploaded_at
                     tcu.save()
-
-
-        

common/djangoapps/student/management/commands/pearson_export_ead.py

@@ -1,15 +1,17 @@
 import csv
+import os
 from collections import OrderedDict
 from datetime import datetime
-from os.path import isdir, join
 from optparse import make_option
 
-from django.core.management.base import BaseCommand
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+
+from student.models import TestCenterRegistration, ACCOMMODATION_REJECTED_CODE
 
-from student.models import TestCenterRegistration
 
 class Command(BaseCommand):
-    
+
     CSV_TO_MODEL_FIELDS = OrderedDict([
         ('AuthorizationTransactionType', 'authorization_transaction_type'),
         ('AuthorizationID', 'authorization_id'),
@@ -20,51 +22,60 @@ class Command(BaseCommand):
         ('Accommodations', 'accommodation_code'),
         ('EligibilityApptDateFirst', 'eligibility_appointment_date_first'),
         ('EligibilityApptDateLast', 'eligibility_appointment_date_last'),
-        ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
+        ("LastUpdate", "user_updated_at"),  # in UTC, so same as what we store
     ])
 
-    args = '<output_file_or_dir>'
-    help = """
-    Export user registration information from TestCenterRegistration model into a tab delimited
-    text file with a format that Pearson expects.
-    """
-
     option_list = BaseCommand.option_list + (
-        make_option(
-            '--dump_all',
-            action='store_true',
-            dest='dump_all',
-        ),
-        make_option(
-            '--force_add',
-            action='store_true',
-            dest='force_add',
-        ),
+        make_option('--dest-from-settings',
+                    action='store_true',
+                    dest='dest-from-settings',
+                    default=False,
+                    help='Retrieve the destination to export to from django.'),
+        make_option('--destination',
+                    action='store',
+                    dest='destination',
+                    default=None,
+                    help='Where to store the exported files'),
+        make_option('--dump_all',
+                    action='store_true',
+                    dest='dump_all',
+                    default=False,
+                    ),
+        make_option('--force_add',
+                    action='store_true',
+                    dest='force_add',
+                    default=False,
+                    ),
     )
-    
-    
-    def handle(self, *args, **kwargs):
-        if len(args) < 1:
-            print Command.help
-            return
 
-        # update time should use UTC in order to be comparable to the user_updated_at 
+    def handle(self, **options):
+        # update time should use UTC in order to be comparable to the user_updated_at
         # field
         uploaded_at = datetime.utcnow()
 
-        # if specified destination is an existing directory, then 
+        # if specified destination is an existing directory, then
         # create a filename for it automatically.  If it doesn't exist,
-        # or exists as a file, then we will just write to it.
+        # then we will create the directory.
         # Name will use timestamp -- this is UTC, so it will look funny,
-        # but it should at least be consistent with the other timestamps 
+        # but it should at least be consistent with the other timestamps
         # used in the system.
-        dest = args[0]
-        if isdir(dest):
-            destfile = join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
+        if 'dest-from-settings' in options and options['dest-from-settings']:
+            if 'LOCAL_EXPORT' in settings.PEARSON:
+                dest = settings.PEARSON['LOCAL_EXPORT']
+            else:
+                raise CommandError('--dest-from-settings was enabled but the'
+                                   'PEARSON[LOCAL_EXPORT] setting was not set.')
+        elif 'destination' in options and options['destination']:
+            dest = options['destination']
         else:
-            destfile = dest
+            raise CommandError('--destination or --dest-from-settings must be used')
 
-        dump_all = kwargs['dump_all']
+        if not os.path.isdir(dest):
+            os.makedirs(dest)
+
+        destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
+
+        dump_all = options['dump_all']
 
         with open(destfile, "wb") as outfile:
             writer = csv.DictWriter(outfile,
@@ -81,13 +92,11 @@ class Command(BaseCommand):
                     record["LastUpdate"] = record["LastUpdate"].strftime("%Y/%m/%d %H:%M:%S")
                     record["EligibilityApptDateFirst"] = record["EligibilityApptDateFirst"].strftime("%Y/%m/%d")
                     record["EligibilityApptDateLast"] = record["EligibilityApptDateLast"].strftime("%Y/%m/%d")
-                    if kwargs['force_add']:
+                    if record["Accommodations"] == ACCOMMODATION_REJECTED_CODE:
+                        record["Accommodations"] = ""
+                    if options['force_add']:
                         record['AuthorizationTransactionType'] = 'Add'
 
                     writer.writerow(record)
                     tcr.uploaded_at = uploaded_at
                     tcr.save()
-
-
-
-        

common/djangoapps/student/management/commands/pearson_import_conf_zip.py

@@ -0,0 +1,119 @@
+import csv
+
+from zipfile import ZipFile, is_zipfile
+from time import strptime, strftime
+
+from collections import OrderedDict
+from datetime import datetime
+from os.path import isdir
+from optparse import make_option
+from dogapi import dog_http_api, dog_stats_api
+
+from django.core.management.base import BaseCommand, CommandError
+from django.conf import settings
+
+from student.models import TestCenterUser, TestCenterRegistration
+
+
+class Command(BaseCommand):
+
+    dog_http_api.api_key = settings.DATADOG_API
+    args = '<input zip file>'
+    help = """
+    Import Pearson confirmation files and update TestCenterUser
+    and TestCenterRegistration tables with status.
+    """
+
+    @staticmethod
+    def datadog_error(string, tags):
+        dog_http_api.event("Pearson Import", string, alert_type='error', tags=[tags])
+
+    def handle(self, *args, **kwargs):
+        if len(args) < 1:
+            print Command.help
+            return
+
+        source_zip = args[0]
+        if not is_zipfile(source_zip):
+            error = "Input file is not a zipfile: \"{}\"".format(source_zip)
+            Command.datadog_error(error, source_zip)
+            raise CommandError(error)
+
+        # loop through all files in zip, and process them based on filename prefix:
+        with ZipFile(source_zip, 'r') as zipfile:
+            for fileinfo in zipfile.infolist():
+                with zipfile.open(fileinfo) as zipentry:
+                    if fileinfo.filename.startswith("eac-"):
+                        self.process_eac(zipentry)
+                    elif fileinfo.filename.startswith("vcdc-"):
+                        self.process_vcdc(zipentry)
+                    else:
+                        error = "Unrecognized confirmation file type\"{}\" in confirmation zip file \"{}\"".format(fileinfo.filename, zipfile)
+                        Command.datadog_error(error, source_zip)
+                        raise CommandError(error)
+
+    def process_eac(self, eacfile):
+        print "processing eac"
+        reader = csv.DictReader(eacfile, delimiter="\t")
+        for row in reader:
+            client_authorization_id = row['ClientAuthorizationID']
+            if not client_authorization_id:
+                if row['Status'] == 'Error':
+                    Command.datadog_error("Error in EAD file processing ({}): {}".format(row['Date'], row['Message']), eacfile.name)
+                else:
+                    Command.datadog_error("Encountered bad record: {}".format(row), eacfile.name)
+            else:
+                try:
+                    registration = TestCenterRegistration.objects.get(client_authorization_id=client_authorization_id)
+                    Command.datadog_error("Found authorization record for user {}".format(registration.testcenter_user.user.username), eacfile)
+                    # now update the record:
+                    registration.upload_status = row['Status']
+                    registration.upload_error_message =  row['Message']
+                    try:
+                        registration.processed_at = strftime('%Y-%m-%d %H:%M:%S', strptime(row['Date'], '%Y/%m/%d %H:%M:%S'))
+                    except ValueError as ve:
+                        Command.datadog_error("Bad Date value found for {}: message {}".format(client_authorization_id, ve), eacfile.name)
+                    # store the authorization Id if one is provided.  (For debugging)
+                    if row['AuthorizationID']:
+                        try:
+                            registration.authorization_id = int(row['AuthorizationID'])
+                        except ValueError as ve:
+                            Command.datadog_error("Bad AuthorizationID value found for {}: message {}".format(client_authorization_id, ve), eacfile.name)
+
+                    registration.confirmed_at = datetime.utcnow()
+                    registration.save()
+                except TestCenterRegistration.DoesNotExist:
+                    Command.datadog_error("Failed to find record for client_auth_id {}".format(client_authorization_id), eacfile.name)
+
+    def process_vcdc(self, vcdcfile):
+        print "processing vcdc"
+        reader = csv.DictReader(vcdcfile, delimiter="\t")
+        for row in reader:
+            client_candidate_id = row['ClientCandidateID']
+            if not client_candidate_id:
+                if row['Status'] == 'Error':
+                    Command.datadog_error("Error in CDD file processing ({}): {}".format(row['Date'], row['Message']), vcdcfile.name)
+                else:
+                    Command.datadog_error("Encountered bad record: {}".format(row), vcdcfile.name)
+            else:
+                try:
+                    tcuser = TestCenterUser.objects.get(client_candidate_id=client_candidate_id)
+                    Command.datadog_error("Found demographics record for user {}".format(tcuser.user.username), vcdcfile.name)
+                    # now update the record:
+                    tcuser.upload_status = row['Status']
+                    tcuser.upload_error_message = row['Message']
+                    try:
+                        tcuser.processed_at = strftime('%Y-%m-%d %H:%M:%S', strptime(row['Date'], '%Y/%m/%d %H:%M:%S'))
+                    except ValueError as ve:
+                        Command.datadog_error("Bad Date value found for {}: message {}".format(client_candidate_id, ve), vcdcfile.name)
+                    # store the candidate Id if one is provided.  (For debugging)
+                    if row['CandidateID']:
+                        try:
+                            tcuser.candidate_id = int(row['CandidateID'])
+                        except ValueError as ve:
+                            Command.datadog_error("Bad CandidateID value found for {}: message {}".format(client_candidate_id, ve), vcdcfile.name)
+                    tcuser.confirmed_at = datetime.utcnow()
+                    tcuser.save()
+                except TestCenterUser.DoesNotExist:
+                    Command.datadog_error(" Failed to find record for client_candidate_id {}".format(client_candidate_id), vcdcfile.name)
+

common/djangoapps/student/management/commands/pearson_make_tc_registration.py

@@ -71,6 +71,12 @@ class Command(BaseCommand):
             dest='ignore_registration_dates',
             help='find exam info for course based on exam_series_code, even if the exam is not active.'
         ),   
+        make_option(
+            '--create_dummy_exam',
+            action='store_true',
+            dest='create_dummy_exam',
+            help='create dummy exam info for course, even if course exists'
+        ),   
     )
     args = "<student_username course_id>"
     help = "Create or modify a TestCenterRegistration entry for a given Student"
@@ -98,15 +104,20 @@ class Command(BaseCommand):
         except TestCenterUser.DoesNotExist:
             raise CommandError("User \"{}\" does not have an existing demographics record".format(username))
             
-        # check to see if a course_id was specified, and use information from that:
-        try:
-            course = course_from_id(course_id)
-            if 'ignore_registration_dates' in our_options:
-                examlist = [exam for exam in course.test_center_exams if exam.exam_series_code == our_options.get('exam_series_code')]
-                exam = examlist[0] if len(examlist) > 0 else None
-            else:
-                exam = course.current_test_center_exam
-        except ItemNotFoundError: 
+        # get an "exam" object.  Check to see if a course_id was specified, and use information from that:
+        exam = None
+        create_dummy_exam = 'create_dummy_exam' in our_options and our_options['create_dummy_exam']
+        if not create_dummy_exam:
+            try:
+                course = course_from_id(course_id)
+                if 'ignore_registration_dates' in our_options:
+                    examlist = [exam for exam in course.test_center_exams if exam.exam_series_code == our_options.get('exam_series_code')]
+                    exam = examlist[0] if len(examlist) > 0 else None
+                else:
+                    exam = course.current_test_center_exam
+            except ItemNotFoundError: 
+                pass
+        else:
             # otherwise use explicit values (so we don't have to define a course):    
             exam_name = "Dummy Placeholder Name"
             exam_info = { 'Exam_Series_Code': our_options['exam_series_code'],
@@ -120,7 +131,7 @@ class Command(BaseCommand):
             our_options['eligibility_appointment_date_last'] = strftime("%Y-%m-%d", exam.last_eligible_appointment_date)
 
         if exam is None:
-            raise CommandError("Exam for course_id {%s} does not exist".format(course_id))
+            raise CommandError("Exam for course_id {} does not exist".format(course_id))
 
         exam_code = exam.exam_series_code
             

common/djangoapps/student/management/commands/pearson_make_tc_user.py

@@ -1,7 +1,7 @@
 from optparse import make_option
 
 from django.contrib.auth.models import User
-from django.core.management.base import BaseCommand
+from django.core.management.base import BaseCommand, CommandError
 
 from student.models import TestCenterUser, TestCenterUserForm
 
@@ -161,15 +161,16 @@ class Command(BaseCommand):
             if form.is_valid():
                 form.update_and_save()
             else:
+                errorlist = []
                 if (len(form.errors) > 0):
-                    print "Field Form errors encountered:"
-                for fielderror in form.errors:
-                    print "Field Form Error:  %s" % fielderror
-                    if (len(form.non_field_errors()) > 0):
-                        print "Non-field Form errors encountered:"
-                        for nonfielderror in form.non_field_errors:
-                            print "Non-field Form Error:  %s" % nonfielderror
-                    
+                    errorlist.append("Field Form errors encountered:")
+                    for fielderror in form.errors:
+                        errorlist.append("Field Form Error:  {}".format(fielderror))
+                if (len(form.non_field_errors()) > 0):
+                    errorlist.append("Non-field Form errors encountered:")
+                    for nonfielderror in form.non_field_errors:
+                        errorlist.append("Non-field Form Error:  {}".format(nonfielderror))
+                raise CommandError("\n".join(errorlist))    
         else:
             print "No changes necessary to make to existing user's demographics."
             

common/djangoapps/student/management/commands/pearson_transfer.py

@@ -0,0 +1,163 @@
+import os
+from optparse import make_option
+from stat import S_ISDIR
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+from django.core.management import call_command
+from dogapi import dog_http_api, dog_stats_api
+import paramiko
+import boto
+
+dog_http_api.api_key = settings.DATADOG_API
+
+
+class Command(BaseCommand):
+    help = """
+    This command handles the importing and exporting of student records for
+    Pearson.  It uses some other Django commands to export and import the
+    files and then uploads over SFTP to Pearson and stuffs the entry in an
+    S3 bucket for archive purposes.
+
+    Usage: django-admin.py pearson-transfer --mode [import|export|both]
+    """
+
+    option_list = BaseCommand.option_list + (
+        make_option('--mode',
+                    action='store',
+                    dest='mode',
+                    default='both',
+                    choices=('import', 'export', 'both'),
+                    help='mode is import, export, or both'),
+    )
+
+    def handle(self, **options):
+
+        if not hasattr(settings, 'PEARSON'):
+            raise CommandError('No PEARSON entries in auth/env.json.')
+
+        # check settings needed for either import or export:
+        for value in ['SFTP_HOSTNAME', 'SFTP_USERNAME', 'SFTP_PASSWORD', 'S3_BUCKET']:
+            if value not in settings.PEARSON:
+                raise CommandError('No entry in the PEARSON settings'
+                                   '(env/auth.json) for {0}'.format(value))
+
+        for value in ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']:
+            if not hasattr(settings, value):
+                raise CommandError('No entry in the AWS settings'
+                                   '(env/auth.json) for {0}'.format(value))
+                
+        # check additional required settings for import and export:
+        if options['mode'] in ('export', 'both'):
+            for value in ['LOCAL_EXPORT','SFTP_EXPORT']: 
+                if value not in settings.PEARSON:
+                    raise CommandError('No entry in the PEARSON settings'
+                                       '(env/auth.json) for {0}'.format(value))
+            # make sure that the import directory exists or can be created:
+            source_dir = settings.PEARSON['LOCAL_EXPORT']
+            if not os.path.isdir(source_dir):
+                os.makedirs(source_dir)
+                    
+        if options['mode'] in ('import', 'both'):
+            for value in ['LOCAL_IMPORT','SFTP_IMPORT']: 
+                if value not in settings.PEARSON:
+                    raise CommandError('No entry in the PEARSON settings'
+                                       '(env/auth.json) for {0}'.format(value))
+            # make sure that the import directory exists or can be created:
+            dest_dir = settings.PEARSON['LOCAL_IMPORT']
+            if not os.path.isdir(dest_dir):
+                os.makedirs(dest_dir)
+
+
+        def sftp(files_from, files_to, mode, deleteAfterCopy=False):
+            with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'):
+                try:
+                    t = paramiko.Transport((settings.PEARSON['SFTP_HOSTNAME'], 22))
+                    t.connect(username=settings.PEARSON['SFTP_USERNAME'],
+                              password=settings.PEARSON['SFTP_PASSWORD'])
+                    sftp = paramiko.SFTPClient.from_transport(t)
+                    
+                    if mode == 'export':
+                        try:
+                            sftp.chdir(files_to)
+                        except IOError:
+                            raise CommandError('SFTP destination path does not exist: {}'.format(files_to))
+                        for filename in os.listdir(files_from):
+                            sftp.put(files_from + '/' + filename, filename)
+                            if deleteAfterCopy:
+                                os.remove(os.path.join(files_from, filename))
+                    else:
+                        try:
+                            sftp.chdir(files_from)
+                        except IOError:
+                            raise CommandError('SFTP source path does not exist: {}'.format(files_from))
+                        for filename in sftp.listdir('.'):
+                            # skip subdirectories 
+                            if not S_ISDIR(sftp.stat(filename).st_mode):
+                                sftp.get(filename, files_to + '/' + filename)
+                                # delete files from sftp server once they are successfully pulled off:
+                                if deleteAfterCopy:
+                                    sftp.remove(filename)
+                except:
+                    dog_http_api.event('pearson {0}'.format(mode),
+                                       'sftp uploading failed',
+                                       alert_type='error')
+                    raise
+                finally:
+                    sftp.close()
+                    t.close()
+
+        def s3(files_from, bucket, mode, deleteAfterCopy=False):
+            with dog_stats_api.timer('pearson.{0}'.format(mode), tags='s3'):
+                try:
+                    for filename in os.listdir(files_from):
+                        source_file = os.path.join(files_from, filename)
+                        # use mode as name of directory into which to write files 
+                        dest_file = os.path.join(mode, filename)
+                        upload_file_to_s3(bucket, source_file, dest_file)
+                        if deleteAfterCopy:
+                            os.remove(files_from + '/' + filename)
+                except:
+                    dog_http_api.event('pearson {0}'.format(mode),
+                                       's3 archiving failed')
+                    raise
+
+        def upload_file_to_s3(bucket, source_file, dest_file):
+            """
+            Upload file to S3
+            """
+            s3 = boto.connect_s3(settings.AWS_ACCESS_KEY_ID,
+                                 settings.AWS_SECRET_ACCESS_KEY)
+            from boto.s3.key import Key
+            b = s3.get_bucket(bucket)
+            k = Key(b)
+            k.key = "{filename}".format(filename=dest_file)
+            k.set_contents_from_filename(source_file)
+
+        def export_pearson():
+            options = { 'dest-from-settings' : True }
+            call_command('pearson_export_cdd', **options)
+            call_command('pearson_export_ead', **options)
+            mode = 'export'
+            sftp(settings.PEARSON['LOCAL_EXPORT'], settings.PEARSON['SFTP_EXPORT'], mode, deleteAfterCopy = False)
+            s3(settings.PEARSON['LOCAL_EXPORT'], settings.PEARSON['S3_BUCKET'], mode, deleteAfterCopy=True)
+
+        def import_pearson():
+            mode = 'import'
+            try:
+                sftp(settings.PEARSON['SFTP_IMPORT'], settings.PEARSON['LOCAL_IMPORT'], mode, deleteAfterCopy = True)
+                s3(settings.PEARSON['LOCAL_IMPORT'], settings.PEARSON['S3_BUCKET'], mode, deleteAfterCopy=False)
+            except Exception as e:
+                dog_http_api.event('Pearson Import failure', str(e))
+                raise e
+            else:
+                for filename in os.listdir(settings.PEARSON['LOCAL_IMPORT']):
+                    filepath = os.path.join(settings.PEARSON['LOCAL_IMPORT'], filename)
+                    call_command('pearson_import_conf_zip', filepath)
+                    os.remove(filepath)
+
+        # actually do the work!
+        if options['mode'] in ('export', 'both'):
+            export_pearson()
+        if options['mode'] in ('import', 'both'):
+            import_pearson()

common/djangoapps/student/management/commands/tests/test_pearson.py

@@ -0,0 +1,382 @@
+'''
+Created on Jan 17, 2013
+
+@author: brian
+'''
+import logging
+import os
+from tempfile import mkdtemp
+import cStringIO
+import sys
+
+from django.test import TestCase
+from django.core.management import call_command
+from nose.plugins.skip import SkipTest
+
+from student.models import User, TestCenterRegistration, TestCenterUser, get_testcenter_registration
+
+log = logging.getLogger(__name__)
+
+def create_tc_user(username):
+    user = User.objects.create_user(username, '{}@edx.org'.format(username), 'fakepass')
+    options = {
+                   'first_name' : 'TestFirst',
+                   'last_name' : 'TestLast',
+                   'address_1' : 'Test Address',
+                   'city' : 'TestCity',
+                   'state' : 'Alberta',
+                   'postal_code' : 'A0B 1C2',
+                   'country' : 'CAN',
+                   'phone' : '252-1866',
+                   'phone_country_code' : '1',
+                    }
+    call_command('pearson_make_tc_user', username, **options)
+    return TestCenterUser.objects.get(user=user)
+    
+    
+def create_tc_registration(username, course_id = 'org1/course1/term1', exam_code = 'exam1', accommodation_code = None):
+    
+    options = { 'exam_series_code' : exam_code,
+               'eligibility_appointment_date_first' : '2013-01-01T00:00',
+               'eligibility_appointment_date_last' : '2013-12-31T23:59',
+               'accommodation_code' : accommodation_code, 
+               }
+
+    call_command('pearson_make_tc_registration', username, course_id, **options)
+    user = User.objects.get(username=username)
+    registrations = get_testcenter_registration(user, course_id, exam_code)
+    return registrations[0]
+
+def create_multiple_registrations(prefix='test'):
+    username1 = '{}_multiple1'.format(prefix)
+    create_tc_user(username1)
+    create_tc_registration(username1)
+    create_tc_registration(username1, course_id = 'org1/course2/term1')
+    create_tc_registration(username1, exam_code = 'exam2')
+    username2 = '{}_multiple2'.format(prefix)
+    create_tc_user(username2)
+    create_tc_registration(username2)
+    username3 = '{}_multiple3'.format(prefix)
+    create_tc_user(username3)
+    create_tc_registration(username3, course_id = 'org1/course2/term1')
+    username4 = '{}_multiple4'.format(prefix)
+    create_tc_user(username4)
+    create_tc_registration(username4, exam_code = 'exam2')
+
+def get_command_error_text(*args, **options):
+    stderr_string = None
+    old_stderr = sys.stderr
+    sys.stderr = cStringIO.StringIO()
+    try:
+        call_command(*args, **options)
+    except SystemExit, why1:
+        # The goal here is to catch CommandError calls.
+        # But these are actually translated into nice messages,
+        # and sys.exit(1) is then called.  For testing, we
+        # want to catch what sys.exit throws, and get the
+        # relevant text either from stdout or stderr. 
+        if (why1.message > 0):
+            stderr_string = sys.stderr.getvalue()
+        else:
+            raise why1
+    except Exception, why:
+        raise why
+    
+    finally:
+        sys.stderr = old_stderr
+            
+    if stderr_string is None:
+        raise Exception("Expected call to {} to fail, but it succeeded!".format(args[0]))            
+    return stderr_string
+  
+def get_error_string_for_management_call(*args, **options):
+    stdout_string = None
+    old_stdout = sys.stdout
+    old_stderr = sys.stderr
+    sys.stdout = cStringIO.StringIO()
+    sys.stderr = cStringIO.StringIO()
+    try:
+        call_command(*args, **options)
+    except SystemExit, why1:
+        # The goal here is to catch CommandError calls.
+        # But these are actually translated into nice messages,
+        # and sys.exit(1) is then called.  For testing, we
+        # want to catch what sys.exit throws, and get the
+        # relevant text either from stdout or stderr. 
+        if (why1.message == 1):
+            stdout_string = sys.stdout.getvalue()
+            stderr_string = sys.stderr.getvalue()
+        else:
+            raise why1
+    except Exception, why:
+        raise why
+    
+    finally:
+        sys.stdout = old_stdout
+        sys.stderr = old_stderr
+            
+    if stdout_string is None:
+        raise Exception("Expected call to {} to fail, but it succeeded!".format(args[0]))            
+    return stdout_string, stderr_string
+        
+
+def get_file_info(dirpath):
+    filelist = os.listdir(dirpath)
+    print 'Files found: {}'.format(filelist)
+    numfiles = len(filelist)
+    if numfiles == 1:
+        filepath = os.path.join(dirpath, filelist[0])
+        with open(filepath, 'r') as cddfile:
+            filecontents = cddfile.readlines()
+            numlines = len(filecontents)
+            return filepath, numlines
+    else:
+        raise Exception("Expected to find a single file in {}, but found {}".format(dirpath,filelist))
+            
+class PearsonTestCase(TestCase):
+    '''
+    Base class for tests running Pearson-related commands
+    '''
+    import_dir = mkdtemp(prefix="import")
+    export_dir = mkdtemp(prefix="export")
+    
+    def assertErrorContains(self, error_message, expected):
+        self.assertTrue(error_message.find(expected) >= 0, 'error message "{}" did not contain "{}"'.format(error_message, expected))
+        
+    def tearDown(self):
+        def delete_temp_dir(dirname):
+            if os.path.exists(dirname):
+                for filename in os.listdir(dirname):
+                    os.remove(os.path.join(dirname, filename))
+                os.rmdir(dirname)
+            
+        # clean up after any test data was dumped to temp directory
+        delete_temp_dir(self.import_dir)
+        delete_temp_dir(self.export_dir)
+        
+        # and clean up the database:
+#        TestCenterUser.objects.all().delete()
+#        TestCenterRegistration.objects.all().delete()
+
+class PearsonCommandTestCase(PearsonTestCase):
+
+    def test_missing_demographic_fields(self):
+        # We won't bother to test all details of form validation here.  
+        # It is enough to show that it works here, but deal with test cases for the form
+        # validation in the student tests, not these management tests.
+        username = 'baduser'
+        User.objects.create_user(username, '{}@edx.org'.format(username), 'fakepass')
+        options = {}
+        error_string = get_command_error_text('pearson_make_tc_user', username, **options) 
+        self.assertTrue(error_string.find('Field Form errors encountered:') >= 0)
+        self.assertTrue(error_string.find('Field Form Error:  city') >= 0)
+        self.assertTrue(error_string.find('Field Form Error:  first_name') >= 0)
+        self.assertTrue(error_string.find('Field Form Error:  last_name') >= 0)
+        self.assertTrue(error_string.find('Field Form Error:  country') >= 0)
+        self.assertTrue(error_string.find('Field Form Error:  phone_country_code') >= 0)
+        self.assertTrue(error_string.find('Field Form Error:  phone') >= 0)
+        self.assertTrue(error_string.find('Field Form Error:  address_1') >= 0)
+        self.assertErrorContains(error_string, 'Field Form Error:  address_1')
+        
+    def test_create_good_testcenter_user(self):
+        testcenter_user = create_tc_user("test1")
+        self.assertIsNotNone(testcenter_user)
+        
+    def test_create_good_testcenter_registration(self):
+        username = 'test1'
+        create_tc_user(username)
+        registration = create_tc_registration(username)
+        self.assertIsNotNone(registration)
+
+    def test_cdd_missing_option(self):
+        error_string = get_command_error_text('pearson_export_cdd', **{})
+        self.assertErrorContains(error_string, 'Error: --destination or --dest-from-settings must be used')
+   
+    def test_ead_missing_option(self):
+        error_string = get_command_error_text('pearson_export_ead', **{})
+        self.assertErrorContains(error_string, 'Error: --destination or --dest-from-settings must be used')
+
+    def test_export_single_cdd(self):
+        # before we generate any tc_users, we expect there to be nothing to output:
+        options = { 'dest-from-settings' : True }
+        with self.settings(PEARSON={ 'LOCAL_EXPORT' : self.export_dir }):
+            call_command('pearson_export_cdd', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 1, "Expect cdd file to have no non-header lines")
+            os.remove(filepath)
+
+            # generating a tc_user should result in a line in the output    
+            username = 'test_single_cdd'
+            create_tc_user(username)
+            call_command('pearson_export_cdd', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 2, "Expect cdd file to have one non-header line")
+            os.remove(filepath)
+
+            # output after registration should not have any entries again.
+            call_command('pearson_export_cdd', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 1, "Expect cdd file to have no non-header lines")
+            os.remove(filepath)
+
+            # if we modify the record, then it should be output again:
+            user_options = { 'first_name' : 'NewTestFirst', }
+            call_command('pearson_make_tc_user', username, **user_options)
+            call_command('pearson_export_cdd', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 2, "Expect cdd file to have one non-header line")
+            os.remove(filepath)
+            
+    def test_export_single_ead(self):
+        # before we generate any registrations, we expect there to be nothing to output:
+        options = { 'dest-from-settings' : True }
+        with self.settings(PEARSON={ 'LOCAL_EXPORT' : self.export_dir }):
+            call_command('pearson_export_ead', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 1, "Expect ead file to have no non-header lines")
+            os.remove(filepath)
+
+            # generating a registration should result in a line in the output    
+            username = 'test_single_ead'
+            create_tc_user(username)
+            create_tc_registration(username)
+            call_command('pearson_export_ead', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 2, "Expect ead file to have one non-header line")
+            os.remove(filepath)
+
+            # output after registration should not have any entries again.
+            call_command('pearson_export_ead', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 1, "Expect ead file to have no non-header lines")
+            os.remove(filepath)
+        
+            # if we modify the record, then it should be output again:
+            create_tc_registration(username, accommodation_code='EQPMNT')
+            call_command('pearson_export_ead', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 2, "Expect ead file to have one non-header line")
+            os.remove(filepath)
+
+    def test_export_multiple(self):
+        create_multiple_registrations("export")
+        with self.settings(PEARSON={ 'LOCAL_EXPORT' : self.export_dir }):
+            options = { 'dest-from-settings' : True }
+            call_command('pearson_export_cdd', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 5, "Expect cdd file to have four non-header lines: total was {}".format(numlines))
+            os.remove(filepath)
+
+            call_command('pearson_export_ead', **options)
+            (filepath, numlines) = get_file_info(self.export_dir)
+            self.assertEquals(numlines, 7, "Expect ead file to have six non-header lines: total was {}".format(numlines))
+            os.remove(filepath)
+
+
+#    def test_bad_demographic_option(self):
+#        username = 'nonuser'
+#        output_string, stderrmsg = get_error_string_for_management_call('pearson_make_tc_user', username, **{'--garbage' : None })
+#        print stderrmsg
+#        self.assertErrorContains(stderrmsg, 'Unexpected option')
+#
+#    def test_missing_demographic_user(self):
+#        username = 'nonuser'
+#        output_string, error_string = get_error_string_for_management_call('pearson_make_tc_user', username, **{})
+#        self.assertErrorContains(error_string, 'User matching query does not exist')
+
+# credentials for a test SFTP site:
+SFTP_HOSTNAME = 'ec2-23-20-150-101.compute-1.amazonaws.com'
+SFTP_USERNAME = 'pearsontest'
+SFTP_PASSWORD = 'password goes here'
+
+S3_BUCKET = 'edx-pearson-archive'
+AWS_ACCESS_KEY_ID = 'put yours here'
+AWS_SECRET_ACCESS_KEY = 'put yours here'
+
+class PearsonTransferTestCase(PearsonTestCase):
+    '''
+    Class for tests running Pearson transfers
+    '''
+
+    def test_transfer_config(self):
+        with self.settings(DATADOG_API='FAKE_KEY'):
+            # TODO: why is this failing with the wrong error message?!
+            stderrmsg = get_command_error_text('pearson_transfer', **{'mode' : 'garbage'})
+            self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries')
+        with self.settings(DATADOG_API='FAKE_KEY'):
+            stderrmsg = get_command_error_text('pearson_transfer')
+            self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries')
+        with self.settings(DATADOG_API='FAKE_KEY',
+                           PEARSON={'LOCAL_EXPORT' : self.export_dir,
+                                    'LOCAL_IMPORT' : self.import_dir }):
+            stderrmsg = get_command_error_text('pearson_transfer')
+            self.assertErrorContains(stderrmsg, 'Error: No entry in the PEARSON settings')
+
+    def test_transfer_export_missing_dest_dir(self):
+        raise SkipTest()
+        create_multiple_registrations('export_missing_dest')
+        with self.settings(DATADOG_API='FAKE_KEY',
+                           PEARSON={'LOCAL_EXPORT' : self.export_dir,
+                                    'SFTP_EXPORT' : 'this/does/not/exist',
+                                    'SFTP_HOSTNAME' : SFTP_HOSTNAME,
+                                    'SFTP_USERNAME' : SFTP_USERNAME,
+                                    'SFTP_PASSWORD' : SFTP_PASSWORD, 
+                                    'S3_BUCKET' : S3_BUCKET,
+                                    },
+                           AWS_ACCESS_KEY_ID = AWS_ACCESS_KEY_ID,
+                           AWS_SECRET_ACCESS_KEY = AWS_SECRET_ACCESS_KEY):
+            options = { 'mode' : 'export'}
+            stderrmsg = get_command_error_text('pearson_transfer', **options)
+            self.assertErrorContains(stderrmsg, 'Error: SFTP destination path does not exist')
+
+    def test_transfer_export(self):
+        raise SkipTest()
+        create_multiple_registrations("transfer_export")
+        with self.settings(DATADOG_API='FAKE_KEY',
+                           PEARSON={'LOCAL_EXPORT' : self.export_dir,
+                                    'SFTP_EXPORT' : 'results/topvue',
+                                    'SFTP_HOSTNAME' : SFTP_HOSTNAME,
+                                    'SFTP_USERNAME' : SFTP_USERNAME,
+                                    'SFTP_PASSWORD' : SFTP_PASSWORD, 
+                                    'S3_BUCKET' : S3_BUCKET,
+                                    },
+                           AWS_ACCESS_KEY_ID = AWS_ACCESS_KEY_ID,
+                           AWS_SECRET_ACCESS_KEY = AWS_SECRET_ACCESS_KEY):
+            options = { 'mode' : 'export'}
+#            call_command('pearson_transfer', **options)
+#            # confirm that the export directory is still empty:
+#            self.assertEqual(len(os.listdir(self.export_dir)), 0, "expected export directory to be empty")
+
+    def test_transfer_import_missing_source_dir(self):
+        raise SkipTest()
+        create_multiple_registrations('import_missing_src')
+        with self.settings(DATADOG_API='FAKE_KEY',
+                           PEARSON={'LOCAL_IMPORT' : self.import_dir,
+                                    'SFTP_IMPORT' : 'this/does/not/exist',
+                                    'SFTP_HOSTNAME' : SFTP_HOSTNAME,
+                                    'SFTP_USERNAME' : SFTP_USERNAME,
+                                    'SFTP_PASSWORD' : SFTP_PASSWORD, 
+                                    'S3_BUCKET' : S3_BUCKET,
+                                    },
+                           AWS_ACCESS_KEY_ID = AWS_ACCESS_KEY_ID,
+                           AWS_SECRET_ACCESS_KEY = AWS_SECRET_ACCESS_KEY):
+            options = { 'mode' : 'import'}
+            stderrmsg = get_command_error_text('pearson_transfer', **options)
+            self.assertErrorContains(stderrmsg, 'Error: SFTP source path does not exist')
+
+    def test_transfer_import(self):
+        raise SkipTest()
+        create_multiple_registrations('import_missing_src')
+        with self.settings(DATADOG_API='FAKE_KEY',
+                           PEARSON={'LOCAL_IMPORT' : self.import_dir,
+                                    'SFTP_IMPORT' : 'results',
+                                    'SFTP_HOSTNAME' : SFTP_HOSTNAME,
+                                    'SFTP_USERNAME' : SFTP_USERNAME,
+                                    'SFTP_PASSWORD' : SFTP_PASSWORD, 
+                                    'S3_BUCKET' : S3_BUCKET,
+                                    },
+                           AWS_ACCESS_KEY_ID = AWS_ACCESS_KEY_ID,
+                           AWS_SECRET_ACCESS_KEY = AWS_SECRET_ACCESS_KEY):
+            options = { 'mode' : 'import'}
+            call_command('pearson_transfer', **options)
+            self.assertEqual(len(os.listdir(self.import_dir)), 0, "expected import directory to be empty")

common/djangoapps/student/models.py

@@ -1,30 +1,5 @@
 """
-Models for Student Information
-
-Replication Notes
-
-TODO: Update this to be consistent with reality  (no portal servers, no more askbot)
-
-In our live deployment, we intend to run in a scenario where there is a pool of
-Portal servers that hold the canoncial user information and that user
-information is replicated to slave Course server pools. Each Course has a set of
-servers that serves only its content and has users that are relevant only to it.
-
-We replicate the following tables into the Course DBs where the user is
-enrolled. Only the Portal servers should ever write to these models.
-* UserProfile
-* CourseEnrollment
-
-We do a partial replication of:
-* User -- Askbot extends this and uses the extra fields, so we replicate only
-          the stuff that comes with basic django_auth and ignore the rest.)
-
-There are a couple different scenarios:
-
-1. There's an update of User or UserProfile -- replicate it to all Course DBs
-   that the user is enrolled in (found via CourseEnrollment).
-2. There's a change in CourseEnrollment. We need to push copies of UserProfile,
-   CourseEnrollment, and the base fields in User
+Models for User Information (students, staff, etc)
 
 Migration Notes
 
@@ -146,8 +121,8 @@ class TestCenterUser(models.Model):
     The field names and lengths are modeled on the conventions and constraints
     of Pearson's data import system, including oddities such as suffix having
     a limit of 255 while last_name only gets 50.
-    
-    Also storing here the confirmation information received from Pearson (if any) 
+
+    Also storing here the confirmation information received from Pearson (if any)
     as to the success or failure of the upload.  (VCDC file)
     """
     # Our own record keeping...
@@ -197,7 +172,7 @@ class TestCenterUser(models.Model):
     uploaded_at = models.DateTimeField(null=True, blank=True, db_index=True)
 
     # confirmation back from the test center, as well as timestamps
-    # on when they processed the request, and when we received 
+    # on when they processed the request, and when we received
     # confirmation back.
     processed_at = models.DateTimeField(null=True, db_index=True)
     upload_status = models.CharField(max_length=20, blank=True, db_index=True)  # 'Error' or 'Accepted'
@@ -211,52 +186,52 @@ class TestCenterUser(models.Model):
     @property
     def needs_uploading(self):
         return self.uploaded_at is None or self.uploaded_at < self.user_updated_at
-    
+
     @staticmethod
     def user_provided_fields():
-        return [ 'first_name', 'middle_name', 'last_name', 'suffix', 'salutation', 
-                'address_1', 'address_2', 'address_3', 'city', 'state', 'postal_code', 'country', 
+        return [ 'first_name', 'middle_name', 'last_name', 'suffix', 'salutation',
+                'address_1', 'address_2', 'address_3', 'city', 'state', 'postal_code', 'country',
                 'phone', 'extension', 'phone_country_code', 'fax', 'fax_country_code', 'company_name']
-        
+
     @property
     def email(self):
         return self.user.email
-    
+
     def needs_update(self, fields):
         for fieldname in TestCenterUser.user_provided_fields():
             if fieldname in fields and getattr(self, fieldname) != fields[fieldname]:
                 return True
-            
-        return False    
-                       
+
+        return False
+
     @staticmethod
     def _generate_edx_id(prefix):
         NUM_DIGITS = 12
         return u"{}{:012}".format(prefix, randint(1, 10**NUM_DIGITS-1))
-    
+
     @staticmethod
     def _generate_candidate_id():
         return TestCenterUser._generate_edx_id("edX")
-        
+
     @classmethod
     def create(cls, user):
         testcenter_user = cls(user=user)
-        # testcenter_user.candidate_id remains unset    
+        # testcenter_user.candidate_id remains unset
         # assign an ID of our own:
         cand_id = cls._generate_candidate_id()
         while TestCenterUser.objects.filter(client_candidate_id=cand_id).exists():
             cand_id = cls._generate_candidate_id()
-        testcenter_user.client_candidate_id = cand_id  
+        testcenter_user.client_candidate_id = cand_id
         return testcenter_user
 
     @property
     def is_accepted(self):
         return self.upload_status == TEST_CENTER_STATUS_ACCEPTED
-  
+
     @property
     def is_rejected(self):
         return self.upload_status == TEST_CENTER_STATUS_ERROR
-    
+
     @property
     def is_pending(self):
         return not self.is_accepted and not self.is_rejected
@@ -264,26 +239,26 @@ class TestCenterUser(models.Model):
 class TestCenterUserForm(ModelForm):
     class Meta:
         model = TestCenterUser
-        fields = ( 'first_name', 'middle_name', 'last_name', 'suffix', 'salutation', 
-                'address_1', 'address_2', 'address_3', 'city', 'state', 'postal_code', 'country', 
+        fields = ( 'first_name', 'middle_name', 'last_name', 'suffix', 'salutation',
+                'address_1', 'address_2', 'address_3', 'city', 'state', 'postal_code', 'country',
                 'phone', 'extension', 'phone_country_code', 'fax', 'fax_country_code', 'company_name')
-        
+
     def update_and_save(self):
         new_user = self.save(commit=False)
         # create additional values here:
         new_user.user_updated_at = datetime.utcnow()
         new_user.upload_status = ''
         new_user.save()
-        log.info("Updated demographic information for user's test center exam registration: username \"{}\" ".format(new_user.user.username)) 
-        
+        log.info("Updated demographic information for user's test center exam registration: username \"{}\" ".format(new_user.user.username))
+
     # add validation:
-    
+
     def clean_country(self):
         code = self.cleaned_data['country']
         if code and len(code) != 3:
             raise forms.ValidationError(u'Must be three characters (ISO 3166-1):  e.g. USA, CAN, MNG')
         return code
-                
+
     def clean(self):
         def _can_encode_as_latin(fieldvalue):
             try:
@@ -291,40 +266,40 @@ class TestCenterUserForm(ModelForm):
             except UnicodeEncodeError:
                 return False
             return True
-        
+
         cleaned_data = super(TestCenterUserForm, self).clean()
-        
+
         # check for interactions between fields:
         if 'country' in cleaned_data:
             country = cleaned_data.get('country')
             if country == 'USA' or country == 'CAN':
                 if 'state' in cleaned_data and len(cleaned_data['state']) == 0:
-                    self._errors['state'] = self.error_class([u'Required if country is USA or CAN.'])                
+                    self._errors['state'] = self.error_class([u'Required if country is USA or CAN.'])
                     del cleaned_data['state']
 
                 if 'postal_code' in cleaned_data and len(cleaned_data['postal_code']) == 0:
-                    self._errors['postal_code'] = self.error_class([u'Required if country is USA or CAN.'])                
+                    self._errors['postal_code'] = self.error_class([u'Required if country is USA or CAN.'])
                     del cleaned_data['postal_code']
-                    
+
         if 'fax' in cleaned_data and len(cleaned_data['fax']) > 0 and 'fax_country_code' in cleaned_data and len(cleaned_data['fax_country_code']) == 0:
-            self._errors['fax_country_code'] = self.error_class([u'Required if fax is specified.'])                
+            self._errors['fax_country_code'] = self.error_class([u'Required if fax is specified.'])
             del cleaned_data['fax_country_code']
 
         # check encoding for all fields:
         cleaned_data_fields = [fieldname for fieldname in cleaned_data]
         for fieldname in cleaned_data_fields:
             if not _can_encode_as_latin(cleaned_data[fieldname]):
-                self._errors[fieldname] = self.error_class([u'Must only use characters in Latin-1 (iso-8859-1) encoding'])                
+                self._errors[fieldname] = self.error_class([u'Must only use characters in Latin-1 (iso-8859-1) encoding'])
                 del cleaned_data[fieldname]
 
         # Always return the full collection of cleaned data.
         return cleaned_data
-        
-# our own code to indicate that a request has been rejected. 
-ACCOMMODATION_REJECTED_CODE = 'NONE'        
-   
+
+# our own code to indicate that a request has been rejected.
+ACCOMMODATION_REJECTED_CODE = 'NONE'
+
 ACCOMMODATION_CODES = (
-                      (ACCOMMODATION_REJECTED_CODE, 'No Accommodation Granted'), 
+                      (ACCOMMODATION_REJECTED_CODE, 'No Accommodation Granted'),
                       ('EQPMNT', 'Equipment'),
                       ('ET12ET', 'Extra Time - 1/2 Exam Time'),
                       ('ET30MN', 'Extra Time - 30 Minutes'),
@@ -334,11 +309,11 @@ ACCOMMODATION_CODES = (
                       ('SRRERC', 'Separate Room and Reader/Recorder'),
                       ('SRRECR', 'Separate Room and Recorder'),
                       ('SRSEAN', 'Separate Room and Service Animal'),
-                      ('SRSGNR', 'Separate Room and Sign Language Interpreter'), 
+                      ('SRSGNR', 'Separate Room and Sign Language Interpreter'),
                       )
 
 ACCOMMODATION_CODE_DICT = { code : name for (code, name) in ACCOMMODATION_CODES }
-    
+
 class TestCenterRegistration(models.Model):
     """
     This is our representation of a user's registration for in-person testing,
@@ -353,20 +328,20 @@ class TestCenterRegistration(models.Model):
     of Pearson's data import system.
     """
     # to find an exam registration, we key off of the user and course_id.
-    # If multiple exams per course are possible, we would also need to add the 
+    # If multiple exams per course are possible, we would also need to add the
     # exam_series_code.
     testcenter_user = models.ForeignKey(TestCenterUser, default=None)
     course_id = models.CharField(max_length=128, db_index=True)
-    
+
     created_at = models.DateTimeField(auto_now_add=True, db_index=True)
     updated_at = models.DateTimeField(auto_now=True, db_index=True)
     # user_updated_at happens only when the user makes a change to their data,
     # and is something Pearson needs to know to manage updates. Unlike
     # updated_at, this will not get incremented when we do a batch data import.
-    # The appointment dates, the exam count, and the accommodation codes can be updated, 
+    # The appointment dates, the exam count, and the accommodation codes can be updated,
     # but hopefully this won't happen often.
     user_updated_at = models.DateTimeField(db_index=True)
-    # "client_authorization_id" is our unique identifier for the authorization.  
+    # "client_authorization_id" is our unique identifier for the authorization.
     # This must be present for an update or delete to be sent to Pearson.
     client_authorization_id = models.CharField(max_length=20, unique=True, db_index=True)
 
@@ -376,10 +351,10 @@ class TestCenterRegistration(models.Model):
     eligibility_appointment_date_last = models.DateField(db_index=True)
 
     # this is really a list of codes, using an '*' as a delimiter.
-    # So it's not a choice list.  We use the special value of ACCOMMODATION_REJECTED_CODE 
+    # So it's not a choice list.  We use the special value of ACCOMMODATION_REJECTED_CODE
     # to indicate the rejection of an accommodation request.
     accommodation_code = models.CharField(max_length=64, blank=True)
-    
+
     # store the original text of the accommodation request.
     accommodation_request = models.CharField(max_length=1024, blank=True, db_index=True)
 
@@ -387,7 +362,7 @@ class TestCenterRegistration(models.Model):
     uploaded_at = models.DateTimeField(null=True, db_index=True)
 
     # confirmation back from the test center, as well as timestamps
-    # on when they processed the request, and when we received 
+    # on when they processed the request, and when we received
     # confirmation back.
     processed_at = models.DateTimeField(null=True, db_index=True)
     upload_status = models.CharField(max_length=20, blank=True, db_index=True)  # 'Error' or 'Accepted'
@@ -397,11 +372,11 @@ class TestCenterRegistration(models.Model):
     # (However, it may never be set if we are always initiating such candidate creation.)
     authorization_id = models.IntegerField(null=True, db_index=True)
     confirmed_at = models.DateTimeField(null=True, db_index=True)
-    
+
     @property
     def candidate_id(self):
         return self.testcenter_user.candidate_id
-    
+
     @property
     def client_candidate_id(self):
         return self.testcenter_user.client_candidate_id
@@ -414,20 +389,24 @@ class TestCenterRegistration(models.Model):
             return 'Add'
         else:
             # TODO: decide what to send when we have uploaded an initial version,
-            # but have not received confirmation back from that upload.  If the 
+            # but have not received confirmation back from that upload.  If the
             # registration here has been changed, then we don't know if this changed
-            # registration should be submitted as an 'add' or an 'update'. 
+            # registration should be submitted as an 'add' or an 'update'.
             #
-            # If the first registration were lost or in error (e.g. bad code), 
+            # If the first registration were lost or in error (e.g. bad code),
             # the second should be an "Add".  If the first were processed successfully,
             # then the second should be an "Update".  We just don't know....
             return 'Update'
-        
+
     @property
     def exam_authorization_count(self):
         # TODO: figure out if this should really go in the database (with a default value).
         return 1
     
+    @property
+    def needs_uploading(self):
+        return self.uploaded_at is None or self.uploaded_at < self.user_updated_at
+    
     @classmethod
     def create(cls, testcenter_user, exam, accommodation_request):
         registration = cls(testcenter_user = testcenter_user)
@@ -443,7 +422,7 @@ class TestCenterRegistration(models.Model):
     @staticmethod
     def _generate_authorization_id():
         return TestCenterUser._generate_edx_id("edXexam")
-        
+
     @staticmethod
     def _create_client_authorization_id():
         """
@@ -455,8 +434,8 @@ class TestCenterRegistration(models.Model):
         while TestCenterRegistration.objects.filter(client_authorization_id=auth_id).exists():
             auth_id = TestCenterRegistration._generate_authorization_id()
         return auth_id
-            
-    # methods for providing registration status details on registration page:        
+
+    # methods for providing registration status details on registration page:
     @property
     def demographics_is_accepted(self):
         return self.testcenter_user.is_accepted
@@ -464,7 +443,7 @@ class TestCenterRegistration(models.Model):
     @property
     def demographics_is_rejected(self):
         return self.testcenter_user.is_rejected
-                
+
     @property
     def demographics_is_pending(self):
         return self.testcenter_user.is_pending
@@ -476,7 +455,7 @@ class TestCenterRegistration(models.Model):
     @property
     def accommodation_is_rejected(self):
         return len(self.accommodation_request) > 0 and self.accommodation_code == ACCOMMODATION_REJECTED_CODE
-            
+
     @property
     def accommodation_is_pending(self):
         return len(self.accommodation_request) > 0 and len(self.accommodation_code) == 0
@@ -488,20 +467,20 @@ class TestCenterRegistration(models.Model):
     @property
     def registration_is_accepted(self):
         return self.upload_status == TEST_CENTER_STATUS_ACCEPTED
-  
+
     @property
     def registration_is_rejected(self):
         return self.upload_status == TEST_CENTER_STATUS_ERROR
-    
+
     @property
     def registration_is_pending(self):
         return not self.registration_is_accepted and not self.registration_is_rejected
 
-    # methods for providing registration status summary on dashboard page:        
+    # methods for providing registration status summary on dashboard page:
     @property
     def is_accepted(self):
         return self.registration_is_accepted and self.demographics_is_accepted
-  
+
     @property
     def is_rejected(self):
         return self.registration_is_rejected or self.demographics_is_rejected
@@ -509,17 +488,17 @@ class TestCenterRegistration(models.Model):
     @property
     def is_pending(self):
         return not self.is_accepted and not self.is_rejected
-    
+
     def get_accommodation_codes(self):
         return self.accommodation_code.split('*')
 
     def get_accommodation_names(self):
-        return [ ACCOMMODATION_CODE_DICT.get(code, "Unknown code " + code) for code in self.get_accommodation_codes() ]         
+        return [ ACCOMMODATION_CODE_DICT.get(code, "Unknown code " + code) for code in self.get_accommodation_codes() ]
 
     @property
     def registration_signup_url(self):
         return settings.PEARSONVUE_SIGNINPAGE_URL
-    
+
 class TestCenterRegistrationForm(ModelForm):
     class Meta:
         model = TestCenterRegistration
@@ -530,33 +509,37 @@ class TestCenterRegistrationForm(ModelForm):
         if code and len(code) > 0:
             return code.strip()
         return code
-        
+
     def update_and_save(self):
         registration = self.save(commit=False)
         # create additional values here:
         registration.user_updated_at = datetime.utcnow()
         registration.upload_status = ''
         registration.save()
-        log.info("Updated registration information for user's test center exam registration: username \"{}\" course \"{}\", examcode \"{}\"".format(registration.testcenter_user.user.username, registration.course_id, registration.exam_series_code)) 
+        log.info("Updated registration information for user's test center exam registration: username \"{}\" course \"{}\", examcode \"{}\"".format(registration.testcenter_user.user.username, registration.course_id, registration.exam_series_code))
 
     # TODO: add validation code for values added to accommodation_code field.
-    
-    
-    
+
+
+
 def get_testcenter_registration(user, course_id, exam_series_code):
     try:
         tcu = TestCenterUser.objects.get(user=user)
     except TestCenterUser.DoesNotExist:
         return []
     return TestCenterRegistration.objects.filter(testcenter_user=tcu, course_id=course_id, exam_series_code=exam_series_code)
-        
+
+# nosetests thinks that anything with _test_ in the name is a test.
+# Correct this (https://nose.readthedocs.org/en/latest/finding_tests.html)
+get_testcenter_registration.__test__ = False
+   
 def unique_id_for_user(user):
     """
     Return a unique id for a user, suitable for inserting into
     e.g. personalized survey links.
     """
     # include the secret key as a salt, and to make the ids unique across
-    # different LMS installs.    
+    # different LMS installs.
     h = hashlib.md5()
     h.update(settings.SECRET_KEY)
     h.update(str(user.id))
@@ -638,7 +621,20 @@ class CourseEnrollmentAllowed(models.Model):
 
 #cache_relation(User.profile)
 
-#### Helper methods for use from python manage.py shell.
+#### Helper methods for use from python manage.py shell and other classes.
+
+def get_user_by_username_or_email(username_or_email):
+    """
+    Return a User object, looking up by email if username_or_email contains a
+    '@', otherwise by username.
+
+    Raises:
+        User.DoesNotExist is lookup fails.
+    """
+    if '@' in username_or_email:
+        return User.objects.get(email=username_or_email)
+    else:
+        return User.objects.get(username=username_or_email)
 
 
 def get_user(email):
@@ -729,167 +725,3 @@ def update_user_information(sender, instance, created, **kwargs):
         log.error(unicode(e))
         log.error("update user info to discussion failed for user with id: " + str(instance.id))
 
-
-########################## REPLICATION SIGNALS #################################
-# @receiver(post_save, sender=User)
-def replicate_user_save(sender, **kwargs):
-    user_obj = kwargs['instance']
-    if not should_replicate(user_obj):
-        return
-    for course_db_name in db_names_to_replicate_to(user_obj.id):
-        replicate_user(user_obj, course_db_name)
-
-
-# @receiver(post_save, sender=CourseEnrollment)
-def replicate_enrollment_save(sender, **kwargs):
-    """This is called when a Student enrolls in a course. It has to do the
-    following:
-
-    1. Make sure the User is copied into the Course DB. It may already exist
-       (someone deleting and re-adding a course). This has to happen first or
-       the foreign key constraint breaks.
-    2. Replicate the CourseEnrollment.
-    3. Replicate the UserProfile.
-    """
-    if not is_portal():
-        return
-
-    enrollment_obj = kwargs['instance']
-    log.debug("Replicating user because of new enrollment")
-    for course_db_name in db_names_to_replicate_to(enrollment_obj.user.id):
-        replicate_user(enrollment_obj.user, course_db_name)
-
-    log.debug("Replicating enrollment because of new enrollment")
-    replicate_model(CourseEnrollment.save, enrollment_obj, enrollment_obj.user_id)
-
-    log.debug("Replicating user profile because of new enrollment")
-    user_profile = UserProfile.objects.get(user_id=enrollment_obj.user_id)
-    replicate_model(UserProfile.save, user_profile, enrollment_obj.user_id)
-
-
-# @receiver(post_delete, sender=CourseEnrollment)
-def replicate_enrollment_delete(sender, **kwargs):
-    enrollment_obj = kwargs['instance']
-    return replicate_model(CourseEnrollment.delete, enrollment_obj, enrollment_obj.user_id)
-
-
-# @receiver(post_save, sender=UserProfile)
-def replicate_userprofile_save(sender, **kwargs):
-    """We just updated the UserProfile (say an update to the name), so push that
-    change to all Course DBs that we're enrolled in."""
-    user_profile_obj = kwargs['instance']
-    return replicate_model(UserProfile.save, user_profile_obj, user_profile_obj.user_id)
-
-
-######### Replication functions #########
-USER_FIELDS_TO_COPY = ["id", "username", "first_name", "last_name", "email",
-                       "password", "is_staff", "is_active", "is_superuser",
-                       "last_login", "date_joined"]
-
-
-def replicate_user(portal_user, course_db_name):
-    """Replicate a User to the correct Course DB. This is more complicated than
-    it should be because Askbot extends the auth_user table and adds its own
-    fields. So we need to only push changes to the standard fields and leave
-    the rest alone so that Askbot changes at the Course DB level don't get
-    overridden.
-    """
-    try:
-        course_user = User.objects.using(course_db_name).get(id=portal_user.id)
-        log.debug("User {0} found in Course DB, replicating fields to {1}"
-                  .format(course_user, course_db_name))
-    except User.DoesNotExist:
-        log.debug("User {0} not found in Course DB, creating copy in {1}"
-                  .format(portal_user, course_db_name))
-        course_user = User()
-
-    for field in USER_FIELDS_TO_COPY:
-        setattr(course_user, field, getattr(portal_user, field))
-
-    mark_handled(course_user)
-    course_user.save(using=course_db_name)
-    unmark(course_user)
-
-
-def replicate_model(model_method, instance, user_id):
-    """
-    model_method is the model action that we want replicated. For instance,
-                 UserProfile.save
-    """
-    if not should_replicate(instance):
-        return
-
-    course_db_names = db_names_to_replicate_to(user_id)
-    log.debug("Replicating {0} for user {1} to DBs: {2}"
-              .format(model_method, user_id, course_db_names))
-
-    mark_handled(instance)
-    for db_name in course_db_names:
-        model_method(instance, using=db_name)
-    unmark(instance)
-
-
-######### Replication Helpers #########
-
-
-def is_valid_course_id(course_id):
-    """Right now, the only database that's not a course database is 'default'.
-    I had nicer checking in here originally -- it would scan the courses that
-    were in the system and only let you choose that. But it was annoying to run
-    tests with, since we don't have course data for some for our course test
-    databases. Hence the lazy version.
-    """
-    return course_id != 'default'
-
-
-def is_portal():
-    """Are we in the portal pool? Only Portal servers are allowed to replicate
-    their changes. For now, only Portal servers see multiple DBs, so we use
-    that to decide."""
-    return len(settings.DATABASES) > 1
-
-
-def db_names_to_replicate_to(user_id):
-    """Return a list of DB names that this user_id is enrolled in."""
-    return [c.course_id
-            for c in CourseEnrollment.objects.filter(user_id=user_id)
-            if is_valid_course_id(c.course_id)]
-
-
-def marked_handled(instance):
-    """Have we marked this instance as being handled to avoid infinite loops
-    caused by saving models in post_save hooks for the same models?"""
-    return hasattr(instance, '_do_not_copy_to_course_db') and instance._do_not_copy_to_course_db
-
-
-def mark_handled(instance):
-    """You have to mark your instance with this function or else we'll go into
-    an infinite loop since we're putting listeners on Model saves/deletes and
-    the act of replication requires us to call the same model method.
-
-    We create a _replicated attribute to differentiate the first save of this
-    model vs. the duplicate save we force on to the course database. Kind of
-    a hack -- suggestions welcome.
-    """
-    instance._do_not_copy_to_course_db = True
-
-
-def unmark(instance):
-    """If we don't unmark a model after we do replication, then consecutive
-    save() calls won't be properly replicated."""
-    instance._do_not_copy_to_course_db = False
-
-
-def should_replicate(instance):
-    """Should this instance be replicated? We need to be a Portal server and
-    the instance has to not have been marked_handled."""
-    if marked_handled(instance):
-        # Basically, avoid an infinite loop. You should
-        log.debug("{0} should not be replicated because it's been marked"
-                  .format(instance))
-        return False
-    if not is_portal():
-        log.debug("{0} should not be replicated because we're not a portal."
-                  .format(instance))
-        return False
-    return True

common/djangoapps/student/tests.py

@@ -5,16 +5,11 @@ when you run "manage.py test".
 Replace this with more appropriate tests for your application.
 """
 import logging
-from datetime import datetime
-from hashlib import sha1
 
 from django.test import TestCase
-from mock import patch, Mock
-from nose.plugins.skip import SkipTest
+from mock import Mock
 
-from .models import (User, UserProfile, CourseEnrollment,
-                     replicate_user, USER_FIELDS_TO_COPY,
-                     unique_id_for_user)
+from .models import unique_id_for_user
 from .views import process_survey_link, _cert_info
 
 COURSE_1 = 'edX/toy/2012_Fall'
@@ -22,185 +17,6 @@ COURSE_2 = 'edx/full/6.002_Spring_2012'
 
 log = logging.getLogger(__name__)
 
-class ReplicationTest(TestCase):
-
-    multi_db = True
-
-    def test_user_replication(self):
-        """Test basic user replication."""
-        raise SkipTest()
-        portal_user = User.objects.create_user('rusty', 'rusty@edx.org', 'fakepass')
-        portal_user.first_name='Rusty'
-        portal_user.last_name='Skids'
-        portal_user.is_staff=True
-        portal_user.is_active=True
-        portal_user.is_superuser=True
-        portal_user.last_login=datetime(2012, 1, 1)
-        portal_user.date_joined=datetime(2011, 1, 1)
-        # This is an Askbot field and will break if askbot is not included
-
-        if hasattr(portal_user, 'seen_response_count'):
-            portal_user.seen_response_count = 10
-
-        portal_user.save(using='default')
-
-        # We replicate this user to Course 1, then pull the same user and verify
-        # that the fields copied over properly.
-        replicate_user(portal_user, COURSE_1)
-        course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
-
-        # Make sure the fields we care about got copied over for this user.
-        for field in USER_FIELDS_TO_COPY:
-            self.assertEqual(getattr(portal_user, field),
-                             getattr(course_user, field),
-                             "{0} not copied from {1} to {2}".format(
-                                 field, portal_user, course_user
-                             ))
-
-        # This hasattr lameness is here because we don't want this test to be
-        # triggered when we're being run by CMS tests (Askbot doesn't exist
-        # there, so the test will fail).
-        #
-        # seen_response_count isn't a field we care about, so it shouldn't have
-        # been copied over.
-        if hasattr(portal_user, 'seen_response_count'):
-            portal_user.seen_response_count = 20
-            replicate_user(portal_user, COURSE_1)
-            course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
-            self.assertEqual(portal_user.seen_response_count, 20)
-            self.assertEqual(course_user.seen_response_count, 0)
-
-        # Another replication should work for an email change however, since
-        # it's a field we care about.
-        portal_user.email = "clyde@edx.org"
-        replicate_user(portal_user, COURSE_1)
-        course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
-        self.assertEqual(portal_user.email, course_user.email)
-
-        # During this entire time, the user data should never have made it over
-        # to COURSE_2
-        self.assertRaises(User.DoesNotExist,
-                          User.objects.using(COURSE_2).get,
-                          id=portal_user.id)
-
-
-    def test_enrollment_for_existing_user_info(self):
-        """Test the effect of Enrolling in a class if you've already got user
-        data to be copied over."""
-        raise SkipTest()
-        # Create our User
-        portal_user = User.objects.create_user('jack', 'jack@edx.org', 'fakepass')
-        portal_user.first_name = "Jack"
-        portal_user.save()
-
-        # Set up our UserProfile info
-        portal_user_profile = UserProfile.objects.create(
-                                  user=portal_user,
-                                  name="Jack Foo",
-                                  level_of_education=None,
-                                  gender='m',
-                                  mailing_address=None,
-                                  goals="World domination",
-                              )
-        portal_user_profile.save()
-
-        # Now let's see if creating a CourseEnrollment copies all the relevant
-        # data.
-        portal_enrollment = CourseEnrollment.objects.create(user=portal_user,
-                                                            course_id=COURSE_1)
-        portal_enrollment.save()
-
-        # Grab all the copies we expect
-        course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
-        self.assertEquals(portal_user, course_user)
-        self.assertRaises(User.DoesNotExist,
-                          User.objects.using(COURSE_2).get,
-                          id=portal_user.id)
-
-        course_enrollment = CourseEnrollment.objects.using(COURSE_1).get(id=portal_enrollment.id)
-        self.assertEquals(portal_enrollment, course_enrollment)
-        self.assertRaises(CourseEnrollment.DoesNotExist,
-                          CourseEnrollment.objects.using(COURSE_2).get,
-                          id=portal_enrollment.id)
-
-        course_user_profile = UserProfile.objects.using(COURSE_1).get(id=portal_user_profile.id)
-        self.assertEquals(portal_user_profile, course_user_profile)
-        self.assertRaises(UserProfile.DoesNotExist,
-                          UserProfile.objects.using(COURSE_2).get,
-                          id=portal_user_profile.id)
-
-        log.debug("Make sure our seen_response_count is not replicated.")
-        if hasattr(portal_user, 'seen_response_count'):
-            portal_user.seen_response_count = 200
-            course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
-            self.assertEqual(portal_user.seen_response_count, 200)
-            self.assertEqual(course_user.seen_response_count, 0)
-            portal_user.save()
-
-            course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
-            self.assertEqual(portal_user.seen_response_count, 200)
-            self.assertEqual(course_user.seen_response_count, 0)
-
-            portal_user.email = 'jim@edx.org'
-            portal_user.save()
-            course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
-            self.assertEqual(portal_user.email, 'jim@edx.org')
-            self.assertEqual(course_user.email, 'jim@edx.org')
-
-
-
-    def test_enrollment_for_user_info_after_enrollment(self):
-        """Test the effect of modifying User data after you've enrolled."""
-        raise SkipTest()
-
-        # Create our User
-        portal_user = User.objects.create_user('patty', 'patty@edx.org', 'fakepass')
-        portal_user.first_name = "Patty"
-        portal_user.save()
-
-        # Set up our UserProfile info
-        portal_user_profile = UserProfile.objects.create(
-                                  user=portal_user,
-                                  name="Patty Foo",
-                                  level_of_education=None,
-                                  gender='f',
-                                  mailing_address=None,
-                                  goals="World peace",
-                              )
-        portal_user_profile.save()
-
-        # Now let's see if creating a CourseEnrollment copies all the relevant
-        # data when things are saved.
-        portal_enrollment = CourseEnrollment.objects.create(user=portal_user,
-                                                            course_id=COURSE_1)
-        portal_enrollment.save()
-
-        portal_user.last_name = "Bar"
-        portal_user.save()
-        portal_user_profile.gender = 'm'
-        portal_user_profile.save()
-
-        # Grab all the copies we expect, and make sure it doesn't end up in
-        # places we don't expect.
-        course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
-        self.assertEquals(portal_user, course_user)
-        self.assertRaises(User.DoesNotExist,
-                          User.objects.using(COURSE_2).get,
-                          id=portal_user.id)
-
-        course_enrollment = CourseEnrollment.objects.using(COURSE_1).get(id=portal_enrollment.id)
-        self.assertEquals(portal_enrollment, course_enrollment)
-        self.assertRaises(CourseEnrollment.DoesNotExist,
-                          CourseEnrollment.objects.using(COURSE_2).get,
-                          id=portal_enrollment.id)
-
-        course_user_profile = UserProfile.objects.using(COURSE_1).get(id=portal_user_profile.id)
-        self.assertEquals(portal_user_profile, course_user_profile)
-        self.assertRaises(UserProfile.DoesNotExist,
-                          UserProfile.objects.using(COURSE_2).get,
-                          id=portal_user_profile.id)
-
-
 class CourseEndingTest(TestCase):
     """Test things related to course endings: certificates, surveys, etc"""
 

common/lib/xmodule/xmodule/course_module.py

@@ -223,7 +223,7 @@ class CourseDescriptor(SequenceDescriptor):
 
         return policy_str
 
-    
+
     @classmethod
     def from_xml(cls, xml_data, system, org=None, course=None):
         instance = super(CourseDescriptor, cls).from_xml(xml_data, system, org, course)
@@ -248,7 +248,7 @@ class CourseDescriptor(SequenceDescriptor):
         except ValueError:
             system.error_tracker("Unable to decode grading policy as json")
             policy = None
-        
+
         # cdodge: import the grading policy information that is on disk and put into the
         # descriptor 'definition' bucket as a dictionary so that it is persisted in the DB
         instance.definition['data']['grading_policy'] = policy
@@ -303,28 +303,28 @@ class CourseDescriptor(SequenceDescriptor):
     @property
     def enrollment_start(self):
         return self._try_parse_time("enrollment_start")
-        
+
     @enrollment_start.setter
     def enrollment_start(self, value):
         if isinstance(value, time.struct_time):
             self.metadata['enrollment_start'] = stringify_time(value)
     @property
-    def enrollment_end(self):        
+    def enrollment_end(self):
         return self._try_parse_time("enrollment_end")
-        
+
     @enrollment_end.setter
     def enrollment_end(self, value):
         if isinstance(value, time.struct_time):
             self.metadata['enrollment_end'] = stringify_time(value)
-        
+
     @property
     def grader(self):
         return self._grading_policy['GRADER']
-    
+
     @property
     def raw_grader(self):
         return self._grading_policy['RAW_GRADER']
-    
+
     @raw_grader.setter
     def raw_grader(self, value):
         # NOTE WELL: this change will not update the processed graders. If we need that, this needs to call grader_from_conf
@@ -334,12 +334,12 @@ class CourseDescriptor(SequenceDescriptor):
     @property
     def grade_cutoffs(self):
         return self._grading_policy['GRADE_CUTOFFS']
-    
+
     @grade_cutoffs.setter
     def grade_cutoffs(self, value):
         self._grading_policy['GRADE_CUTOFFS'] = value
         self.definition['data'].setdefault('grading_policy',{})['GRADE_CUTOFFS'] = value
-    
+
 
     @property
     def lowest_passing_grade(self):
@@ -360,6 +360,41 @@ class CourseDescriptor(SequenceDescriptor):
     def show_calculator(self):
         return self.metadata.get("show_calculator", None) == "Yes"
 
+    @property
+    def is_cohorted(self):
+        """
+        Return whether the course is cohorted.
+        """
+        config = self.metadata.get("cohort_config")
+        if config is None:
+            return False
+
+        return bool(config.get("cohorted"))
+
+    @property
+    def top_level_discussion_topic_ids(self):
+        """
+        Return list of topic ids defined in course policy.
+        """
+        topics = self.metadata.get("discussion_topics", {})
+        return [d["id"] for d in topics.values()] 
+
+
+    @property
+    def cohorted_discussions(self):
+        """
+        Return the set of discussions that is explicitly cohorted.  It may be
+        the empty set.  Note that all inline discussions are automatically
+        cohorted based on the course's is_cohorted setting.
+        """
+        config = self.metadata.get("cohort_config")
+        if config is None:
+            return set()
+
+        return set(config.get("cohorted_discussions", []))
+
+
+
     @property
     def is_new(self):
         """

common/lib/xmodule/xmodule/discussion_module.py

@@ -18,8 +18,10 @@ class DiscussionModule(XModule):
         }
         return self.system.render_template('discussion/_discussion_module.html', context)
 
-    def __init__(self, system, location, definition, descriptor, instance_state=None, shared_state=None, **kwargs):
-        XModule.__init__(self, system, location, definition, descriptor, instance_state, shared_state, **kwargs)
+    def __init__(self, system, location, definition, descriptor,
+                 instance_state=None, shared_state=None, **kwargs):
+        XModule.__init__(self, system, location, definition, descriptor,
+                         instance_state, shared_state, **kwargs)
 
         if isinstance(instance_state, str):
             instance_state = json.loads(instance_state)

common/lib/xmodule/xmodule/js/src/video/display/video_player.coffee

@@ -45,6 +45,7 @@ class @VideoPlayer extends Subview
       modestbranding: 1
     if @video.start
       @playerVars.start = @video.start
+      @playerVars.wmode = 'window'
     if @video.end
       # work in AS3, not HMLT5. but iframe use AS3
       @playerVars.end = @video.end

common/lib/xmodule/xmodule/tests/test_import.py

@@ -45,13 +45,24 @@ class DummySystem(ImportSystem):
             raise Exception("Shouldn't be called")
 
 
-class ImportTestCase(unittest.TestCase):
+class BaseCourseTestCase(unittest.TestCase):
     '''Make sure module imports work properly, including for malformed inputs'''
     @staticmethod
     def get_system(load_error_modules=True):
         '''Get a dummy system'''
         return DummySystem(load_error_modules)
 
+    def get_course(self, name):
+        """Get a test course by directory name.  If there's more than one, error."""
+        print "Importing {0}".format(name)
+
+        modulestore = XMLModuleStore(DATA_DIR, course_dirs=[name])
+        courses = modulestore.get_courses()
+        self.assertEquals(len(courses), 1)
+        return courses[0]
+
+class ImportTestCase(BaseCourseTestCase):
+
     def test_fallback(self):
         '''Check that malformed xml loads as an ErrorDescriptor.'''
 
@@ -207,11 +218,7 @@ class ImportTestCase(unittest.TestCase):
         """Make sure that metadata is inherited properly"""
 
         print "Starting import"
-        initial_import = XMLModuleStore(DATA_DIR, course_dirs=['toy'])
-
-        courses = initial_import.get_courses()
-        self.assertEquals(len(courses), 1)
-        course = courses[0]
+        course = self.get_course('toy')
 
         def check_for_key(key, node):
             "recursive check for presence of key"
@@ -227,16 +234,8 @@ class ImportTestCase(unittest.TestCase):
         """Make sure that when two courses share content with the same
         org and course names, policy applies to the right one."""
 
-        def get_course(name):
-            print "Importing {0}".format(name)
-
-            modulestore = XMLModuleStore(DATA_DIR, course_dirs=[name])
-            courses = modulestore.get_courses()
-            self.assertEquals(len(courses), 1)
-            return courses[0]
-
-        toy = get_course('toy')
-        two_toys = get_course('two_toys')
+        toy = self.get_course('toy')
+        two_toys = self.get_course('two_toys')
 
         self.assertEqual(toy.url_name, "2012_Fall")
         self.assertEqual(two_toys.url_name, "TT_2012_Fall")
@@ -279,8 +278,8 @@ class ImportTestCase(unittest.TestCase):
         """Ensure that colons in url_names convert to file paths properly"""
 
         print "Starting import"
+        # Not using get_courses because we need the modulestore object too afterward
         modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy'])
-
         courses = modulestore.get_courses()
         self.assertEquals(len(courses), 1)
         course = courses[0]
@@ -317,7 +316,7 @@ class ImportTestCase(unittest.TestCase):
 
         toy_id = "edX/toy/2012_Fall"
 
-        course = modulestore.get_courses()[0]
+        course = modulestore.get_course(toy_id)
         chapters = course.get_children()
         ch1 = chapters[0]
         sections = ch1.get_children()
@@ -355,3 +354,30 @@ class ImportTestCase(unittest.TestCase):
         <slider var="a" style="width:400px;float:left;"/>\
 <plot style="margin-top:15px;margin-bottom:15px;"/>""".strip()
         self.assertEqual(gst_sample.definition['render'], render_string_from_sample_gst_xml)
+
+    def test_cohort_config(self):
+        """
+        Check that cohort config parsing works right.
+        """
+        modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy'])
+
+        toy_id = "edX/toy/2012_Fall"
+
+        course = modulestore.get_course(toy_id)
+
+        # No config -> False
+        self.assertFalse(course.is_cohorted)
+
+        # empty config -> False
+        course.metadata['cohort_config'] = {}
+        self.assertFalse(course.is_cohorted)
+
+        # false config -> False
+        course.metadata['cohort_config'] = {'cohorted': False}
+        self.assertFalse(course.is_cohorted)
+
+        # and finally...
+        course.metadata['cohort_config'] = {'cohorted': True}
+        self.assertTrue(course.is_cohorted)
+
+

common/static/js/course_groups/cohorts.js

@@ -0,0 +1,256 @@
+// structure stolen from http://briancray.com/posts/javascript-module-pattern
+
+var CohortManager = (function ($) {
+    // private variables and functions
+
+    // using jQuery
+    function getCookie(name) {
+        var cookieValue = null;
+        if (document.cookie && document.cookie != '') {
+            var cookies = document.cookie.split(';');
+            for (var i = 0; i < cookies.length; i++) {
+                var cookie = $.trim(cookies[i]);
+                // Does this cookie string begin with the name we want?
+                if (cookie.substring(0, name.length + 1) == (name + '=')) {
+                    cookieValue = decodeURIComponent(cookie.substring(name.length + 1));
+                    break;
+                }
+            }
+        }
+        return cookieValue;
+    }
+    var csrftoken = getCookie('csrftoken');
+    
+    function csrfSafeMethod(method) {
+        // these HTTP methods do not require CSRF protection
+        return (/^(GET|HEAD|OPTIONS|TRACE)$/.test(method));
+    }  
+    $.ajaxSetup({
+        crossDomain: false, // obviates need for sameOrigin test
+        beforeSend: function(xhr, settings) {
+            if (!csrfSafeMethod(settings.type)) {
+                xhr.setRequestHeader("X-CSRFToken", csrftoken);
+            }
+        }
+    });
+
+    // constructor
+    var module = function () {
+        var el = $(".cohort_manager");
+        // localized jquery
+        var $$ = function (selector) {
+            return $(selector, el)
+        }
+        var state_init = "init";
+        var state_summary = "summary";
+        var state_detail = "detail";
+        var state = state_init;
+        
+        var url = el.data('ajax_url');
+        var self = this;
+
+        // Pull out the relevant parts of the html
+        // global stuff
+        var errors = $$(".errors");
+
+        // cohort summary display
+        var summary = $$(".summary");
+        var cohorts = $$(".cohorts");
+        var show_cohorts_button = $$(".controls .show_cohorts");
+        var add_cohort_input = $$(".cohort_name");
+        var add_cohort_button = $$(".add_cohort");
+        
+        // single cohort user display
+        var detail = $$(".detail");
+        var detail_header = $(".header", detail);
+        var detail_users = $$(".users");
+        var detail_page_num = $$(".page_num");
+        var users_area = $$(".users_area");
+        var add_members_button = $$(".add_members");
+        var op_results = $$(".op_results");
+        var cohort_id = null;
+        var cohort_title = null;
+        var detail_url = null;
+        var page = null;
+
+        // *********** Summary view methods
+
+        function show_cohort(item) {
+            // item is a li that has a data-href link to the cohort base url
+            var el = $(this);
+            cohort_title = el.text();
+            detail_url = el.data('href');
+            cohort_id = el.data('id');
+            state = state_detail;
+            render();
+        }
+
+        function add_to_cohorts_list(item) {
+            var li = $('<li><a></a></li>');
+            $("a", li).text(item.name)
+                .data('href', url + '/' + item.id)
+                .addClass('link')
+                .click(show_cohort);
+            cohorts.append(li);
+        };
+
+        function log_error(msg) {
+            errors.empty();
+            errors.append($("<li />").text(msg).addClass("error"));
+        };
+        
+        function load_cohorts(response) {
+            cohorts.empty();
+            if (response && response.success) { 
+                response.cohorts.forEach(add_to_cohorts_list);
+            } else {
+                log_error(response.msg || "There was an error loading cohorts");
+            }
+            summary.show();
+        };
+
+
+        function added_cohort(response) {
+            if (response && response.success) {
+                add_to_cohorts_list(response.cohort);
+            } else {
+                log_error(response.msg || "There was an error adding a cohort");
+            }                
+        }
+
+        // *********** Detail view methods
+
+        function remove_user_from_cohort(username, cohort_id, row) {
+            var delete_url = detail_url + '/delete';
+            var data = {'username': username}
+            $.post(delete_url, data).done(function() {row.remove()})
+                .fail(function(jqXHR, status, error) {
+                    log_error('Error removing user ' + username + 
+                              ' from cohort. ' + status + ' ' + error);
+                });
+        }
+
+        function add_to_users_list(item) {
+            var tr = $('<tr><td class="name"></td><td class="username"></td>' + 
+                       '<td class="email"></td>' + 
+                       '<td class="remove"></td></tr>');
+            var current_cohort_id = cohort_id;
+
+            $(".name", tr).text(item.name);
+            $(".username", tr).text(item.username);
+            $(".email", tr).text(item.email);
+
+            $(".remove", tr).html('<a href="#">remove</a>')
+                .click(function() { 
+                    remove_user_from_cohort(item.username, current_cohort_id, tr);
+                });
+            
+            detail_users.append(tr);
+        };
+
+                
+        function show_users(response) {
+            detail_users.html("<tr><th>Name</th><th>Username</th><th>Email</th></tr>");
+            if (response && response.success) { 
+                response.users.forEach(add_to_users_list);
+                detail_page_num.text("Page " + response.page + " of " + response.num_pages);
+            } else {
+                log_error(response.msg || 
+                          "There was an error loading users for " + cohort.title);
+            }
+            detail.show();
+        }
+            
+
+        function added_users(response) {
+            function adder(note, color) {
+                return function(item) {
+                    var li = $('<li></li>')
+                    if (typeof item === "object" && item.username) {
+                        li.text(note + ' ' + item.name + ', ' + item.username + ', ' + item.email);
+                    } else if (typeof item === "object" && item.msg) {
+                        li.text(note + ' ' + item.username_or_email + ', ' + item.msg);
+                    } else {
+                        // string
+                        li.text(note + ' ' + item);
+                    }
+                    li.css('color', color);
+                    op_results.append(li);
+                }
+            }
+            op_results.empty();
+            if (response && response.success) {
+                response.added.forEach(adder("Added", "green"));
+                response.present.forEach(adder("Already present:", "black"));
+                response.conflict.forEach(adder("In another cohort:", "purple"));
+                response.unknown.forEach(adder("Unknown user:", "red"));
+                users_area.val('')
+            } else {
+                log_error(response.msg || "There was an error adding users");
+            }                
+        }
+
+        // ******* Rendering
+
+            
+        function render() {
+            // Load and render the right thing based on the state
+             
+            // start with both divs hidden
+            summary.hide();
+            detail.hide();
+            // and clear out the errors
+            errors.empty();
+            if (state == state_summary) {
+                $.ajax(url).done(load_cohorts).fail(function() {
+                    log_error("Error trying to load cohorts");
+                });
+            } else if (state == state_detail) {
+                detail_header.text("Members of " + cohort_title);
+                $.ajax(detail_url).done(show_users).fail(function() {
+                    log_error("Error trying to load users in cohort");
+                });
+            }
+        }
+
+        show_cohorts_button.click(function() { 
+            state = state_summary;
+            render();
+        });
+        
+        add_cohort_input.change(function() {
+            if (!$(this).val()) {
+                add_cohort_button.removeClass('button').addClass('button-disabled');
+            } else {
+                add_cohort_button.removeClass('button-disabled').addClass('button');
+            }
+        });
+
+        add_cohort_button.click(function() { 
+            var add_url = url + '/add';
+            data = {'name': add_cohort_input.val()}
+            $.post(add_url, data).done(added_cohort);
+        });
+
+        add_members_button.click(function() { 
+            var add_url = detail_url + '/add';
+            data = {'users': users_area.val()}
+            $.post(add_url, data).done(added_users);
+        });
+
+
+    };
+
+    // prototype
+    module.prototype = {
+        constructor: module,
+    };
+
+    // return module
+    return module;
+})(jQuery);
+
+$(window).load(function () {
+    var my_module = new CohortManager();
+})
+

doc/xml-format.md

@@ -258,6 +258,11 @@ Supported fields at the course level:
 * "discussion_blackouts" -- An array of time intervals during which you want to disable a student's ability to create or edit posts in the forum. Moderators, Community TAs, and Admins are unaffected. You might use this during exam periods, but please be aware that the forum is often a very good place to catch mistakes and clarify points to students. The better long term solution would be to have better flagging/moderation mechanisms, but this is the hammer we have today. Format by example: [["2012-10-29T04:00", "2012-11-03T04:00"], ["2012-12-30T04:00", "2013-01-02T04:00"]]
 * "show_calculator" (value "Yes" if desired)
 * "days_early_for_beta" -- number of days (floating point ok) early that students in the beta-testers group get to see course content.  Can also be specified for any other course element, and overrides values set at higher levels.
+* "cohort_config" : dictionary with keys 
+    - "cohorted" : boolean.  Set to true if this course uses student cohorts.  If so, all inline discussions are automatically cohorted, and top-level discussion topics are configurable with an optional 'cohorted': bool parameter (with default value false).  
+    - "cohorted_discussions": list of discussions that should be cohorted.  
+    - ... more to come.  ('auto_cohort', how to auto cohort, etc)
+
 * TODO: there are others
 
 ### Grading policy file contents

github-requirements.txt

@@ -3,3 +3,4 @@
 -e git://github.com/MITx/django-pipeline.git#egg=django-pipeline
 -e git://github.com/MITx/django-wiki.git@e2e84558#egg=django-wiki
 -e git://github.com/dementrock/pystache_custom.git@776973740bdaad83a3b029f96e415a7d1e8bec2f#egg=pystache_custom-dev
+-e git://github.com/MITx/dogapi.git@003a4fc9#egg=dogapi

lms/djangoapps/courseware/courses.py

@@ -83,7 +83,7 @@ def get_opt_course_with_access(user, course_id, action):
         return None
     return get_course_with_access(user, course_id, action)
 
-
+    
 def course_image_url(course):
     """Try to look up the image url for the course.  If it's not found,
     log an error and return the dead link"""

lms/djangoapps/courseware/views.py

@@ -39,6 +39,7 @@ log = logging.getLogger("mitx.courseware")
 template_imports = {'urllib': urllib}
 
 
+
 def user_groups(user):
     """
     TODO (vshnayder): This is not used. When we have a new plan for groups, adjust appropriately.

lms/djangoapps/django_comment_client/base/urls.py

@@ -24,9 +24,9 @@ urlpatterns = patterns('django_comment_client.base.views',
     url(r'comments/(?P<comment_id>[\w\-]+)/downvote$', 'vote_for_comment', {'value': 'down'}, name='downvote_comment'),
     url(r'comments/(?P<comment_id>[\w\-]+)/unvote$', 'undo_vote_for_comment', name='undo_vote_for_comment'),
     
-    url(r'(?P<commentable_id>[\w\-]+)/threads/create$', 'create_thread', name='create_thread'),
+    url(r'^(?P<commentable_id>[\w\-.]+)/threads/create$', 'create_thread', name='create_thread'),
     # TODO should we search within the board?
-    url(r'(?P<commentable_id>[\w\-]+)/threads/search_similar$', 'search_similar_threads', name='search_similar_threads'),
-    url(r'(?P<commentable_id>[\w\-]+)/follow$', 'follow_commentable', name='follow_commentable'),
-    url(r'(?P<commentable_id>[\w\-]+)/unfollow$', 'unfollow_commentable', name='unfollow_commentable'),
+    url(r'^(?P<commentable_id>[\w\-.]+)/threads/search_similar$', 'search_similar_threads', name='search_similar_threads'),
+    url(r'^(?P<commentable_id>[\w\-.]+)/follow$', 'follow_commentable', name='follow_commentable'),
+    url(r'^(?P<commentable_id>[\w\-.]+)/unfollow$', 'unfollow_commentable', name='unfollow_commentable'),
 )

lms/djangoapps/django_comment_client/base/views.py

@@ -21,12 +21,15 @@ from django.contrib.auth.models import User
 
 from mitxmako.shortcuts import render_to_response, render_to_string
 from courseware.courses import get_course_with_access
+from course_groups.cohorts import get_cohort_id, is_commentable_cohorted
 
 from django_comment_client.utils import JsonResponse, JsonError, extract, get_courseware_context
 
-from django_comment_client.permissions import check_permissions_by_view
+from django_comment_client.permissions import check_permissions_by_view, cached_has_permission
 from django_comment_client.models import Role
 
+log = logging.getLogger(__name__)
+
 def permitted(fn):
     @functools.wraps(fn)
     def wrapper(request, *args, **kwargs):
@@ -58,10 +61,12 @@ def ajax_content_response(request, course_id, content, template_name):
         'annotated_content_info': annotated_content_info,
     })
 
+
 @require_POST
 @login_required
 @permitted
 def create_thread(request, course_id, commentable_id):
+    log.debug("Creating new thread in %r, id %r", course_id, commentable_id)
     course = get_course_with_access(request.user, course_id, 'load')
     post = request.POST
 
@@ -83,6 +88,23 @@ def create_thread(request, course_id, commentable_id):
         'course_id'          : course_id,
         'user_id'            : request.user.id,
     })
+
+
+    # Cohort the thread if the commentable is cohorted.
+    if is_commentable_cohorted(course_id, commentable_id):
+        user_group_id = get_cohort_id(request.user, course_id)
+        # TODO (vshnayder): once we have more than just cohorts, we'll want to
+        # change this to a single get_group_for_user_and_commentable function
+        # that can do different things depending on the commentable_id
+        if cached_has_permission(request.user, "see_all_cohorts", course_id):
+            # admins can optionally choose what group to post as
+            group_id = post.get('group_id', user_group_id)
+        else:
+            # regular users always post with their own id.
+            group_id = user_group_id
+
+        thread.update_attributes(group_id=group_id)
+
     thread.save()
     if post.get('auto_subscribe', 'false').lower() == 'true':
         user = cc.User.from_django_user(request.user)

lms/djangoapps/django_comment_client/forum/urls.py

@@ -4,7 +4,7 @@ import django_comment_client.forum.views
 urlpatterns = patterns('django_comment_client.forum.views',
     url(r'users/(?P<user_id>\w+)/followed$', 'followed_threads', name='followed_threads'),
     url(r'users/(?P<user_id>\w+)$', 'user_profile', name='user_profile'),
-    url(r'(?P<discussion_id>[\w\-]+)/threads/(?P<thread_id>\w+)$', 'single_thread', name='single_thread'),
-    url(r'(?P<discussion_id>[\w\-]+)/inline$', 'inline_discussion', name='inline_discussion'),
+    url(r'^(?P<discussion_id>[\w\-.]+)/threads/(?P<thread_id>\w+)$', 'single_thread', name='single_thread'),
+    url(r'^(?P<discussion_id>[\w\-.]+)/inline$', 'inline_discussion', name='inline_discussion'),
     url(r'', 'forum_form_discussion', name='forum_form_discussion'),
 )

lms/djangoapps/django_comment_client/forum/views.py

@@ -11,12 +11,14 @@ from django.contrib.auth.models import User
 
 from mitxmako.shortcuts import render_to_response, render_to_string
 from courseware.courses import get_course_with_access
+from course_groups.cohorts import get_cohort_id
 from courseware.access import has_access
 
 from urllib import urlencode
 from operator import methodcaller
 from django_comment_client.permissions import check_permissions_by_view
-from django_comment_client.utils import merge_dict, extract, strip_none, strip_blank, get_courseware_context
+from django_comment_client.utils import (merge_dict, extract, strip_none,
+                                         strip_blank, get_courseware_context)
 
 import django_comment_client.utils as utils
 import comment_client as cc
@@ -33,7 +35,6 @@ def get_threads(request, course_id, discussion_id=None, per_page=THREADS_PER_PAG
     This may raise cc.utils.CommentClientError or
     cc.utils.CommentClientUnknownError if something goes wrong.
     """
-
     default_query_params = {
         'page': 1,
         'per_page': per_page,
@@ -58,8 +59,17 @@ def get_threads(request, course_id, discussion_id=None, per_page=THREADS_PER_PAG
         user.default_sort_key = request.GET.get('sort_key')
         user.save()
 
+
+    #if the course-user is cohorted, then add the group id
+    group_id = get_cohort_id(user,course_id)
+    if group_id:
+        default_query_params["group_id"] = group_id
+
     query_params = merge_dict(default_query_params,
-                              strip_none(extract(request.GET, ['page', 'sort_key', 'sort_order', 'text', 'tags', 'commentable_ids'])))
+                              strip_none(extract(request.GET,
+                                                 ['page', 'sort_key',
+                                                  'sort_order', 'text',
+                                                  'tags', 'commentable_ids'])))
 
     threads, page, num_pages = cc.Thread.search(query_params)
 
@@ -218,7 +228,7 @@ def single_thread(request, course_id, discussion_id, thread_id):
         #    course_id,
         #)
 
-        
+
         annotated_content_info = utils.get_metadata_for_threads(course_id, threads, request.user, user_info)
 
         context = {

lms/djangoapps/django_comment_client/management/commands/seed_permissions_roles.py

@@ -23,7 +23,7 @@ class Command(BaseCommand):
             student_role.add_permission(per)
 
         for per in ["edit_content", "delete_thread", "openclose_thread",
-                        "endorse_comment", "delete_comment"]:
+                        "endorse_comment", "delete_comment", "see_all_cohorts"]:
             moderator_role.add_permission(per)
 
         for per in ["manage_moderator"]:

lms/djangoapps/django_comment_client/tests.py

@@ -1,11 +1,17 @@
-from django.contrib.auth.models import User
+from django.contrib.auth.models import User, Group
+from django.core.urlresolvers import reverse
 from django.test import TestCase
-from student.models import CourseEnrollment, \
-                           replicate_enrollment_save, \
-                           replicate_enrollment_delete, \
-                           update_user_information, \
-                           replicate_user_save
-                           
+from django.test.client import RequestFactory
+from django.conf import settings
+
+from mock import Mock
+
+from override_settings import override_settings
+
+import xmodule.modulestore.django
+
+from student.models import CourseEnrollment
+
 from django.db.models.signals import m2m_changed, pre_delete, pre_save, post_delete, post_save
 from django.dispatch.dispatcher import _make_id
 import string
@@ -13,6 +19,57 @@ import random
 from .permissions import has_permission
 from .models import Role, Permission
 
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore import Location
+from xmodule.modulestore.xml_importer import import_from_xml
+from xmodule.modulestore.xml import XMLModuleStore
+
+import comment_client
+
+from courseware.tests.tests import PageLoader, TEST_DATA_XML_MODULESTORE
+
+#@override_settings(MODULESTORE=TEST_DATA_XML_MODULESTORE)
+#class TestCohorting(PageLoader):
+#    """Check that cohorting works properly"""
+#
+#    def setUp(self):
+#        xmodule.modulestore.django._MODULESTORES = {}
+#
+#        # Assume courses are there
+#        self.toy = modulestore().get_course("edX/toy/2012_Fall")
+#
+#        # Create two accounts
+#        self.student = 'view@test.com'
+#        self.student2 = 'view2@test.com'
+#        self.password = 'foo'
+#        self.create_account('u1', self.student, self.password)
+#        self.create_account('u2', self.student2, self.password)
+#        self.activate_user(self.student)
+#        self.activate_user(self.student2)
+#
+#    def test_create_thread(self):
+#        my_save = Mock()
+#        comment_client.perform_request = my_save
+#
+#        resp = self.client.post(
+#            reverse('django_comment_client.base.views.create_thread',
+#                    kwargs={'course_id': 'edX/toy/2012_Fall',
+#                            'commentable_id': 'General'}),
+#                                        {'some': "some",
+#                                         'data': 'data'})
+#        self.assertTrue(my_save.called)
+#
+#        #self.assertEqual(resp.status_code, 200)
+#        #self.assertEqual(my_save.something, "expected", "complaint if not true")
+#
+#        self.toy.metadata["cohort_config"] = {"cohorted": True}
+#
+#        # call the view again ...
+#
+#       # assert that different things happened
+
+
+
 class PermissionsTestCase(TestCase):
     def random_str(self, length=15, chars=string.ascii_uppercase + string.digits):
         return ''.join(random.choice(chars) for x in range(length))

lms/djangoapps/django_comment_client/utils.py

@@ -207,6 +207,9 @@ def initialize_discussion_info(course):
                                                       "sort_key": entry["sort_key"],
                                                       "start_date": entry["start_date"]}
 
+    # TODO.  BUG! : course location is not unique across multiple course runs!
+    # (I think Kevin already noticed this)  Need to send course_id with requests, store it
+    # in the backend.
     default_topics = {'General': {'id' :course.location.html_id()}}
     discussion_topics = course.metadata.get('discussion_topics', default_topics)
     for topic, entry in discussion_topics.items():

lms/djangoapps/instructor/views.py

@@ -6,6 +6,7 @@ import itertools
 import json
 import logging
 import os
+import re
 import requests
 import urllib
 import json
@@ -24,6 +25,7 @@ from courseware import grades
 from courseware.access import (has_access, get_access_group_name,
                                course_beta_test_group_name)
 from courseware.courses import get_course_with_access
+from courseware.models import StudentModule
 from django_comment_client.models import (Role,
                                           FORUM_ROLE_ADMINISTRATOR,
                                           FORUM_ROLE_MODERATOR,
@@ -31,7 +33,6 @@ from django_comment_client.models import (Role,
 from django_comment_client.utils import has_forum_access
 from psychometrics import psychoanalyze
 from student.models import CourseEnrollment, CourseEnrollmentAllowed
-from courseware.models import StudentModule
 from xmodule.course_module import CourseDescriptor
 from xmodule.modulestore import Location
 from xmodule.modulestore.django import modulestore
@@ -49,6 +50,9 @@ template_imports = {'urllib': urllib}
 FORUM_ROLE_ADD = 'add'
 FORUM_ROLE_REMOVE = 'remove'
 
+def split_by_comma_and_whitespace(s):
+    return re.split(r'[\s,]', s)
+
 @ensure_csrf_cookie
 @cache_control(no_cache=True, no_store=True, must_revalidate=True)
 def instructor_dashboard(request, course_id):
@@ -193,8 +197,8 @@ def instructor_dashboard(request, course_id):
         # get the form data
         unique_student_identifier=request.POST.get('unique_student_identifier','')
         problem_to_reset=request.POST.get('problem_to_reset','')
-        
-        if problem_to_reset[-4:]==".xml": 
+
+        if problem_to_reset[-4:]==".xml":
             problem_to_reset=problem_to_reset[:-4]
 
         # try to uniquely id student by email address or username
@@ -213,8 +217,8 @@ def instructor_dashboard(request, course_id):
             try:
                 (org, course_name, run)=course_id.split("/")
                 module_state_key="i4x://"+org+"/"+course_name+"/problem/"+problem_to_reset
-                module_to_reset=StudentModule.objects.get(student_id=student_to_reset.id, 
-                                                          course_id=course_id, 
+                module_to_reset=StudentModule.objects.get(student_id=student_to_reset.id,
+                                                          course_id=course_id,
                                                           module_state_key=module_state_key)
                 msg+="Found module to reset.  "
             except Exception as e:
@@ -230,14 +234,14 @@ def instructor_dashboard(request, course_id):
             # save
             module_to_reset.state=json.dumps(problem_state)
             module_to_reset.save()
-            track.views.server_track(request, 
+            track.views.server_track(request,
                                     '{instructor} reset attempts from {old_attempts} to 0 for {student} on problem {problem} in {course}'.format(
                                         old_attempts=old_number_of_attempts,
                                         student=student_to_reset,
                                         problem=module_to_reset.module_state_key,
                                         instructor=request.user,
                                         course=course_id),
-                                    {}, 
+                                    {},
                                     page='idashboard')
             msg+="<font color='green'>Module state successfully reset!</font>"
         except:
@@ -252,12 +256,12 @@ def instructor_dashboard(request, course_id):
             else:
                 student_to_reset=User.objects.get(username=unique_student_identifier)
             progress_url=reverse('student_progress',kwargs={'course_id':course_id,'student_id': student_to_reset.id})
-            track.views.server_track(request, 
+            track.views.server_track(request,
                                     '{instructor} requested progress page for {student} in {course}'.format(
                                         student=student_to_reset,
                                         instructor=request.user,
                                         course=course_id),
-                                    {}, 
+                                    {},
                                     page='idashboard')
             msg+="<a href='{0}' target='_blank'> Progress page for username: {1} with email address: {2}</a>.".format(progress_url,student_to_reset.username,student_to_reset.email)
         except:
@@ -392,14 +396,14 @@ def instructor_dashboard(request, course_id):
         users = request.POST['betausers']
         log.debug("users: {0!r}".format(users))
         group = get_beta_group(course)
-        for username_or_email in _split_by_comma_and_whitespace(users):
+        for username_or_email in split_by_comma_and_whitespace(users):
             msg += "<p>{0}</p>".format(
                 add_user_to_group(request, username_or_email, group, 'beta testers', 'beta-tester'))
 
     elif action == 'Remove beta testers':
         users = request.POST['betausers']
         group = get_beta_group(course)
-        for username_or_email in _split_by_comma_and_whitespace(users):
+        for username_or_email in split_by_comma_and_whitespace(users):
             msg += "<p>{0}</p>".format(
                 remove_user_from_group(request, username_or_email, group, 'beta testers', 'beta-tester'))
 
@@ -562,6 +566,7 @@ def instructor_dashboard(request, course_id):
                'djangopid' : os.getpid(),
                'mitx_version' : getattr(settings,'MITX_VERSION_STRING',''),
                'offline_grade_log' : offline_grades_available(course_id),
+               'cohorts_ajax_url' : reverse('cohorts', kwargs={'course_id': course_id}),
                }
 
     return render_to_response('courseware/instructor_dashboard.html', context)
@@ -870,21 +875,11 @@ def grade_summary(request, course_id):
 #-----------------------------------------------------------------------------
 # enrollment
 
-
-def _split_by_comma_and_whitespace(s):
-    """
-    Split a string both by on commas and whitespice.
-    """
-    # Note: split() with no args removes empty strings from output
-    lists = [x.split() for x in s.split(',')]
-    # return all of them
-    return itertools.chain(*lists)
-
 def _do_enroll_students(course, course_id, students, overload=False):
     """Do the actual work of enrolling multiple students, presented as a string
     of emails separated by commas or returns"""
 
-    new_students = _split_by_comma_and_whitespace(students)
+    new_students = split_by_comma_and_whitespace(students)
     new_students = [str(s.strip()) for s in new_students]
     new_students_lc = [x.lower() for x in new_students]
 

lms/envs/aws.py

@@ -37,6 +37,7 @@ with open(ENV_ROOT / "env.json") as env_file:
 
 SITE_NAME = ENV_TOKENS['SITE_NAME']
 SESSION_COOKIE_DOMAIN = ENV_TOKENS.get('SESSION_COOKIE_DOMAIN')
+CSRF_COOKIE_DOMAIN = ENV_TOKENS.get('SESSION_COOKIE_DOMAIN')
 
 BOOK_URL = ENV_TOKENS['BOOK_URL']
 MEDIA_URL = ENV_TOKENS['MEDIA_URL']
@@ -88,3 +89,9 @@ PEER_GRADING_INTERFACE = AUTH_TOKENS.get('PEER_GRADING_INTERFACE', PEER_GRADING_
 
 PEARSON_TEST_USER = "pearsontest"
 PEARSON_TEST_PASSWORD = AUTH_TOKENS.get("PEARSON_TEST_PASSWORD")
+
+# Pearson hash for import/export
+PEARSON = AUTH_TOKENS.get("PEARSON")
+
+# Datadog for events!
+DATADOG_API = AUTH_TOKENS.get("DATADOG_API")

lms/envs/common.py

@@ -577,6 +577,7 @@ INSTALLED_APPS = (
     'open_ended_grading',
     'psychometrics',
     'licenses',
+    'course_groups',
 
     #For the wiki
     'wiki', # The new django-wiki from benjaoming

lms/envs/test.py

@@ -17,7 +17,7 @@ from path import path
 MITX_FEATURES['DISABLE_START_DATES'] = True
 
 # Until we have discussion actually working in test mode, just turn it off
-MITX_FEATURES['ENABLE_DISCUSSION_SERVICE'] = False
+#MITX_FEATURES['ENABLE_DISCUSSION_SERVICE'] = True
 
 # Need wiki for courseware views to work. TODO (vshnayder): shouldn't need it.
 WIKI_ENABLED = True

lms/lib/comment_client/thread.py

@@ -10,12 +10,12 @@ class Thread(models.Model):
         'closed', 'tags', 'votes', 'commentable_id', 'username', 'user_id',
         'created_at', 'updated_at', 'comments_count', 'unread_comments_count',
         'at_position_list', 'children', 'type', 'highlighted_title',
-        'highlighted_body', 'endorsed', 'read'
+        'highlighted_body', 'endorsed', 'read', 'group_id'
     ]
 
     updatable_fields = [
         'title', 'body', 'anonymous', 'anonymous_to_peers', 'course_id',
-        'closed', 'tags', 'user_id', 'commentable_id',
+        'closed', 'tags', 'user_id', 'commentable_id', 'group_id'
     ]
 
     initializable_fields = updatable_fields

lms/static/sass/ie.scss

@@ -182,4 +182,4 @@ div.course-wrapper {
     float: left !important;
     width: 50px;
   }
-}
+}

lms/templates/course_groups/cohort_management.html

@@ -0,0 +1,41 @@
+<section class="cohort_manager" data-ajax_url="${cohorts_ajax_url}">
+<h3>Cohort groups</h3>
+
+<div class="controls" style="padding-top:15px">
+  <a href="#" class="button show_cohorts">Show cohorts</a>
+</div>
+
+<ul class="errors">
+</ul>
+
+<div class="summary" style="display:none">
+  <h3>Cohorts in the course</h3>
+  <ul class="cohorts">
+  </ul>
+
+  <p>
+  <input class="cohort_name"/>
+  <a href="#" class="button add_cohort">Add cohort</a>
+  </p>
+  
+</div>
+
+<div class="detail" style="display:none">
+  <h3 class="header"></h3>
+  <table class="users">
+  </table>
+  <span class="page_num"></span>
+  
+  <p>
+  Add users by username or email.  One per line or comma-separated.
+  </p>
+  <textarea cols="50" row="30" class="users_area" style="height: 200px"></textarea>
+  <a href="#" class="button add_members">Add cohort members</a>
+
+  <ul class="op_results">
+  </ul>
+  
+</div>
+
+
+</section>

lms/templates/course_groups/debug.html

@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <%block name="title"><title>edX</title></%block>
+
+  <script type="text/javascript" src="/static/js/vendor/jquery.min.js"></script>
+  <script type="text/javascript" src="/static/js/course_groups/cohorts.js"></script>
+
+</head>
+
+<body class="<%block name='bodyclass'/>">
+<%include file="/course_groups/cohort_management.html" />
+
+</body>
+</html>
+

lms/templates/courseware/instructor_dashboard.html

@@ -6,6 +6,8 @@
   <%static:css group='course'/>
   <script type="text/javascript" src="${static.url('js/vendor/flot/jquery.flot.js')}"></script>
   <script type="text/javascript" src="${static.url('js/vendor/flot/jquery.flot.axislabels.js')}"></script>
+  <script type="text/javascript" src="${static.url('js/course_groups/cohorts.js')}"></script>
+
 </%block>
 
 <%include file="/courseware/course_navigation.html" args="active_page='instructor'" />
@@ -282,16 +284,21 @@ function goto( mode)
     <input type="submit" name="action" value="Add beta testers">
     </p>
     <hr width="40%" style="align:left">
+
+    %if course.is_cohorted:
+    <%include file="/course_groups/cohort_management.html" />
+    %endif
+
   %endif
 %endif
 
     </form>
+##----------------------------------------------------------------------------- 
 
 %if msg:
     <p></p><p>${msg}</p>
 %endif
 ##----------------------------------------------------------------------------- 
-##----------------------------------------------------------------------------- 
 
 %if datatable and modeflag.get('Psychometrics') is None:
 

lms/urls.py

@@ -274,6 +274,26 @@ if settings.COURSEWARE_ENABLED:
             'open_ended_grading.peer_grading_service.save_grade', name='peer_grading_save_grade'),
         url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/peer_grading/save_calibration_essay$',
             'open_ended_grading.peer_grading_service.save_calibration_essay', name='peer_grading_save_calibration_essay'),
+
+        # Cohorts management
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/cohorts$',
+            'course_groups.views.list_cohorts', name="cohorts"),
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/cohorts/add$',
+            'course_groups.views.add_cohort',
+            name="add_cohort"),
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/cohorts/(?P<cohort_id>[0-9]+)$',
+            'course_groups.views.users_in_cohort',
+            name="list_cohort"),
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/cohorts/(?P<cohort_id>[0-9]+)/add$',
+            'course_groups.views.add_users_to_cohort',
+            name="add_to_cohort"),
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/cohorts/(?P<cohort_id>[0-9]+)/delete$',
+            'course_groups.views.remove_user_from_cohort',
+            name="remove_from_cohort"),
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/cohorts/debug$',
+            'course_groups.views.debug_cohort_mgmt',
+            name="debug_cohort_mgmt"),
+
     )
 
     # discussion forums live within courseware, so courseware must be enabled first

requirements.txt

@@ -58,4 +58,4 @@ factory_boy
 Shapely==1.2.16
 ipython==0.13.1
 xmltodict==0.4.1
-
+paramiko==1.9.0