Support cohorting students via a CSV File.

TNL-735

common/djangoapps/student/management/commands/create_random_users.py

@@ -1,12 +1,27 @@
 """
 A script to create some dummy users
 """
+import uuid
+
 from django.core.management.base import BaseCommand
 from student.models import CourseEnrollment
 from opaque_keys import InvalidKeyError
 from opaque_keys.edx.keys import CourseKey
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
-from student.views import _do_create_account, get_random_post_override
+from student.views import _do_create_account
+
+
+def get_random_post_override():
+    """
+    Generate unique user data for dummy users.
+    """
+    identification = uuid.uuid4().hex[:8]
+    return {
+        'username': 'user_{id}'.format(id=identification),
+        'email': 'email_{id}@example.com'.format(id=identification),
+        'password': '12345',
+        'name': 'User {id}'.format(id=identification),
+    }
 
 
 def create(num, course_key):

common/djangoapps/util/file.py

@@ -0,0 +1,168 @@
+"""
+Utility methods related to file handling.
+"""
+
+from datetime import datetime
+import os
+from pytz import UTC
+
+from django.core.exceptions import PermissionDenied
+from django.core.files.storage import DefaultStorage, get_valid_filename
+from django.utils.translation import ugettext as _
+from django.utils.translation import ungettext
+
+
+class FileValidationException(Exception):
+    """
+    An exception thrown during file validation.
+    """
+    pass
+
+
+def store_uploaded_file(
+        request, file_key, allowed_file_types, base_storage_filename, max_file_size, validator=None,
+):
+    """
+    Stores an uploaded file to django file storage.
+
+    Args:
+        request (HttpRequest): A request object from which a file will be retrieved.
+        file_key (str): The key for retrieving the file from `request.FILES`. If no entry exists with this
+            key, a `ValueError` will be thrown.
+        allowed_file_types (list): a list of allowable file type extensions. These should start with a period
+            and be specified in lower-case. For example, ['.txt', '.csv']. If the uploaded file does not end
+            with one of these extensions, a `PermissionDenied` exception will be thrown. Note that the uploaded file
+            extension does not need to be lower-case.
+        base_storage_filename (str): the filename to be used for the stored file, not including the extension.
+            The same extension as the uploaded file will be appended to this value.
+        max_file_size (int): the maximum file size in bytes that the uploaded file can be. If the uploaded file
+            is larger than this size, a `PermissionDenied` exception will be thrown.
+        validator (function): an optional validation method that, if defined, will be passed the stored file (which
+            is copied from the uploaded file). This method can do validation on the contents of the file and throw
+            a `FileValidationException` if the file is not properly formatted. If any exception is thrown, the stored
+            file will be deleted before the exception is re-raised. Note that the implementor of the validator function
+            should take care to close the stored file if they open it for reading.
+
+    Returns:
+        Storage: the file storage object where the file can be retrieved from
+        str: stored_file_name: the name of the stored file (including extension)
+
+    """
+
+    if file_key not in request.FILES:
+        raise ValueError("No file uploaded with key '" + file_key + "'.")
+
+    uploaded_file = request.FILES[file_key]
+    try:
+        file_extension = os.path.splitext(uploaded_file.name)[1].lower()
+        if not file_extension in allowed_file_types:
+            file_types = "', '".join(allowed_file_types)
+            msg = ungettext(
+                "The file must end with the extension '{file_types}'.",
+                "The file must end with one of the following extensions: '{file_types}'.",
+                len(allowed_file_types)).format(file_types=file_types)
+            raise PermissionDenied(msg)
+
+        if uploaded_file.size > max_file_size:
+            msg = _("Maximum upload file size is {file_size} bytes.").format(file_size=max_file_size)
+            raise PermissionDenied(msg)
+
+        stored_file_name = base_storage_filename + file_extension
+
+        file_storage = DefaultStorage()
+        file_storage.save(stored_file_name, uploaded_file)
+
+        if validator:
+            try:
+                validator(file_storage, stored_file_name)
+            except:
+                file_storage.delete(stored_file_name)
+                raise
+
+    finally:
+        uploaded_file.close()
+
+    return file_storage, stored_file_name
+
+
+# pylint: disable=invalid-name
+def course_filename_prefix_generator(course_id, separator='_'):
+    """
+    Generates a course-identifying unicode string for use in a file
+    name.
+
+    Args:
+        course_id (object): A course identification object.
+    Returns:
+        str: A unicode string which can safely be inserted into a
+            filename.
+    """
+    return get_valid_filename(unicode(separator).join([course_id.org, course_id.course, course_id.run]))
+
+
+# pylint: disable=invalid-name
+def course_and_time_based_filename_generator(course_id, base_name):
+    """
+    Generates a filename (without extension) based on the current time and the supplied filename.
+
+    Args:
+        course_id (object): A course identification object (must have org, course, and run).
+        base_name (str): A name describing what type of file this is. Any characters that are not safe for
+            filenames will be converted per django.core.files.storage.get_valid_filename (Specifically,
+            leading and trailing spaces are removed; other  spaces are converted to underscores; and anything
+            that is not a unicode alphanumeric, dash, underscore, or dot, is removed).
+
+    Returns:
+        str: a concatenation of the org, course and run from the input course_id, the input base_name,
+            and the current time. Note that there will be no extension.
+
+    """
+    return u"{course_prefix}_{base_name}_{timestamp_str}".format(
+        course_prefix=course_filename_prefix_generator(course_id),
+        base_name=get_valid_filename(base_name),
+        timestamp_str=datetime.now(UTC).strftime("%Y-%m-%d-%H%M%S")  # pylint: disable=maybe-no-member
+    )
+
+
+class UniversalNewlineIterator(object):
+    """
+    This iterable class can be used as a wrapper around a file-like
+    object which does not inherently support being read in
+    universal-newline mode.  It returns a line at a time.
+    """
+    def __init__(self, original_file, buffer_size=4096):
+        self.original_file = original_file
+        self.buffer_size = buffer_size
+
+    def __iter__(self):
+        return self.generate_lines()
+
+    @staticmethod
+    def sanitize(string):
+        """
+        Replace CR and CRLF with LF within `string`.
+        """
+        return string.replace('\r\n', '\n').replace('\r', '\n')
+
+    def generate_lines(self):
+        """
+        Return data from `self.original_file` a line at a time,
+        replacing CR and CRLF with LF.
+        """
+        buf = self.original_file.read(self.buffer_size)
+        line = ''
+        while buf:
+            for char in buf:
+                if line.endswith('\r') and char == '\n':
+                    last_line = line
+                    line = ''
+                    yield self.sanitize(last_line)
+                elif line.endswith('\r') or line.endswith('\n'):
+                    last_line = line
+                    line = char
+                    yield self.sanitize(last_line)
+                else:
+                    line += char
+            buf = self.original_file.read(self.buffer_size)
+            if not buf and line:
+                yield self.sanitize(line)

common/djangoapps/util/tests/test_file.py

@@ -0,0 +1,257 @@
+# -*- coding: utf-8 -*-
+"""
+Tests for file.py
+"""
+import ddt
+from io import StringIO
+
+from django.test import TestCase
+from datetime import datetime
+from django.utils.timezone import UTC
+from mock import patch, Mock
+from django.http import HttpRequest
+from django.core.files.uploadedfile import SimpleUploadedFile
+import util.file
+from util.file import (
+    course_and_time_based_filename_generator,
+    course_filename_prefix_generator,
+    store_uploaded_file,
+    FileValidationException,
+    UniversalNewlineIterator
+)
+from opaque_keys.edx.locations import CourseLocator, SlashSeparatedCourseKey
+from django.core import exceptions
+import os
+
+
+@ddt.ddt
+class FilenamePrefixGeneratorTestCase(TestCase):
+    """
+    Tests for course_filename_prefix_generator
+    """
+    @ddt.data(CourseLocator, SlashSeparatedCourseKey)
+    def test_locators(self, course_key_class):
+        self.assertEqual(
+            course_filename_prefix_generator(course_key_class(org='foo', course='bar', run='baz')),
+            u'foo_bar_baz'
+        )
+
+    @ddt.data(CourseLocator, SlashSeparatedCourseKey)
+    def test_custom_separator(self, course_key_class):
+        self.assertEqual(
+            course_filename_prefix_generator(course_key_class(org='foo', course='bar', run='baz'), separator='-'),
+            u'foo-bar-baz'
+        )
+
+
+@ddt.ddt
+class FilenameGeneratorTestCase(TestCase):
+    """
+    Tests for course_and_time_based_filename_generator
+    """
+    NOW = datetime.strptime('1974-06-22T01:02:03', '%Y-%m-%dT%H:%M:%S').replace(tzinfo=UTC())
+
+    def setUp(self):
+        datetime_patcher = patch.object(
+            util.file, 'datetime',
+            Mock(wraps=datetime)
+        )
+        mocked_datetime = datetime_patcher.start()
+        mocked_datetime.now.return_value = self.NOW
+        self.addCleanup(datetime_patcher.stop)
+
+    @ddt.data(CourseLocator, SlashSeparatedCourseKey)
+    def test_filename_generator(self, course_key_class):
+        """
+        Tests that the generator creates names based on course_id, base name, and date.
+        """
+        self.assertEqual(
+            u'foo_bar_baz_file_1974-06-22-010203',
+            course_and_time_based_filename_generator(course_key_class(org='foo', course='bar', run='baz'), 'file')
+        )
+
+        self.assertEqual(
+            u'foo_bar_baz_base_name_ø_1974-06-22-010203',
+            course_and_time_based_filename_generator(
+                course_key_class(org='foo', course='bar', run='baz'), ' base` name ø '
+            )
+        )
+
+
+class StoreUploadedFileTestCase(TestCase):
+    """
+    Tests for store_uploaded_file.
+    """
+
+    def setUp(self):
+        self.request = Mock(spec=HttpRequest)
+        self.file_content = "test file content"
+        self.request.FILES = {"uploaded_file": SimpleUploadedFile("tempfile.csv", self.file_content)}
+        self.stored_file_name = None
+        self.file_storage = None
+        self.default_max_size = 2000000
+
+    def tearDown(self):
+        if self.file_storage and self.stored_file_name:
+            self.file_storage.delete(self.stored_file_name)
+
+    def verify_exception(self, expected_message, error):
+        """
+        Helper method to verify exception text.
+        """
+        self.assertEqual(expected_message, error.exception.message)
+
+    def test_error_conditions(self):
+        """
+        Verifies that exceptions are thrown in the expected cases.
+        """
+        with self.assertRaises(ValueError) as error:
+            store_uploaded_file(self.request, "wrong_key", [".txt", ".csv"], "stored_file", self.default_max_size)
+        self.verify_exception("No file uploaded with key 'wrong_key'.", error)
+
+        with self.assertRaises(exceptions.PermissionDenied) as error:
+            store_uploaded_file(self.request, "uploaded_file", [], "stored_file", self.default_max_size)
+        self.verify_exception("The file must end with one of the following extensions: ''.", error)
+
+        with self.assertRaises(exceptions.PermissionDenied) as error:
+            store_uploaded_file(self.request, "uploaded_file", [".bar"], "stored_file", self.default_max_size)
+        self.verify_exception("The file must end with the extension '.bar'.", error)
+
+        with self.assertRaises(exceptions.PermissionDenied) as error:
+            store_uploaded_file(self.request, "uploaded_file", [".xxx", ".bar"], "stored_file", self.default_max_size)
+        self.verify_exception("The file must end with one of the following extensions: '.xxx', '.bar'.", error)
+
+        with self.assertRaises(exceptions.PermissionDenied) as error:
+            store_uploaded_file(self.request, "uploaded_file", [".csv"], "stored_file", 2)
+        self.verify_exception("Maximum upload file size is 2 bytes.", error)
+
+    def test_validator(self):
+        """
+        Verify that a validator function can throw an exception.
+        """
+        validator_data = {}
+
+        def verify_file_presence(should_exist):
+            """ Verify whether or not the stored file, passed to the validator, exists. """
+            self.assertEqual(should_exist, validator_data["storage"].exists(validator_data["filename"]))
+
+        def store_file_data(storage, filename):
+            """ Stores file validator data for testing after validation is complete. """
+            validator_data["storage"] = storage
+            validator_data["filename"] = filename
+            verify_file_presence(True)
+
+        def exception_validator(storage, filename):
+            """ Validation test function that throws an exception """
+            self.assertEqual("error_file.csv", os.path.basename(filename))
+            with storage.open(filename, 'rU') as f:
+                self.assertEqual(self.file_content, f.read())
+            store_file_data(storage, filename)
+            raise FileValidationException("validation failed")
+
+        def success_validator(storage, filename):
+            """ Validation test function that is a no-op """
+            self.assertEqual("success_file.csv", os.path.basename(filename))
+            store_file_data(storage, filename)
+
+        with self.assertRaises(FileValidationException) as error:
+            store_uploaded_file(
+                self.request, "uploaded_file", [".csv"], "error_file",
+                self.default_max_size, validator=exception_validator
+            )
+        self.verify_exception("validation failed", error)
+        # Verify the file was deleted.
+        verify_file_presence(False)
+
+        store_uploaded_file(
+            self.request, "uploaded_file", [".csv"], "success_file", self.default_max_size, validator=success_validator
+        )
+        # Verify the file still exists
+        verify_file_presence(True)
+
+    def test_file_upload_lower_case_extension(self):
+        """
+        Tests uploading a file with lower case extension. Verifies that the stored file contents are correct.
+        """
+        self.file_storage, self.stored_file_name = store_uploaded_file(
+            self.request, "uploaded_file", [".csv"], "stored_file", self.default_max_size
+        )
+        self._verify_successful_upload()
+
+    def test_file_upload_upper_case_extension(self):
+        """
+        Tests uploading a file with upper case extension. Verifies that the stored file contents are correct.
+        """
+        self.request.FILES = {"uploaded_file": SimpleUploadedFile("tempfile.CSV", self.file_content)}
+        self.file_storage, self.stored_file_name = store_uploaded_file(
+            self.request, "uploaded_file", [".gif", ".csv"], "second_stored_file", self.default_max_size
+        )
+        self._verify_successful_upload()
+
+    def _verify_successful_upload(self):
+        """ Helper method that checks that the stored version of the uploaded file has the correct content """
+        self.assertTrue(self.file_storage.exists(self.stored_file_name))
+        with self.file_storage.open(self.stored_file_name, 'r') as f:
+            self.assertEqual(self.file_content, f.read())
+
+
+@ddt.ddt
+class TestUniversalNewlineIterator(TestCase):
+    """
+    Tests for the UniversalNewlineIterator class.
+    """
+    @ddt.data(1, 2, 999)
+    def test_line_feeds(self, buffer_size):
+        self.assertEqual(
+            [thing for thing in UniversalNewlineIterator(StringIO(u'foo\nbar\n'), buffer_size=buffer_size)],
+            ['foo\n', 'bar\n']
+        )
+
+    @ddt.data(1, 2, 999)
+    def test_carriage_returns(self, buffer_size):
+        self.assertEqual(
+            [thing for thing in UniversalNewlineIterator(StringIO(u'foo\rbar\r'), buffer_size=buffer_size)],
+            ['foo\n', 'bar\n']
+        )
+
+    @ddt.data(1, 2, 999)
+    def test_carriage_returns_and_line_feeds(self, buffer_size):
+        self.assertEqual(
+            [thing for thing in UniversalNewlineIterator(StringIO(u'foo\r\nbar\r\n'), buffer_size=buffer_size)],
+            ['foo\n', 'bar\n']
+        )
+
+    @ddt.data(1, 2, 999)
+    def test_no_trailing_newline(self, buffer_size):
+        self.assertEqual(
+            [thing for thing in UniversalNewlineIterator(StringIO(u'foo\nbar'), buffer_size=buffer_size)],
+            ['foo\n', 'bar']
+        )
+
+    @ddt.data(1, 2, 999)
+    def test_only_one_line(self, buffer_size):
+        self.assertEqual(
+            [thing for thing in UniversalNewlineIterator(StringIO(u'foo\n'), buffer_size=buffer_size)],
+            ['foo\n']
+        )
+
+    @ddt.data(1, 2, 999)
+    def test_only_one_line_no_trailing_newline(self, buffer_size):
+        self.assertEqual(
+            [thing for thing in UniversalNewlineIterator(StringIO(u'foo'), buffer_size=buffer_size)],
+            ['foo']
+        )
+
+    @ddt.data(1, 2, 999)
+    def test_empty_file(self, buffer_size):
+        self.assertEqual(
+            [thing for thing in UniversalNewlineIterator(StringIO(u''), buffer_size=buffer_size)],
+            []
+        )
+
+    @ddt.data(1, 2, 999)
+    def test_unicode_data(self, buffer_size):
+        self.assertEqual(
+            [thing for thing in UniversalNewlineIterator(StringIO(u'héllø wo®ld'), buffer_size=buffer_size)],
+            [u'héllø wo®ld']
+        )

common/static/js/spec_helpers/ajax_helpers.js

@@ -77,13 +77,20 @@ define(['sinon', 'underscore'], function(sinon, _) {
             JSON.stringify(jsonResponse));
     };
 
-    respondWithError = function(requests, requestIndex) {
+    respondWithError = function(requests, statusCode, jsonResponse, requestIndex) {
         if (_.isUndefined(requestIndex)) {
             requestIndex = requests.length - 1;
         }
-        requests[requestIndex].respond(500,
+        if (_.isUndefined(statusCode)) {
+            statusCode = 500;
+        }
+        if (_.isUndefined(jsonResponse)) {
+            jsonResponse = {};
+        }
+        requests[requestIndex].respond(statusCode,
             { 'Content-Type': 'application/json' },
-            JSON.stringify({ }));
+            JSON.stringify(jsonResponse)
+        );
     };
 
     respondToDelete = function(requests, requestIndex) {

common/test/acceptance/pages/lms/instructor_dashboard.py

@@ -26,6 +26,37 @@ class InstructorDashboardPage(CoursePage):
         membership_section.wait_for_page()
         return membership_section
 
+    def select_data_download(self):
+        """
+        Selects the data download tab and returns a DataDownloadPage.
+        """
+        self.q(css='a[data-section=data_download]').first.click()
+        data_download_section = DataDownloadPage(self.browser)
+        data_download_section.wait_for_page()
+        return data_download_section
+
+    @staticmethod
+    def get_asset_path(file_name):
+        """
+        Returns the full path of the file to upload.
+        These files have been placed in edx-platform/common/test/data/uploads/
+        """
+
+        # Separate the list of folders in the path reaching to the current file,
+        # e.g.  '... common/test/acceptance/pages/lms/instructor_dashboard.py' will result in
+        #       [..., 'common', 'test', 'acceptance', 'pages', 'lms', 'instructor_dashboard.py']
+        folders_list_in_path = __file__.split(os.sep)
+
+        # Get rid of the last 4 elements: 'acceptance', 'pages', 'lms', and 'instructor_dashboard.py'
+        # to point to the 'test' folder, a shared point in the path's tree.
+        folders_list_in_path = folders_list_in_path[:-4]
+
+        # Append the folders in the asset's path
+        folders_list_in_path.extend(['data', 'uploads', file_name])
+
+        # Return the joined path of the required asset.
+        return os.sep.join(folders_list_in_path)
+
 
 class MembershipPage(PageObject):
     """
@@ -38,15 +69,39 @@ class MembershipPage(PageObject):
 
     def select_auto_enroll_section(self):
         """
-        returns the MembershipPageAutoEnrollSection
+        Returns the MembershipPageAutoEnrollSection page object.
         """
         return MembershipPageAutoEnrollSection(self.browser)
 
+    def select_cohort_management_section(self):
+        """
+        Returns the MembershipPageCohortManagementSection page object.
+        """
+        return MembershipPageCohortManagementSection(self.browser)
+
+
+class MembershipPageCohortManagementSection(PageObject):
+    """
+    The cohort management subsection of the Membership section of the Instructor dashboard.
+    """
+    url = None
+    csv_browse_button_selector = '.csv-upload #file-upload-form-file'
+    csv_upload_button_selector = '.csv-upload #file-upload-form-submit'
+
+    def is_browser_on_page(self):
+        return self.q(css='.cohort-management.membership-section').present
+
+    def _bounded_selector(self, selector):
+        """
+        Return `selector`, but limited to the cohort management context.
+        """
+        return '.cohort-management.membership-section {}'.format(selector)
+
     def _get_cohort_options(self):
         """
         Returns the available options in the cohort dropdown, including the initial "Select a cohort group".
         """
-        return self.q(css=".cohort-management #cohort-select option")
+        return self.q(css=self._bounded_selector("#cohort-select option"))
 
     def _cohort_name(self, label):
         """
@@ -89,7 +144,7 @@ class MembershipPage(PageObject):
         """
         Selects the given cohort in the drop-down.
         """
-        self.q(css=".cohort-management #cohort-select option").filter(
+        self.q(css=self._bounded_selector("#cohort-select option")).filter(
             lambda el: self._cohort_name(el.text) == cohort_name
         ).first.click()
 
@@ -97,45 +152,64 @@ class MembershipPage(PageObject):
         """
         Adds a new manual cohort with the specified name.
         """
-        self.q(css="div.cohort-management-nav .action-create").first.click()
-        textinput = self.q(css="#cohort-create-name").results[0]
+        self.q(css=self._bounded_selector("div.cohort-management-nav .action-create")).first.click()
+        textinput = self.q(css=self._bounded_selector("#cohort-create-name")).results[0]
         textinput.send_keys(cohort_name)
-        self.q(css="div.form-actions .action-save").first.click()
+        self.q(css=self._bounded_selector("div.form-actions .action-save")).first.click()
 
     def get_cohort_group_setup(self):
         """
         Returns the description of the current cohort
         """
-        return self.q(css='.cohort-management-group-setup .setup-value').first.text[0]
+        return self.q(css=self._bounded_selector('.cohort-management-group-setup .setup-value')).first.text[0]
 
     def select_edit_settings(self):
-        self.q(css=".action-edit").first.click()
+        self.q(css=self._bounded_selector(".action-edit")).first.click()
 
     def add_students_to_selected_cohort(self, users):
         """
         Adds a list of users (either usernames or email addresses) to the currently selected cohort.
         """
-        textinput = self.q(css="#cohort-management-group-add-students").results[0]
+        textinput = self.q(css=self._bounded_selector("#cohort-management-group-add-students")).results[0]
         for user in users:
             textinput.send_keys(user)
             textinput.send_keys(",")
-        self.q(css="div.cohort-management-group-add .action-primary").first.click()
+        self.q(css=self._bounded_selector("div.cohort-management-group-add .action-primary")).first.click()
 
     def get_cohort_student_input_field_value(self):
         """
         Returns the contents of the input field where students can be added to a cohort.
         """
-        return self.q(css="#cohort-management-group-add-students").results[0].get_attribute("value")
+        return self.q(
+            css=self._bounded_selector("#cohort-management-group-add-students")
+        ).results[0].get_attribute("value")
 
     def _get_cohort_messages(self, type):
         """
-        Returns array of messages for given type.
+        Returns array of messages related to manipulating cohorts directly through the UI for the given type.
         """
-        message_title = self.q(css="div.cohort-management-group-add .cohort-" + type + " .message-title")
+        title_css = "div.cohort-management-group-add .cohort-" + type + " .message-title"
+        detail_css = "div.cohort-management-group-add .cohort-" + type + " .summary-item"
+
+        return self._get_messages(title_css, detail_css)
+
+    def get_csv_messages(self):
+        """
+        Returns array of messages related to a CSV upload of cohort assignments.
+        """
+        title_css = ".csv-upload .message-title"
+        detail_css = ".csv-upload .summary-item"
+        return self._get_messages(title_css, detail_css)
+
+    def _get_messages(self, title_css, details_css):
+        """
+        Helper method to get messages given title and details CSS.
+        """
+        message_title = self.q(css=self._bounded_selector(title_css))
         if len(message_title.results) == 0:
             return []
         messages = [message_title.first.text[0]]
-        details = self.q(css="div.cohort-management-group-add .cohort-" + type + " .summary-item").results
+        details = self.q(css=self._bounded_selector(details_css)).results
         for detail in details:
             messages.append(detail.text)
         return messages
@@ -158,7 +232,20 @@ class MembershipPage(PageObject):
         """
         Click on the link to the Data Download Page.
         """
-        self.q(css="a.link-cross-reference[data-section=data_download]").first.click()
+        self.q(css=self._bounded_selector("a.link-cross-reference[data-section=data_download]")).first.click()
+
+    def upload_cohort_file(self, filename):
+        """
+        Uploads a file with cohort assignment information.
+        """
+        # If the CSV upload section has not yet been toggled on, click on the toggle link.
+        cvs_upload_toggle = self.q(css=self._bounded_selector(".toggle-cohort-management-secondary")).first
+        if cvs_upload_toggle:
+            cvs_upload_toggle.click()
+        path = InstructorDashboardPage.get_asset_path(filename)
+        file_input = self.q(css=self._bounded_selector(self.csv_browse_button_selector)).results[0]
+        file_input.send_keys(path)
+        self.q(css=self._bounded_selector(self.csv_upload_button_selector)).first.click()
 
 
 class MembershipPageAutoEnrollSection(PageObject):
@@ -215,49 +302,30 @@ class MembershipPageAutoEnrollSection(PageObject):
         self.wait_for_element_presence(error_message_selector, "%s message" % section_type.title())
         return self.q(css=error_message_selector).text[0]
 
-    def get_asset_path(self, file_name):
-        """
-        Returns the full path of the file to upload.
-        These files have been placed in edx-platform/common/test/data/uploads/
-        """
-
-        # Separate the list of folders in the path reaching to the current file,
-        # e.g.  '... common/test/acceptance/pages/lms/instructor_dashboard.py' will result in
-        #       [..., 'common', 'test', 'acceptance', 'pages', 'lms', 'instructor_dashboard.py']
-        folders_list_in_path = __file__.split(os.sep)
-
-        # Get rid of the last 4 elements: 'acceptance', 'pages', 'lms', and 'instructor_dashboard.py'
-        # to point to the 'test' folder, a shared point in the path's tree.
-        folders_list_in_path = folders_list_in_path[:-4]
-
-        # Append the folders in the asset's path
-        folders_list_in_path.extend(['data', 'uploads', file_name])
-
-        # Return the joined path of the required asset.
-        return os.sep.join(folders_list_in_path)
-
     def upload_correct_csv_file(self):
         """
         Selects the correct file and clicks the upload button.
         """
-        correct_files_path = self.get_asset_path('auto_reg_enrollment.csv')
-        self.q(css=self.auto_enroll_browse_button_selector).results[0].send_keys(correct_files_path)
-        self.click_upload_file_button()
+        self._upload_file('auto_reg_enrollment.csv')
 
     def upload_csv_file_with_errors_warnings(self):
         """
         Selects the file which will generate errors and warnings and clicks the upload button.
         """
-        errors_warnings_files_path = self.get_asset_path('auto_reg_enrollment_errors_warnings.csv')
-        self.q(css=self.auto_enroll_browse_button_selector).results[0].send_keys(errors_warnings_files_path)
-        self.click_upload_file_button()
+        self._upload_file('auto_reg_enrollment_errors_warnings.csv')
 
     def upload_non_csv_file(self):
         """
         Selects an image file and clicks the upload button.
         """
-        errors_warnings_files_path = self.get_asset_path('image.jpg')
-        self.q(css=self.auto_enroll_browse_button_selector).results[0].send_keys(errors_warnings_files_path)
+        self._upload_file('image.jpg')
+
+    def _upload_file(self, filename):
+        """
+        Helper method to upload a file with registration and enrollment information.
+        """
+        file_path = InstructorDashboardPage.get_asset_path(filename)
+        self.q(css=self.auto_enroll_browse_button_selector).results[0].send_keys(file_path)
         self.click_upload_file_button()
 
 
@@ -269,3 +337,10 @@ class DataDownloadPage(PageObject):
 
     def is_browser_on_page(self):
         return self.q(css='a[data-section=data_download].active-section').present
+
+    def get_available_reports_for_download(self):
+        """
+        Returns a list of all the available reports for download.
+        """
+        reports = self.q(css="#report-downloads-table .file-download-link>a").map(lambda el: el.text)
+        return reports.results

common/test/acceptance/tests/discussion/test_cohort_management.py

@@ -7,6 +7,7 @@ from datetime import datetime
 
 from pymongo import MongoClient
 
+from pytz import UTC, utc
 from bok_choy.promise import EmptyPromise
 from .helpers import CohortTestMixin
 from ..helpers import UniqueCourseTest
@@ -41,28 +42,37 @@ class CohortConfigurationTest(UniqueCourseTest, CohortTestMixin):
         # create a non-instructor who will be registered for the course and in the manual cohort.
         self.student_name = "student_user"
         self.student_id = AutoAuthPage(
-            self.browser, username=self.student_name, course_id=self.course_id, staff=False
+            self.browser, username=self.student_name, email="student_user@example.com",
+            course_id=self.course_id, staff=False
         ).visit().get_user_id()
         self.add_user_to_cohort(self.course_fixture, self.student_name, self.manual_cohort_id)
 
+        # create a user with unicode characters in their username
+        self.unicode_student_id = AutoAuthPage(
+            self.browser, username="Ωπ", email="unicode_student_user@example.com",
+            course_id=self.course_id, staff=False
+        ).visit().get_user_id()
+
         # login as an instructor
         self.instructor_name = "instructor_user"
         self.instructor_id = AutoAuthPage(
-            self.browser, username=self.instructor_name, course_id=self.course_id, staff=True
+            self.browser, username=self.instructor_name, email="instructor_user@example.com",
+            course_id=self.course_id, staff=True
         ).visit().get_user_id()
 
         # go to the membership page on the instructor dashboard
-        instructor_dashboard_page = InstructorDashboardPage(self.browser, self.course_id)
-        instructor_dashboard_page.visit()
-        self.membership_page = instructor_dashboard_page.select_membership()
+        self.instructor_dashboard_page = InstructorDashboardPage(self.browser, self.course_id)
+        self.instructor_dashboard_page.visit()
+        membership_page = self.instructor_dashboard_page.select_membership()
+        self.cohort_management_page = membership_page.select_cohort_management_section()
 
     def verify_cohort_description(self, cohort_name, expected_description):
         """
         Selects the cohort with the given name and verifies the expected description is presented.
         """
-        self.membership_page.select_cohort(cohort_name)
-        self.assertEquals(self.membership_page.get_selected_cohort(), cohort_name)
-        self.assertIn(expected_description, self.membership_page.get_cohort_group_setup())
+        self.cohort_management_page.select_cohort(cohort_name)
+        self.assertEquals(self.cohort_management_page.get_selected_cohort(), cohort_name)
+        self.assertIn(expected_description, self.cohort_management_page.get_cohort_group_setup())
 
     def test_cohort_description(self):
         """
@@ -93,8 +103,8 @@ class CohortConfigurationTest(UniqueCourseTest, CohortTestMixin):
         When I view the cohort in the LMS instructor dashboard
         There is a link to take me to the Studio Advanced Settings for the course
         """
-        self.membership_page.select_cohort(self.manual_cohort_name)
-        self.membership_page.select_edit_settings()
+        self.cohort_management_page.select_cohort(self.manual_cohort_name)
+        self.cohort_management_page.select_edit_settings()
         advanced_settings_page = AdvancedSettingsPage(
             self.browser, self.course_info['org'], self.course_info['number'], self.course_info['run']
         )
@@ -114,19 +124,19 @@ class CohortConfigurationTest(UniqueCourseTest, CohortTestMixin):
         And the user input field is empty
         And appropriate events have been emitted
         """
-        start_time = datetime.now()
-        self.membership_page.select_cohort(self.auto_cohort_name)
-        self.assertEqual(0, self.membership_page.get_selected_cohort_count())
-        self.membership_page.add_students_to_selected_cohort([self.student_name, self.instructor_name])
+        start_time = datetime.now(UTC)
+        self.cohort_management_page.select_cohort(self.auto_cohort_name)
+        self.assertEqual(0, self.cohort_management_page.get_selected_cohort_count())
+        self.cohort_management_page.add_students_to_selected_cohort([self.student_name, self.instructor_name])
         # Wait for the number of users in the cohort to change, indicating that the add operation is complete.
         EmptyPromise(
-            lambda: 2 == self.membership_page.get_selected_cohort_count(), 'Waiting for added students'
+            lambda: 2 == self.cohort_management_page.get_selected_cohort_count(), 'Waiting for added students'
         ).fulfill()
-        confirmation_messages = self.membership_page.get_cohort_confirmation_messages()
+        confirmation_messages = self.cohort_management_page.get_cohort_confirmation_messages()
         self.assertEqual(2, len(confirmation_messages))
         self.assertEqual("2 students have been added to this cohort group", confirmation_messages[0])
         self.assertEqual("1 student was removed from " + self.manual_cohort_name, confirmation_messages[1])
-        self.assertEqual("", self.membership_page.get_cohort_student_input_field_value())
+        self.assertEqual("", self.cohort_management_page.get_cohort_student_input_field_value())
         self.assertEqual(
             self.event_collection.find({
                 "name": "edx.cohort.user_added",
@@ -178,27 +188,27 @@ class CohortConfigurationTest(UniqueCourseTest, CohortTestMixin):
         And I get a notification that one user is unknown
         And the user input field still contains the incorrect email addresses
         """
-        self.membership_page.select_cohort(self.manual_cohort_name)
-        self.assertEqual(1, self.membership_page.get_selected_cohort_count())
-        self.membership_page.add_students_to_selected_cohort([self.student_name, "unknown_user"])
+        self.cohort_management_page.select_cohort(self.manual_cohort_name)
+        self.assertEqual(1, self.cohort_management_page.get_selected_cohort_count())
+        self.cohort_management_page.add_students_to_selected_cohort([self.student_name, "unknown_user"])
         # Wait for notification messages to appear, indicating that the add operation is complete.
         EmptyPromise(
-            lambda: 2 == len(self.membership_page.get_cohort_confirmation_messages()), 'Waiting for notification'
+            lambda: 2 == len(self.cohort_management_page.get_cohort_confirmation_messages()), 'Waiting for notification'
         ).fulfill()
-        self.assertEqual(1, self.membership_page.get_selected_cohort_count())
+        self.assertEqual(1, self.cohort_management_page.get_selected_cohort_count())
 
-        confirmation_messages = self.membership_page.get_cohort_confirmation_messages()
+        confirmation_messages = self.cohort_management_page.get_cohort_confirmation_messages()
         self.assertEqual(2, len(confirmation_messages))
         self.assertEqual("0 students have been added to this cohort group", confirmation_messages[0])
         self.assertEqual("1 student was already in the cohort group", confirmation_messages[1])
 
-        error_messages = self.membership_page.get_cohort_error_messages()
+        error_messages = self.cohort_management_page.get_cohort_error_messages()
         self.assertEqual(2, len(error_messages))
         self.assertEqual("There was an error when trying to add students:", error_messages[0])
         self.assertEqual("Unknown user: unknown_user", error_messages[1])
         self.assertEqual(
             self.student_name + ",unknown_user,",
-            self.membership_page.get_cohort_student_input_field_value()
+            self.cohort_management_page.get_cohort_student_input_field_value()
         )
 
     def test_add_new_cohort(self):
@@ -212,19 +222,19 @@ class CohortConfigurationTest(UniqueCourseTest, CohortTestMixin):
         Then the cohort has 1 user
         And appropriate events have been emitted
         """
-        start_time = datetime.now()
+        start_time = datetime.now(UTC)
         new_cohort = str(uuid.uuid4().get_hex()[0:20])
-        self.assertFalse(new_cohort in self.membership_page.get_cohorts())
-        self.membership_page.add_cohort(new_cohort)
+        self.assertFalse(new_cohort in self.cohort_management_page.get_cohorts())
+        self.cohort_management_page.add_cohort(new_cohort)
         # After adding the cohort, it should automatically be selected
         EmptyPromise(
-            lambda: new_cohort == self.membership_page.get_selected_cohort(), "Waiting for new cohort to appear"
+            lambda: new_cohort == self.cohort_management_page.get_selected_cohort(), "Waiting for new cohort to appear"
         ).fulfill()
-        self.assertEqual(0, self.membership_page.get_selected_cohort_count())
-        self.membership_page.add_students_to_selected_cohort([self.instructor_name])
+        self.assertEqual(0, self.cohort_management_page.get_selected_cohort_count())
+        self.cohort_management_page.add_students_to_selected_cohort([self.instructor_name])
         # Wait for the number of users in the cohort to change, indicating that the add operation is complete.
         EmptyPromise(
-            lambda: 1 == self.membership_page.get_selected_cohort_count(), 'Waiting for student to be added'
+            lambda: 1 == self.cohort_management_page.get_selected_cohort_count(), 'Waiting for student to be added'
         ).fulfill()
         self.assertEqual(
             self.event_collection.find({
@@ -252,6 +262,162 @@ class CohortConfigurationTest(UniqueCourseTest, CohortTestMixin):
         When I view the cohort in the LMS instructor dashboard
         There is a link to take me to the Data Download section of the Instructor Dashboard.
         """
-        self.membership_page.select_data_download()
+        self.cohort_management_page.select_data_download()
         data_download_page = DataDownloadPage(self.browser)
         data_download_page.wait_for_page()
+
+    def test_cohort_by_csv_both_columns(self):
+        """
+        Scenario: the instructor can upload a file with user and cohort assignments, using both emails and usernames.
+
+        Given I have a course with two cohorts defined
+        When I go to the cohort management section of the instructor dashboard
+        I can upload a CSV file with assignments of users to cohorts via both usernames and emails
+        Then I can download a file with results
+        And appropriate events have been emitted
+        """
+        # cohort_users_both_columns.csv adds instructor_user to ManualCohort1 via username and
+        # student_user to AutoCohort1 via email
+        self._verify_csv_upload_acceptable_file("cohort_users_both_columns.csv")
+
+    def test_cohort_by_csv_only_email(self):
+        """
+        Scenario: the instructor can upload a file with user and cohort assignments, using only emails.
+
+        Given I have a course with two cohorts defined
+        When I go to the cohort management section of the instructor dashboard
+        I can upload a CSV file with assignments of users to cohorts via only emails
+        Then I can download a file with results
+        And appropriate events have been emitted
+        """
+        # cohort_users_only_email.csv adds instructor_user to ManualCohort1 and student_user to AutoCohort1 via email
+        self._verify_csv_upload_acceptable_file("cohort_users_only_email.csv")
+
+    def test_cohort_by_csv_only_username(self):
+        """
+        Scenario: the instructor can upload a file with user and cohort assignments, using only usernames.
+
+        Given I have a course with two cohorts defined
+        When I go to the cohort management section of the instructor dashboard
+        I can upload a CSV file with assignments of users to cohorts via only usernames
+        Then I can download a file with results
+        And appropriate events have been emitted
+        """
+        # cohort_users_only_username.csv adds instructor_user to ManualCohort1 and
+        # student_user to AutoCohort1 via username
+        self._verify_csv_upload_acceptable_file("cohort_users_only_username.csv")
+
+    def _verify_csv_upload_acceptable_file(self, filename):
+        """
+        Helper method to verify cohort assignments after a successful CSV upload.
+        """
+        start_time = datetime.now(UTC)
+        self.cohort_management_page.upload_cohort_file(filename)
+        self._verify_cohort_by_csv_notification(
+            "Your file '{}' has been uploaded. Please allow a few minutes for processing.".format(filename)
+        )
+
+        # student_user is moved from manual cohort group to auto cohort group
+        self.assertEqual(
+            self.event_collection.find({
+                "name": "edx.cohort.user_added",
+                "time": {"$gt": start_time},
+                "event.user_id": {"$in": [int(self.student_id)]},
+                "event.cohort_name": self.auto_cohort_name,
+            }).count(),
+            1
+        )
+        self.assertEqual(
+            self.event_collection.find({
+                "name": "edx.cohort.user_removed",
+                "time": {"$gt": start_time},
+                "event.user_id": int(self.student_id),
+                "event.cohort_name": self.manual_cohort_name,
+            }).count(),
+            1
+        )
+        # instructor_user (previously unassigned) is added to manual cohort group
+        self.assertEqual(
+            self.event_collection.find({
+                "name": "edx.cohort.user_added",
+                "time": {"$gt": start_time},
+                "event.user_id": {"$in": [int(self.instructor_id)]},
+                "event.cohort_name": self.manual_cohort_name,
+            }).count(),
+            1
+        )
+        # unicode_student_user (previously unassigned) is added to manual cohort group
+        self.assertEqual(
+            self.event_collection.find({
+                "name": "edx.cohort.user_added",
+                "time": {"$gt": start_time},
+                "event.user_id": {"$in": [int(self.unicode_student_id)]},
+                "event.cohort_name": self.manual_cohort_name,
+            }).count(),
+            1
+        )
+
+        # Verify the results can be downloaded.
+        data_download = self.instructor_dashboard_page.select_data_download()
+        EmptyPromise(
+            lambda: 1 == len(data_download.get_available_reports_for_download()), 'Waiting for downloadable report'
+        ).fulfill()
+        report = data_download.get_available_reports_for_download()[0]
+        base_file_name = "cohort_results_"
+        self.assertIn("{}_{}".format(
+            '_'.join([self.course_info['org'], self.course_info['number'], self.course_info['run']]), base_file_name
+        ), report)
+        report_datetime = datetime.strptime(
+            report[report.index(base_file_name) + len(base_file_name):-len(".csv")],
+            "%Y-%m-%d-%H%M"
+        )
+        self.assertLessEqual(start_time.replace(second=0, microsecond=0), utc.localize(report_datetime))
+
+    def test_cohort_by_csv_wrong_file_type(self):
+        """
+        Scenario: if the instructor uploads a non-csv file, an error message is presented.
+
+        Given I have a course with cohorting enabled
+        When I go to the cohort management section of the instructor dashboard
+        And I upload a file without the CSV extension
+        Then I get an error message stating that the file must have a CSV extension
+        """
+        self.cohort_management_page.upload_cohort_file("image.jpg")
+        self._verify_cohort_by_csv_notification("The file must end with the extension '.csv'.")
+
+    def test_cohort_by_csv_missing_cohort(self):
+        """
+        Scenario: if the instructor uploads a csv file with no cohort column, an error message is presented.
+
+        Given I have a course with cohorting enabled
+        When I go to the cohort management section of the instructor dashboard
+        And I upload a CSV file that is missing the cohort column
+        Then I get an error message stating that the file must have a cohort column
+        """
+        self.cohort_management_page.upload_cohort_file("cohort_users_missing_cohort_column.csv")
+        self._verify_cohort_by_csv_notification("The file must contain a 'cohort' column containing cohort names.")
+
+    def test_cohort_by_csv_missing_user(self):
+        """
+        Scenario: if the instructor uploads a csv file with no username or email column, an error message is presented.
+
+        Given I have a course with cohorting enabled
+        When I go to the cohort management section of the instructor dashboard
+        And I upload a CSV file that is missing both the username and email columns
+        Then I get an error message stating that the file must have either a username or email column
+        """
+        self.cohort_management_page.upload_cohort_file("cohort_users_missing_user_columns.csv")
+        self._verify_cohort_by_csv_notification(
+            "The file must contain a 'username' column, an 'email' column, or both."
+        )
+
+    def _verify_cohort_by_csv_notification(self, expected_message):
+        """
+        Helper method to check the CSV file upload notification message.
+        """
+        # Wait for notification message to appear, indicating file has been uploaded.
+        EmptyPromise(
+            lambda: 1 == len(self.cohort_management_page.get_csv_messages()), 'Waiting for notification'
+        ).fulfill()
+        messages = self.cohort_management_page.get_csv_messages()
+        self.assertEquals(expected_message, messages[0])

common/test/acceptance/tests/lms/test_lms_staff_view.py

@@ -107,10 +107,7 @@ class StaffDebugTest(UniqueCourseTest):
         staff_debug_page = staff_page.open_staff_debug_info()
         staff_debug_page.rescore()
         msg = staff_debug_page.idash_msg[0]
-        # Since we aren't running celery stuff, this will fail badly
-        # for now, but is worth excercising that bad of a response
-        self.assertEqual(u'Failed to rescore problem. '
-                         'Unknown Error Occurred.', msg)
+        self.assertEqual(u'Successfully rescored problem for user STAFF_TESTER', msg)
 
     def test_student_state_delete(self):
         """
@@ -176,10 +173,7 @@ class StaffDebugTest(UniqueCourseTest):
         staff_debug_page = staff_page.open_staff_debug_info()
         staff_debug_page.rescore()
         msg = staff_debug_page.idash_msg[0]
-        # Since we aren't running celery stuff, this will fail badly
-        # for now, but is worth excercising that bad of a response
-        self.assertEqual(u'Failed to rescore problem. '
-                         'Unknown Error Occurred.', msg)
+        self.assertEqual(u'Successfully rescored problem for user STAFF_TESTER', msg)
 
     def test_student_state_delete_for_problem_loaded_via_ajax(self):
         """

common/test/data/uploads/cohort_users_both_columns.csv

@@ -0,0 +1,4 @@
+username,email,ignored_column,cohort
+instructor_user,,June,ManualCohort1
+,student_user@example.com,Spring,AutoCohort1
+Ωπ,,Fall,ManualCohort1

common/test/data/uploads/cohort_users_missing_cohort_column.csv

@@ -0,0 +1,3 @@
+username,email
+instructor_user,
+,student_user@example.com

common/test/data/uploads/cohort_users_missing_user_columns.csv

@@ -0,0 +1,3 @@
+cohort
+ManualCohort1
+AutoCohort1

common/test/data/uploads/cohort_users_only_email.csv

@@ -0,0 +1,5 @@
+email,cohort
+instructor_user@example.com,ManualCohort1
+student_user@example.com,AutoCohort1
+unicode_student_user@example.com,ManualCohort1
+

common/test/data/uploads/cohort_users_only_username.csv

@@ -0,0 +1,4 @@
+username,cohort
+instructor_user,ManualCohort1
+student_user,AutoCohort1
+Ωπ,ManualCohort1