fix conflict

apt-packages.txt

@@ -0,0 +1,25 @@
+python-software-properties
+pkg-config
+curl
+git
+python-virtualenv
+build-essential
+python-dev
+gfortran
+liblapack-dev
+libfreetype6-dev
+libpng12-dev
+libxml2-dev
+libxslt-dev
+yui-compressor
+graphviz
+graphviz-dev
+mysql-server
+libmysqlclient-dev
+libgeos-dev
+libreadline6
+libreadline6-dev
+mongodb
+nodejs
+npm
+coffeescript

apt-repos.txt

@@ -0,0 +1,3 @@
+ppa:chris-lea/node.js
+ppa:chris-lea/node.js-libs
+ppa:chris-lea/libjs-underscore

brew-formulas.txt

@@ -1,10 +1,12 @@
-readline 
-sqlite 
-gdbm 
-pkg-config 
-gfortran 
-python 
-yuicompressor 
+readline
+sqlite
+gdbm
+pkg-config
+gfortran
+python
+yuicompressor
 node
 graphviz
 mysql
+geos
+mongodb

common/djangoapps/mitxmako/shortcuts.py

@@ -12,10 +12,6 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
-import logging
-
-log = logging.getLogger("mitx." + __name__)
-
 from django.template import Context
 from django.http import HttpResponse
 
@@ -42,7 +38,7 @@ def render_to_string(template_name, dictionary, context=None, namespace='main'):
         context_dictionary.update(context)
     # fetch and render template
     template = middleware.lookup[namespace].get_template(template_name)
-    return template.render(**context_dictionary)
+    return template.render_unicode(**context_dictionary)
 
 
 def render_to_response(template_name, dictionary, context_instance=None, namespace='main', **kwargs):

common/djangoapps/mitxmako/template.py

@@ -54,5 +54,4 @@ class Template(MakoTemplate):
         context_dictionary['MITX_ROOT_URL'] = settings.MITX_ROOT_URL
         context_dictionary['django_context'] = context_instance
                 
-        return super(Template, self).render(**context_dictionary)
-        
+        return super(Template, self).render_unicode(**context_dictionary)

common/djangoapps/student/models.py

@@ -36,7 +36,7 @@ file and check it in at the same time as your model changes. To do that,
 3. Add the migration file created in mitx/common/djangoapps/student/migrations/
 """
 from datetime import datetime
-from hashlib import sha1
+import hashlib
 import json
 import logging
 import uuid
@@ -197,14 +197,13 @@ def unique_id_for_user(user):
     """
     Return a unique id for a user, suitable for inserting into
     e.g. personalized survey links.
-
-    Currently happens to be implemented as a sha1 hash of the username
-    (and thus assumes that usernames don't change).
     """
-    # Using the user id as the salt because it's sort of random, and is already
-    # in the db.
-    salt = str(user.id)
-    return sha1(salt + user.username).hexdigest()
+    # include the secret key as a salt, and to make the ids unique accross
+    # different LMS installs.    
+    h = hashlib.md5()
+    h.update(settings.SECRET_KEY)
+    h.update(str(user.id))
+    return h.hexdigest()
 
 
 ## TODO: Should be renamed to generic UserGroup, and possibly

common/djangoapps/util/json_request.py

@@ -4,6 +4,11 @@ import json
 
 
 def expect_json(view_function):
+    """
+    View decorator for simplifying handing of requests that expect json.  If the request's
+    CONTENT_TYPE is application/json, parses the json dict from request.body, and updates
+    request.POST with the contents.
+    """
     @wraps(view_function)
     def expect_json_with_cloned_request(request, *args, **kwargs):
         # cdodge: fix postback errors in CMS. The POST 'content-type' header can include additional information

common/lib/capa/capa/capa_problem.py

@@ -33,6 +33,7 @@ from xml.sax.saxutils import unescape
 import chem
 import chem.chemcalc
 import chem.chemtools
+import chem.miller
 
 import calc
 from correctmap import CorrectMap
@@ -52,7 +53,7 @@ response_tag_dict = dict([(x.response_tag, x) for x in responsetypes.__all__])
 solution_tags = ['solution']
 
 # these get captured as student responses
-response_properties = ["codeparam", "responseparam", "answer"]
+response_properties = ["codeparam", "responseparam", "answer", "openendedparam"]
 
 # special problem tags which should be turned into innocuous HTML
 html_transforms = {'problem': {'tag': 'div'},
@@ -67,10 +68,11 @@ global_context = {'random': random,
                   'calc': calc,
                   'eia': eia,
                   'chemcalc': chem.chemcalc,
-                  'chemtools': chem.chemtools}
+                  'chemtools': chem.chemtools,
+                  'miller': chem.miller}
 
 # These should be removed from HTML output, including all subelements
-html_problem_semantics = ["codeparam", "responseparam", "answer", "script", "hintgroup"]
+html_problem_semantics = ["codeparam", "responseparam", "answer", "script", "hintgroup", "openendedparam","openendedrubric"]
 
 log = logging.getLogger('mitx.' + __name__)
 

common/lib/capa/capa/chem/miller.py

@@ -0,0 +1,267 @@
+""" Calculation of Miller indices """
+
+import numpy as np
+import math
+import fractions as fr
+import decimal
+import json
+
+
+def lcm(a, b):
+    """
+    Returns least common multiple of a, b
+
+    Args:
+        a, b: floats
+
+    Returns:
+        float
+    """
+    return a * b / fr.gcd(a, b)
+
+
+def segment_to_fraction(distance):
+    """
+    Converts lengths of which the plane cuts the axes to fraction.
+
+    Tries convert distance to closest nicest fraction with denominator less or
+    equal than 10. It is
+    purely for simplicity and clearance of learning purposes. Jenny: 'In typical
+    courses students usually do not encounter indices any higher than 6'.
+
+    If distance is not a number (numpy nan), it means that plane is parallel to
+    axis or contains it. Inverted fraction to nan (nan is 1/0) = 0 / 1 is
+    returned
+
+    Generally (special cases):
+
+    a) if distance is smaller than some constant, i.g. 0.01011,
+    than fraction's denominator usually much greater than 10.
+
+    b) Also, if student will set point on 0.66 -> 1/3, so it is 333 plane,
+    But if he will slightly move the mouse and click on 0.65 -> it will be
+    (16,15,16) plane. That's why we are doing adjustments for points coordinates,
+    to the closest tick, tick + tick / 2 value. And now UI sends to server only
+    values multiple to 0.05  (half of tick). Same rounding is implemented for
+    unittests.
+
+    But if one will want to calculate miller indices with exact coordinates and
+    with nice fractions (which produce small Miller indices), he may want shift
+    to new origin if segments are like S = (0.015, > 0.05, >0.05) - close to zero
+    in one coordinate. He may update S to (0, >0.05, >0.05) and shift origin.
+    In this way he can recieve nice small fractions. Also there is can be
+    degenerated case when S = (0.015, 0.012, >0.05) - if update S to (0, 0, >0.05) -
+    it is a line. This case  should be considered separately. Small nice Miller
+    numbers and possibility to create very small segments can not be implemented
+    at same time).
+
+
+    Args:
+        distance: float distance that plane cuts on axis, it must not be 0.
+        Distance is multiple of 0.05.
+
+    Returns:
+         Inverted fraction.
+         0 / 1 if distance is nan
+
+    """
+    if np.isnan(distance):
+        return fr.Fraction(0, 1)
+    else:
+        fract = fr.Fraction(distance).limit_denominator(10)
+        return fr.Fraction(fract.denominator, fract.numerator)
+
+
+def sub_miller(segments):
+    '''
+    Calculates Miller indices from segments.
+
+    Algorithm:
+
+    1. Obtain inverted fraction from segments
+
+    2. Find common denominator of inverted fractions
+
+    3. Lead fractions to common denominator and throws denominator away.
+
+    4. Return obtained values.
+
+    Args:
+        List of 3 floats, meaning distances that plane cuts on x, y, z axes.
+        Any float not equals zero, it means that plane does not intersect origin,
+        i. e. shift of origin has already been done.
+
+    Returns:
+        String that represents Miller indices, e.g: (-6,3,-6) or (2,2,2)
+    '''
+    fracts = [segment_to_fraction(segment) for segment in segments]
+    common_denominator = reduce(lcm, [fract.denominator for fract in fracts])
+    miller = ([fract.numerator * math.fabs(common_denominator) /
+                                fract.denominator for fract in fracts])
+    return'(' + ','.join(map(str, map(decimal.Decimal, miller))) + ')'
+
+
+def miller(points):
+    """
+    Calculates Miller indices from points.
+
+    Algorithm:
+
+    1. Calculate normal vector to a plane that goes trough all points.
+
+    2. Set origin.
+
+    3. Create Cartesian coordinate system (Ccs).
+
+    4. Find the lengths of segments of which the plane cuts the axes. Equation
+       of a line for axes: Origin + (Coordinate_vector - Origin) * parameter.
+
+    5. If plane goes trough Origin:
+
+       a) Find new random origin: find unit cube vertex, not crossed by a plane.
+
+       b) Repeat 2-4.
+
+       c) Fix signs of segments after Origin shift. This means to consider
+          original directions of axes. I.g.: Origin was 0,0,0 and became
+          new_origin. If new_origin has same Y coordinate as Origin, then segment
+          does not change its sign. But if new_origin has another Y coordinate than
+          origin (was 0, became 1), than segment has to change its sign (it now
+          lies on negative side of Y axis). New Origin 0 value of X or Y or Z
+          coordinate means that segment does not change sign, 1 value -> does
+          change. So new sign  is (1 - 2 * new_origin): 0 -> 1, 1 -> -1
+
+    6. Run function that calculates miller indices from segments.
+
+    Args:
+        List of points. Each point is list of float coordinates. Order of
+        coordinates in point's list: x, y, z. Points are different!
+
+    Returns:
+        String that represents Miller indices, e.g: (-6,3,-6) or (2,2,2)
+    """
+
+    N = np.cross(points[1] - points[0], points[2] - points[0])
+    O = np.array([0, 0, 0])
+    P = points[0]  # point of plane
+    Ccs = map(np.array, [[1.0, 0, 0], [0, 1.0, 0], [0, 0, 1.0]])
+    segments = ([np.dot(P - O, N) / np.dot(ort, N) if np.dot(ort, N) != 0 else
+                                            np.nan for ort in Ccs])
+    if any(x == 0 for x in segments):  # Plane goes through origin.
+        vertices = [  # top:
+                    np.array([1.0, 1.0, 1.0]),
+                    np.array([0.0, 0.0, 1.0]),
+                    np.array([1.0, 0.0, 1.0]),
+                    np.array([0.0, 1.0, 1.0]),
+                    # bottom, except 0,0,0:
+                    np.array([1.0, 0.0, 0.0]),
+                    np.array([0.0, 1.0, 0.0]),
+                    np.array([1.0, 1.0, 1.0]),
+                ]
+        for vertex in vertices:
+            if np.dot(vertex - O, N) != 0:  # vertex not in plane
+                new_origin = vertex
+                break
+        # obtain new axes with center in new origin
+        X = np.array([1 - new_origin[0], new_origin[1], new_origin[2]])
+        Y = np.array([new_origin[0], 1 - new_origin[1], new_origin[2]])
+        Z = np.array([new_origin[0], new_origin[1], 1 - new_origin[2]])
+        new_Ccs = [X - new_origin, Y - new_origin, Z - new_origin]
+        segments = ([np.dot(P - new_origin, N) / np.dot(ort, N) if
+                    np.dot(ort, N) != 0 else np.nan for ort in new_Ccs])
+        # fix signs of indices: 0 -> 1, 1 -> -1 (
+        segments = (1 - 2 * new_origin) * segments
+
+    return sub_miller(segments)
+
+
+def grade(user_input, correct_answer):
+    '''
+    Grade crystallography problem.
+
+    Returns true if lattices are the same and Miller indices are same or minus
+    same. E.g. (2,2,2) = (2, 2, 2) or (-2, -2, -2). Because sign depends only
+    on student's selection of origin.
+
+    Args:
+        user_input, correct_answer: json. Format:
+
+        user_input: {"lattice":"sc","points":[["0.77","0.00","1.00"],
+        ["0.78","1.00","0.00"],["0.00","1.00","0.72"]]}
+
+        correct_answer: {'miller': '(00-1)', 'lattice': 'bcc'}
+
+        "lattice" is one of: "", "sc", "bcc", "fcc"
+
+    Returns:
+        True or false.
+    '''
+    def negative(m):
+        """
+        Change sign of Miller indices.
+
+        Args:
+            m: string with meaning of Miller indices. E.g.:
+            (-6,3,-6) -> (6, -3, 6)
+
+        Returns:
+            String with changed signs.
+        """
+        output = ''
+        i = 1
+        while i in range(1, len(m) - 1):
+            if m[i] in (',', ' '):
+                output += m[i]
+            elif m[i] not in ('-', '0'):
+                output += '-' + m[i]
+            elif m[i] == '0':
+                output += m[i]
+            else:
+                i += 1
+                output += m[i]
+            i += 1
+        return '(' + output + ')'
+
+    def round0_25(point):
+        """
+        Rounds point coordinates to closest 0.5 value.
+
+        Args:
+            point: list of float coordinates. Order of coordinates: x, y, z.
+
+        Returns:
+            list of coordinates rounded to closes 0.5 value
+        """
+        rounded_points = []
+        for coord in point:
+            base = math.floor(coord * 10)
+            fractional_part = (coord * 10 - base)
+            aliquot0_25 = math.floor(fractional_part / 0.25)
+            if aliquot0_25 == 0.0:
+                rounded_points.append(base / 10)
+            if aliquot0_25 in (1.0, 2.0):
+                rounded_points.append(base / 10 + 0.05)
+            if aliquot0_25 == 3.0:
+                rounded_points.append(base / 10 + 0.1)
+        return rounded_points
+
+    user_answer = json.loads(user_input)
+
+    if user_answer['lattice'] != correct_answer['lattice']:
+        return False
+
+    points = [map(float, p) for p in user_answer['points']]
+
+    if len(points) < 3:
+        return False
+
+    # round point to closes 0.05 value
+    points = [round0_25(point) for point in points]
+
+    points = [np.array(point) for point in points]
+    # print miller(points), (correct_answer['miller'].replace(' ', ''),
+    #     negative(correct_answer['miller']).replace(' ', ''))
+    if miller(points) in (correct_answer['miller'].replace(' ', ''), negative(correct_answer['miller']).replace(' ', '')):
+        return True
+
+    return False

common/lib/capa/capa/chem/tests.py

@@ -1,13 +1,15 @@
 import codecs
 from fractions import Fraction
-from pyparsing import ParseException
 import unittest
 
 from chemcalc import (compare_chemical_expression, divide_chemical_expression,
                       render_to_html, chemical_equations_equal)
 
+import miller
+
 local_debug = None
 
+
 def log(s, output_type=None):
     if local_debug:
         print s
@@ -37,7 +39,6 @@ class Test_Compare_Equations(unittest.TestCase):
         self.assertFalse(chemical_equations_equal('2H2 + O2 -> H2O2',
                                                  '2O2 + 2H2 -> 2H2O2'))
 
-
     def test_different_arrows(self):
         self.assertTrue(chemical_equations_equal('H2 + O2 -> H2O2',
                                                  '2O2 + 2H2 -> 2H2O2'))
@@ -56,7 +57,6 @@ class Test_Compare_Equations(unittest.TestCase):
         self.assertTrue(chemical_equations_equal('H2 + O2 -> H2O2',
                                                  'O2 + H2 -> H2O2', exact=True))
 
-
     def test_syntax_errors(self):
         self.assertFalse(chemical_equations_equal('H2 + O2 a-> H2O2',
                                                   '2O2 + 2H2 -> 2H2O2'))
@@ -311,7 +311,6 @@ class Test_Render_Equations(unittest.TestCase):
         log(out + ' ------- ' + correct, 'html')
         self.assertEqual(out, correct)
 
-        
     def test_render_eq3(self):
         s = "H^+ + OH^- <= H2O"   # unsupported arrow
         out = render_to_html(s)
@@ -320,10 +319,148 @@ class Test_Render_Equations(unittest.TestCase):
         self.assertEqual(out, correct)
 
 
+class Test_Crystallography_Miller(unittest.TestCase):
+    ''' Tests  for crystallography grade function.'''
+
+    def test_empty_points(self):
+        user_input = '{"lattice": "bcc", "points": []}'
+        self.assertFalse(miller.grade(user_input, {'miller': '(2,2,2)', 'lattice': 'bcc'}))
+
+    def test_only_one_point(self):
+        user_input = '{"lattice": "bcc", "points": [["0.50", "0.00", "0.00"]]}'
+        self.assertFalse(miller.grade(user_input, {'miller': '(2,2,2)', 'lattice': 'bcc'}))
+
+    def test_only_two_points(self):
+        user_input = '{"lattice": "bcc", "points": [["0.50", "0.00", "0.00"], ["0.00", "0.50", "0.00"]]}'
+        self.assertFalse(miller.grade(user_input, {'miller': '(2,2,2)', 'lattice': 'bcc'}))
+
+    def test_1(self):
+        user_input = '{"lattice": "bcc", "points": [["0.50", "0.00", "0.00"], ["0.00", "0.50", "0.00"], ["0.00", "0.00", "0.50"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(2,2,2)', 'lattice': 'bcc'}))
+
+    def test_2(self):
+        user_input = '{"lattice": "bcc", "points": [["1.00", "0.00", "0.00"], ["0.00", "1.00", "0.00"], ["0.00", "0.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(1,1,1)', 'lattice': 'bcc'}))
+
+    def test_3(self):
+        user_input = '{"lattice": "bcc", "points": [["1.00", "0.50", "1.00"], ["1.00", "1.00", "0.50"], ["0.50", "1.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(2,2,2)', 'lattice': 'bcc'}))
+
+    def test_4(self):
+        user_input = '{"lattice": "bcc", "points": [["0.33", "1.00", "0.00"], ["0.00", "0.664", "0.00"], ["0.00", "1.00", "0.33"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(-3, 3, -3)', 'lattice': 'bcc'}))
+
+    def test_5(self):
+        """ return true only in case points coordinates are exact.
+        But if they transform to closest 0.05 value it is not true"""
+        user_input = '{"lattice": "bcc", "points": [["0.33", "1.00", "0.00"], ["0.00", "0.33", "0.00"], ["0.00", "1.00", "0.33"]]}'
+        self.assertFalse(miller.grade(user_input, {'miller': '(-6,3,-6)', 'lattice': 'bcc'}))
+
+    def test_6(self):
+        user_input = '{"lattice": "bcc", "points": [["0.00", "0.25", "0.00"], ["0.25", "0.00", "0.00"], ["0.00", "0.00", "0.25"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(4,4,4)', 'lattice': 'bcc'}))
+
+    def test_7(self):  # goes throug origin
+        user_input = '{"lattice": "bcc", "points": [["0.00", "1.00", "0.00"], ["1.00", "0.00", "0.00"], ["0.50", "1.00", "0.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(0,0,-1)', 'lattice': 'bcc'}))
+
+    def test_8(self):
+        user_input = '{"lattice": "bcc", "points": [["0.00", "1.00", "0.50"], ["1.00", "0.00", "0.50"], ["0.50", "1.00", "0.50"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(0,0,2)', 'lattice': 'bcc'}))
+
+    def test_9(self):
+        user_input = '{"lattice": "bcc", "points": [["1.00", "0.00", "1.00"], ["0.00", "1.00", "1.00"], ["1.00", "0.00", "0.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(1,1,0)', 'lattice': 'bcc'}))
+
+    def test_10(self):
+        user_input = '{"lattice": "bcc", "points": [["1.00", "0.00", "1.00"], ["0.00", "0.00", "0.00"], ["0.00", "1.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(1,1,-1)', 'lattice': 'bcc'}))
+
+    def test_11(self):
+        user_input = '{"lattice": "bcc", "points": [["1.00", "0.00", "0.50"], ["1.00", "1.00", "0.00"], ["0.00", "1.00", "0.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(0,1,2)', 'lattice': 'bcc'}))
+
+    def test_12(self):
+        user_input = '{"lattice": "bcc", "points": [["1.00", "0.00", "0.50"], ["0.00", "0.00", "0.50"], ["1.00", "1.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(0,1,-2)', 'lattice': 'bcc'}))
+
+    def test_13(self):
+        user_input = '{"lattice": "bcc", "points": [["0.50", "0.00", "0.00"], ["0.50", "1.00", "0.00"], ["0.00", "0.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(2,0,1)', 'lattice': 'bcc'}))
+
+    def test_14(self):
+        user_input = '{"lattice": "bcc", "points": [["0.00", "0.00", "0.00"], ["0.00", "0.00", "1.00"], ["0.50", "1.00", "0.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(2,-1,0)', 'lattice': 'bcc'}))
+
+    def test_15(self):
+        user_input = '{"lattice": "bcc", "points": [["0.00", "0.00", "0.00"], ["1.00", "1.00", "0.00"], ["0.00", "1.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(1,-1,1)', 'lattice': 'bcc'}))
+
+    def test_16(self):
+        user_input = '{"lattice": "bcc", "points": [["1.00", "0.00", "0.00"], ["0.00", "1.00", "0.00"], ["1.00", "1.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(1,1,-1)', 'lattice': 'bcc'}))
+
+    def test_17(self):
+        user_input = '{"lattice": "bcc", "points": [["0.00", "0.00", "0.00"], ["1.00", "0.00", "1.00"], ["1.00", "1.00", "0.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(-1,1,1)', 'lattice': 'bcc'}))
+
+    def test_18(self):
+        user_input = '{"lattice": "bcc", "points": [["0.00", "0.00", "0.00"], ["1.00", "1.00", "0.00"], ["0.00", "1.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(1,-1,1)', 'lattice': 'bcc'}))
+
+    def test_19(self):
+        user_input = '{"lattice": "bcc", "points": [["0.00", "0.00", "0.00"], ["1.00", "1.00", "0.00"], ["0.00", "0.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(-1,1,0)', 'lattice': 'bcc'}))
+
+    def test_20(self):
+        user_input = '{"lattice": "bcc", "points": [["1.00", "0.00", "0.00"], ["1.00", "1.00", "0.00"], ["0.00", "0.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(1,0,1)', 'lattice': 'bcc'}))
+
+    def test_21(self):
+        user_input = '{"lattice": "bcc", "points": [["0.00", "0.00", "0.00"], ["0.00", "1.00", "0.00"], ["1.00", "0.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(-1,0,1)', 'lattice': 'bcc'}))
+
+    def test_22(self):
+        user_input = '{"lattice": "bcc", "points": [["0.00", "1.00", "0.00"], ["1.00", "1.00", "0.00"], ["0.00", "0.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(0,1,1)', 'lattice': 'bcc'}))
+
+    def test_23(self):
+        user_input = '{"lattice": "bcc", "points": [["0.00", "0.00", "0.00"], ["1.00", "0.00", "0.00"], ["1.00", "1.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(0,-1,1)', 'lattice': 'bcc'}))
+
+    def test_24(self):
+        user_input = '{"lattice": "bcc", "points": [["0.66", "0.00", "0.00"], ["0.00", "0.66", "0.00"], ["0.00", "0.00", "0.66"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(3,3,3)', 'lattice': 'bcc'}))
+
+    def test_25(self):
+        user_input = u'{"lattice":"","points":[["0.00","0.00","0.01"],["1.00","1.00","0.01"],["0.00","1.00","1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(1,-1,1)', 'lattice': ''}))
+
+    def test_26(self):
+        user_input = u'{"lattice":"","points":[["0.00","0.01","0.00"],["1.00","0.00","0.00"],["0.00","0.00","1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(0,-1,0)', 'lattice': ''}))
+
+    def test_27(self):
+        """ rounding to 0.35"""
+        user_input = u'{"lattice":"","points":[["0.33","0.00","0.00"],["0.00","0.33","0.00"],["0.00","0.00","0.33"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(3,3,3)', 'lattice': ''}))
+
+    def test_28(self):
+        """ rounding to 0.30"""
+        user_input = u'{"lattice":"","points":[["0.30","0.00","0.00"],["0.00","0.30","0.00"],["0.00","0.00","0.30"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(10,10,10)', 'lattice': ''}))
+
+    def test_wrong_lattice(self):
+        user_input = '{"lattice": "bcc", "points": [["0.00", "0.00", "0.00"], ["1.00", "0.00", "0.00"], ["1.00", "1.00", "1.00"]]}'
+        self.assertFalse(miller.grade(user_input, {'miller': '(3,3,3)', 'lattice': 'fcc'}))
+
 
 def suite():
 
-    testcases = [Test_Compare_Expressions, Test_Divide_Expressions, Test_Render_Equations]
+    testcases = [Test_Compare_Expressions,
+                 Test_Divide_Expressions,
+                 Test_Render_Equations,
+                 Test_Crystallography_Miller]
     suites = []
     for testcase in testcases:
         suites.append(unittest.TestLoader().loadTestsFromTestCase(testcase))

common/lib/capa/capa/inputtypes.py

@@ -671,18 +671,15 @@ class Crystallography(InputTypeBase):
         """
         Note: height, width are required.
         """
-        return [Attribute('size', None),
-                Attribute('height'),
+        return [Attribute('height'),
                 Attribute('width'),
-
-                # can probably be removed (textline should prob be always-hidden)
-                Attribute('hidden', ''),
                 ]
 
 registry.register(Crystallography)
 
 # -------------------------------------------------------------------------
 
+
 class VseprInput(InputTypeBase):
     """
     Input for molecular geometry--show possible structures, let student
@@ -736,3 +733,53 @@ class ChemicalEquationInput(InputTypeBase):
         return {'previewer': '/static/js/capa/chemical_equation_preview.js',}
 
 registry.register(ChemicalEquationInput)
+
+#-----------------------------------------------------------------------------
+
+class OpenEndedInput(InputTypeBase):
+    """
+    A text area input for code--uses codemirror, does syntax highlighting, special tab handling,
+    etc.
+    """
+
+    template = "openendedinput.html"
+    tags = ['openendedinput']
+
+    # pulled out for testing
+    submitted_msg = ("Feedback not yet available.  Reload to check again. "
+                     "Once the problem is graded, this message will be "
+                     "replaced with the grader's feedback")
+
+    @classmethod
+    def get_attributes(cls):
+        """
+        Convert options to a convenient format.
+        """
+        return [Attribute('rows', '30'),
+                Attribute('cols', '80'),
+                Attribute('hidden', ''),
+                ]
+
+    def setup(self):
+        """
+        Implement special logic: handle queueing state, and default input.
+        """
+        # if no student input yet, then use the default input given by the problem
+        if not self.value:
+            self.value = self.xml.text
+
+        # Check if problem has been queued
+        self.queue_len = 0
+        # Flag indicating that the problem has been queued, 'msg' is length of queue
+        if self.status == 'incomplete':
+            self.status = 'queued'
+            self.queue_len = self.msg
+            self.msg = self.submitted_msg
+
+    def _extra_context(self):
+        """Defined queue_len, add it """
+        return {'queue_len': self.queue_len,}
+
+registry.register(OpenEndedInput)
+
+#-----------------------------------------------------------------------------

common/lib/capa/capa/responsetypes.py

@@ -8,22 +8,25 @@ Used by capa_problem.py
 '''
 
 # standard library imports
+import abc
 import cgi
+import hashlib
 import inspect
 import json
 import logging
 import numbers
 import numpy
+import os
 import random
 import re
 import requests
-import traceback
-import hashlib
-import abc
-import os
 import subprocess
+import traceback
 import xml.sax.saxutils as saxutils
 
+from collections import namedtuple
+from shapely.geometry import Point, MultiPoint
+
 # specific library imports
 from calc import evaluator, UndefinedVariable
 from correctmap import CorrectMap
@@ -1100,6 +1103,15 @@ class SymbolicResponse(CustomResponse):
 
 #-----------------------------------------------------------------------------
 
+"""
+valid:       Flag indicating valid score_msg format (Boolean)
+correct:     Correctness of submission (Boolean)
+score:       Points to be assigned (numeric, can be float)
+msg:         Message from grader to display to student (string)
+"""
+ScoreMessage = namedtuple('ScoreMessage',
+                          ['valid', 'correct', 'points', 'msg'])
+
 
 class CodeResponse(LoncapaResponse):
     """
@@ -1139,7 +1151,7 @@ class CodeResponse(LoncapaResponse):
         else:
             self._parse_coderesponse_xml(codeparam)
 
-    def _parse_coderesponse_xml(self,codeparam):
+    def _parse_coderesponse_xml(self, codeparam):
         '''
         Parse the new CodeResponse XML format. When successful, sets:
             self.initial_display
@@ -1151,17 +1163,9 @@ class CodeResponse(LoncapaResponse):
         grader_payload = grader_payload.text if grader_payload is not None else ''
         self.payload = {'grader_payload': grader_payload}
 
-        answer_display = codeparam.find('answer_display')
-        if answer_display is not None:
-            self.answer = answer_display.text
-        else:
-            self.answer = 'No answer provided.'
-
-        initial_display = codeparam.find('initial_display')
-        if initial_display is not None:
-            self.initial_display = initial_display.text
-        else:
-            self.initial_display = ''
+        self.initial_display = find_with_default(codeparam, 'initial_display', '')
+        self.answer = find_with_default(codeparam, 'answer_display',
+                                        'No answer provided.')
 
     def _parse_externalresponse_xml(self):
         '''
@@ -1308,8 +1312,6 @@ class CodeResponse(LoncapaResponse):
             # Sanity check on returned points
             if points < 0:
                 points = 0
-            elif points > self.maxpoints[self.answer_id]:
-                points = self.maxpoints[self.answer_id]
             # Queuestate is consumed
             oldcmap.set(self.answer_id, npoints=points, correctness=correctness,
                         msg=msg.replace('&nbsp;', '&#160;'), queuestate=None)
@@ -1717,15 +1719,38 @@ class ImageResponse(LoncapaResponse):
     which produces an [x,y] coordinate pair.  The click is correct if it falls
     within a region specified.  This region is a union of rectangles.
 
-    Lon-CAPA requires that each <imageresponse> has a <foilgroup> inside it.  That
-    doesn't make sense to me (Ike).  Instead, let's have it such that <imageresponse>
-    should contain one or more <imageinput> stanzas. Each <imageinput> should specify
-    a rectangle, given as an attribute, defining the correct answer.
+    Lon-CAPA requires that each <imageresponse> has a <foilgroup> inside it.
+    That doesn't make sense to me (Ike).  Instead, let's have it such that
+    <imageresponse> should contain one or more <imageinput> stanzas.
+    Each <imageinput> should specify a rectangle(s) or region(s), given as an
+    attribute, defining the correct answer.
+
+    <imageinput src="/static/images/Lecture2/S2_p04.png" width="811" height="610"
+    rectangle="(10,10)-(20,30);(12,12)-(40,60)"
+    regions="[[[10,10], [20,30], [40, 10]], [[100,100], [120,130], [110,150]]]"/>
+
+    Regions is list of lists [region1, region2, region3, ...] where regionN
+    is disordered list of points: [[1,1], [100,100], [50,50], [20, 70]].
+
+    If there is only one region in the list, simpler notation can be used:
+    regions="[[10,10], [30,30], [10, 30], [30, 10]]" (without explicitly
+        setting outer list)
+
+    Returns:
+        True, if click is inside any region or rectangle. Otherwise False.
     """
     snippets = [{'snippet': '''<imageresponse>
-      <imageinput src="image1.jpg" width="200" height="100" rectangle="(10,10)-(20,30)" />
-      <imageinput src="image2.jpg" width="210" height="130" rectangle="(12,12)-(40,60)" />
-      <imageinput src="image2.jpg" width="210" height="130" rectangle="(10,10)-(20,30);(12,12)-(40,60)" />
+      <imageinput src="image1.jpg" width="200" height="100"
+      rectangle="(10,10)-(20,30)" />
+      <imageinput src="image2.jpg" width="210" height="130"
+      rectangle="(12,12)-(40,60)" />
+      <imageinput src="image3.jpg" width="210" height="130"
+      rectangle="(10,10)-(20,30);(12,12)-(40,60)" />
+      <imageinput src="image4.jpg" width="811" height="610"
+      rectangle="(10,10)-(20,30);(12,12)-(40,60)"
+      regions="[[[10,10], [20,30], [40, 10]], [[100,100], [120,130], [110,150]]]"/>
+      <imageinput src="image5.jpg" width="200" height="200"
+      regions="[[[10,10], [20,30], [40, 10]], [[100,100], [120,130], [110,150]]]"/>
     </imageresponse>'''}]
 
     response_tag = 'imageresponse'
@@ -1733,19 +1758,17 @@ class ImageResponse(LoncapaResponse):
 
     def setup_response(self):
         self.ielements = self.inputfields
-        self.answer_ids = [ie.get('id')  for ie in self.ielements]
+        self.answer_ids = [ie.get('id') for ie in self.ielements]
 
     def get_score(self, student_answers):
         correct_map = CorrectMap()
         expectedset = self.get_answers()
-
-        for aid in self.answer_ids:	 # loop through IDs of <imageinput> fields in our stanza
-            given = student_answers[aid]	 # this should be a string of the form '[x,y]'
-
+        for aid in self.answer_ids:	 # loop through IDs of <imageinput>
+        #  fields in our stanza
+            given = student_answers[aid]  # this should be a string of the form '[x,y]'
             correct_map.set(aid, 'incorrect')
-            if not given: # No answer to parse. Mark as incorrect and move on
+            if not given:  # No answer to parse. Mark as incorrect and move on
                 continue
-
             # parse given answer
             m = re.match('\[([0-9]+),([0-9]+)]', given.strip().replace(' ', ''))
             if not m:
@@ -1753,28 +1776,384 @@ class ImageResponse(LoncapaResponse):
                                 'error grading %s (input=%s)' % (aid, given))
             (gx, gy) = [int(x) for x in m.groups()]
 
-            # Check whether given point lies in any of the solution rectangles
-            solution_rectangles = expectedset[aid].split(';')
-            for solution_rectangle in solution_rectangles:
-                # parse expected answer
-                # TODO: Compile regexp on file load
-                m = re.match('[\(\[]([0-9]+),([0-9]+)[\)\]]-[\(\[]([0-9]+),([0-9]+)[\)\]]',
-                             solution_rectangle.strip().replace(' ', ''))
-                if not m:
-                    msg = 'Error in problem specification! cannot parse rectangle in %s' % (
-                        etree.tostring(self.ielements[aid], pretty_print=True))
-                    raise Exception('[capamodule.capa.responsetypes.imageinput] ' + msg)
-                (llx, lly, urx, ury) = [int(x) for x in m.groups()]
-
-                # answer is correct if (x,y) is within the specified rectangle
-                if (llx <= gx <= urx) and (lly <= gy <= ury):
-                    correct_map.set(aid, 'correct')
-                    break
+            rectangles, regions = expectedset
+            if rectangles[aid]:  # rectangles part - for backward compatibility
+                # Check whether given point lies in any of the solution rectangles
+                solution_rectangles = rectangles[aid].split(';')
+                for solution_rectangle in solution_rectangles:
+                    # parse expected answer
+                    # TODO: Compile regexp on file load
+                    m = re.match('[\(\[]([0-9]+),([0-9]+)[\)\]]-[\(\[]([0-9]+),([0-9]+)[\)\]]',
+                                 solution_rectangle.strip().replace(' ', ''))
+                    if not m:
+                        msg = 'Error in problem specification! cannot parse rectangle in %s' % (
+                            etree.tostring(self.ielements[aid], pretty_print=True))
+                        raise Exception('[capamodule.capa.responsetypes.imageinput] ' + msg)
+                    (llx, lly, urx, ury) = [int(x) for x in m.groups()]
 
+                    # answer is correct if (x,y) is within the specified rectangle
+                    if (llx <= gx <= urx) and (lly <= gy <= ury):
+                        correct_map.set(aid, 'correct')
+                        break
+            if correct_map[aid]['correctness'] != 'correct' and regions[aid]:
+                parsed_region = json.loads(regions[aid])
+                if parsed_region:
+                    if type(parsed_region[0][0]) != list:
+                        # we have [[1,2],[3,4],[5,6]] - single region
+                        # instead of [[[1,2],[3,4],[5,6], [[1,2],[3,4],[5,6]]]
+                        # or [[[1,2],[3,4],[5,6]]] - multiple regions syntax
+                        parsed_region = [parsed_region]
+                    for region in parsed_region:
+                        polygon = MultiPoint(region).convex_hull
+                        if (polygon.type == 'Polygon' and
+                                polygon.contains(Point(gx, gy))):
+                            correct_map.set(aid, 'correct')
+                            break
         return correct_map
 
     def get_answers(self):
-        return dict([(ie.get('id'), ie.get('rectangle')) for ie in self.ielements])
+        return (dict([(ie.get('id'), ie.get('rectangle')) for ie in self.ielements]),
+                dict([(ie.get('id'), ie.get('regions')) for ie in self.ielements]))
+#-----------------------------------------------------------------------------
+
+class OpenEndedResponse(LoncapaResponse):
+    """
+    Grade student open ended responses using an external grading system,
+    accessed through the xqueue system.
+
+    Expects 'xqueue' dict in ModuleSystem with the following keys that are
+    needed by OpenEndedResponse:
+
+        system.xqueue = { 'interface': XqueueInterface object,
+                          'callback_url': Per-StudentModule callback URL
+                                          where results are posted (string),
+                        }
+
+    External requests are only submitted for student submission grading
+        (i.e. and not for getting reference answers)
+
+    By default, uses the OpenEndedResponse.DEFAULT_QUEUE queue.
+    """
+
+    DEFAULT_QUEUE = 'open-ended'
+    response_tag = 'openendedresponse'
+    allowed_inputfields = ['openendedinput']
+    max_inputfields = 1
+
+    def setup_response(self):
+        '''
+        Configure OpenEndedResponse from XML.
+        '''
+        xml = self.xml
+        self.url = xml.get('url', None)
+        self.queue_name = xml.get('queuename', self.DEFAULT_QUEUE)
+
+        # The openendedparam tag encapsulates all grader settings
+        oeparam = self.xml.find('openendedparam')
+        prompt = self.xml.find('prompt')
+        rubric = self.xml.find('openendedrubric')
+
+        if oeparam is None:
+            raise ValueError("No oeparam found in problem xml.")
+        if prompt is None:
+            raise ValueError("No prompt found in problem xml.")
+        if rubric is None:
+            raise ValueError("No rubric found in problem xml.")
+
+        self._parse(oeparam, prompt, rubric)
+
+    @staticmethod
+    def stringify_children(node):
+        """
+        Modify code from stringify_children in xmodule.  Didn't import directly
+        in order to avoid capa depending on xmodule (seems to be avoided in
+        code)
+        """
+        parts=[node.text if node.text is not None else '']
+        for p in node.getchildren():
+            parts.append(etree.tostring(p, with_tail=True, encoding='unicode'))
+
+        return ' '.join(parts)
+
+    def _parse(self, oeparam, prompt, rubric):
+        '''
+        Parse OpenEndedResponse XML:
+            self.initial_display
+            self.payload - dict containing keys --
+            'grader' : path to grader settings file, 'problem_id' : id of the problem
+
+            self.answer - What to display when show answer is clicked
+        '''
+        # Note that OpenEndedResponse is agnostic to the specific contents of grader_payload
+        prompt_string = self.stringify_children(prompt)
+        rubric_string = self.stringify_children(rubric)
+
+        grader_payload = oeparam.find('grader_payload')
+        grader_payload = grader_payload.text if grader_payload is not None else ''
+
+        #Update grader payload with student id.  If grader payload not json, error.
+        try:
+            parsed_grader_payload = json.loads(grader_payload)
+            # NOTE: self.system.location is valid because the capa_module
+            # __init__ adds it (easiest way to get problem location into
+            # response types)
+        except TypeError, ValueError:
+            log.exception("Grader payload %r is not a json object!", grader_payload)
+        parsed_grader_payload.update({
+            'location' : self.system.location,
+            'course_id' : self.system.course_id,
+            'prompt' : prompt_string,
+            'rubric' : rubric_string,
+        })
+        updated_grader_payload = json.dumps(parsed_grader_payload)
+
+        self.payload = {'grader_payload': updated_grader_payload}
+
+        self.initial_display = find_with_default(oeparam, 'initial_display', '')
+        self.answer = find_with_default(oeparam, 'answer_display', 'No answer given.')
+        try:
+            self.max_score = int(find_with_default(oeparam, 'max_score', 1))
+        except ValueError:
+            self.max_score = 1
+
+    def get_score(self, student_answers):
+
+        try:
+            submission = student_answers[self.answer_id]
+        except KeyError:
+            msg = ('Cannot get student answer for answer_id: {0}. student_answers {1}'
+                   .format(self.answer_id, student_answers))
+            log.exception(msg)
+            raise LoncapaProblemError(msg)
+
+        # Prepare xqueue request
+        #------------------------------------------------------------
+
+        qinterface = self.system.xqueue['interface']
+        qtime = datetime.strftime(datetime.now(), xqueue_interface.dateformat)
+
+        anonymous_student_id = self.system.anonymous_student_id
+
+        # Generate header
+        queuekey = xqueue_interface.make_hashkey(str(self.system.seed) + qtime +
+                                                 anonymous_student_id +
+                                                 self.answer_id)
+
+        xheader = xqueue_interface.make_xheader(lms_callback_url=self.system.xqueue['callback_url'],
+            lms_key=queuekey,
+            queue_name=self.queue_name)
+
+        self.context.update({'submission': submission})
+
+        contents = self.payload.copy()
+
+        # Metadata related to the student submission revealed to the external grader
+        student_info = {'anonymous_student_id': anonymous_student_id,
+                        'submission_time': qtime,
+                        }
+
+        #Update contents with student response and student info
+        contents.update({
+            'student_info': json.dumps(student_info),
+            'student_response': submission,
+            'max_score' : self.max_score
+            })
+
+        # Submit request. When successful, 'msg' is the prior length of the queue
+        (error, msg) = qinterface.send_to_queue(header=xheader,
+            body=json.dumps(contents))
+
+        # State associated with the queueing request
+        queuestate = {'key': queuekey,
+                      'time': qtime,}
+
+        cmap = CorrectMap()
+        if error:
+            cmap.set(self.answer_id, queuestate=None,
+                msg='Unable to deliver your submission to grader. (Reason: {0}.)'
+                    ' Please try again later.'.format(msg))
+        else:
+            # Queueing mechanism flags:
+            #   1) Backend: Non-null CorrectMap['queuestate'] indicates that
+            #      the problem has been queued
+            #   2) Frontend: correctness='incomplete' eventually trickles down
+            #      through inputtypes.textbox and .filesubmission to inform the
+            #      browser that the submission is queued (and it could e.g. poll)
+            cmap.set(self.answer_id, queuestate=queuestate,
+                     correctness='incomplete', msg=msg)
+
+        return cmap
+
+    def update_score(self, score_msg, oldcmap, queuekey):
+        log.debug(score_msg)
+        score_msg = self._parse_score_msg(score_msg)
+        if not score_msg.valid:
+            oldcmap.set(self.answer_id,
+                msg = 'Invalid grader reply. Please contact the course staff.')
+            return oldcmap
+
+        correctness = 'correct' if score_msg.correct else 'incorrect'
+
+        # TODO: Find out how this is used elsewhere, if any
+        self.context['correct'] = correctness
+
+        # Replace 'oldcmap' with new grading results if queuekey matches.  If queuekey
+        # does not match, we keep waiting for the score_msg whose key actually matches
+        if oldcmap.is_right_queuekey(self.answer_id, queuekey):
+            # Sanity check on returned points
+            points = score_msg.points
+            if points < 0:
+                points = 0
+
+            # Queuestate is consumed, so reset it to None
+            oldcmap.set(self.answer_id, npoints=points, correctness=correctness,
+                msg = score_msg.msg.replace('&nbsp;', '&#160;'), queuestate=None)
+        else:
+            log.debug('OpenEndedResponse: queuekey {0} does not match for answer_id={1}.'.format(
+                queuekey, self.answer_id))
+
+        return oldcmap
+
+    def get_answers(self):
+        anshtml = '<span class="openended-answer"><pre><code>{0}</code></pre></span>'.format(self.answer)
+        return {self.answer_id: anshtml}
+
+    def get_initial_display(self):
+        return {self.answer_id: self.initial_display}
+
+    def _convert_longform_feedback_to_html(self, response_items):
+        """
+        Take in a dictionary, and return html strings for display to student.
+        Input:
+            response_items: Dictionary with keys success, feedback.
+                if success is True, feedback should be a dictionary, with keys for
+                   types of feedback, and the corresponding feedback values.
+                if success is False, feedback is actually an error string.
+
+                NOTE: this will need to change when we integrate peer grading, because
+                that will have more complex feedback.
+
+        Output:
+            String -- html that can be displayed to the student.
+        """
+
+        # We want to display available feedback in a particular order.
+        # This dictionary specifies which goes first--lower first.
+        priorities = {# These go at the start of the feedback
+                      'spelling': 0,
+                      'grammar': 1,
+                      # needs to be after all the other feedback
+                      'markup_text': 3}
+
+        default_priority = 2
+
+        def get_priority(elt):
+            """
+            Args:
+                elt: a tuple of feedback-type, feedback
+            Returns:
+                the priority for this feedback type
+            """
+            return priorities.get(elt[0], default_priority)
+
+        def format_feedback(feedback_type, value):
+            return """
+            <div class="{feedback_type}">
+            {value}
+            </div>
+            """.format(feedback_type, value)
+
+        # TODO (vshnayder): design and document the details of this format so
+        # that we can do proper escaping here (e.g. are the graders allowed to
+        # include HTML?)
+
+        for tag in ['success', 'feedback']:
+            if tag not in response_items:
+                return format_feedback('errors', 'Error getting feedback')
+
+        feedback_items = response_items['feedback']
+        try:
+            feedback = json.loads(feedback_items)
+        except (TypeError, ValueError):
+            log.exception("feedback_items have invalid json %r", feedback_items)
+            return format_feedback('errors', 'Could not parse feedback')
+
+        if response_items['success']:
+            if len(feedback) == 0:
+                return format_feedback('errors', 'No feedback available')
+
+            feedback_lst = sorted(feedback.items(), key=get_priority)
+            return u"\n".join(format_feedback(k, v) for k, v in feedback_lst)
+        else:
+            return format_feedback('errors', response_items['feedback'])
+
+
+    def _format_feedback(self, response_items):
+        """
+        Input:
+            Dictionary called feedback.  Must contain keys seen below.
+        Output:
+            Return error message or feedback template
+        """
+
+        feedback = self._convert_longform_feedback_to_html(response_items)
+
+        if not response_items['success']:
+            return self.system.render_template("open_ended_error.html",
+                                               {'errors' : feedback})
+
+        feedback_template = self.system.render_template("open_ended_feedback.html", {
+            'grader_type': response_items['grader_type'],
+            'score': response_items['score'],
+            'feedback': feedback,
+        })
+
+        return feedback_template
+
+
+    def _parse_score_msg(self, score_msg):
+        """
+         Grader reply is a JSON-dump of the following dict
+           { 'correct': True/False,
+             'score': Numeric value (floating point is okay) to assign to answer
+             'msg': grader_msg
+             'feedback' : feedback from grader
+             }
+
+        Returns (valid_score_msg, correct, score, msg):
+            valid_score_msg: Flag indicating valid score_msg format (Boolean)
+            correct:         Correctness of submission (Boolean)
+            score:           Points to be assigned (numeric, can be float)
+        """
+        fail = ScoreMessage(valid=False, correct=False, points=0, msg='')
+        try:
+            score_result = json.loads(score_msg)
+        except (TypeError, ValueError):
+            log.error("External grader message should be a JSON-serialized dict."
+                      " Received score_msg = {0}".format(score_msg))
+            return fail
+
+        if not isinstance(score_result, dict):
+            log.error("External grader message should be a JSON-serialized dict."
+                      " Received score_result = {0}".format(score_result))
+            return fail
+
+        for tag in ['score', 'feedback', 'grader_type', 'success']:
+            if tag not in score_result:
+                log.error("External grader message is missing required tag: {0}"
+                          .format(tag))
+                return fail
+
+        feedback = self._format_feedback(score_result)
+
+        # HACK: for now, just assume it's correct if you got more than 2/3.
+        # Also assumes that score_result['score'] is an integer.
+        score_ratio = int(score_result['score']) / self.max_score
+        correct = (score_ratio >= 0.66)
+
+        #Currently ignore msg and only return feedback (which takes the place of msg)
+        return ScoreMessage(valid=True, correct=correct,
+                            score=score_result['score'], msg=feedback)
 
 #-----------------------------------------------------------------------------
 # TEMPORARY: List of all response subclasses
@@ -1793,4 +2172,5 @@ __all__ = [CodeResponse,
            ChoiceResponse,
            MultipleChoiceResponse,
            TrueFalseResponse,
-           JavascriptResponse]
+           JavascriptResponse,
+           OpenEndedResponse]

common/lib/capa/capa/templates/crystallography.html

@@ -1,34 +1,28 @@
 <section id="inputtype_${id}" class="capa_inputtype" >
-     <div id="holder" style="width:${width};height:${height}"></div>
+    <div class="crystalography_problem" style="width:${width};height:${height}"></div>
+
+    <div class="input_lattice">
+        Lattice: <select></select>
+    </div>
+
     <div class="script_placeholder" data-src="/static/js/raphael.js"></div>
     <div class="script_placeholder" data-src="/static/js/sylvester.js"></div>
-    <div class="script_placeholder" data-src="/static/js/underscore-min.js"></div>
     <div class="script_placeholder" data-src="/static/js/crystallography.js"></div>
 
-
     % if status == 'unsubmitted':
-    <div class="unanswered" id="status_${id}">
+        <div class="unanswered" id="status_${id}">
     % elif status == 'correct':
-    <div class="correct" id="status_${id}">
+        <div class="correct" id="status_${id}">
     % elif status == 'incorrect':
-    <div class="incorrect" id="status_${id}">
+        <div class="incorrect" id="status_${id}">
     % elif status == 'incomplete':
-    <div class="incorrect" id="status_${id}">
-    % endif
-    % if hidden:
-      <div style="display:none;" name="${hidden}" inputid="input_${id}" />
+        <div class="incorrect" id="status_${id}">
     % endif
 
-  <input type="text" name="input_${id}" id="input_${id}" value="${value|h}"
-   % if size:
-   size="${size}"
-   % endif
-   % if hidden:
-    style="display:none;"
-   % endif
-   />
 
-      <p class="status">
+    <input type="text" name="input_${id}" id="input_${id}" value="${value|h}" style="display:none;"/>
+
+    <p class="status">
         % if status == 'unsubmitted':
         unanswered
         % elif status == 'correct':
@@ -38,14 +32,15 @@
         % elif status == 'incomplete':
         incomplete
         % endif
-      </p>
+    </p>
 
-  <p id="answer_${id}" class="answer"></p>
+    <p id="answer_${id}" class="answer"></p>
 
-  % if msg:
-      <span class="message">${msg|n}</span>
-  % endif
-% if status in ['unsubmitted', 'correct', 'incorrect', 'incomplete']:
-</div>
-% endif
+    % if msg:
+        <span class="message">${msg|n}</span>
+    % endif
+
+    % if status in ['unsubmitted', 'correct', 'incorrect', 'incomplete']:
+        </div>
+    % endif
 </section>

common/lib/capa/capa/templates/openendedinput.html

@@ -0,0 +1,32 @@
+<section id="openended_${id}" class="openended">
+  <textarea rows="${rows}" cols="${cols}" name="input_${id}" class="short-form-response" id="input_${id}"
+       % if hidden:
+      	    style="display:none;"
+       % endif
+  >${value|h}</textarea>
+
+  <div class="grader-status">
+    % if status == 'unsubmitted':
+      <span class="unanswered" style="display:inline-block;" id="status_${id}">Unanswered</span>
+    % elif status == 'correct':
+      <span class="correct" id="status_${id}">Correct</span>
+    % elif status == 'incorrect':
+      <span class="incorrect" id="status_${id}">Incorrect</span>
+    % elif status == 'queued':
+      <span class="grading" id="status_${id}">Submitted for grading</span>
+    % endif
+
+    % if hidden:
+      <div style="display:none;" name="${hidden}" inputid="input_${id}" />
+    % endif
+  </div>
+
+  <span id="answer_${id}"></span>
+
+    % if status == 'queued':
+    <input name="reload" class="reload" type="button" value="Recheck for Feedback" onclick="document.location.reload(true);" />
+    % endif
+  <div class="external-grader-message">
+    ${msg|n}
+  </div>
+</section>

common/lib/capa/capa/tests/test_files/imageresponse.xml

@@ -18,4 +18,23 @@ Hello</p></text>
 <text><p>Use conservation of energy.</p></text>
 </hintgroup>
 </imageresponse>
+
+
+<imageresponse max="1" loncapaid="12">
+<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" rectangle="(490,11)-(556,98)" regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"/>
+<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" rectangle="(490,11)-(556,98)" regions='[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]'/>
+<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"/>
+<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"/>
+<text>Click on either of the two positions as discussed previously.</text>
+<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" regions="[[[10,10], [20,10], [20, 30]]]"/>
+<text>Click on either of the two positions as discussed previously.</text>
+<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" regions="[[10,10], [30,30], [15, 15]]"/>
+<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" regions="[[10,10], [30,30], [10, 30], [30, 10]]"/>
+<text>Click on either of the two positions as discussed previously.</text>
+<hintgroup showoncorrect="no">
+<text><p>Use conservation of energy.</p></text>
+</hintgroup>
+</imageresponse>
+
+
 </problem>

common/lib/capa/capa/tests/test_inputtypes.py

@@ -407,13 +407,11 @@ class CrystallographyTest(unittest.TestCase):
     def test_rendering(self):
         height = '12'
         width = '33'
-        size = '10'
 
         xml_str = """<crystallography id="prob_1_2"
         height="{h}"
         width="{w}"
-        size="{s}"
-        />""".format(h=height, w=width, s=size)
+        />""".format(h=height, w=width)
 
         element = etree.fromstring(xml_str)
 
@@ -428,9 +426,7 @@ class CrystallographyTest(unittest.TestCase):
         expected = {'id': 'prob_1_2',
                     'value': value,
                     'status': 'unsubmitted',
-                    'size': size,
                     'msg': '',
-                    'hidden': '',
                     'width': width,
                     'height': height,
                    }

common/lib/capa/capa/tests/test_responsetypes.py

@@ -52,24 +52,57 @@ class ImageResponseTest(unittest.TestCase):
     def test_ir_grade(self):
         imageresponse_file = os.path.dirname(__file__) + "/test_files/imageresponse.xml"
         test_lcp = lcp.LoncapaProblem(open(imageresponse_file).read(), '1', system=test_system)
-        correct_answers = {'1_2_1': '(490,11)-(556,98)',
-                           '1_2_2': '(242,202)-(296,276)',
-                           '1_2_3': '(490,11)-(556,98);(242,202)-(296,276)',
-                           '1_2_4': '(490,11)-(556,98);(242,202)-(296,276)',
-                           '1_2_5': '(490,11)-(556,98);(242,202)-(296,276)',
+        # testing regions only
+        correct_answers = {
+           #regions
+           '1_2_1': '(490,11)-(556,98)',
+           '1_2_2': '(242,202)-(296,276)',
+           '1_2_3': '(490,11)-(556,98);(242,202)-(296,276)',
+           '1_2_4': '(490,11)-(556,98);(242,202)-(296,276)',
+           '1_2_5': '(490,11)-(556,98);(242,202)-(296,276)',
+           #testing regions and rectanges
+           '1_3_1': 'rectangle="(490,11)-(556,98)" \
+           regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"',
+           '1_3_2': 'rectangle="(490,11)-(556,98)" \
+           regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"',
+           '1_3_3': 'regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"',
+           '1_3_4': 'regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"',
+           '1_3_5': 'regions="[[[10,10], [20,10], [20, 30]]]"',
+           '1_3_6': 'regions="[[10,10], [30,30], [15, 15]]"',
+           '1_3_7': 'regions="[[10,10], [30,30], [10, 30], [30, 10]]"',
                           }
-        test_answers = {'1_2_1': '[500,20]',
-                        '1_2_2': '[250,300]',
-                        '1_2_3': '[500,20]',
-                        '1_2_4': '[250,250]',
-                        '1_2_5': '[10,10]',
+        test_answers = {
+            '1_2_1': '[500,20]',
+            '1_2_2': '[250,300]',
+            '1_2_3': '[500,20]',
+            '1_2_4': '[250,250]',
+            '1_2_5': '[10,10]',
+
+            '1_3_1': '[500,20]',
+            '1_3_2': '[15,15]',
+            '1_3_3': '[500,20]',
+            '1_3_4': '[115,115]',
+            '1_3_5': '[15,15]',
+            '1_3_6': '[20,20]',
+            '1_3_7': '[20,15]',
                         }
+
+        # regions
         self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_2_1'), 'correct')
         self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_2_2'), 'incorrect')
         self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_2_3'), 'correct')
         self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_2_4'), 'correct')
         self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_2_5'), 'incorrect')
 
+        # regions and rectangles
+        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_3_1'), 'correct')
+        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_3_2'), 'correct')
+        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_3_3'), 'incorrect')
+        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_3_4'), 'correct')
+        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_3_5'), 'correct')
+        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_3_6'), 'incorrect')
+        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_3_7'), 'correct')
+
 
 class SymbolicResponseTest(unittest.TestCase):
     def test_sr_grade(self):

common/lib/capa/capa/util.py

@@ -65,3 +65,25 @@ def is_file(file_to_test):
     Duck typing to check if 'file_to_test' is a File object
     '''
     return all(hasattr(file_to_test, method) for method in ['read', 'name'])
+
+
+def find_with_default(node, path, default):
+    """
+    Look for a child of node using , and return its text if found.
+    Otherwise returns default.
+
+    Arguments:
+       node: lxml node
+       path: xpath search expression
+       default: value to return if nothing found
+
+    Returns:
+       node.find(path).text if the find succeeds, default otherwise.
+       
+    """
+    v = node.find(path)
+    if v is not None:
+        return v.text
+    else:
+        return default
+

common/lib/capa/capa/xqueue_interface.py

@@ -49,6 +49,7 @@ def parse_xreply(xreply):
 
     return_code = xreply['return_code']
     content = xreply['content']
+
     return (return_code, content)
 
 
@@ -80,7 +81,11 @@ class XQueueInterface(object):
 
         # Log in, then try again
         if error and (msg == 'login_required'):
-            self._login()
+            (error, content) = self._login()
+            if error != 0:
+                # when the login fails
+                log.debug("Failed to login to queue: %s", content)
+                return (error, content)
             if files_to_upload is not None:
                 # Need to rewind file pointers
                 for f in files_to_upload:

common/lib/xmodule/xmodule/capa_module.py

@@ -146,6 +146,11 @@ class CapaModule(XModule):
         else:
             self.seed = None
 
+        # Need the problem location in openendedresponse to send out.  Adding
+        # it to the system here seems like the least clunky way to get it
+        # there.
+        self.system.set('location', self.location.url())
+
         try:
             # TODO (vshnayder): move as much as possible of this work and error
             # checking to descriptor load time

common/lib/xmodule/xmodule/course_module.py

@@ -149,6 +149,10 @@ class CourseDescriptor(SequenceDescriptor):
     def grade_cutoffs(self):
         return self._grading_policy['GRADE_CUTOFFS']
 
+    @property
+    def lowest_passing_grade(self):
+        return min(self._grading_policy['GRADE_CUTOFFS'].values())
+
     @property
     def tabs(self):
         """
@@ -292,7 +296,7 @@ class CourseDescriptor(SequenceDescriptor):
                     return False
         except:
             log.exception("Error parsing discussion_blackouts for course {0}".format(self.id))
-        
+
         return True
 
     @property

common/lib/xmodule/xmodule/css/capa/display.scss

@@ -121,16 +121,6 @@ section.problem {
       }
     }
 
-    &.processing {
-      p.status {
-        @include inline-block();
-        background: url('../images/spinner.gif') center center no-repeat;
-        height: 20px;
-        width: 20px;
-        text-indent: -9999px;
-      }
-    }
-
     &.correct, &.ui-icon-check {
       p.status {
         @include inline-block();
@@ -250,6 +240,13 @@ section.problem {
       }
     }
 
+    .reload
+    {
+      float:right;
+      margin: 10px;
+    }
+
+
     .grader-status {
       padding: 9px;
       background: #F6F6F6;
@@ -266,6 +263,13 @@ section.problem {
         margin: -7px 7px 0 0;
       }
 
+      .grading {
+        background: url('../images/info-icon.png') left center no-repeat;
+        padding-left: 25px;
+        text-indent: 0px;
+        margin: 0px 7px 0 0;
+      }
+
       p {
         line-height: 20px;
         text-transform: capitalize;
@@ -685,6 +689,21 @@ section.problem {
             color: #B00;
           }
         }
+
+        .markup-text{
+          margin: 5px;
+          padding: 20px 0px 15px 50px;
+          border-top: 1px solid #DDD;
+          border-left: 20px solid #FAFAFA;
+
+          bs {
+            color: #BB0000;
+          }
+
+          bg {
+            color: #BDA046;
+          }
+        }
       }
     }
   }

common/lib/xmodule/xmodule/error_module.py

@@ -149,14 +149,14 @@ class ErrorDescriptor(JSONEditingDescriptor):
         '''
         try:
             xml = etree.fromstring(self.definition['data']['contents'])
-            return etree.tostring(xml)
+            return etree.tostring(xml, encoding='unicode')
         except etree.XMLSyntaxError:
             # still not valid.
             root = etree.Element('error')
             root.text = self.definition['data']['contents']
             err_node = etree.SubElement(root, 'error_msg')
             err_node.text = self.definition['data']['error_msg']
-            return etree.tostring(root)
+            return etree.tostring(root, encoding='unicode')
 
 
 class NonStaffErrorDescriptor(ErrorDescriptor):

common/lib/xmodule/xmodule/html_module.py

@@ -7,15 +7,14 @@ from lxml import etree
 from lxml.html import rewrite_links
 from path import path
 
-from .x_module import XModule
 from pkg_resources import resource_string
-from .xml_module import XmlDescriptor, name_to_pathname
-from .editing_module import EditingDescriptor
-from .stringify import stringify_children
-from .html_checker import check_html
-from xmodule.modulestore import Location
-
 from xmodule.contentstore.content import XASSET_SRCREF_PREFIX, StaticContent
+from xmodule.editing_module import EditingDescriptor
+from xmodule.html_checker import check_html
+from xmodule.modulestore import Location
+from xmodule.stringify import stringify_children
+from xmodule.x_module import XModule
+from xmodule.xml_module import XmlDescriptor, name_to_pathname
 
 log = logging.getLogger("mitx.courseware")
 
@@ -123,7 +122,7 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
 
             try:
                 with system.resources_fs.open(filepath) as file:
-                    html = file.read()
+                    html = file.read().decode('utf-8')
                     # Log a warning if we can't parse the file, but don't error
                     if not check_html(html):
                         msg = "Couldn't parse html in {0}.".format(filepath)
@@ -164,7 +163,7 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
 
         resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
         with resource_fs.open(filepath, 'w') as file:
-            file.write(self.definition['data'])
+            file.write(self.definition['data'].encode('utf-8'))
 
         # write out the relative name
         relname = path(pathname).basename()

common/lib/xmodule/xmodule/js/src/capa/schematic.js

@@ -1953,7 +1953,7 @@ cktsim = (function() {
 	var module = {
 	    'Circuit': Circuit,
 	    'parse_number': parse_number,
-	    'parse_source': parse_source,
+	    'parse_source': parse_source
 	}
 	return module;
     }());
@@ -2068,7 +2068,7 @@ schematic = (function() {
 	    'n': [NFet, 'NFet'],
 	    'p': [PFet, 'PFet'],
 	    's': [Probe, 'Voltage Probe'],
-	    'a': [Ammeter, 'Current Probe'],
+	    'a': [Ammeter, 'Current Probe']
 	};
 
 	// global clipboard
@@ -5502,7 +5502,7 @@ schematic = (function() {
 	    'magenta' : 'rgb(255,64,255)',
 	    'yellow': 'rgb(255,255,64)',
 	    'black': 'rgb(0,0,0)',
-	    'x-axis': undefined,
+	    'x-axis': undefined
 	};
 
 	function Probe(x,y,rotation,color,offset) {
@@ -6100,7 +6100,7 @@ schematic = (function() {
 		    'Amplitude',
 		    'Frequency (Hz)',
 		    'Delay until sin starts (secs)',
-		    'Phase offset (degrees)'],
+		    'Phase offset (degrees)']
 	}
 
 	// build property editor div
@@ -6300,7 +6300,7 @@ schematic = (function() {
 
 	var module = {
 	    'Schematic': Schematic,
-	    'component_slider': component_slider,
+	    'component_slider': component_slider
 	}
 	return module;
     }());

common/lib/xmodule/xmodule/js/src/selfassessment/display.coffee

@@ -120,7 +120,7 @@ class @SelfAssessment
     if @state == 'done'
       $.postWithPrefix "#{@ajax_url}/reset", {}, (response) =>
         if response.success
-          @answer_area.html('')
+          @answer_area.val('')
           @rubric_wrapper.html('')
           @hint_wrapper.html('')
           @message_wrapper.html('')

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -339,6 +339,12 @@ class ModuleStore(object):
         '''
         raise NotImplementedError
 
+    def get_course(self, course_id):
+        '''
+        Look for a specific course id.  Returns the course descriptor, or None if not found.
+        '''
+        raise NotImplementedError
+
     def get_parent_locations(self, location):
         '''Find all locations that are the parents of this location.  Needed
         for path_to_location().
@@ -399,3 +405,10 @@ class ModuleStoreBase(ModuleStore):
 
         errorlog = self._get_errorlog(location)
         return errorlog.errors
+
+    def get_course(self, course_id):
+        """Default impl--linear search through course list"""
+        for c in self.get_courses():
+            if c.id == course_id:
+                return c
+        return None

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -152,7 +152,7 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
                 make_name_unique(xml_data)
 
                 descriptor = XModuleDescriptor.load_from_xml(
-                    etree.tostring(xml_data), self, self.org,
+                    etree.tostring(xml_data, encoding='unicode'), self, self.org,
                     self.course, xmlstore.default_class)
             except Exception as err:
                 print err, self.load_error_modules
@@ -436,7 +436,7 @@ class XMLModuleStore(ModuleStoreBase):
                 self.load_error_modules,
             )
 
-            course_descriptor = system.process_xml(etree.tostring(course_data))
+            course_descriptor = system.process_xml(etree.tostring(course_data, encoding='unicode'))
 
             # NOTE: The descriptors end up loading somewhat bottom up, which
             # breaks metadata inheritance via get_children().  Instead

common/lib/xmodule/xmodule/raw_module.py

@@ -13,7 +13,7 @@ class RawDescriptor(XmlDescriptor, XMLEditingDescriptor):
     """
     @classmethod
     def definition_from_xml(cls, xml_object, system):
-        return {'data': etree.tostring(xml_object, pretty_print=True)}
+        return {'data': etree.tostring(xml_object, pretty_print=True,encoding='unicode')}
 
     def definition_to_xml(self, resource_fs):
         try:

common/lib/xmodule/xmodule/self_assessment_module.py

@@ -7,20 +7,21 @@ Parses xml definition file--see below for exact format.
 
 import copy
 from fs.errors import ResourceNotFoundError
+import itertools
+import json
 import logging
-import os
-import sys
 from lxml import etree
 from lxml.html import rewrite_links
 from path import path
-import json
-from progress import Progress
+import os
+import sys
 
 from pkg_resources import resource_string
 
 from .capa_module import only_one, ComplexEncoder
 from .editing_module import EditingDescriptor
 from .html_checker import check_html
+from progress import Progress
 from .stringify import stringify_children
 from .x_module import XModule
 from .xml_module import XmlDescriptor
@@ -52,6 +53,8 @@ class SelfAssessmentModule(XModule):
          submissions too.)
     """
 
+    STATE_VERSION = 1
+
     # states
     INITIAL = 'initial'
     ASSESSING = 'assessing'
@@ -102,35 +105,130 @@ class SelfAssessmentModule(XModule):
         else:
             instance_state = {}
 
-        # Note: score responses are on scale from 0 to max_score
-        self.student_answers = instance_state.get('student_answers', [])
-        self.scores = instance_state.get('scores', [])
-        self.hints = instance_state.get('hints', [])
+        instance_state = self.convert_state_to_current_format(instance_state)
+
+        # History is a list of tuples of (answer, score, hint), where hint may be
+        # None for any element, and score and hint can be None for the last (current)
+        # element.
+        # Scores are on scale from 0 to max_score
+        self.history = instance_state.get('history', [])
 
         self.state = instance_state.get('state', 'initial')
 
+        self.attempts = instance_state.get('attempts', 0)
+        self.max_attempts = int(self.metadata.get('attempts', MAX_ATTEMPTS))
+
         # Used for progress / grading.  Currently get credit just for
         # completion (doesn't matter if you self-assessed correct/incorrect).
-
         self._max_score = int(self.metadata.get('max_score', MAX_SCORE))
 
-        self.attempts = instance_state.get('attempts', 0)
-
-        self.max_attempts = int(self.metadata.get('attempts', MAX_ATTEMPTS))
-
         self.rubric = definition['rubric']
         self.prompt = definition['prompt']
         self.submit_message = definition['submitmessage']
         self.hint_prompt = definition['hintprompt']
 
+
+    def latest_answer(self):
+        """None if not available"""
+        if not self.history:
+            return None
+        return self.history[-1].get('answer')
+
+    def latest_score(self):
+        """None if not available"""
+        if not self.history:
+            return None
+        return self.history[-1].get('score')
+
+    def latest_hint(self):
+        """None if not available"""
+        if not self.history:
+            return None
+        return self.history[-1].get('hint')
+
+    def new_history_entry(self, answer):
+        self.history.append({'answer': answer})
+
+    def record_latest_score(self, score):
+        """Assumes that state is right, so we're adding a score to the latest
+        history element"""
+        self.history[-1]['score'] = score
+
+    def record_latest_hint(self, hint):
+        """Assumes that state is right, so we're adding a score to the latest
+        history element"""
+        self.history[-1]['hint'] = hint
+
+
+    def change_state(self, new_state):
+        """
+        A centralized place for state changes--allows for hooks.  If the
+        current state matches the old state, don't run any hooks.
+        """
+        if self.state == new_state:
+            return
+
+        self.state = new_state
+
+        if self.state == self.DONE:
+            self.attempts += 1
+
+    @staticmethod
+    def convert_state_to_current_format(old_state):
+        """
+        This module used to use a problematic state representation. This method
+        converts that into the new format.
+
+        Args:
+            old_state: dict of state, as passed in.  May be old.
+
+        Returns:
+            new_state: dict of new state
+        """
+        if old_state.get('version', 0) == SelfAssessmentModule.STATE_VERSION:
+            # already current
+            return old_state
+
+        # for now, there's only one older format.
+
+        new_state = {'version': SelfAssessmentModule.STATE_VERSION}
+
+        def copy_if_present(key):
+            if key in old_state:
+                new_state[key] = old_state[key]
+
+        for to_copy in ['attempts', 'state']:
+            copy_if_present(to_copy)
+
+        # The answers, scores, and hints need to be kept together to avoid them
+        # getting out of sync.
+
+        # NOTE: Since there's only one problem with a few hundred submissions
+        # in production so far, not trying to be smart about matching up hints
+        # and submissions in cases where they got out of sync.
+
+        student_answers = old_state.get('student_answers', [])
+        scores = old_state.get('scores', [])
+        hints = old_state.get('hints', [])
+
+        new_state['history'] = [
+            {'answer': answer,
+             'score': score,
+             'hint': hint}
+             for answer, score, hint in itertools.izip_longest(
+                     student_answers, scores, hints)]
+        return new_state
+
+
     def _allow_reset(self):
         """Can the module be reset?"""
         return self.state == self.DONE and self.attempts < self.max_attempts
 
     def get_html(self):
         #set context variables and render template
-        if self.state != self.INITIAL and self.student_answers:
-            previous_answer = self.student_answers[-1]
+        if self.state != self.INITIAL:
+            latest = self.latest_answer()
+            previous_answer = latest if latest is not None else ''
         else:
             previous_answer = ''
 
@@ -149,26 +247,19 @@ class SelfAssessmentModule(XModule):
         # cdodge: perform link substitutions for any references to course static content (e.g. images)
         return rewrite_links(html, self.rewrite_content_links)
 
-    def get_score(self):
-        """
-        Returns dict with 'score' key
-        """
-        return {'score': self.get_last_score()}
-
     def max_score(self):
         """
         Return max_score
         """
         return self._max_score
 
-    def get_last_score(self):
+    def get_score(self):
         """
         Returns the last score in the list
         """
-        last_score=0
-        if(len(self.scores)>0):
-            last_score=self.scores[len(self.scores)-1]
-        return last_score
+        score = self.latest_score()
+        return {'score': score if score is not None else 0,
+                'total': self._max_score}
 
     def get_progress(self):
         '''
@@ -176,7 +267,7 @@ class SelfAssessmentModule(XModule):
         '''
         if self._max_score > 0:
             try:
-                return Progress(self.get_last_score(), self._max_score)
+                return Progress(self.get_score()['score'], self._max_score)
             except Exception as err:
                 log.exception("Got bad progress")
                 return None
@@ -250,9 +341,10 @@ class SelfAssessmentModule(XModule):
         if self.state in (self.INITIAL, self.ASSESSING):
             return ''
 
-        if self.state == self.DONE and len(self.hints) > 0:
+        if self.state == self.DONE:
             # display the previous hint
-            hint = self.hints[-1]
+            latest = self.latest_hint()
+            hint = latest if latest is not None else ''
         else:
             hint = ''
 
@@ -281,6 +373,14 @@ class SelfAssessmentModule(XModule):
     def save_answer(self, get):
         """
         After the answer is submitted, show the rubric.
+
+        Args:
+            get: the GET dictionary passed to the ajax request.  Should contain
+                a key 'student_answer'
+
+        Returns:
+            Dictionary with keys 'success' and either 'error' (if not success),
+            or 'rubric_html' (if success).
         """
         # Check to see if attempts are less than max
         if self.attempts > self.max_attempts:
@@ -295,8 +395,9 @@ class SelfAssessmentModule(XModule):
         if self.state != self.INITIAL:
             return self.out_of_sync_error(get)
 
-        self.student_answers.append(get['student_answer'])
-        self.state = self.ASSESSING
+        # add new history element with answer and empty score and hint.
+        self.new_history_entry(get['student_answer'])
+        self.change_state(self.ASSESSING)
 
         return {
             'success': True,
@@ -318,27 +419,24 @@ class SelfAssessmentModule(XModule):
         'message_html' only if success is true
         """
 
-        n_answers = len(self.student_answers)
-        n_scores = len(self.scores)
-        if (self.state != self.ASSESSING or n_answers !=  n_scores + 1):
-            msg = "%d answers, %d scores" % (n_answers, n_scores)
-            return self.out_of_sync_error(get, msg)
+        if self.state != self.ASSESSING:
+            return self.out_of_sync_error(get)
 
         try:
             score = int(get['assessment'])
-        except:
+        except ValueError:
             return {'success': False, 'error': "Non-integer score value"}
 
-        self.scores.append(score)
+        self.record_latest_score(score)
 
         d = {'success': True,}
 
         if score == self.max_score():
-            self.state = self.DONE
+            self.change_state(self.DONE)
             d['message_html'] = self.get_message_html()
             d['allow_reset'] = self._allow_reset()
         else:
-            self.state = self.REQUEST_HINT
+            self.change_state(self.REQUEST_HINT)
             d['hint_html'] = self.get_hint_html()
 
         d['state'] = self.state
@@ -360,19 +458,15 @@ class SelfAssessmentModule(XModule):
             # the same number of hints and answers.
             return self.out_of_sync_error(get)
 
-        self.hints.append(get['hint'].lower())
-        self.state = self.DONE
-
-        # increment attempts
-        self.attempts = self.attempts + 1
+        self.record_latest_hint(get['hint'])
+        self.change_state(self.DONE)
 
         # To the tracking logs!
         event_info = {
             'selfassessment_id': self.location.url(),
             'state': {
-                'student_answers': self.student_answers,
-                'score': self.scores,
-                'hints': self.hints,
+                'version': self.STATE_VERSION,
+                'history': self.history,
                 }
             }
         self.system.track_function('save_hint', event_info)
@@ -397,7 +491,7 @@ class SelfAssessmentModule(XModule):
                 'success': False,
                 'error': 'Too many attempts.'
             }
-        self.state = self.INITIAL
+        self.change_state(self.INITIAL)
         return {'success': True}
 
 
@@ -407,12 +501,11 @@ class SelfAssessmentModule(XModule):
         """
 
         state = {
-                 'student_answers': self.student_answers,
-                 'hints': self.hints,
+                 'version': self.STATE_VERSION,
+                 'history': self.history,
                  'state': self.state,
-                 'scores': self.scores,
                  'max_score': self._max_score,
-                 'attempts': self.attempts
+                 'attempts': self.attempts,
         }
         return json.dumps(state)
 

common/lib/xmodule/xmodule/seq_module.py

@@ -10,7 +10,7 @@ from xmodule.progress import Progress
 from xmodule.exceptions import NotFoundError
 from pkg_resources import resource_string
 
-log = logging.getLogger("mitx.common.lib.seq_module")
+log = logging.getLogger(__name__)
 
 # HACK: This shouldn't be hard-coded to two types
 # OBSOLETE: This obsoletes 'type'
@@ -124,7 +124,7 @@ class SequenceDescriptor(MakoModuleDescriptor, XmlDescriptor):
         children = []
         for child in xml_object:
             try:
-                children.append(system.process_xml(etree.tostring(child)).location.url())
+                children.append(system.process_xml(etree.tostring(child, encoding='unicode')).location.url())
             except:
                 log.exception("Unable to load child when parsing Sequence. Continuing...")
                 continue

common/lib/xmodule/xmodule/stringify.py

@@ -22,7 +22,7 @@ def stringify_children(node):
     #                 next element.
     parts = [node.text]
     for c in node.getchildren():
-        parts.append(etree.tostring(c, with_tail=True))
+        parts.append(etree.tostring(c, with_tail=True, encoding='unicode'))
 
     # filter removes possible Nones in texts and tails
-    return ''.join(filter(None, parts))
+    return u''.join(filter(None, parts))

common/lib/xmodule/xmodule/template_module.py

@@ -58,7 +58,7 @@ class CustomTagDescriptor(RawDescriptor):
         params = dict(xmltree.items())
         with system.resources_fs.open('custom_tags/{name}'
                                    .format(name=template_name)) as template:
-            return Template(template.read()).render(**params)
+            return Template(template.read().decode('utf-8')).render(**params)
 
 
     def __init__(self, system, definition, **kwargs):

common/lib/xmodule/xmodule/tests/__init__.py

@@ -4,7 +4,7 @@ unittests for xmodule
 Run like this:
 
     rake test_common/lib/xmodule
-    
+
 """
 
 import unittest
@@ -19,11 +19,12 @@ import xmodule
 from xmodule.x_module import ModuleSystem
 from mock import Mock
 
-i4xs = ModuleSystem(
+test_system = ModuleSystem(
     ajax_url='courses/course_id/modx/a_location',
     track_function=Mock(),
     get_module=Mock(),
-    render_template=Mock(),
+    # "render" to just the context...
+    render_template=lambda template, context: str(context),
     replace_urls=Mock(),
     user=Mock(),
     filestore=Mock(),

common/lib/xmodule/xmodule/tests/test_progress.py

@@ -5,7 +5,7 @@ import unittest
 from xmodule.progress import Progress
 from xmodule import x_module
 
-from . import i4xs
+from . import test_system
 
 class ProgressTest(unittest.TestCase):
     ''' Test that basic Progress objects work.  A Progress represents a
@@ -133,6 +133,6 @@ class ModuleProgressTest(unittest.TestCase):
     '''
     def test_xmodule_default(self):
         '''Make sure default get_progress exists, returns None'''
-        xm = x_module.XModule(i4xs, 'a://b/c/d/e', None, {})
+        xm = x_module.XModule(test_system, 'a://b/c/d/e', None, {})
         p = xm.get_progress()
         self.assertEqual(p, None)

common/lib/xmodule/xmodule/tests/test_self_assessment.py

@@ -0,0 +1,54 @@
+import json
+from mock import Mock
+import unittest
+
+from xmodule.self_assessment_module import SelfAssessmentModule
+from xmodule.modulestore import Location
+
+from . import test_system
+
+class SelfAssessmentTest(unittest.TestCase):
+
+    definition = {'rubric': 'A rubric',
+                  'prompt': 'Who?',
+                  'submitmessage': 'Shall we submit now?',
+                  'hintprompt': 'Consider this...',
+                  }
+
+    location = Location(["i4x", "edX", "sa_test", "selfassessment",
+                         "SampleQuestion"])
+
+    metadata = {'attempts': '10'}
+
+    descriptor = Mock()
+
+    def test_import(self):
+        state = json.dumps({'student_answers': ["Answer 1", "answer 2", "answer 3"],
+                            'scores': [0, 1],
+                            'hints': ['o hai'],
+                            'state': SelfAssessmentModule.ASSESSING,
+                            'attempts': 2})
+
+        module = SelfAssessmentModule(test_system, self.location,
+                                      self.definition, self.descriptor,
+                                      state, {}, metadata=self.metadata)
+
+        self.assertEqual(module.get_score()['score'], 0)
+
+        self.assertTrue('answer 3' in module.get_html())
+        self.assertFalse('answer 2' in module.get_html())
+
+        module.save_assessment({'assessment': '0'})
+        self.assertEqual(module.state, module.REQUEST_HINT)
+
+        module.save_hint({'hint': 'hint for ans 3'})
+        self.assertEqual(module.state, module.DONE)
+
+        d = module.reset({})
+        self.assertTrue(d['success'])
+        self.assertEqual(module.state, module.INITIAL)
+
+        # if we now assess as right, skip the REQUEST_HINT state
+        module.save_answer({'student_answer': 'answer 4'})
+        module.save_assessment({'assessment': '1'})
+        self.assertEqual(module.state, module.DONE)

common/lib/xmodule/xmodule/x_module.py

@@ -233,17 +233,17 @@ class XModule(HTMLSnippet):
             self._loaded_children = [c for c in children if c is not None]
 
         return self._loaded_children
-    
+
     def get_children_locations(self):
         '''
         Returns the locations of each of child modules.
-        
+
         Overriding this changes the behavior of get_children and
         anything that uses get_children, such as get_display_items.
-        
+
         This method will not instantiate the modules of the children
         unless absolutely necessary, so it is cheaper to call than get_children
-        
+
         These children will be the same children returned by the
         descriptor unless descriptor.has_dynamic_children() is true.
         '''
@@ -288,8 +288,20 @@ class XModule(HTMLSnippet):
         return '{}'
 
     def get_score(self):
-        ''' Score the student received on the problem.
-        '''
+        """
+        Score the student received on the problem, or None if there is no
+        score.
+
+        Returns:
+          dictionary
+             {'score': integer, from 0 to get_max_score(),
+              'total': get_max_score()}
+
+          NOTE (vshnayder): not sure if this was the intended return value, but
+          that's what it's doing now.  I suspect that we really want it to just
+          return a number.  Would need to change (at least) capa and
+          modx_dispatch to match if we did that.
+        """
         return None
 
     def max_score(self):
@@ -319,7 +331,7 @@ class XModule(HTMLSnippet):
             get is a dictionary-like object '''
         return ""
 
-    # cdodge: added to support dynamic substitutions of 
+    # cdodge: added to support dynamic substitutions of
     # links for courseware assets (e.g. images). <link> is passed through from lxml.html parser
     def rewrite_content_links(self, link):
         # see if we start with our format, e.g. 'xasset:<filename>'
@@ -408,7 +420,7 @@ class XModuleDescriptor(Plugin, HTMLSnippet, ResourceTemplates):
     # cdodge: this is a list of metadata names which are 'system' metadata
     # and should not be edited by an end-user
     system_metadata_fields = [ 'data_dir' ]
-    
+
     # A list of descriptor attributes that must be equal for the descriptors to
     # be equal
     equality_attributes = ('definition', 'metadata', 'location',
@@ -562,18 +574,18 @@ class XModuleDescriptor(Plugin, HTMLSnippet, ResourceTemplates):
             self,
             metadata=self.metadata
         )
-    
-    
+
+
     def has_dynamic_children(self):
         """
         Returns True if this descriptor has dynamic children for a given
         student when the module is created.
-        
+
         Returns False if the children of this descriptor are the same
-        children that the module will return for any student. 
+        children that the module will return for any student.
         """
         return False
-        
+
 
     # ================================= JSON PARSING ===========================
     @staticmethod
@@ -797,7 +809,8 @@ class ModuleSystem(object):
                  debug=False,
                  xqueue=None,
                  node_path="",
-                 anonymous_student_id=''):
+                 anonymous_student_id='',
+                 course_id=None):
         '''
         Create a closure around the system environment.
 
@@ -832,6 +845,8 @@ class ModuleSystem(object):
                          ajax results.
 
         anonymous_student_id - Used for tracking modules with student id
+
+        course_id - the course_id containing this module
         '''
         self.ajax_url = ajax_url
         self.xqueue = xqueue
@@ -844,6 +859,7 @@ class ModuleSystem(object):
         self.replace_urls = replace_urls
         self.node_path = node_path
         self.anonymous_student_id = anonymous_student_id
+        self.course_id = course_id
         self.user_is_staff = user is not None and user.is_staff
 
     def get(self, attr):

common/lib/xmodule/xmodule/xml_module.py

@@ -1,19 +1,20 @@
-from xmodule.x_module import (XModuleDescriptor, policy_key)
-from xmodule.modulestore import Location
-from lxml import etree
 import json
 import copy
 import logging
-import traceback
-from collections import namedtuple
-from fs.errors import ResourceNotFoundError
 import os
 import sys
+from collections import namedtuple
+from lxml import etree
+
+from xmodule.x_module import (XModuleDescriptor, policy_key)
+from xmodule.modulestore import Location
 
 log = logging.getLogger(__name__)
 
+# assume all XML files are persisted as utf-8.
 edx_xml_parser = etree.XMLParser(dtd_validation=False, load_dtd=False,
-                                 remove_comments=True, remove_blank_text=True)
+                                 remove_comments=True, remove_blank_text=True,
+                                 encoding='utf-8')
 
 def name_to_pathname(name):
     """
@@ -366,7 +367,7 @@ class XmlDescriptor(XModuleDescriptor):
             filepath = self.__class__._format_filepath(self.category, url_path)
             resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
             with resource_fs.open(filepath, 'w') as file:
-                file.write(etree.tostring(xml_object, pretty_print=True))
+                file.write(etree.tostring(xml_object, pretty_print=True, encoding='utf-8'))
 
             # And return just a pointer with the category and filename.
             record_object = etree.Element(self.category)
@@ -381,7 +382,7 @@ class XmlDescriptor(XModuleDescriptor):
             record_object.set('org', self.location.org)
             record_object.set('course', self.location.course)
 
-        return etree.tostring(record_object, pretty_print=True)
+        return etree.tostring(record_object, pretty_print=True, encoding='utf-8')
 
     def definition_to_xml(self, resource_fs):
         """

common/templates/mathjax_include.html

@@ -33,4 +33,4 @@
 <!-- This must appear after all mathjax-config blocks, so it is after the imports from the other templates.
      It can't be run through static.url because MathJax uses crazy url introspection to do lazy loading of
      MathJax extension libraries -->
-<script type="text/javascript" src="/static/js/vendor/mathjax-MathJax-c9db6ac/MathJax.js?config=TeX-MML-AM_HTMLorMML-full"></script>
+<script type="text/javascript" src="https://edx-static.s3.amazonaws.com/mathjax-MathJax-07669ac/MathJax.js?config=TeX-MML-AM_HTMLorMML-full"></script>

common/test/data/full/about/faq.html

@@ -6,7 +6,7 @@
     <p>No - anyone and everyone is welcome to take this course.</p>
   </li>
   <li>What textbook should I buy?
-    <p>Although the lectures are designed to be self-contained, we recommend (but do not require) that students refer to the book Worlds Together, Worlds Apart: A History of the World: From 1000 CE to the Present (W W Norton, 3rd edition) -- Volume II, which was written specifically for this course.</p>
+    <p>Although the lectures are designed to be self-contained, we recommend (but do not require) that students refer to the book Worlds Together, Worlds Apart: A History of the World: From 1000 CE to the Present (W W Norton, 3rd edition) &mdash; Volume II, which was written specifically for this course.</p>
   </li>
   <li>Does Harvard award credentials or reports regarding my work in this course?
     <p>Princeton does not award credentials or issue reports for student work in this course. However, Coursera could maintain a record of your score on the assessments and, with your permission, verify that score for authorized parties.</p>

common/test/data/full/chapter/Overview.xml

@@ -2,7 +2,7 @@
   <video url_name="welcome"/>
   <sequential filename="System_Usage_Sequence" slug="System_Usage_Sequence" format="Lecture Sequence" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never" name="System Usage Sequence"/>
   <vertical slug="Lab0_Using_the_tools" format="Lab" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never" name="Lab0: Using the tools">
-    <html slug="html_19" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never"> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab </html>
+    <html slug="html_19" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never"> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab… </html>
     <html slug="html_5555" filename="html_5555"/>
     <problem filename="Lab_0_Using_the_Tools" slug="Lab_0_Using_the_Tools" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="false" name="Lab 0: Using the Tools"/>
   </vertical>

common/test/data/full/custom_tags/book

@@ -1 +1 @@
-More information given in <a href="/book/${page}">the text</a>. 
+More information given in… <a href="/book/${page}">the text</a>. 

common/test/data/full/custom_tags/discuss

@@ -1 +1 @@
-<a href='https://6002x.mitx.mit.edu/discussion/questions/scope:all/sort:activity-desc/tags:${tag}/page:1/'> Discussion: ${tag} </a>
+<a href='https://6002x.mitx.mit.edu/discussion/questions/scope:all/sort:activity-desc/tags:${tag}/page:1/'> Discussion: ${tag}… </a>

common/test/data/full/custom_tags/slides

@@ -1 +1 @@
-Lecture Slides Handout [<a href="">Clean </a>][<a href="">Annotated</a>]
+Lecture Slides Handout [<a href="">Clean… </a>][<a href="">Annotated…</a>]

common/test/data/full/html/FirstOrderSystemHint.xml

@@ -1,4 +1,4 @@
-Hint
+Hint…
 <br/><br/>
 Remember that the time evolution of any variable \(x(t)\) governed by
 a first-order system with a time-constant \(\tau\) for a time \(t) between an initial

common/test/data/full/html/Week_13_Tutorials.html

@@ -4,14 +4,14 @@
     <section class="tutorials">
       <h2> Basic Tutorials </h2>
       <ul>
-        <li><a href="/section/wk13_solder">Soldering</a> -- Steve
+        <li><a href="/section/wk13_solder">Soldering</a> &mdash; Steve
        Finberg, one of the pioneers in from Draper Lab, talks about
        soldering. </li>
       </ul>
       <h2> Bonus Tutorials </h2>
       <ul>
         <li><a href="/section/wk13_FreqResp">Frequency Response
-       Curves</a> -- We explain several techniques for understanding
+       Curves</a> &mdash; We explain several techniques for understanding
        and approximating Bode plots. </li>
       </ul>
     </section>

common/test/data/full/html/Week_1_Tutorials.html

@@ -41,7 +41,7 @@
         <li><a href="/section/problem_1_3">OCW Problem 1-3 </a> - Reverse engineer a black-box resistor network</li>
       </ul>
       <hr/>
-      <p>  Since the course has students from a diverse set of backgrounds, the first week's tutorials includes several extra segments, worked out with greater detail, to help bring everyone up to speed. </p>
+      <p>  Since the course has students from a diverse set of backgrounds, the first week's tutorials includes several extra segments, worked out with greater detail, to help bring everyone up to speed. Gratuitous &ge; entity.</p>
     </section>
   </body>
 </html>

common/test/data/full/html/html_5555.html

@@ -1 +1 @@
-    <html slug="html_5555" filename="html_5555> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab </html>
+    <html slug="html_5555" filename="html_5555> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab. </html>

common/test/data/full/html/linearity_clarify.html

@@ -34,6 +34,6 @@
     the Thevenin or Norton theorems to summarize the behavior at
     a pair of exposed terminals.
     </p><p>
-    Sorry for the confusion of words -- natural language is like
+    Sorry for the confusion of words &mdash; natural language is like
     that!
   </p>

common/test/data/full/html/linearity_clarify.xml

@@ -34,6 +34,6 @@
     the Thevenin or Norton theorems to summarize the behavior at
     a pair of exposed terminals.
     </p><p>
-    Sorry for the confusion of words -- natural language is like
+    Sorry for the confusion of words &mdash; natural language is like
     that!
   </p>

common/test/data/full/html/schematic_tutorial.html

@@ -9,14 +9,14 @@ the right of the diagram area) and drag it onto the diagram.  Release
 the mouse when the component is in the correct position.
 </td>
 </tr>
-
+<!-- note that entities like &mdash; may be used. -->
 <tr>
 <td>Move a component</td>
 <td>Click to select a component in the diagram (it will turn green)
 and then drag it to its new location.  You can use shift-click to add
 a component to the current selection.  Or you can click somewhere in
 the diagram that is not on top of a component and drag out a selection
-rectangle -- components intersecting the rectangle will be added to
+rectangle &mdash; components intersecting the rectangle will be added to
 the current selection.
 </td>
 </tr>
@@ -63,7 +63,7 @@ engineeering notation:
 <td>Add a wire</td>
 <td>Wires start at connection points, the open circles that
 appear at the terminals of components or the ends of wires.
-Click on a connection point to start a wire -- a green wire
+Click on a connection point to start a wire &mdash; a green wire
 will appear with one end anchored at the starting point.
 Drag the mouse and release the mouse button when the other
 end of the wire is positioned as you wish.  Once a wire has

common/test/data/full/html/units_hint.html

@@ -1,4 +1,4 @@
-Hint
+Hint…
 <br/><br/>
 Be careful of units here.  Make sure you notice multipliers such
-as u, k, m, M.
+as u (or &mu;), k, m, M.

common/test/data/full/info/updates.html

@@ -9,8 +9,9 @@
 <li> <h2>May 2 </h2>
 <section class="update-description">
 <ul>
-<li> We have opened the show-answer button on the midterm. </li>
-<li> There was a four hour outage in posting ability on the discussion board Monday night. It has been fixed. We apologise for the inconvenience.</li>
+<!-- utf-8 characters are acceptable… as are HTML entities -->
+<li> We have opened the show-answer button on the midterm… </li>
+<li> There was a four hour outage in posting ability on the discussion board Monday night&hellip; It has been fixed. We apologise for the inconvenience.</li>
 </ul>
 </li>
 <li> <h2>April 30 </h2>

common/test/data/full/problem/Circuit_Sandbox.xml

@@ -1,6 +1,6 @@
-<problem><startouttext/><p/>Here's a sandbox where you can experiment with all the components
+<problem><!-- include ellipses to test non-ascii characters --><startouttext/><p/>Here's a sandbox where you can experiment with all the components
 we'll discuss in 6.002x.  If you click on CHECK below, your diagram
-will be saved on the server and you can return at some later time.
+will be saved on the server and you can return at some later time…
 <endouttext/><schematicresponse><p/><center><schematic name="work" value="" width="800" height="600"/></center><answer type="loncapa/python">
 correct = ['correct']
 </answer></schematicresponse></problem>

common/test/data/full/problem/H1P3_Poor_Workmanship.xml

@@ -78,7 +78,8 @@ So the total heating power in Joe's shop was:
 <numericalresponse answer="$Pbad"><responseparam type="tolerance" default="5%" name="tol" description="Numerical Tolerance"/><textline/></numericalresponse>
 <startouttext/>
 <br/>
-No wonder Joe was cold.
+<!-- add non-ascii utf-8 character here -->
+No wonder Joe was cold…
 <endouttext/>
 
 </problem>

common/test/data/full/problem/Lab_0_Using_the_Tools.xml

@@ -94,7 +94,7 @@ scope probes to nodes A, B and C and edit their properties so that the
 plots will be different colors. Now run a transient analysis for 5ms.
 Move the mouse over the plot until the marker (a vertical dashed line
 that follows the mouse when it's over the plot) is at approximately
-1.25ms.  Please report the measured voltages for nodes A, B and C.
+1.25ms.  Please report the measured voltages for nodes A, B and C…
 
 <br/>
 <div style="margin-left: 4em;">

common/test/data/full/problem/Sample_Algebraic_Problem.xml

@@ -6,7 +6,7 @@ z = "A*x^2 + sqrt(y)"
 Enter the algebraic expression \(A x^2 + \sqrt{y}\) in the box below.  The
 entry is case sensitive.  The product must be indicated with an
 asterisk, and the exponentation with a caret, so you must write 
-"A*x^2 + sqrt(y)".
+"A*x^2 + sqrt(y)"…
 <endouttext/>
 <formularesponse type="cs" samples="A,x,y@1,1,1:3,3,3#10" answer="$z"><responseparam description="Numerical Tolerance" type="tolerance" default="0.00001" name="tol"/><textline size="40"/></formularesponse>
 

common/test/data/full/problem/Sample_Numeric_Problem.xml

@@ -1,6 +1,6 @@
 <problem><startouttext/>
 Enter the numerical value of the expression \(x + y\) where 
-\(x = 3\) and \(y = 5\).
+\(x = 3\) and \(y = 5\)…
 <endouttext/>
 
 <numericalresponse answer="8"><responseparam type="tolerance" default="5%" name="tol" description="Numerical Tolerance"/><textline/></numericalresponse>

common/test/data/full/problem/choiceresponse_demo.xml

@@ -1,19 +1,20 @@
 <problem display_name="S3E2: Lorentz Force">
 
 <startouttext/>
-<p>Consider a hypothetical magnetic field pointing out of your computer screen. Now imagine an electron traveling from right to leftin the plane of your screen. A diagram of this situation is show below.</p>
+<p>Consider a hypothetical magnetic field pointing out of your computer screen. Now imagine an electron traveling from right to left in the plane of your screen. A diagram of this situation is show below…</p>
 <center><img width="400" src="/static/images/LSQimages/LSQ_W01_8.png"/></center>
 
 <p>a. The magnitude of the force experienced by the electron is proportional the product of which of the following? (Select all that apply.)</p>
 <endouttext/>
 <choiceresponse>
 <checkboxgroup>
-<choice correct="true"><text>Magnetic field strength</text></choice>
-<choice correct="false"><text>Electric field strength</text></choice>
-<choice correct="true"><text>Electric charge of the electron</text></choice>
-<choice correct="false"><text>Radius of the electron</text></choice>
-<choice correct="false"><text>Mass of the electron</text></choice>
-<choice correct="true"><text>Velocity of the electron</text></choice>
+<!-- include ellipses to test non-ascii characters -->
+<choice correct="true"><text>Magnetic field strength…</text></choice>
+<choice correct="false"><text>Electric field strength…</text></choice>
+<choice correct="true"><text>Electric charge of the electron…</text></choice>
+<choice correct="false"><text>Radius of the electron…</text></choice>
+<choice correct="false"><text>Mass of the electron…</text></choice>
+<choice correct="true"><text>Velocity of the electron…</text></choice>
 
 </checkboxgroup>
 </choiceresponse>

common/test/data/full/problem/codeinput_demo.xml

@@ -2,7 +2,8 @@
 <problem display_name="L4 Problem 1">
   <text>
     <p>
-      <b class="bfseries">Part 1: Function Types</b>
+      <!-- include ellipses to test non-ascii characters -->
+      <b class="bfseries">Part 1: Function Types…</b>
     </p>
     <p>
 For each of the following functions, specify the type of its <b class="bfseries">output</b>. You can assume each function is called with an appropriate argument, as specified by its docstring. </p>

common/test/data/full/sequential/Administrivia_and_Circuit_Elements.xml

@@ -3,12 +3,13 @@
   <vertical slug="vertical_66" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never">
     <problem filename="S1E3_AC_power" slug="S1E3_AC_power" name="S1E3: AC power"/>
     <customtag tag="S1E3" slug="discuss_67" impl="discuss"/>
-    <html slug="html_68"> S1E4 has been removed. </html>
+    <!-- utf-8 characters acceptable, but not HTML entities -->
+    <html slug="html_68"> S1E4 has been removed…</html>
   </vertical>
   <vertical filename="vertical_89" slug="vertical_89" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never"/>
   <vertical slug="vertical_94" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never">
     <video youtube="0.75:XNh13VZhThQ,1.0:XbDRmF6J0K0,1.25:JDty12WEQWk,1.50:wELKGj-5iyM" slug="What_s_next" name="What's next"/>
-    <html slug="html_95">Minor correction: Six elements (five resistors)</html>
+    <html slug="html_95">Minor correction: Six elements (five resistors)…</html>
     <customtag tag="S1" slug="discuss_96" impl="discuss"/>
   </vertical>
 </sequential>

common/test/data/full/vertical/vertical_89.xml

@@ -1,6 +1,7 @@
 <sequential>
   <html slug="html_90">
-    <h1> </h1>
+    <!-- UTF-8 characters are acceptable… HTML entities are not -->
+    <h1>Inline content…</h1>
   </html>
   <video youtube="1.50:vl9xrfxcr38,1.25:qxNX4REGqx4,1.0:BGU1poJDgOY,0.75:8rK9vnpystQ" slug="S1V14_Summary" name="S1V14: Summary"/>
   <customtag tag="S1" slug="discuss_91" impl="discuss"/>

common/test/data/full/video/welcome.xml

@@ -1 +1 @@
-<video youtube="0.75:izygArpw-Qo,1.0:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8" format="Video" display_name="Welcome"/>
+<video youtube="0.75:izygArpw-Qo,1.0:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8" format="Video" display_name="Welcome…"/>

common/test/data/toy/html/secret/toylab.html

@@ -1,3 +1,13 @@
 <b>Lab 2A: Superposition Experiment</b>
 
+<<<<<<< Updated upstream
 <p>Isn't the toy course great?</p>
+
+<p>Let's add some markup that uses non-ascii characters.
+For example, we should be able to write words like encyclop&aelig;dia, or foreign words like fran&ccedil;ais.
+Looking beyond latin-1, we should handle math symbols:  &pi;r&sup2 &le; &#8734.
+And it shouldn't matter if we use entities or numeric codes &mdash; &Omega; &ne; &pi; &equiv; &#937; &#8800; &#960;.
+</p>
+=======
+<p>Isn't the toy course great? — &le;</p>
+>>>>>>> Stashed changes

create-dev-env.sh

@@ -98,8 +98,9 @@ RUBY_VER="1.9.3"
 NUMPY_VER="1.6.2"
 SCIPY_VER="0.10.1"
 BREW_FILE="$BASE/mitx/brew-formulas.txt"
+APT_REPOS_FILE="$BASE/mitx/apt-repos.txt"
+APT_PKGS_FILE="$BASE/mitx/apt-packages.txt"
 LOG="/var/tmp/install-$(date +%Y%m%d-%H%M%S).log"
-APT_PKGS="pkg-config curl git python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor nodejs npm graphviz graphviz-dev mysql-server libmysqlclient-dev"
 
 if [[ $EUID -eq 0 ]]; then
     error "This script should not be run using sudo or as the root user"
@@ -182,16 +183,22 @@ case `uname -s` in
             error "Please install lsb-release."
             exit 1
         }
+
         distro=`lsb_release -cs`
         case $distro in
             maya|lisa|natty|oneiric|precise|quantal)
                 output "Installing ubuntu requirements"
-                sudo apt-get install python-software-properties
-                sudo add-apt-repository ppa:chris-lea/node.js
-                sudo apt-get -y update
+
                 # DEBIAN_FRONTEND=noninteractive is required for silent mysql-server installation
-                sudo DEBIAN_FRONTEND=noninteractive apt-get -y install $APT_PKGS
-                sudo npm install coffee-script
+                export DEBIAN_FRONTEND=noninteractive
+
+                # add repositories
+                cat $APT_REPOS_FILE | xargs -n 1 sudo add-apt-repository -y
+                sudo apt-get -y update
+
+                # install packages listed in APT_PKGS_FILE
+                cat $APT_PKGS_FILE | xargs sudo apt-get -y install
+
                 clone_repos
                 ;;
             *)
@@ -272,7 +279,7 @@ output "Installing rvm and ruby"
 curl -sL get.rvm.io | bash -s -- --version 1.15.7
 source $RUBY_DIR/scripts/rvm
 # skip the intro
-LESS="-E" rvm install $RUBY_VER
+LESS="-E" rvm install $RUBY_VER --with-readline
 output "Installing gem bundler"
 gem install bundler
 output "Installing ruby packages"

doc/development.md

@@ -67,6 +67,15 @@ To run a single nose test:
 
 Very handy: if you uncomment the `--pdb` argument in `NOSE_ARGS` in `lms/envs/test.py`, it will drop you into pdb on error.  This lets you go up and down the stack and see what the values of the variables are.  Check out http://docs.python.org/library/pdb.html
 
+## Testing using queue servers
+
+When testing problems that use a queue server on AWS (e.g. sandbox-xqueue.edx.org), you'll need to run your server on your public IP, like so.
+
+`django-admin.py runserver --settings=lms.envs.dev --pythonpath=. 0.0.0.0:8000`
+
+When you connect to the LMS, you need to use the public ip.  Use `ifconfig` to figure out the numnber, and connect e.g. to `http://18.3.4.5:8000/`
+
+
 ## Content development
 
 If you change course content, while running the LMS in dev mode, it is unnecessary to restart to refresh the modulestore.  

doc/xml-format.md

@@ -418,6 +418,10 @@ If you want to customize the courseware tabs displayed for your course, specify
 * "external_link".  Parameters "name", "link".
 * "textbooks".  No parameters--generates tab names from book titles.
 * "progress".  Parameter "name".
+* "static_tab".  Parameters "name", 'url_slug'--will look for tab contents in
+                       'tabs/{course_url_name}/{tab url_slug}.html'
+* "staff_grading". No parameters.  If specified, displays the staff grading tab for instructors.
+
 
 # Tips for content developers
 
@@ -429,9 +433,7 @@ before the week 1 material to make it easy to find in the file.
 
 * Come up with a consistent pattern for url_names, so that it's easy to know where to look for any piece of content.  It will also help to come up with a standard way of splitting your content files.  As a point of departure, we suggest splitting chapters, sequences, html, and problems into separate files.
 
-* A heads up: our content management system will allow you to develop content through a web browser, but will be backed by this same xml at first.  Once that happens, every element will be in its own file to make access and updates faster.
-
-* Prefer the most "semantic" name for containers: e.g., use problemset rather than vertical for a problem set.  That way, if we decide to display problem sets differently, we don't have to change the xml.
+* Prefer the most "semantic" name for containers: e.g., use problemset rather than sequential for a problem set.  That way, if we decide to display problem sets differently, we don't have to change the xml.
 
 # Other file locations (info and about)
 

docs/source/capa.rst

@@ -1,9 +1,15 @@
 *******************************************
 Capa module
 *******************************************
+Contents:
 
 .. module:: capa
 
+.. toctree::
+    :maxdepth: 2
+
+    chem.rst
+
 Calc
 ====
 

docs/source/chem.rst

@@ -0,0 +1,69 @@
+*******************************************
+Chem module
+*******************************************
+
+.. module:: chem
+
+Miller
+======
+
+.. automodule:: capa.chem.miller
+    :members:
+    :show-inheritance:
+
+UI part and inputtypes
+----------------------
+Miller module is used in the system in crystallography problems.
+Crystallography is a class in :mod:`capa` inputtypes module.
+It uses *crystallography.html* for rendering and **crystallography.js**
+for UI part.
+
+Documentation from **crystallography.js**::
+
+    For a crystallographic problem of the type
+
+     Given a plane definition via miller indexes, specify it by plotting points on the edges
+     of a 3D cube. Additionally, select the correct Bravais cubic lattice type depending on the
+     physical crystal mentioned in the problem.
+
+    we create a graph which contains a cube, and a 3D Cartesian coordinate system. The interface
+    will allow to plot 3 points anywhere along the edges of the cube, and select which type of
+    Bravais lattice should be displayed along with the basic cube outline.
+
+    When 3 points are successfully plotted, an intersection of the resulting plane (defined by
+    the 3 plotted points), and the cube, will be automatically displayed for clarity.
+
+    After lotting the three points, it is possible to continue plotting additional points. By
+    doing so, the point that was plotted first (from the three that already exist), will be
+    removed, and the new point will be added. The intersection of the resulting new plane and
+    the cube will be redrawn.
+
+    The UI has been designed in such a way, that the user is able to determine which point will
+    be removed next (if adding a new point). This is achieved via filling the to-be-removed point
+    with a different color.
+
+
+
+Chemcalc
+========
+
+.. automodule:: capa.chem.chemcalc
+    :members:
+    :show-inheritance:
+
+Chemtools
+=========
+
+.. automodule:: capa.chem.chemtools
+    :members:
+    :show-inheritance:
+
+
+Tests
+=====
+
+.. automodule:: capa.chem.tests
+    :members:
+    :show-inheritance:
+
+

jenkins/test.sh

@@ -0,0 +1,48 @@
+#! /bin/bash
+
+set -e
+set -x
+
+function github_status {
+    gcli status create mitx mitx $GIT_COMMIT \
+         --params=$1 \
+                  target_url:$BUILD_URL \
+                  description:"Build #$BUILD_NUMBER $2" \
+         -f csv
+}
+
+function github_mark_failed_on_exit {
+    trap '[ $? == "0" ] || github_status state:failure "failed"' EXIT
+}
+
+github_mark_failed_on_exit
+github_status state:pending "is running"
+
+# Reset the submodule, in case it changed
+git submodule foreach 'git reset --hard HEAD'
+
+# Set the IO encoding to UTF-8 so that askbot will start
+export PYTHONIOENCODING=UTF-8
+
+GIT_BRANCH=${GIT_BRANCH/HEAD/master}
+
+pip install -q -r pre-requirements.txt
+pip install -q -r test-requirements.txt
+yes w | pip install -q -r requirements.txt
+
+rake clobber
+TESTS_FAILED=0
+# Don't run the studio tests until feature/cale/cms-master is merged in
+# rake test_cms[false] || TESTS_FAILED=1
+rake test_lms[false] || TESTS_FAILED=1
+rake test_common/lib/capa || TESTS_FAILED=1
+rake test_common/lib/xmodule || TESTS_FAILED=1
+rake phantomjs_jasmine_lms || true
+# Don't run the studio tests until feature/cale/cms-master is merged in
+# rake phantomjs_jasmine_cms || true
+rake coverage:xml coverage:html
+
+[ $TESTS_FAILED == '0' ]
+rake autodeploy_properties
+
+github_status state:success "passed"

jenkins/test_edge.sh

@@ -1,29 +0,0 @@
-#! /bin/bash
-
-set -e
-set -x
-
-# Reset the submodule, in case it changed
-git submodule foreach 'git reset --hard HEAD'
-
-# Set the IO encoding to UTF-8 so that askbot will start
-export PYTHONIOENCODING=UTF-8
-
-GIT_BRANCH=${GIT_BRANCH/HEAD/master}
-
-pip install -q -r pre-requirements.txt
-yes w | pip install -q -r requirements.txt
-[ ! -d askbot ] || pip install -q -r askbot/askbot_requirements.txt
-
-rake clobber
-TESTS_FAILED=0
-rake test_cms[false] || TESTS_FAILED=1
-rake test_lms[false] || TESTS_FAILED=1
-rake test_common/lib/capa || TESTS_FAILED=1
-rake test_common/lib/xmodule || TESTS_FAILED=1
-rake phantomjs_jasmine_lms || true
-rake phantomjs_jasmine_cms || true
-rake coverage:xml coverage:html
-
-[ $TESTS_FAILED == '0' ]
-rake autodeploy_properties

jenkins/test_lms.sh

@@ -1,27 +0,0 @@
-#! /bin/bash
-
-set -e
-set -x
-
-# Reset the submodule, in case it changed
-git submodule foreach 'git reset --hard HEAD'
-
-# Set the IO encoding to UTF-8 so that askbot will start
-export PYTHONIOENCODING=UTF-8
-
-GIT_BRANCH=${GIT_BRANCH/HEAD/master}
-
-pip install -q -r pre-requirements.txt
-yes w | pip install -q -r requirements.txt
-[ ! -d askbot ] || pip install -q -r askbot/askbot_requirements.txt
-
-rake clobber
-TESTS_FAILED=0
-rake test_lms[false] || TESTS_FAILED=1
-rake test_common/lib/capa || TESTS_FAILED=1
-rake test_common/lib/xmodule || TESTS_FAILED=1
-rake phantomjs_jasmine_lms || true
-rake coverage:xml coverage:html
-
-[ $TESTS_FAILED == '0' ]
-rake autodeploy_properties

lms/djangoapps/courseware/access.py

@@ -34,7 +34,8 @@ def has_access(user, obj, action):
 
     user: a Django user object. May be anonymous.
 
-    obj: The object to check access for.  For now, a module or descriptor.
+    obj: The object to check access for.  A module, descriptor, location, or
+                    certain special strings (e.g. 'global')
 
     action: A string specifying the action that the client is trying to perform.
 

lms/djangoapps/courseware/module_render.py

@@ -1,4 +1,3 @@
-import hashlib
 import json
 import logging
 import pyparsing
@@ -20,6 +19,7 @@ from mitxmako.shortcuts import render_to_string
 from models import StudentModule, StudentModuleCache
 from psychometrics.psychoanalyze import make_psychometrics_data_update_handler
 from static_replace import replace_urls
+from student.models import unique_id_for_user
 from xmodule.errortracker import exc_info_to_str
 from xmodule.exceptions import NotFoundError
 from xmodule.modulestore import Location
@@ -152,12 +152,6 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
     if not has_access(user, descriptor, 'load'):
         return None
 
-    # Anonymized student identifier
-    h = hashlib.md5()
-    h.update(settings.SECRET_KEY)
-    h.update(str(user.id))
-    anonymous_student_id = h.hexdigest()
-
     # Only check the cache if this module can possibly have state
     instance_module = None
     shared_module = None
@@ -230,7 +224,8 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
                           # by the replace_static_urls code below
                           replace_urls=replace_urls,
                           node_path=settings.NODE_PATH,
-                          anonymous_student_id=anonymous_student_id,
+                          anonymous_student_id=unique_id_for_user(user),
+                          course_id=course_id,
                           )
     # pass position specified in URL to module through ModuleSystem
     system.set('position', position)

lms/djangoapps/courseware/tabs.py

@@ -36,7 +36,7 @@ CourseTab = namedtuple('CourseTab', 'name link is_active')
 #    wrong.  (e.g. "is there a 'name' field?).  Validators can assume
 #    that the type field is valid.
 #
-#  - a function that takes a config, a user, and a course, and active_page and
+#  - a function that takes a config, a user, and a course, an active_page and
 #    return a list of CourseTabs.  (e.g. "return a CourseTab with specified
 #    name, link to courseware, and is_active=True/False").  The function can
 #    assume that it is only called with configs of the appropriate type that
@@ -97,6 +97,14 @@ def _textbooks(tab, user, course, active_page):
                 for index, textbook in enumerate(course.textbooks)]
     return []
 
+
+def _staff_grading(tab, user, course, active_page):
+    if has_access(user, course, 'staff'):
+        link = reverse('staff_grading', args=[course.id])
+        return [CourseTab('Staff grading', link, active_page == "staff_grading")]
+    return []
+
+
 #### Validators
 
 
@@ -132,6 +140,7 @@ VALID_TAB_TYPES = {
     'textbooks': TabImpl(null_validator, _textbooks),
     'progress': TabImpl(need_name, _progress),
     'static_tab': TabImpl(key_checker(['name', 'url_slug']), _static_tab),
+    'staff_grading': TabImpl(null_validator, _staff_grading),
     }
 
 

lms/djangoapps/courseware/tests/tests.py

@@ -1,21 +1,16 @@
-import copy
+import logging
+log = logging.getLogger("mitx." + __name__)
+
 import json
-import os
-import sys
 import time
 
-from nose import SkipTest
-from path import path
-from pprint import pprint
 from urlparse import urlsplit, urlunsplit
 
 from django.contrib.auth.models import User, Group
-from django.core.handlers.wsgi import WSGIRequest
 from django.test import TestCase
-from django.test.client import Client, RequestFactory
+from django.test.client import RequestFactory
 from django.conf import settings
 from django.core.urlresolvers import reverse
-from mock import patch, Mock
 from override_settings import override_settings
 
 import xmodule.modulestore.django
@@ -26,9 +21,11 @@ from courseware.access import _course_staff_group_name
 from courseware.models import StudentModuleCache
 
 from student.models import Registration
+from xmodule.error_module import ErrorDescriptor
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore import Location
 from xmodule.modulestore.xml_importer import import_from_xml
+from xmodule.modulestore.xml import XMLModuleStore
 from xmodule.timeparse import stringify_time
 
 def parse_json(response):
@@ -45,26 +42,6 @@ def registration(email):
     '''look up registration object by email'''
     return Registration.objects.get(user__email=email)
 
-
-# A bit of a hack--want mongo modulestore for these tests, until
-# jump_to works with the xmlmodulestore or we have an even better solution
-# NOTE: this means this test requires mongo to be running.
-
-def mongo_store_config(data_dir):
-    return {
-    'default': {
-        'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
-        'OPTIONS': {
-            'default_class': 'xmodule.raw_module.RawDescriptor',
-            'host': 'localhost',
-            'db': 'xmodule',
-            'collection': 'modulestore',
-            'fs_root': data_dir,
-            'render_template': 'mitxmako.shortcuts.render_to_string',
-        }
-    }
-}
-
 def xml_store_config(data_dir):
     return {
     'default': {
@@ -76,14 +53,9 @@ def xml_store_config(data_dir):
     }
 }
 
-
 TEST_DATA_DIR = settings.COMMON_TEST_DATA_ROOT
-TEST_DATA_MONGO_MODULESTORE = mongo_store_config(TEST_DATA_DIR)
 TEST_DATA_XML_MODULESTORE = xml_store_config(TEST_DATA_DIR)
 
-REAL_DATA_DIR = settings.GITHUB_REPO_ROOT
-REAL_DATA_MODULESTORE = mongo_store_config(REAL_DATA_DIR)
-
 class ActivateLoginTestCase(TestCase):
     '''Check that we can activate and log in'''
 
@@ -221,22 +193,43 @@ class PageLoader(ActivateLoginTestCase):
 
     def check_for_get_code(self, code, url):
         """
-        Check that we got the expected code.  Hacks around our broken 404
-        handling.
+        Check that we got the expected code when accessing url via GET.
+        Returns the response.
         """
         resp = self.client.get(url)
         self.assertEqual(resp.status_code, code,
                          "got code {0} for url '{1}'. Expected code {2}"
                          .format(resp.status_code, url, code))
+        return resp
+
+
+    def check_for_post_code(self, code, url, data={}):
+        """
+        Check that we got the expected code when accessing url via POST.
+        Returns the response.
+        """
+        resp = self.client.post(url, data)
+        self.assertEqual(resp.status_code, code,
+                         "got code {0} for url '{1}'. Expected code {2}"
+                         .format(resp.status_code, url, code))
+        return resp
+
 
 
     def check_pages_load(self, course_name, data_dir, modstore):
         """Make all locations in course load"""
         print "Checking course {0} in {1}".format(course_name, data_dir)
-        import_from_xml(modstore, data_dir, [course_name])
+        default_class='xmodule.hidden_module.HiddenDescriptor'
+        load_error_modules=True
+        module_store = XMLModuleStore(
+                                      data_dir,
+                                      default_class=default_class,
+                                      course_dirs=[course_name],
+                                      load_error_modules=load_error_modules,
+                                      )
 
-        # enroll in the course before trying to access pages
-        courses = modstore.get_courses()
+       # enroll in the course before trying to access pages
+        courses = module_store.get_courses()
         self.assertEqual(len(courses), 1)
         course = courses[0]
         self.enroll(course)
@@ -245,36 +238,55 @@ class PageLoader(ActivateLoginTestCase):
         n = 0
         num_bad = 0
         all_ok = True
-        for descriptor in modstore.get_items(
-                Location(None, None, None, None, None)):
+        for descriptor in module_store.modules[course_id].itervalues(): 
             n += 1
             print "Checking ", descriptor.location.url()
             #print descriptor.__class__, descriptor.location
             resp = self.client.get(reverse('jump_to',
                                    kwargs={'course_id': course_id,
-                                           'location': descriptor.location.url()}))
+                                           'location': descriptor.location.url()}), follow=True)
+            # check status codes first
             msg = str(resp.status_code)
+            if resp.status_code != 200:
+                msg = "ERROR " + msg  + ": " + descriptor.location.url()
+                all_ok = False
+                num_bad += 1
+            elif resp.redirect_chain[0][1] != 302:
+                msg = "ERROR on redirect from " + descriptor.location.url()
+                all_ok = False
+                num_bad += 1
 
-            if resp.status_code != 302:
-                msg = "ERROR " + msg
+            # check content to make sure there were no rendering failures
+            content = resp.content
+            if content.find("this module is temporarily unavailable")>=0:
+                msg = "ERROR unavailable module " 
+                all_ok = False
+                num_bad += 1
+            elif isinstance(descriptor, ErrorDescriptor):
+                msg = "ERROR error descriptor loaded: " 
+                msg = msg + descriptor.definition['data']['error_msg']
                 all_ok = False
                 num_bad += 1
             print msg
             self.assertTrue(all_ok)  # fail fast
 
         print "{0}/{1} good".format(n - num_bad, n)
+        log.info( "{0}/{1} good".format(n - num_bad, n))
         self.assertTrue(all_ok)
 
 
-@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE)
+
+@override_settings(MODULESTORE=TEST_DATA_XML_MODULESTORE)
 class TestCoursesLoadTestCase(PageLoader):
     '''Check that all pages in test courses load properly'''
 
     def setUp(self):
         ActivateLoginTestCase.setUp(self)
         xmodule.modulestore.django._MODULESTORES = {}
-        xmodule.modulestore.django.modulestore().collection.drop()
-
+#        xmodule.modulestore.django.modulestore().collection.drop()
+#        store = xmodule.modulestore.django.modulestore()
+        # is there a way to empty the store?
+        
     def test_toy_course_loads(self):
         self.check_pages_load('toy', TEST_DATA_DIR, modulestore())
 
@@ -288,14 +300,10 @@ class TestNavigation(PageLoader):
 
     def setUp(self):
         xmodule.modulestore.django._MODULESTORES = {}
-        courses = modulestore().get_courses()
 
-        def find_course(course_id):
-            """Assumes the course is present"""
-            return [c for c in courses if c.id==course_id][0]
-
-        self.full = find_course("edX/full/6.002_Spring_2012")
-        self.toy = find_course("edX/toy/2012_Fall")
+        # Assume courses are there
+        self.full = modulestore().get_course("edX/full/6.002_Spring_2012")
+        self.toy = modulestore().get_course("edX/toy/2012_Fall")
 
         # Create two accounts
         self.student = 'view@test.com'
@@ -346,14 +354,9 @@ class TestViewAuth(PageLoader):
 
     def setUp(self):
         xmodule.modulestore.django._MODULESTORES = {}
-        courses = modulestore().get_courses()
 
-        def find_course(course_id):
-            """Assumes the course is present"""
-            return [c for c in courses if c.id==course_id][0]
-
-        self.full = find_course("edX/full/6.002_Spring_2012")
-        self.toy = find_course("edX/toy/2012_Fall")
+        self.full = modulestore().get_course("edX/full/6.002_Spring_2012")
+        self.toy = modulestore().get_course("edX/toy/2012_Fall")
 
         # Create two accounts
         self.student = 'view@test.com'
@@ -615,35 +618,6 @@ class TestViewAuth(PageLoader):
         self.unenroll(self.toy)
         self.assertTrue(self.try_enroll(self.toy))
 
-
-@override_settings(MODULESTORE=REAL_DATA_MODULESTORE)
-class RealCoursesLoadTestCase(PageLoader):
-    '''Check that all pages in real courses load properly'''
-
-    def setUp(self):
-        ActivateLoginTestCase.setUp(self)
-        xmodule.modulestore.django._MODULESTORES = {}
-        xmodule.modulestore.django.modulestore().collection.drop()
-
-    def test_real_courses_loads(self):
-        '''See if any real courses are available at the REAL_DATA_DIR.
-        If they are, check them.'''
-
-        # TODO: Disabled test for now..  Fix once things are cleaned up.
-        raise SkipTest
-        # TODO: adjust staticfiles_dirs
-        if not os.path.isdir(REAL_DATA_DIR):
-            # No data present.  Just pass.
-            return
-
-        courses = [course_dir for course_dir in os.listdir(REAL_DATA_DIR)
-                   if os.path.isdir(REAL_DATA_DIR / course_dir)]
-        for course in courses:
-            self.check_pages_load(course, REAL_DATA_DIR, modulestore())
-
-
-    # ========= TODO: check ajax interaction here too?
-
 @override_settings(MODULESTORE=TEST_DATA_XML_MODULESTORE)
 class TestCourseGrader(PageLoader):
     """Check that a course gets graded properly"""
@@ -660,46 +634,46 @@ class TestCourseGrader(PageLoader):
             return [c for c in courses if c.id==course_id][0]
 
         self.graded_course = find_course("edX/graded/2012_Fall")
-        
+
         # create a test student
         self.student = 'view@test.com'
         self.password = 'foo'
         self.create_account('u1', self.student, self.password)
         self.activate_user(self.student)
         self.enroll(self.graded_course)
-        
+
         self.student_user = user(self.student)
-        
+
         self.factory = RequestFactory()
-    
+
     def get_grade_summary(self):
         student_module_cache = StudentModuleCache.cache_for_descriptor_descendents(
             self.graded_course.id, self.student_user, self.graded_course)
-        
-        fake_request = self.factory.get(reverse('progress',
-                                       kwargs={'course_id': self.graded_course.id}))
-        
-        return grades.grade(self.student_user, fake_request, 
-                            self.graded_course, student_module_cache)
-    
-    def get_homework_scores(self):
-        return self.get_grade_summary()['totaled_scores']['Homework']
-    
-    def get_progress_summary(self):
-        student_module_cache = StudentModuleCache.cache_for_descriptor_descendents(
-            self.graded_course.id, self.student_user, self.graded_course)
-        
+
         fake_request = self.factory.get(reverse('progress',
                                        kwargs={'course_id': self.graded_course.id}))
 
-        progress_summary = grades.progress_summary(self.student_user, fake_request, 
+        return grades.grade(self.student_user, fake_request,
+                            self.graded_course, student_module_cache)
+
+    def get_homework_scores(self):
+        return self.get_grade_summary()['totaled_scores']['Homework']
+
+    def get_progress_summary(self):
+        student_module_cache = StudentModuleCache.cache_for_descriptor_descendents(
+            self.graded_course.id, self.student_user, self.graded_course)
+
+        fake_request = self.factory.get(reverse('progress',
+                                       kwargs={'course_id': self.graded_course.id}))
+
+        progress_summary = grades.progress_summary(self.student_user, fake_request,
                                                    self.graded_course, student_module_cache)
         return progress_summary
-        
+
     def check_grade_percent(self, percent):
         grade_summary = self.get_grade_summary()
-        self.assertEqual(percent, grade_summary['percent'])
-    
+        self.assertEqual(grade_summary['percent'], percent)
+
     def submit_question_answer(self, problem_url_name, responses):
         """
         The field names of a problem are hard to determine. This method only works
@@ -709,96 +683,96 @@ class TestCourseGrader(PageLoader):
         input_i4x-edX-graded-problem-H1P3_2_2
         """
         problem_location = "i4x://edX/graded/problem/{0}".format(problem_url_name)
-        
-        modx_url = reverse('modx_dispatch', 
+
+        modx_url = reverse('modx_dispatch',
                             kwargs={
                                 'course_id' : self.graded_course.id,
                                 'location' : problem_location,
                                 'dispatch' : 'problem_check', }
                           )
-        
+
         resp = self.client.post(modx_url, {
             'input_i4x-edX-graded-problem-{0}_2_1'.format(problem_url_name): responses[0],
             'input_i4x-edX-graded-problem-{0}_2_2'.format(problem_url_name): responses[1],
             })
         print "modx_url" , modx_url, "responses" , responses
         print "resp" , resp
-        
+
         return resp
-    
+
     def problem_location(self, problem_url_name):
         return "i4x://edX/graded/problem/{0}".format(problem_url_name)
-    
+
     def reset_question_answer(self, problem_url_name):
         problem_location = self.problem_location(problem_url_name)
-        
-        modx_url = reverse('modx_dispatch', 
+
+        modx_url = reverse('modx_dispatch',
                             kwargs={
                                 'course_id' : self.graded_course.id,
                                 'location' : problem_location,
                                 'dispatch' : 'problem_reset', }
                           )
-        
+
         resp = self.client.post(modx_url)
-        return resp    
-     
+        return resp
+
     def test_get_graded(self):
         #### Check that the grader shows we have 0% in the course
         self.check_grade_percent(0)
-        
+
         #### Submit the answers to a few problems as ajax calls
         def earned_hw_scores():
             """Global scores, each Score is a Problem Set"""
             return [s.earned for s in self.get_homework_scores()]
-        
+
         def score_for_hw(hw_url_name):
             hw_section = [section for section
                           in self.get_progress_summary()[0]['sections']
                           if section.get('url_name') == hw_url_name][0]
             return [s.earned for s in hw_section['scores']]
-        
+
         # Only get half of the first problem correct
         self.submit_question_answer('H1P1', ['Correct', 'Incorrect'])
         self.check_grade_percent(0.06)
         self.assertEqual(earned_hw_scores(), [1.0, 0, 0]) # Order matters
         self.assertEqual(score_for_hw('Homework1'), [1.0, 0.0])
-        
+
         # Get both parts of the first problem correct
         self.reset_question_answer('H1P1')
         self.submit_question_answer('H1P1', ['Correct', 'Correct'])
         self.check_grade_percent(0.13)
         self.assertEqual(earned_hw_scores(), [2.0, 0, 0])
         self.assertEqual(score_for_hw('Homework1'), [2.0, 0.0])
-        
+
         # This problem is shown in an ABTest
         self.submit_question_answer('H1P2', ['Correct', 'Correct'])
         self.check_grade_percent(0.25)
         self.assertEqual(earned_hw_scores(), [4.0, 0.0, 0])
-        self.assertEqual(score_for_hw('Homework1'), [2.0, 2.0])        
-        
+        self.assertEqual(score_for_hw('Homework1'), [2.0, 2.0])
+
         # This problem is hidden in an ABTest. Getting it correct doesn't change total grade
         self.submit_question_answer('H1P3', ['Correct', 'Correct'])
         self.check_grade_percent(0.25)
         self.assertEqual(score_for_hw('Homework1'), [2.0, 2.0])
-        
+
         # On the second homework, we only answer half of the questions.
         # Then it will be dropped when homework three becomes the higher percent
         # This problem is also weighted to be 4 points (instead of default of 2)
-        # If the problem was unweighted the percent would have been 0.38 so we 
+        # If the problem was unweighted the percent would have been 0.38 so we
         # know it works.
         self.submit_question_answer('H2P1', ['Correct', 'Correct'])
         self.check_grade_percent(0.42)
-        self.assertEqual(earned_hw_scores(), [4.0, 4.0, 0])        
-        
+        self.assertEqual(earned_hw_scores(), [4.0, 4.0, 0])
+
         # Third homework
         self.submit_question_answer('H3P1', ['Correct', 'Correct'])
         self.check_grade_percent(0.42) # Score didn't change
-        self.assertEqual(earned_hw_scores(), [4.0, 4.0, 2.0])        
-        
+        self.assertEqual(earned_hw_scores(), [4.0, 4.0, 2.0])
+
         self.submit_question_answer('H3P2', ['Correct', 'Correct'])
         self.check_grade_percent(0.5) # Now homework2 dropped. Score changes
-        self.assertEqual(earned_hw_scores(), [4.0, 4.0, 4.0])                
-        
+        self.assertEqual(earned_hw_scores(), [4.0, 4.0, 4.0])
+
         # Now we answer the final question (worth half of the grade)
         self.submit_question_answer('FinalQuestion', ['Correct', 'Correct'])
         self.check_grade_percent(1.0) # Hooray! We got 100%

lms/djangoapps/courseware/views.py

@@ -293,7 +293,6 @@ def index(request, course_id, chapter=None, section=None,
 
     return result
 
-
 @ensure_csrf_cookie
 def jump_to(request, course_id, location):
     '''
@@ -318,12 +317,18 @@ def jump_to(request, course_id, location):
     except NoPathToItem:
         raise Http404("This location is not in any class: {0}".format(location))
 
+    # choose the appropriate view (and provide the necessary args) based on the 
+    # args provided by the redirect.
     # Rely on index to do all error handling and access control.
-    return redirect('courseware_position',
-                    course_id=course_id,
-                    chapter=chapter,
-                    section=section,
-                    position=position)
+    if chapter is None:
+        return redirect('courseware', course_id=course_id)
+    elif section is None:
+        return redirect('courseware_chapter', course_id=course_id, chapter=chapter)
+    elif position is None:
+        return redirect('courseware_section', course_id=course_id, chapter=chapter, section=section)
+    else:
+        return redirect('courseware_position', course_id=course_id, chapter=chapter, section=section, position=position)
+        
 @ensure_csrf_cookie
 def course_info(request, course_id):
     """

lms/djangoapps/django_comment_client/models.py

@@ -1,9 +1,15 @@
-from django.db import models
-from django.contrib.auth.models import User
 import logging
 
+from django.db import models
+from django.contrib.auth.models import User
+
 from courseware.courses import get_course_by_id
 
+FORUM_ROLE_ADMINISTRATOR = 'Administrator'
+FORUM_ROLE_MODERATOR = 'Moderator'
+FORUM_ROLE_COMMUNITY_TA = 'Community TA'
+FORUM_ROLE_STUDENT = 'Student'
+
 class Role(models.Model):
     name = models.CharField(max_length=30, null=False, blank=False)
     users = models.ManyToManyField(User, related_name="roles")
@@ -15,8 +21,8 @@ class Role(models.Model):
     def inherit_permissions(self, role): # TODO the name of this method is a little bit confusing,
                                          # since it's one-off and doesn't handle inheritance later
         if role.course_id and role.course_id != self.course_id:
-            logging.warning("%s cannot inheret permissions from %s due to course_id inconsistency" % 
-                            (self, role))
+            logging.warning("{0} cannot inherit permissions from {1} due to course_id inconsistency", \
+                            self, role)
         for per in role.permissions.all():
             self.add_permission(per)
 
@@ -25,10 +31,10 @@ class Role(models.Model):
 
     def has_permission(self, permission):
         course = get_course_by_id(self.course_id)
-        if self.name == "Student" and \
+        if self.name == FORUM_ROLE_STUDENT and \
            (permission.startswith('edit') or permission.startswith('update') or permission.startswith('create')) and \
            (not course.forum_posts_allowed):
-           return False
+            return False
         
         return self.permissions.filter(name=permission).exists()
 

lms/djangoapps/django_comment_client/utils.py

@@ -1,28 +1,22 @@
 import time
-import logging
 from collections import defaultdict
-from importlib import import_module
+import logging
+import time
+import urllib
+
+from django.contrib.auth.models import User
+from django.core.urlresolvers import reverse
+from django.db import connection
+from django.http import HttpResponse
+from django.utils import simplejson
+from django_comment_client.models import Role
+from django_comment_client.permissions import check_permissions_by_view
+from mitxmako import middleware
+import pystache_custom as pystache
 
-from courseware.models import StudentModuleCache
-from courseware.module_render import get_module
 from xmodule.modulestore import Location
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.search import path_to_location
-from django.http import HttpResponse
-from django.utils import simplejson
-from django.db import connection
-from django.conf import settings
-from django.core.urlresolvers import reverse
-from django.contrib.auth.models import User
-from django_comment_client.permissions import check_permissions_by_view
-from django_comment_client.models import Role
-from mitxmako import middleware
-
-import logging
-import operator
-import itertools
-import urllib
-import pystache_custom as pystache
 
 
 # TODO these should be cached via django's caching rather than in-memory globals
@@ -48,9 +42,16 @@ def get_role_ids(course_id):
     staff = list(User.objects.filter(is_staff=True).values_list('id', flat=True))
     roles_with_ids = {'Staff': staff}
     for role in roles:
-      roles_with_ids[role.name] = list(role.users.values_list('id', flat=True))
+        roles_with_ids[role.name] = list(role.users.values_list('id', flat=True))
     return roles_with_ids
 
+def has_forum_access(uname, course_id, rolename):
+    try:
+        role = Role.objects.get(name=rolename, course_id=course_id)
+    except Role.DoesNotExist:
+        return False
+    return role.users.filter(username=uname).exists()
+
 def get_full_modules():
     global _FULLMODULES
     if not _FULLMODULES:
@@ -133,8 +134,6 @@ def initialize_discussion_info(course):
         return
 
     course_id = course.id
-    url_course_id = course_id.replace('/', '_').replace('.', '_')
-
     all_modules = get_full_modules()[course_id]
 
     discussion_id_map = {}

lms/djangoapps/instructor/grading.py

@@ -0,0 +1,25 @@
+"""
+LMS part of instructor grading:
+
+- views + ajax handling
+- calls the instructor grading service
+"""
+
+import json
+import logging
+
+log = logging.getLogger(__name__)
+
+
+class StaffGrading(object):
+    """
+    Wrap up functionality for staff grading of submissions--interface exposes get_html, ajax views.
+    """
+    def __init__(self, course):
+        self.course = course
+
+    def get_html(self):
+        return "<b>Instructor grading!</b>"
+        # context = {}
+        # return render_to_string('courseware/instructor_grading_view.html', context)
+

lms/djangoapps/instructor/staff_grading_service.py

@@ -0,0 +1,358 @@
+"""
+This module provides views that proxy to the staff grading backend service.
+"""
+
+import json
+import logging
+import requests
+from requests.exceptions import RequestException, ConnectionError, HTTPError
+import sys
+
+from django.conf import settings
+from django.http import HttpResponse, Http404
+
+from courseware.access import has_access
+from util.json_request import expect_json
+from xmodule.course_module import CourseDescriptor
+
+log = logging.getLogger(__name__)
+
+
+class GradingServiceError(Exception):
+    pass
+
+
+class MockStaffGradingService(object):
+    """
+    A simple mockup of a staff grading service, testing.
+    """
+    def __init__(self):
+        self.cnt = 0
+
+    def get_next(self,course_id, location, grader_id):
+        self.cnt += 1
+        return json.dumps({'success': True,
+                           'submission_id': self.cnt,
+                           'submission': 'Test submission {cnt}'.format(cnt=self.cnt),
+                           'num_graded': 3,
+                           'min_for_ml': 5,
+                           'num_pending': 4,
+                           'prompt': 'This is a fake prompt',
+                           'ml_error_info': 'ML info',
+                           'max_score': 2 + self.cnt % 3,
+                           'rubric': 'A rubric'})
+
+    def save_grade(self, course_id, grader_id, submission_id, score, feedback):
+        return self.get_next(course_id, 'fake location', grader_id)
+
+
+class StaffGradingService(object):
+    """
+    Interface to staff grading backend.
+    """
+    def __init__(self, config):
+        self.username = config['username']
+        self.password = config['password']
+        self.url = config['url']
+
+        self.login_url = self.url + '/login/'
+        self.get_next_url = self.url + '/get_next_submission/'
+        self.save_grade_url = self.url + '/save_grade/'
+        self.get_problem_list_url = self.url + '/get_problem_list/'
+
+        self.session = requests.session()
+
+
+    def _login(self):
+        """
+        Log into the staff grading service.
+
+        Raises requests.exceptions.HTTPError if something goes wrong.
+
+        Returns the decoded json dict of the response.
+        """
+        response = self.session.post(self.login_url,
+                                     {'username': self.username,
+                                      'password': self.password,})
+
+        response.raise_for_status()
+
+        return response.json
+
+
+    def _try_with_login(self, operation):
+        """
+        Call operation(), which should return a requests response object.  If
+        the request fails with a 'login_required' error, call _login() and try
+        the operation again.
+
+        Returns the result of operation().  Does not catch exceptions.
+        """
+        response = operation()
+        if (response.json
+            and response.json.get('success') == False
+            and response.json.get('error') == 'login_required'):
+            # apparrently we aren't logged in.  Try to fix that.
+            r = self._login()
+            if r and not r.get('success'):
+                log.warning("Couldn't log into staff_grading backend. Response: %s",
+                            r)
+            # try again
+            return operation()
+
+        return response
+
+    def get_problem_list(self, course_id, grader_id):
+        """
+        Get the list of problems for a given course.
+
+        Args:
+            course_id: course id that we want the problems of
+            grader_id: who is grading this?  The anonymous user_id of the grader.
+
+        Returns:
+            json string with the response from the service.  (Deliberately not
+            writing out the fields here--see the docs on the staff_grading view
+            in the grading_controller repo)
+
+        Raises:
+            GradingServiceError: something went wrong with the connection.
+        """
+        op = lambda: self.session.get(self.get_problem_list_url,
+                                        allow_redirects = False,
+                                        params={'course_id': course_id,
+                                            'grader_id': grader_id})
+        try:
+            r = self._try_with_login(op)
+        except (RequestException, ConnectionError, HTTPError) as err:
+            # reraise as promised GradingServiceError, but preserve stacktrace.
+            raise GradingServiceError, str(err), sys.exc_info()[2]
+
+        return r.text
+
+
+    def get_next(self, course_id, location, grader_id):
+        """
+        Get the next thing to grade.
+
+        Args:
+            course_id: the course that this problem belongs to
+            location: location of the problem that we are grading and would like the
+                next submission for
+            grader_id: who is grading this?  The anonymous user_id of the grader.
+
+        Returns:
+            json string with the response from the service.  (Deliberately not
+            writing out the fields here--see the docs on the staff_grading view
+            in the grading_controller repo)
+
+        Raises:
+            GradingServiceError: something went wrong with the connection.
+        """
+        op = lambda: self.session.get(self.get_next_url,
+                                      allow_redirects=False,
+                                      params={'location': location,
+                                              'grader_id': grader_id})
+        try:
+            r = self._try_with_login(op)
+        except (RequestException, ConnectionError, HTTPError) as err:
+            # reraise as promised GradingServiceError, but preserve stacktrace.
+            raise GradingServiceError, str(err), sys.exc_info()[2]
+
+        return r.text
+
+
+    def save_grade(self, course_id, grader_id, submission_id, score, feedback):
+        """
+        Save a score and feedback for a submission.
+
+        Returns:
+            json dict with keys
+                'success': bool
+                'error': error msg, if something went wrong.
+
+        Raises:
+            GradingServiceError if there's a problem connecting.
+        """
+        try:
+            data = {'course_id': course_id,
+                    'submission_id': submission_id,
+                    'score': score,
+                    'feedback': feedback,
+                    'grader_id': grader_id}
+
+            op = lambda: self.session.post(self.save_grade_url, data=data,
+                                           allow_redirects=False)
+            r = self._try_with_login(op)
+        except (RequestException, ConnectionError, HTTPError) as err:
+            # reraise as promised GradingServiceError, but preserve stacktrace.
+            raise GradingServiceError, str(err), sys.exc_info()[2]
+
+        return r.text
+
+# don't initialize until grading_service() is called--means that just
+# importing this file doesn't create objects that may not have the right config
+_service = None
+
+def grading_service():
+    """
+    Return a staff grading service instance--if settings.MOCK_STAFF_GRADING is True,
+    returns a mock one, otherwise a real one.
+
+    Caches the result, so changing the setting after the first call to this
+    function will have no effect.
+    """
+    global _service
+    if _service is not None:
+        return _service
+
+    if settings.MOCK_STAFF_GRADING:
+        _service = MockStaffGradingService()
+    else:
+        _service = StaffGradingService(settings.STAFF_GRADING_INTERFACE)
+
+    return _service
+
+def _err_response(msg):
+    """
+    Return a HttpResponse with a json dump with success=False, and the given error message.
+    """
+    return HttpResponse(json.dumps({'success': False, 'error': msg}),
+                        mimetype="application/json")
+
+
+def _check_access(user, course_id):
+    """
+    Raise 404 if user doesn't have staff access to course_id
+    """
+    course_location = CourseDescriptor.id_to_location(course_id)
+    if not has_access(user, course_location, 'staff'):
+        raise Http404
+
+    return
+
+
+def get_next(request, course_id):
+    """
+    Get the next thing to grade for course_id and with the location specified
+    in the .
+
+    Returns a json dict with the following keys:
+
+    'success': bool
+
+    'submission_id': a unique identifier for the submission, to be passed back
+                     with the grade.
+
+    'submission': the submission, rendered as read-only html for grading
+
+    'rubric': the rubric, also rendered as html.
+
+    'message': if there was no submission available, but nothing went wrong,
+            there will be a message field.
+
+    'error': if success is False, will have an error message with more info.
+    """
+    _check_access(request.user, course_id)
+
+    required = set(['location'])
+    if request.method != 'POST':
+        raise Http404
+    actual = set(request.POST.keys())
+    missing = required - actual
+    if len(missing) > 0:
+        return _err_response('Missing required keys {0}'.format(
+            ', '.join(missing)))
+    grader_id = request.user.id
+    p = request.POST
+    location = p['location']
+
+    return HttpResponse(_get_next(course_id, request.user.id, location),
+                        mimetype="application/json")
+
+
+def get_problem_list(request, course_id):
+    """
+    Get all the problems for the given course id
+    TODO: fill in all of this stuff
+    """
+    _check_access(request.user, course_id)
+    try:
+        response = grading_service().get_problem_list(course_id, request.user.id)
+        return HttpResponse(response,
+                mimetype="application/json")
+    except GradingServiceError:
+        log.exception("Error from grading service.  server url: {0}"
+                      .format(grading_service().url))
+        return HttpResponse(json.dumps({'success': False,
+                           'error': 'Could not connect to grading service'}))
+
+
+def _get_next(course_id, grader_id, location):
+    """
+    Implementation of get_next (also called from save_grade) -- returns a json string
+    """
+    try:
+        return grading_service().get_next(course_id, location, grader_id)
+    except GradingServiceError:
+        log.exception("Error from grading service.  server url: {0}"
+                      .format(grading_service().url))
+        return json.dumps({'success': False,
+                           'error': 'Could not connect to grading service'})
+
+
+@expect_json
+def save_grade(request, course_id):
+    """
+    Save the grade and feedback for a submission, and, if all goes well, return
+    the next thing to grade.
+
+    Expects the following POST parameters:
+    'score': int
+    'feedback': string
+    'submission_id': int
+
+    Returns the same thing as get_next, except that additional error messages
+    are possible if something goes wrong with saving the grade.
+    """
+    _check_access(request.user, course_id)
+
+    if request.method != 'POST':
+        raise Http404
+
+    required = set(['score', 'feedback', 'submission_id', 'location'])
+    actual = set(request.POST.keys())
+    log.debug(actual)
+    missing = required - actual
+    if len(missing) > 0:
+        return _err_response('Missing required keys {0}'.format(
+            ', '.join(missing)))
+
+    grader_id = request.user.id
+    p = request.POST
+    location = p['location']
+
+    try:
+        result_json = grading_service().save_grade(course_id,
+                                          grader_id,
+                                          p['submission_id'],
+                                          p['score'],
+                                          p['feedback'])
+    except GradingServiceError:
+        log.exception("Error saving grade")
+        return _err_response('Could not connect to grading service')
+
+    try:
+        result = json.loads(result_json)
+    except ValueError:
+        log.exception("save_grade returned broken json: %s", result_json)
+        return _err_response('Grading service returned mal-formatted data.')
+
+    if not result.get('success', False):
+        log.warning('Got success=False from grading service.  Response: %s', result_json)
+        return _err_response('Grading service failed')
+
+    # Ok, save_grade seemed to work.  Get the next submission to grade.
+    return HttpResponse(_get_next(course_id, grader_id, location),
+                        mimetype="application/json")
+

lms/djangoapps/instructor/tests.py

@@ -10,19 +10,26 @@ django-admin.py test --settings=lms.envs.test --pythonpath=. lms/djangoapps/inst
 
 import courseware.tests.tests as ct
 
+import json
+
 from nose import SkipTest
 from mock import patch, Mock
+
 from override_settings import override_settings
 
 # Need access to internal func to put users in the right group
-from courseware.access import _course_staff_group_name
-from django.contrib.auth.models import User, Group
-from django.conf import settings
+from django.contrib.auth.models import Group
+
 from django.core.urlresolvers import reverse
+from django_comment_client.models import Role, FORUM_ROLE_ADMINISTRATOR, \
+    FORUM_ROLE_MODERATOR, FORUM_ROLE_COMMUNITY_TA, FORUM_ROLE_STUDENT
+from django_comment_client.utils import has_forum_access
 
-import xmodule.modulestore.django
-
+from instructor import staff_grading_service
+from courseware.access import _course_staff_group_name
+import courseware.tests.tests as ct
 from xmodule.modulestore.django import modulestore
+import xmodule.modulestore.django
 
 
 @override_settings(MODULESTORE=ct.TEST_DATA_XML_MODULESTORE)
@@ -31,16 +38,83 @@ class TestInstructorDashboardGradeDownloadCSV(ct.PageLoader):
     Check for download of csv
     '''
 
+    def setUp(self):
+        xmodule.modulestore.django._MODULESTORES = {}
+
+        self.full = modulestore().get_course("edX/full/6.002_Spring_2012")
+        self.toy = modulestore().get_course("edX/toy/2012_Fall")
+
+        # Create two accounts
+        self.student = 'view@test.com'
+        self.instructor = 'view2@test.com'
+        self.password = 'foo'
+        self.create_account('u1', self.student, self.password)
+        self.create_account('u2', self.instructor, self.password)
+        self.activate_user(self.student)
+        self.activate_user(self.instructor)
+
+        def make_instructor(course):
+            group_name = _course_staff_group_name(course.location)
+            g = Group.objects.create(name=group_name)
+            g.user_set.add(ct.user(self.instructor))
+
+        make_instructor(self.toy)
+
+        self.logout()
+        self.login(self.instructor, self.password)
+        self.enroll(self.toy)
+
+
+    def test_download_grades_csv(self):
+        course = self.toy
+        url = reverse('instructor_dashboard', kwargs={'course_id': course.id})
+        msg = "url = {0}\n".format(url)
+        response = self.client.post(url, {'action': 'Download CSV of all student grades for this course'})
+        msg += "instructor dashboard download csv grades: response = '{0}'\n".format(response)
+
+        self.assertEqual(response['Content-Type'],'text/csv',msg)
+
+        cdisp = response['Content-Disposition']
+        msg += "Content-Disposition = '%s'\n" % cdisp
+        self.assertEqual(cdisp, 'attachment; filename=grades_{0}.csv'.format(course.id), msg)
+
+        body = response.content.replace('\r','')
+        msg += "body = '{0}'\n".format(body)
+
+        # All the not-actually-in-the-course hw and labs come from the
+        # default grading policy string in graders.py
+        expected_body = '''"ID","Username","Full Name","edX email","External email","HW 01","HW 02","HW 03","HW 04","HW 05","HW 06","HW 07","HW 08","HW 09","HW 10","HW 11","HW 12","HW Avg","Lab 01","Lab 02","Lab 03","Lab 04","Lab 05","Lab 06","Lab 07","Lab 08","Lab 09","Lab 10","Lab 11","Lab 12","Lab Avg","Midterm","Final"
+"2","u2","Fred Weasley","view2@test.com","","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0.0","0.0"
+'''
+        self.assertEqual(body, expected_body, msg)
+
+
+FORUM_ROLES = [ FORUM_ROLE_ADMINISTRATOR, FORUM_ROLE_MODERATOR, FORUM_ROLE_COMMUNITY_TA ]
+FORUM_ADMIN_ACTION_SUFFIX = { FORUM_ROLE_ADMINISTRATOR : 'admin', FORUM_ROLE_MODERATOR : 'moderator', FORUM_ROLE_COMMUNITY_TA : 'community TA'}
+FORUM_ADMIN_USER = { FORUM_ROLE_ADMINISTRATOR : 'forumadmin', FORUM_ROLE_MODERATOR : 'forummoderator', FORUM_ROLE_COMMUNITY_TA : 'forummoderator'}
+
+def action_name(operation, rolename):
+    if operation == 'List':
+        return '{0} course forum {1}s'.format(operation, FORUM_ADMIN_ACTION_SUFFIX[rolename])
+    else:
+        return '{0} forum {1}'.format(operation, FORUM_ADMIN_ACTION_SUFFIX[rolename])
+
+
+_mock_service = staff_grading_service.MockStaffGradingService()
+
+@override_settings(MODULESTORE=ct.TEST_DATA_XML_MODULESTORE)
+class TestInstructorDashboardForumAdmin(ct.PageLoader):
+    '''
+    Check for change in forum admin role memberships
+    '''
+
     def setUp(self):
         xmodule.modulestore.django._MODULESTORES = {}
         courses = modulestore().get_courses()
 
-        def find_course(name):
-            """Assumes the course is present"""
-            return [c for c in courses if c.location.course==name][0]
 
-        self.full = find_course("full")
-        self.toy = find_course("toy")
+        self.course_id = "edX/toy/2012_Fall"
+        self.toy = modulestore().get_course(self.course_id)
 
         # Create two accounts
         self.student = 'view@test.com'
@@ -60,25 +134,164 @@ class TestInstructorDashboardGradeDownloadCSV(ct.PageLoader):
         self.enroll(self.toy)
 
 
-    def test_download_grades_csv(self):
-        print "running test_download_grades_csv"
+
+    def initialize_roles(self, course_id):
+        self.admin_role = Role.objects.get_or_create(name=FORUM_ROLE_ADMINISTRATOR, course_id=course_id)[0]
+        self.moderator_role = Role.objects.get_or_create(name=FORUM_ROLE_MODERATOR, course_id=course_id)[0]
+        self.community_ta_role = Role.objects.get_or_create(name=FORUM_ROLE_COMMUNITY_TA, course_id=course_id)[0]
+
+    def test_add_forum_admin_users_for_unknown_user(self):
         course = self.toy
         url = reverse('instructor_dashboard', kwargs={'course_id': course.id})
-        msg = "url = %s\n" % url
-        response = self.client.post(url, {'action': 'Download CSV of all student grades for this course',
-                                          })
-        msg += "instructor dashboard download csv grades: response = '%s'\n" % response
+        username = 'unknown'
+        for action in ['Add', 'Remove']:
+            for rolename in FORUM_ROLES:
+                response = self.client.post(url, {'action': action_name(action, rolename), FORUM_ADMIN_USER[rolename]: username})
+                self.assertTrue(response.content.find('Error: unknown username "{0}"'.format(username))>=0)
 
-        self.assertEqual(response['Content-Type'],'text/csv',msg)
+    def test_add_forum_admin_users_for_missing_roles(self):
+        course = self.toy
+        url = reverse('instructor_dashboard', kwargs={'course_id': course.id})
+        username = 'u1'
+        for action in ['Add', 'Remove']:
+            for rolename in FORUM_ROLES:
+                response = self.client.post(url, {'action': action_name(action, rolename), FORUM_ADMIN_USER[rolename]: username})
+                self.assertTrue(response.content.find('Error: unknown rolename "{0}"'.format(rolename))>=0)
 
-        cdisp = response['Content-Disposition'].replace('TT_2012','2012')  # jenkins course_id is TT_2012_Fall instead of 2012_Fall?
-        msg += "cdisp = '%s'\n" % cdisp
-        self.assertEqual(cdisp,'attachment; filename=grades_edX/toy/2012_Fall.csv',msg)
+    def test_remove_forum_admin_users_for_missing_users(self):
+        course = self.toy
+        self.initialize_roles(course.id)
+        url = reverse('instructor_dashboard', kwargs={'course_id': course.id})
+        username = 'u1'
+        action = 'Remove'
+        for rolename in FORUM_ROLES:
+            response = self.client.post(url, {'action': action_name(action, rolename), FORUM_ADMIN_USER[rolename]: username})
+            self.assertTrue(response.content.find('Error: user "{0}" does not have rolename "{1}"'.format(username, rolename))>=0)
+
+    def test_add_and_remove_forum_admin_users(self):
+        course = self.toy
+        self.initialize_roles(course.id)
+        url = reverse('instructor_dashboard', kwargs={'course_id': course.id})
+        username = 'u2'
+        for rolename in FORUM_ROLES:
+            response = self.client.post(url, {'action': action_name('Add', rolename), FORUM_ADMIN_USER[rolename]: username})
+            self.assertTrue(response.content.find('Added "{0}" to "{1}" forum role = "{2}"'.format(username, course.id, rolename))>=0)
+            self.assertTrue(has_forum_access(username, course.id, rolename))
+            response = self.client.post(url, {'action': action_name('Remove', rolename), FORUM_ADMIN_USER[rolename]: username})
+            self.assertTrue(response.content.find('Removed "{0}" from "{1}" forum role = "{2}"'.format(username, course.id, rolename))>=0)
+            self.assertFalse(has_forum_access(username, course.id, rolename))
+
+    def test_add_and_readd_forum_admin_users(self):
+        course = self.toy
+        self.initialize_roles(course.id)
+        url = reverse('instructor_dashboard', kwargs={'course_id': course.id})
+        username = 'u2'
+        for rolename in FORUM_ROLES:
+            # perform an add, and follow with a second identical add:
+            self.client.post(url, {'action': action_name('Add', rolename), FORUM_ADMIN_USER[rolename]: username})
+            response = self.client.post(url, {'action': action_name('Add', rolename), FORUM_ADMIN_USER[rolename]: username})
+            self.assertTrue(response.content.find('Error: user "{0}" already has rolename "{1}", cannot add'.format(username, rolename))>=0)
+            self.assertTrue(has_forum_access(username, course.id, rolename))
+
+    def test_add_nonstaff_forum_admin_users(self):
+        course = self.toy
+        self.initialize_roles(course.id)
+        url = reverse('instructor_dashboard', kwargs={'course_id': course.id})
+        username = 'u1'
+        rolename = FORUM_ROLE_ADMINISTRATOR
+        response = self.client.post(url, {'action': action_name('Add', rolename), FORUM_ADMIN_USER[rolename]: username})
+        self.assertTrue(response.content.find('Error: user "{0}" should first be added as staff'.format(username))>=0)
+
+    def test_list_forum_admin_users(self):
+        course = self.toy
+        self.initialize_roles(course.id)
+        url = reverse('instructor_dashboard', kwargs={'course_id': course.id})
+        username = 'u2'
+        added_roles = [FORUM_ROLE_STUDENT]  # u2 is already added as a student to the discussion forums
+        self.assertTrue(has_forum_access(username, course.id, 'Student'))
+        for rolename in FORUM_ROLES:
+            response = self.client.post(url, {'action': action_name('Add', rolename), FORUM_ADMIN_USER[rolename]: username})
+            self.assertTrue(has_forum_access(username, course.id, rolename))
+            response = self.client.post(url, {'action': action_name('List', rolename), FORUM_ADMIN_USER[rolename]: username})
+            for header in ['Username', 'Full name', 'Roles']:
+                self.assertTrue(response.content.find('<th>{0}</th>'.format(header))>0)
+            self.assertTrue(response.content.find('<td>{0}</td>'.format(username))>=0)
+            # concatenate all roles for user, in sorted order:
+            added_roles.append(rolename)
+            added_roles.sort()
+            roles = ', '.join(added_roles)
+            self.assertTrue(response.content.find('<td>{0}</td>'.format(roles))>=0, 'not finding roles "{0}"'.format(roles))
+
+
+@override_settings(MODULESTORE=ct.TEST_DATA_XML_MODULESTORE)
+class TestStaffGradingService(ct.PageLoader):
+    '''
+    Check that staff grading service proxy works.  Basically just checking the
+    access control and error handling logic -- all the actual work is on the
+    backend.
+    '''
+    def setUp(self):
+        xmodule.modulestore.django._MODULESTORES = {}
+
+        self.student = 'view@test.com'
+        self.instructor = 'view2@test.com'
+        self.password = 'foo'
+        self.location = 'TestLocation'
+        self.create_account('u1', self.student, self.password)
+        self.create_account('u2', self.instructor, self.password)
+        self.activate_user(self.student)
+        self.activate_user(self.instructor)
+        
+        self.course_id = "edX/toy/2012_Fall"
+        self.toy = modulestore().get_course(self.course_id)
+        def make_instructor(course):
+            group_name = _course_staff_group_name(course.location)
+            g = Group.objects.create(name=group_name)
+            g.user_set.add(ct.user(self.instructor))
+
+        make_instructor(self.toy)
+
+        self.mock_service = staff_grading_service.grading_service()
+
+        self.logout()
+
+    def test_access(self):
+        """
+        Make sure only staff have access.
+        """
+        self.login(self.student, self.password)
+
+        # both get and post should return 404
+        for view_name in ('staff_grading_get_next', 'staff_grading_save_grade'):
+            url = reverse(view_name, kwargs={'course_id': self.course_id})
+            self.check_for_get_code(404, url)
+            self.check_for_post_code(404, url)
+
+
+    def test_get_next(self):
+        self.login(self.instructor, self.password)
+
+        url = reverse('staff_grading_get_next', kwargs={'course_id': self.course_id})
+        data = {'location': self.location}
+
+        r = self.check_for_post_code(200, url, data)
+        d = json.loads(r.content)
+        self.assertTrue(d['success'])
+        self.assertEquals(d['submission_id'], self.mock_service.cnt)
+
+
+    def test_save_grade(self):
+        self.login(self.instructor, self.password)
+
+        url = reverse('staff_grading_save_grade', kwargs={'course_id': self.course_id})
+
+        data = {'score': '12',
+                'feedback': 'great!',
+                'submission_id': '123',
+                'location': self.location}
+        r = self.check_for_post_code(200, url, data)
+        d = json.loads(r.content)
+        self.assertTrue(d['success'], str(d))
+        self.assertEquals(d['submission_id'], self.mock_service.cnt)
 
-        body = response.content.replace('\r','')
-        msg += "body = '%s'\n" % body
 
-        expected_body = '''"ID","Username","Full Name","edX email","External email","HW 01","HW 02","HW 03","HW 04","HW 05","HW 06","HW 07","HW 08","HW 09","HW 10","HW 11","HW 12","HW Avg","Lab 01","Lab 02","Lab 03","Lab 04","Lab 05","Lab 06","Lab 07","Lab 08","Lab 09","Lab 10","Lab 11","Lab 12","Lab Avg","Midterm","Final"
-"2","u2","Fred Weasley","view2@test.com","","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0.0","0.0"
-'''
-        self.assertEqual(body, expected_body, msg)

lms/djangoapps/instructor/views.py

@@ -1,59 +1,56 @@
 # ======== Instructor views =============================================================================
 
+from collections import defaultdict
 import csv
-import itertools
-import json
 import logging
 import os
 import urllib
 
-import track.views
-
-from functools import partial
-from collections import defaultdict
-
 from django.conf import settings
-from django.core.context_processors import csrf
-from django.core.urlresolvers import reverse
 from django.contrib.auth.models import User, Group
-from django.contrib.auth.decorators import login_required
-from django.http import Http404, HttpResponse
-from django.shortcuts import redirect
-from mitxmako.shortcuts import render_to_response, render_to_string
-#from django.views.decorators.csrf import ensure_csrf_cookie
+from django.http import HttpResponse
 from django_future.csrf import ensure_csrf_cookie
 from django.views.decorators.cache import cache_control
+from mitxmako.shortcuts import render_to_response
+from django.core.urlresolvers import reverse
 
 from courseware import grades
 from courseware.access import has_access, get_access_group_name
-from courseware.courses import (get_course_with_access, get_courses_by_university)
+from courseware.courses import get_course_with_access 
+from django_comment_client.models import Role, FORUM_ROLE_ADMINISTRATOR, FORUM_ROLE_MODERATOR, FORUM_ROLE_COMMUNITY_TA
+from django_comment_client.utils import has_forum_access
 from psychometrics import psychoanalyze
-from student.models import UserProfile
-
-from student.models import UserTestGroup, CourseEnrollment
-from util.cache import cache, cache_if_anonymous
+from student.models import CourseEnrollment
 from xmodule.course_module import CourseDescriptor
 from xmodule.modulestore import Location
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError, NoPathToItem
 from xmodule.modulestore.search import path_to_location
+import track.views
 
-log = logging.getLogger("mitx.courseware")
+from .grading import StaffGrading
+
+
+log = logging.getLogger(__name__)
 
 template_imports = {'urllib': urllib}
 
+# internal commands for managing forum roles:
+FORUM_ROLE_ADD = 'add'
+FORUM_ROLE_REMOVE = 'remove'
 
 @ensure_csrf_cookie
 @cache_control(no_cache=True, no_store=True, must_revalidate=True)
+
 def instructor_dashboard(request, course_id):
     """Display the instructor dashboard for a course."""
     course = get_course_with_access(request.user, course_id, 'staff')
 
     instructor_access = has_access(request.user, course, 'instructor')   # an instructor can manage staff lists
+    
+    forum_admin_access = has_forum_access(request.user, course_id, FORUM_ROLE_ADMINISTRATOR)
 
     msg = ''
-    #msg += ('POST=%s' % dict(request.POST)).replace('<','&lt;')
-
     problems = []
     plots = []
 
@@ -81,7 +78,7 @@ def instructor_dashboard(request, course_id):
 
     def return_csv(fn, datatable):
         response = HttpResponse(mimetype='text/csv')
-        response['Content-Disposition'] = 'attachment; filename=%s' % fn
+        response['Content-Disposition'] = 'attachment; filename={0}'.format(fn)
         writer = csv.writer(response, dialect='excel', quotechar='"', quoting=csv.QUOTE_ALL)
         writer.writerow(datatable['header'])
         for datarow in datatable['data']:
@@ -94,7 +91,7 @@ def instructor_dashboard(request, course_id):
         try:
             group = Group.objects.get(name=staffgrp)
         except Group.DoesNotExist:
-            group = Group(name=staffgrp)		# create the group
+            group = Group(name=staffgrp)     # create the group
             group.save()
         return group
 
@@ -104,75 +101,75 @@ def instructor_dashboard(request, course_id):
     if settings.MITX_FEATURES['ENABLE_MANUAL_GIT_RELOAD']:
         if 'GIT pull' in action:
             data_dir = course.metadata['data_dir']
-            log.debug('git pull %s' % (data_dir))
+            log.debug('git pull {0}'.format(data_dir))
             gdir = settings.DATA_DIR / data_dir
             if not os.path.exists(gdir):
-                msg += "====> ERROR in gitreload - no such directory %s" % gdir
+                msg += "====> ERROR in gitreload - no such directory {0}".format(gdir)
             else:
-                cmd = "cd %s; git reset --hard HEAD; git clean -f -d; git pull origin; chmod g+w course.xml" % gdir
-                msg += "git pull on %s:<p>" % data_dir
-                msg += "<pre>%s</pre></p>" % escape(os.popen(cmd).read())
-                track.views.server_track(request, 'git pull %s' % data_dir, {}, page='idashboard')
+                cmd = "cd {0}; git reset --hard HEAD; git clean -f -d; git pull origin; chmod g+w course.xml".format(gdir)
+                msg += "git pull on {0}:<p>".format(data_dir)
+                msg += "<pre>{0}</pre></p>".format(escape(os.popen(cmd).read()))
+                track.views.server_track(request, 'git pull {0}'.format(data_dir), {}, page='idashboard')
 
         if 'Reload course' in action:
-            log.debug('reloading %s (%s)' % (course_id, course))
+            log.debug('reloading {0} ({1})'.format(course_id, course))
             try:
                 data_dir = course.metadata['data_dir']
                 modulestore().try_load_course(data_dir)
-                msg += "<br/><p>Course reloaded from %s</p>" % data_dir
-                track.views.server_track(request, 'reload %s' % data_dir, {}, page='idashboard')
+                msg += "<br/><p>Course reloaded from {0}</p>".format(data_dir)
+                track.views.server_track(request, 'reload {0}'.format(data_dir), {}, page='idashboard')
                 course_errors = modulestore().get_item_errors(course.location)
                 msg += '<ul>'
                 for cmsg, cerr in course_errors:
-                    msg += "<li>%s: <pre>%s</pre>" % (cmsg,escape(cerr))
+                    msg += "<li>{0}: <pre>{1}</pre>".format(cmsg,escape(cerr))
                 msg += '</ul>'
             except Exception as err:
-                msg += '<br/><p>Error: %s</p>' % escape(err)
+                msg += '<br/><p>Error: {0}</p>'.format(escape(err))
 
     if action == 'Dump list of enrolled students':
         log.debug(action)
         datatable = get_student_grade_summary_data(request, course, course_id, get_grades=False)
-        datatable['title'] = 'List of students enrolled in %s' % course_id
+        datatable['title'] = 'List of students enrolled in {0}'.format(course_id)
         track.views.server_track(request, 'list-students', {}, page='idashboard')
 
     elif 'Dump Grades' in action:
         log.debug(action)
         datatable = get_student_grade_summary_data(request, course, course_id, get_grades=True)
-        datatable['title'] = 'Summary Grades of students enrolled in %s' % course_id
+        datatable['title'] = 'Summary Grades of students enrolled in {0}'.format(course_id)
         track.views.server_track(request, 'dump-grades', {}, page='idashboard')
 
     elif 'Dump all RAW grades' in action:
         log.debug(action)
         datatable = get_student_grade_summary_data(request, course, course_id, get_grades=True,
                                                    get_raw_scores=True)
-        datatable['title'] = 'Raw Grades of students enrolled in %s' % course_id
+        datatable['title'] = 'Raw Grades of students enrolled in {0}'.format(course_id)
         track.views.server_track(request, 'dump-grades-raw', {}, page='idashboard')
 
     elif 'Download CSV of all student grades' in action:
         track.views.server_track(request, 'dump-grades-csv', {}, page='idashboard')
-        return return_csv('grades_%s.csv' % course_id,
+        return return_csv('grades_{0}.csv'.format(course_id),
                           get_student_grade_summary_data(request, course, course_id))
 
     elif 'Download CSV of all RAW grades' in action:
         track.views.server_track(request, 'dump-grades-csv-raw', {}, page='idashboard')
-        return return_csv('grades_%s_raw.csv' % course_id,
+        return return_csv('grades_{0}_raw.csv'.format(course_id),
                           get_student_grade_summary_data(request, course, course_id, get_raw_scores=True))
 
     elif 'Download CSV of answer distributions' in action:
         track.views.server_track(request, 'dump-answer-dist-csv', {}, page='idashboard')
-        return return_csv('answer_dist_%s.csv' % course_id, get_answers_distribution(request, course_id))
+        return return_csv('answer_dist_{0}.csv'.format(course_id), get_answers_distribution(request, course_id))
 
     #----------------------------------------
     # Admin
 
     elif 'List course staff' in action:
         group = get_staff_group(course)
-        msg += 'Staff group = %s' % group.name
-        log.debug('staffgrp=%s' % group.name)
+        msg += 'Staff group = {0}'.format(group.name)
+        log.debug('staffgrp={0}'.format(group.name))
         uset = group.user_set.all()
         datatable = {'header': ['Username', 'Full name']}
         datatable['data'] = [[x.username, x.profile.name] for x in uset]
-        datatable['title'] = 'List of Staff in course %s' % course_id
+        datatable['title'] = 'List of Staff in course {0}'.format(course_id)
         track.views.server_track(request, 'list-staff', {}, page='idashboard')
 
     elif action == 'Add course staff':
@@ -180,28 +177,86 @@ def instructor_dashboard(request, course_id):
         try:
             user = User.objects.get(username=uname)
         except User.DoesNotExist:
-            msg += '<font color="red">Error: unknown username "%s"</font>' % uname
+            msg += '<font color="red">Error: unknown username "{0}"</font>'.format(uname)
             user = None
         if user is not None:
             group = get_staff_group(course)
-            msg += '<font color="green">Added %s to staff group = %s</font>' % (user, group.name)
-            log.debug('staffgrp=%s' % group.name)
+            msg += '<font color="green">Added {0} to staff group = {1}</font>'.format(user, group.name)
+            log.debug('staffgrp={0}'.format(group.name))
             user.groups.add(group)
-            track.views.server_track(request, 'add-staff %s' % user, {}, page='idashboard')
+            track.views.server_track(request, 'add-staff {0}'.format(user), {}, page='idashboard')
 
     elif action == 'Remove course staff':
         uname = request.POST['staffuser']
         try:
             user = User.objects.get(username=uname)
         except User.DoesNotExist:
-            msg += '<font color="red">Error: unknown username "%s"</font>' % uname
+            msg += '<font color="red">Error: unknown username "{0}"</font>'.format(uname)
             user = None
         if user is not None:
             group = get_staff_group(course)
-            msg += '<font color="green">Removed %s from staff group = %s</font>' % (user, group.name)
-            log.debug('staffgrp=%s' % group.name)
+            msg += '<font color="green">Removed {0} from staff group = {1}</font>'.format(user, group.name)
+            log.debug('staffgrp={0}'.format(group.name))
             user.groups.remove(group)
-            track.views.server_track(request, 'remove-staff %s' % user, {}, page='idashboard')
+            track.views.server_track(request, 'remove-staff {0}'.format(user), {}, page='idashboard')
+
+    #----------------------------------------
+    # forum administration
+  
+    elif action == 'List course forum admins':
+        rolename = FORUM_ROLE_ADMINISTRATOR
+        datatable = {}
+        msg += _list_course_forum_members(course_id, rolename, datatable)
+        track.views.server_track(request, 'list-{0}'.format(rolename), {}, page='idashboard')
+        
+    
+    elif action == 'Remove forum admin':
+        uname = request.POST['forumadmin']
+        msg += _update_forum_role_membership(uname, course, FORUM_ROLE_ADMINISTRATOR, FORUM_ROLE_REMOVE)
+        track.views.server_track(request, '{0} {1} as {2} for {3}'.format(FORUM_ROLE_REMOVE, uname, FORUM_ROLE_ADMINISTRATOR, course_id), 
+                                 {}, page='idashboard')
+
+    elif action == 'Add forum admin':
+        uname = request.POST['forumadmin']
+        msg += _update_forum_role_membership(uname, course, FORUM_ROLE_ADMINISTRATOR, FORUM_ROLE_ADD)
+        track.views.server_track(request, '{0} {1} as {2} for {3}'.format(FORUM_ROLE_ADD, uname, FORUM_ROLE_ADMINISTRATOR, course_id), 
+                                 {}, page='idashboard')
+
+    elif action == 'List course forum moderators':
+        rolename = FORUM_ROLE_MODERATOR
+        datatable = {}
+        msg += _list_course_forum_members(course_id, rolename, datatable)
+        track.views.server_track(request, 'list-{0}'.format(rolename), {}, page='idashboard')
+    
+    elif action == 'Remove forum moderator':
+        uname = request.POST['forummoderator']
+        msg += _update_forum_role_membership(uname, course, FORUM_ROLE_MODERATOR, FORUM_ROLE_REMOVE)
+        track.views.server_track(request, '{0} {1} as {2} for {3}'.format(FORUM_ROLE_REMOVE, uname, FORUM_ROLE_MODERATOR, course_id), 
+                                 {}, page='idashboard')
+    
+    elif action == 'Add forum moderator':
+        uname = request.POST['forummoderator']
+        msg += _update_forum_role_membership(uname, course, FORUM_ROLE_MODERATOR, FORUM_ROLE_ADD)
+        track.views.server_track(request, '{0} {1} as {2} for {3}'.format(FORUM_ROLE_ADD, uname, FORUM_ROLE_MODERATOR, course_id), 
+                                 {}, page='idashboard')
+    
+    elif action == 'List course forum community TAs':
+        rolename = FORUM_ROLE_COMMUNITY_TA
+        datatable = {}
+        msg += _list_course_forum_members(course_id, rolename, datatable)
+        track.views.server_track(request, 'list-{0}'.format(rolename), {}, page='idashboard')
+    
+    elif action == 'Remove forum community TA':
+        uname = request.POST['forummoderator']
+        msg += _update_forum_role_membership(uname, course, FORUM_ROLE_COMMUNITY_TA, FORUM_ROLE_REMOVE)
+        track.views.server_track(request, '{0} {1} as {2} for {3}'.format(FORUM_ROLE_REMOVE, uname, FORUM_ROLE_COMMUNITY_TA, course_id), 
+                                 {}, page='idashboard')
+    
+    elif action == 'Add forum community TA':
+        uname = request.POST['forummoderator']
+        msg += _update_forum_role_membership(uname, course, FORUM_ROLE_COMMUNITY_TA, FORUM_ROLE_ADD)
+        track.views.server_track(request, '{0} {1} as {2} for {3}'.format(FORUM_ROLE_ADD, uname, FORUM_ROLE_COMMUNITY_TA, course_id), 
+                                 {}, page='idashboard')
 
     #----------------------------------------
     # psychometrics
@@ -210,17 +265,20 @@ def instructor_dashboard(request, course_id):
         problem = request.POST['Problem']
         nmsg, plots = psychoanalyze.generate_plots_for_problem(problem)
         msg += nmsg
-        track.views.server_track(request, 'psychometrics %s' % problem, {}, page='idashboard')
+        track.views.server_track(request, 'psychometrics {0}'.format(problem), {}, page='idashboard')
 
     if idash_mode=='Psychometrics':
         problems = psychoanalyze.problems_with_psychometric_data(course_id)
 
+
+
     #----------------------------------------
     # context for rendering
     context = {'course': course,
                'staff_access': True,
                'admin_access': request.user.is_staff,
                'instructor_access': instructor_access,
+               'forum_admin_access': forum_admin_access,
                'datatable': datatable,
                'msg': msg,
                'modeflag': {idash_mode: 'selectedmode'},
@@ -232,6 +290,75 @@ def instructor_dashboard(request, course_id):
 
     return render_to_response('courseware/instructor_dashboard.html', context)
 
+def _list_course_forum_members(course_id, rolename, datatable):
+    ''' 
+    Fills in datatable with forum membership information, for a given role,
+    so that it will be displayed on instructor dashboard.
+    
+      course_ID = course's ID string
+      rolename = one of "Administrator", "Moderator", "Community TA"
+    
+    Returns message status string to append to displayed message, if role is unknown.
+    '''
+    # make sure datatable is set up properly for display first, before checking for errors
+    datatable['header'] = ['Username', 'Full name', 'Roles']
+    datatable['title'] = 'List of Forum {0}s in course {1}'.format(rolename, course_id)
+    datatable['data'] = [];
+    try:
+        role = Role.objects.get(name=rolename, course_id=course_id)
+    except Role.DoesNotExist:
+        return '<font color="red">Error: unknown rolename "{0}"</font>'.format(rolename)
+    uset = role.users.all().order_by('username')
+    msg = 'Role = {0}'.format(rolename)
+    log.debug('role={0}'.format(rolename))
+    datatable['data'] = [[x.username, x.profile.name, ', '.join([r.name for r in x.roles.filter(course_id=course_id).order_by('name')])] for x in uset]
+    return msg
+
+
+def _update_forum_role_membership(uname, course, rolename, add_or_remove):
+    '''
+    Supports adding a user to a course's forum role
+    
+      uname = username string for user
+      course = course object 
+      rolename = one of "Administrator", "Moderator", "Community TA"
+      add_or_remove = one of "add" or "remove"
+      
+    Returns message status string to append to displayed message,  Status is returned if user 
+    or role is unknown, or if entry already exists when adding, or if entry doesn't exist when removing.
+    '''
+    # check that username and rolename are valid:
+    try:
+        user = User.objects.get(username=uname)
+    except User.DoesNotExist:
+        return '<font color="red">Error: unknown username "{0}"</font>'.format(uname)
+    try:
+        role = Role.objects.get(name=rolename, course_id=course.id)
+    except Role.DoesNotExist:
+        return '<font color="red">Error: unknown rolename "{0}"</font>'.format(rolename)
+
+    # check whether role already has the specified user:
+    alreadyexists = role.users.filter(username=uname).exists()
+    msg = ''
+    log.debug('rolename={0}'.format(rolename))
+    if add_or_remove == FORUM_ROLE_REMOVE:
+        if not alreadyexists:
+            msg ='<font color="red">Error: user "{0}" does not have rolename "{1}", cannot remove</font>'.format(uname, rolename)
+        else: 
+            user.roles.remove(role)
+            msg = '<font color="green">Removed "{0}" from "{1}" forum role = "{2}"</font>'.format(user, course.id, rolename)
+    else:
+        if alreadyexists:
+            msg = '<font color="red">Error: user "{0}" already has rolename "{1}", cannot add</font>'.format(uname, rolename)
+        else: 
+            if (rolename == FORUM_ROLE_ADMINISTRATOR and not has_access(user, course, 'staff')):   
+                msg = '<font color="red">Error: user "{0}" should first be added as staff before adding as a forum administrator, cannot add</font>'.format(uname)
+            else:
+                user.roles.add(role)
+                msg = '<font color="green">Added "{0}" to "{1}" forum role = "{2}"</font>'.format(user, course.id, rolename)
+
+    return msg
+    
 
 def get_student_grade_summary_data(request, course, course_id, get_grades=True, get_raw_scores=False):
     '''
@@ -254,10 +381,10 @@ def get_student_grade_summary_data(request, course, course_id, get_grades=True,
     enrolled_students = User.objects.filter(courseenrollment__course_id=course_id).prefetch_related("groups").order_by('username')
 
     header = ['ID', 'Username', 'Full Name', 'edX email', 'External email']
-    if get_grades:
+    if get_grades and enrolled_students.count() > 0:
         # just to construct the header
         gradeset = grades.grade(enrolled_students[0], request, course, keep_raw_scores=get_raw_scores)
-        # log.debug('student %s gradeset %s' % (enrolled_students[0], gradeset))
+        # log.debug('student {0} gradeset {1}'.format(enrolled_students[0], gradeset))
         if get_raw_scores:
             header += [score.section for score in gradeset['raw_scores']]
         else:
@@ -275,7 +402,7 @@ def get_student_grade_summary_data(request, course, course_id, get_grades=True,
 
         if get_grades:
             gradeset = grades.grade(student, request, course, keep_raw_scores=get_raw_scores)
-            # log.debug('student=%s, gradeset=%s' % (student,gradeset))
+            # log.debug('student={0}, gradeset={1}'.format(student,gradeset))
             if get_raw_scores:
                 datarow += [score.earned for score in gradeset['raw_scores']]
             else:
@@ -286,6 +413,29 @@ def get_student_grade_summary_data(request, course, course_id, get_grades=True,
     return datatable
 
 
+
+@cache_control(no_cache=True, no_store=True, must_revalidate=True)
+def staff_grading(request, course_id):
+    """
+    Show the instructor grading interface.
+    """
+    course = get_course_with_access(request.user, course_id, 'staff')
+
+    grading = StaffGrading(course)
+
+    ajax_url = reverse('staff_grading', kwargs={'course_id': course_id})
+    if not ajax_url.endswith('/'):
+        ajax_url += '/'
+        
+    return render_to_response('instructor/staff_grading.html', {
+        'view_html': grading.get_html(),
+        'course': course,
+        'course_id': course_id,
+        'ajax_url': ajax_url,
+        # Checked above
+        'staff_access': True, })
+
+
 @cache_control(no_cache=True, no_store=True, must_revalidate=True)
 def gradebook(request, course_id):
     """

lms/envs/aws.py

@@ -76,5 +76,8 @@ DATABASES = AUTH_TOKENS['DATABASES']
 
 XQUEUE_INTERFACE = AUTH_TOKENS['XQUEUE_INTERFACE']
 
+STAFF_GRADING_INTERFACE = AUTH_TOKENS.get('STAFF_GRADING_INTERFACE')
+
+
 PEARSON_TEST_USER = "pearsontest"
 PEARSON_TEST_PASSWORD = AUTH_TOKENS.get("PEARSON_TEST_PASSWORD")

lms/envs/common.py

@@ -322,6 +322,13 @@ WIKI_USE_BOOTSTRAP_SELECT_WIDGET = False
 WIKI_LINK_LIVE_LOOKUPS = False
 WIKI_LINK_DEFAULT_LEVEL = 2
 
+################################# Staff grading config  #####################
+
+STAFF_GRADING_INTERFACE = None
+# Used for testing, debugging
+MOCK_STAFF_GRADING = False
+
+
 ################################# Jasmine ###################################
 JASMINE_TEST_DIRECTORY = PROJECT_ROOT + '/static/coffee'
 
@@ -397,6 +404,7 @@ courseware_only_js += [
 ]
 
 main_vendor_js = [
+  'js/vendor/json2.js',
   'js/vendor/jquery.min.js',
   'js/vendor/jquery-ui.min.js',
   'js/vendor/jquery.cookie.js',
@@ -406,6 +414,9 @@ main_vendor_js = [
 
 discussion_js = sorted(glob2.glob(PROJECT_ROOT / 'static/coffee/src/discussion/**/*.coffee'))
 
+staff_grading_js = sorted(glob2.glob(PROJECT_ROOT / 'static/coffee/src/staff_grading/**/*.coffee'))
+
+
 # Load javascript from all of the available xmodules, and
 # prep it for use in pipeline js
 from xmodule.x_module import XModuleDescriptor
@@ -468,7 +479,8 @@ with open(module_styles_path, 'w') as module_styles:
 
 PIPELINE_JS = {
     'application': {
-        # Application will contain all paths not in courseware_only_js
+        # Application will contain all paths not in courseware_only_js or
+        # discussion_js or staff_grading_js
         'source_filenames': [
             pth.replace(COMMON_ROOT / 'static/', '')
             for pth
@@ -476,7 +488,9 @@ PIPELINE_JS = {
         ] + [
             pth.replace(PROJECT_ROOT / 'static/', '')
             for pth in sorted(glob2.glob(PROJECT_ROOT / 'static/coffee/src/**/*.coffee'))\
-            if pth not in courseware_only_js and pth not in discussion_js
+            if (pth not in courseware_only_js and
+                pth not in discussion_js and
+                pth not in staff_grading_js)
         ] + [
             'js/form.ext.js',
             'js/my_courses_dropdown.js',
@@ -505,7 +519,12 @@ PIPELINE_JS = {
     'discussion' : {
         'source_filenames': [pth.replace(PROJECT_ROOT / 'static/', '')  for pth in discussion_js],
         'output_filename': 'js/discussion.js'
+    },
+    'staff_grading' : {
+        'source_filenames': [pth.replace(PROJECT_ROOT / 'static/', '')  for pth in staff_grading_js],
+        'output_filename': 'js/staff_grading.js'
     }
+
 }
 
 PIPELINE_DISABLE_WRAPPER = True

lms/envs/dev.py

@@ -39,7 +39,7 @@ DATABASES = {
 }
 
 CACHES = {
-    # This is the cache used for most things. 
+    # This is the cache used for most things.
     # In staging/prod envs, the sessions also live here.
     'default': {
         'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
@@ -102,7 +102,13 @@ SUBDOMAIN_BRANDING = {
 
 COMMENTS_SERVICE_KEY = "PUT_YOUR_API_KEY_HERE"
 
+################################# Staff grading config  #####################
 
+STAFF_GRADING_INTERFACE = {
+    'url': 'http://127.0.0.1:3033/staff_grading',
+    'username': 'lms',
+    'password': 'abcd',
+    }
 
 ################################ LMS Migration #################################
 MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True

lms/envs/logsettings.py

@@ -40,7 +40,7 @@ def get_logger_config(log_dir,
                         logging_env=logging_env, hostname=hostname)
 
     handlers = ['console', 'local'] if debug else ['console',
-                                'syslogger-remote', 'local', 'newrelic']
+                                'syslogger-remote', 'local']
 
     logger_config = {
         'version': 1,

lms/envs/test.py

@@ -65,6 +65,10 @@ XQUEUE_INTERFACE = {
 }
 XQUEUE_WAITTIME_BETWEEN_REQUESTS = 5 # seconds
 
+
+# Don't rely on a real staff grading backend
+MOCK_STAFF_GRADING = True
+
 # TODO (cpennington): We need to figure out how envs/test.py can inject things
 # into common.py so that we don't have to repeat this sort of thing
 STATICFILES_DIRS = [
@@ -99,7 +103,7 @@ DATABASES = {
 }
 
 CACHES = {
-    # This is the cache used for most things. 
+    # This is the cache used for most things.
     # In staging/prod envs, the sessions also live here.
     'default': {
         'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',

lms/static/admin/css/ie.css

@@ -1,63 +0,0 @@
-/* IE 6 & 7 */
-
-/* Proper fixed width for dashboard in IE6 */
-
-.dashboard #content {
-    *width: 768px;
-}
-
-.dashboard #content-main {
-    *width: 535px;
-}
-
-/* IE 6 ONLY */
-
-/* Keep header from flowing off the page */
-
-#container {
-    _position: static;
-}
-
-/* Put the right sidebars back on the page */
-
-.colMS #content-related {
-    _margin-right: 0;
-    _margin-left: 10px;
-    _position: static;
-}
-
-/* Put the left sidebars back on the page */
-
-.colSM #content-related {
-    _margin-right: 10px;
-    _margin-left: -115px;
-    _position: static;
-}
-
-.form-row {
-    _height: 1%;
-}
-
-/* Fix right margin for changelist filters in IE6 */
-
-#changelist-filter ul {
-    _margin-right: -10px;
-}
-
-/* IE ignores min-height, but treats height as if it were min-height */
-
-.change-list .filtered {
-    _height: 400px;
-}
-
-/* IE doesn't know alpha transparency in PNGs */
-
-.inline-deletelink {
-    background: transparent url(../img/inline-delete-8bit.png) no-repeat;
-}
-
-/* IE7 doesn't support inline-block */
-.change-list ul.toplinks li {
-    zoom: 1;
-    *display: inline;
-}

lms/static/coffee/src/courseware.coffee

@@ -18,5 +18,6 @@ class @Courseware
         histg = new Histogram id, $(this).data('histogram')
       catch error
         histg = error
-        console.log(error)
+        if console?
+          console.log(error)
       return histg

lms/static/coffee/src/staff_grading/staff_grading.coffee

@@ -0,0 +1,391 @@
+# wrap everything in a class in case we want to use inside xmodules later
+
+get_random_int: (min, max) ->
+  return Math.floor(Math.random() * (max - min + 1)) + min
+
+# states
+state_grading = "grading"
+state_graded = "graded"
+state_no_data = "no_data"
+state_error = "error"
+
+class StaffGradingBackend
+  constructor: (ajax_url, mock_backend) ->
+    @ajax_url = ajax_url
+    @mock_backend = mock_backend
+    if @mock_backend
+      @mock_cnt = 0
+
+  mock: (cmd, data) ->
+    # Return a mock response to cmd and data
+    # should take a location as an argument
+    if cmd == 'get_next'
+      @mock_cnt++
+      switch data.location
+        when 'i4x://MITx/3.091x/problem/open_ended_demo1'
+          response =
+            success: true
+            problem_name: 'Problem 1'
+            num_graded: 3
+            min_for_ml: 5
+            num_pending: 4
+            prompt: '''
+            	<h2>S11E3: Metal Bands</h2>
+<p>Shown below are schematic band diagrams for two different metals. Both diagrams appear different, yet both of the elements are undisputably metallic in nature.</p>
+<img width="480" src="/static/images/LSQimages/shaded_metal_bands.png"/>
+<p>* Why is it that both sodium and magnesium behave as metals, even though the s-band of magnesium is filled? </p>
+<p>This is a self-assessed open response question. Please use as much space as you need in the box below to answer the question.</p>
+            '''
+            submission: '''
+            Contrary to popular belief, Lorem Ipsum is not simply random text. It has roots in a piece of classical Latin literature from 45 BC, making it over 2000 years old. Richard McClintock, a Latin professor at Hampden-Sydney College in Virginia, looked up one of the more obscure Latin words, consectetur, from a Lorem Ipsum passage, and going through the cites of the word in classical literature, discovered the undoubtable source. Lorem Ipsum comes from sections 1.10.32 and 1.10.33 of "de Finibus Bonorum et Malorum" (The Extremes of Good and Evil) by Cicero, written in 45 BC. This book is a treatise on the theory of ethics, very popular during the Renaissance. The first line of Lorem Ipsum, "Lorem ipsum dolor sit amet..", comes from a line in section 1.10.32.
+
+The standard chunk of Lorem Ipsum used since the 1500s is reproduced below for those interested. Sections 1.10.32 and 1.10.33 from "de Finibus Bonorum et Malorum" by Cicero are also reproduced in their exact original form, accompanied by English versions from the 1914 translation by H. Rackham.
+            '''
+            rubric: '''
+<ul>
+<li>Metals tend to be good electronic conductors, meaning that they have a large number of electrons which are able to access empty (mobile) energy states within the material.</li>
+<li>Sodium has a half-filled s-band, so there are a number of empty states immediately above the highest occupied energy levels within the band.</li>
+<li>Magnesium has a full s-band, but the the s-band and p-band overlap in magnesium. Thus are still a large number of available energy states immediately above the s-band highest occupied energy level.</li>
+</ul>
+
+<p>Please score your response according to how many of the above components you identified:</p>
+            '''
+            submission_id: @mock_cnt
+            max_score: 2 + @mock_cnt % 3
+            ml_error_info : 'ML accuracy info: ' + @mock_cnt
+        when 'i4x://MITx/3.091x/problem/open_ended_demo2'
+          response =
+            success: true
+            problem_name: 'Problem 2'
+            num_graded: 2
+            min_for_ml: 5
+            num_pending: 4
+            prompt: 'This is a fake second problem'
+            submission: 'This is the best submission ever! ' + @mock_cnt
+            rubric: 'I am a rubric for grading things! ' + @mock_cnt
+            submission_id: @mock_cnt
+            max_score: 2 + @mock_cnt % 3
+            ml_error_info : 'ML accuracy info: ' + @mock_cnt
+        else
+          response = 
+            success: false
+          
+
+    else if cmd == 'save_grade'
+      console.log("eval: #{data.score} pts,  Feedback: #{data.feedback}")
+      response =
+        @mock('get_next', {location: data.location})
+    # get_problem_list
+    # should get back a list of problem_ids, problem_names, num_graded, min_for_ml
+    else if cmd == 'get_problem_list'
+      @mock_cnt = 1
+      response = 
+        success: true
+        problem_list: [
+          {location: 'i4x://MITx/3.091x/problem/open_ended_demo1', \
+            problem_name: "Problem 1", num_graded: 3, num_pending: 5, min_for_ml: 10},
+          {location: 'i4x://MITx/3.091x/problem/open_ended_demo2', \
+            problem_name: "Problem 2", num_graded: 1, num_pending: 5, min_for_ml: 10}
+        ]
+    else
+      response =
+        success: false
+        error: 'Unknown command ' + cmd
+
+    if @mock_cnt % 5 == 0
+        response = 
+          success: true
+          message: 'No more submissions'
+
+
+    if @mock_cnt % 7 == 0
+      response =
+        success: false
+        error: 'An error for testing'
+        
+    return response
+
+
+  post: (cmd, data, callback) ->
+    if @mock_backend
+      callback(@mock(cmd, data))
+    else
+      # TODO: replace with postWithPrefix when that's loaded
+      $.post(@ajax_url + cmd, data, callback)
+
+
+class StaffGrading
+  constructor: (backend) ->
+    @backend = backend
+
+    # all the jquery selectors
+
+    @problem_list_container = $('.problem-list-container')
+    @problem_list = $('.problem-list')
+
+    @error_container = $('.error-container')
+    @message_container = $('.message-container')
+
+    @prompt_name_container = $('.prompt-name')
+    @prompt_container = $('.prompt-container')
+    @prompt_wrapper = $('.prompt-wrapper')
+
+    @submission_container = $('.submission-container')
+    @submission_wrapper = $('.submission-wrapper')
+
+    @rubric_container = $('.rubric-container')
+    @rubric_wrapper = $('.rubric-wrapper')
+    @grading_wrapper = $('.grading-wrapper')
+
+    @feedback_area = $('.feedback-area')
+    @score_selection_container = $('.score-selection-container')        
+    @submit_button = $('.submit-button')
+    @action_button = $('.action-button')
+
+    @problem_meta_info = $('.problem-meta-info-container')
+    @meta_info_wrapper = $('.meta-info-wrapper')
+    @ml_error_info_container = $('.ml-error-info-container')
+
+    @breadcrumbs = $('.breadcrumbs')
+    
+    # model state
+    @state = state_no_data
+    @submission_id = null
+    @prompt = ''
+    @submission = ''
+    @rubric = ''
+    @error_msg = ''
+    @message = ''
+    @max_score = 0
+    @ml_error_info= ''
+    @location = ''
+    @prompt_name = ''
+    @min_for_ml = 0
+    @num_graded = 0
+    @num_pending = 0
+
+    @score = null
+    @problems = null
+
+    # action handlers
+    @submit_button.click @submit
+    # TODO: fix this to do something more intelligent
+    @action_button.click @submit
+
+    # send initial request automatically
+    @get_problem_list()
+
+
+  setup_score_selection: =>
+    # first, get rid of all the old inputs, if any.
+    @score_selection_container.html('Choose score: ')
+
+    # Now create new labels and inputs for each possible score.
+    for score in [0..@max_score]
+      id = 'score-' + score
+      label = """<label for="#{id}">#{score}</label>"""
+      
+      input = """
+              <input type="radio" name="score-selection" id="#{id}" value="#{score}"/>
+              """       # "  fix broken parsing in emacs
+      @score_selection_container.append(input + label)
+
+    # And now hook up an event handler again
+    $("input[name='score-selection']").change @graded_callback
+    
+
+  set_button_text: (text) =>
+    @action_button.attr('value', text)
+
+  graded_callback: (event) =>
+    @score = event.target.value
+    @state = state_graded
+    @message = ''
+    @render_view()
+
+  ajax_callback: (response) =>
+    # always clear out errors and messages on transition.
+    @error_msg = ''
+    @message = ''
+    
+    if response.success
+      if response.problem_list
+        @problems = response.problem_list
+      else if response.submission
+        @data_loaded(response)
+      else
+        @no_more(response.message)
+    else
+      @error(response.error)
+
+    @render_view()
+       
+  get_next_submission: (location) ->
+    @location = location
+    @list_view = false
+    @backend.post('get_next', {location: location}, @ajax_callback)
+
+  get_problem_list: () ->
+    @list_view = true
+    @backend.post('get_problem_list', {}, @ajax_callback)
+
+  submit_and_get_next: () ->
+    data =
+      score: @score
+      feedback: @feedback_area.val()
+      submission_id: @submission_id
+      location: @location
+    
+    @backend.post('save_grade', data, @ajax_callback)
+
+  error: (msg) ->
+    @error_msg = msg
+    @state = state_error
+
+  data_loaded: (response) ->
+    @prompt = response.prompt
+    @submission = response.submission
+    @rubric = response.rubric
+    @submission_id = response.submission_id
+    @feedback_area.val('')
+    @max_score = response.max_score
+    @score = null
+    @ml_error_info=response.ml_error_info
+    @prompt_name = response.problem_name
+    @num_graded = response.num_graded
+    @min_for_ml = response.min_for_ml
+    @num_pending = response.num_pending
+    @state = state_grading
+    if not @max_score?
+      @error("No max score specified for submission.")
+
+  no_more: (message) ->
+    @prompt = null
+    @prompt_name = ''
+    @num_graded = 0
+    @min_for_ml = 0
+    @submission = null
+    @rubric = null
+    @ml_error_info = null
+    @submission_id = null
+    @message = message
+    @score = null
+    @max_score = 0
+    @state = state_no_data
+
+
+  render_view: () ->
+    # clear the problem list and breadcrumbs
+    @problem_list.html('')
+    @breadcrumbs.html('')
+    @problem_list_container.toggle(@list_view)
+    if @backend.mock_backend
+      @message = @message + "<p>NOTE: Mocking backend.</p>"
+    @message_container.html(@message)
+    @error_container.html(@error_msg)
+    @message_container.toggle(@message != "")
+    @error_container.toggle(@error_msg != "")
+
+
+    # only show the grading elements when we are not in list view or the state
+    # is invalid
+    show_grading_elements = !(@list_view || @state == state_error || 
+      @state == state_no_data)
+    @prompt_wrapper.toggle(show_grading_elements)
+    @submission_wrapper.toggle(show_grading_elements)
+    @rubric_wrapper.toggle(show_grading_elements)
+    @grading_wrapper.toggle(show_grading_elements)
+    @meta_info_wrapper.toggle(show_grading_elements)
+    @action_button.hide()
+    
+    if @list_view
+      @render_list()
+    else
+      @render_problem()
+
+  problem_link:(problem) ->
+    link = $('<a>').attr('href', "javascript:void(0)").append(
+      "#{problem.problem_name} (#{problem.num_graded} graded, #{problem.num_pending} pending)")
+        .click =>
+          @get_next_submission problem.location
+
+  make_paragraphs: (text) ->
+    paragraph_split = text.split(/\n\s*\n/)
+    new_text = ''
+    for paragraph in paragraph_split
+      new_text += "<p>#{paragraph}</p>"
+    return new_text
+
+  render_list: () ->
+    for problem in @problems
+      @problem_list.append($('<li>').append(@problem_link(problem)))
+
+  render_problem: () ->
+    # make the view elements match the state.  Idempotent.
+    show_submit_button = true
+    show_action_button = true
+
+    problem_list_link = $('<a>').attr('href', 'javascript:void(0);')
+      .append("< Back to problem list")
+      .click => @get_problem_list()
+
+    # set up the breadcrumbing
+    @breadcrumbs.append(problem_list_link)
+      
+
+    if @state == state_error
+      @set_button_text('Try loading again')
+      show_action_button = true
+
+    else if @state == state_grading
+      @ml_error_info_container.html(@ml_error_info)
+      meta_list = $("<ul>")
+      meta_list.append("<li><span class='meta-info'>Pending - </span> #{@num_pending}</li>")
+      meta_list.append("<li><span class='meta-info'>Graded - </span> #{@num_graded}</li>")
+      meta_list.append("<li><span class='meta-info'>Needed for ML - </span> #{Math.max(@min_for_ml - @num_graded)}</li>")
+      @problem_meta_info.html(meta_list)
+
+      @prompt_container.html(@prompt)
+      @prompt_name_container.html("#{@prompt_name}")
+      @submission_container.html(@make_paragraphs(@submission))
+      @rubric_container.html(@rubric)
+
+      # no submit button until user picks grade.
+      show_submit_button = false
+      show_action_button = false
+
+      @setup_score_selection()
+      
+    else if @state == state_graded
+      @set_button_text('Submit')
+      show_action_button = false
+
+    else if @state == state_no_data
+      @message_container.html(@message)
+      @set_button_text('Re-check for submissions')
+
+    else
+      @error('System got into invalid state ' + @state)
+
+    @submit_button.toggle(show_submit_button)
+    @action_button.toggle(show_action_button)
+
+  submit: (event) =>
+    event.preventDefault()
+    
+    if @state == state_error
+      @get_next_submission(@location)
+    else if @state == state_graded
+      @submit_and_get_next()
+    else if @state == state_no_data
+      @get_next_submission(@location)
+    else
+      @error('System got into invalid state for submission: ' + @state)
+  
+
+# for now, just create an instance and load it...
+mock_backend = false
+ajax_url = $('.staff-grading').data('ajax_url')
+backend = new StaffGradingBackend(ajax_url, mock_backend)
+
+$(document).ready(() -> new StaffGrading(backend))

lms/static/coffee/src/staff_grading/test_grading.html

@@ -0,0 +1,45 @@
+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML//EN">
+<html> <head>
+<title></title>
+<!-- <script src="http://code.jquery.com/jquery-latest.js"></script> -->
+<script src="../../../admin/js/jquery.min.js"></script> 
+<script type="text/javascript" src="staff_grading.js"></script>
+</head>
+
+<body>
+<div class="staff-grading" data-ajax_url="/some_url/">
+<h1>Staff grading</h1>
+
+<div class="error-container">
+</div>
+
+<div class="message-container">
+</div>
+
+<section class="submission-wrapper">
+<h3>Submission</h3>
+<div class="submission-container">
+</div>
+</section>
+
+<section class="rubric-wrapper">
+<h3>Rubric</h3>
+<div class="rubric-container">
+</div>
+
+<div class="evaluation">
+  <textarea name="feedback" placeholder="Feedback for student..."
+            class="feedback-area" cols="70" rows="10"></textarea>
+  <p class="score-selection-container">
+  </p>
+</div>
+  
+</section>
+
+<div class="submission">
+<input type="button" value="Submit" class="submit-button" name="show"/>
+</div>
+
+</div>
+
+</body> </html>